diff --git a/sys/dev/bnxt/bnxt_re/bnxt_re-abi.h b/sys/dev/bnxt/bnxt_re/bnxt_re-abi.h
new file mode 100644
index 000000000000..8f48609e7f6f
--- /dev/null
+++ b/sys/dev/bnxt/bnxt_re/bnxt_re-abi.h
@@ -0,0 +1,177 @@
+/*
+ * Copyright (c) 2015-2024, Broadcom. All rights reserved.  The term
+ * Broadcom refers to Broadcom Limited and/or its subsidiaries.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS''
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
+ * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
+ * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * Description: Uverbs ABI header file
+ */
+
+#ifndef __BNXT_RE_UVERBS_ABI_H__
+#define __BNXT_RE_UVERBS_ABI_H__
+
+#include <asm/types.h>
+#include <linux/types.h>
+
+#define BNXT_RE_ABI_VERSION	6
+
+enum {
+	BNXT_RE_COMP_MASK_UCNTX_WC_DPI_ENABLED = 0x01,
+	BNXT_RE_COMP_MASK_UCNTX_POW2_DISABLED = 0x02,
+	BNXT_RE_COMP_MASK_UCNTX_RSVD_WQE_DISABLED = 0x04,
+	BNXT_RE_COMP_MASK_UCNTX_MQP_EX_SUPPORTED = 0x08,
+	BNXT_RE_COMP_MASK_UCNTX_DBR_PACING_ENABLED = 0x10,
+	BNXT_RE_COMP_MASK_UCNTX_DBR_RECOVERY_ENABLED = 0x20,
+	BNXT_RE_COMP_MASK_UCNTX_HW_RETX_ENABLED = 0x40
+};
+
+enum {
+	BNXT_RE_COMP_MASK_REQ_UCNTX_POW2_SUPPORT = 0x01,
+	BNXT_RE_COMP_MASK_REQ_UCNTX_RSVD_WQE = 0x02
+};
+
+struct bnxt_re_uctx_req {
+	__aligned_u64 comp_mask;
+};
+
+#define BNXT_RE_CHIP_ID0_CHIP_NUM_SFT		0x00
+#define BNXT_RE_CHIP_ID0_CHIP_REV_SFT		0x10
+#define BNXT_RE_CHIP_ID0_CHIP_MET_SFT		0x18
+struct bnxt_re_uctx_resp {
+	__u32 dev_id;
+	__u32 max_qp;
+	__u32 pg_size;
+	__u32 cqe_sz;
+	__u32 max_cqd;
+	__u32 chip_id0;
+	__u32 chip_id1;
+	__u32 modes;
+	__aligned_u64 comp_mask;
+} __attribute__((packed));
+
+enum {
+	BNXT_RE_COMP_MASK_PD_HAS_WC_DPI = 0x01,
+	BNXT_RE_COMP_MASK_PD_HAS_DBR_BAR_ADDR = 0x02,
+};
+
+struct bnxt_re_pd_resp {
+	__u32 pdid;
+	__u32 dpi;
+	__u64 dbr;
+	__u64 comp_mask;
+	__u32 wcdpi;
+	__u64 dbr_bar_addr;
+} __attribute__((packed));
+
+enum {
+	BNXT_RE_COMP_MASK_CQ_HAS_DB_INFO = 0x01,
+	BNXT_RE_COMP_MASK_CQ_HAS_WC_DPI = 0x02,
+	BNXT_RE_COMP_MASK_CQ_HAS_CQ_PAGE = 0x04,
+};
+
+enum {
+	BNXT_RE_COMP_MASK_CQ_REQ_HAS_CAP_MASK = 0x1
+};
+
+enum {
+	BNXT_RE_COMP_MASK_CQ_REQ_CAP_DBR_RECOVERY = 0x1,
+	BNXT_RE_COMP_MASK_CQ_REQ_CAP_DBR_PACING_NOTIFY = 0x2
+};
+
+#define BNXT_RE_IS_DBR_PACING_NOTIFY_CQ(_req)				\
+	(_req.comp_mask & BNXT_RE_COMP_MASK_CQ_REQ_HAS_CAP_MASK &&	\
+	 _req.cq_capability & BNXT_RE_COMP_MASK_CQ_REQ_CAP_DBR_PACING_NOTIFY)
+
+#define BNXT_RE_IS_DBR_RECOV_CQ(_req)					\
+	(_req.comp_mask & BNXT_RE_COMP_MASK_CQ_REQ_HAS_CAP_MASK &&	\
+	 _req.cq_capability & BNXT_RE_COMP_MASK_CQ_REQ_CAP_DBR_RECOVERY)
+
+struct bnxt_re_cq_req {
+	__u64 cq_va;
+	__u64 cq_handle;
+	__aligned_u64 comp_mask;
+	__u16 cq_capability;
+} __attribute__((packed));
+
+struct bnxt_re_cq_resp {
+	__u32 cqid;
+	__u32 tail;
+	__u32 phase;
+	__u32 rsvd;
+	__aligned_u64 comp_mask;
+	__u32 dpi;
+	__u64 dbr;
+	__u32 wcdpi;
+	__u64 uctx_cq_page;
+} __attribute__((packed));
+
+struct bnxt_re_resize_cq_req {
+	__u64 cq_va;
+} __attribute__((packed));
+
+struct bnxt_re_qp_req {
+	__u64 qpsva;
+	__u64 qprva;
+	__u64 qp_handle;
+} __attribute__((packed));
+
+struct bnxt_re_qp_resp {
+	__u32 qpid;
+} __attribute__((packed));
+
+struct bnxt_re_srq_req {
+	__u64 srqva;
+	__u64 srq_handle;
+} __attribute__((packed));
+
+struct bnxt_re_srq_resp {
+	__u32 srqid;
+} __attribute__((packed));
+
+/* Modify QP */
+enum {
+	BNXT_RE_COMP_MASK_MQP_EX_PPP_REQ_EN_MASK = 0x1,
+	BNXT_RE_COMP_MASK_MQP_EX_PPP_REQ_EN	 = 0x1,
+	BNXT_RE_COMP_MASK_MQP_EX_PATH_MTU_MASK	 = 0x2
+};
+
+struct bnxt_re_modify_qp_ex_req {
+	__aligned_u64 comp_mask;
+	__u32 dpi;
+	__u32 rsvd;
+} __packed;
+
+struct bnxt_re_modify_qp_ex_resp {
+	__aligned_u64 comp_mask;
+	__u32 ppp_st_idx;
+	__u32 path_mtu;
+} __packed;
+
+enum bnxt_re_shpg_offt {
+	BNXT_RE_BEG_RESV_OFFT	= 0x00,
+	BNXT_RE_AVID_OFFT	= 0x10,
+	BNXT_RE_AVID_SIZE	= 0x04,
+	BNXT_RE_END_RESV_OFFT	= 0xFF0
+};
+#endif
diff --git a/sys/dev/bnxt/bnxt_re/bnxt_re.h b/sys/dev/bnxt/bnxt_re/bnxt_re.h
new file mode 100644
index 000000000000..3bf0bbeb1061
--- /dev/null
+++ b/sys/dev/bnxt/bnxt_re/bnxt_re.h
@@ -0,0 +1,1075 @@
+/*
+ * Copyright (c) 2015-2024, Broadcom. All rights reserved.  The term
+ * Broadcom refers to Broadcom Limited and/or its subsidiaries.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS''
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
+ * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
+ * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * Description: main (header)
+ */
+
+#ifndef __BNXT_RE_H__
+#define __BNXT_RE_H__
+
+#include <linux/module.h>
+#include <linux/netdevice.h>
+#include <linux/mutex.h>
+#include <linux/list.h>
+#include <linux/rculist.h>
+#include <linux/spinlock.h>
+#include <net/ipv6.h>
+#include <linux/if_ether.h>
+#include <linux/debugfs.h>
+#include <linux/seq_file.h>
+#include <linux/interrupt.h>
+#include <linux/vmalloc.h>
+#include <linux/delay.h>
+#include <rdma/ib_verbs.h>
+#include <rdma/ib_user_verbs.h>
+#include <rdma/ib_umem.h>
+#include <rdma/ib_addr.h>
+#include <rdma/ib_mad.h>
+#include <rdma/ib_cache.h>
+#include <linux/pci.h>
+
+#include "bnxt.h"
+#include "bnxt_ulp.h"
+#include "hsi_struct_def.h"
+#include "qplib_res.h"
+#include "qplib_sp.h"
+#include "qplib_fp.h"
+#include "qplib_rcfw.h"
+#include "ib_verbs.h"
+#include "stats.h"
+
+#define ROCE_DRV_MODULE_NAME		"bnxt_re"
+#define ROCE_DRV_MODULE_VERSION		"230.0.133.0"
+#define ROCE_DRV_MODULE_RELDATE		"April 22, 2024"
+
+#define BNXT_RE_REF_WAIT_COUNT		20
+#define BNXT_RE_ROCE_V1_ETH_TYPE	0x8915
+#define BNXT_RE_ROCE_V2_PORT_NO		4791
+#define BNXT_RE_RES_FREE_WAIT_COUNT	1000
+
+#define BNXT_RE_PAGE_SHIFT_4K		(12)
+#define BNXT_RE_PAGE_SHIFT_8K		(13)
+#define BNXT_RE_PAGE_SHIFT_64K		(16)
+#define BNXT_RE_PAGE_SHIFT_2M		(21)
+#define BNXT_RE_PAGE_SHIFT_8M		(23)
+#define BNXT_RE_PAGE_SHIFT_1G		(30)
+
+#define BNXT_RE_PAGE_SIZE_4K		BIT(BNXT_RE_PAGE_SHIFT_4K)
+#define BNXT_RE_PAGE_SIZE_8K		BIT(BNXT_RE_PAGE_SHIFT_8K)
+#define BNXT_RE_PAGE_SIZE_64K		BIT(BNXT_RE_PAGE_SHIFT_64K)
+#define BNXT_RE_PAGE_SIZE_2M		BIT(BNXT_RE_PAGE_SHIFT_2M)
+#define BNXT_RE_PAGE_SIZE_8M		BIT(BNXT_RE_PAGE_SHIFT_8M)
+#define BNXT_RE_PAGE_SIZE_1G		BIT(BNXT_RE_PAGE_SHIFT_1G)
+
+#define BNXT_RE_MAX_MR_SIZE_LOW		BIT(BNXT_RE_PAGE_SHIFT_1G)
+#define BNXT_RE_MAX_MR_SIZE_HIGH	BIT(39)
+#define BNXT_RE_MAX_MR_SIZE		BNXT_RE_MAX_MR_SIZE_HIGH
+
+/* Number of MRs to reserve for PF, leaving remainder for VFs */
+#define BNXT_RE_RESVD_MR_FOR_PF		(32 * 1024)
+#define BNXT_RE_MAX_GID_PER_VF		128
+
+#define BNXT_RE_MAX_VF_QPS_PER_PF	(6 * 1024)
+
+/**
+ * min_not_zero - return the minimum that is _not_ zero, unless both are zero
+ * @x: value1
+ * @y: value2
+ */
+#define min_not_zero(x, y) ({			\
+	typeof(x) __x = (x);			\
+	typeof(y) __y = (y);			\
+	__x == 0 ? __y : ((__y == 0) ? __x : min(__x, __y)); })
+
+struct ib_mr_init_attr {
+	int		max_reg_descriptors;
+	u32		flags;
+};
+
+struct bnxt_re_dev;
+
+int bnxt_re_register_netdevice_notifier(struct notifier_block *nb);
+int bnxt_re_unregister_netdevice_notifier(struct notifier_block *nb);
+int ib_register_device_compat(struct bnxt_re_dev *rdev);
+
+#ifndef __struct_group
+#define __struct_group(TAG, NAME, ATTRS, MEMBERS...) \
+	union { \
+		struct { MEMBERS } ATTRS; \
+		struct TAG { MEMBERS } ATTRS NAME; \
+	}
+#endif /* __struct_group */
+#ifndef struct_group_attr
+#define struct_group_attr(NAME, ATTRS, MEMBERS...) \
+	__struct_group(/* no tag */, NAME, ATTRS, MEMBERS)
+#endif /* struct_group_attr */
+/*
+ * Percentage of resources of each type reserved for PF.
+ * Remaining resources are divided equally among VFs.
+ * [0, 100]
+ */
+
+#define BNXT_RE_RQ_WQE_THRESHOLD	32
+#define BNXT_RE_UD_QP_HW_STALL		0x400000
+
+/*
+ * Setting the default ack delay value to 16, which means
+ * the default timeout is approx. 260ms(4 usec * 2 ^(timeout))
+ */
+
+#define BNXT_RE_DEFAULT_ACK_DELAY	16
+#define BNXT_RE_BOND_PF_MAX		2
+
+#define BNXT_RE_STATS_CTX_UPDATE_TIMER	250
+#define BNXT_RE_30SEC_MSEC		(30 * 1000)
+
+#define BNXT_RE_BOND_RESCHED_CNT	10
+
+#define BNXT_RE_CHIP_NUM_57454         0xC454
+#define BNXT_RE_CHIP_NUM_57452         0xC452
+
+#define BNXT_RE_CHIP_NUM_5745X(chip_num)          \
+	((chip_num) == BNXT_RE_CHIP_NUM_57454 ||       \
+	 (chip_num) == BNXT_RE_CHIP_NUM_57452)
+
+#define BNXT_RE_MIN_KERNEL_QP_TX_DEPTH	4096
+#define BNXT_RE_STOP_QPS_BUDGET		200
+
+#define BNXT_RE_HWRM_CMD_TIMEOUT(rdev) \
+		((rdev)->chip_ctx->hwrm_cmd_max_timeout * 1000)
+
+extern unsigned int min_tx_depth;
+extern struct mutex bnxt_re_dev_lock;
+extern struct mutex bnxt_re_mutex;
+extern struct list_head bnxt_re_dev_list;
+
+struct bnxt_re_ring_attr {
+	dma_addr_t	*dma_arr;
+	int		pages;
+	int	 	type;
+	u32		depth;
+	u32		lrid; /* Logical ring id */
+	u16		flags;
+	u8		mode;
+	u8		rsvd;
+};
+
+#define BNXT_RE_MAX_DEVICES		256
+#define BNXT_RE_MSIX_FROM_MOD_PARAM	-1
+#define BNXT_RE_MIN_MSIX		2
+#define BNXT_RE_MAX_MSIX_VF		2
+#define BNXT_RE_MAX_MSIX_PF		9
+#define BNXT_RE_MAX_MSIX_NPAR_PF	5
+#define BNXT_RE_MAX_MSIX		64
+#define BNXT_RE_MAX_MSIX_GEN_P5_PF	BNXT_RE_MAX_MSIX
+#define BNXT_RE_GEN_P5_MAX_VF		64
+
+struct bnxt_re_nq_record {
+	struct bnxt_msix_entry	msix_entries[BNXT_RE_MAX_MSIX];
+	/* FP Notification Queue (CQ & SRQ) */
+	struct bnxt_qplib_nq    nq[BNXT_RE_MAX_MSIX];
+	int			num_msix;
+	int			max_init;
+	struct mutex		load_lock;
+};
+
+struct bnxt_re_work {
+	struct work_struct	work;
+	unsigned long		event;
+	struct bnxt_re_dev      *rdev;
+	struct ifnet		*vlan_dev;
+	bool do_lag;
+
+	/* netdev where we received the event */
+	struct ifnet *netdev;
+	struct auxiliary_device *adev;
+};
+
+/*
+ * Data structure and defines to handle
+ * recovery
+ */
+#define BNXT_RE_RECOVERY_IB_UNINIT_WAIT_RETRY   20
+#define BNXT_RE_RECOVERY_IB_UNINIT_WAIT_TIME_MS 30000 /* 30sec timeout */
+#define BNXT_RE_PRE_RECOVERY_REMOVE 0x1
+#define BNXT_RE_COMPLETE_REMOVE 0x2
+#define BNXT_RE_POST_RECOVERY_INIT 0x4
+#define BNXT_RE_COMPLETE_INIT 0x8
+#define BNXT_RE_COMPLETE_SHUTDOWN 0x10
+
+/* QP1 SQ entry data strucutre */
+struct bnxt_re_sqp_entries {
+	u64 wrid;
+	struct bnxt_qplib_sge sge;
+	/* For storing the actual qp1 cqe */
+	struct bnxt_qplib_cqe cqe;
+	struct bnxt_re_qp *qp1_qp;
+};
+
+/* GSI QP mode enum */
+enum bnxt_re_gsi_mode {
+	BNXT_RE_GSI_MODE_INVALID = 0,
+	BNXT_RE_GSI_MODE_ALL = 1,
+	BNXT_RE_GSI_MODE_ROCE_V1,
+	BNXT_RE_GSI_MODE_ROCE_V2_IPV4,
+	BNXT_RE_GSI_MODE_ROCE_V2_IPV6,
+	BNXT_RE_GSI_MODE_UD
+};
+
+enum bnxt_re_roce_cap {
+	BNXT_RE_FLAG_ROCEV1_CAP = 1,
+	BNXT_RE_FLAG_ROCEV2_CAP,
+	BNXT_RE_FLAG_ROCEV1_V2_CAP,
+};
+
+#define BNXT_RE_MAX_GSI_SQP_ENTRIES	1024
+struct bnxt_re_gsi_context {
+	u8	gsi_qp_mode;
+	bool	first_cq_created;
+	/* Start: used only in gsi_mode_all */
+	struct	bnxt_re_qp *gsi_qp;
+	struct	bnxt_re_qp *gsi_sqp;
+	struct	bnxt_re_ah *gsi_sah;
+	struct	bnxt_re_sqp_entries *sqp_tbl;
+	/* End: used only in gsi_mode_all */
+};
+
+struct bnxt_re_tc_rec {
+	u8 cos_id_roce;
+	u8 tc_roce;
+	u8 cos_id_cnp;
+	u8 tc_cnp;
+	u8 tc_def;
+	u8 cos_id_def;
+	u8 max_tc;
+	u8 roce_prio;
+	u8 cnp_prio;
+	u8 roce_dscp;
+	u8 cnp_dscp;
+	u8 prio_valid;
+	u8 dscp_valid;
+	bool ecn_enabled;
+	bool serv_type_enabled;
+	u64 cnp_dscp_bv;
+	u64 roce_dscp_bv;
+};
+
+struct bnxt_re_dscp2pri {
+	u8 dscp;
+	u8 mask;
+	u8 pri;
+};
+
+struct bnxt_re_cos2bw_cfg {
+	u8	pad[3];
+	struct_group_attr(cfg, __packed,
+		u8	queue_id;
+		__le32	min_bw;
+		__le32	max_bw;
+		u8	tsa;
+		u8	pri_lvl;
+		u8	bw_weight;
+	);
+	u8	unused;
+};
+
+#define BNXT_RE_AEQ_IDX			0
+#define BNXT_RE_MAX_SGID_ENTRIES	256
+
+#define BNXT_RE_DBGFS_FILE_MEM          65536
+enum {
+	BNXT_RE_STATS_QUERY = 1,
+	BNXT_RE_QP_QUERY = 2,
+	BNXT_RE_SERVICE_FN_QUERY = 3,
+};
+
+struct bnxt_re_dbg_file {
+	struct bnxt_re_dev *rdev;
+	u32 type;
+	union {
+		struct bnxt_qplib_query_stats_info sinfo;
+		struct bnxt_qplib_query_fn_info fninfo;
+	}params;
+	char dbg_buf[BNXT_RE_DBGFS_FILE_MEM];
+};
+
+struct bnxt_re_debug_entries {
+	/* Dir entries */
+	struct dentry *qpinfo_dir;
+	struct dentry *service_fn_dir;
+	/* file entries */
+	struct dentry *stat_query;
+	struct bnxt_re_dbg_file stat_file;
+	struct dentry *qplist_query;
+	struct bnxt_re_dbg_file qp_file;
+	struct dentry *service_fn_query;
+	struct bnxt_re_dbg_file service_fn_file;
+};
+
+struct bnxt_re_en_dev_info {
+	struct list_head en_list;
+	struct bnxt_en_dev *en_dev;
+	struct bnxt_re_dev *rdev;
+	unsigned long flags;
+#define BNXT_RE_FLAG_EN_DEV_NETDEV_REG		0
+#define BNXT_RE_FLAG_EN_DEV_PRIMARY_DEV		1
+#define BNXT_RE_FLAG_EN_DEV_SECONDARY_DEV	2
+	u8 wqe_mode;
+	u8 gsi_mode;
+	bool te_bypass;
+	bool ib_uninit_done;
+	u32 num_msix_requested;
+	wait_queue_head_t waitq;
+};
+
+#define BNXT_RE_DB_FIFO_ROOM_MASK_P5	0x1FFF8000
+#define BNXT_RE_MAX_FIFO_DEPTH_P5	0x2c00
+#define BNXT_RE_DB_FIFO_ROOM_SHIFT	15
+
+#define BNXT_RE_DB_FIFO_ROOM_MASK_P7	0x3FFF8000
+#define BNXT_RE_MAX_FIFO_DEPTH_P7	0x8000
+
+#define BNXT_RE_DB_FIFO_ROOM_MASK(ctx)	\
+	(_is_chip_p7((ctx)) ? \
+	 BNXT_RE_DB_FIFO_ROOM_MASK_P7 :\
+	 BNXT_RE_DB_FIFO_ROOM_MASK_P5)
+#define BNXT_RE_MAX_FIFO_DEPTH(ctx)	\
+	(_is_chip_p7((ctx)) ? \
+	 BNXT_RE_MAX_FIFO_DEPTH_P7 :\
+	 BNXT_RE_MAX_FIFO_DEPTH_P5)
+
+struct bnxt_dbq_nq_list {
+	int num_nql_entries;
+	u16 nq_id[16];
+};
+
+#define BNXT_RE_ASYNC_ERR_REP_BASE(_type)				\
+	 (ASYNC_EVENT_CMPL_ERROR_REPORT_BASE_EVENT_DATA1_ERROR_##_type)
+
+#define BNXT_RE_ASYNC_ERR_DBR_TRESH(_type)				\
+	 (ASYNC_EVENT_CMPL_ERROR_REPORT_DOORBELL_DROP_THRESHOLD_##_type)
+
+#define BNXT_RE_EVENT_DBR_EPOCH(data)					\
+	(((data) &							\
+	  BNXT_RE_ASYNC_ERR_DBR_TRESH(EVENT_DATA1_EPOCH_MASK)) >>	\
+	 BNXT_RE_ASYNC_ERR_DBR_TRESH(EVENT_DATA1_EPOCH_SFT))
+
+#define BNXT_RE_EVENT_ERROR_REPORT_TYPE(data1)				\
+	(((data1) &							\
+	  BNXT_RE_ASYNC_ERR_REP_BASE(TYPE_MASK))  >>			\
+	 BNXT_RE_ASYNC_ERR_REP_BASE(TYPE_SFT))
+
+#define BNXT_RE_DBR_LIST_ADD(_rdev, _res, _type)			\
+{									\
+	spin_lock(&(_rdev)->res_list[_type].lock);			\
+	list_add_tail(&(_res)->dbr_list,				\
+		      &(_rdev)->res_list[_type].head);			\
+	spin_unlock(&(_rdev)->res_list[_type].lock);			\
+}
+
+#define BNXT_RE_DBR_LIST_DEL(_rdev, _res, _type)			\
+{									\
+	spin_lock(&(_rdev)->res_list[_type].lock);			\
+	list_del(&(_res)->dbr_list);					\
+	spin_unlock(&(_rdev)->res_list[_type].lock);			\
+}
+
+#define BNXT_RE_CQ_PAGE_LIST_ADD(_uctx, _cq)				\
+{									\
+	mutex_lock(&(_uctx)->cq_lock);					\
+	list_add_tail(&(_cq)->cq_list, &(_uctx)->cq_list);		\
+	mutex_unlock(&(_uctx)->cq_lock);				\
+}
+
+#define BNXT_RE_CQ_PAGE_LIST_DEL(_uctx, _cq)				\
+{									\
+	mutex_lock(&(_uctx)->cq_lock);					\
+	list_del(&(_cq)->cq_list);					\
+	mutex_unlock(&(_uctx)->cq_lock);				\
+}
+
+#define BNXT_RE_NETDEV_EVENT(event, x)					\
+	do {								\
+		if ((event) == (x))					\
+			return #x;					\
+	} while (0)
+
+/* Do not change the seq of this enum which is followed by dbr recov */
+enum {
+	BNXT_RE_RES_TYPE_CQ = 0,
+	BNXT_RE_RES_TYPE_UCTX,
+	BNXT_RE_RES_TYPE_QP,
+	BNXT_RE_RES_TYPE_SRQ,
+	BNXT_RE_RES_TYPE_MAX
+};
+
+struct bnxt_re_dbr_res_list {
+	struct list_head head;
+	spinlock_t lock;
+};
+
+struct bnxt_re_dbr_drop_recov_work {
+	struct work_struct work;
+	struct bnxt_re_dev *rdev;
+	u32 curr_epoch;
+};
+
+struct bnxt_re_aer_work {
+	struct work_struct work;
+	struct bnxt_re_dev *rdev;
+};
+
+struct bnxt_re_dbq_stats {
+	u64 fifo_occup_slab_1;
+	u64 fifo_occup_slab_2;
+	u64 fifo_occup_slab_3;
+	u64 fifo_occup_slab_4;
+	u64 fifo_occup_water_mark;
+	u64 do_pacing_slab_1;
+	u64 do_pacing_slab_2;
+	u64 do_pacing_slab_3;
+	u64 do_pacing_slab_4;
+	u64 do_pacing_slab_5;
+	u64 do_pacing_water_mark;
+};
+
+/* Device debug statistics */
+struct bnxt_re_drv_dbg_stats {
+	struct bnxt_re_dbq_stats dbq;
+};
+
+/* DB pacing counters */
+struct bnxt_re_dbr_sw_stats {
+	u64 dbq_int_recv;
+	u64 dbq_int_en;
+	u64 dbq_pacing_resched;
+	u64 dbq_pacing_complete;
+	u64 dbq_pacing_alerts;
+	u64 dbr_drop_recov_events;
+	u64 dbr_drop_recov_timeouts;
+	u64 dbr_drop_recov_timeout_users;
+	u64 dbr_drop_recov_event_skips;
+};
+
+struct bnxt_re_dev {
+	struct ib_device		ibdev;
+	struct list_head		list;
+	atomic_t			ref_count;
+	atomic_t			sched_count;
+	unsigned long			flags;
+#define BNXT_RE_FLAG_NETDEV_REGISTERED		0
+#define BNXT_RE_FLAG_IBDEV_REGISTERED		1
+#define BNXT_RE_FLAG_GOT_MSIX			2
+#define BNXT_RE_FLAG_HAVE_L2_REF		3
+#define BNXT_RE_FLAG_ALLOC_RCFW			4
+#define BNXT_RE_FLAG_NET_RING_ALLOC		5
+#define BNXT_RE_FLAG_RCFW_CHANNEL_EN		6
+#define BNXT_RE_FLAG_ALLOC_CTX			7
+#define BNXT_RE_FLAG_STATS_CTX_ALLOC		8
+#define BNXT_RE_FLAG_STATS_CTX2_ALLOC		9
+#define BNXT_RE_FLAG_RCFW_CHANNEL_INIT		10
+#define BNXT_RE_FLAG_WORKER_REG			11
+#define BNXT_RE_FLAG_TBLS_ALLOCINIT		12
+#define BNXT_RE_FLAG_SETUP_NQ			13
+#define BNXT_RE_FLAG_BOND_DEV_REGISTERED	14
+#define BNXT_RE_FLAG_PER_PORT_DEBUG_INFO	15
+#define BNXT_RE_FLAG_DEV_LIST_INITIALIZED	16
+#define BNXT_RE_FLAG_ERR_DEVICE_DETACHED	17
+#define BNXT_RE_FLAG_INIT_DCBX_CC_PARAM		18
+#define BNXT_RE_FLAG_STOP_IN_PROGRESS		20
+#define BNXT_RE_FLAG_ISSUE_ROCE_STATS		29
+#define BNXT_RE_FLAG_ISSUE_CFA_FLOW_STATS	30
+	struct ifnet			*netdev;
+	struct auxiliary_device		*adev;
+	struct bnxt_qplib_chip_ctx	*chip_ctx;
+	struct bnxt_en_dev		*en_dev;
+	struct bnxt_re_nq_record	nqr;
+	int				id;
+	struct delayed_work		worker;
+	u16				worker_30s;
+	struct bnxt_re_tc_rec		tc_rec[2];
+	u8				cur_prio_map;
+	/* RCFW Channel */
+	struct bnxt_qplib_rcfw		rcfw;
+	/* Device Resources */
+	struct bnxt_qplib_dev_attr	*dev_attr;
+	struct bnxt_qplib_res		qplib_res;
+	struct bnxt_qplib_dpi		dpi_privileged;
+	struct bnxt_qplib_cc_param	cc_param;
+	struct mutex			cc_lock;
+	struct mutex			qp_lock;
+	struct list_head		qp_list;
+	u8				roce_mode;
+
+	/* Max of 2 lossless traffic class supported per port */
+	u16				cosq[2];
+	/* Start: QP for handling QP1 packets */
+	struct bnxt_re_gsi_context	gsi_ctx;
+	/* End: QP for handling QP1 packets */
+	bool				is_virtfn;
+	u32				num_vfs;
+	u32				espeed;
+	/*
+	 * For storing the speed of slave interfaces.
+	 * Same as espeed when bond is not configured
+	 */
+	u32				sl_espeed;
+	/* To be used for a workaround for ISER stack */
+	u32				min_tx_depth;
+	/* To enable qp debug info. Disabled during driver load */
+	u32				en_qp_dbg;
+	/* Array to handle gid mapping */
+	char				*gid_map;
+
+	struct bnxt_re_device_stats	stats;
+	struct bnxt_re_drv_dbg_stats	*dbg_stats;
+	/* debugfs to expose per port information*/
+	struct dentry                   *port_debug_dir;
+	struct dentry                   *info;
+	struct dentry                   *drv_dbg_stats;
+	struct dentry                   *sp_perf_stats;
+	struct dentry                   *pdev_debug_dir;
+	struct dentry                   *pdev_qpinfo_dir;
+	struct bnxt_re_debug_entries	*dbg_ent;
+	struct workqueue_struct		*resolve_wq;
+	struct list_head		mac_wq_list;
+	struct workqueue_struct		*dcb_wq;
+	struct workqueue_struct		*aer_wq;
+	u32				event_bitmap[3];
+	bool unreg_sched;
+	u64	dbr_throttling_reg_off;
+	u64	dbr_aeq_arm_reg_off;
+	u64	dbr_db_fifo_reg_off;
+	void *dbr_page;
+	u64 dbr_bar_addr;
+	u32 pacing_algo_th;
+	u32 pacing_en_int_th;
+	u32 do_pacing_save;
+	struct workqueue_struct		*dbq_wq;
+	struct workqueue_struct		*dbr_drop_recov_wq;
+	struct work_struct		dbq_fifo_check_work;
+	struct delayed_work		dbq_pacing_work;
+	/* protect DB pacing */
+	struct mutex dbq_lock;
+	/* Control DBR pacing feature. Set if enabled */
+	bool dbr_pacing;
+	/* Control DBR recovery feature. Set if enabled */
+	bool dbr_drop_recov;
+	bool user_dbr_drop_recov;
+	/* DBR recovery feature. Set if running */
+	bool dbr_recovery_on;
+	u32 user_dbr_drop_recov_timeout;
+	 /*
+	  * Value used for pacing algo when pacing is active
+	  */
+#define BNXT_RE_MAX_DBR_DO_PACING 0xFFFF
+	u32 dbr_do_pacing;
+	u32 dbq_watermark; /* Current watermark set in HW registers */
+	u32 dbq_nq_id; /* Current NQ ID for DBQ events */
+	u32 dbq_pacing_time; /* ms */
+	u32 dbr_def_do_pacing; /* do_pacing when no congestion */
+	u32 dbr_evt_curr_epoch;
+	bool dbq_int_disable;
+
+	bool mod_exit;
+	struct bnxt_re_dbr_sw_stats *dbr_sw_stats;
+	struct bnxt_re_dbr_res_list res_list[BNXT_RE_RES_TYPE_MAX];
+	struct bnxt_dbq_nq_list nq_list;
+	char dev_name[IB_DEVICE_NAME_MAX];
+	atomic_t dbq_intr_running;
+	u32 num_msix_requested;
+	unsigned char	*dev_addr; /* For netdev->dev_addr */
+};
+
+#define BNXT_RE_RESOLVE_RETRY_COUNT_US	5000000 /* 5 sec */
+struct bnxt_re_resolve_dmac_work{
+	struct work_struct      work;
+	struct list_head	list;
+	struct bnxt_re_dev 	*rdev;
+	struct ib_ah_attr	*ah_attr;
+	struct bnxt_re_ah_info *ah_info;
+	atomic_t		status_wait;
+};
+
+static inline u8 bnxt_re_get_prio(u8 prio_map)
+{
+	u8 prio = 0xFF;
+
+	for (prio = 0; prio < 8; prio++)
+		if (prio_map & (1UL << prio))
+			break;
+	return prio;
+}
+
+/* This should be called with bnxt_re_dev_lock mutex held */
+static inline bool __bnxt_re_is_rdev_valid(struct bnxt_re_dev *rdev)
+{
+	struct bnxt_re_dev *tmp_rdev;
+
+	list_for_each_entry(tmp_rdev, &bnxt_re_dev_list, list) {
+		if (rdev == tmp_rdev)
+			return true;
+	}
+	return false;
+}
+
+static inline bool bnxt_re_is_rdev_valid(struct bnxt_re_dev *rdev)
+{
+	struct bnxt_re_dev *tmp_rdev;
+
+	mutex_lock(&bnxt_re_dev_lock);
+	list_for_each_entry(tmp_rdev, &bnxt_re_dev_list, list) {
+		if (rdev == tmp_rdev) {
+			mutex_unlock(&bnxt_re_dev_lock);
+			return true;
+		}
+	}
+	mutex_unlock(&bnxt_re_dev_lock);
+
+	pr_debug("bnxt_re: %s : Invalid rdev received rdev = %p\n",
+		 __func__, rdev);
+	return false;
+}
+
+int bnxt_re_send_hwrm_cmd(struct bnxt_re_dev *rdev, void *cmd,
+			  int cmdlen);
+void bnxt_re_stopqps_and_ib_uninit(struct bnxt_re_dev *rdev);
+int bnxt_re_set_hwrm_dscp2pri(struct bnxt_re_dev *rdev,
+			      struct bnxt_re_dscp2pri *d2p, u16 count,
+			      u16 target_id);
+int bnxt_re_query_hwrm_dscp2pri(struct bnxt_re_dev *rdev,
+				struct bnxt_re_dscp2pri *d2p, u16 *count,
+				u16 target_id);
+int bnxt_re_query_hwrm_qportcfg(struct bnxt_re_dev *rdev,
+				struct bnxt_re_tc_rec *cnprec, u16 tid);
+int bnxt_re_hwrm_cos2bw_qcfg(struct bnxt_re_dev *rdev, u16 target_id,
+			     struct bnxt_re_cos2bw_cfg *cfg);
+int bnxt_re_hwrm_cos2bw_cfg(struct bnxt_re_dev *rdev, u16 target_id,
+			    struct bnxt_re_cos2bw_cfg *cfg);
+int bnxt_re_hwrm_pri2cos_cfg(struct bnxt_re_dev *rdev,
+			     u16 target_id, u16 port_id,
+			     u8 *cos_id_map, u8 pri_map);
+int bnxt_re_prio_vlan_tx_update(struct bnxt_re_dev *rdev);
+int bnxt_re_get_slot_pf_count(struct bnxt_re_dev *rdev);
+struct bnxt_re_dev *bnxt_re_get_peer_pf(struct bnxt_re_dev *rdev);
+struct bnxt_re_dev *bnxt_re_from_netdev(struct ifnet *netdev);
+u8 bnxt_re_get_priority_mask(struct bnxt_re_dev *rdev, u8 selector);
+struct bnxt_qplib_nq * bnxt_re_get_nq(struct bnxt_re_dev *rdev);
+void bnxt_re_put_nq(struct bnxt_re_dev *rdev, struct bnxt_qplib_nq *nq);
+
+#define to_bnxt_re(ptr, type, member)	\
+	container_of(ptr, type, member)
+
+#define to_bnxt_re_dev(ptr, member)	\
+	container_of((ptr), struct bnxt_re_dev, member)
+
+/* Even number functions from port 0 and odd number from port 1 */
+#define BNXT_RE_IS_PORT0(rdev) (!(rdev->en_dev->pdev->devfn & 1))
+
+#define BNXT_RE_ROCE_V1_PACKET		0
+#define BNXT_RE_ROCEV2_IPV4_PACKET	2
+#define BNXT_RE_ROCEV2_IPV6_PACKET	3
+#define BNXT_RE_ACTIVE_MAP_PORT1    0x1  /*port-1 active */
+#define BNXT_RE_ACTIVE_MAP_PORT2    0x2  /*port-2 active */
+
+#define BNXT_RE_MEMBER_PORT_MAP		(BNXT_RE_ACTIVE_MAP_PORT1 | \
+					BNXT_RE_ACTIVE_MAP_PORT2)
+
+#define	rdev_to_dev(rdev)	((rdev) ? (&(rdev)->ibdev.dev) : NULL)
+
+void bnxt_re_set_dma_device(struct ib_device *ibdev, struct bnxt_re_dev *rdev);
+bool bnxt_re_is_rdev_valid(struct bnxt_re_dev *rdev);
+
+#define bnxt_re_rdev_ready(rdev)	(bnxt_re_is_rdev_valid(rdev) && \
+					 (test_bit(BNXT_RE_FLAG_IBDEV_REGISTERED, &rdev->flags)))
+#define BNXT_RE_SRIOV_CFG_TIMEOUT 6
+
+int bnxt_re_get_device_stats(struct bnxt_re_dev *rdev);
+void bnxt_re_remove_device(struct bnxt_re_dev *rdev, u8 removal_type,
+			   struct auxiliary_device *aux_dev);
+void bnxt_re_destroy_lag(struct bnxt_re_dev **rdev);
+int bnxt_re_add_device(struct bnxt_re_dev **rdev,
+		       struct ifnet *netdev,
+		       u8 qp_mode, u8 op_type, u8 wqe_mode, u32 num_msix_requested,
+		       struct auxiliary_device *aux_dev);
+void bnxt_re_create_base_interface(bool primary);
+int bnxt_re_schedule_work(struct bnxt_re_dev *rdev, unsigned long event,
+			  struct ifnet *vlan_dev,
+			  struct ifnet *netdev,
+			  struct auxiliary_device *aux_dev);
+void bnxt_re_get_link_speed(struct bnxt_re_dev *rdev);
+int _bnxt_re_ib_init(struct bnxt_re_dev *rdev);
+int _bnxt_re_ib_init2(struct bnxt_re_dev *rdev);
+void bnxt_re_init_resolve_wq(struct bnxt_re_dev *rdev);
+void bnxt_re_uninit_resolve_wq(struct bnxt_re_dev *rdev);
+
+/* The rdev ref_count is to protect immature removal of the device */
+static inline void bnxt_re_hold(struct bnxt_re_dev *rdev)
+{
+	atomic_inc(&rdev->ref_count);
+	dev_dbg(rdev_to_dev(rdev),
+		"Hold ref_count = 0x%x", atomic_read(&rdev->ref_count));
+}
+
+static inline void bnxt_re_put(struct bnxt_re_dev *rdev)
+{
+	atomic_dec(&rdev->ref_count);
+	dev_dbg(rdev_to_dev(rdev),
+		"Put ref_count = 0x%x", atomic_read(&rdev->ref_count));
+}
+
+/*
+* Responder Error reason codes
+* FIXME: Remove these when the defs
+* are properly included in hsi header
+*/
+enum res_err_state_reason {
+	/* No error. */
+	CFCQ_RES_ERR_STATE_REASON_NO_ERROR = 0,
+	/*
+	 * Incoming Send, RDMA write, or RDMA read exceeds the maximum
+	 * transfer length. Detected on RX first and only packets for
+	 * write. Detected on RX request for read. This is an RX
+	 * Detected Error.
+	 */
+	CFCQ_RES_ERR_STATE_REASON_RES_EXCEED_MAX,
+	/*
+	 * RDMA write payload size does not match write length. Detected
+	 * when total write payload is not equal to the RDMA write
+	 * length that was given in the first or only packet of the
+	 * request. This is an RX Detected Error.
+	 */
+	CFCQ_RES_ERR_STATE_REASON_RES_PAYLOAD_LENGTH_MISMATCH,
+	/*
+	 * Send payload exceeds RQ/SRQ WQE buffer capacity. The total
+	 * send payload that arrived is more than the size of the WQE
+	 * buffer that was fetched from the RQ/SRQ. This is an RX
+	 * Detected Error.
+	 */
+	CFCQ_RES_ERR_STATE_REASON_RES_EXCEEDS_WQE,
+	/*
+	 * Responder detected opcode error. * First, only, middle, last
+	 * for incoming requests are improperly ordered with respect to
+	 * previous (PSN) packet. * First or middle packet is not full
+	 * MTU size. This is an RX Detected Error.
+	 */
+	CFCQ_RES_ERR_STATE_REASON_RES_OPCODE_ERROR,
+	/*
+	 * PSN sequence error retry limit exceeded. The responder
+	 * encountered a PSN sequence error for the same PSN too many
+	 * times. This can occur via implicit or explicit NAK. This is
+	 * an RX Detected Error.
+	 */
+	CFCQ_RES_ERR_STATE_REASON_RES_PSN_SEQ_ERROR_RETRY_LIMIT,
+	/*
+	 * Invalid R_Key. An incoming request contained an R_Key that
+	 * did not reference a valid MR/MW. This error may be detected
+	 * by the RX engine for RDMA write or by the TX engine for RDMA
+	 * read (detected while servicing IRRQ). This is an RX Detected
+	 * Error.
+	 */
+	CFCQ_RES_ERR_STATE_REASON_RES_RX_INVALID_R_KEY,
+	/*
+	 * Domain error. An incoming request specified an R_Key which
+	 * referenced a MR/MW that was not in the same PD as the QP on
+	 * which the request arrived. This is an RX Detected Error.
+	 */
+	CFCQ_RES_ERR_STATE_REASON_RES_RX_DOMAIN_ERROR,
+	/*
+	 * No permission. An incoming request contained an R_Key that
+	 * referenced a MR/MW which did not have the access permission
+	 * needed for the operation. This is an RX Detected Error.
+	 */
+	CFCQ_RES_ERR_STATE_REASON_RES_RX_NO_PERMISSION,
+	/*
+	 * Range error. An incoming request had a combination of R_Key,
+	 * VA, and length that was out of bounds of the associated
+	 * MR/MW. This is an RX Detected Error.
+	 */
+	CFCQ_RES_ERR_STATE_REASON_RES_RX_RANGE_ERROR,
+	/*
+	 * Invalid R_Key. An incoming request contained an R_Key that
+	 * did not reference a valid MR/MW. This error may be detected
+	 * by the RX engine for RDMA write or by the TX engine for RDMA
+	 * read (detected while servicing IRRQ). This is a TX Detected
+	 * Error.
+	 */
+	CFCQ_RES_ERR_STATE_REASON_RES_TX_INVALID_R_KEY,
+	/*
+	 * Domain error. An incoming request specified an R_Key which
+	 * referenced a MR/MW that was not in the same PD as the QP on
+	 * which the request arrived. This is a TX Detected Error.
+	 */
+	CFCQ_RES_ERR_STATE_REASON_RES_TX_DOMAIN_ERROR,
+	/*
+	 * No permission. An incoming request contained an R_Key that
+	 * referenced a MR/MW which did not have the access permission
+	 * needed for the operation. This is a TX Detected Error.
+	 */
+	CFCQ_RES_ERR_STATE_REASON_RES_TX_NO_PERMISSION,
+	/*
+	 * Range error. An incoming request had a combination of R_Key,
+	 * VA, and length that was out of bounds of the associated
+	 * MR/MW. This is a TX Detected Error.
+	 */
+	CFCQ_RES_ERR_STATE_REASON_RES_TX_RANGE_ERROR,
+	/*
+	 * IRRQ overflow. The peer sent us more RDMA read or atomic
+	 * requests than the negotiated maximum. This is an RX Detected
+	 * Error.
+	 */
+	CFCQ_RES_ERR_STATE_REASON_RES_IRRQ_OFLOW,
+	/*
+	 * Unsupported opcode. The peer sent us a request with an opcode
+	 * for a request type that is not supported on this QP. This is
+	 * an RX Detected Error.
+	 */
+	CFCQ_RES_ERR_STATE_REASON_RES_UNSUPPORTED_OPCODE,
+	/*
+	 * Unaligned atomic operation. The VA of an atomic request is on
+	 * a memory boundary that prevents atomic execution. This is an
+	 * RX Detected Error.
+	 */
+	CFCQ_RES_ERR_STATE_REASON_RES_UNALIGN_ATOMIC,
+	/*
+	 * Remote invalidate error. A send with invalidate request
+	 * arrived in which the R_Key to invalidate did not describe a
+	 * MR/MW which could be invalidated. RQ WQE completes with error
+	 * status. This error is only reported if the send operation did
+	 * not fail. If the send operation failed then the remote
+	 * invalidate error is not reported. This is an RX Detected
+	 * Error.
+	 */
+	CFCQ_RES_ERR_STATE_REASON_RES_REM_INVALIDATE,
+	/*
+	 * Local memory error. An RQ/SRQ SGE described an inaccessible
+	 * memory. This is an RX Detected Error.
+	 */
+	CFCQ_RES_ERR_STATE_REASON_RES_MEMORY_ERROR,
+	/*
+	 * SRQ in error. The QP is moving to error state because it
+	 * found SRQ it uses in error. This is an RX Detected Error.
+	 */
+	CFCQ_RES_ERR_STATE_REASON_RES_SRQ_ERROR,
+	/*
+	 * Completion error. No CQE space available on queue or CQ not
+	 * in VALID state. This is a Completion Detected Error.
+	 */
+	CFCQ_RES_ERR_STATE_REASON_RES_CMP_ERROR,
+	/*
+	 * Invalid R_Key while resending responses to duplicate request.
+	 * This is a TX Detected Error.
+	 */
+	CFCQ_RES_ERR_STATE_REASON_RES_IVALID_DUP_RKEY,
+	/*
+	 * Problem was found in the format of a WQE in the RQ/SRQ. This
+	 * is an RX Detected Error.
+	 */
+	CFCQ_RES_ERR_STATE_REASON_RES_WQE_FORMAT_ERROR,
+	/*
+	 * A load error occurred on an attempt to load the CQ Context.
+	 * This is a Completion Detected Error.
+	 */
+	CFCQ_RES_ERR_STATE_REASON_RES_CQ_LOAD_ERROR = 0x18,
+	/*
+	 * A load error occurred on an attempt to load the SRQ Context.
+	 * This is an RX Detected Error.
+	 */
+	CFCQ_RES_ERR_STATE_REASON_RES_SRQ_LOAD_ERROR,
+	/*
+	 * A fatal error was detected on an attempt to read from or
+	 * write to PCIe on the transmit side. This error is detected by
+	 * the TX side, but has the priority of a Completion Detected
+	 * Error.
+	 */
+	CFCQ_RES_ERR_STATE_REASON_RES_TX_PCI_ERROR = 0x1b,
+	/*
+	 * A fatal error was detected on an attempt to read from or
+	 * write to PCIe on the receive side. This error is detected by
+	 * the RX side (or CAGR), but has the priority of a Completion
+	 * Detected Error.
+	 */
+	CFCQ_RES_ERR_STATE_REASON_RES_RX_PCI_ERROR = 0x1c
+};
+
+int bnxt_re_host_pf_id_query(struct bnxt_re_dev *rdev,
+			     struct bnxt_qplib_query_fn_info *fn_info,
+			     u32 *pf_mask, u32 *first_pf);
+
+/* Default DCBx and CC values */
+#define BNXT_RE_DEFAULT_CNP_DSCP	48
+#define BNXT_RE_DEFAULT_CNP_PRI		7
+#define BNXT_RE_DEFAULT_ROCE_DSCP	26
+#define BNXT_RE_DEFAULT_ROCE_PRI	3
+
+#define BNXT_RE_DEFAULT_L2_BW		50
+#define BNXT_RE_DEFAULT_ROCE_BW		50
+
+#define ROCE_PRIO_VALID	0x0
+#define CNP_PRIO_VALID	0x1
+#define ROCE_DSCP_VALID	0x0
+#define CNP_DSCP_VALID	0x1
+
+int bnxt_re_get_pri_dscp_settings(struct bnxt_re_dev *rdev,
+				  u16 target_id,
+				  struct bnxt_re_tc_rec *tc_rec);
+
+int bnxt_re_setup_dscp(struct bnxt_re_dev *rdev);
+int bnxt_re_clear_dscp(struct bnxt_re_dev *rdev);
+int bnxt_re_setup_cnp_cos(struct bnxt_re_dev *rdev, bool reset);
+
+static inline enum ib_port_state bnxt_re_get_link_state(struct bnxt_re_dev *rdev)
+{
+	if (rdev->netdev->if_drv_flags & IFF_DRV_RUNNING &&
+	    rdev->netdev->if_link_state == LINK_STATE_UP)
+		return IB_PORT_ACTIVE;
+	return IB_PORT_DOWN;
+}
+
+static inline int bnxt_re_link_state(struct bnxt_re_dev *rdev)
+{
+	return bnxt_re_get_link_state(rdev) == IB_PORT_ACTIVE ? 1:0;
+}
+
+static inline int is_cc_enabled(struct bnxt_re_dev *rdev)
+{
+	return rdev->cc_param.enable;
+}
+
+static inline void bnxt_re_init_hwrm_hdr(struct bnxt_re_dev *rdev,
+					 struct input *hdr, u16 opcd,
+					 u16 crid, u16 trid)
+{
+	hdr->req_type = cpu_to_le16(opcd);
+	hdr->cmpl_ring = cpu_to_le16(crid);
+	hdr->target_id = cpu_to_le16(trid);
+}
+
+static inline void bnxt_re_fill_fw_msg(struct bnxt_fw_msg *fw_msg,
+				       void *msg, int msg_len, void *resp,
+				       int resp_max_len, int timeout)
+{
+	fw_msg->msg = msg;
+	fw_msg->msg_len = msg_len;
+	fw_msg->resp = resp;
+	fw_msg->resp_max_len = resp_max_len;
+	fw_msg->timeout = timeout;
+}
+
+static inline bool is_qport_service_type_supported(struct bnxt_re_dev *rdev)
+{
+	return rdev->tc_rec[0].serv_type_enabled;
+}
+
+static inline bool is_bnxt_roce_queue(struct bnxt_re_dev *rdev, u8 ser_prof, u8 prof_type)
+{
+	if (is_qport_service_type_supported(rdev))
+		return (prof_type & HWRM_QUEUE_QPORTCFG_OUTPUT_QUEUE_ID1_SERVICE_PROFILE_TYPE_ROCE);
+	else
+		return (ser_prof == HWRM_QUEUE_QPORTCFG_OUTPUT_QUEUE_ID0_SERVICE_PROFILE_LOSSLESS_ROCE);
+}
+
+static inline bool is_bnxt_cnp_queue(struct bnxt_re_dev *rdev, u8 ser_prof, u8 prof_type)
+{
+	if (is_qport_service_type_supported(rdev))
+		return (prof_type & HWRM_QUEUE_QPORTCFG_OUTPUT_QUEUE_ID1_SERVICE_PROFILE_TYPE_CNP);
+	else
+		return (ser_prof == HWRM_QUEUE_QPORTCFG_OUTPUT_QUEUE_ID0_SERVICE_PROFILE_LOSSY_ROCE_CNP);
+}
+
+#define BNXT_RE_MAP_SH_PAGE		0x0
+#define BNXT_RE_MAP_WC			0x1
+#define BNXT_RE_DBR_PAGE		0x2
+#define BNXT_RE_MAP_DB_RECOVERY_PAGE	0x3
+
+#define BNXT_RE_DBR_RECOV_USERLAND_TIMEOUT (20)  /*  20 ms */
+#define BNXT_RE_DBR_INT_TIME 5 /* ms */
+#define BNXT_RE_PACING_EN_INT_THRESHOLD 50 /* Entries in DB FIFO */
+#define BNXT_RE_PACING_ALGO_THRESHOLD 250 /* Entries in DB FIFO */
+/* Percentage of DB FIFO depth */
+#define BNXT_RE_PACING_DBQ_THRESHOLD BNXT_RE_PACING_DBQ_HIGH_WATERMARK
+
+#define BNXT_RE_PACING_ALARM_TH_MULTIPLE(ctx) (_is_chip_p7(ctx) ? 0 : 2)
+
+/*
+ * Maximum Percentage of configurable DB FIFO depth.
+ * The Doorbell FIFO depth is 0x2c00. But the DBR_REG_DB_THROTTLING register has only 12 bits
+ * to program the high watermark. This means user can configure maximum 36% only(4095/11264).
+ */
+#define BNXT_RE_PACING_DBQ_HIGH_WATERMARK 36
+
+/* Default do_pacing value when there is no congestion */
+#define BNXT_RE_DBR_DO_PACING_NO_CONGESTION 0x7F /* 1 in 512 probability */
+
+enum {
+	BNXT_RE_DBQ_EVENT_SCHED = 0,
+	BNXT_RE_DBR_PACING_EVENT = 1,
+	BNXT_RE_DBR_NQ_PACING_NOTIFICATION = 2,
+};
+
+struct bnxt_re_dbq_work {
+	struct work_struct work;
+	struct bnxt_re_dev *rdev;
+	struct hwrm_async_event_cmpl cmpl;
+	u32 event;
+};
+
+int bnxt_re_hwrm_qcaps(struct bnxt_re_dev *rdev);
+int bnxt_re_enable_dbr_pacing(struct bnxt_re_dev *rdev);
+int bnxt_re_disable_dbr_pacing(struct bnxt_re_dev *rdev);
+int bnxt_re_set_dbq_throttling_reg(struct bnxt_re_dev *rdev,
+				   u16 nq_id, u32 throttle);
+void bnxt_re_pacing_alert(struct bnxt_re_dev *rdev);
+int bnxt_re_hwrm_pri2cos_qcfg(struct bnxt_re_dev *rdev, struct bnxt_re_tc_rec *tc_rec,
+			      u16 target_id);
+void writel_fbsd(struct bnxt_softc *bp, u32, u8, u32);
+u32 readl_fbsd(struct bnxt_softc *bp, u32, u8);
+
+static inline unsigned int bnxt_re_get_total_mr_mw_count(struct bnxt_re_dev *rdev)
+{
+	return (atomic_read(&rdev->stats.rsors.mr_count) +
+		atomic_read(&rdev->stats.rsors.mw_count));
+}
+
+static inline void bnxt_re_set_def_pacing_threshold(struct bnxt_re_dev *rdev)
+{
+	rdev->qplib_res.pacing_data->pacing_th = rdev->pacing_algo_th;
+	rdev->qplib_res.pacing_data->alarm_th =
+		rdev->pacing_algo_th * BNXT_RE_PACING_ALARM_TH_MULTIPLE(rdev->chip_ctx);
+}
+
+static inline void bnxt_re_set_def_do_pacing(struct bnxt_re_dev *rdev)
+{
+	rdev->qplib_res.pacing_data->do_pacing = rdev->dbr_def_do_pacing;
+}
+
+static inline void bnxt_re_set_pacing_dev_state(struct bnxt_re_dev *rdev)
+{
+	rdev->qplib_res.pacing_data->dev_err_state =
+		test_bit(BNXT_RE_FLAG_ERR_DEVICE_DETACHED, &rdev->flags);
+}
+#endif
diff --git a/sys/dev/bnxt/bnxt_re/ib_verbs.c b/sys/dev/bnxt/bnxt_re/ib_verbs.c
new file mode 100644
index 000000000000..8d43fa96c048
--- /dev/null
+++ b/sys/dev/bnxt/bnxt_re/ib_verbs.c
@@ -0,0 +1,5498 @@
+/*
+ * Copyright (c) 2015-2024, Broadcom. All rights reserved.  The term
+ * Broadcom refers to Broadcom Limited and/or its subsidiaries.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS''
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
+ * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
+ * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * Description: IB Verbs interpreter
+ */
+
+#include <linux/if_ether.h>
+#include <linux/etherdevice.h>
+#include <rdma/uverbs_ioctl.h>
+
+#include "bnxt_re.h"
+#include "ib_verbs.h"
+
+static inline
+struct scatterlist *get_ib_umem_sgl(struct ib_umem *umem, u32 *nmap)
+{
+
+	*nmap = umem->nmap;
+	return umem->sg_head.sgl;
+}
+
+static inline void bnxt_re_peer_mem_release(struct ib_umem *umem)
+{
+	dev_dbg(NULL, "ib_umem_release getting invoked \n");
+	ib_umem_release(umem);
+}
+
+void bnxt_re_resolve_dmac_task(struct work_struct *work)
+{
+	int rc = -1;
+	struct bnxt_re_dev *rdev;
+	struct ib_ah_attr	*ah_attr;
+	struct bnxt_re_resolve_dmac_work *dmac_work =
+			container_of(work, struct bnxt_re_resolve_dmac_work, work);
+
+	rdev = dmac_work->rdev;
+	ah_attr = dmac_work->ah_attr;
+	rc = ib_resolve_eth_dmac(&rdev->ibdev, ah_attr);
+	if (rc)
+		dev_err(rdev_to_dev(dmac_work->rdev),
+			"Failed to resolve dest mac rc = %d\n", rc);
+	atomic_set(&dmac_work->status_wait, rc << 8);
+}
+
+static int __from_ib_access_flags(int iflags)
+{
+	int qflags = 0;
+
+	if (iflags & IB_ACCESS_LOCAL_WRITE)
+		qflags |= BNXT_QPLIB_ACCESS_LOCAL_WRITE;
+	if (iflags & IB_ACCESS_REMOTE_READ)
+		qflags |= BNXT_QPLIB_ACCESS_REMOTE_READ;
+	if (iflags & IB_ACCESS_REMOTE_WRITE)
+		qflags |= BNXT_QPLIB_ACCESS_REMOTE_WRITE;
+	if (iflags & IB_ACCESS_REMOTE_ATOMIC)
+		qflags |= BNXT_QPLIB_ACCESS_REMOTE_ATOMIC;
+	if (iflags & IB_ACCESS_MW_BIND)
+		qflags |= BNXT_QPLIB_ACCESS_MW_BIND;
+	if (iflags & IB_ZERO_BASED)
+		qflags |= BNXT_QPLIB_ACCESS_ZERO_BASED;
+	if (iflags & IB_ACCESS_ON_DEMAND)
+		qflags |= BNXT_QPLIB_ACCESS_ON_DEMAND;
+	return qflags;
+};
+
+static enum ib_access_flags __to_ib_access_flags(int qflags)
+{
+	enum ib_access_flags iflags = 0;
+
+	if (qflags & BNXT_QPLIB_ACCESS_LOCAL_WRITE)
+		iflags |= IB_ACCESS_LOCAL_WRITE;
+	if (qflags & BNXT_QPLIB_ACCESS_REMOTE_WRITE)
+		iflags |= IB_ACCESS_REMOTE_WRITE;
+	if (qflags & BNXT_QPLIB_ACCESS_REMOTE_READ)
+		iflags |= IB_ACCESS_REMOTE_READ;
+	if (qflags & BNXT_QPLIB_ACCESS_REMOTE_ATOMIC)
+		iflags |= IB_ACCESS_REMOTE_ATOMIC;
+	if (qflags & BNXT_QPLIB_ACCESS_MW_BIND)
+		iflags |= IB_ACCESS_MW_BIND;
+	if (qflags & BNXT_QPLIB_ACCESS_ZERO_BASED)
+		iflags |= IB_ZERO_BASED;
+	if (qflags & BNXT_QPLIB_ACCESS_ON_DEMAND)
+		iflags |= IB_ACCESS_ON_DEMAND;
+	return iflags;
+};
+
+static int bnxt_re_copy_to_udata(struct bnxt_re_dev *rdev, void *data,
+				 int len, struct ib_udata *udata)
+{
+	int rc;
+
+	rc = ib_copy_to_udata(udata, data, len);
+	if (rc)
+		dev_err(rdev_to_dev(rdev),
+			"ucontext copy failed from %ps rc %d\n",
+			__builtin_return_address(0), rc);
+
+	return rc;
+}
+
+struct ifnet *bnxt_re_get_netdev(struct ib_device *ibdev,
+				 u8 port_num)
+{
+	struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev);
+	struct ifnet *netdev = NULL;
+
+	rcu_read_lock();
+
+	if (!rdev || !rdev->netdev)
+		goto end;
+
+	netdev = rdev->netdev;
+
+	/* In case of active-backup bond mode, return active slave */
+	if (netdev)
+		dev_hold(netdev);
+
+end:
+	rcu_read_unlock();
+	return netdev;
+}
+
+int bnxt_re_query_device(struct ib_device *ibdev,
+			 struct ib_device_attr *ib_attr,
+			 struct ib_udata *udata)
+{
+	struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev);
+	struct bnxt_qplib_dev_attr *dev_attr = rdev->dev_attr;
+
+	memset(ib_attr, 0, sizeof(*ib_attr));
+
+	memcpy(&ib_attr->fw_ver, dev_attr->fw_ver, 4);
+	bnxt_qplib_get_guid(rdev->dev_addr, (u8 *)&ib_attr->sys_image_guid);
+	ib_attr->max_mr_size = BNXT_RE_MAX_MR_SIZE;
+	ib_attr->page_size_cap = dev_attr->page_size_cap;
+	ib_attr->vendor_id = rdev->en_dev->pdev->vendor;
+	ib_attr->vendor_part_id = rdev->en_dev->pdev->device;
+	ib_attr->hw_ver = rdev->en_dev->pdev->subsystem_device;
+	ib_attr->max_qp = dev_attr->max_qp;
+	ib_attr->max_qp_wr = dev_attr->max_qp_wqes;
+	/*
+	 * Read and set from the module param 'min_tx_depth'
+	 * only once after the driver load
+	 */
+	if (rdev->min_tx_depth == 1 &&
+	    min_tx_depth < dev_attr->max_qp_wqes)
+		rdev->min_tx_depth = min_tx_depth;
+	ib_attr->device_cap_flags =
+				    IB_DEVICE_CURR_QP_STATE_MOD
+				    | IB_DEVICE_RC_RNR_NAK_GEN
+				    | IB_DEVICE_SHUTDOWN_PORT
+				    | IB_DEVICE_SYS_IMAGE_GUID
+				    | IB_DEVICE_LOCAL_DMA_LKEY
+				    | IB_DEVICE_RESIZE_MAX_WR
+				    | IB_DEVICE_PORT_ACTIVE_EVENT
+				    | IB_DEVICE_N_NOTIFY_CQ
+				    | IB_DEVICE_MEM_WINDOW
+				    | IB_DEVICE_MEM_WINDOW_TYPE_2B
+				    | IB_DEVICE_MEM_MGT_EXTENSIONS;
+	ib_attr->max_send_sge = dev_attr->max_qp_sges;
+	ib_attr->max_recv_sge = dev_attr->max_qp_sges;
+	ib_attr->max_sge_rd = dev_attr->max_qp_sges;
+	ib_attr->max_cq = dev_attr->max_cq;
+	ib_attr->max_cqe = dev_attr->max_cq_wqes;
+	ib_attr->max_mr = dev_attr->max_mr;
+	ib_attr->max_pd = dev_attr->max_pd;
+	ib_attr->max_qp_rd_atom = dev_attr->max_qp_rd_atom;
+	ib_attr->max_qp_init_rd_atom = dev_attr->max_qp_init_rd_atom;
+	if (dev_attr->is_atomic) {
+		ib_attr->atomic_cap = IB_ATOMIC_GLOB;
+		ib_attr->masked_atomic_cap = IB_ATOMIC_GLOB;
+	}
+	ib_attr->max_ee_rd_atom = 0;
+	ib_attr->max_res_rd_atom = 0;
+	ib_attr->max_ee_init_rd_atom = 0;
+	ib_attr->max_ee = 0;
+	ib_attr->max_rdd = 0;
+	ib_attr->max_mw = dev_attr->max_mw;
+	ib_attr->max_raw_ipv6_qp = 0;
+	ib_attr->max_raw_ethy_qp = dev_attr->max_raw_ethy_qp;
+	ib_attr->max_mcast_grp = 0;
+	ib_attr->max_mcast_qp_attach = 0;
+	ib_attr->max_total_mcast_qp_attach = 0;
+	ib_attr->max_ah = dev_attr->max_ah;
+	ib_attr->max_srq = dev_attr->max_srq;
+	ib_attr->max_srq_wr = dev_attr->max_srq_wqes;
+	ib_attr->max_srq_sge = dev_attr->max_srq_sges;
+
+	ib_attr->max_fast_reg_page_list_len = MAX_PBL_LVL_1_PGS;
+	ib_attr->max_pkeys = 1;
+	ib_attr->local_ca_ack_delay = BNXT_RE_DEFAULT_ACK_DELAY;
+	ib_attr->sig_prot_cap = 0;
+	ib_attr->sig_guard_cap = 0;
+	ib_attr->odp_caps.general_caps = 0;
+
+	return 0;
+}
+
+int bnxt_re_modify_device(struct ib_device *ibdev,
+			  int device_modify_mask,
+			  struct ib_device_modify *device_modify)
+{
+	dev_dbg(rdev_to_dev(rdev), "Modify device with mask 0x%x\n",
+		device_modify_mask);
+
+	switch (device_modify_mask) {
+	case IB_DEVICE_MODIFY_SYS_IMAGE_GUID:
+		/* Modify the GUID requires the modification of the GID table */
+		/* GUID should be made as READ-ONLY */
+		break;
+	case IB_DEVICE_MODIFY_NODE_DESC:
+		/* Node Desc should be made as READ-ONLY */
+		break;
+	default:
+		break;
+	}
+	return 0;
+}
+
+static void __to_ib_speed_width(u32 espeed, u8 *speed, u8 *width)
+{
+	switch (espeed) {
+	case SPEED_1000:
+		*speed = IB_SPEED_SDR;
+		*width = IB_WIDTH_1X;
+		break;
+	case SPEED_10000:
+		*speed = IB_SPEED_QDR;
+		*width = IB_WIDTH_1X;
+		break;
+	case SPEED_20000:
+		*speed = IB_SPEED_DDR;
+		*width = IB_WIDTH_4X;
+		break;
+	case SPEED_25000:
+		*speed = IB_SPEED_EDR;
+		*width = IB_WIDTH_1X;
+		break;
+	case SPEED_40000:
+		*speed = IB_SPEED_QDR;
+		*width = IB_WIDTH_4X;
+		break;
+	case SPEED_50000:
+		*speed = IB_SPEED_EDR;
+		*width = IB_WIDTH_2X;
+		break;
+	case SPEED_100000:
+		*speed = IB_SPEED_EDR;
+		*width = IB_WIDTH_4X;
+		break;
+	case SPEED_200000:
+		*speed = IB_SPEED_HDR;
+		*width = IB_WIDTH_4X;
+		break;
+	default:
+		*speed = IB_SPEED_SDR;
+		*width = IB_WIDTH_1X;
+		break;
+	}
+}
+
+/* Port */
+int bnxt_re_query_port(struct ib_device *ibdev, u8 port_num,
+		       struct ib_port_attr *port_attr)
+{
+	struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev);
+	struct bnxt_qplib_dev_attr *dev_attr = rdev->dev_attr;
+	u8 active_speed = 0, active_width = 0;
+
+	dev_dbg(rdev_to_dev(rdev), "QUERY PORT with port_num 0x%x\n", port_num);
+	memset(port_attr, 0, sizeof(*port_attr));
+
+	port_attr->phys_state = IB_PORT_PHYS_STATE_DISABLED;
+	port_attr->state = bnxt_re_get_link_state(rdev);
+	if (port_attr->state == IB_PORT_ACTIVE)
+		port_attr->phys_state = IB_PORT_PHYS_STATE_LINK_UP;
+	port_attr->max_mtu = IB_MTU_4096;
+	port_attr->active_mtu = iboe_get_mtu(rdev->netdev->if_mtu);
+	port_attr->gid_tbl_len = dev_attr->max_sgid;
+	port_attr->port_cap_flags = IB_PORT_CM_SUP | IB_PORT_REINIT_SUP |
+				    IB_PORT_DEVICE_MGMT_SUP |
+				    IB_PORT_VENDOR_CLASS_SUP |
+				    IB_PORT_IP_BASED_GIDS;
+
+	port_attr->max_msg_sz = (u32)BNXT_RE_MAX_MR_SIZE_LOW;
+	port_attr->bad_pkey_cntr = 0;
+	port_attr->qkey_viol_cntr = 0;
+	port_attr->pkey_tbl_len = dev_attr->max_pkey;
+	port_attr->lid = 0;
+	port_attr->sm_lid = 0;
+	port_attr->lmc = 0;
+	port_attr->max_vl_num = 4;
+	port_attr->sm_sl = 0;
+	port_attr->subnet_timeout = 0;
+	port_attr->init_type_reply = 0;
+	rdev->espeed = rdev->en_dev->espeed;
+
+	if (test_bit(BNXT_RE_FLAG_IBDEV_REGISTERED, &rdev->flags))
+		__to_ib_speed_width(rdev->espeed, &active_speed,
+				    &active_width);
+
+	port_attr->active_speed = active_speed;
+	port_attr->active_width = active_width;
+
+	return 0;
+}
+
+int bnxt_re_modify_port(struct ib_device *ibdev, u8 port_num,
+			int port_modify_mask,
+			struct ib_port_modify *port_modify)
+{
+	dev_dbg(rdev_to_dev(rdev), "Modify port with mask 0x%x\n",
+		port_modify_mask);
+
+	switch (port_modify_mask) {
+	case IB_PORT_SHUTDOWN:
+		break;
+	case IB_PORT_INIT_TYPE:
+		break;
+	case IB_PORT_RESET_QKEY_CNTR:
+		break;
+	default:
+		break;
+	}
+	return 0;
+}
+
+int bnxt_re_get_port_immutable(struct ib_device *ibdev, u8 port_num,
+			       struct ib_port_immutable *immutable)
+{
+	struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev);
+	struct ib_port_attr port_attr;
+
+	if (bnxt_re_query_port(ibdev, port_num, &port_attr))
+		return -EINVAL;
+
+	immutable->pkey_tbl_len = port_attr.pkey_tbl_len;
+	immutable->gid_tbl_len = port_attr.gid_tbl_len;
+	if (rdev->roce_mode == BNXT_RE_FLAG_ROCEV1_CAP)
+		immutable->core_cap_flags = RDMA_CORE_PORT_IBA_ROCE;
+	else if (rdev->roce_mode == BNXT_RE_FLAG_ROCEV2_CAP)
+		immutable->core_cap_flags = RDMA_CORE_PORT_IBA_ROCE_UDP_ENCAP;
+	else
+		immutable->core_cap_flags = RDMA_CORE_PORT_IBA_ROCE |
+					    RDMA_CORE_PORT_IBA_ROCE_UDP_ENCAP;
+	immutable->max_mad_size = IB_MGMT_MAD_SIZE;
+	return 0;
+}
+
+void bnxt_re_compat_qfwstr(void)
+{
+	struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev);
+
+	sprintf(str, "%d.%d.%d.%d", rdev->dev_attr->fw_ver[0],
+		rdev->dev_attr->fw_ver[1], rdev->dev_attr->fw_ver[2],
+		rdev->dev_attr->fw_ver[3]);
+}
+
+int bnxt_re_query_pkey(struct ib_device *ibdev, u8 port_num,
+		       u16 index, u16 *pkey)
+{
+	if (index > 0)
+		return -EINVAL;
+
+	*pkey = IB_DEFAULT_PKEY_FULL;
+
+	return 0;
+}
+
+int bnxt_re_query_gid(struct ib_device *ibdev, u8 port_num,
+		      int index, union ib_gid *gid)
+{
+	struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev);
+	int rc = 0;
+
+	/* Ignore port_num */
+	memset(gid, 0, sizeof(*gid));
+	rc = bnxt_qplib_get_sgid(&rdev->qplib_res,
+				 &rdev->qplib_res.sgid_tbl, index,
+				 (struct bnxt_qplib_gid *)gid);
+	return rc;
+}
+
+int bnxt_re_del_gid(struct ib_device *ibdev, u8 port_num,
+		    unsigned int index, void **context)
+{
+	int rc = 0;
+	struct bnxt_re_gid_ctx *ctx, **ctx_tbl;
+	struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev);
+	struct bnxt_qplib_sgid_tbl *sgid_tbl = &rdev->qplib_res.sgid_tbl;
+	struct bnxt_qplib_gid *gid_to_del;
+	u16 vlan_id = 0xFFFF;
+
+	/* Delete the entry from the hardware */
+	ctx = *context;
+	if (!ctx) {
+		dev_err(rdev_to_dev(rdev), "GID entry has no ctx?!\n");
+		return -EINVAL;
+	}
+	if (sgid_tbl && sgid_tbl->active) {
+		if (ctx->idx >= sgid_tbl->max) {
+			dev_dbg(rdev_to_dev(rdev), "GID index out of range?!\n");
+			return -EINVAL;
+		}
+		gid_to_del = &sgid_tbl->tbl[ctx->idx].gid;
+		vlan_id = sgid_tbl->tbl[ctx->idx].vlan_id;
+		ctx->refcnt--;
+		/* DEL_GID is called via WQ context(netdevice_event_work_handler)
+		 * or via the ib_unregister_device path. In the former case QP1
+		 * may not be destroyed yet, in which case just return as FW
+		 * needs that entry to be present and will fail it's deletion.
+		 * We could get invoked again after QP1 is destroyed OR get an
+		 * ADD_GID call with a different GID value for the same index
+		 * where we issue MODIFY_GID cmd to update the GID entry -- TBD
+		 */
+		if (ctx->idx == 0 &&
+		    rdma_link_local_addr((struct in6_addr *)gid_to_del) &&
+		    (rdev->gsi_ctx.gsi_sqp ||
+		     rdev->gsi_ctx.gsi_qp_mode == BNXT_RE_GSI_MODE_UD)) {
+			dev_dbg(rdev_to_dev(rdev),
+				"Trying to delete GID0 while QP1 is alive\n");
+			if (!ctx->refcnt) {
+				rdev->gid_map[index] = -1;
+				ctx_tbl = sgid_tbl->ctx;
+				ctx_tbl[ctx->idx] = NULL;
+				kfree(ctx);
+			}
+			return 0;
+		}
+		rdev->gid_map[index] = -1;
+		if (!ctx->refcnt) {
+			rc = bnxt_qplib_del_sgid(sgid_tbl, gid_to_del,
+						 vlan_id, true);
+			if (!rc) {
+				dev_dbg(rdev_to_dev(rdev), "GID remove success\n");
+				ctx_tbl = sgid_tbl->ctx;
+				ctx_tbl[ctx->idx] = NULL;
+				kfree(ctx);
+			} else {
+				dev_err(rdev_to_dev(rdev),
+					"Remove GID failed rc = 0x%x\n", rc);
+			}
+		}
+	} else {
+		dev_dbg(rdev_to_dev(rdev), "GID sgid_tbl does not exist!\n");
+		return -EINVAL;
+	}
+	return rc;
+}
+
+int bnxt_re_add_gid(struct ib_device *ibdev, u8 port_num,
+		    unsigned int index, const union ib_gid *gid,
+		    const struct ib_gid_attr *attr, void **context)
+{
+	int rc;
+	u32 tbl_idx = 0;
+	u16 vlan_id = 0xFFFF;
+	struct bnxt_re_gid_ctx *ctx, **ctx_tbl;
+	struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev);
+	struct bnxt_qplib_sgid_tbl *sgid_tbl = &rdev->qplib_res.sgid_tbl;
+	if ((attr->ndev) && is_vlan_dev(attr->ndev))
+		vlan_id = vlan_dev_vlan_id(attr->ndev);
+
+	rc = bnxt_qplib_add_sgid(sgid_tbl, gid,
+				 rdev->dev_addr,
+				 vlan_id, true, &tbl_idx);
+	if (rc == -EALREADY) {
+		dev_dbg(rdev_to_dev(rdev), "GID %pI6 is already present\n", gid);
+		ctx_tbl = sgid_tbl->ctx;
+		if (!ctx_tbl[tbl_idx]) {
+			ctx = kmalloc(sizeof(*ctx), GFP_KERNEL);
+			if (!ctx)
+				return -ENOMEM;
+			ctx->idx = tbl_idx;
+			ctx->refcnt = 1;
+			ctx_tbl[tbl_idx] = ctx;
+		} else {
+			ctx_tbl[tbl_idx]->refcnt++;
+		}
+		*context = ctx_tbl[tbl_idx];
+		/* tbl_idx is the HW table index and index is the stack index */
+		rdev->gid_map[index] = tbl_idx;
+		return 0;
+	} else if (rc < 0) {
+		dev_err(rdev_to_dev(rdev), "Add GID failed rc = 0x%x\n", rc);
+		return rc;
+	} else {
+		ctx = kmalloc(sizeof(*ctx), GFP_KERNEL);
+		if (!ctx) {
+			dev_err(rdev_to_dev(rdev), "Add GID ctx failed\n");
+			return -ENOMEM;
+		}
+		ctx_tbl = sgid_tbl->ctx;
+		ctx->idx = tbl_idx;
+		ctx->refcnt = 1;
+		ctx_tbl[tbl_idx] = ctx;
+		/* tbl_idx is the HW table index and index is the stack index */
+		rdev->gid_map[index] = tbl_idx;
+		*context = ctx;
+	}
+	return rc;
+}
+
+enum rdma_link_layer bnxt_re_get_link_layer(struct ib_device *ibdev,
+					    u8 port_num)
+{
+	return IB_LINK_LAYER_ETHERNET;
+}
+
+static void bnxt_re_legacy_create_fence_wqe(struct bnxt_re_pd *pd)
+{
+	struct bnxt_re_legacy_fence_data *fence = &pd->fence;
+	struct ib_mr *ib_mr = &fence->mr->ib_mr;
+	struct bnxt_qplib_swqe *wqe = &fence->bind_wqe;
+	struct bnxt_re_dev *rdev = pd->rdev;
+
+	if (!_is_chip_gen_p5_p7(rdev->chip_ctx))
+		return;
+
+	memset(wqe, 0, sizeof(*wqe));
+	wqe->type = BNXT_QPLIB_SWQE_TYPE_BIND_MW;
+	wqe->wr_id = BNXT_QPLIB_FENCE_WRID;
+	wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP;
+	wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE;
+	wqe->bind.zero_based = false;
+	wqe->bind.parent_l_key = ib_mr->lkey;
+	wqe->bind.va = (u64)fence->va;
+	wqe->bind.length = fence->size;
+	wqe->bind.access_cntl = __from_ib_access_flags(IB_ACCESS_REMOTE_READ);
+	wqe->bind.mw_type = SQ_BIND_MW_TYPE_TYPE1;
+
+	/* Save the initial rkey in fence structure for now;
+	 * wqe->bind.r_key will be set at (re)bind time.
+	 */
+	fence->bind_rkey = ib_inc_rkey(fence->mw->rkey);
+}
+
+static int bnxt_re_legacy_bind_fence_mw(struct bnxt_qplib_qp *qplib_qp)
+{
+	struct bnxt_re_qp *qp = container_of(qplib_qp, struct bnxt_re_qp,
+					     qplib_qp);
+	struct ib_pd *ib_pd = qp->ib_qp.pd;
+	struct bnxt_re_pd *pd = to_bnxt_re(ib_pd, struct bnxt_re_pd, ibpd);
+	struct bnxt_re_legacy_fence_data *fence = &pd->fence;
+	struct bnxt_qplib_swqe *fence_wqe = &fence->bind_wqe;
+	struct bnxt_qplib_swqe wqe;
+	int rc;
+
+	/* TODO: Need SQ locking here when Fence WQE
+	 * posting moves up into bnxt_re from bnxt_qplib.
+	 */
+	memcpy(&wqe, fence_wqe, sizeof(wqe));
+	wqe.bind.r_key = fence->bind_rkey;
+	fence->bind_rkey = ib_inc_rkey(fence->bind_rkey);
+
+	dev_dbg(rdev_to_dev(qp->rdev),
+		"Posting bind fence-WQE: rkey: %#x QP: %d PD: %p\n",
+		wqe.bind.r_key, qp->qplib_qp.id, pd);
+	rc = bnxt_qplib_post_send(&qp->qplib_qp, &wqe);
+	if (rc) {
+		dev_err(rdev_to_dev(qp->rdev), "Failed to bind fence-WQE\n");
+		return rc;
+	}
+	bnxt_qplib_post_send_db(&qp->qplib_qp);
+
+	return rc;
+}
+
+static int bnxt_re_legacy_create_fence_mr(struct bnxt_re_pd *pd)
+{
+	int mr_access_flags = IB_ACCESS_LOCAL_WRITE | IB_ACCESS_MW_BIND;
+	struct bnxt_re_legacy_fence_data *fence = &pd->fence;
+	struct bnxt_re_dev *rdev = pd->rdev;
+	struct bnxt_qplib_mrinfo mrinfo;
+	struct bnxt_re_mr *mr = NULL;
+	struct ib_mw *ib_mw = NULL;
+	dma_addr_t dma_addr = 0;
+	u32 max_mr_count;
+	u64 pbl_tbl;
+	int rc;
+
+	if (!_is_chip_gen_p5_p7(rdev->chip_ctx))
+		return 0;
+
+	memset(&mrinfo, 0, sizeof(mrinfo));
+	/* Allocate a small chunk of memory and dma-map it */
+	fence->va = kzalloc(BNXT_RE_LEGACY_FENCE_BYTES, GFP_KERNEL);
+	if (!fence->va)
+		return -ENOMEM;
+	dma_addr = ib_dma_map_single(&rdev->ibdev, fence->va,
+				     BNXT_RE_LEGACY_FENCE_BYTES,
+				     DMA_BIDIRECTIONAL);
+	rc = ib_dma_mapping_error(&rdev->ibdev, dma_addr);
+	if (rc) {
+		dev_err(rdev_to_dev(rdev), "Failed to dma-map fence-MR-mem\n");
+		rc = -EIO;
+		fence->dma_addr = 0;
+		goto free_va;
+	}
+	fence->dma_addr = dma_addr;
+
+	/* Allocate a MR */
+	mr = kzalloc(sizeof(*mr), GFP_KERNEL);
+	if (!mr)
+		goto free_dma_addr;
+	fence->mr = mr;
+	mr->rdev = rdev;
+	mr->qplib_mr.pd = &pd->qplib_pd;
+	mr->qplib_mr.type = CMDQ_ALLOCATE_MRW_MRW_FLAGS_PMR;
+	mr->qplib_mr.flags = __from_ib_access_flags(mr_access_flags);
+	if (!_is_alloc_mr_unified(rdev->qplib_res.dattr)) {
+		rc = bnxt_qplib_alloc_mrw(&rdev->qplib_res, &mr->qplib_mr);
+		if (rc) {
+			dev_err(rdev_to_dev(rdev), "Failed to alloc fence-HW-MR\n");
+			goto free_mr;
+		}
+		/* Register MR */
+		mr->ib_mr.lkey = mr->qplib_mr.lkey;
+	}
+	mr->qplib_mr.va         = (u64)fence->va;
+	mr->qplib_mr.total_size = BNXT_RE_LEGACY_FENCE_BYTES;
+	pbl_tbl = dma_addr;
+
+	mrinfo.mrw = &mr->qplib_mr;
+	mrinfo.ptes = &pbl_tbl;
+	mrinfo.sg.npages = BNXT_RE_LEGACY_FENCE_PBL_SIZE;
+
+	mrinfo.sg.nmap = 0;
+	mrinfo.sg.sghead = 0;
+	mrinfo.sg.pgshft = PAGE_SHIFT;
+	mrinfo.sg.pgsize = PAGE_SIZE;
+	rc = bnxt_qplib_reg_mr(&rdev->qplib_res, &mrinfo, false);
+	if (rc) {
+		dev_err(rdev_to_dev(rdev), "Failed to register fence-MR\n");
+		goto free_mr;
+	}
+	mr->ib_mr.lkey = mr->qplib_mr.lkey;
+	mr->ib_mr.rkey = mr->qplib_mr.rkey;
+	atomic_inc(&rdev->stats.rsors.mr_count);
+	max_mr_count =  atomic_read(&rdev->stats.rsors.mr_count);
+	if (max_mr_count > (atomic_read(&rdev->stats.rsors.max_mr_count)))
+		atomic_set(&rdev->stats.rsors.max_mr_count, max_mr_count);
+
+	ib_mw = bnxt_re_alloc_mw(&pd->ibpd, IB_MW_TYPE_1, NULL);
+	/* Create a fence MW only for kernel consumers */
+	if (!ib_mw) {
+		dev_err(rdev_to_dev(rdev),
+			"Failed to create fence-MW for PD: %p\n", pd);
+		rc = -EINVAL;
+		goto free_mr;
+	}
+	fence->mw = ib_mw;
+
+	bnxt_re_legacy_create_fence_wqe(pd);
+	return 0;
+
+free_mr:
+	if (mr->ib_mr.lkey) {
+		bnxt_qplib_free_mrw(&rdev->qplib_res, &mr->qplib_mr);
+		atomic_dec(&rdev->stats.rsors.mr_count);
+	}
+	kfree(mr);
+	fence->mr = NULL;
+
+free_dma_addr:
+	ib_dma_unmap_single(&rdev->ibdev, fence->dma_addr,
+			    BNXT_RE_LEGACY_FENCE_BYTES, DMA_BIDIRECTIONAL);
+	fence->dma_addr = 0;
+
+free_va:
+	kfree(fence->va);
+	fence->va = NULL;
+	return rc;
+}
+
+static void bnxt_re_legacy_destroy_fence_mr(struct bnxt_re_pd *pd)
+{
+	struct bnxt_re_legacy_fence_data *fence = &pd->fence;
+	struct bnxt_re_dev *rdev = pd->rdev;
+	struct bnxt_re_mr *mr = fence->mr;
+
+	if (!_is_chip_gen_p5_p7(rdev->chip_ctx))
+		return;
+
+	if (fence->mw) {
+		bnxt_re_dealloc_mw(fence->mw);
+		fence->mw = NULL;
+	}
+	if (mr) {
+		if (mr->ib_mr.rkey)
+			bnxt_qplib_dereg_mrw(&rdev->qplib_res, &mr->qplib_mr,
+					     false);
+		if (mr->ib_mr.lkey)
+			bnxt_qplib_free_mrw(&rdev->qplib_res, &mr->qplib_mr);
+		kfree(mr);
+		fence->mr = NULL;
+		atomic_dec(&rdev->stats.rsors.mr_count);
+	}
+	if (fence->dma_addr) {
+		ib_dma_unmap_single(&rdev->ibdev, fence->dma_addr,
+				    BNXT_RE_LEGACY_FENCE_BYTES,
+				    DMA_BIDIRECTIONAL);
+		fence->dma_addr = 0;
+	}
+	kfree(fence->va);
+	fence->va = NULL;
+}
+
+
+static int bnxt_re_get_user_dpi(struct bnxt_re_dev *rdev,
+				struct bnxt_re_ucontext *cntx)
+{
+	struct bnxt_qplib_chip_ctx *cctx = rdev->chip_ctx;
+	int ret = 0;
+	u8 type;
+	/* Allocate DPI in alloc_pd or in create_cq to avoid failing of
+	 * ibv_devinfo and family of application when DPIs are depleted.
+	 */
+	type = BNXT_QPLIB_DPI_TYPE_UC;
+	ret = bnxt_qplib_alloc_dpi(&rdev->qplib_res, &cntx->dpi, cntx, type);
+	if (ret) {
+		dev_err(rdev_to_dev(rdev), "Alloc doorbell page failed!\n");
+		goto out;
+	}
+
+	if (cctx->modes.db_push) {
+		type = BNXT_QPLIB_DPI_TYPE_WC;
+		ret = bnxt_qplib_alloc_dpi(&rdev->qplib_res, &cntx->wcdpi,
+					   cntx, type);
+		if (ret)
+			dev_err(rdev_to_dev(rdev), "push dp alloc failed\n");
+	}
+out:
+	return ret;
+}
+
+/* Protection Domains */
+void bnxt_re_dealloc_pd(struct ib_pd *ib_pd, struct ib_udata *udata)
+{
+	struct bnxt_re_pd *pd = to_bnxt_re(ib_pd, struct bnxt_re_pd, ibpd);
+	struct bnxt_re_dev *rdev = pd->rdev;
+	int rc;
+
+	bnxt_re_legacy_destroy_fence_mr(pd);
+
+	rc = bnxt_qplib_dealloc_pd(&rdev->qplib_res,
+				   &rdev->qplib_res.pd_tbl,
+				   &pd->qplib_pd);
+	if (rc)
+		dev_err_ratelimited(rdev_to_dev(rdev),
+				    "%s failed rc = %d\n", __func__, rc);
+	atomic_dec(&rdev->stats.rsors.pd_count);
+
+	return;
+}
+
+int bnxt_re_alloc_pd(struct ib_pd *pd_in,
+		     struct ib_udata *udata)
+{
+	struct ib_pd *ibpd = pd_in;
+	struct ib_device *ibdev = ibpd->device;
+	struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev);
+	struct bnxt_re_ucontext *ucntx =
+		rdma_udata_to_drv_context(udata, struct bnxt_re_ucontext,
+					  ibucontext);
+	u32 max_pd_count;
+	int rc;
+	struct bnxt_re_pd *pd = container_of(ibpd, struct bnxt_re_pd, ibpd);
+
+	pd->rdev = rdev;
+	if (bnxt_qplib_alloc_pd(&rdev->qplib_res, &pd->qplib_pd)) {
+		dev_err(rdev_to_dev(rdev),
+			"Allocate HW Protection Domain failed!\n");
+		rc = -ENOMEM;
+		goto fail;
+	}
+
+	if (udata) {
+		struct bnxt_re_pd_resp resp = {};
+
+		if (!ucntx->dpi.dbr) {
+			rc = bnxt_re_get_user_dpi(rdev, ucntx);
+			if (rc)
+				goto dbfail;
+		}
+
+		resp.pdid = pd->qplib_pd.id;
+		/* Still allow mapping this DBR to the new user PD. */
+		resp.dpi = ucntx->dpi.dpi;
+		resp.dbr = (u64)ucntx->dpi.umdbr;
+		/* Copy only on a valid wcpdi */
+		if (ucntx->wcdpi.dpi) {
+			resp.wcdpi = ucntx->wcdpi.dpi;
+			resp.comp_mask = BNXT_RE_COMP_MASK_PD_HAS_WC_DPI;
+		}
+		if (rdev->dbr_pacing) {
+			WARN_ON(!rdev->dbr_bar_addr);
+			resp.dbr_bar_addr = (u64)rdev->dbr_bar_addr;
+			resp.comp_mask |= BNXT_RE_COMP_MASK_PD_HAS_DBR_BAR_ADDR;
+		}
+
+		rc = bnxt_re_copy_to_udata(rdev, &resp,
+					   min(udata->outlen, sizeof(resp)),
+					   udata);
+		if (rc)
+			goto dbfail;
+	}
+
+	if (!udata)
+		if (bnxt_re_legacy_create_fence_mr(pd))
+			dev_warn(rdev_to_dev(rdev),
+				 "Failed to create Fence-MR\n");
+
+	atomic_inc(&rdev->stats.rsors.pd_count);
+	max_pd_count = atomic_read(&rdev->stats.rsors.pd_count);
+	if (max_pd_count > atomic_read(&rdev->stats.rsors.max_pd_count))
+		atomic_set(&rdev->stats.rsors.max_pd_count, max_pd_count);
+
+	return 0;
+dbfail:
+	(void)bnxt_qplib_dealloc_pd(&rdev->qplib_res, &rdev->qplib_res.pd_tbl,
+				    &pd->qplib_pd);
+fail:
+	return rc;
+}
+
+/* Address Handles */
+void bnxt_re_destroy_ah(struct ib_ah *ib_ah, u32 flags)
+{
+	struct bnxt_re_ah *ah = to_bnxt_re(ib_ah, struct bnxt_re_ah, ibah);
+	struct bnxt_re_dev *rdev = ah->rdev;
+	int rc = 0;
+	bool block = true;
+
+	block = !(flags & RDMA_DESTROY_AH_SLEEPABLE);
+
+	rc = bnxt_qplib_destroy_ah(&rdev->qplib_res, &ah->qplib_ah, block);
+	if (rc)
+		dev_err_ratelimited(rdev_to_dev(rdev),
+				   "%s id = %d blocking %d failed rc = %d\n",
+				    __func__, ah->qplib_ah.id, block, rc);
+	atomic_dec(&rdev->stats.rsors.ah_count);
+
+	return;
+}
+
+static u8 _to_bnxt_re_nw_type(enum rdma_network_type ntype)
+{
+	u8 nw_type;
+	switch (ntype) {
+		case RDMA_NETWORK_IPV4:
+			nw_type = CMDQ_CREATE_AH_TYPE_V2IPV4;
+			break;
+		case RDMA_NETWORK_IPV6:
+			nw_type = CMDQ_CREATE_AH_TYPE_V2IPV6;
+			break;
+		default:
+			nw_type = CMDQ_CREATE_AH_TYPE_V1;
+			break;
+	}
+	return nw_type;
+}
+
+static inline int
+bnxt_re_get_cached_gid(struct ib_device *dev, u8 port_num, int index,
+		       union ib_gid *sgid, struct ib_gid_attr **sgid_attr,
+		       struct ib_global_route *grh, struct ib_ah *ah)
+{
+	int ret = 0;
+
+	ret = ib_get_cached_gid(dev, port_num, index, sgid, *sgid_attr);
+	return ret;
+}
+
+static inline enum rdma_network_type
+bnxt_re_gid_to_network_type(struct ib_gid_attr *sgid_attr,
+			    union ib_gid *sgid)
+{
+	return ib_gid_to_network_type(sgid_attr->gid_type, sgid);
+}
+
+static int bnxt_re_get_ah_info(struct bnxt_re_dev *rdev,
+			       struct ib_ah_attr *ah_attr,
+			       struct bnxt_re_ah_info *ah_info)
+{
+	struct ib_gid_attr *gattr;
+	enum rdma_network_type ib_ntype;
+	u8 ntype;
+	union ib_gid *gid;
+	int rc = 0;
+
+	gid = &ah_info->sgid;
+	gattr = &ah_info->sgid_attr;
+
+	rc = bnxt_re_get_cached_gid(&rdev->ibdev, 1, ah_attr->grh.sgid_index,
+				    gid, &gattr, &ah_attr->grh, NULL);
+	if (rc)
+		return rc;
+
+	/* Get vlan tag */
+	if (gattr->ndev) {
+		if (is_vlan_dev(gattr->ndev))
+			ah_info->vlan_tag = vlan_dev_vlan_id(gattr->ndev);
+		if_rele(gattr->ndev);
+	}
+
+	/* Get network header type for this GID */
+
+	ib_ntype = bnxt_re_gid_to_network_type(gattr, gid);
+	ntype = _to_bnxt_re_nw_type(ib_ntype);
+	ah_info->nw_type = ntype;
+
+	return rc;
+}
+
+static u8 _get_sgid_index(struct bnxt_re_dev *rdev, u8 gindx)
+{
+	gindx = rdev->gid_map[gindx];
+	return gindx;
+}
+
+static int bnxt_re_init_dmac(struct bnxt_re_dev *rdev, struct ib_ah_attr *ah_attr,
+			     struct bnxt_re_ah_info *ah_info, bool is_user,
+			     struct bnxt_re_ah *ah)
+{
+	int rc = 0;
+	u8 *dmac;
+
+	if (is_user && !rdma_is_multicast_addr((struct in6_addr *)
+						ah_attr->grh.dgid.raw) &&
+	    !rdma_link_local_addr((struct in6_addr *)ah_attr->grh.dgid.raw)) {
+
+		u32 retry_count = BNXT_RE_RESOLVE_RETRY_COUNT_US;
+		struct bnxt_re_resolve_dmac_work *resolve_dmac_work;
+
+
+		resolve_dmac_work = kzalloc(sizeof(*resolve_dmac_work), GFP_ATOMIC);
+
+		resolve_dmac_work->rdev = rdev;
+		resolve_dmac_work->ah_attr = ah_attr;
+		resolve_dmac_work->ah_info = ah_info;
+
+		atomic_set(&resolve_dmac_work->status_wait, 1);
+		INIT_WORK(&resolve_dmac_work->work, bnxt_re_resolve_dmac_task);
+		queue_work(rdev->resolve_wq, &resolve_dmac_work->work);
+
+		do {
+			rc = atomic_read(&resolve_dmac_work->status_wait) & 0xFF;
+			if (!rc)
+				break;
+			udelay(1);
+		} while (--retry_count);
+		if (atomic_read(&resolve_dmac_work->status_wait)) {
+			INIT_LIST_HEAD(&resolve_dmac_work->list);
+			list_add_tail(&resolve_dmac_work->list,
+					&rdev->mac_wq_list);
+			return -EFAULT;
+		}
+		kfree(resolve_dmac_work);
+	}
+	dmac = ROCE_DMAC(ah_attr);
+	if (dmac)
+		memcpy(ah->qplib_ah.dmac, dmac, ETH_ALEN);
+	return rc;
+}
+
+int bnxt_re_create_ah(struct ib_ah *ah_in, struct ib_ah_attr *attr,
+		      u32 flags, struct ib_udata *udata)
+{
+
+	struct ib_ah *ib_ah = ah_in;
+	struct ib_pd *ib_pd = ib_ah->pd;
+	struct bnxt_re_ah *ah = container_of(ib_ah, struct bnxt_re_ah, ibah);
+	struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ibpd);
+	struct bnxt_re_dev *rdev = pd->rdev;
+	struct bnxt_re_ah_info ah_info;
+	u32 max_ah_count;
+	bool is_user;
+	int rc;
+	bool block = true;
+	struct ib_ah_attr *ah_attr = attr;
+	block = !(flags & RDMA_CREATE_AH_SLEEPABLE);
+
+	if (!(ah_attr->ah_flags & IB_AH_GRH))
+		dev_err(rdev_to_dev(rdev), "ah_attr->ah_flags GRH is not set\n");
+
+	ah->rdev = rdev;
+	ah->qplib_ah.pd = &pd->qplib_pd;
+	is_user = ib_pd->uobject ? true : false;
+
+	/* Supply the configuration for the HW */
+	memcpy(ah->qplib_ah.dgid.data, ah_attr->grh.dgid.raw,
+			sizeof(union ib_gid));
+	ah->qplib_ah.sgid_index = _get_sgid_index(rdev, ah_attr->grh.sgid_index);
+	if (ah->qplib_ah.sgid_index == 0xFF) {
+		dev_err(rdev_to_dev(rdev), "invalid sgid_index!\n");
+		rc = -EINVAL;
+		goto fail;
+	}
+	ah->qplib_ah.host_sgid_index = ah_attr->grh.sgid_index;
+	ah->qplib_ah.traffic_class = ah_attr->grh.traffic_class;
+	ah->qplib_ah.flow_label = ah_attr->grh.flow_label;
+	ah->qplib_ah.hop_limit = ah_attr->grh.hop_limit;
+	ah->qplib_ah.sl = ah_attr->sl;
+	rc = bnxt_re_get_ah_info(rdev, ah_attr, &ah_info);
+	if (rc)
+		goto fail;
+	ah->qplib_ah.nw_type = ah_info.nw_type;
+
+	rc = bnxt_re_init_dmac(rdev, ah_attr, &ah_info, is_user, ah);
+	if (rc)
+		goto fail;
+
+	rc = bnxt_qplib_create_ah(&rdev->qplib_res, &ah->qplib_ah, block);
+	if (rc) {
+		dev_err(rdev_to_dev(rdev),
+			"Allocate HW Address Handle failed!\n");
+		goto fail;
+	}
+
+	/* Write AVID to shared page. */
+	if (ib_pd->uobject) {
+		struct ib_ucontext *ib_uctx = ib_pd->uobject->context;
+		struct bnxt_re_ucontext *uctx;
+		unsigned long flag;
+		u32 *wrptr;
+
+		uctx = to_bnxt_re(ib_uctx, struct bnxt_re_ucontext, ibucontext);
+		spin_lock_irqsave(&uctx->sh_lock, flag);
+		wrptr = (u32 *)((u8 *)uctx->shpg + BNXT_RE_AVID_OFFT);
+		*wrptr = ah->qplib_ah.id;
+		wmb(); /* make sure cache is updated. */
+		spin_unlock_irqrestore(&uctx->sh_lock, flag);
+	}
+	atomic_inc(&rdev->stats.rsors.ah_count);
+	max_ah_count = atomic_read(&rdev->stats.rsors.ah_count);
+	if (max_ah_count > atomic_read(&rdev->stats.rsors.max_ah_count))
+		atomic_set(&rdev->stats.rsors.max_ah_count, max_ah_count);
+
+	return 0;
+fail:
+	return rc;
+}
+
+int bnxt_re_modify_ah(struct ib_ah *ib_ah, struct ib_ah_attr *ah_attr)
+{
+	return 0;
+}
+
+int bnxt_re_query_ah(struct ib_ah *ib_ah, struct ib_ah_attr *ah_attr)
+{
+	struct bnxt_re_ah *ah = to_bnxt_re(ib_ah, struct bnxt_re_ah, ibah);
+
+	memcpy(ah_attr->grh.dgid.raw, ah->qplib_ah.dgid.data,
+	       sizeof(union ib_gid));
+	ah_attr->grh.sgid_index = ah->qplib_ah.host_sgid_index;
+	ah_attr->grh.traffic_class = ah->qplib_ah.traffic_class;
+	ah_attr->sl = ah->qplib_ah.sl;
+	memcpy(ROCE_DMAC(ah_attr), ah->qplib_ah.dmac, ETH_ALEN);
+	ah_attr->ah_flags = IB_AH_GRH;
+	ah_attr->port_num = 1;
+	ah_attr->static_rate = 0;
+
+	return 0;
+}
+
+/* Shared Receive Queues */
+void bnxt_re_destroy_srq(struct ib_srq *ib_srq,
+			 struct ib_udata *udata)
+{
+	struct bnxt_re_srq *srq = to_bnxt_re(ib_srq, struct bnxt_re_srq, ibsrq);
+	struct bnxt_re_dev *rdev = srq->rdev;
+	struct bnxt_qplib_srq *qplib_srq = &srq->qplib_srq;
+	int rc = 0;
+
+
+	rc = bnxt_qplib_destroy_srq(&rdev->qplib_res, qplib_srq);
+	if (rc)
+		dev_err_ratelimited(rdev_to_dev(rdev),
+				   "%s id = %d failed rc = %d\n",
+				    __func__, qplib_srq->id, rc);
+
+	if (srq->umem && !IS_ERR(srq->umem))
+		ib_umem_release(srq->umem);
+
+	atomic_dec(&rdev->stats.rsors.srq_count);
+
+	return;
+}
+
+static u16 _max_rwqe_sz(int nsge)
+{
+	return sizeof(struct rq_wqe_hdr) + (nsge * sizeof(struct sq_sge));
+}
+
+static u16 bnxt_re_get_rwqe_size(struct bnxt_qplib_qp *qplqp,
+				 int rsge, int max)
+{
+	if (qplqp->wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC)
+		rsge = max;
+
+	return _max_rwqe_sz(rsge);
+}
+
+static inline
+struct ib_umem *ib_umem_get_compat(struct bnxt_re_dev *rdev,
+				   struct ib_ucontext *ucontext,
+				   struct ib_udata *udata,
+				   unsigned long addr,
+				   size_t size, int access, int dmasync)
+{
+	return ib_umem_get(ucontext, addr, size, access, dmasync);
+}
+
+static inline
+struct ib_umem *ib_umem_get_flags_compat(struct bnxt_re_dev *rdev,
+					 struct ib_ucontext *ucontext,
+					 struct ib_udata *udata,
+					 unsigned long addr,
+					 size_t size, int access, int dmasync)
+{
+	return ib_umem_get_compat(rdev, ucontext, udata, addr, size,
+				  access, 0);
+}
+
+static inline size_t ib_umem_num_pages_compat(struct ib_umem *umem)
+{
+	return ib_umem_num_pages(umem);
+}
+
+static int bnxt_re_init_user_srq(struct bnxt_re_dev *rdev,
+				 struct bnxt_re_pd *pd,
+				 struct bnxt_re_srq *srq,
+				 struct ib_udata *udata)
+{
+	struct bnxt_qplib_sg_info *sginfo;
+	struct bnxt_qplib_srq *qplib_srq;
+	struct bnxt_re_ucontext *cntx;
+	struct ib_ucontext *context;
+	struct bnxt_re_srq_req ureq;
+	struct ib_umem *umem;
+	int rc, bytes = 0;
+
+	context = pd->ibpd.uobject->context;
+	cntx = to_bnxt_re(context, struct bnxt_re_ucontext, ibucontext);
+	qplib_srq = &srq->qplib_srq;
+	sginfo = &qplib_srq->sginfo;
+
+	if (udata->inlen < sizeof(ureq))
+		dev_warn(rdev_to_dev(rdev),
+			 "Update the library ulen %d klen %d\n",
+			 (unsigned int)udata->inlen,
+			 (unsigned int)sizeof(ureq));
+
+	rc = ib_copy_from_udata(&ureq, udata,
+				min(udata->inlen, sizeof(ureq)));
+	if (rc)
+		return rc;
+
+	bytes = (qplib_srq->max_wqe * qplib_srq->wqe_size);
+	bytes = PAGE_ALIGN(bytes);
+	umem = ib_umem_get_compat(rdev, context, udata, ureq.srqva, bytes,
+				  IB_ACCESS_LOCAL_WRITE, 1);
+	if (IS_ERR(umem)) {
+		dev_err(rdev_to_dev(rdev), "%s: ib_umem_get failed with %ld\n",
+			__func__, PTR_ERR(umem));
+		return PTR_ERR(umem);
+	}
+
+	srq->umem = umem;
+	sginfo->sghead = get_ib_umem_sgl(umem, &sginfo->nmap);
+	sginfo->npages = ib_umem_num_pages_compat(umem);
+	qplib_srq->srq_handle = ureq.srq_handle;
+	qplib_srq->dpi = &cntx->dpi;
+	qplib_srq->is_user = true;
+
+	return 0;
+}
+
+int bnxt_re_create_srq(struct ib_srq *srq_in, struct ib_srq_init_attr *srq_init_attr,
+		       struct ib_udata *udata)
+{
+	struct bnxt_qplib_dev_attr *dev_attr;
+	struct bnxt_re_ucontext *cntx = NULL;
+	struct ib_ucontext *context;
+	struct bnxt_re_dev *rdev;
+	struct bnxt_re_pd *pd;
+	int rc, entries;
+	struct ib_srq *ib_srq = srq_in;
+	struct ib_pd *ib_pd = ib_srq->pd;
+	struct bnxt_re_srq *srq =
+		container_of(ib_srq, struct bnxt_re_srq, ibsrq);
+	u32 max_srq_count;
+
+	pd = to_bnxt_re(ib_pd, struct bnxt_re_pd, ibpd);
+	rdev = pd->rdev;
+	dev_attr = rdev->dev_attr;
+
+	if (rdev->mod_exit) {
+		dev_dbg(rdev_to_dev(rdev), "%s(): in mod_exit, just return!\n", __func__);
+		rc = -EIO;
+		goto exit;
+	}
+
+	if (srq_init_attr->srq_type != IB_SRQT_BASIC) {
+		dev_err(rdev_to_dev(rdev), "SRQ type not supported\n");
+		rc = -ENOTSUPP;
+		goto exit;
+	}
+
+	if (udata) {
+		context = pd->ibpd.uobject->context;
+		cntx = to_bnxt_re(context, struct bnxt_re_ucontext, ibucontext);
+	}
+
+	if (atomic_read(&rdev->stats.rsors.srq_count) >= dev_attr->max_srq) {
+		dev_err(rdev_to_dev(rdev), "Create SRQ failed - max exceeded(SRQs)\n");
+		rc = -EINVAL;
+		goto exit;
+	}
+
+	if (srq_init_attr->attr.max_wr >= dev_attr->max_srq_wqes) {
+		dev_err(rdev_to_dev(rdev), "Create SRQ failed - max exceeded(SRQ_WQs)\n");
+		rc = -EINVAL;
+		goto exit;
+	}
+
+	srq->rdev = rdev;
+	srq->qplib_srq.pd = &pd->qplib_pd;
+	srq->qplib_srq.dpi = &rdev->dpi_privileged;
+
+	/* Allocate 1 more than what's provided so posting max doesn't
+	   mean empty */
+	entries = srq_init_attr->attr.max_wr + 1;
+	entries = bnxt_re_init_depth(entries, cntx);
+	if (entries > dev_attr->max_srq_wqes + 1)
+		entries = dev_attr->max_srq_wqes + 1;
+
+	srq->qplib_srq.wqe_size = _max_rwqe_sz(6); /* 128 byte wqe size */
+	srq->qplib_srq.max_wqe = entries;
+	srq->qplib_srq.max_sge = srq_init_attr->attr.max_sge;
+	srq->qplib_srq.threshold = srq_init_attr->attr.srq_limit;
+	srq->srq_limit = srq_init_attr->attr.srq_limit;
+	srq->qplib_srq.eventq_hw_ring_id = rdev->nqr.nq[0].ring_id;
+	srq->qplib_srq.sginfo.pgsize = PAGE_SIZE;
+	srq->qplib_srq.sginfo.pgshft = PAGE_SHIFT;
+
+	if (udata) {
+		rc = bnxt_re_init_user_srq(rdev, pd, srq, udata);
+		if (rc)
+			goto fail;
+	}
+
+	rc = bnxt_qplib_create_srq(&rdev->qplib_res, &srq->qplib_srq);
+	if (rc) {
+		dev_err(rdev_to_dev(rdev), "Create HW SRQ failed!\n");
+		goto fail;
+	}
+
+	if (udata) {
+		struct bnxt_re_srq_resp resp;
+
+		resp.srqid = srq->qplib_srq.id;
+		rc = bnxt_re_copy_to_udata(rdev, &resp,
+					   min(udata->outlen, sizeof(resp)),
+					   udata);
+		if (rc) {
+			bnxt_qplib_destroy_srq(&rdev->qplib_res, &srq->qplib_srq);
+			goto fail;
+		}
+	}
+	atomic_inc(&rdev->stats.rsors.srq_count);
+	max_srq_count = atomic_read(&rdev->stats.rsors.srq_count);
+	if (max_srq_count > atomic_read(&rdev->stats.rsors.max_srq_count))
+		atomic_set(&rdev->stats.rsors.max_srq_count, max_srq_count);
+	spin_lock_init(&srq->lock);
+
+	return 0;
+fail:
+	if (udata && srq->umem && !IS_ERR(srq->umem)) {
+		ib_umem_release(srq->umem);
+		srq->umem = NULL;
+	}
+exit:
+	return rc;
+}
+
+int bnxt_re_modify_srq(struct ib_srq *ib_srq, struct ib_srq_attr *srq_attr,
+		       enum ib_srq_attr_mask srq_attr_mask,
+		       struct ib_udata *udata)
+{
+	struct bnxt_re_srq *srq = to_bnxt_re(ib_srq, struct bnxt_re_srq,
+					     ibsrq);
+	struct bnxt_re_dev *rdev = srq->rdev;
+	int rc;
+
+	switch (srq_attr_mask) {
+	case IB_SRQ_MAX_WR:
+		/* SRQ resize is not supported */
+		break;
+	case IB_SRQ_LIMIT:
+		/* Change the SRQ threshold */
+		if (srq_attr->srq_limit > srq->qplib_srq.max_wqe)
+			return -EINVAL;
+
+		srq->qplib_srq.threshold = srq_attr->srq_limit;
+		rc = bnxt_qplib_modify_srq(&rdev->qplib_res, &srq->qplib_srq);
+		if (rc) {
+			dev_err(rdev_to_dev(rdev), "Modify HW SRQ failed!\n");
+			return rc;
+		}
+		/* On success, update the shadow */
+		srq->srq_limit = srq_attr->srq_limit;
+
+		if (udata) {
+			/* Build and send response back to udata */
+			rc = bnxt_re_copy_to_udata(rdev, srq, 0, udata);
+			if (rc)
+				return rc;
+		}
+		break;
+	default:
+		dev_err(rdev_to_dev(rdev),
+			"Unsupported srq_attr_mask 0x%x\n", srq_attr_mask);
+		return -EINVAL;
+	}
+	return 0;
+}
+
+int bnxt_re_query_srq(struct ib_srq *ib_srq, struct ib_srq_attr *srq_attr)
+{
+	struct bnxt_re_srq *srq = to_bnxt_re(ib_srq, struct bnxt_re_srq,
+					     ibsrq);
+	struct bnxt_re_dev *rdev = srq->rdev;
+	int rc;
+
+	rc = bnxt_qplib_query_srq(&rdev->qplib_res, &srq->qplib_srq);
+	if (rc) {
+		dev_err(rdev_to_dev(rdev), "Query HW SRQ (0x%x) failed! rc = %d\n",
+			srq->qplib_srq.id, rc);
+		return rc;
+	}
+	srq_attr->max_wr = srq->qplib_srq.max_wqe;
+	srq_attr->max_sge = srq->qplib_srq.max_sge;
+	srq_attr->srq_limit = srq->qplib_srq.threshold;
+
+	return 0;
+}
+
+int bnxt_re_post_srq_recv(struct ib_srq *ib_srq, const struct ib_recv_wr *wr,
+			  const struct ib_recv_wr **bad_wr)
+{
+	struct bnxt_re_srq *srq = to_bnxt_re(ib_srq, struct bnxt_re_srq,
+					     ibsrq);
+	struct bnxt_qplib_swqe wqe = {};
+	unsigned long flags;
+	int rc = 0;
+
+	spin_lock_irqsave(&srq->lock, flags);
+	while (wr) {
+		/* Transcribe each ib_recv_wr to qplib_swqe */
+		wqe.num_sge = wr->num_sge;
+		wqe.sg_list = (struct bnxt_qplib_sge *)wr->sg_list;
+		wqe.wr_id = wr->wr_id;
+		wqe.type = BNXT_QPLIB_SWQE_TYPE_RECV;
+		rc = bnxt_qplib_post_srq_recv(&srq->qplib_srq, &wqe);
+		if (rc) {
+			*bad_wr = wr;
+			break;
+		}
+		wr = wr->next;
+	}
+	spin_unlock_irqrestore(&srq->lock, flags);
+
+	return rc;
+}
+
+unsigned long bnxt_re_lock_cqs(struct bnxt_re_qp *qp)
+{
+	unsigned long flags;
+
+	spin_lock_irqsave(&qp->scq->cq_lock, flags);
+	if (qp->rcq && qp->rcq != qp->scq)
+		spin_lock(&qp->rcq->cq_lock);
+
+	return flags;
+}
+
+void bnxt_re_unlock_cqs(struct bnxt_re_qp *qp,
+				  unsigned long flags)
+{
+	if (qp->rcq && qp->rcq != qp->scq)
+		spin_unlock(&qp->rcq->cq_lock);
+	spin_unlock_irqrestore(&qp->scq->cq_lock, flags);
+}
+
+/* Queue Pairs */
+static int bnxt_re_destroy_gsi_sqp(struct bnxt_re_qp *qp)
+{
+	struct bnxt_re_qp *gsi_sqp;
+	struct bnxt_re_ah *gsi_sah;
+	struct bnxt_re_dev *rdev;
+	unsigned long flags;
+	int rc = 0;
+
+	rdev = qp->rdev;
+	gsi_sqp = rdev->gsi_ctx.gsi_sqp;
+	gsi_sah = rdev->gsi_ctx.gsi_sah;
+
+	/* remove from active qp list */
+	mutex_lock(&rdev->qp_lock);
+	list_del(&gsi_sqp->list);
+	mutex_unlock(&rdev->qp_lock);
+
+	if (gsi_sah) {
+		dev_dbg(rdev_to_dev(rdev), "Destroy the shadow AH\n");
+		rc = bnxt_qplib_destroy_ah(&rdev->qplib_res, &gsi_sah->qplib_ah,
+					   true);
+		if (rc)
+			dev_err(rdev_to_dev(rdev),
+				"Destroy HW AH for shadow QP failed!\n");
+		atomic_dec(&rdev->stats.rsors.ah_count);
+	}
+
+	dev_dbg(rdev_to_dev(rdev), "Destroy the shadow QP\n");
+	rc = bnxt_qplib_destroy_qp(&rdev->qplib_res, &gsi_sqp->qplib_qp);
+	if (rc)
+		dev_err(rdev_to_dev(rdev), "Destroy Shadow QP failed\n");
+
+	/* Clean the CQ for shadow QP completions */
+	flags = bnxt_re_lock_cqs(gsi_sqp);
+	bnxt_qplib_clean_qp(&gsi_sqp->qplib_qp);
+	bnxt_re_unlock_cqs(gsi_sqp, flags);
+
+	bnxt_qplib_free_qp_res(&rdev->qplib_res, &gsi_sqp->qplib_qp);
+	bnxt_qplib_free_hdr_buf(&rdev->qplib_res, &gsi_sqp->qplib_qp);
+	kfree(rdev->gsi_ctx.sqp_tbl);
+	kfree(gsi_sah);
+	kfree(gsi_sqp);
+	rdev->gsi_ctx.gsi_sqp = NULL;
+	rdev->gsi_ctx.gsi_sah = NULL;
+	rdev->gsi_ctx.sqp_tbl = NULL;
+	atomic_dec(&rdev->stats.rsors.qp_count);
+
+	return 0;
+}
+
+static void bnxt_re_dump_debug_stats(struct bnxt_re_dev *rdev, u32 active_qps)
+{
+	u32	total_qp = 0;
+	u64	avg_time = 0;
+	int	i;
+
+	if (!rdev->rcfw.sp_perf_stats_enabled)
+		return;
+
+	switch (active_qps) {
+	case 1:
+		/* Potential hint for Test Stop */
+		for (i = 0; i < RCFW_MAX_STAT_INDEX; i++) {
+			if (rdev->rcfw.qp_destroy_stats[i]) {
+				total_qp++;
+				avg_time += rdev->rcfw.qp_destroy_stats[i];
+			}
+		}
+		if (total_qp >= 0 || avg_time >= 0)
+			dev_dbg(rdev_to_dev(rdev),
+				"Perf Debug: %ps Total (%d) QP destroyed in (%ld) msec\n",
+				__builtin_return_address(0), total_qp,
+				(long)jiffies_to_msecs(avg_time));
+		break;
+	case 2:
+		/* Potential hint for Test Start */
+		dev_dbg(rdev_to_dev(rdev),
+			"Perf Debug: %ps active_qps = %d\n",
+			__builtin_return_address(0), active_qps);
+		break;
+	default:
+		/* Potential hint to know latency of QP destroy.
+		 * Average time taken for 1K QP Destroy.
+		 */
+		if (active_qps > 1024 && !(active_qps % 1024))
+			dev_dbg(rdev_to_dev(rdev),
+				"Perf Debug: %ps Active QP (%d) Watermark (%d)\n",
+				__builtin_return_address(0), active_qps,
+				atomic_read(&rdev->stats.rsors.max_qp_count));
+		break;
+	}
+}
+
+int bnxt_re_destroy_qp(struct ib_qp *ib_qp, struct ib_udata *udata)
+{
+	struct bnxt_re_qp *qp = to_bnxt_re(ib_qp, struct bnxt_re_qp, ib_qp);
+	struct bnxt_re_dev *rdev = qp->rdev;
+	unsigned long flags;
+	u32 active_qps;
+	int rc;
+
+	mutex_lock(&rdev->qp_lock);
+	list_del(&qp->list);
+	active_qps = atomic_dec_return(&rdev->stats.rsors.qp_count);
+	if (qp->qplib_qp.type == CMDQ_CREATE_QP_TYPE_RC)
+		atomic_dec(&rdev->stats.rsors.rc_qp_count);
+	else if (qp->qplib_qp.type == CMDQ_CREATE_QP_TYPE_UD)
+		atomic_dec(&rdev->stats.rsors.ud_qp_count);
+	mutex_unlock(&rdev->qp_lock);
+
+	rc = bnxt_qplib_destroy_qp(&rdev->qplib_res, &qp->qplib_qp);
+	if (rc)
+		dev_err_ratelimited(rdev_to_dev(rdev),
+				   "%s id = %d failed rc = %d\n",
+				    __func__, qp->qplib_qp.id, rc);
+
+	if (!ib_qp->uobject) {
+		flags = bnxt_re_lock_cqs(qp);
+		bnxt_qplib_clean_qp(&qp->qplib_qp);
+		bnxt_re_unlock_cqs(qp, flags);
+	}
+
+	bnxt_qplib_free_qp_res(&rdev->qplib_res, &qp->qplib_qp);
+	if (ib_qp->qp_type == IB_QPT_GSI &&
+	    rdev->gsi_ctx.gsi_qp_mode != BNXT_RE_GSI_MODE_UD) {
+		if (rdev->gsi_ctx.gsi_qp_mode == BNXT_RE_GSI_MODE_ALL &&
+		    rdev->gsi_ctx.gsi_sqp) {
+			bnxt_re_destroy_gsi_sqp(qp);
+		}
+		bnxt_qplib_free_hdr_buf(&rdev->qplib_res, &qp->qplib_qp);
+	}
+
+	if (qp->rumem && !IS_ERR(qp->rumem))
+		ib_umem_release(qp->rumem);
+	if (qp->sumem && !IS_ERR(qp->sumem))
+		ib_umem_release(qp->sumem);
+	kfree(qp);
+
+	bnxt_re_dump_debug_stats(rdev, active_qps);
+
+	return 0;
+}
+
+static u8 __from_ib_qp_type(enum ib_qp_type type)
+{
+	switch (type) {
+	case IB_QPT_GSI:
+		return CMDQ_CREATE_QP1_TYPE_GSI;
+	case IB_QPT_RC:
+		return CMDQ_CREATE_QP_TYPE_RC;
+	case IB_QPT_UD:
+		return CMDQ_CREATE_QP_TYPE_UD;
+	case IB_QPT_RAW_ETHERTYPE:
+		return CMDQ_CREATE_QP_TYPE_RAW_ETHERTYPE;
+	default:
+		return IB_QPT_MAX;
+	}
+}
+
+static u16 _get_swqe_sz(int nsge)
+{
+	return sizeof(struct sq_send_hdr) + nsge * sizeof(struct sq_sge);
+}
+
+static int bnxt_re_get_swqe_size(int ilsize, int nsge)
+{
+	u16 wqe_size, calc_ils;
+
+	wqe_size = _get_swqe_sz(nsge);
+	if (ilsize) {
+		calc_ils = (sizeof(struct sq_send_hdr) + ilsize);
+		wqe_size = max_t(int, calc_ils, wqe_size);
+		wqe_size = ALIGN(wqe_size, 32);
+	}
+	return wqe_size;
+}
+
+static int bnxt_re_setup_swqe_size(struct bnxt_re_qp *qp,
+				   struct ib_qp_init_attr *init_attr)
+{
+	struct bnxt_qplib_dev_attr *dev_attr;
+	struct bnxt_qplib_qp *qplqp;
+	struct bnxt_re_dev *rdev;
+	struct bnxt_qplib_q *sq;
+	int align, ilsize;
+
+	rdev = qp->rdev;
+	qplqp = &qp->qplib_qp;
+	sq = &qplqp->sq;
+	dev_attr = rdev->dev_attr;
+
+	align = sizeof(struct sq_send_hdr);
+	ilsize = ALIGN(init_attr->cap.max_inline_data, align);
+
+	sq->wqe_size = bnxt_re_get_swqe_size(ilsize, sq->max_sge);
+	if (sq->wqe_size > _get_swqe_sz(dev_attr->max_qp_sges))
+		return -EINVAL;
+	/* For Cu/Wh and gen p5 backward compatibility mode
+	 * wqe size is fixed to 128 bytes
+	 */
+	if (sq->wqe_size < _get_swqe_sz(dev_attr->max_qp_sges) &&
+	    qplqp->wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC)
+		sq->wqe_size = _get_swqe_sz(dev_attr->max_qp_sges);
+
+	if (init_attr->cap.max_inline_data) {
+		qplqp->max_inline_data = sq->wqe_size -
+					 sizeof(struct sq_send_hdr);
+		init_attr->cap.max_inline_data = qplqp->max_inline_data;
+		if (qplqp->wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC)
+			sq->max_sge = qplqp->max_inline_data /
+				      sizeof(struct sq_sge);
+	}
+
+	return 0;
+}
+
+static int bnxt_re_init_user_qp(struct bnxt_re_dev *rdev,
+				struct bnxt_re_pd *pd, struct bnxt_re_qp *qp,
+				struct ib_udata *udata)
+{
+	struct bnxt_qplib_sg_info *sginfo;
+	struct bnxt_qplib_qp *qplib_qp;
+	struct bnxt_re_ucontext *cntx;
+	struct ib_ucontext *context;
+	struct bnxt_re_qp_req ureq;
+	struct ib_umem *umem;
+	int rc, bytes = 0;
+	int psn_nume;
+	int psn_sz;
+
+	qplib_qp = &qp->qplib_qp;
+	context = pd->ibpd.uobject->context;
+	cntx = to_bnxt_re(context, struct bnxt_re_ucontext, ibucontext);
+	sginfo = &qplib_qp->sq.sginfo;
+
+	if (udata->inlen < sizeof(ureq))
+		dev_warn(rdev_to_dev(rdev),
+			 "Update the library ulen %d klen %d\n",
+			 (unsigned int)udata->inlen,
+			 (unsigned int)sizeof(ureq));
+
+	rc = ib_copy_from_udata(&ureq, udata,
+				min(udata->inlen, sizeof(ureq)));
+	if (rc)
+		return rc;
+
+	bytes = (qplib_qp->sq.max_wqe * qplib_qp->sq.wqe_size);
+	/* Consider mapping PSN search memory only for RC QPs. */
+	if (qplib_qp->type == CMDQ_CREATE_QP_TYPE_RC) {
+		psn_sz = _is_chip_gen_p5_p7(rdev->chip_ctx) ?
+				sizeof(struct sq_psn_search_ext) :
+				sizeof(struct sq_psn_search);
+		if (rdev->dev_attr && BNXT_RE_HW_RETX(rdev->dev_attr->dev_cap_flags))
+			psn_sz = sizeof(struct sq_msn_search);
+		psn_nume = (qplib_qp->wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC) ?
+			    qplib_qp->sq.max_wqe :
+			    ((qplib_qp->sq.max_wqe * qplib_qp->sq.wqe_size) /
+			     sizeof(struct bnxt_qplib_sge));
+		if (BNXT_RE_HW_RETX(rdev->dev_attr->dev_cap_flags))
+			psn_nume = roundup_pow_of_two(psn_nume);
+
+		bytes += (psn_nume * psn_sz);
+	}
+	bytes = PAGE_ALIGN(bytes);
+	umem = ib_umem_get_compat(rdev, context, udata, ureq.qpsva, bytes,
+				  IB_ACCESS_LOCAL_WRITE, 1);
+	if (IS_ERR(umem)) {
+		dev_err(rdev_to_dev(rdev), "%s: ib_umem_get failed with %ld\n",
+			__func__, PTR_ERR(umem));
+		return PTR_ERR(umem);
+	}
+
+	qp->sumem = umem;
+	/* pgsize and pgshft were initialize already. */
+	sginfo->sghead = get_ib_umem_sgl(umem, &sginfo->nmap);
+	sginfo->npages = ib_umem_num_pages_compat(umem);
+	qplib_qp->qp_handle = ureq.qp_handle;
+
+	if (!qp->qplib_qp.srq) {
+		sginfo = &qplib_qp->rq.sginfo;
+		bytes = (qplib_qp->rq.max_wqe * qplib_qp->rq.wqe_size);
+		bytes = PAGE_ALIGN(bytes);
+		umem = ib_umem_get_compat(rdev,
+					  context, udata, ureq.qprva, bytes,
+					  IB_ACCESS_LOCAL_WRITE, 1);
+		if (IS_ERR(umem)) {
+			dev_err(rdev_to_dev(rdev),
+				"%s: ib_umem_get failed ret =%ld\n",
+				__func__, PTR_ERR(umem));
+			goto rqfail;
+		}
+		qp->rumem = umem;
+		/* pgsize and pgshft were initialize already. */
+		sginfo->sghead = get_ib_umem_sgl(umem, &sginfo->nmap);
+		sginfo->npages = ib_umem_num_pages_compat(umem);
+	}
+
+	qplib_qp->dpi = &cntx->dpi;
+	qplib_qp->is_user = true;
+
+	return 0;
+rqfail:
+	ib_umem_release(qp->sumem);
+	qp->sumem = NULL;
+	qplib_qp->sq.sginfo.sghead = NULL;
+	qplib_qp->sq.sginfo.nmap = 0;
+
+	return PTR_ERR(umem);
+}
+
+static struct bnxt_re_ah *bnxt_re_create_shadow_qp_ah(struct bnxt_re_pd *pd,
+					       struct bnxt_qplib_res *qp1_res,
+					       struct bnxt_qplib_qp *qp1_qp)
+{
+	struct bnxt_re_dev *rdev = pd->rdev;
+	struct bnxt_re_ah *ah;
+	union ib_gid sgid;
+	int rc;
+
+	ah = kzalloc(sizeof(*ah), GFP_KERNEL);
+	if (!ah) {
+		dev_err(rdev_to_dev(rdev), "Allocate Address Handle failed!\n");
+		return NULL;
+	}
+	memset(ah, 0, sizeof(*ah));
+	ah->rdev = rdev;
+	ah->qplib_ah.pd = &pd->qplib_pd;
+
+	rc = bnxt_re_query_gid(&rdev->ibdev, 1, 0, &sgid);
+	if (rc)
+		goto fail;
+
+	/* supply the dgid data same as sgid */
+	memcpy(ah->qplib_ah.dgid.data, &sgid.raw,
+	       sizeof(union ib_gid));
+	ah->qplib_ah.sgid_index = 0;
+
+	ah->qplib_ah.traffic_class = 0;
+	ah->qplib_ah.flow_label = 0;
+	ah->qplib_ah.hop_limit = 1;
+	ah->qplib_ah.sl = 0;
+	/* Have DMAC same as SMAC */
+	ether_addr_copy(ah->qplib_ah.dmac, rdev->dev_addr);
+	dev_dbg(rdev_to_dev(rdev), "ah->qplib_ah.dmac = %x:%x:%x:%x:%x:%x\n",
+		ah->qplib_ah.dmac[0], ah->qplib_ah.dmac[1], ah->qplib_ah.dmac[2],
+		ah->qplib_ah.dmac[3], ah->qplib_ah.dmac[4], ah->qplib_ah.dmac[5]);
+
+	rc = bnxt_qplib_create_ah(&rdev->qplib_res, &ah->qplib_ah, true);
+	if (rc) {
+		dev_err(rdev_to_dev(rdev),
+			"Allocate HW AH for Shadow QP failed!\n");
+		goto fail;
+	}
+	dev_dbg(rdev_to_dev(rdev), "AH ID = %d\n", ah->qplib_ah.id);
+	atomic_inc(&rdev->stats.rsors.ah_count);
+
+	return ah;
+fail:
+	kfree(ah);
+	return NULL;
+}
+
+void bnxt_re_update_shadow_ah(struct bnxt_re_dev *rdev)
+{
+	struct bnxt_re_qp *gsi_qp;
+	struct bnxt_re_ah *sah;
+	struct bnxt_re_pd *pd;
+	struct ib_pd *ib_pd;
+	int rc;
+
+	if (!rdev)
+		return;
+
+	sah = rdev->gsi_ctx.gsi_sah;
+
+	dev_dbg(rdev_to_dev(rdev), "Updating the AH\n");
+	if (sah) {
+		/* Check if the AH created with current mac address */
+		if (!compare_ether_header(sah->qplib_ah.dmac, rdev->dev_addr)) {
+			dev_dbg(rdev_to_dev(rdev),
+				"Not modifying shadow AH during AH update\n");
+			return;
+		}
+
+		gsi_qp = rdev->gsi_ctx.gsi_qp;
+		ib_pd = gsi_qp->ib_qp.pd;
+		pd = to_bnxt_re(ib_pd, struct bnxt_re_pd, ibpd);
+		rc = bnxt_qplib_destroy_ah(&rdev->qplib_res,
+					   &sah->qplib_ah, false);
+		if (rc) {
+			dev_err(rdev_to_dev(rdev),
+				"Failed to destroy shadow AH during AH update\n");
+			return;
+		}
+		atomic_dec(&rdev->stats.rsors.ah_count);
+		kfree(sah);
+		rdev->gsi_ctx.gsi_sah = NULL;
+
+		sah = bnxt_re_create_shadow_qp_ah(pd, &rdev->qplib_res,
+						  &gsi_qp->qplib_qp);
+		if (!sah) {
+			dev_err(rdev_to_dev(rdev),
+				"Failed to update AH for ShadowQP\n");
+			return;
+		}
+		rdev->gsi_ctx.gsi_sah = sah;
+		atomic_inc(&rdev->stats.rsors.ah_count);
+	}
+}
+
+static struct bnxt_re_qp *bnxt_re_create_shadow_qp(struct bnxt_re_pd *pd,
+					    struct bnxt_qplib_res *qp1_res,
+					    struct bnxt_qplib_qp *qp1_qp)
+{
+	struct bnxt_re_dev *rdev = pd->rdev;
+	struct bnxt_re_qp *qp;
+	int rc;
+
+	qp = kzalloc(sizeof(*qp), GFP_KERNEL);
+	if (!qp) {
+		dev_err(rdev_to_dev(rdev), "Allocate internal UD QP failed!\n");
+		return NULL;
+	}
+	memset(qp, 0, sizeof(*qp));
+	qp->rdev = rdev;
+
+	/* Initialize the shadow QP structure from the QP1 values */
+	ether_addr_copy(qp->qplib_qp.smac, rdev->dev_addr);
+	qp->qplib_qp.pd = &pd->qplib_pd;
+	qp->qplib_qp.qp_handle = (u64)&qp->qplib_qp;
+	qp->qplib_qp.type = IB_QPT_UD;
+
+	qp->qplib_qp.max_inline_data = 0;
+	qp->qplib_qp.sig_type = true;
+
+	/* Shadow QP SQ depth should be same as QP1 RQ depth */
+	qp->qplib_qp.sq.wqe_size = bnxt_re_get_swqe_size(0, 6);
+	qp->qplib_qp.sq.max_wqe = qp1_qp->rq.max_wqe;
+	qp->qplib_qp.sq.max_sge = 2;
+	/* Q full delta can be 1 since it is internal QP */
+	qp->qplib_qp.sq.q_full_delta = 1;
+	qp->qplib_qp.sq.sginfo.pgsize = PAGE_SIZE;
+	qp->qplib_qp.sq.sginfo.pgshft = PAGE_SHIFT;
+
+	qp->qplib_qp.scq = qp1_qp->scq;
+	qp->qplib_qp.rcq = qp1_qp->rcq;
+
+	qp->qplib_qp.rq.wqe_size = _max_rwqe_sz(6); /* 128 Byte wqe size */
+	qp->qplib_qp.rq.max_wqe = qp1_qp->rq.max_wqe;
+	qp->qplib_qp.rq.max_sge = qp1_qp->rq.max_sge;
+	qp->qplib_qp.rq.sginfo.pgsize = PAGE_SIZE;
+	qp->qplib_qp.rq.sginfo.pgshft = PAGE_SHIFT;
+	/* Q full delta can be 1 since it is internal QP */
+	qp->qplib_qp.rq.q_full_delta = 1;
+	qp->qplib_qp.mtu = qp1_qp->mtu;
+	qp->qplib_qp.dpi = &rdev->dpi_privileged;
+
+	rc = bnxt_qplib_alloc_hdr_buf(qp1_res, &qp->qplib_qp, 0,
+				      BNXT_QPLIB_MAX_GRH_HDR_SIZE_IPV6);
+	if (rc)
+		goto fail;
+
+	rc = bnxt_qplib_create_qp(qp1_res, &qp->qplib_qp);
+	if (rc) {
+		dev_err(rdev_to_dev(rdev), "create HW QP failed!\n");
+		goto qp_fail;
+	}
+
+	dev_dbg(rdev_to_dev(rdev), "Created shadow QP with ID = %d\n",
+		qp->qplib_qp.id);
+	spin_lock_init(&qp->sq_lock);
+	INIT_LIST_HEAD(&qp->list);
+	mutex_lock(&rdev->qp_lock);
+	list_add_tail(&qp->list, &rdev->qp_list);
+	atomic_inc(&rdev->stats.rsors.qp_count);
+	mutex_unlock(&rdev->qp_lock);
+	return qp;
+qp_fail:
+	bnxt_qplib_free_hdr_buf(qp1_res, &qp->qplib_qp);
+fail:
+	kfree(qp);
+	return NULL;
+}
+
+static int bnxt_re_init_rq_attr(struct bnxt_re_qp *qp,
+				struct ib_qp_init_attr *init_attr, void *cntx)
+{
+	struct bnxt_qplib_dev_attr *dev_attr;
+	struct bnxt_qplib_qp *qplqp;
+	struct bnxt_re_dev *rdev;
+	struct bnxt_qplib_q *rq;
+	int entries;
+
+	rdev = qp->rdev;
+	qplqp = &qp->qplib_qp;
+	rq = &qplqp->rq;
+	dev_attr = rdev->dev_attr;
+
+	if (init_attr->srq) {
+		struct bnxt_re_srq *srq;
+
+		srq = to_bnxt_re(init_attr->srq, struct bnxt_re_srq, ibsrq);
+		if (!srq) {
+			dev_err(rdev_to_dev(rdev), "SRQ not found\n");
+			return -EINVAL;
+		}
+		qplqp->srq = &srq->qplib_srq;
+		rq->max_wqe = 0;
+	} else {
+		rq->max_sge = init_attr->cap.max_recv_sge;
+		if (rq->max_sge > dev_attr->max_qp_sges)
+			rq->max_sge = dev_attr->max_qp_sges;
+		init_attr->cap.max_recv_sge = rq->max_sge;
+		rq->wqe_size = bnxt_re_get_rwqe_size(qplqp, rq->max_sge,
+						     dev_attr->max_qp_sges);
+
+		/* Allocate 1 more than what's provided so posting max doesn't
+		   mean empty */
+		entries = init_attr->cap.max_recv_wr + 1;
+		entries = bnxt_re_init_depth(entries, cntx);
+		rq->max_wqe = min_t(u32, entries, dev_attr->max_qp_wqes + 1);
+		rq->q_full_delta = 0;
+		rq->sginfo.pgsize = PAGE_SIZE;
+		rq->sginfo.pgshft = PAGE_SHIFT;
+	}
+
+	return 0;
+}
+
+static void bnxt_re_adjust_gsi_rq_attr(struct bnxt_re_qp *qp)
+{
+	struct bnxt_qplib_dev_attr *dev_attr;
+	struct bnxt_qplib_qp *qplqp;
+	struct bnxt_re_dev *rdev;
+
+	rdev = qp->rdev;
+	qplqp = &qp->qplib_qp;
+	dev_attr = rdev->dev_attr;
+
+	if (rdev->gsi_ctx.gsi_qp_mode != BNXT_RE_GSI_MODE_UD)
+		qplqp->rq.max_sge = dev_attr->max_qp_sges;
+}
+
+static int bnxt_re_init_sq_attr(struct bnxt_re_qp *qp,
+				struct ib_qp_init_attr *init_attr,
+				void *cntx)
+{
+	struct bnxt_qplib_dev_attr *dev_attr;
+	struct bnxt_qplib_qp *qplqp;
+	struct bnxt_re_dev *rdev;
+	struct bnxt_qplib_q *sq;
+	int diff = 0;
+	int entries;
+	int rc;
+
+	rdev = qp->rdev;
+	qplqp = &qp->qplib_qp;
+	sq = &qplqp->sq;
+	dev_attr = rdev->dev_attr;
+
+	sq->max_sge = init_attr->cap.max_send_sge;
+	if (sq->max_sge > dev_attr->max_qp_sges) {
+		sq->max_sge = dev_attr->max_qp_sges;
+		init_attr->cap.max_send_sge = sq->max_sge;
+	}
+	rc = bnxt_re_setup_swqe_size(qp, init_attr);
+	if (rc)
+		return rc;
+	/*
+	 * Change the SQ depth if user has requested minimum using
+	 * configfs. Only supported for kernel consumers. Setting
+	 * min_tx_depth to 4096 to handle iser SQ full condition
+	 * in most of the newer OS distros
+	 */
+	entries = init_attr->cap.max_send_wr;
+	if (!cntx && rdev->min_tx_depth && init_attr->qp_type != IB_QPT_GSI) {
+		/*
+		 * If users specify any value greater than 1 use min_tx_depth
+		 * provided by user for comparison. Else, compare it with the
+		 * BNXT_RE_MIN_KERNEL_QP_TX_DEPTH and adjust it accordingly.
+		 */
+		if (rdev->min_tx_depth > 1 && entries < rdev->min_tx_depth)
+			entries = rdev->min_tx_depth;
+		else if (entries < BNXT_RE_MIN_KERNEL_QP_TX_DEPTH)
+			entries = BNXT_RE_MIN_KERNEL_QP_TX_DEPTH;
+	}
+	diff = bnxt_re_get_diff(cntx, rdev->chip_ctx);
+	entries = bnxt_re_init_depth(entries + diff + 1, cntx);
+	sq->max_wqe = min_t(u32, entries, dev_attr->max_qp_wqes + diff + 1);
+	sq->q_full_delta = diff + 1;
+	/*
+	 * Reserving one slot for Phantom WQE. Application can
+	 * post one extra entry in this case. But allowing this to avoid
+	 * unexpected Queue full condition
+	 */
+	sq->q_full_delta -= 1; /* becomes 0 for gen-p5 */
+	sq->sginfo.pgsize = PAGE_SIZE;
+	sq->sginfo.pgshft = PAGE_SHIFT;
+	return 0;
+}
+
+static void bnxt_re_adjust_gsi_sq_attr(struct bnxt_re_qp *qp,
+				       struct ib_qp_init_attr *init_attr,
+				       void *cntx)
+{
+	struct bnxt_qplib_dev_attr *dev_attr;
+	struct bnxt_qplib_qp *qplqp;
+	struct bnxt_re_dev *rdev;
+	int entries;
+
+	rdev = qp->rdev;
+	qplqp = &qp->qplib_qp;
+	dev_attr = rdev->dev_attr;
+
+	if (rdev->gsi_ctx.gsi_qp_mode != BNXT_RE_GSI_MODE_UD) {
+		entries = init_attr->cap.max_send_wr + 1;
+		entries = bnxt_re_init_depth(entries, cntx);
+		qplqp->sq.max_wqe = min_t(u32, entries,
+					  dev_attr->max_qp_wqes + 1);
+		qplqp->sq.q_full_delta = qplqp->sq.max_wqe -
+					 init_attr->cap.max_send_wr;
+		qplqp->sq.max_sge++; /* Need one extra sge to put UD header */
+		if (qplqp->sq.max_sge > dev_attr->max_qp_sges)
+			qplqp->sq.max_sge = dev_attr->max_qp_sges;
+	}
+}
+
+static int bnxt_re_init_qp_type(struct bnxt_re_dev *rdev,
+				struct ib_qp_init_attr *init_attr)
+{
+	struct bnxt_qplib_chip_ctx *chip_ctx;
+	struct bnxt_re_gsi_context *gsi_ctx;
+	int qptype;
+
+	chip_ctx = rdev->chip_ctx;
+	gsi_ctx = &rdev->gsi_ctx;
+
+	qptype = __from_ib_qp_type(init_attr->qp_type);
+	if (qptype == IB_QPT_MAX) {
+		dev_err(rdev_to_dev(rdev), "QP type 0x%x not supported\n",
+			qptype);
+		qptype = -EINVAL;
+		goto out;
+	}
+
+	if (_is_chip_gen_p5_p7(chip_ctx) && init_attr->qp_type == IB_QPT_GSI) {
+		/* For Thor always force UD mode. */
+		qptype = CMDQ_CREATE_QP_TYPE_GSI;
+		gsi_ctx->gsi_qp_mode = BNXT_RE_GSI_MODE_UD;
+	}
+out:
+	return qptype;
+}
+
+static int bnxt_re_init_qp_wqe_mode(struct bnxt_re_dev *rdev)
+{
+	return rdev->chip_ctx->modes.wqe_mode;
+}
+
+static int bnxt_re_init_qp_attr(struct bnxt_re_qp *qp, struct bnxt_re_pd *pd,
+				struct ib_qp_init_attr *init_attr,
+				struct ib_udata *udata)
+{
+	struct bnxt_qplib_dev_attr *dev_attr;
+	struct bnxt_re_ucontext *cntx = NULL;
+	struct ib_ucontext *context;
+	struct bnxt_qplib_qp *qplqp;
+	struct bnxt_re_dev *rdev;
+	struct bnxt_re_cq *cq;
+	int rc = 0, qptype;
+
+	rdev = qp->rdev;
+	qplqp = &qp->qplib_qp;
+	dev_attr = rdev->dev_attr;
+
+	if (udata) {
+		context = pd->ibpd.uobject->context;
+		cntx = to_bnxt_re(context, struct bnxt_re_ucontext, ibucontext);
+	}
+
+	/* Setup misc params */
+	qplqp->is_user = false;
+	qplqp->pd = &pd->qplib_pd;
+	qplqp->qp_handle = (u64)qplqp;
+	qplqp->sig_type = ((init_attr->sq_sig_type == IB_SIGNAL_ALL_WR) ?
+			    true : false);
+	qptype = bnxt_re_init_qp_type(rdev, init_attr);
+	if (qptype < 0) {
+		rc = qptype;
+		goto out;
+	}
+	qplqp->type = (u8)qptype;
+	qplqp->wqe_mode = bnxt_re_init_qp_wqe_mode(rdev);
+	ether_addr_copy(qplqp->smac, rdev->dev_addr);
+
+	if (init_attr->qp_type == IB_QPT_RC) {
+		qplqp->max_rd_atomic = dev_attr->max_qp_rd_atom;
+		qplqp->max_dest_rd_atomic = dev_attr->max_qp_init_rd_atom;
+	}
+	qplqp->mtu = ib_mtu_enum_to_int(iboe_get_mtu(rdev->netdev->if_mtu));
+	qplqp->dpi = &rdev->dpi_privileged; /* Doorbell page */
+	if (init_attr->create_flags) {
+		dev_dbg(rdev_to_dev(rdev),
+			"QP create flags 0x%x not supported\n",
+			init_attr->create_flags);
+		return -EOPNOTSUPP;
+	}
+
+	/* Setup CQs */
+	if (init_attr->send_cq) {
+		cq = to_bnxt_re(init_attr->send_cq, struct bnxt_re_cq, ibcq);
+		if (!cq) {
+			dev_err(rdev_to_dev(rdev), "Send CQ not found\n");
+			rc = -EINVAL;
+			goto out;
+		}
+		qplqp->scq = &cq->qplib_cq;
+		qp->scq = cq;
+	}
+
+	if (init_attr->recv_cq) {
+		cq = to_bnxt_re(init_attr->recv_cq, struct bnxt_re_cq, ibcq);
+		if (!cq) {
+			dev_err(rdev_to_dev(rdev), "Receive CQ not found\n");
+			rc = -EINVAL;
+			goto out;
+		}
+		qplqp->rcq = &cq->qplib_cq;
+		qp->rcq = cq;
+	}
+
+	/* Setup RQ/SRQ */
+	rc = bnxt_re_init_rq_attr(qp, init_attr, cntx);
+	if (rc)
+		goto out;
+	if (init_attr->qp_type == IB_QPT_GSI)
+		bnxt_re_adjust_gsi_rq_attr(qp);
+
+	/* Setup SQ */
+	rc = bnxt_re_init_sq_attr(qp, init_attr, cntx);
+	if (rc)
+		goto out;
+	if (init_attr->qp_type == IB_QPT_GSI)
+		bnxt_re_adjust_gsi_sq_attr(qp, init_attr, cntx);
+
+	if (udata) /* This will update DPI and qp_handle */
+		rc = bnxt_re_init_user_qp(rdev, pd, qp, udata);
+out:
+	return rc;
+}
+
+static int bnxt_re_create_shadow_gsi(struct bnxt_re_qp *qp,
+				     struct bnxt_re_pd *pd)
+{
+	struct bnxt_re_sqp_entries *sqp_tbl = NULL;
+	struct bnxt_re_dev *rdev;
+	struct bnxt_re_qp *sqp;
+	struct bnxt_re_ah *sah;
+	int rc = 0;
+
+	rdev = qp->rdev;
+	/* Create a shadow QP to handle the QP1 traffic */
+	sqp_tbl = kzalloc(sizeof(*sqp_tbl) * BNXT_RE_MAX_GSI_SQP_ENTRIES,
+			  GFP_KERNEL);
+	if (!sqp_tbl)
+		return -ENOMEM;
+	rdev->gsi_ctx.sqp_tbl = sqp_tbl;
+
+	sqp = bnxt_re_create_shadow_qp(pd, &rdev->qplib_res, &qp->qplib_qp);
+	if (!sqp) {
+		rc = -ENODEV;
+		dev_err(rdev_to_dev(rdev),
+			"Failed to create Shadow QP for QP1\n");
+		goto out;
+	}
+	rdev->gsi_ctx.gsi_sqp = sqp;
+
+	sqp->rcq = qp->rcq;
+	sqp->scq = qp->scq;
+	sah = bnxt_re_create_shadow_qp_ah(pd, &rdev->qplib_res,
+			&qp->qplib_qp);
+	if (!sah) {
+		bnxt_qplib_destroy_qp(&rdev->qplib_res,
+				&sqp->qplib_qp);
+		rc = -ENODEV;
+		dev_err(rdev_to_dev(rdev),
+				"Failed to create AH entry for ShadowQP\n");
+		goto out;
+	}
+	rdev->gsi_ctx.gsi_sah = sah;
+
+	return 0;
+out:
+	kfree(sqp_tbl);
+	return rc;
+}
+
+static int __get_rq_hdr_buf_size(u8 gsi_mode)
+{
+	return (gsi_mode == BNXT_RE_GSI_MODE_ALL) ?
+		BNXT_QPLIB_MAX_QP1_RQ_HDR_SIZE_V2 :
+		BNXT_QPLIB_MAX_QP1_RQ_HDR_SIZE;
+}
+
+static int __get_sq_hdr_buf_size(u8 gsi_mode)
+{
+	return (gsi_mode != BNXT_RE_GSI_MODE_ROCE_V1) ?
+		BNXT_QPLIB_MAX_QP1_SQ_HDR_SIZE_V2 :
+		BNXT_QPLIB_MAX_QP1_SQ_HDR_SIZE;
+}
+
+static int bnxt_re_create_gsi_qp(struct bnxt_re_qp *qp, struct bnxt_re_pd *pd)
+{
+	struct bnxt_qplib_qp *qplqp;
+	struct bnxt_qplib_res *res;
+	struct bnxt_re_dev *rdev;
+	u32 sstep, rstep;
+	u8 gsi_mode;
+	int rc = 0;
+
+	rdev = qp->rdev;
+	qplqp = &qp->qplib_qp;
+	res = &rdev->qplib_res;
+	gsi_mode = rdev->gsi_ctx.gsi_qp_mode;
+
+	rstep = __get_rq_hdr_buf_size(gsi_mode);
+	sstep = __get_sq_hdr_buf_size(gsi_mode);
+	rc = bnxt_qplib_alloc_hdr_buf(res, qplqp, sstep, rstep);
+	if (rc)
+		goto out;
+
+	rc = bnxt_qplib_create_qp1(res, qplqp);
+	if (rc) {
+		dev_err(rdev_to_dev(rdev), "create HW QP1 failed!\n");
+		goto out;
+	}
+
+	if (gsi_mode == BNXT_RE_GSI_MODE_ALL)
+		rc = bnxt_re_create_shadow_gsi(qp, pd);
+out:
+	return rc;
+}
+
+static bool bnxt_re_test_qp_limits(struct bnxt_re_dev *rdev,
+				   struct ib_qp_init_attr *init_attr,
+				   struct bnxt_qplib_dev_attr *dev_attr)
+{
+	bool rc = true;
+	int ilsize;
+
+	ilsize = ALIGN(init_attr->cap.max_inline_data, sizeof(struct sq_sge));
+	if ((init_attr->cap.max_send_wr > dev_attr->max_qp_wqes) ||
+	    (init_attr->cap.max_recv_wr > dev_attr->max_qp_wqes) ||
+	    (init_attr->cap.max_send_sge > dev_attr->max_qp_sges) ||
+	    (init_attr->cap.max_recv_sge > dev_attr->max_qp_sges) ||
+	    (ilsize > dev_attr->max_inline_data)) {
+		dev_err(rdev_to_dev(rdev), "Create QP failed - max exceeded! "
+			"0x%x/0x%x 0x%x/0x%x 0x%x/0x%x "
+			"0x%x/0x%x 0x%x/0x%x\n",
+			init_attr->cap.max_send_wr, dev_attr->max_qp_wqes,
+			init_attr->cap.max_recv_wr, dev_attr->max_qp_wqes,
+			init_attr->cap.max_send_sge, dev_attr->max_qp_sges,
+			init_attr->cap.max_recv_sge, dev_attr->max_qp_sges,
+			init_attr->cap.max_inline_data,
+			dev_attr->max_inline_data);
+		rc = false;
+	}
+	return rc;
+}
+
+static inline struct
+bnxt_re_qp *__get_qp_from_qp_in(struct ib_pd *qp_in,
+				struct bnxt_re_dev *rdev)
+{
+	struct bnxt_re_qp *qp;
+
+	qp = kzalloc(sizeof(*qp), GFP_KERNEL);
+	if (!qp)
+		dev_err(rdev_to_dev(rdev), "Allocate QP failed!\n");
+	return qp;
+}
+
+struct ib_qp *bnxt_re_create_qp(struct ib_pd *qp_in,
+			       struct ib_qp_init_attr *qp_init_attr,
+			       struct ib_udata *udata)
+{
+	struct bnxt_re_pd *pd;
+	struct ib_pd *ib_pd = qp_in;
+	struct bnxt_qplib_dev_attr *dev_attr;
+	struct bnxt_re_dev *rdev;
+	u32 active_qps, tmp_qps;
+	struct bnxt_re_qp *qp;
+	int rc;
+
+	pd = to_bnxt_re(ib_pd, struct bnxt_re_pd, ibpd);
+	rdev = pd->rdev;
+	dev_attr = rdev->dev_attr;
+	if (rdev->mod_exit) {
+		rc = -EIO;
+		dev_dbg(rdev_to_dev(rdev), "%s(): in mod_exit, just return!\n", __func__);
+		goto exit;
+	}
+
+	if (atomic_read(&rdev->stats.rsors.qp_count) >= dev_attr->max_qp) {
+		dev_err(rdev_to_dev(rdev), "Create QP failed - max exceeded(QPs Alloc'd %u of max %u)\n",
+			atomic_read(&rdev->stats.rsors.qp_count), dev_attr->max_qp);
+		rc = -EINVAL;
+		goto exit;
+	}
+
+	rc = bnxt_re_test_qp_limits(rdev, qp_init_attr, dev_attr);
+	if (!rc) {
+		rc = -EINVAL;
+		goto exit;
+	}
+	qp = __get_qp_from_qp_in(qp_in, rdev);
+	if (!qp) {
+		rc = -ENOMEM;
+		goto exit;
+	}
+	qp->rdev = rdev;
+
+	rc = bnxt_re_init_qp_attr(qp, pd, qp_init_attr, udata);
+	if (rc)
+		goto fail;
+
+	if (qp_init_attr->qp_type == IB_QPT_GSI &&
+	    !_is_chip_gen_p5_p7(rdev->chip_ctx)) {
+		rc = bnxt_re_create_gsi_qp(qp, pd);
+		if (rc == -ENODEV)
+			goto qp_destroy;
+		if (rc)
+			goto fail;
+	} else {
+		rc = bnxt_qplib_create_qp(&rdev->qplib_res, &qp->qplib_qp);
+		if (rc) {
+			dev_err(rdev_to_dev(rdev), "create HW QP failed!\n");
+			goto free_umem;
+		}
+
+		if (udata) {
+			struct bnxt_re_qp_resp resp;
+
+			resp.qpid = qp->qplib_qp.id;
+			rc = bnxt_re_copy_to_udata(rdev, &resp,
+						   min(udata->outlen, sizeof(resp)),
+						   udata);
+			if (rc)
+				goto qp_destroy;
+		}
+	}
+
+	qp->ib_qp.qp_num = qp->qplib_qp.id;
+	if (qp_init_attr->qp_type == IB_QPT_GSI)
+		rdev->gsi_ctx.gsi_qp = qp;
+	spin_lock_init(&qp->sq_lock);
+	spin_lock_init(&qp->rq_lock);
+	INIT_LIST_HEAD(&qp->list);
+	mutex_lock(&rdev->qp_lock);
+	list_add_tail(&qp->list, &rdev->qp_list);
+	mutex_unlock(&rdev->qp_lock);
+	atomic_inc(&rdev->stats.rsors.qp_count);
+	active_qps = atomic_read(&rdev->stats.rsors.qp_count);
+	if (active_qps > atomic_read(&rdev->stats.rsors.max_qp_count))
+		atomic_set(&rdev->stats.rsors.max_qp_count, active_qps);
+
+	bnxt_re_dump_debug_stats(rdev, active_qps);
+
+	/* Get the counters for RC QPs and UD QPs */
+	if (qp_init_attr->qp_type == IB_QPT_RC) {
+		tmp_qps = atomic_inc_return(&rdev->stats.rsors.rc_qp_count);
+		if (tmp_qps > atomic_read(&rdev->stats.rsors.max_rc_qp_count))
+			atomic_set(&rdev->stats.rsors.max_rc_qp_count, tmp_qps);
+	} else if (qp_init_attr->qp_type == IB_QPT_UD) {
+		tmp_qps = atomic_inc_return(&rdev->stats.rsors.ud_qp_count);
+		if (tmp_qps > atomic_read(&rdev->stats.rsors.max_ud_qp_count))
+			atomic_set(&rdev->stats.rsors.max_ud_qp_count, tmp_qps);
+	}
+
+	return &qp->ib_qp;
+
+qp_destroy:
+	bnxt_qplib_destroy_qp(&rdev->qplib_res, &qp->qplib_qp);
+free_umem:
+	if (udata) {
+		if (qp->rumem && !IS_ERR(qp->rumem))
+			ib_umem_release(qp->rumem);
+		if (qp->sumem && !IS_ERR(qp->sumem))
+			ib_umem_release(qp->sumem);
+	}
+fail:
+	kfree(qp);
+exit:
+	return ERR_PTR(rc);
+}
+
+static int bnxt_re_modify_shadow_qp(struct bnxt_re_dev *rdev,
+			     struct bnxt_re_qp *qp1_qp,
+			     int qp_attr_mask)
+{
+	struct bnxt_re_qp *qp = rdev->gsi_ctx.gsi_sqp;
+	int rc = 0;
+
+	if (qp_attr_mask & IB_QP_STATE) {
+		qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_STATE;
+		qp->qplib_qp.state = qp1_qp->qplib_qp.state;
+	}
+	if (qp_attr_mask & IB_QP_PKEY_INDEX) {
+		qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_PKEY;
+		qp->qplib_qp.pkey_index = qp1_qp->qplib_qp.pkey_index;
+	}
+
+	if (qp_attr_mask & IB_QP_QKEY) {
+		qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_QKEY;
+		/* Using a Random  QKEY */
+		qp->qplib_qp.qkey = BNXT_RE_QP_RANDOM_QKEY;
+	}
+	if (qp_attr_mask & IB_QP_SQ_PSN) {
+		qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_SQ_PSN;
+		qp->qplib_qp.sq.psn = qp1_qp->qplib_qp.sq.psn;
+	}
+
+	rc = bnxt_qplib_modify_qp(&rdev->qplib_res, &qp->qplib_qp);
+	if (rc)
+		dev_err(rdev_to_dev(rdev), "Modify Shadow QP for QP1 failed\n");
+	return rc;
+}
+
+static u32 ipv4_from_gid(u8 *gid)
+{
+	return (gid[15] << 24 | gid[14] << 16 | gid[13] << 8 | gid[12]);
+}
+
+static u16 get_source_port(struct bnxt_re_dev *rdev,
+			   struct bnxt_re_qp *qp)
+{
+	u8 ip_off, data[48], smac[ETH_ALEN];
+	u16 crc = 0, buf_len = 0, i;
+	u8 addr_len;
+	u32 qpn;
+
+	if (qp->qplib_qp.nw_type == CMDQ_MODIFY_QP_NETWORK_TYPE_ROCEV2_IPV6) {
+		addr_len = 6;
+		ip_off = 10;
+	} else {
+		addr_len = 4;
+		ip_off = 12;
+	}
+
+	memcpy(smac, qp->qplib_qp.smac, ETH_ALEN);
+
+	memset(data, 0, 48);
+	memcpy(data, qp->qplib_qp.ah.dmac, ETH_ALEN);
+	buf_len += ETH_ALEN;
+
+	memcpy(data + buf_len, smac, ETH_ALEN);
+	buf_len += ETH_ALEN;
+
+	memcpy(data + buf_len, qp->qplib_qp.ah.dgid.data + ip_off, addr_len);
+	buf_len += addr_len;
+
+	memcpy(data + buf_len, qp->qp_info_entry.sgid.raw + ip_off, addr_len);
+	buf_len += addr_len;
+
+	qpn = htonl(qp->qplib_qp.dest_qpn);
+	memcpy(data + buf_len, (u8 *)&qpn + 1, 3);
+	buf_len += 3;
+
+	for (i = 0; i < buf_len; i++)
+		crc = crc16(crc, (data + i), 1);
+
+	return crc;
+}
+
+static void bnxt_re_update_qp_info(struct bnxt_re_dev *rdev, struct bnxt_re_qp *qp)
+{
+	u16 type;
+
+	type = __from_hw_to_ib_qp_type(qp->qplib_qp.type);
+
+	/* User-space can extract ip address with sgid_index. */
+	if (ipv6_addr_v4mapped((struct in6_addr *)&qp->qplib_qp.ah.dgid)) {
+		qp->qp_info_entry.s_ip.ipv4_addr = ipv4_from_gid(qp->qp_info_entry.sgid.raw);
+		qp->qp_info_entry.d_ip.ipv4_addr = ipv4_from_gid(qp->qplib_qp.ah.dgid.data);
+	} else {
+		memcpy(&qp->qp_info_entry.s_ip.ipv6_addr, qp->qp_info_entry.sgid.raw,
+		       sizeof(qp->qp_info_entry.s_ip.ipv6_addr));
+		memcpy(&qp->qp_info_entry.d_ip.ipv6_addr, qp->qplib_qp.ah.dgid.data,
+		       sizeof(qp->qp_info_entry.d_ip.ipv6_addr));
+	}
+
+	if (type == IB_QPT_RC &&
+	    (qp->qplib_qp.nw_type == CMDQ_MODIFY_QP_NETWORK_TYPE_ROCEV2_IPV4 ||
+	     qp->qplib_qp.nw_type == CMDQ_MODIFY_QP_NETWORK_TYPE_ROCEV2_IPV6)) {
+		qp->qp_info_entry.s_port = get_source_port(rdev, qp);
+	}
+	qp->qp_info_entry.d_port = BNXT_RE_QP_DEST_PORT;
+}
+
+static void bnxt_qplib_manage_flush_qp(struct bnxt_re_qp *qp)
+{
+	struct bnxt_qplib_q *rq, *sq;
+	unsigned long flags;
+
+	if (qp->sumem)
+		return;
+
+	if (qp->qplib_qp.state == CMDQ_MODIFY_QP_NEW_STATE_ERR) {
+		rq = &qp->qplib_qp.rq;
+		sq = &qp->qplib_qp.sq;
+
+		dev_dbg(rdev_to_dev(qp->rdev),
+			"Move QP = %p to flush list\n", qp);
+		flags = bnxt_re_lock_cqs(qp);
+		bnxt_qplib_add_flush_qp(&qp->qplib_qp);
+		bnxt_re_unlock_cqs(qp, flags);
+
+		if (sq->hwq.prod != sq->hwq.cons)
+			bnxt_re_handle_cqn(&qp->scq->qplib_cq);
+
+		if (qp->rcq && (qp->rcq != qp->scq) &&
+		    (rq->hwq.prod != rq->hwq.cons))
+			bnxt_re_handle_cqn(&qp->rcq->qplib_cq);
+	}
+
+	if (qp->qplib_qp.state == CMDQ_MODIFY_QP_NEW_STATE_RESET) {
+		dev_dbg(rdev_to_dev(qp->rdev),
+			"Move QP = %p out of flush list\n", qp);
+		flags = bnxt_re_lock_cqs(qp);
+		bnxt_qplib_clean_qp(&qp->qplib_qp);
+		bnxt_re_unlock_cqs(qp, flags);
+	}
+}
+
+bool ib_modify_qp_is_ok_compat(enum ib_qp_state cur_state,
+			       enum ib_qp_state next_state,
+			       enum ib_qp_type type,
+			       enum ib_qp_attr_mask mask)
+{
+		return (ib_modify_qp_is_ok(cur_state, next_state,
+					   type, mask));
+}
+
+int bnxt_re_modify_qp(struct ib_qp *ib_qp, struct ib_qp_attr *qp_attr,
+		      int qp_attr_mask, struct ib_udata *udata)
+{
+	enum ib_qp_state curr_qp_state, new_qp_state;
+	struct bnxt_re_modify_qp_ex_resp resp = {};
+	struct bnxt_re_modify_qp_ex_req ureq = {};
+	struct bnxt_qplib_dev_attr *dev_attr;
+	struct bnxt_qplib_ppp *ppp = NULL;
+	struct bnxt_re_dev *rdev;
+	struct bnxt_re_qp *qp;
+	struct ib_gid_attr *sgid_attr;
+	struct ib_gid_attr gid_attr;
+	union ib_gid sgid, *gid_ptr = NULL;
+	u8 nw_type;
+	int rc, entries, status;
+	bool is_copy_to_udata = false;
+	bool is_qpmtu_high = false;
+
+	qp = to_bnxt_re(ib_qp, struct bnxt_re_qp, ib_qp);
+	rdev = qp->rdev;
+	dev_attr = rdev->dev_attr;
+
+	qp->qplib_qp.modify_flags = 0;
+	ppp = &qp->qplib_qp.ppp;
+	if (qp_attr_mask & IB_QP_STATE) {
+		curr_qp_state = __to_ib_qp_state(qp->qplib_qp.cur_qp_state);
+		new_qp_state = qp_attr->qp_state;
+		if (!ib_modify_qp_is_ok_compat(curr_qp_state, new_qp_state,
+					       ib_qp->qp_type, qp_attr_mask)) {
+			dev_err(rdev_to_dev(rdev),"invalid attribute mask=0x%x"
+				" specified for qpn=0x%x of type=0x%x"
+				" current_qp_state=0x%x, new_qp_state=0x%x\n",
+				qp_attr_mask, ib_qp->qp_num, ib_qp->qp_type,
+				curr_qp_state, new_qp_state);
+			return -EINVAL;
+		}
+		dev_dbg(rdev_to_dev(rdev), "%s:%d INFO attribute mask=0x%x qpn=0x%x "
+			"of type=0x%x current_qp_state=0x%x, new_qp_state=0x%x\n",
+			__func__, __LINE__, qp_attr_mask, ib_qp->qp_num,
+			ib_qp->qp_type, curr_qp_state, new_qp_state);
+		qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_STATE;
+		qp->qplib_qp.state = __from_ib_qp_state(qp_attr->qp_state);
+
+		if (udata && curr_qp_state == IB_QPS_RESET &&
+		    new_qp_state == IB_QPS_INIT) {
+			if (!ib_copy_from_udata(&ureq, udata, sizeof(ureq))) {
+				if (ureq.comp_mask &
+				    BNXT_RE_COMP_MASK_MQP_EX_PPP_REQ_EN_MASK) {
+					ppp->req = BNXT_QPLIB_PPP_REQ;
+					ppp->dpi = ureq.dpi;
+				}
+			}
+		}
+	}
+	if (qp_attr_mask & IB_QP_EN_SQD_ASYNC_NOTIFY) {
+		qp->qplib_qp.modify_flags |=
+				CMDQ_MODIFY_QP_MODIFY_MASK_EN_SQD_ASYNC_NOTIFY;
+		qp->qplib_qp.en_sqd_async_notify = true;
+	}
+	if (qp_attr_mask & IB_QP_ACCESS_FLAGS) {
+		qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_ACCESS;
+		qp->qplib_qp.access =
+			__from_ib_access_flags(qp_attr->qp_access_flags);
+		/* LOCAL_WRITE access must be set to allow RC receive */
+		qp->qplib_qp.access |= BNXT_QPLIB_ACCESS_LOCAL_WRITE;
+		qp->qplib_qp.access |= CMDQ_MODIFY_QP_ACCESS_REMOTE_WRITE;
+		qp->qplib_qp.access |= CMDQ_MODIFY_QP_ACCESS_REMOTE_READ;
+	}
+	if (qp_attr_mask & IB_QP_PKEY_INDEX) {
+		qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_PKEY;
+		qp->qplib_qp.pkey_index = qp_attr->pkey_index;
+	}
+	if (qp_attr_mask & IB_QP_QKEY) {
+		qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_QKEY;
+		qp->qplib_qp.qkey = qp_attr->qkey;
+	}
+	if (qp_attr_mask & IB_QP_AV) {
+		qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_DGID |
+				     CMDQ_MODIFY_QP_MODIFY_MASK_FLOW_LABEL |
+				     CMDQ_MODIFY_QP_MODIFY_MASK_SGID_INDEX |
+				     CMDQ_MODIFY_QP_MODIFY_MASK_HOP_LIMIT |
+				     CMDQ_MODIFY_QP_MODIFY_MASK_TRAFFIC_CLASS |
+				     CMDQ_MODIFY_QP_MODIFY_MASK_DEST_MAC |
+				     CMDQ_MODIFY_QP_MODIFY_MASK_VLAN_ID;
+		memcpy(qp->qplib_qp.ah.dgid.data, qp_attr->ah_attr.grh.dgid.raw,
+		       sizeof(qp->qplib_qp.ah.dgid.data));
+		qp->qplib_qp.ah.flow_label = qp_attr->ah_attr.grh.flow_label;
+		qp->qplib_qp.ah.sgid_index = _get_sgid_index(rdev,
+						qp_attr->ah_attr.grh.sgid_index);
+		qp->qplib_qp.ah.host_sgid_index = qp_attr->ah_attr.grh.sgid_index;
+		qp->qplib_qp.ah.hop_limit = qp_attr->ah_attr.grh.hop_limit;
+		qp->qplib_qp.ah.traffic_class =
+					qp_attr->ah_attr.grh.traffic_class;
+		qp->qplib_qp.ah.sl = qp_attr->ah_attr.sl;
+		ether_addr_copy(qp->qplib_qp.ah.dmac, ROCE_DMAC(&qp_attr->ah_attr));
+		sgid_attr = &gid_attr;
+		status = bnxt_re_get_cached_gid(&rdev->ibdev, 1,
+						qp_attr->ah_attr.grh.sgid_index,
+						&sgid, &sgid_attr,
+						&qp_attr->ah_attr.grh, NULL);
+		if (!status)
+			if_rele(sgid_attr->ndev);
+		gid_ptr = &sgid;
+		if (sgid_attr->ndev) {
+			memcpy(qp->qplib_qp.smac, rdev->dev_addr,
+			       ETH_ALEN);
+			nw_type = bnxt_re_gid_to_network_type(sgid_attr, &sgid);
+			dev_dbg(rdev_to_dev(rdev),
+				 "Connection using the nw_type %d\n", nw_type);
+			switch (nw_type) {
+			case RDMA_NETWORK_IPV4:
+				qp->qplib_qp.nw_type =
+					CMDQ_MODIFY_QP_NETWORK_TYPE_ROCEV2_IPV4;
+				break;
+			case RDMA_NETWORK_IPV6:
+				qp->qplib_qp.nw_type =
+					CMDQ_MODIFY_QP_NETWORK_TYPE_ROCEV2_IPV6;
+				break;
+			default:
+				qp->qplib_qp.nw_type =
+					CMDQ_MODIFY_QP_NETWORK_TYPE_ROCEV1;
+				break;
+			}
+		}
+		memcpy(&qp->qp_info_entry.sgid, gid_ptr, sizeof(qp->qp_info_entry.sgid));
+	}
+
+	/* MTU settings allowed only during INIT -> RTR */
+	if (qp_attr->qp_state == IB_QPS_RTR) {
+		bnxt_re_init_qpmtu(qp, rdev->netdev->if_mtu, qp_attr_mask, qp_attr,
+				   &is_qpmtu_high);
+		if (udata && !ib_copy_from_udata(&ureq, udata, sizeof(ureq))) {
+			if (ureq.comp_mask & BNXT_RE_COMP_MASK_MQP_EX_PATH_MTU_MASK) {
+				resp.comp_mask |= BNXT_RE_COMP_MASK_MQP_EX_PATH_MTU_MASK;
+				resp.path_mtu = qp->qplib_qp.mtu;
+				is_copy_to_udata = true;
+			} else if (is_qpmtu_high) {
+				dev_err(rdev_to_dev(rdev), "qp %#x invalid mtu\n",
+					qp->qplib_qp.id);
+				return -EINVAL;
+			}
+		}
+	}
+
+	if (qp_attr_mask & IB_QP_TIMEOUT) {
+		qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_TIMEOUT;
+		qp->qplib_qp.timeout = qp_attr->timeout;
+	}
+	if (qp_attr_mask & IB_QP_RETRY_CNT) {
+		qp->qplib_qp.modify_flags |=
+				CMDQ_MODIFY_QP_MODIFY_MASK_RETRY_CNT;
+		qp->qplib_qp.retry_cnt = qp_attr->retry_cnt;
+	}
+	if (qp_attr_mask & IB_QP_RNR_RETRY) {
+		qp->qplib_qp.modify_flags |=
+				CMDQ_MODIFY_QP_MODIFY_MASK_RNR_RETRY;
+		qp->qplib_qp.rnr_retry = qp_attr->rnr_retry;
+	}
+	if (qp_attr_mask & IB_QP_MIN_RNR_TIMER) {
+		qp->qplib_qp.modify_flags |=
+				CMDQ_MODIFY_QP_MODIFY_MASK_MIN_RNR_TIMER;
+		qp->qplib_qp.min_rnr_timer = qp_attr->min_rnr_timer;
+	}
+	if (qp_attr_mask & IB_QP_RQ_PSN) {
+		qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_RQ_PSN;
+		qp->qplib_qp.rq.psn = qp_attr->rq_psn;
+	}
+	if (qp_attr_mask & IB_QP_MAX_QP_RD_ATOMIC) {
+		qp->qplib_qp.modify_flags |=
+				CMDQ_MODIFY_QP_MODIFY_MASK_MAX_RD_ATOMIC;
+		/* Cap the max_rd_atomic to device max */
+		if (qp_attr->max_rd_atomic > dev_attr->max_qp_rd_atom)
+			dev_dbg(rdev_to_dev(rdev),
+				"max_rd_atomic requested %d is > device max %d\n",
+				qp_attr->max_rd_atomic,
+				dev_attr->max_qp_rd_atom);
+		qp->qplib_qp.max_rd_atomic = min_t(u32, qp_attr->max_rd_atomic,
+						   dev_attr->max_qp_rd_atom);
+	}
+	if (qp_attr_mask & IB_QP_SQ_PSN) {
+		qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_SQ_PSN;
+		qp->qplib_qp.sq.psn = qp_attr->sq_psn;
+	}
+	if (qp_attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) {
+		if (qp_attr->max_dest_rd_atomic >
+		    dev_attr->max_qp_init_rd_atom) {
+			dev_err(rdev_to_dev(rdev),
+				"max_dest_rd_atomic requested %d is > device max %d\n",
+				qp_attr->max_dest_rd_atomic,
+				dev_attr->max_qp_init_rd_atom);
+			return -EINVAL;
+		}
+		qp->qplib_qp.modify_flags |=
+				CMDQ_MODIFY_QP_MODIFY_MASK_MAX_DEST_RD_ATOMIC;
+		qp->qplib_qp.max_dest_rd_atomic = qp_attr->max_dest_rd_atomic;
+	}
+	if (qp_attr_mask & IB_QP_CAP) {
+		qp->qplib_qp.modify_flags |=
+				CMDQ_MODIFY_QP_MODIFY_MASK_SQ_SIZE |
+				CMDQ_MODIFY_QP_MODIFY_MASK_RQ_SIZE |
+				CMDQ_MODIFY_QP_MODIFY_MASK_SQ_SGE |
+				CMDQ_MODIFY_QP_MODIFY_MASK_RQ_SGE |
+				CMDQ_MODIFY_QP_MODIFY_MASK_MAX_INLINE_DATA;
+		if ((qp_attr->cap.max_send_wr >= dev_attr->max_qp_wqes) ||
+		    (qp_attr->cap.max_recv_wr >= dev_attr->max_qp_wqes) ||
+		    (qp_attr->cap.max_send_sge >= dev_attr->max_qp_sges) ||
+		    (qp_attr->cap.max_recv_sge >= dev_attr->max_qp_sges) ||
+		    (qp_attr->cap.max_inline_data >=
+						dev_attr->max_inline_data)) {
+			dev_err(rdev_to_dev(rdev),
+				"Create QP failed - max exceeded\n");
+			return -EINVAL;
+		}
+		entries = roundup_pow_of_two(qp_attr->cap.max_send_wr);
+		if (entries > dev_attr->max_qp_wqes)
+			entries = dev_attr->max_qp_wqes;
+		entries = min_t(u32, entries, dev_attr->max_qp_wqes);
+		qp->qplib_qp.sq.max_wqe = entries;
+		qp->qplib_qp.sq.q_full_delta = qp->qplib_qp.sq.max_wqe -
+						qp_attr->cap.max_send_wr;
+		/*
+		 * Reserving one slot for Phantom WQE. Some application can
+		 * post one extra entry in this case. Allowing this to avoid
+		 * unexpected Queue full condition
+		 */
+		qp->qplib_qp.sq.q_full_delta -= 1;
+		qp->qplib_qp.sq.max_sge = qp_attr->cap.max_send_sge;
+		if (qp->qplib_qp.rq.max_wqe) {
+			entries = roundup_pow_of_two(qp_attr->cap.max_recv_wr);
+			if (entries > dev_attr->max_qp_wqes)
+				entries = dev_attr->max_qp_wqes;
+			qp->qplib_qp.rq.max_wqe = entries;
+			qp->qplib_qp.rq.q_full_delta = qp->qplib_qp.rq.max_wqe -
+						       qp_attr->cap.max_recv_wr;
+			qp->qplib_qp.rq.max_sge = qp_attr->cap.max_recv_sge;
+		} else {
+			/* SRQ was used prior, just ignore the RQ caps */
+		}
+	}
+	if (qp_attr_mask & IB_QP_DEST_QPN) {
+		qp->qplib_qp.modify_flags |=
+				CMDQ_MODIFY_QP_MODIFY_MASK_DEST_QP_ID;
+		qp->qplib_qp.dest_qpn = qp_attr->dest_qp_num;
+	}
+
+	rc = bnxt_qplib_modify_qp(&rdev->qplib_res, &qp->qplib_qp);
+	if (rc) {
+		dev_err(rdev_to_dev(rdev), "Modify HW QP failed!\n");
+		return rc;
+	}
+	if (qp_attr_mask & IB_QP_STATE)
+		bnxt_qplib_manage_flush_qp(qp);
+	if (ureq.comp_mask & BNXT_RE_COMP_MASK_MQP_EX_PPP_REQ_EN_MASK &&
+	    ppp->st_idx_en & CREQ_MODIFY_QP_RESP_PINGPONG_PUSH_ENABLED) {
+		resp.comp_mask |= BNXT_RE_COMP_MASK_MQP_EX_PPP_REQ_EN;
+		resp.ppp_st_idx = ppp->st_idx_en >>
+				  BNXT_QPLIB_PPP_ST_IDX_SHIFT;
+		is_copy_to_udata = true;
+	}
+
+	if (is_copy_to_udata) {
+		rc = bnxt_re_copy_to_udata(rdev, &resp,
+					   min(udata->outlen, sizeof(resp)),
+					   udata);
+		if (rc)
+			return rc;
+	}
+
+	if (ib_qp->qp_type == IB_QPT_GSI &&
+	    rdev->gsi_ctx.gsi_qp_mode == BNXT_RE_GSI_MODE_ALL &&
+	    rdev->gsi_ctx.gsi_sqp)
+		rc = bnxt_re_modify_shadow_qp(rdev, qp, qp_attr_mask);
+	/*
+	 * Update info when qp_info_info
+	 */
+	bnxt_re_update_qp_info(rdev, qp);
+	return rc;
+}
+
+int bnxt_re_query_qp(struct ib_qp *ib_qp, struct ib_qp_attr *qp_attr,
+		     int qp_attr_mask, struct ib_qp_init_attr *qp_init_attr)
+{
+	struct bnxt_re_qp *qp = to_bnxt_re(ib_qp, struct bnxt_re_qp, ib_qp);
+	struct bnxt_re_dev *rdev = qp->rdev;
+	struct bnxt_qplib_qp *qplib_qp;
+	int rc;
+
+	qplib_qp = kcalloc(1, sizeof(*qplib_qp), GFP_KERNEL);
+	if (!qplib_qp)
+		return -ENOMEM;
+
+	qplib_qp->id = qp->qplib_qp.id;
+	qplib_qp->ah.host_sgid_index = qp->qplib_qp.ah.host_sgid_index;
+
+	rc = bnxt_qplib_query_qp(&rdev->qplib_res, qplib_qp);
+	if (rc) {
+		dev_err(rdev_to_dev(rdev), "Query HW QP (0x%x) failed! rc = %d\n",
+			qplib_qp->id, rc);
+		goto free_mem;
+	}
+	qp_attr->qp_state = __to_ib_qp_state(qplib_qp->state);
+	qp_attr->cur_qp_state = __to_ib_qp_state(qplib_qp->cur_qp_state);
+	qp_attr->en_sqd_async_notify = qplib_qp->en_sqd_async_notify ? 1 : 0;
+	qp_attr->qp_access_flags = __to_ib_access_flags(qplib_qp->access);
+	qp_attr->pkey_index = qplib_qp->pkey_index;
+	qp_attr->qkey = qplib_qp->qkey;
+	memcpy(qp_attr->ah_attr.grh.dgid.raw, qplib_qp->ah.dgid.data,
+	       sizeof(qplib_qp->ah.dgid.data));
+	qp_attr->ah_attr.grh.flow_label = qplib_qp->ah.flow_label;
+	qp_attr->ah_attr.grh.sgid_index = qplib_qp->ah.host_sgid_index;
+	qp_attr->ah_attr.grh.hop_limit = qplib_qp->ah.hop_limit;
+	qp_attr->ah_attr.grh.traffic_class = qplib_qp->ah.traffic_class;
+	qp_attr->ah_attr.sl = qplib_qp->ah.sl;
+	ether_addr_copy(ROCE_DMAC(&qp_attr->ah_attr), qplib_qp->ah.dmac);
+	qp_attr->path_mtu = __to_ib_mtu(qplib_qp->path_mtu);
+	qp_attr->timeout = qplib_qp->timeout;
+	qp_attr->retry_cnt = qplib_qp->retry_cnt;
+	qp_attr->rnr_retry = qplib_qp->rnr_retry;
+	qp_attr->min_rnr_timer = qplib_qp->min_rnr_timer;
+	qp_attr->rq_psn = qplib_qp->rq.psn;
+	qp_attr->max_rd_atomic = qplib_qp->max_rd_atomic;
+	qp_attr->sq_psn = qplib_qp->sq.psn;
+	qp_attr->max_dest_rd_atomic = qplib_qp->max_dest_rd_atomic;
+	qp_init_attr->sq_sig_type = qplib_qp->sig_type ? IB_SIGNAL_ALL_WR :
+							IB_SIGNAL_REQ_WR;
+	qp_attr->dest_qp_num = qplib_qp->dest_qpn;
+
+	qp_attr->cap.max_send_wr = qp->qplib_qp.sq.max_wqe;
+	qp_attr->cap.max_send_sge = qp->qplib_qp.sq.max_sge;
+	qp_attr->cap.max_recv_wr = qp->qplib_qp.rq.max_wqe;
+	qp_attr->cap.max_recv_sge = qp->qplib_qp.rq.max_sge;
+	qp_attr->cap.max_inline_data = qp->qplib_qp.max_inline_data;
+	qp_init_attr->cap = qp_attr->cap;
+
+free_mem:
+	kfree(qplib_qp);
+	return rc;
+}
+
+/* Builders */
+
+/* For Raw, the application is responsible to build the entire packet */
+static void bnxt_re_build_raw_send(const struct ib_send_wr *wr,
+				   struct bnxt_qplib_swqe *wqe)
+{
+	switch (wr->send_flags) {
+	case IB_SEND_IP_CSUM:
+		wqe->rawqp1.lflags |= SQ_SEND_RAWETH_QP1_LFLAGS_IP_CHKSUM;
+		break;
+	default:
+		/* Pad HW RoCE iCRC */
+		wqe->rawqp1.lflags |= SQ_SEND_RAWETH_QP1_LFLAGS_ROCE_CRC;
+		break;
+	}
+}
+
+/* For QP1, the driver must build the entire RoCE (v1/v2) packet hdr
+ * as according to the sgid and AV
+ */
+static int bnxt_re_build_qp1_send(struct bnxt_re_qp *qp, const struct ib_send_wr *wr,
+				  struct bnxt_qplib_swqe *wqe, int payload_size)
+{
+	struct bnxt_re_ah *ah = to_bnxt_re(ud_wr(wr)->ah, struct bnxt_re_ah,
+					   ibah);
+	struct bnxt_qplib_ah *qplib_ah = &ah->qplib_ah;
+	struct bnxt_qplib_sge sge;
+	int i, rc = 0;
+	union ib_gid sgid;
+	u16 vlan_id;
+	u8 *ptmac;
+	void *buf;
+
+	memset(&qp->qp1_hdr, 0, sizeof(qp->qp1_hdr));
+
+	/* Get sgid */
+	rc = bnxt_re_query_gid(&qp->rdev->ibdev, 1, qplib_ah->sgid_index, &sgid);
+	if (rc)
+		return rc;
+
+	/* ETH */
+	qp->qp1_hdr.eth_present = 1;
+	ptmac = ah->qplib_ah.dmac;
+	memcpy(qp->qp1_hdr.eth.dmac_h, ptmac, 4);
+	ptmac += 4;
+	memcpy(qp->qp1_hdr.eth.dmac_l, ptmac, 2);
+
+	ptmac = qp->qplib_qp.smac;
+	memcpy(qp->qp1_hdr.eth.smac_h, ptmac, 2);
+	ptmac += 2;
+	memcpy(qp->qp1_hdr.eth.smac_l, ptmac, 4);
+
+	qp->qp1_hdr.eth.type = cpu_to_be16(BNXT_QPLIB_ETHTYPE_ROCEV1);
+
+	/* For vlan, check the sgid for vlan existence */
+	vlan_id = rdma_get_vlan_id(&sgid);
+	if (vlan_id && vlan_id < 0x1000) {
+		qp->qp1_hdr.vlan_present = 1;
+		qp->qp1_hdr.eth.type = cpu_to_be16(ETH_P_8021Q);
+	}
+	/* GRH */
+	qp->qp1_hdr.grh_present = 1;
+	qp->qp1_hdr.grh.ip_version = 6;
+	qp->qp1_hdr.grh.payload_length =
+		cpu_to_be16((IB_BTH_BYTES + IB_DETH_BYTES + payload_size + 7)
+			    & ~3);
+	qp->qp1_hdr.grh.next_header = 0x1b;
+	memcpy(qp->qp1_hdr.grh.source_gid.raw, sgid.raw, sizeof(sgid));
+	memcpy(qp->qp1_hdr.grh.destination_gid.raw, qplib_ah->dgid.data,
+	       sizeof(sgid));
+
+	/* BTH */
+	if (wr->opcode == IB_WR_SEND_WITH_IMM) {
+		qp->qp1_hdr.bth.opcode = IB_OPCODE_UD_SEND_ONLY_WITH_IMMEDIATE;
+		qp->qp1_hdr.immediate_present = 1;
+	} else {
+		qp->qp1_hdr.bth.opcode = IB_OPCODE_UD_SEND_ONLY;
+	}
+	if (wr->send_flags & IB_SEND_SOLICITED)
+		qp->qp1_hdr.bth.solicited_event = 1;
+	qp->qp1_hdr.bth.pad_count = (4 - payload_size) & 3;
+	/* P_key for QP1 is for all members */
+	qp->qp1_hdr.bth.pkey = cpu_to_be16(0xFFFF);
+	qp->qp1_hdr.bth.destination_qpn = IB_QP1;
+	qp->qp1_hdr.bth.ack_req = 0;
+	qp->send_psn++;
+	qp->send_psn &= BTH_PSN_MASK;
+	qp->qp1_hdr.bth.psn = cpu_to_be32(qp->send_psn);
+	/* DETH */
+	/* Use the priviledged Q_Key for QP1 */
+	qp->qp1_hdr.deth.qkey = cpu_to_be32(IB_QP1_QKEY);
+	qp->qp1_hdr.deth.source_qpn = IB_QP1;
+
+	/* Pack the QP1 to the transmit buffer */
+	buf = bnxt_qplib_get_qp1_sq_buf(&qp->qplib_qp, &sge);
+	if (!buf) {
+		dev_err(rdev_to_dev(qp->rdev), "QP1 buffer is empty!\n");
+		rc = -ENOMEM;
+	}
+	for (i = wqe->num_sge; i; i--) {
+		wqe->sg_list[i].addr = wqe->sg_list[i - 1].addr;
+		wqe->sg_list[i].lkey = wqe->sg_list[i - 1].lkey;
+		wqe->sg_list[i].size = wqe->sg_list[i - 1].size;
+	}
+	wqe->sg_list[0].addr = sge.addr;
+	wqe->sg_list[0].lkey = sge.lkey;
+	wqe->sg_list[0].size = sge.size;
+	wqe->num_sge++;
+
+	return rc;
+}
+
+static int bnxt_re_build_gsi_send(struct bnxt_re_qp *qp,
+				  const struct ib_send_wr *wr,
+				  struct bnxt_qplib_swqe *wqe)
+{
+	struct bnxt_re_dev *rdev;
+	int rc, indx, len = 0;
+
+	rdev = qp->rdev;
+
+	/* Mode UD is applicable to Gen P5 only */
+	if (rdev->gsi_ctx.gsi_qp_mode == BNXT_RE_GSI_MODE_UD)
+		return 0;
+
+	for (indx = 0; indx < wr->num_sge; indx++) {
+		wqe->sg_list[indx].addr = wr->sg_list[indx].addr;
+		wqe->sg_list[indx].lkey = wr->sg_list[indx].lkey;
+		wqe->sg_list[indx].size = wr->sg_list[indx].length;
+		len += wr->sg_list[indx].length;
+	}
+	rc = bnxt_re_build_qp1_send(qp, wr, wqe, len);
+	wqe->rawqp1.lflags |= SQ_SEND_RAWETH_QP1_LFLAGS_ROCE_CRC;
+
+	return rc;
+}
+
+/* For the MAD layer, it only provides the recv SGE the size of
+   ib_grh + MAD datagram.  No Ethernet headers, Ethertype, BTH, DETH,
+   nor RoCE iCRC.  The Cu+ solution must provide buffer for the entire
+   receive packet (334 bytes) with no VLAN and then copy the GRH
+   and the MAD datagram out to the provided SGE.
+*/
+
+static int bnxt_re_build_qp1_recv(struct bnxt_re_qp *qp,
+				  const struct ib_recv_wr *wr,
+				  struct bnxt_qplib_swqe *wqe)
+{
+	struct bnxt_re_dev *rdev = qp->rdev;
+	struct bnxt_qplib_sge ref, sge;
+	u8 udp_hdr_size = 0;
+	u8 ip_hdr_size = 0;
+	int rc = 0;
+	int size;
+
+	if (bnxt_qplib_get_qp1_rq_buf(&qp->qplib_qp, &sge)) {
+		/* Create 5 SGEs as according to the following:
+		 * Ethernet header (14)
+		 * ib_grh (40) - as provided from the wr
+		 * ib_bth + ib_deth + UDP(RoCE v2 only)  (28)
+		 * MAD (256) - as provided from the wr
+		 * iCRC (4)
+		 */
+
+		/* Set RoCE v2 header size and offsets */
+		if (rdev->gsi_ctx.gsi_qp_mode == BNXT_RE_GSI_MODE_ROCE_V2_IPV4)
+			ip_hdr_size = 20;
+		if (rdev->gsi_ctx.gsi_qp_mode != BNXT_RE_GSI_MODE_ROCE_V1)
+			udp_hdr_size = 8;
+
+		/* Save the reference from ULP */
+		ref.addr = wr->sg_list[0].addr;
+		ref.lkey = wr->sg_list[0].lkey;
+		ref.size = wr->sg_list[0].length;
+
+		/* SGE 1 */
+		size = sge.size;
+		wqe->sg_list[0].addr = sge.addr;
+		wqe->sg_list[0].lkey = sge.lkey;
+		wqe->sg_list[0].size = BNXT_QPLIB_MAX_QP1_RQ_ETH_HDR_SIZE;
+		size -= wqe->sg_list[0].size;
+		if (size <= 0) {
+			dev_err(rdev_to_dev(qp->rdev),"QP1 rq buffer is empty!\n");
+			rc = -ENOMEM;
+			goto done;
+		}
+		sge.size = (u32)size;
+		sge.addr += wqe->sg_list[0].size;
+
+		/* SGE 2 */
+		/* In case of RoCE v2 ipv4 lower 20 bytes should have IP hdr */
+		wqe->sg_list[1].addr = ref.addr + ip_hdr_size;
+		wqe->sg_list[1].lkey = ref.lkey;
+		wqe->sg_list[1].size = sizeof(struct ib_grh) - ip_hdr_size;
+		ref.size -= wqe->sg_list[1].size;
+		if (ref.size <= 0) {
+			dev_err(rdev_to_dev(qp->rdev),
+				"QP1 ref buffer is empty!\n");
+			rc = -ENOMEM;
+			goto done;
+		}
+		ref.addr += wqe->sg_list[1].size + ip_hdr_size;
+
+		/* SGE 3 */
+		wqe->sg_list[2].addr = sge.addr;
+		wqe->sg_list[2].lkey = sge.lkey;
+		wqe->sg_list[2].size = BNXT_QPLIB_MAX_QP1_RQ_BDETH_HDR_SIZE +
+				       udp_hdr_size;
+		size -= wqe->sg_list[2].size;
+		if (size <= 0) {
+			dev_err(rdev_to_dev(qp->rdev),
+				"QP1 rq buffer is empty!\n");
+			rc = -ENOMEM;
+			goto done;
+		}
+		sge.size = (u32)size;
+		sge.addr += wqe->sg_list[2].size;
+
+		/* SGE 4 */
+		wqe->sg_list[3].addr = ref.addr;
+		wqe->sg_list[3].lkey = ref.lkey;
+		wqe->sg_list[3].size = ref.size;
+		ref.size -= wqe->sg_list[3].size;
+		if (ref.size) {
+			dev_err(rdev_to_dev(qp->rdev),
+				"QP1 ref buffer is incorrect!\n");
+			rc = -ENOMEM;
+			goto done;
+		}
+		/* SGE 5 */
+		wqe->sg_list[4].addr = sge.addr;
+		wqe->sg_list[4].lkey = sge.lkey;
+		wqe->sg_list[4].size = sge.size;
+		size -= wqe->sg_list[4].size;
+		if (size) {
+			dev_err(rdev_to_dev(qp->rdev),
+				"QP1 rq buffer is incorrect!\n");
+			rc = -ENOMEM;
+			goto done;
+		}
+		sge.size = (u32)size;
+		wqe->num_sge = 5;
+	} else {
+		dev_err(rdev_to_dev(qp->rdev), "QP1 buffer is empty!\n");
+		rc = -ENOMEM;
+	}
+done:
+	return rc;
+}
+
+static int bnxt_re_build_qp1_shadow_qp_recv(struct bnxt_re_qp *qp,
+					    const struct ib_recv_wr *wr,
+					    struct bnxt_qplib_swqe *wqe)
+{
+	struct bnxt_re_sqp_entries *sqp_entry;
+	struct bnxt_qplib_sge sge;
+	struct bnxt_re_dev *rdev;
+	u32 rq_prod_index;
+	int rc = 0;
+
+	rdev = qp->rdev;
+
+	rq_prod_index = bnxt_qplib_get_rq_prod_index(&qp->qplib_qp);
+
+	if (bnxt_qplib_get_qp1_rq_buf(&qp->qplib_qp, &sge)) {
+		/* Create 1 SGE to receive the entire
+		 * ethernet packet
+		 */
+		/* SGE 1 */
+		wqe->sg_list[0].addr = sge.addr;
+		/* TODO check the lkey to be used */
+		wqe->sg_list[0].lkey = sge.lkey;
+		wqe->sg_list[0].size = BNXT_QPLIB_MAX_QP1_RQ_HDR_SIZE_V2;
+		if (sge.size < wqe->sg_list[0].size) {
+			dev_err(rdev_to_dev(qp->rdev),
+				"QP1 rq buffer is empty!\n");
+			rc = -ENOMEM;
+			goto done;
+		}
+
+		sqp_entry = &rdev->gsi_ctx.sqp_tbl[rq_prod_index];
+		sqp_entry->sge.addr = wr->sg_list[0].addr;
+		sqp_entry->sge.lkey = wr->sg_list[0].lkey;
+		sqp_entry->sge.size = wr->sg_list[0].length;
+		/* Store the wrid for reporting completion */
+		sqp_entry->wrid = wqe->wr_id;
+		/* change the wqe->wrid to table index */
+		wqe->wr_id = rq_prod_index;
+	}
+done:
+	return rc;
+}
+
+static bool is_ud_qp(struct bnxt_re_qp *qp)
+{
+	return (qp->qplib_qp.type == CMDQ_CREATE_QP_TYPE_UD ||
+		qp->qplib_qp.type == CMDQ_CREATE_QP_TYPE_GSI);
+}
+
+static int bnxt_re_build_send_wqe(struct bnxt_re_qp *qp,
+				  const struct ib_send_wr *wr,
+				  struct bnxt_qplib_swqe *wqe)
+{
+	struct bnxt_re_ah *ah = NULL;
+
+	if(is_ud_qp(qp)) {
+		ah = to_bnxt_re(ud_wr(wr)->ah, struct bnxt_re_ah, ibah);
+		wqe->send.q_key = ud_wr(wr)->remote_qkey;
+		wqe->send.dst_qp = ud_wr(wr)->remote_qpn;
+		wqe->send.avid = ah->qplib_ah.id;
+	}
+	switch (wr->opcode) {
+	case IB_WR_SEND:
+		wqe->type = BNXT_QPLIB_SWQE_TYPE_SEND;
+		break;
+	case IB_WR_SEND_WITH_IMM:
+		wqe->type = BNXT_QPLIB_SWQE_TYPE_SEND_WITH_IMM;
+		wqe->send.imm_data = wr->ex.imm_data;
+		break;
+	case IB_WR_SEND_WITH_INV:
+		wqe->type = BNXT_QPLIB_SWQE_TYPE_SEND_WITH_INV;
+		wqe->send.inv_key = wr->ex.invalidate_rkey;
+		break;
+	default:
+		dev_err(rdev_to_dev(qp->rdev), "%s Invalid opcode %d!\n",
+			__func__, wr->opcode);
+		return -EINVAL;
+	}
+	if (wr->send_flags & IB_SEND_SIGNALED)
+		wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP;
+	if (wr->send_flags & IB_SEND_FENCE)
+		wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE;
+	if (wr->send_flags & IB_SEND_SOLICITED)
+		wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SOLICIT_EVENT;
+	if (wr->send_flags & IB_SEND_INLINE)
+		wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_INLINE;
+
+	return 0;
+}
+
+static int bnxt_re_build_rdma_wqe(const struct ib_send_wr *wr,
+				  struct bnxt_qplib_swqe *wqe)
+{
+	switch (wr->opcode) {
+	case IB_WR_RDMA_WRITE:
+		wqe->type = BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE;
+		break;
+	case IB_WR_RDMA_WRITE_WITH_IMM:
+		wqe->type = BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE_WITH_IMM;
+		wqe->rdma.imm_data = wr->ex.imm_data;
+		break;
+	case IB_WR_RDMA_READ:
+		wqe->type = BNXT_QPLIB_SWQE_TYPE_RDMA_READ;
+		wqe->rdma.inv_key = wr->ex.invalidate_rkey;
+		break;
+	default:
+		return -EINVAL;
+	}
+	wqe->rdma.remote_va = rdma_wr(wr)->remote_addr;
+	wqe->rdma.r_key = rdma_wr(wr)->rkey;
+	if (wr->send_flags & IB_SEND_SIGNALED)
+		wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP;
+	if (wr->send_flags & IB_SEND_FENCE)
+		wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE;
+	if (wr->send_flags & IB_SEND_SOLICITED)
+		wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SOLICIT_EVENT;
+	if (wr->send_flags & IB_SEND_INLINE)
+		wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_INLINE;
+
+	return 0;
+}
+
+static int bnxt_re_build_atomic_wqe(const struct ib_send_wr *wr,
+				    struct bnxt_qplib_swqe *wqe)
+{
+	switch (wr->opcode) {
+	case IB_WR_ATOMIC_CMP_AND_SWP:
+		wqe->type = BNXT_QPLIB_SWQE_TYPE_ATOMIC_CMP_AND_SWP;
+		wqe->atomic.cmp_data = atomic_wr(wr)->compare_add;
+		wqe->atomic.swap_data = atomic_wr(wr)->swap;
+		break;
+	case IB_WR_ATOMIC_FETCH_AND_ADD:
+		wqe->type = BNXT_QPLIB_SWQE_TYPE_ATOMIC_FETCH_AND_ADD;
+		wqe->atomic.cmp_data = atomic_wr(wr)->compare_add;
+		break;
+	default:
+		return -EINVAL;
+	}
+	wqe->atomic.remote_va = atomic_wr(wr)->remote_addr;
+	wqe->atomic.r_key = atomic_wr(wr)->rkey;
+	if (wr->send_flags & IB_SEND_SIGNALED)
+		wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP;
+	if (wr->send_flags & IB_SEND_FENCE)
+		wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE;
+	if (wr->send_flags & IB_SEND_SOLICITED)
+		wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SOLICIT_EVENT;
+	return 0;
+}
+
+static int bnxt_re_build_inv_wqe(const struct ib_send_wr *wr,
+				 struct bnxt_qplib_swqe *wqe)
+{
+	wqe->type = BNXT_QPLIB_SWQE_TYPE_LOCAL_INV;
+	wqe->local_inv.inv_l_key = wr->ex.invalidate_rkey;
+	if (wr->send_flags & IB_SEND_SIGNALED)
+		wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP;
+	if (wr->send_flags & IB_SEND_FENCE)
+		wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE;
+	if (wr->send_flags & IB_SEND_SOLICITED)
+		wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SOLICIT_EVENT;
+
+	return 0;
+}
+
+static int bnxt_re_build_reg_wqe(const struct ib_reg_wr *wr,
+				 struct bnxt_qplib_swqe *wqe)
+{
+	struct bnxt_re_mr *mr = to_bnxt_re(wr->mr, struct bnxt_re_mr, ib_mr);
+	struct bnxt_qplib_frpl *qplib_frpl = &mr->qplib_frpl;
+	int reg_len, i, access = wr->access;
+
+	if (mr->npages > qplib_frpl->max_pg_ptrs) {
+		dev_err_ratelimited(rdev_to_dev(mr->rdev),
+			" %s: failed npages %d > %d\n", __func__,
+			mr->npages, qplib_frpl->max_pg_ptrs);
+		return -EINVAL;
+	}
+
+	wqe->frmr.pbl_ptr = (__le64 *)qplib_frpl->hwq.pbl_ptr[0];
+	wqe->frmr.pbl_dma_ptr = qplib_frpl->hwq.pbl_dma_ptr[0];
+	wqe->frmr.levels = qplib_frpl->hwq.level;
+	wqe->frmr.page_list = mr->pages;
+	wqe->frmr.page_list_len = mr->npages;
+	wqe->type = BNXT_QPLIB_SWQE_TYPE_REG_MR;
+
+	if (wr->wr.send_flags & IB_SEND_SIGNALED)
+		wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP;
+	if (access & IB_ACCESS_LOCAL_WRITE)
+		wqe->frmr.access_cntl |= SQ_FR_PMR_ACCESS_CNTL_LOCAL_WRITE;
+	if (access & IB_ACCESS_REMOTE_READ)
+		wqe->frmr.access_cntl |= SQ_FR_PMR_ACCESS_CNTL_REMOTE_READ;
+	if (access & IB_ACCESS_REMOTE_WRITE)
+		wqe->frmr.access_cntl |= SQ_FR_PMR_ACCESS_CNTL_REMOTE_WRITE;
+	if (access & IB_ACCESS_REMOTE_ATOMIC)
+		wqe->frmr.access_cntl |= SQ_FR_PMR_ACCESS_CNTL_REMOTE_ATOMIC;
+	if (access & IB_ACCESS_MW_BIND)
+		wqe->frmr.access_cntl |= SQ_FR_PMR_ACCESS_CNTL_WINDOW_BIND;
+
+	/* TODO: OFED provides the rkey of the MR instead of the lkey */
+	wqe->frmr.l_key = wr->key;
+	wqe->frmr.length = wr->mr->length;
+	wqe->frmr.pbl_pg_sz_log = ilog2(PAGE_SIZE >> PAGE_SHIFT_4K);
+	wqe->frmr.pg_sz_log = ilog2(wr->mr->page_size >> PAGE_SHIFT_4K);
+	wqe->frmr.va = wr->mr->iova;
+	reg_len = wqe->frmr.page_list_len * wr->mr->page_size;
+
+	if (wqe->frmr.length > reg_len) {
+		dev_err_ratelimited(rdev_to_dev(mr->rdev),
+				    "%s: bnxt_re_mr 0x%px  len (%d > %d)\n",
+				    __func__, (void *)mr, wqe->frmr.length,
+				    reg_len);
+
+		for (i = 0; i < mr->npages; i++)
+			dev_dbg(rdev_to_dev(mr->rdev),
+				"%s: build_reg_wqe page[%d] = 0x%llx\n",
+				__func__, i, mr->pages[i]);
+
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static void bnxt_re_set_sg_list(const struct ib_send_wr *wr,
+				struct bnxt_qplib_swqe *wqe)
+{
+	wqe->sg_list = (struct bnxt_qplib_sge *)wr->sg_list;
+	wqe->num_sge = wr->num_sge;
+}
+
+static void bnxt_ud_qp_hw_stall_workaround(struct bnxt_re_qp *qp)
+{
+	if ((qp->ib_qp.qp_type == IB_QPT_UD || qp->ib_qp.qp_type == IB_QPT_GSI ||
+	    qp->ib_qp.qp_type == IB_QPT_RAW_ETHERTYPE) &&
+	    qp->qplib_qp.wqe_cnt == BNXT_RE_UD_QP_HW_STALL) {
+		int qp_attr_mask;
+		struct ib_qp_attr qp_attr;
+
+		qp_attr_mask = IB_QP_STATE;
+		qp_attr.qp_state = IB_QPS_RTS;
+		bnxt_re_modify_qp(&qp->ib_qp, &qp_attr, qp_attr_mask, NULL);
+		qp->qplib_qp.wqe_cnt = 0;
+	}
+}
+
+static int bnxt_re_post_send_shadow_qp(struct bnxt_re_dev *rdev,
+				       struct bnxt_re_qp *qp,
+				       const struct ib_send_wr *wr)
+{
+	struct bnxt_qplib_swqe wqe;
+	unsigned long flags;
+	int rc = 0;
+
+	spin_lock_irqsave(&qp->sq_lock, flags);
+	while (wr) {
+		/* House keeping */
+		memset(&wqe, 0, sizeof(wqe));
+		/* Common */
+		if (wr->num_sge > qp->qplib_qp.sq.max_sge) {
+			dev_err(rdev_to_dev(rdev),
+				"Limit exceeded for Send SGEs\n");
+			rc = -EINVAL;
+			break;
+		}
+
+		bnxt_re_set_sg_list(wr, &wqe);
+		wqe.wr_id = wr->wr_id;
+		wqe.type = BNXT_QPLIB_SWQE_TYPE_SEND;
+		rc = bnxt_re_build_send_wqe(qp, wr, &wqe);
+		if (rc)
+			break;
+
+		rc = bnxt_qplib_post_send(&qp->qplib_qp, &wqe);
+		if (rc) {
+			dev_err(rdev_to_dev(rdev),
+				"bad_wr seen with opcode = 0x%x rc = %d\n",
+				wr->opcode, rc);
+			break;
+		}
+		wr = wr->next;
+	}
+	bnxt_qplib_post_send_db(&qp->qplib_qp);
+	bnxt_ud_qp_hw_stall_workaround(qp);
+	spin_unlock_irqrestore(&qp->sq_lock, flags);
+	return rc;
+}
+
+static void bnxt_re_legacy_set_uc_fence(struct bnxt_qplib_swqe *wqe)
+{
+	/* Need unconditional fence for non-wire memory opcode
+	 * to work as expected.
+	 */
+	if (wqe->type == BNXT_QPLIB_SWQE_TYPE_LOCAL_INV ||
+	    wqe->type == BNXT_QPLIB_SWQE_TYPE_FAST_REG_MR ||
+	    wqe->type == BNXT_QPLIB_SWQE_TYPE_REG_MR ||
+	    wqe->type == BNXT_QPLIB_SWQE_TYPE_BIND_MW)
+		wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE;
+}
+
+int bnxt_re_post_send(struct ib_qp *ib_qp, const struct ib_send_wr *wr,
+		      const struct ib_send_wr **bad_wr)
+{
+	struct bnxt_re_qp *qp = to_bnxt_re(ib_qp, struct bnxt_re_qp, ib_qp);
+	struct bnxt_qplib_sge sge[6];
+	struct bnxt_qplib_swqe wqe;
+	struct bnxt_re_dev *rdev;
+	unsigned long flags;
+	int rc = 0;
+
+	rdev = qp->rdev;
+	spin_lock_irqsave(&qp->sq_lock, flags);
+	while (wr) {
+		/* House keeping */
+		memset(&wqe, 0, sizeof(wqe));
+		/* Common */
+		if (wr->num_sge > qp->qplib_qp.sq.max_sge) {
+			dev_err(rdev_to_dev(rdev),
+				"Limit exceeded for Send SGEs\n");
+			rc = -EINVAL;
+			goto bad;
+		}
+
+		bnxt_re_set_sg_list(wr, &wqe);
+		wqe.wr_id = wr->wr_id;
+
+		switch (wr->opcode) {
+		case IB_WR_SEND:
+		case IB_WR_SEND_WITH_IMM:
+			if (ib_qp->qp_type == IB_QPT_GSI &&
+			    rdev->gsi_ctx.gsi_qp_mode != BNXT_RE_GSI_MODE_UD) {
+				memset(sge, 0, sizeof(sge));
+				wqe.sg_list = sge;
+				rc = bnxt_re_build_gsi_send(qp, wr, &wqe);
+				if (rc)
+					goto bad;
+			} else if (ib_qp->qp_type == IB_QPT_RAW_ETHERTYPE) {
+				bnxt_re_build_raw_send(wr, &wqe);
+			}
+			switch (wr->send_flags) {
+			case IB_SEND_IP_CSUM:
+				wqe.rawqp1.lflags |=
+					SQ_SEND_RAWETH_QP1_LFLAGS_IP_CHKSUM;
+				break;
+			default:
+				break;
+			}
+			fallthrough;
+		case IB_WR_SEND_WITH_INV:
+			rc = bnxt_re_build_send_wqe(qp, wr, &wqe);
+			break;
+		case IB_WR_RDMA_WRITE:
+		case IB_WR_RDMA_WRITE_WITH_IMM:
+		case IB_WR_RDMA_READ:
+			rc = bnxt_re_build_rdma_wqe(wr, &wqe);
+			break;
+		case IB_WR_ATOMIC_CMP_AND_SWP:
+		case IB_WR_ATOMIC_FETCH_AND_ADD:
+			rc = bnxt_re_build_atomic_wqe(wr, &wqe);
+			break;
+		case IB_WR_RDMA_READ_WITH_INV:
+			dev_err(rdev_to_dev(rdev),
+				"RDMA Read with Invalidate is not supported\n");
+			rc = -EINVAL;
+			goto bad;
+		case IB_WR_LOCAL_INV:
+			rc = bnxt_re_build_inv_wqe(wr, &wqe);
+			break;
+		case IB_WR_REG_MR:
+			rc = bnxt_re_build_reg_wqe(reg_wr(wr), &wqe);
+			break;
+		default:
+			/* Unsupported WRs */
+			dev_err(rdev_to_dev(rdev),
+				"WR (0x%x) is not supported\n", wr->opcode);
+			rc = -EINVAL;
+			goto bad;
+		}
+
+		if (likely(!rc)) {
+			if (!_is_chip_gen_p5_p7(rdev->chip_ctx))
+				bnxt_re_legacy_set_uc_fence(&wqe);
+			rc = bnxt_qplib_post_send(&qp->qplib_qp, &wqe);
+		}
+bad:
+		if (unlikely(rc)) {
+			dev_err(rdev_to_dev(rdev),
+				"bad_wr seen with opcode = 0x%x\n", wr->opcode);
+			*bad_wr = wr;
+			break;
+		}
+		wr = wr->next;
+	}
+	bnxt_qplib_post_send_db(&qp->qplib_qp);
+	if (!_is_chip_gen_p5_p7(rdev->chip_ctx))
+		bnxt_ud_qp_hw_stall_workaround(qp);
+	spin_unlock_irqrestore(&qp->sq_lock, flags);
+
+	return rc;
+}
+
+static int bnxt_re_post_recv_shadow_qp(struct bnxt_re_dev *rdev,
+				struct bnxt_re_qp *qp,
+				struct ib_recv_wr *wr)
+{
+	struct bnxt_qplib_swqe wqe;
+	int rc = 0;
+
+	/* rq lock can be pardoned here. */
+	while (wr) {
+		/* House keeping */
+		memset(&wqe, 0, sizeof(wqe));
+		/* Common */
+		if (wr->num_sge > qp->qplib_qp.rq.max_sge) {
+			dev_err(rdev_to_dev(rdev),
+				"Limit exceeded for Receive SGEs\n");
+			rc = -EINVAL;
+			goto bad;
+		}
+
+		wqe.sg_list = (struct bnxt_qplib_sge *)wr->sg_list;
+		wqe.num_sge = wr->num_sge;
+		wqe.wr_id = wr->wr_id;
+		wqe.type = BNXT_QPLIB_SWQE_TYPE_RECV;
+		rc = bnxt_qplib_post_recv(&qp->qplib_qp, &wqe);
+bad:
+		if (rc) {
+			dev_err(rdev_to_dev(rdev),
+				"bad_wr seen with RQ post\n");
+			break;
+		}
+		wr = wr->next;
+	}
+	bnxt_qplib_post_recv_db(&qp->qplib_qp);
+	return rc;
+}
+
+static int bnxt_re_build_gsi_recv(struct bnxt_re_qp *qp,
+				  const struct ib_recv_wr *wr,
+				  struct bnxt_qplib_swqe *wqe)
+{
+	struct bnxt_re_dev *rdev = qp->rdev;
+	int rc = 0;
+
+	if (rdev->gsi_ctx.gsi_qp_mode == BNXT_RE_GSI_MODE_ALL)
+		rc = bnxt_re_build_qp1_shadow_qp_recv(qp, wr, wqe);
+	else
+		rc = bnxt_re_build_qp1_recv(qp, wr, wqe);
+
+	return rc;
+}
+
+int bnxt_re_post_recv(struct ib_qp *ib_qp, const struct ib_recv_wr *wr,
+		      const struct ib_recv_wr **bad_wr)
+{
+	struct bnxt_re_qp *qp = to_bnxt_re(ib_qp, struct bnxt_re_qp, ib_qp);
+	struct bnxt_qplib_sge sge[6];
+	struct bnxt_qplib_swqe wqe;
+	unsigned long flags;
+	u32 count = 0;
+	int rc = 0;
+
+	spin_lock_irqsave(&qp->rq_lock, flags);
+	while (wr) {
+		memset(&wqe, 0, sizeof(wqe));
+		if (wr->num_sge > qp->qplib_qp.rq.max_sge) {
+			dev_err(rdev_to_dev(qp->rdev),
+				"Limit exceeded for Receive SGEs\n");
+			rc = -EINVAL;
+			goto bad;
+		}
+		wqe.num_sge = wr->num_sge;
+		wqe.sg_list = (struct bnxt_qplib_sge *)wr->sg_list;
+		wqe.wr_id = wr->wr_id;
+		wqe.type = BNXT_QPLIB_SWQE_TYPE_RECV;
+
+		if (ib_qp->qp_type == IB_QPT_GSI &&
+		    qp->rdev->gsi_ctx.gsi_qp_mode != BNXT_RE_GSI_MODE_UD) {
+			memset(sge, 0, sizeof(sge));
+			wqe.sg_list = sge;
+			rc = bnxt_re_build_gsi_recv(qp, wr, &wqe);
+			if (rc)
+				goto bad;
+		}
+		rc = bnxt_qplib_post_recv(&qp->qplib_qp, &wqe);
+bad:
+		if (rc) {
+			dev_err(rdev_to_dev(qp->rdev),
+				"bad_wr seen with RQ post\n");
+			*bad_wr = wr;
+			break;
+		}
+		/* Ring DB if the RQEs posted reaches a threshold value */
+		if (++count >= BNXT_RE_RQ_WQE_THRESHOLD) {
+			bnxt_qplib_post_recv_db(&qp->qplib_qp);
+			count = 0;
+		}
+		wr = wr->next;
+	}
+
+	if (count)
+		bnxt_qplib_post_recv_db(&qp->qplib_qp);
+	spin_unlock_irqrestore(&qp->rq_lock, flags);
+
+	return rc;
+}
+
+/* Completion Queues */
+void bnxt_re_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata)
+{
+	struct bnxt_re_cq *cq = to_bnxt_re(ib_cq, struct bnxt_re_cq, ibcq);
+	struct bnxt_re_dev *rdev = cq->rdev;
+	int rc =  0;
+
+	if (cq->uctx_cq_page) {
+		BNXT_RE_CQ_PAGE_LIST_DEL(cq->uctx, cq);
+		free_page((u64)cq->uctx_cq_page);
+		cq->uctx_cq_page = NULL;
+	}
+	if (cq->is_dbr_soft_cq && cq->uctx) {
+		void *dbr_page;
+
+		if (cq->uctx->dbr_recov_cq) {
+			dbr_page = cq->uctx->dbr_recov_cq_page;
+			cq->uctx->dbr_recov_cq_page = NULL;
+			cq->uctx->dbr_recov_cq = NULL;
+			free_page((unsigned long)dbr_page);
+		}
+		goto end;
+	}
+	/* CQ getting destroyed. Set this state for cqn handler */
+	spin_lock_bh(&cq->qplib_cq.compl_lock);
+	cq->qplib_cq.destroyed = true;
+	spin_unlock_bh(&cq->qplib_cq.compl_lock);
+	if (ib_cq->poll_ctx == IB_POLL_WORKQUEUE ||
+	    ib_cq->poll_ctx == IB_POLL_UNBOUND_WORKQUEUE)
+		cancel_work_sync(&ib_cq->work);
+
+	rc = bnxt_qplib_destroy_cq(&rdev->qplib_res, &cq->qplib_cq);
+	if (rc)
+		dev_err_ratelimited(rdev_to_dev(rdev),
+				   "%s id = %d failed rc = %d\n",
+				   __func__, cq->qplib_cq.id, rc);
+
+	bnxt_re_put_nq(rdev, cq->qplib_cq.nq);
+	if (cq->umem && !IS_ERR(cq->umem))
+		ib_umem_release(cq->umem);
+
+	kfree(cq->cql);
+	atomic_dec(&rdev->stats.rsors.cq_count);
+end:
+	return;
+}
+
+static inline struct
+bnxt_re_cq *__get_cq_from_cq_in(struct ib_cq *cq_in,
+				struct bnxt_re_dev *rdev)
+{
+	struct bnxt_re_cq *cq;
+	cq = container_of(cq_in, struct bnxt_re_cq, ibcq);
+	return cq;
+}
+
+int bnxt_re_create_cq(struct ib_cq *cq_in,
+		      const struct ib_cq_init_attr *attr,
+		      struct ib_udata *udata)
+{
+	struct bnxt_qplib_dev_attr *dev_attr;
+	struct bnxt_re_ucontext *uctx = NULL;
+	struct ib_ucontext *context = NULL;
+	struct bnxt_qplib_cq *qplcq;
+	struct bnxt_re_cq_req ureq;
+	struct bnxt_re_dev *rdev;
+	int rc, entries;
+	struct bnxt_re_cq *cq;
+	u32 max_active_cqs;
+	int cqe = attr->cqe;
+
+	if (attr->flags)
+		return -EOPNOTSUPP;
+
+	rdev = rdev_from_cq_in(cq_in);
+	if (rdev->mod_exit) {
+		rc = -EIO;
+		dev_dbg(rdev_to_dev(rdev), "%s(): in mod_exit, just return!\n", __func__);
+		goto exit;
+	}
+	if (udata) {
+		uctx = rdma_udata_to_drv_context(udata,
+						 struct bnxt_re_ucontext,
+						 ibucontext);
+		context = &uctx->ibucontext;
+	}
+	dev_attr = rdev->dev_attr;
+
+	if (atomic_read(&rdev->stats.rsors.cq_count) >= dev_attr->max_cq) {
+		dev_err(rdev_to_dev(rdev), "Create CQ failed - max exceeded(CQs)\n");
+		rc = -EINVAL;
+		goto exit;
+	}
+	/* Validate CQ fields */
+	if (cqe < 1 || cqe > dev_attr->max_cq_wqes) {
+		dev_err(rdev_to_dev(rdev), "Create CQ failed - max exceeded(CQ_WQs)\n");
+		rc = -EINVAL;
+		goto exit;
+	}
+
+	cq = __get_cq_from_cq_in(cq_in, rdev);
+	if (!cq) {
+		rc = -ENOMEM;
+		goto exit;
+	}
+	cq->rdev = rdev;
+	cq->uctx = uctx;
+	qplcq = &cq->qplib_cq;
+	qplcq->cq_handle = (u64)qplcq;
+	/*
+	 * Since CQ is for QP1 is shared with Shadow CQ, the size
+	 * should be double the size. There is no way to identify
+	 * whether this CQ is for GSI QP. So assuming that the first
+	 * CQ created is for QP1
+	 */
+	if (!udata && !rdev->gsi_ctx.first_cq_created &&
+	    rdev->gsi_ctx.gsi_qp_mode == BNXT_RE_GSI_MODE_ALL) {
+		rdev->gsi_ctx.first_cq_created = true;
+		/*
+		 * Total CQE required for the CQ = CQE for QP1 RQ +
+		 * CQE for Shadow QP SQEs + CQE for Shadow QP RQEs.
+		 * Max entries of shadow QP SQ and RQ = QP1 RQEs = cqe
+		 */
+		cqe *= 3;
+	}
+
+	entries = bnxt_re_init_depth(cqe + 1, uctx);
+	if (entries > dev_attr->max_cq_wqes + 1)
+		entries = dev_attr->max_cq_wqes + 1;
+
+	qplcq->sginfo.pgshft = PAGE_SHIFT;
+	qplcq->sginfo.pgsize = PAGE_SIZE;
+	if (udata) {
+		if (udata->inlen < sizeof(ureq))
+			dev_warn(rdev_to_dev(rdev),
+				 "Update the library ulen %d klen %d\n",
+				 (unsigned int)udata->inlen,
+				 (unsigned int)sizeof(ureq));
+
+		rc = ib_copy_from_udata(&ureq, udata,
+					min(udata->inlen, sizeof(ureq)));
+		if (rc)
+			goto fail;
+
+		if (BNXT_RE_IS_DBR_PACING_NOTIFY_CQ(ureq)) {
+			cq->is_dbr_soft_cq = true;
+			goto success;
+		}
+
+		if (BNXT_RE_IS_DBR_RECOV_CQ(ureq)) {
+			void *dbr_page;
+			u32 *epoch;
+
+			dbr_page = (void *)__get_free_page(GFP_KERNEL);
+			if (!dbr_page) {
+				dev_err(rdev_to_dev(rdev),
+					"DBR recov CQ page allocation failed!");
+				rc = -ENOMEM;
+				goto fail;
+			}
+
+			/* memset the epoch and epoch_ack to 0 */
+			epoch = dbr_page;
+			epoch[0] = 0x0;
+			epoch[1] = 0x0;
+
+			uctx->dbr_recov_cq = cq;
+			uctx->dbr_recov_cq_page = dbr_page;
+
+			cq->is_dbr_soft_cq = true;
+			goto success;
+		}
+
+		cq->umem = ib_umem_get_compat
+				      (rdev, context, udata, ureq.cq_va,
+				       entries * sizeof(struct cq_base),
+				       IB_ACCESS_LOCAL_WRITE, 1);
+		if (IS_ERR(cq->umem)) {
+			rc = PTR_ERR(cq->umem);
+			dev_err(rdev_to_dev(rdev),
+				"%s: ib_umem_get failed! rc = %d\n",
+				__func__, rc);
+			goto fail;
+		}
+		qplcq->sginfo.sghead = get_ib_umem_sgl(cq->umem,
+						       &qplcq->sginfo.nmap);
+		qplcq->sginfo.npages = ib_umem_num_pages_compat(cq->umem);
+		if (!uctx->dpi.dbr) {
+			rc = bnxt_re_get_user_dpi(rdev, uctx);
+			if (rc)
+				goto c2fail;
+		}
+		qplcq->dpi = &uctx->dpi;
+	} else {
+		cq->max_cql = entries > MAX_CQL_PER_POLL ? MAX_CQL_PER_POLL : entries;
+		cq->cql = kcalloc(cq->max_cql, sizeof(struct bnxt_qplib_cqe),
+				  GFP_KERNEL);
+		if (!cq->cql) {
+			dev_err(rdev_to_dev(rdev),
+				"Allocate CQL for %d failed!\n", cq->max_cql);
+			rc = -ENOMEM;
+			goto fail;
+		}
+		qplcq->dpi = &rdev->dpi_privileged;
+	}
+	/*
+	 * Allocating the NQ in a round robin fashion. nq_alloc_cnt is a
+	 * used for getting the NQ index.
+	 */
+	qplcq->max_wqe = entries;
+	qplcq->nq = bnxt_re_get_nq(rdev);
+	qplcq->cnq_hw_ring_id = qplcq->nq->ring_id;
+
+	rc = bnxt_qplib_create_cq(&rdev->qplib_res, qplcq);
+	if (rc) {
+		dev_err(rdev_to_dev(rdev), "Create HW CQ failed!\n");
+		goto fail;
+	}
+
+	INIT_LIST_HEAD(&cq->cq_list);
+	cq->ibcq.cqe = entries;
+	cq->cq_period = qplcq->period;
+
+	atomic_inc(&rdev->stats.rsors.cq_count);
+	max_active_cqs = atomic_read(&rdev->stats.rsors.cq_count);
+	if (max_active_cqs > atomic_read(&rdev->stats.rsors.max_cq_count))
+		atomic_set(&rdev->stats.rsors.max_cq_count, max_active_cqs);
+	spin_lock_init(&cq->cq_lock);
+
+	if (udata) {
+		struct bnxt_re_cq_resp resp;
+
+		resp.cqid = qplcq->id;
+		resp.tail = qplcq->hwq.cons;
+		resp.phase = qplcq->period;
+		resp.comp_mask = 0;
+		resp.dbr = (u64)uctx->dpi.umdbr;
+		resp.dpi = uctx->dpi.dpi;
+		resp.comp_mask |= BNXT_RE_COMP_MASK_CQ_HAS_DB_INFO;
+		/* Copy only on a valid wcpdi */
+		if (uctx->wcdpi.dpi) {
+			resp.wcdpi = uctx->wcdpi.dpi;
+			resp.comp_mask |= BNXT_RE_COMP_MASK_CQ_HAS_WC_DPI;
+		}
+
+		if (_is_chip_p7(rdev->chip_ctx)) {
+			cq->uctx_cq_page = (void *)__get_free_page(GFP_KERNEL);
+
+			if (!cq->uctx_cq_page) {
+				dev_err(rdev_to_dev(rdev),
+					"CQ page allocation failed!\n");
+				bnxt_qplib_destroy_cq(&rdev->qplib_res, qplcq);
+				rc = -ENOMEM;
+				goto c2fail;
+			}
+
+			resp.uctx_cq_page = (u64)cq->uctx_cq_page;
+			resp.comp_mask |= BNXT_RE_COMP_MASK_CQ_HAS_CQ_PAGE;
+		}
+
+		rc = bnxt_re_copy_to_udata(rdev, &resp,
+					   min(udata->outlen, sizeof(resp)),
+					   udata);
+		if (rc) {
+			free_page((u64)cq->uctx_cq_page);
+			cq->uctx_cq_page = NULL;
+			bnxt_qplib_destroy_cq(&rdev->qplib_res, qplcq);
+			goto c2fail;
+		}
+
+		if (cq->uctx_cq_page)
+			BNXT_RE_CQ_PAGE_LIST_ADD(uctx, cq);
+	}
+
+success:
+	return 0;
+c2fail:
+	if (udata && cq->umem && !IS_ERR(cq->umem))
+		ib_umem_release(cq->umem);
+fail:
+	if (cq) {
+		if (cq->cql)
+			kfree(cq->cql);
+	}
+exit:
+	return rc;
+}
+
+int bnxt_re_modify_cq(struct ib_cq *ib_cq, u16 cq_count, u16 cq_period)
+{
+	struct bnxt_re_cq *cq = to_bnxt_re(ib_cq, struct bnxt_re_cq, ibcq);
+	struct bnxt_re_dev *rdev = cq->rdev;
+	int rc;
+
+	if ((cq->cq_count != cq_count) || (cq->cq_period != cq_period)) {
+		cq->qplib_cq.count = cq_count;
+		cq->qplib_cq.period = cq_period;
+		rc = bnxt_qplib_modify_cq(&rdev->qplib_res, &cq->qplib_cq);
+		if (rc) {
+			dev_err(rdev_to_dev(rdev), "Modify HW CQ %#x failed!\n",
+				cq->qplib_cq.id);
+			return rc;
+		}
+		/* On success, update the shadow */
+		cq->cq_count = cq_count;
+		cq->cq_period = cq_period;
+	}
+	return 0;
+}
+
+static void bnxt_re_resize_cq_complete(struct bnxt_re_cq *cq)
+{
+	struct bnxt_re_dev *rdev = cq->rdev;
+
+	bnxt_qplib_resize_cq_complete(&rdev->qplib_res, &cq->qplib_cq);
+
+	cq->qplib_cq.max_wqe = cq->resize_cqe;
+	if (cq->resize_umem) {
+		ib_umem_release(cq->umem);
+		cq->umem = cq->resize_umem;
+		cq->resize_umem = NULL;
+		cq->resize_cqe = 0;
+	}
+}
+
+int bnxt_re_resize_cq(struct ib_cq *ib_cq, int cqe, struct ib_udata *udata)
+{
+	struct bnxt_qplib_sg_info sginfo = {};
+	struct bnxt_qplib_dpi *orig_dpi = NULL;
+	struct bnxt_qplib_dev_attr *dev_attr;
+	struct bnxt_re_ucontext *uctx = NULL;
+	struct bnxt_re_resize_cq_req ureq;
+	struct ib_ucontext *context = NULL;
+	struct bnxt_re_dev *rdev;
+	struct bnxt_re_cq *cq;
+	int rc, entries;
+
+	/* Don't allow more than one resize request at the same time.
+	 * TODO: need a mutex here when we support kernel consumers of resize.
+	 */
+	cq =  to_bnxt_re(ib_cq, struct bnxt_re_cq, ibcq);
+	rdev = cq->rdev;
+	dev_attr = rdev->dev_attr;
+	if (ib_cq->uobject) {
+		uctx = rdma_udata_to_drv_context(udata,
+						 struct bnxt_re_ucontext,
+						 ibucontext);
+		context = &uctx->ibucontext;
+	}
+
+	if (cq->resize_umem) {
+		dev_err(rdev_to_dev(rdev), "Resize CQ %#x failed - Busy\n",
+			cq->qplib_cq.id);
+		return -EBUSY;
+	}
+
+	/* Check the requested cq depth out of supported depth */
+	if (cqe < 1 || cqe > dev_attr->max_cq_wqes) {
+		dev_err(rdev_to_dev(rdev), "Resize CQ %#x failed - max exceeded\n",
+			cq->qplib_cq.id);
+		return -EINVAL;
+	}
+
+	entries = bnxt_re_init_depth(cqe + 1, uctx);
+	entries = min_t(u32, (u32)entries, dev_attr->max_cq_wqes + 1);
+
+	/* Check to see if the new requested size can be handled by already
+	 * existing CQ
+	 */
+	if (entries == cq->ibcq.cqe) {
+		dev_info(rdev_to_dev(rdev), "CQ is already at size %d\n", cqe);
+		return 0;
+	}
+
+	if (ib_cq->uobject && udata) {
+		if (udata->inlen < sizeof(ureq))
+			dev_warn(rdev_to_dev(rdev),
+				 "Update the library ulen %d klen %d\n",
+				 (unsigned int)udata->inlen,
+				 (unsigned int)sizeof(ureq));
+
+		rc = ib_copy_from_udata(&ureq, udata,
+					min(udata->inlen, sizeof(ureq)));
+		if (rc)
+			goto fail;
+
+		dev_dbg(rdev_to_dev(rdev), "%s: va %p\n", __func__,
+			(void *)ureq.cq_va);
+		cq->resize_umem = ib_umem_get_compat
+				       (rdev,
+					context, udata, ureq.cq_va,
+					entries * sizeof(struct cq_base),
+					IB_ACCESS_LOCAL_WRITE, 1);
+		if (IS_ERR(cq->resize_umem)) {
+			rc = PTR_ERR(cq->resize_umem);
+			cq->resize_umem = NULL;
+			dev_err(rdev_to_dev(rdev), "%s: ib_umem_get failed! rc = %d\n",
+				__func__, rc);
+			goto fail;
+		}
+		cq->resize_cqe = entries;
+		dev_dbg(rdev_to_dev(rdev), "%s: ib_umem_get() success\n",
+			__func__);
+		memcpy(&sginfo, &cq->qplib_cq.sginfo, sizeof(sginfo));
+		orig_dpi = cq->qplib_cq.dpi;
+
+		cq->qplib_cq.sginfo.sghead = get_ib_umem_sgl(cq->resize_umem,
+						&cq->qplib_cq.sginfo.nmap);
+		cq->qplib_cq.sginfo.npages =
+				ib_umem_num_pages_compat(cq->resize_umem);
+		cq->qplib_cq.sginfo.pgsize = PAGE_SIZE;
+		cq->qplib_cq.sginfo.pgshft = PAGE_SHIFT;
+		cq->qplib_cq.dpi = &uctx->dpi;
+	} else {
+		/* TODO: kernel consumer */
+	}
+
+	rc = bnxt_qplib_resize_cq(&rdev->qplib_res, &cq->qplib_cq, entries);
+	if (rc) {
+		dev_err(rdev_to_dev(rdev), "Resize HW CQ %#x failed!\n",
+			cq->qplib_cq.id);
+		goto fail;
+	}
+
+	cq->ibcq.cqe = cq->resize_cqe;
+	/* For kernel consumers complete resize here. For uverbs consumers,
+	 * we complete it in the context of ibv_poll_cq().
+	 */
+	if (!cq->resize_umem)
+		bnxt_qplib_resize_cq_complete(&rdev->qplib_res, &cq->qplib_cq);
+
+	atomic_inc(&rdev->stats.rsors.resize_count);
+	return 0;
+
+fail:
+	if (cq->resize_umem) {
+		ib_umem_release(cq->resize_umem);
+		cq->resize_umem = NULL;
+		cq->resize_cqe = 0;
+		memcpy(&cq->qplib_cq.sginfo, &sginfo, sizeof(sginfo));
+		cq->qplib_cq.dpi = orig_dpi;
+	}
+	return rc;
+}
+
+static enum ib_wc_status __req_to_ib_wc_status(u8 qstatus)
+{
+	switch(qstatus) {
+	case CQ_REQ_STATUS_OK:
+		return IB_WC_SUCCESS;
+	case CQ_REQ_STATUS_BAD_RESPONSE_ERR:
+		return IB_WC_BAD_RESP_ERR;
+	case CQ_REQ_STATUS_LOCAL_LENGTH_ERR:
+		return IB_WC_LOC_LEN_ERR;
+	case CQ_REQ_STATUS_LOCAL_QP_OPERATION_ERR:
+		return IB_WC_LOC_QP_OP_ERR;
+	case CQ_REQ_STATUS_LOCAL_PROTECTION_ERR:
+		return IB_WC_LOC_PROT_ERR;
+	case CQ_REQ_STATUS_MEMORY_MGT_OPERATION_ERR:
+		return IB_WC_GENERAL_ERR;
+	case CQ_REQ_STATUS_REMOTE_INVALID_REQUEST_ERR:
+		return IB_WC_REM_INV_REQ_ERR;
+	case CQ_REQ_STATUS_REMOTE_ACCESS_ERR:
+		return IB_WC_REM_ACCESS_ERR;
+	case CQ_REQ_STATUS_REMOTE_OPERATION_ERR:
+		return IB_WC_REM_OP_ERR;
+	case CQ_REQ_STATUS_RNR_NAK_RETRY_CNT_ERR:
+		return IB_WC_RNR_RETRY_EXC_ERR;
+	case CQ_REQ_STATUS_TRANSPORT_RETRY_CNT_ERR:
+		return IB_WC_RETRY_EXC_ERR;
+	case CQ_REQ_STATUS_WORK_REQUEST_FLUSHED_ERR:
+		return IB_WC_WR_FLUSH_ERR;
+	default:
+		return IB_WC_GENERAL_ERR;
+	}
+	return 0;
+}
+
+static enum ib_wc_status __rawqp1_to_ib_wc_status(u8 qstatus)
+{
+	switch(qstatus) {
+	case CQ_RES_RAWETH_QP1_STATUS_OK:
+		return IB_WC_SUCCESS;
+	case CQ_RES_RAWETH_QP1_STATUS_LOCAL_ACCESS_ERROR:
+		return IB_WC_LOC_ACCESS_ERR;
+	case CQ_RES_RAWETH_QP1_STATUS_HW_LOCAL_LENGTH_ERR:
+		return IB_WC_LOC_LEN_ERR;
+	case CQ_RES_RAWETH_QP1_STATUS_LOCAL_PROTECTION_ERR:
+		return IB_WC_LOC_PROT_ERR;
+	case CQ_RES_RAWETH_QP1_STATUS_LOCAL_QP_OPERATION_ERR:
+		return IB_WC_LOC_QP_OP_ERR;
+	case CQ_RES_RAWETH_QP1_STATUS_MEMORY_MGT_OPERATION_ERR:
+		return IB_WC_GENERAL_ERR;
+	case CQ_RES_RAWETH_QP1_STATUS_WORK_REQUEST_FLUSHED_ERR:
+		return IB_WC_WR_FLUSH_ERR;
+	case CQ_RES_RAWETH_QP1_STATUS_HW_FLUSH_ERR:
+		return IB_WC_WR_FLUSH_ERR;
+	default:
+		return IB_WC_GENERAL_ERR;
+	}
+}
+
+static enum ib_wc_status __rc_to_ib_wc_status(u8 qstatus)
+{
+	switch(qstatus) {
+	case CQ_RES_RC_STATUS_OK:
+		return IB_WC_SUCCESS;
+	case CQ_RES_RC_STATUS_LOCAL_ACCESS_ERROR:
+		return IB_WC_LOC_ACCESS_ERR;
+	case CQ_RES_RC_STATUS_LOCAL_LENGTH_ERR:
+		return IB_WC_LOC_LEN_ERR;
+	case CQ_RES_RC_STATUS_LOCAL_PROTECTION_ERR:
+		return IB_WC_LOC_PROT_ERR;
+	case CQ_RES_RC_STATUS_LOCAL_QP_OPERATION_ERR:
+		return IB_WC_LOC_QP_OP_ERR;
+	case CQ_RES_RC_STATUS_MEMORY_MGT_OPERATION_ERR:
+		return IB_WC_GENERAL_ERR;
+	case CQ_RES_RC_STATUS_REMOTE_INVALID_REQUEST_ERR:
+		return IB_WC_REM_INV_REQ_ERR;
+	case CQ_RES_RC_STATUS_WORK_REQUEST_FLUSHED_ERR:
+		return IB_WC_WR_FLUSH_ERR;
+	case CQ_RES_RC_STATUS_HW_FLUSH_ERR:
+		return IB_WC_WR_FLUSH_ERR;
+	default:
+		return IB_WC_GENERAL_ERR;
+	}
+}
+
+static void bnxt_re_process_req_wc(struct ib_wc *wc, struct bnxt_qplib_cqe *cqe)
+{
+	switch (cqe->type) {
+	case BNXT_QPLIB_SWQE_TYPE_SEND:
+		wc->opcode = IB_WC_SEND;
+		break;
+	case BNXT_QPLIB_SWQE_TYPE_SEND_WITH_IMM:
+		wc->opcode = IB_WC_SEND;
+		wc->wc_flags |= IB_WC_WITH_IMM;
+		break;
+	case BNXT_QPLIB_SWQE_TYPE_SEND_WITH_INV:
+		wc->opcode = IB_WC_SEND;
+		wc->wc_flags |= IB_WC_WITH_INVALIDATE;
+		break;
+	case BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE:
+		wc->opcode = IB_WC_RDMA_WRITE;
+		break;
+	case BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE_WITH_IMM:
+		wc->opcode = IB_WC_RDMA_WRITE;
+		wc->wc_flags |= IB_WC_WITH_IMM;
+		break;
+	case BNXT_QPLIB_SWQE_TYPE_RDMA_READ:
+		wc->opcode = IB_WC_RDMA_READ;
+		break;
+	case BNXT_QPLIB_SWQE_TYPE_ATOMIC_CMP_AND_SWP:
+		wc->opcode = IB_WC_COMP_SWAP;
+		break;
+	case BNXT_QPLIB_SWQE_TYPE_ATOMIC_FETCH_AND_ADD:
+		wc->opcode = IB_WC_FETCH_ADD;
+		break;
+	case BNXT_QPLIB_SWQE_TYPE_LOCAL_INV:
+		wc->opcode = IB_WC_LOCAL_INV;
+		break;
+	case BNXT_QPLIB_SWQE_TYPE_REG_MR:
+		wc->opcode = IB_WC_REG_MR;
+		break;
+	default:
+		wc->opcode = IB_WC_SEND;
+		break;
+	}
+
+	wc->status = __req_to_ib_wc_status(cqe->status);
+}
+
+static int bnxt_re_check_packet_type(u16 raweth_qp1_flags, u16 raweth_qp1_flags2)
+{
+	bool is_ipv6 = false, is_ipv4 = false;
+
+	/* raweth_qp1_flags Bit 9-6 indicates itype */
+
+	if ((raweth_qp1_flags & CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_ROCE)
+	    != CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_ROCE)
+		return -1;
+
+	if (raweth_qp1_flags2 &
+	    CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_IP_CS_CALC &&
+	    raweth_qp1_flags2 &
+	    CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_L4_CS_CALC) {
+		/* raweth_qp1_flags2 Bit 8 indicates ip_type. 0-v4 1 - v6 */
+		(raweth_qp1_flags2 &
+		 CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_IP_TYPE) ?
+			(is_ipv6 = true) : (is_ipv4 = true);
+		return ((is_ipv6) ?
+			 BNXT_RE_ROCEV2_IPV6_PACKET :
+			 BNXT_RE_ROCEV2_IPV4_PACKET);
+	} else {
+		return BNXT_RE_ROCE_V1_PACKET;
+	}
+}
+
+static bool bnxt_re_is_loopback_packet(struct bnxt_re_dev *rdev,
+					    void *rq_hdr_buf)
+{
+	u8 *tmp_buf = NULL;
+	struct ethhdr *eth_hdr;
+	u16 eth_type;
+	bool rc = false;
+
+	tmp_buf = (u8 *)rq_hdr_buf;
+	/*
+	 * If dest mac is not same as I/F mac, this could be a
+	 * loopback address or multicast address, check whether
+	 * it is a loopback packet
+	 */
+	if (!ether_addr_equal(tmp_buf, rdev->dev_addr)) {
+		tmp_buf += 4;
+		/* Check the  ether type */
+		eth_hdr = (struct ethhdr *)tmp_buf;
+		eth_type = ntohs(eth_hdr->h_proto);
+		switch (eth_type) {
+		case BNXT_QPLIB_ETHTYPE_ROCEV1:
+			rc = true;
+			break;
+		default:
+			break;
+		}
+	}
+
+	return rc;
+}
+
+static bool bnxt_re_is_vlan_in_packet(struct bnxt_re_dev *rdev,
+				      void *rq_hdr_buf,
+				      struct bnxt_qplib_cqe *cqe)
+{
+	struct vlan_hdr *vlan_hdr;
+	struct ethhdr *eth_hdr;
+	u8 *tmp_buf = NULL;
+	u16 eth_type;
+
+	tmp_buf = (u8 *)rq_hdr_buf;
+	/* Check the  ether type */
+	eth_hdr = (struct ethhdr *)tmp_buf;
+	eth_type = ntohs(eth_hdr->h_proto);
+	if (eth_type == ETH_P_8021Q) {
+		tmp_buf += sizeof(struct ethhdr);
+		vlan_hdr = (struct vlan_hdr *)tmp_buf;
+		cqe->raweth_qp1_metadata =
+			ntohs(vlan_hdr->h_vlan_TCI) |
+			(eth_type <<
+			 CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_TPID_SFT);
+		cqe->raweth_qp1_flags2 |=
+			CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_META_FORMAT_VLAN;
+		return true;
+	}
+
+	return false;
+}
+
+static int bnxt_re_process_raw_qp_packet_receive(struct bnxt_re_qp *gsi_qp,
+						 struct bnxt_qplib_cqe *cqe)
+{
+	struct bnxt_re_sqp_entries *sqp_entry = NULL;
+	struct bnxt_qplib_hdrbuf *hdr_buf;
+	dma_addr_t shrq_hdr_buf_map;
+	struct ib_sge s_sge[2] = {};
+	struct ib_sge r_sge[2] = {};
+	struct ib_recv_wr rwr = {};
+	struct bnxt_re_ah *gsi_sah;
+	struct bnxt_re_qp *gsi_sqp;
+	dma_addr_t rq_hdr_buf_map;
+	struct bnxt_re_dev *rdev;
+	struct ib_send_wr *swr;
+	u32 skip_bytes = 0;
+	void *rq_hdr_buf;
+	int pkt_type = 0;
+	u32 offset = 0;
+	u32 tbl_idx;
+	int rc;
+	struct ib_ud_wr udwr = {};
+
+	swr = &udwr.wr;
+	rdev = gsi_qp->rdev;
+	gsi_sqp = rdev->gsi_ctx.gsi_sqp;
+	tbl_idx = cqe->wr_id;
+
+	hdr_buf = gsi_qp->qplib_qp.rq_hdr_buf;
+	rq_hdr_buf = (u8 *) hdr_buf->va + tbl_idx * hdr_buf->step;
+	rq_hdr_buf_map = bnxt_qplib_get_qp_buf_from_index(&gsi_qp->qplib_qp,
+							  tbl_idx);
+	/* Shadow QP header buffer */
+	shrq_hdr_buf_map = bnxt_qplib_get_qp_buf_from_index(&gsi_sqp->qplib_qp,
+							    tbl_idx);
+	sqp_entry = &rdev->gsi_ctx.sqp_tbl[tbl_idx];
+
+	/* Find packet type from the cqe */
+	pkt_type = bnxt_re_check_packet_type(cqe->raweth_qp1_flags,
+					     cqe->raweth_qp1_flags2);
+	if (pkt_type < 0) {
+		dev_err(rdev_to_dev(rdev), "Not handling this packet\n");
+		return -EINVAL;
+	}
+
+	/* Adjust the offset for the user buffer and post in the rq */
+
+	if (pkt_type == BNXT_RE_ROCEV2_IPV4_PACKET)
+		offset = 20;
+
+	/*
+	 * QP1 loopback packet has 4 bytes of internal header before
+	 * ether header. Skip these four bytes.
+	 */
+	if (bnxt_re_is_loopback_packet(rdev, rq_hdr_buf))
+		skip_bytes = 4;
+
+	if (bnxt_re_is_vlan_in_packet(rdev, rq_hdr_buf, cqe))
+		skip_bytes += VLAN_HLEN;
+
+	/* Store this cqe */
+	memcpy(&sqp_entry->cqe, cqe, sizeof(struct bnxt_qplib_cqe));
+	sqp_entry->qp1_qp = gsi_qp;
+
+	/* First send SGE . Skip the ether header*/
+	s_sge[0].addr = rq_hdr_buf_map + BNXT_QPLIB_MAX_QP1_RQ_ETH_HDR_SIZE
+			+ skip_bytes;
+	s_sge[0].lkey = 0xFFFFFFFF;
+	s_sge[0].length = offset ? BNXT_QPLIB_MAX_GRH_HDR_SIZE_IPV4 :
+				BNXT_QPLIB_MAX_GRH_HDR_SIZE_IPV6;
+
+	/* Second Send SGE */
+	s_sge[1].addr = s_sge[0].addr + s_sge[0].length +
+			BNXT_QPLIB_MAX_QP1_RQ_BDETH_HDR_SIZE;
+	if (pkt_type != BNXT_RE_ROCE_V1_PACKET)
+		s_sge[1].addr += 8;
+	s_sge[1].lkey = 0xFFFFFFFF;
+	s_sge[1].length = 256;
+
+	/* First recv SGE */
+	r_sge[0].addr = shrq_hdr_buf_map;
+	r_sge[0].lkey = 0xFFFFFFFF;
+	r_sge[0].length = 40;
+
+	r_sge[1].addr = sqp_entry->sge.addr + offset;
+	r_sge[1].lkey = sqp_entry->sge.lkey;
+	r_sge[1].length = BNXT_QPLIB_MAX_GRH_HDR_SIZE_IPV6 + 256 - offset;
+
+	/* Create receive work request */
+	rwr.num_sge = 2;
+	rwr.sg_list = r_sge;
+	rwr.wr_id = tbl_idx;
+	rwr.next = NULL;
+
+	rc = bnxt_re_post_recv_shadow_qp(rdev, gsi_sqp, &rwr);
+	if (rc) {
+		dev_err(rdev_to_dev(rdev),
+			"Failed to post Rx buffers to shadow QP\n");
+		return -ENOMEM;
+	}
+
+	swr->num_sge = 2;
+	swr->sg_list = s_sge;
+	swr->wr_id = tbl_idx;
+	swr->opcode = IB_WR_SEND;
+	swr->next = NULL;
+
+	gsi_sah = rdev->gsi_ctx.gsi_sah;
+	udwr.ah = &gsi_sah->ibah;
+	udwr.remote_qpn = gsi_sqp->qplib_qp.id;
+	udwr.remote_qkey = gsi_sqp->qplib_qp.qkey;
+	/* post data received in the send queue */
+	rc = bnxt_re_post_send_shadow_qp(rdev, gsi_sqp, swr);
+
+	return rc;
+}
+
+static void bnxt_re_process_res_rawqp1_wc(struct ib_wc *wc,
+					  struct bnxt_qplib_cqe *cqe)
+{
+	wc->opcode = IB_WC_RECV;
+	wc->status = __rawqp1_to_ib_wc_status(cqe->status);
+	wc->wc_flags |= IB_WC_GRH;
+}
+
+static void bnxt_re_process_res_rc_wc(struct ib_wc *wc,
+				      struct bnxt_qplib_cqe *cqe)
+{
+	wc->opcode = IB_WC_RECV;
+	wc->status = __rc_to_ib_wc_status(cqe->status);
+
+	if (cqe->flags & CQ_RES_RC_FLAGS_IMM)
+		wc->wc_flags |= IB_WC_WITH_IMM;
+	if (cqe->flags & CQ_RES_RC_FLAGS_INV)
+		wc->wc_flags |= IB_WC_WITH_INVALIDATE;
+	if ((cqe->flags & (CQ_RES_RC_FLAGS_RDMA | CQ_RES_RC_FLAGS_IMM)) ==
+	    (CQ_RES_RC_FLAGS_RDMA | CQ_RES_RC_FLAGS_IMM))
+		wc->opcode = IB_WC_RECV_RDMA_WITH_IMM;
+}
+
+/* Returns TRUE if pkt has valid VLAN and if VLAN id is non-zero */
+static bool bnxt_re_is_nonzero_vlanid_pkt(struct bnxt_qplib_cqe *orig_cqe,
+					  u16 *vid, u8 *sl)
+{
+	u32 metadata;
+	u16 tpid;
+	bool ret = false;
+	metadata = orig_cqe->raweth_qp1_metadata;
+	if (orig_cqe->raweth_qp1_flags2 &
+	    CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_META_FORMAT_VLAN) {
+		tpid = ((metadata &
+			 CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_TPID_MASK) >>
+			 CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_TPID_SFT);
+		if (tpid == ETH_P_8021Q) {
+			*vid = metadata &
+			       CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_VID_MASK;
+			*sl = (metadata &
+			       CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_PRI_MASK) >>
+			       CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_PRI_SFT;
+			ret = !!(*vid);
+		}
+	}
+
+	return ret;
+}
+
+static void bnxt_re_process_res_shadow_qp_wc(struct bnxt_re_qp *gsi_sqp,
+					     struct ib_wc *wc,
+					     struct bnxt_qplib_cqe *cqe)
+{
+	u32 tbl_idx;
+	struct bnxt_re_dev *rdev = gsi_sqp->rdev;
+	struct bnxt_re_qp *gsi_qp = NULL;
+	struct bnxt_qplib_cqe *orig_cqe = NULL;
+	struct bnxt_re_sqp_entries *sqp_entry = NULL;
+	int nw_type;
+	u16 vlan_id;
+	u8 sl;
+
+	tbl_idx = cqe->wr_id;
+
+	sqp_entry = &rdev->gsi_ctx.sqp_tbl[tbl_idx];
+	gsi_qp = sqp_entry->qp1_qp;
+	orig_cqe = &sqp_entry->cqe;
+
+	wc->wr_id = sqp_entry->wrid;
+	wc->byte_len = orig_cqe->length;
+	wc->qp = &gsi_qp->ib_qp;
+
+	wc->ex.imm_data = orig_cqe->immdata;
+	wc->src_qp = orig_cqe->src_qp;
+	memcpy(wc->smac, orig_cqe->smac, ETH_ALEN);
+	if (bnxt_re_is_nonzero_vlanid_pkt(orig_cqe, &vlan_id, &sl)) {
+		if (bnxt_re_check_if_vlan_valid(rdev, vlan_id)) {
+			wc->sl = sl;
+			wc->vlan_id = vlan_id;
+			wc->wc_flags |= IB_WC_WITH_VLAN;
+		}
+	}
+	wc->port_num = 1;
+	wc->vendor_err = orig_cqe->status;
+
+	wc->opcode = IB_WC_RECV;
+	wc->status = __rawqp1_to_ib_wc_status(orig_cqe->status);
+	wc->wc_flags |= IB_WC_GRH;
+
+	nw_type = bnxt_re_check_packet_type(orig_cqe->raweth_qp1_flags,
+					    orig_cqe->raweth_qp1_flags2);
+	if(nw_type >= 0)
+		dev_dbg(rdev_to_dev(rdev), "%s nw_type = %d\n", __func__, nw_type);
+}
+
+static void bnxt_re_process_res_ud_wc(struct bnxt_re_dev *rdev,
+				      struct bnxt_re_qp *qp, struct ib_wc *wc,
+				      struct bnxt_qplib_cqe *cqe)
+{
+	u16 vlan_id = 0;
+
+	wc->opcode = IB_WC_RECV;
+	wc->status = __rc_to_ib_wc_status(cqe->status);
+	if (cqe->flags & CQ_RES_UD_FLAGS_IMM)
+		wc->wc_flags |= IB_WC_WITH_IMM;
+	if (cqe->flags & CQ_RES_RC_FLAGS_INV)
+		wc->wc_flags |= IB_WC_WITH_INVALIDATE;
+	/* report only on GSI QP for Thor */
+	if (rdev->gsi_ctx.gsi_qp->qplib_qp.id == qp->qplib_qp.id &&
+	    rdev->gsi_ctx.gsi_qp_mode == BNXT_RE_GSI_MODE_UD) {
+		wc->wc_flags |= IB_WC_GRH;
+		memcpy(wc->smac, cqe->smac, ETH_ALEN);
+		wc->wc_flags |= IB_WC_WITH_SMAC;
+		if (_is_cqe_v2_supported(rdev->dev_attr->dev_cap_flags)) {
+			if (cqe->flags & CQ_RES_UD_V2_FLAGS_META_FORMAT_MASK) {
+				if (cqe->cfa_meta &
+				    BNXT_QPLIB_CQE_CFA_META1_VALID)
+					vlan_id = (cqe->cfa_meta & 0xFFF);
+			}
+		} else if (cqe->flags & CQ_RES_UD_FLAGS_META_FORMAT_VLAN) {
+			vlan_id = (cqe->cfa_meta & 0xFFF);
+		}
+		/* Mark only if vlan_id is non zero */
+		if (vlan_id && bnxt_re_check_if_vlan_valid(rdev, vlan_id)) {
+			wc->vlan_id = vlan_id;
+			wc->wc_flags |= IB_WC_WITH_VLAN;
+		}
+	}
+}
+
+static int bnxt_re_legacy_send_phantom_wqe(struct bnxt_re_qp *qp)
+{
+	struct bnxt_qplib_qp *lib_qp = &qp->qplib_qp;
+	unsigned long flags;
+	int rc = 0;
+
+	spin_lock_irqsave(&qp->sq_lock, flags);
+
+	rc = bnxt_re_legacy_bind_fence_mw(lib_qp);
+	if (!rc) {
+		lib_qp->sq.phantom_wqe_cnt++;
+		dev_dbg(&lib_qp->sq.hwq.pdev->dev,
+			"qp %#x sq->prod %#x sw_prod %#x phantom_wqe_cnt %d\n",
+			lib_qp->id, lib_qp->sq.hwq.prod,
+			HWQ_CMP(lib_qp->sq.hwq.prod, &lib_qp->sq.hwq),
+			lib_qp->sq.phantom_wqe_cnt);
+	}
+
+	spin_unlock_irqrestore(&qp->sq_lock, flags);
+	return rc;
+}
+
+int bnxt_re_poll_cq(struct ib_cq *ib_cq, int num_entries, struct ib_wc *wc)
+{
+	struct bnxt_re_cq *cq = to_bnxt_re(ib_cq, struct bnxt_re_cq, ibcq);
+	struct bnxt_re_dev *rdev = cq->rdev;
+	struct bnxt_re_qp *qp;
+	struct bnxt_qplib_cqe *cqe;
+	int i, ncqe, budget, init_budget;
+	struct bnxt_qplib_q *sq;
+	struct bnxt_qplib_qp *lib_qp;
+	u32 tbl_idx;
+	struct bnxt_re_sqp_entries *sqp_entry = NULL;
+	unsigned long flags;
+	u8 gsi_mode;
+
+	/*
+	 * DB recovery CQ; only process the door bell pacing alert from
+	 * the user lib
+	 */
+	if (cq->is_dbr_soft_cq) {
+		bnxt_re_pacing_alert(rdev);
+		return 0;
+	}
+
+	/* User CQ; the only processing we do is to
+	 * complete any pending CQ resize operation.
+	 */
+	if (cq->umem) {
+		if (cq->resize_umem)
+			bnxt_re_resize_cq_complete(cq);
+		return 0;
+	}
+
+	spin_lock_irqsave(&cq->cq_lock, flags);
+
+	budget = min_t(u32, num_entries, cq->max_cql);
+	init_budget = budget;
+	if (!cq->cql) {
+		dev_err(rdev_to_dev(rdev), "POLL CQ no CQL to use\n");
+		goto exit;
+	}
+	cqe = &cq->cql[0];
+	gsi_mode = rdev->gsi_ctx.gsi_qp_mode;
+	while (budget) {
+		lib_qp = NULL;
+		ncqe = bnxt_qplib_poll_cq(&cq->qplib_cq, cqe, budget, &lib_qp);
+		if (lib_qp) {
+			sq = &lib_qp->sq;
+			if (sq->legacy_send_phantom == true) {
+				qp = container_of(lib_qp, struct bnxt_re_qp, qplib_qp);
+				if (bnxt_re_legacy_send_phantom_wqe(qp) == -ENOMEM)
+					dev_err(rdev_to_dev(rdev),
+						"Phantom failed! Scheduled to send again\n");
+				else
+					sq->legacy_send_phantom = false;
+			}
+		}
+		if (ncqe < budget)
+			ncqe += bnxt_qplib_process_flush_list(&cq->qplib_cq,
+							      cqe + ncqe,
+							      budget - ncqe);
+
+		if (!ncqe)
+			break;
+
+		for (i = 0; i < ncqe; i++, cqe++) {
+			/* Transcribe each qplib_wqe back to ib_wc */
+			memset(wc, 0, sizeof(*wc));
+
+			wc->wr_id = cqe->wr_id;
+			wc->byte_len = cqe->length;
+			qp = to_bnxt_re((struct bnxt_qplib_qp *)cqe->qp_handle,
+					struct bnxt_re_qp, qplib_qp);
+			if (!qp) {
+				dev_err(rdev_to_dev(rdev),
+					"POLL CQ bad QP handle\n");
+				continue;
+			}
+			wc->qp = &qp->ib_qp;
+			wc->ex.imm_data = cqe->immdata;
+			wc->src_qp = cqe->src_qp;
+			memcpy(wc->smac, cqe->smac, ETH_ALEN);
+			wc->port_num = 1;
+			wc->vendor_err = cqe->status;
+
+			switch(cqe->opcode) {
+			case CQ_BASE_CQE_TYPE_REQ:
+				if (gsi_mode == BNXT_RE_GSI_MODE_ALL &&
+				    qp->qplib_qp.id ==
+				    rdev->gsi_ctx.gsi_sqp->qplib_qp.id) {
+					/* Handle this completion with
+					 * the stored completion */
+					 dev_dbg(rdev_to_dev(rdev),
+						 "Skipping this UD Send CQ\n");
+					memset(wc, 0, sizeof(*wc));
+					continue;
+				}
+				bnxt_re_process_req_wc(wc, cqe);
+				break;
+			case CQ_BASE_CQE_TYPE_RES_RAWETH_QP1:
+				if (gsi_mode == BNXT_RE_GSI_MODE_ALL) {
+					if (!cqe->status) {
+						int rc = 0;
+						rc = bnxt_re_process_raw_qp_packet_receive(qp, cqe);
+						if (!rc) {
+							memset(wc, 0,
+							       sizeof(*wc));
+							continue;
+						}
+						cqe->status = -1;
+					}
+					/* Errors need not be looped back.
+					 * But change the wr_id to the one
+					 * stored in the table
+					 */
+					tbl_idx = cqe->wr_id;
+					sqp_entry = &rdev->gsi_ctx.sqp_tbl[tbl_idx];
+					wc->wr_id = sqp_entry->wrid;
+				}
+
+				bnxt_re_process_res_rawqp1_wc(wc, cqe);
+				break;
+			case CQ_BASE_CQE_TYPE_RES_RC:
+				bnxt_re_process_res_rc_wc(wc, cqe);
+				break;
+			case CQ_BASE_CQE_TYPE_RES_UD:
+				if (gsi_mode == BNXT_RE_GSI_MODE_ALL &&
+				    qp->qplib_qp.id ==
+				    rdev->gsi_ctx.gsi_sqp->qplib_qp.id) {
+					/* Handle this completion with
+					 * the stored completion
+					 */
+					dev_dbg(rdev_to_dev(rdev),
+						"Handling the UD receive CQ\n");
+					if (cqe->status) {
+						/* TODO handle this completion  as a failure in
+						 * loopback porocedure
+						 */
+						continue;
+					} else {
+						bnxt_re_process_res_shadow_qp_wc(qp, wc, cqe);
+						break;
+					}
+				}
+				bnxt_re_process_res_ud_wc(rdev, qp, wc, cqe);
+				break;
+			default:
+				dev_err(rdev_to_dev(cq->rdev),
+					"POLL CQ type 0x%x not handled, skip!\n",
+					cqe->opcode);
+				continue;
+			}
+			wc++;
+			budget--;
+		}
+	}
+exit:
+	spin_unlock_irqrestore(&cq->cq_lock, flags);
+	return init_budget - budget;
+}
+
+int bnxt_re_req_notify_cq(struct ib_cq *ib_cq,
+			  enum ib_cq_notify_flags ib_cqn_flags)
+{
+	struct bnxt_re_cq *cq = to_bnxt_re(ib_cq, struct bnxt_re_cq, ibcq);
+	int type = 0, rc = 0;
+	unsigned long flags;
+
+	spin_lock_irqsave(&cq->cq_lock, flags);
+	/* Trigger on the very next completion */
+	if (ib_cqn_flags & IB_CQ_NEXT_COMP)
+		type = DBC_DBC_TYPE_CQ_ARMALL;
+	/* Trigger on the next solicited completion */
+	else if (ib_cqn_flags & IB_CQ_SOLICITED)
+		type = DBC_DBC_TYPE_CQ_ARMSE;
+
+	bnxt_qplib_req_notify_cq(&cq->qplib_cq, type);
+
+	/* Poll to see if there are missed events */
+	if ((ib_cqn_flags & IB_CQ_REPORT_MISSED_EVENTS) &&
+	    !(bnxt_qplib_is_cq_empty(&cq->qplib_cq)))
+		rc = 1;
+
+	spin_unlock_irqrestore(&cq->cq_lock, flags);
+
+	return rc;
+}
+
+/* Memory Regions */
+struct ib_mr *bnxt_re_get_dma_mr(struct ib_pd *ib_pd, int mr_access_flags)
+{
+	struct bnxt_qplib_mrinfo mrinfo;
+	struct bnxt_re_dev *rdev;
+	struct bnxt_re_mr *mr;
+	struct bnxt_re_pd *pd;
+	u32 max_mr_count;
+	u64 pbl = 0;
+	int rc;
+
+	memset(&mrinfo, 0, sizeof(mrinfo));
+	pd = to_bnxt_re(ib_pd, struct bnxt_re_pd, ibpd);
+	rdev = pd->rdev;
+
+	mr = kzalloc(sizeof(*mr), GFP_KERNEL);
+	if (!mr) {
+		dev_err(rdev_to_dev(rdev),
+			"Allocate memory for DMA MR failed!\n");
+		return ERR_PTR(-ENOMEM);
+	}
+	mr->rdev = rdev;
+	mr->qplib_mr.pd = &pd->qplib_pd;
+	mr->qplib_mr.flags = __from_ib_access_flags(mr_access_flags);
+	mr->qplib_mr.type = CMDQ_ALLOCATE_MRW_MRW_FLAGS_PMR;
+
+	/* Allocate and register 0 as the address */
+	rc = bnxt_qplib_alloc_mrw(&rdev->qplib_res, &mr->qplib_mr);
+	if (rc) {
+		dev_err(rdev_to_dev(rdev), "Allocate DMA MR failed!\n");
+		goto fail;
+	}
+	mr->qplib_mr.total_size = -1; /* Infinite length */
+	mrinfo.ptes = &pbl;
+	mrinfo.sg.npages = 0;
+	mrinfo.sg.pgsize = PAGE_SIZE;
+	mrinfo.sg.pgshft = PAGE_SHIFT;
+	mrinfo.sg.pgsize = PAGE_SIZE;
+	mrinfo.mrw = &mr->qplib_mr;
+	mrinfo.is_dma = true;
+	rc = bnxt_qplib_reg_mr(&rdev->qplib_res, &mrinfo, false);
+	if (rc) {
+		dev_err(rdev_to_dev(rdev), "Register DMA MR failed!\n");
+		goto fail_mr;
+	}
+	mr->ib_mr.lkey = mr->qplib_mr.lkey;
+	if (mr_access_flags & (IB_ACCESS_REMOTE_WRITE | IB_ACCESS_REMOTE_READ |
+			       IB_ACCESS_REMOTE_ATOMIC))
+		mr->ib_mr.rkey = mr->ib_mr.lkey;
+	atomic_inc(&rdev->stats.rsors.mr_count);
+	max_mr_count =  atomic_read(&rdev->stats.rsors.mr_count);
+	if (max_mr_count > atomic_read(&rdev->stats.rsors.max_mr_count))
+		atomic_set(&rdev->stats.rsors.max_mr_count, max_mr_count);
+
+	return &mr->ib_mr;
+
+fail_mr:
+	bnxt_qplib_free_mrw(&rdev->qplib_res, &mr->qplib_mr);
+fail:
+	kfree(mr);
+	return ERR_PTR(rc);
+}
+
+int bnxt_re_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata)
+{
+	struct bnxt_re_mr *mr = to_bnxt_re(ib_mr, struct bnxt_re_mr, ib_mr);
+	struct bnxt_re_dev *rdev = mr->rdev;
+	int rc = 0;
+
+	rc = bnxt_qplib_free_mrw(&rdev->qplib_res, &mr->qplib_mr);
+	if (rc)
+		dev_err(rdev_to_dev(rdev), "Dereg MR failed (%d): rc - %#x\n",
+			mr->qplib_mr.lkey, rc);
+
+	if (mr->pages) {
+		bnxt_qplib_free_fast_reg_page_list(&rdev->qplib_res,
+						   &mr->qplib_frpl);
+		kfree(mr->pages);
+		mr->npages = 0;
+		mr->pages = NULL;
+	}
+	if (!IS_ERR(mr->ib_umem) && mr->ib_umem) {
+		mr->is_invalcb_active = false;
+		bnxt_re_peer_mem_release(mr->ib_umem);
+	}
+	kfree(mr);
+	atomic_dec(&rdev->stats.rsors.mr_count);
+	return 0;
+}
+
+static int bnxt_re_set_page(struct ib_mr *ib_mr, u64 addr)
+{
+	struct bnxt_re_mr *mr = to_bnxt_re(ib_mr, struct bnxt_re_mr, ib_mr);
+
+	if (unlikely(mr->npages == mr->qplib_frpl.max_pg_ptrs))
+		return -ENOMEM;
+
+	mr->pages[mr->npages++] = addr;
+	dev_dbg(NULL, "%s: ibdev %p Set MR pages[%d] = 0x%lx\n",
+		ROCE_DRV_MODULE_NAME, ib_mr->device, mr->npages - 1,
+		mr->pages[mr->npages - 1]);
+	return 0;
+}
+
+int bnxt_re_map_mr_sg(struct ib_mr *ib_mr, struct scatterlist *sg,
+		      int sg_nents, unsigned int *sg_offset)
+{
+	struct bnxt_re_mr *mr = to_bnxt_re(ib_mr, struct bnxt_re_mr, ib_mr);
+
+	mr->npages = 0;
+	return ib_sg_to_pages(ib_mr, sg, sg_nents,
+			      sg_offset, bnxt_re_set_page);
+}
+
+struct ib_mr *bnxt_re_alloc_mr(struct ib_pd *ib_pd, enum ib_mr_type type,
+			       u32 max_num_sg, struct ib_udata *udata)
+{
+	struct bnxt_re_pd *pd = to_bnxt_re(ib_pd, struct bnxt_re_pd, ibpd);
+	struct bnxt_re_dev *rdev = pd->rdev;
+	struct bnxt_re_mr *mr;
+	u32 max_mr_count;
+	int rc;
+
+	dev_dbg(rdev_to_dev(rdev), "Alloc MR\n");
+	if (type != IB_MR_TYPE_MEM_REG) {
+		dev_dbg(rdev_to_dev(rdev), "MR type 0x%x not supported\n", type);
+		return ERR_PTR(-EINVAL);
+	}
+	if (max_num_sg > MAX_PBL_LVL_1_PGS) {
+		dev_dbg(rdev_to_dev(rdev), "Max SG exceeded\n");
+		return ERR_PTR(-EINVAL);
+	}
+	mr = kzalloc(sizeof(*mr), GFP_KERNEL);
+	if (!mr) {
+		dev_err(rdev_to_dev(rdev), "Allocate MR mem failed!\n");
+		return ERR_PTR(-ENOMEM);
+	}
+	mr->rdev = rdev;
+	mr->qplib_mr.pd = &pd->qplib_pd;
+	mr->qplib_mr.flags = BNXT_QPLIB_FR_PMR;
+	mr->qplib_mr.type = CMDQ_ALLOCATE_MRW_MRW_FLAGS_PMR;
+
+	rc = bnxt_qplib_alloc_mrw(&rdev->qplib_res, &mr->qplib_mr);
+	if (rc) {
+		dev_err(rdev_to_dev(rdev), "Allocate MR failed!\n");
+		goto fail;
+	}
+	mr->ib_mr.lkey = mr->qplib_mr.lkey;
+	mr->ib_mr.rkey = mr->ib_mr.lkey;
+	mr->pages = kzalloc(sizeof(u64) * max_num_sg, GFP_KERNEL);
+	if (!mr->pages) {
+		dev_err(rdev_to_dev(rdev),
+			"Allocate MR page list mem failed!\n");
+		rc = -ENOMEM;
+		goto fail_mr;
+	}
+	rc = bnxt_qplib_alloc_fast_reg_page_list(&rdev->qplib_res,
+						 &mr->qplib_frpl, max_num_sg);
+	if (rc) {
+		dev_err(rdev_to_dev(rdev),
+			"Allocate HW Fast reg page list failed!\n");
+		goto free_page;
+	}
+	dev_dbg(rdev_to_dev(rdev), "Alloc MR pages = 0x%p\n", mr->pages);
+
+	atomic_inc(&rdev->stats.rsors.mr_count);
+	max_mr_count =  atomic_read(&rdev->stats.rsors.mr_count);
+	if (max_mr_count > atomic_read(&rdev->stats.rsors.max_mr_count))
+		atomic_set(&rdev->stats.rsors.max_mr_count, max_mr_count);
+	return &mr->ib_mr;
+
+free_page:
+	kfree(mr->pages);
+fail_mr:
+	bnxt_qplib_free_mrw(&rdev->qplib_res, &mr->qplib_mr);
+fail:
+	kfree(mr);
+	return ERR_PTR(rc);
+}
+
+/* Memory Windows */
+struct ib_mw *bnxt_re_alloc_mw(struct ib_pd *ib_pd, enum ib_mw_type type,
+			       struct ib_udata *udata)
+{
+	struct bnxt_re_pd *pd = to_bnxt_re(ib_pd, struct bnxt_re_pd, ibpd);
+	struct bnxt_re_dev *rdev = pd->rdev;
+	struct bnxt_re_mw *mw;
+	u32 max_mw_count;
+	int rc;
+
+	mw = kzalloc(sizeof(*mw), GFP_KERNEL);
+	if (!mw) {
+		dev_err(rdev_to_dev(rdev), "Allocate MW failed!\n");
+		rc = -ENOMEM;
+		goto exit;
+	}
+	mw->rdev = rdev;
+	mw->qplib_mw.pd = &pd->qplib_pd;
+
+	mw->qplib_mw.type = (type == IB_MW_TYPE_1 ?
+			       CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE1 :
+			       CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE2B);
+	rc = bnxt_qplib_alloc_mrw(&rdev->qplib_res, &mw->qplib_mw);
+	if (rc) {
+		dev_err(rdev_to_dev(rdev), "Allocate MW failed!\n");
+		goto fail;
+	}
+	mw->ib_mw.rkey = mw->qplib_mw.rkey;
+	atomic_inc(&rdev->stats.rsors.mw_count);
+	max_mw_count = atomic_read(&rdev->stats.rsors.mw_count);
+	if (max_mw_count > atomic_read(&rdev->stats.rsors.max_mw_count))
+		atomic_set(&rdev->stats.rsors.max_mw_count, max_mw_count);
+
+	return &mw->ib_mw;
+fail:
+	kfree(mw);
+exit:
+	return ERR_PTR(rc);
+}
+
+int bnxt_re_dealloc_mw(struct ib_mw *ib_mw)
+{
+	struct bnxt_re_mw *mw = to_bnxt_re(ib_mw, struct bnxt_re_mw, ib_mw);
+	struct bnxt_re_dev *rdev = mw->rdev;
+	int rc;
+
+	rc = bnxt_qplib_free_mrw(&rdev->qplib_res, &mw->qplib_mw);
+	if (rc) {
+		dev_err(rdev_to_dev(rdev), "Free MW failed: %#x\n", rc);
+		return rc;
+	}
+
+	kfree(mw);
+	atomic_dec(&rdev->stats.rsors.mw_count);
+	return rc;
+}
+
+static int bnxt_re_page_size_ok(int page_shift)
+{
+	switch (page_shift) {
+	case CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_4K:
+	case CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_8K:
+	case CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_64K:
+	case CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_2M:
+	case CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_256K:
+	case CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_1M:
+	case CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_4M:
+	case CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_256MB:
+	case CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_1G:
+		return 1;
+	default:
+		return 0;
+	}
+}
+
+static int bnxt_re_get_page_shift(struct ib_umem *umem,
+				  u64 va, u64 st, u64 cmask)
+{
+	int pgshft;
+
+	pgshft = ilog2(umem->page_size);
+
+	return pgshft;
+}
+
+static int bnxt_re_get_num_pages(struct ib_umem *umem, u64 start, u64 length, int page_shift)
+{
+	int npages = 0;
+
+	if (page_shift == PAGE_SHIFT) {
+		npages = ib_umem_num_pages_compat(umem);
+	} else {
+		npages = ALIGN(length, BIT(page_shift)) / BIT(page_shift);
+		if (start %  BIT(page_shift))
+			npages++;
+	}
+	return npages;
+}
+
+/* uverbs */
+struct ib_mr *bnxt_re_reg_user_mr(struct ib_pd *ib_pd, u64 start, u64 length,
+				  u64 virt_addr, int mr_access_flags,
+				  struct ib_udata *udata)
+{
+	struct bnxt_re_pd *pd = to_bnxt_re(ib_pd, struct bnxt_re_pd, ibpd);
+	struct bnxt_re_dev *rdev = pd->rdev;
+	struct bnxt_qplib_mrinfo mrinfo;
+	int umem_pgs, page_shift, rc;
+	struct bnxt_re_mr *mr;
+	struct ib_umem *umem;
+	u32 max_mr_count;
+	int npages;
+
+	dev_dbg(rdev_to_dev(rdev), "Reg user MR\n");
+
+	if (bnxt_re_get_total_mr_mw_count(rdev) >= rdev->dev_attr->max_mr)
+		return ERR_PTR(-ENOMEM);
+
+	if (rdev->mod_exit) {
+		dev_dbg(rdev_to_dev(rdev), "%s(): in mod_exit, just return!\n", __func__);
+		return ERR_PTR(-EIO);
+	}
+	memset(&mrinfo, 0, sizeof(mrinfo));
+	if (length > BNXT_RE_MAX_MR_SIZE) {
+		dev_err(rdev_to_dev(rdev), "Requested MR Size: %lu "
+			"> Max supported: %ld\n", length, BNXT_RE_MAX_MR_SIZE);
+		return ERR_PTR(-ENOMEM);
+	}
+	mr = kzalloc(sizeof(*mr), GFP_KERNEL);
+	if (!mr) {
+		dev_err(rdev_to_dev(rdev), "Allocate MR failed!\n");
+		return ERR_PTR (-ENOMEM);
+	}
+	mr->rdev = rdev;
+	mr->qplib_mr.pd = &pd->qplib_pd;
+	mr->qplib_mr.flags = __from_ib_access_flags(mr_access_flags);
+	mr->qplib_mr.type = CMDQ_ALLOCATE_MRW_MRW_FLAGS_MR;
+
+	if (!_is_alloc_mr_unified(rdev->qplib_res.dattr)) {
+		rc = bnxt_qplib_alloc_mrw(&rdev->qplib_res, &mr->qplib_mr);
+		if (rc) {
+			dev_err(rdev_to_dev(rdev), "Alloc MR failed!\n");
+			goto fail;
+		}
+		/* The fixed portion of the rkey is the same as the lkey */
+		mr->ib_mr.rkey = mr->qplib_mr.rkey;
+	}
+
+	umem = ib_umem_get_flags_compat(rdev, ib_pd->uobject->context,
+					udata, start, length,
+					mr_access_flags, 0);
+	if (IS_ERR(umem)) {
+		rc = PTR_ERR(umem);
+		dev_err(rdev_to_dev(rdev), "%s: ib_umem_get failed! rc = %d\n",
+			__func__, rc);
+		goto free_mr;
+	}
+	mr->ib_umem = umem;
+
+	mr->qplib_mr.va = virt_addr;
+	umem_pgs = ib_umem_num_pages_compat(umem);
+	if (!umem_pgs) {
+		dev_err(rdev_to_dev(rdev), "umem is invalid!\n");
+		rc = -EINVAL;
+		goto free_umem;
+	}
+	mr->qplib_mr.total_size = length;
+	page_shift = bnxt_re_get_page_shift(umem, virt_addr, start,
+					    rdev->dev_attr->page_size_cap);
+	if (!bnxt_re_page_size_ok(page_shift)) {
+		dev_err(rdev_to_dev(rdev), "umem page size unsupported!\n");
+		rc = -EFAULT;
+		goto free_umem;
+	}
+	npages = bnxt_re_get_num_pages(umem, start, length, page_shift);
+
+	/* Map umem buf ptrs to the PBL */
+	mrinfo.sg.npages = npages;
+	mrinfo.sg.sghead = get_ib_umem_sgl(umem, &mrinfo.sg.nmap);
+	mrinfo.sg.pgshft = page_shift;
+	mrinfo.sg.pgsize = BIT(page_shift);
+
+	mrinfo.mrw = &mr->qplib_mr;
+
+	rc = bnxt_qplib_reg_mr(&rdev->qplib_res, &mrinfo, false);
+	if (rc) {
+		dev_err(rdev_to_dev(rdev), "Reg user MR failed!\n");
+		goto free_umem;
+	}
+
+	mr->ib_mr.lkey = mr->ib_mr.rkey = mr->qplib_mr.lkey;
+	atomic_inc(&rdev->stats.rsors.mr_count);
+	max_mr_count =  atomic_read(&rdev->stats.rsors.mr_count);
+	if (max_mr_count > atomic_read(&rdev->stats.rsors.max_mr_count))
+		atomic_set(&rdev->stats.rsors.max_mr_count, max_mr_count);
+
+	return &mr->ib_mr;
+
+free_umem:
+	bnxt_re_peer_mem_release(mr->ib_umem);
+free_mr:
+	if (!_is_alloc_mr_unified(rdev->qplib_res.dattr))
+		bnxt_qplib_free_mrw(&rdev->qplib_res, &mr->qplib_mr);
+fail:
+	kfree(mr);
+	return ERR_PTR(rc);
+}
+
+int
+bnxt_re_rereg_user_mr(struct ib_mr *ib_mr, int flags, u64 start, u64 length,
+		      u64 virt_addr, int mr_access_flags,
+		      struct ib_pd *ib_pd, struct ib_udata *udata)
+{
+	struct bnxt_re_mr *mr = to_bnxt_re(ib_mr, struct bnxt_re_mr, ib_mr);
+	struct bnxt_re_pd *pd = to_bnxt_re(ib_pd, struct bnxt_re_pd, ibpd);
+	int umem_pgs = 0, page_shift = PAGE_SHIFT, rc;
+	struct bnxt_re_dev *rdev = mr->rdev;
+	struct bnxt_qplib_mrinfo mrinfo;
+	struct ib_umem *umem;
+	u32 npages;
+
+	/* TODO: Must decipher what to modify based on the flags */
+	memset(&mrinfo, 0, sizeof(mrinfo));
+	if (flags & IB_MR_REREG_TRANS) {
+		umem = ib_umem_get_flags_compat(rdev, ib_pd->uobject->context,
+						udata, start, length,
+						mr_access_flags, 0);
+		if (IS_ERR(umem)) {
+			rc = PTR_ERR(umem);
+			dev_err(rdev_to_dev(rdev),
+				"%s: ib_umem_get failed! ret =  %d\n",
+				__func__, rc);
+			goto fail;
+		}
+		mr->ib_umem = umem;
+
+		mr->qplib_mr.va = virt_addr;
+		umem_pgs = ib_umem_num_pages_compat(umem);
+		if (!umem_pgs) {
+			dev_err(rdev_to_dev(rdev), "umem is invalid!\n");
+			rc = -EINVAL;
+			goto fail_free_umem;
+		}
+		mr->qplib_mr.total_size = length;
+		page_shift = bnxt_re_get_page_shift(umem, virt_addr, start,
+					    rdev->dev_attr->page_size_cap);
+		if (!bnxt_re_page_size_ok(page_shift)) {
+			dev_err(rdev_to_dev(rdev),
+				"umem page size unsupported!\n");
+			rc = -EFAULT;
+			goto fail_free_umem;
+		}
+		npages = bnxt_re_get_num_pages(umem, start, length, page_shift);
+		/* Map umem buf ptrs to the PBL */
+		mrinfo.sg.npages = npages;
+		mrinfo.sg.sghead = get_ib_umem_sgl(umem, &mrinfo.sg.nmap);
+		mrinfo.sg.pgshft = page_shift;
+		mrinfo.sg.pgsize = BIT(page_shift);
+	}
+
+	mrinfo.mrw = &mr->qplib_mr;
+	if (flags & IB_MR_REREG_PD)
+		mr->qplib_mr.pd = &pd->qplib_pd;
+
+	if (flags & IB_MR_REREG_ACCESS)
+		mr->qplib_mr.flags = __from_ib_access_flags(mr_access_flags);
+
+	rc = bnxt_qplib_reg_mr(&rdev->qplib_res, &mrinfo, false);
+	if (rc) {
+		dev_err(rdev_to_dev(rdev), "Rereg user MR failed!\n");
+		goto fail_free_umem;
+	}
+	mr->ib_mr.rkey = mr->qplib_mr.rkey;
+
+	return 0;
+
+fail_free_umem:
+	bnxt_re_peer_mem_release(mr->ib_umem);
+fail:
+	return rc;
+}
+
+static int bnxt_re_check_abi_version(struct bnxt_re_dev *rdev)
+{
+	struct ib_device *ibdev = &rdev->ibdev;
+	u32 uverbs_abi_ver;
+
+	uverbs_abi_ver = GET_UVERBS_ABI_VERSION(ibdev);
+	dev_dbg(rdev_to_dev(rdev), "ABI version requested %d\n",
+		uverbs_abi_ver);
+	if (uverbs_abi_ver != BNXT_RE_ABI_VERSION) {
+		dev_dbg(rdev_to_dev(rdev), " is different from the device %d \n",
+			BNXT_RE_ABI_VERSION);
+		return -EPERM;
+	}
+	return 0;
+}
+
+int bnxt_re_alloc_ucontext(struct ib_ucontext *uctx_in,
+			   struct ib_udata *udata)
+{
+	struct ib_ucontext *ctx = uctx_in;
+	struct ib_device *ibdev = ctx->device;
+	struct bnxt_re_ucontext *uctx =
+		container_of(ctx, struct bnxt_re_ucontext, ibucontext);
+
+	struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev);
+	struct bnxt_qplib_dev_attr *dev_attr = rdev->dev_attr;
+	struct bnxt_re_uctx_resp resp = {};
+	struct bnxt_re_uctx_req ureq = {};
+	struct bnxt_qplib_chip_ctx *cctx;
+	u32 chip_met_rev_num;
+	bool genp5 = false;
+	int rc;
+
+	cctx = rdev->chip_ctx;
+	rc = bnxt_re_check_abi_version(rdev);
+	if (rc)
+		goto fail;
+
+	uctx->rdev = rdev;
+	uctx->shpg = (void *)__get_free_page(GFP_KERNEL);
+	if (!uctx->shpg) {
+		dev_err(rdev_to_dev(rdev), "shared memory allocation failed!\n");
+		rc = -ENOMEM;
+		goto fail;
+	}
+	spin_lock_init(&uctx->sh_lock);
+	if (BNXT_RE_ABI_VERSION >= 4) {
+		chip_met_rev_num = cctx->chip_num;
+		chip_met_rev_num |= ((u32)cctx->chip_rev & 0xFF) <<
+				     BNXT_RE_CHIP_ID0_CHIP_REV_SFT;
+		chip_met_rev_num |= ((u32)cctx->chip_metal & 0xFF) <<
+				     BNXT_RE_CHIP_ID0_CHIP_MET_SFT;
+		resp.chip_id0 = chip_met_rev_num;
+		resp.chip_id1 = 0; /* future extension of chip info */
+	}
+
+	if (BNXT_RE_ABI_VERSION != 4) {
+		/*Temp, Use idr_alloc instead*/
+		resp.dev_id = rdev->en_dev->pdev->devfn;
+		resp.max_qp = rdev->qplib_res.hctx->qp_ctx.max;
+	}
+
+	genp5 = _is_chip_gen_p5_p7(cctx);
+	if (BNXT_RE_ABI_VERSION > 5) {
+		resp.modes = genp5 ? cctx->modes.wqe_mode : 0;
+		if (rdev->dev_attr && BNXT_RE_HW_RETX(rdev->dev_attr->dev_cap_flags))
+			resp.comp_mask = BNXT_RE_COMP_MASK_UCNTX_HW_RETX_ENABLED;
+	}
+
+	resp.pg_size = PAGE_SIZE;
+	resp.cqe_sz = sizeof(struct cq_base);
+	resp.max_cqd = dev_attr->max_cq_wqes;
+	if (genp5 && cctx->modes.db_push) {
+		resp.comp_mask |= BNXT_RE_COMP_MASK_UCNTX_WC_DPI_ENABLED;
+		if (_is_chip_p7(cctx) &&
+		    !(dev_attr->dev_cap_flags &
+		      CREQ_QUERY_FUNC_RESP_SB_PINGPONG_PUSH_MODE))
+			resp.comp_mask &=
+				~BNXT_RE_COMP_MASK_UCNTX_WC_DPI_ENABLED;
+	}
+
+	resp.comp_mask |= BNXT_RE_COMP_MASK_UCNTX_MQP_EX_SUPPORTED;
+
+	if (rdev->dbr_pacing)
+		resp.comp_mask |= BNXT_RE_COMP_MASK_UCNTX_DBR_PACING_ENABLED;
+
+	if (rdev->dbr_drop_recov && rdev->user_dbr_drop_recov)
+		resp.comp_mask |= BNXT_RE_COMP_MASK_UCNTX_DBR_RECOVERY_ENABLED;
+
+	if (udata->inlen >= sizeof(ureq)) {
+		rc = ib_copy_from_udata(&ureq, udata,
+					min(udata->inlen, sizeof(ureq)));
+		if (rc)
+			goto cfail;
+		if (bnxt_re_init_pow2_flag(&ureq, &resp))
+			dev_warn(rdev_to_dev(rdev),
+				 "Enabled roundup logic. Library bug?\n");
+		if (bnxt_re_init_rsvd_wqe_flag(&ureq, &resp, genp5))
+			dev_warn(rdev_to_dev(rdev),
+				 "Rsvd wqe in use! Try the updated library.\n");
+	} else {
+		dev_warn(rdev_to_dev(rdev),
+			 "Enabled roundup logic. Update the library!\n");
+		resp.comp_mask &= ~BNXT_RE_COMP_MASK_UCNTX_POW2_DISABLED;
+
+		dev_warn(rdev_to_dev(rdev),
+			 "Rsvd wqe in use. Update the library!\n");
+		resp.comp_mask &= ~BNXT_RE_COMP_MASK_UCNTX_RSVD_WQE_DISABLED;
+	}
+
+	uctx->cmask = (uint64_t)resp.comp_mask;
+	rc = bnxt_re_copy_to_udata(rdev, &resp,
+				   min(udata->outlen, sizeof(resp)),
+				   udata);
+	if (rc)
+		goto cfail;
+
+	INIT_LIST_HEAD(&uctx->cq_list);
+	mutex_init(&uctx->cq_lock);
+
+	return 0;
+cfail:
+	free_page((u64)uctx->shpg);
+	uctx->shpg = NULL;
+fail:
+	return rc;
+}
+
+void bnxt_re_dealloc_ucontext(struct ib_ucontext *ib_uctx)
+{
+	struct bnxt_re_ucontext *uctx = to_bnxt_re(ib_uctx,
+						   struct bnxt_re_ucontext,
+						   ibucontext);
+	struct bnxt_re_dev *rdev = uctx->rdev;
+	int rc = 0;
+
+	if (uctx->shpg)
+		free_page((u64)uctx->shpg);
+
+	if (uctx->dpi.dbr) {
+		/* Free DPI only if this is the first PD allocated by the
+		 * application and mark the context dpi as NULL
+		 */
+		if (_is_chip_gen_p5_p7(rdev->chip_ctx) && uctx->wcdpi.dbr) {
+			rc = bnxt_qplib_dealloc_dpi(&rdev->qplib_res,
+						    &uctx->wcdpi);
+			if (rc)
+				dev_err(rdev_to_dev(rdev),
+						"dealloc push dp failed\n");
+			uctx->wcdpi.dbr = NULL;
+		}
+
+		rc = bnxt_qplib_dealloc_dpi(&rdev->qplib_res,
+					    &uctx->dpi);
+		if (rc)
+			dev_err(rdev_to_dev(rdev), "Deallocte HW DPI failed!\n");
+			/* Don't fail, continue*/
+		uctx->dpi.dbr = NULL;
+	}
+	return;
+}
+
+static struct bnxt_re_cq *is_bnxt_re_cq_page(struct bnxt_re_ucontext *uctx,
+				      u64 pg_off)
+{
+	struct bnxt_re_cq *cq = NULL, *tmp_cq;
+
+	if (!_is_chip_p7(uctx->rdev->chip_ctx))
+		return NULL;
+
+	mutex_lock(&uctx->cq_lock);
+	list_for_each_entry(tmp_cq, &uctx->cq_list, cq_list) {
+		if (((u64)tmp_cq->uctx_cq_page >> PAGE_SHIFT) == pg_off) {
+			cq = tmp_cq;
+			break;
+		}
+	}
+	mutex_unlock(&uctx->cq_lock);
+	return cq;
+}
+
+/* Helper function to mmap the virtual memory from user app */
+int bnxt_re_mmap(struct ib_ucontext *ib_uctx, struct vm_area_struct *vma)
+{
+	struct bnxt_re_ucontext *uctx = to_bnxt_re(ib_uctx,
+						   struct bnxt_re_ucontext,
+						   ibucontext);
+	struct bnxt_re_dev *rdev = uctx->rdev;
+	struct bnxt_re_cq *cq = NULL;
+	int rc = 0;
+	u64 pfn;
+
+	switch (vma->vm_pgoff) {
+	case BNXT_RE_MAP_SH_PAGE:
+		pfn = vtophys(uctx->shpg) >> PAGE_SHIFT;
+		return rdma_user_mmap_io(&uctx->ibucontext, vma, pfn, PAGE_SIZE, vma->vm_page_prot, NULL);
+		dev_dbg(rdev_to_dev(rdev), "%s:%d uctx->shpg 0x%lx, vtophys(uctx->shpg) 0x%lx, pfn = 0x%lx \n",
+				__func__, __LINE__, (u64) uctx->shpg, vtophys(uctx->shpg), pfn);
+		if (rc) {
+			dev_err(rdev_to_dev(rdev), "Shared page mapping failed!\n");
+			rc = -EAGAIN;
+		}
+		return rc;
+	case BNXT_RE_MAP_WC:
+		vma->vm_page_prot =
+			pgprot_writecombine(vma->vm_page_prot);
+		pfn = (uctx->wcdpi.umdbr >> PAGE_SHIFT);
+		if (!pfn)
+			return -EFAULT;
+		break;
+	case BNXT_RE_DBR_PAGE:
+		/* Driver doesn't expect write access request */
+		if (vma->vm_flags & VM_WRITE)
+			return -EFAULT;
+
+		pfn = vtophys(rdev->dbr_page) >> PAGE_SHIFT;
+		if (!pfn)
+			return -EFAULT;
+		break;
+	case BNXT_RE_MAP_DB_RECOVERY_PAGE:
+		pfn = vtophys(uctx->dbr_recov_cq_page) >> PAGE_SHIFT;
+		if (!pfn)
+			return -EFAULT;
+		break;
+	default:
+		cq = is_bnxt_re_cq_page(uctx, vma->vm_pgoff);
+		if (cq) {
+			pfn = vtophys((void *)cq->uctx_cq_page) >> PAGE_SHIFT;
+			rc = rdma_user_mmap_io(&uctx->ibucontext, vma, pfn, PAGE_SIZE, vma->vm_page_prot, NULL);
+			if (rc) {
+				dev_err(rdev_to_dev(rdev),
+					"CQ page mapping failed!\n");
+				rc = -EAGAIN;
+			}
+			goto out;
+		} else {
+			vma->vm_page_prot =
+				pgprot_noncached(vma->vm_page_prot);
+			pfn = vma->vm_pgoff;
+		}
+		break;
+	}
+
+	rc = rdma_user_mmap_io(&uctx->ibucontext, vma, pfn, PAGE_SIZE, vma->vm_page_prot, NULL);
+	if (rc) {
+		dev_err(rdev_to_dev(rdev), "DPI mapping failed!\n");
+		return -EAGAIN;
+	}
+	rc = __bnxt_re_set_vma_data(uctx, vma);
+out:
+	return rc;
+}
+
+int bnxt_re_process_mad(struct ib_device *ibdev, int mad_flags, u8 port_num,
+			const struct ib_wc *wc, const struct ib_grh *grh,
+			const struct ib_mad_hdr *in_mad, size_t in_mad_size,
+			struct ib_mad_hdr *out_mad, size_t *out_mad_size,
+			u16 *out_mad_pkey_index)
+{
+	return IB_MAD_RESULT_SUCCESS;
+}
+
+void bnxt_re_disassociate_ucntx(struct ib_ucontext *ib_uctx)
+{
+}
diff --git a/sys/dev/bnxt/bnxt_re/ib_verbs.h b/sys/dev/bnxt/bnxt_re/ib_verbs.h
new file mode 100644
index 000000000000..cb9f7974e92d
--- /dev/null
+++ b/sys/dev/bnxt/bnxt_re/ib_verbs.h
@@ -0,0 +1,632 @@
+/*
+ * Copyright (c) 2015-2024, Broadcom. All rights reserved.  The term
+ * Broadcom refers to Broadcom Limited and/or its subsidiaries.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS''
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
+ * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
+ * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * Description: IB Verbs interpreter (header)
+ */
+
+#ifndef __BNXT_RE_IB_VERBS_H__
+#define __BNXT_RE_IB_VERBS_H__
+
+#include <rdma/ib_addr.h>
+#include "bnxt_re-abi.h"
+#include "qplib_res.h"
+#include "qplib_fp.h"
+
+struct bnxt_re_dev;
+
+#define BNXT_RE_ROCE_V2_UDP_SPORT	0x8CD1
+#define BNXT_RE_QP_RANDOM_QKEY		0x81818181
+
+#ifndef IB_MTU_8192
+#define IB_MTU_8192 8192
+#endif
+
+#ifndef SPEED_1000
+#define SPEED_1000		1000
+#endif
+
+#ifndef SPEED_10000
+#define SPEED_10000		10000
+#endif
+
+#ifndef SPEED_20000
+#define SPEED_20000		20000
+#endif
+
+#ifndef SPEED_25000
+#define SPEED_25000		25000
+#endif
+
+#ifndef SPEED_40000
+#define SPEED_40000		40000
+#endif
+
+#ifndef SPEED_50000
+#define SPEED_50000		50000
+#endif
+
+#ifndef SPEED_100000
+#define SPEED_100000		100000
+#endif
+
+#ifndef SPEED_200000
+#define SPEED_200000		200000
+#endif
+
+#ifndef IB_SPEED_HDR
+#define IB_SPEED_HDR		64
+#endif
+
+#define RDMA_NETWORK_IPV4	1
+#define RDMA_NETWORK_IPV6	2
+
+#define ROCE_DMAC(x) (x)->dmac
+
+#define dma_rmb()       rmb()
+
+#define compat_ib_alloc_device(size) ib_alloc_device(size);
+
+#define rdev_from_cq_in(cq_in) to_bnxt_re_dev(cq_in->device, ibdev)
+
+#define GET_UVERBS_ABI_VERSION(ibdev)	(ibdev->uverbs_abi_ver)
+
+#define CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_256MB 0x1cUL
+
+#define IB_POLL_UNBOUND_WORKQUEUE       IB_POLL_WORKQUEUE
+
+#define BNXT_RE_LEGACY_FENCE_BYTES	64
+#define BNXT_RE_LEGACY_FENCE_PBL_SIZE	DIV_ROUND_UP(BNXT_RE_LEGACY_FENCE_BYTES, PAGE_SIZE)
+
+
+static inline struct
+bnxt_re_cq *__get_cq_from_cq_in(struct ib_cq *cq_in,
+				struct bnxt_re_dev *rdev);
+static inline struct
+bnxt_re_qp *__get_qp_from_qp_in(struct ib_pd *qp_in,
+				struct bnxt_re_dev *rdev);
+
+static inline bool
+bnxt_re_check_if_vlan_valid(struct bnxt_re_dev *rdev, u16 vlan_id);
+
+#define bnxt_re_compat_qfwstr(void)			\
+	bnxt_re_query_fw_str(struct ib_device *ibdev,	\
+			     char *str, size_t str_len)
+
+static inline
+struct scatterlist *get_ib_umem_sgl(struct ib_umem *umem, u32 *nmap);
+
+struct bnxt_re_gid_ctx {
+	u32			idx;
+	u32			refcnt;
+};
+
+struct bnxt_re_legacy_fence_data {
+	u32 size;
+	void *va;
+	dma_addr_t dma_addr;
+	struct bnxt_re_mr *mr;
+	struct ib_mw *mw;
+	struct bnxt_qplib_swqe bind_wqe;
+	u32 bind_rkey;
+};
+
+struct bnxt_re_pd {
+	struct ib_pd		ibpd;
+	struct bnxt_re_dev	*rdev;
+	struct bnxt_qplib_pd	qplib_pd;
+	struct bnxt_re_legacy_fence_data fence;
+};
+
+struct bnxt_re_ah {
+	struct ib_ah		ibah;
+	struct bnxt_re_dev	*rdev;
+	struct bnxt_qplib_ah	qplib_ah;
+};
+
+struct bnxt_re_srq {
+	struct ib_srq		ibsrq;
+	struct bnxt_re_dev	*rdev;
+	u32			srq_limit;
+	struct bnxt_qplib_srq	qplib_srq;
+	struct ib_umem		*umem;
+	spinlock_t		lock;
+};
+
+union ip_addr {
+	u32 ipv4_addr;
+	u8  ipv6_addr[16];
+};
+
+struct bnxt_re_qp_info_entry {
+	union ib_gid		sgid;
+	union ib_gid 		dgid;
+	union ip_addr		s_ip;
+	union ip_addr		d_ip;
+	u16			s_port;
+#define BNXT_RE_QP_DEST_PORT	4791
+	u16			d_port;
+};
+
+struct bnxt_re_qp {
+	struct ib_qp		ib_qp;
+	struct list_head	list;
+	struct bnxt_re_dev	*rdev;
+	spinlock_t		sq_lock;
+	spinlock_t		rq_lock;
+	struct bnxt_qplib_qp	qplib_qp;
+	struct ib_umem		*sumem;
+	struct ib_umem		*rumem;
+	/* QP1 */
+	u32			send_psn;
+	struct ib_ud_header	qp1_hdr;
+	struct bnxt_re_cq	*scq;
+	struct bnxt_re_cq	*rcq;
+	struct dentry		*qp_info_pdev_dentry;
+	struct bnxt_re_qp_info_entry qp_info_entry;
+	void			*qp_data;
+};
+
+struct bnxt_re_cq {
+	struct ib_cq		ibcq;
+	struct list_head	cq_list;
+	struct bnxt_re_dev	*rdev;
+	struct bnxt_re_ucontext *uctx;
+	spinlock_t              cq_lock;
+	u16			cq_count;
+	u16			cq_period;
+	struct bnxt_qplib_cq	qplib_cq;
+	struct bnxt_qplib_cqe	*cql;
+#define MAX_CQL_PER_POLL	1024
+	u32			max_cql;
+	struct ib_umem		*umem;
+	struct ib_umem		*resize_umem;
+	struct ib_ucontext	*context;
+	int			resize_cqe;
+	/* list of cq per uctx. Used only for Thor-2 */
+	void			*uctx_cq_page;
+	void			*dbr_recov_cq_page;
+	bool			is_dbr_soft_cq;
+};
+
+struct bnxt_re_mr {
+	struct bnxt_re_dev	*rdev;
+	struct ib_mr		ib_mr;
+	struct ib_umem		*ib_umem;
+	struct bnxt_qplib_mrw	qplib_mr;
+	u32			npages;
+	u64			*pages;
+	struct bnxt_qplib_frpl	qplib_frpl;
+	bool                    is_invalcb_active;
+};
+
+struct bnxt_re_frpl {
+	struct bnxt_re_dev		*rdev;
+	struct bnxt_qplib_frpl		qplib_frpl;
+	u64				*page_list;
+};
+
+struct bnxt_re_mw {
+	struct bnxt_re_dev	*rdev;
+	struct ib_mw		ib_mw;
+	struct bnxt_qplib_mrw	qplib_mw;
+};
+
+struct bnxt_re_ucontext {
+	struct ib_ucontext	ibucontext;
+	struct bnxt_re_dev	*rdev;
+	struct list_head	cq_list;
+	struct bnxt_qplib_dpi	dpi;
+	struct bnxt_qplib_dpi	wcdpi;
+	void			*shpg;
+	spinlock_t		sh_lock;
+	uint64_t		cmask;
+	struct mutex		cq_lock;	/* Protect cq list */
+	void			*dbr_recov_cq_page;
+	struct bnxt_re_cq	*dbr_recov_cq;
+};
+
+struct bnxt_re_ah_info {
+	union ib_gid		sgid;
+	struct ib_gid_attr	sgid_attr;
+	u16			vlan_tag;
+	u8			nw_type;
+};
+
+struct ifnet *bnxt_re_get_netdev(struct ib_device *ibdev,
+				 u8 port_num);
+
+int bnxt_re_query_device(struct ib_device *ibdev,
+			 struct ib_device_attr *ib_attr,
+			 struct ib_udata *udata);
+int bnxt_re_modify_device(struct ib_device *ibdev,
+			  int device_modify_mask,
+			  struct ib_device_modify *device_modify);
+int bnxt_re_query_port(struct ib_device *ibdev, u8 port_num,
+		       struct ib_port_attr *port_attr);
+int bnxt_re_modify_port(struct ib_device *ibdev, u8 port_num,
+			int port_modify_mask,
+			struct ib_port_modify *port_modify);
+int bnxt_re_get_port_immutable(struct ib_device *ibdev, u8 port_num,
+			       struct ib_port_immutable *immutable);
+void bnxt_re_compat_qfwstr(void);
+int bnxt_re_query_pkey(struct ib_device *ibdev, u8 port_num,
+		       u16 index, u16 *pkey);
+int bnxt_re_del_gid(struct ib_device *ibdev, u8 port_num,
+		    unsigned int index, void **context);
+int bnxt_re_add_gid(struct ib_device *ibdev, u8 port_num,
+		    unsigned int index, const union ib_gid *gid,
+		    const struct ib_gid_attr *attr, void **context);
+int bnxt_re_query_gid(struct ib_device *ibdev, u8 port_num,
+		      int index, union ib_gid *gid);
+enum rdma_link_layer bnxt_re_get_link_layer(struct ib_device *ibdev,
+					    u8 port_num);
+int bnxt_re_alloc_pd(struct ib_pd *pd_in, struct ib_udata *udata);
+void bnxt_re_dealloc_pd(struct ib_pd *ib_pd, struct ib_udata *udata);
+
+int bnxt_re_create_ah(struct ib_ah *ah_in, struct ib_ah_attr *attr,
+		      u32 flags, struct ib_udata *udata);
+
+int bnxt_re_modify_ah(struct ib_ah *ah, struct ib_ah_attr *ah_attr);
+int bnxt_re_query_ah(struct ib_ah *ah, struct ib_ah_attr *ah_attr);
+
+void bnxt_re_destroy_ah(struct ib_ah *ib_ah, u32 flags);
+int bnxt_re_create_srq(struct ib_srq *srq_in,
+		       struct ib_srq_init_attr *srq_init_attr,
+		       struct ib_udata *udata);
+int bnxt_re_modify_srq(struct ib_srq *srq, struct ib_srq_attr *srq_attr,
+		       enum ib_srq_attr_mask srq_attr_mask,
+		       struct ib_udata *udata);
+int bnxt_re_query_srq(struct ib_srq *srq, struct ib_srq_attr *srq_attr);
+void bnxt_re_destroy_srq(struct ib_srq *ib_srq,
+			 struct ib_udata *udata);
+int bnxt_re_post_srq_recv(struct ib_srq *ib_srq, const struct ib_recv_wr *wr,
+			  const struct ib_recv_wr **bad_wr);
+struct ib_qp *bnxt_re_create_qp(struct ib_pd *qp_in,
+			       struct ib_qp_init_attr *qp_init_attr,
+			       struct ib_udata *udata);
+int bnxt_re_modify_qp(struct ib_qp *qp, struct ib_qp_attr *qp_attr,
+		      int qp_attr_mask, struct ib_udata *udata);
+int bnxt_re_query_qp(struct ib_qp *qp, struct ib_qp_attr *qp_attr,
+		     int qp_attr_mask, struct ib_qp_init_attr *qp_init_attr);
+int bnxt_re_destroy_qp(struct ib_qp *ib_qp, struct ib_udata *udata);
+int bnxt_re_post_send(struct ib_qp *ib_qp, const struct ib_send_wr *wr,
+		      const struct ib_send_wr **bad_wr);
+int bnxt_re_post_recv(struct ib_qp *ib_qp, const struct ib_recv_wr *wr,
+		      const struct ib_recv_wr **bad_wr);
+int bnxt_re_create_cq(struct ib_cq *cq_in,
+		      const struct ib_cq_init_attr *attr,
+		      struct ib_udata *udata);
+void bnxt_re_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata);
+int bnxt_re_modify_cq(struct ib_cq *cq, u16 cq_count, u16 cq_period);
+int bnxt_re_resize_cq(struct ib_cq *cq, int cqe, struct ib_udata *udata);
+int bnxt_re_poll_cq(struct ib_cq *cq, int num_entries, struct ib_wc *wc);
+int bnxt_re_req_notify_cq(struct ib_cq *cq, enum ib_cq_notify_flags flags);
+struct ib_mr *bnxt_re_get_dma_mr(struct ib_pd *pd, int mr_access_flags);
+int bnxt_re_map_mr_sg(struct ib_mr *ib_mr, struct scatterlist *sg,
+		      int sg_nents, unsigned int *sg_offset);
+struct ib_mr *bnxt_re_alloc_mr(struct ib_pd *ib_pd, enum ib_mr_type type,
+			       u32 max_num_sg, struct ib_udata *udata);
+int bnxt_re_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata);
+struct ib_mw *bnxt_re_alloc_mw(struct ib_pd *ib_pd, enum ib_mw_type type,
+			       struct ib_udata *udata);
+int bnxt_re_dealloc_mw(struct ib_mw *mw);
+struct ib_mr *bnxt_re_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
+				  u64 virt_addr, int mr_access_flags,
+				  struct ib_udata *udata);
+int
+bnxt_re_rereg_user_mr(struct ib_mr *mr, int flags, u64 start, u64 length,
+		      u64 virt_addr, int mr_access_flags, struct ib_pd *pd,
+		      struct ib_udata *udata);
+int bnxt_re_alloc_ucontext(struct ib_ucontext *uctx_in,
+			   struct ib_udata *udata);
+void bnxt_re_dealloc_ucontext(struct ib_ucontext *ib_uctx);
+int bnxt_re_mmap(struct ib_ucontext *context, struct vm_area_struct *vma);
+int bnxt_re_process_mad(struct ib_device *ibdev, int mad_flags, u8 port_num,
+			const struct ib_wc *wc, const struct ib_grh *grh,
+			const struct ib_mad_hdr *in_mad, size_t in_mad_size,
+			struct ib_mad_hdr *out_mad, size_t *out_mad_size,
+			u16 *out_mad_pkey_index);
+unsigned long bnxt_re_lock_cqs(struct bnxt_re_qp *qp);
+void bnxt_re_unlock_cqs(struct bnxt_re_qp *qp, unsigned long flags);
+void bnxt_re_disassociate_ucntx(struct ib_ucontext *ibcontext);
+static inline int __bnxt_re_set_vma_data(void *bnxt_re_uctx,
+					 struct vm_area_struct *vma);
+void bnxt_re_update_shadow_ah(struct bnxt_re_dev *rdev);
+void bnxt_re_handle_cqn(struct bnxt_qplib_cq *cq);
+static inline int
+bnxt_re_get_cached_gid(struct ib_device *dev, u8 port_num, int index,
+		       union ib_gid *sgid, struct ib_gid_attr **sgid_attr,
+		       struct ib_global_route *grh, struct ib_ah *ah);
+static inline enum rdma_network_type
+bnxt_re_gid_to_network_type(struct ib_gid_attr *sgid_attr,
+			    union ib_gid *sgid);
+static inline
+struct ib_umem *ib_umem_get_compat(struct bnxt_re_dev *rdev,
+				   struct ib_ucontext *ucontext,
+				   struct ib_udata *udata,
+				   unsigned long addr,
+				   size_t size, int access, int dmasync);
+static inline
+struct ib_umem *ib_umem_get_flags_compat(struct bnxt_re_dev *rdev,
+					 struct ib_ucontext *ucontext,
+					 struct ib_udata *udata,
+					 unsigned long addr,
+					 size_t size, int access, int dmasync);
+static inline size_t ib_umem_num_pages_compat(struct ib_umem *umem);
+static inline void bnxt_re_peer_mem_release(struct ib_umem *umem);
+void bnxt_re_resolve_dmac_task(struct work_struct *work);
+
+static inline enum ib_qp_type  __from_hw_to_ib_qp_type(u8 type)
+{
+	switch (type) {
+	case CMDQ_CREATE_QP1_TYPE_GSI:
+	case CMDQ_CREATE_QP_TYPE_GSI:
+		return IB_QPT_GSI;
+	case CMDQ_CREATE_QP_TYPE_RC:
+		return IB_QPT_RC;
+	case CMDQ_CREATE_QP_TYPE_UD:
+		return IB_QPT_UD;
+	case CMDQ_CREATE_QP_TYPE_RAW_ETHERTYPE:
+		return IB_QPT_RAW_ETHERTYPE;
+	default:
+		return IB_QPT_MAX;
+	}
+}
+
+static inline u8 __from_ib_qp_state(enum ib_qp_state state)
+{
+	switch (state) {
+	case IB_QPS_RESET:
+		return CMDQ_MODIFY_QP_NEW_STATE_RESET;
+	case IB_QPS_INIT:
+		return CMDQ_MODIFY_QP_NEW_STATE_INIT;
+	case IB_QPS_RTR:
+		return CMDQ_MODIFY_QP_NEW_STATE_RTR;
+	case IB_QPS_RTS:
+		return CMDQ_MODIFY_QP_NEW_STATE_RTS;
+	case IB_QPS_SQD:
+		return CMDQ_MODIFY_QP_NEW_STATE_SQD;
+	case IB_QPS_SQE:
+		return CMDQ_MODIFY_QP_NEW_STATE_SQE;
+	case IB_QPS_ERR:
+	default:
+		return CMDQ_MODIFY_QP_NEW_STATE_ERR;
+	}
+}
+
+static inline u32 __from_ib_mtu(enum ib_mtu mtu)
+{
+	switch (mtu) {
+	case IB_MTU_256:
+		return CMDQ_MODIFY_QP_PATH_MTU_MTU_256;
+	case IB_MTU_512:
+		return CMDQ_MODIFY_QP_PATH_MTU_MTU_512;
+	case IB_MTU_1024:
+		return CMDQ_MODIFY_QP_PATH_MTU_MTU_1024;
+	case IB_MTU_2048:
+		return CMDQ_MODIFY_QP_PATH_MTU_MTU_2048;
+	case IB_MTU_4096:
+		return CMDQ_MODIFY_QP_PATH_MTU_MTU_4096;
+	default:
+		return CMDQ_MODIFY_QP_PATH_MTU_MTU_2048;
+	}
+}
+
+static inline enum ib_mtu __to_ib_mtu(u32 mtu)
+{
+	switch (mtu & CREQ_QUERY_QP_RESP_SB_PATH_MTU_MASK) {
+	case CMDQ_MODIFY_QP_PATH_MTU_MTU_256:
+		return IB_MTU_256;
+	case CMDQ_MODIFY_QP_PATH_MTU_MTU_512:
+		return IB_MTU_512;
+	case CMDQ_MODIFY_QP_PATH_MTU_MTU_1024:
+		return IB_MTU_1024;
+	case CMDQ_MODIFY_QP_PATH_MTU_MTU_2048:
+		return IB_MTU_2048;
+	case CMDQ_MODIFY_QP_PATH_MTU_MTU_4096:
+		return IB_MTU_4096;
+	case CMDQ_MODIFY_QP_PATH_MTU_MTU_8192:
+		return IB_MTU_8192;
+	default:
+		return IB_MTU_2048;
+	}
+}
+
+static inline enum ib_qp_state __to_ib_qp_state(u8 state)
+{
+	switch (state) {
+	case CMDQ_MODIFY_QP_NEW_STATE_RESET:
+		return IB_QPS_RESET;
+	case CMDQ_MODIFY_QP_NEW_STATE_INIT:
+		return IB_QPS_INIT;
+	case CMDQ_MODIFY_QP_NEW_STATE_RTR:
+		return IB_QPS_RTR;
+	case CMDQ_MODIFY_QP_NEW_STATE_RTS:
+		return IB_QPS_RTS;
+	case CMDQ_MODIFY_QP_NEW_STATE_SQD:
+		return IB_QPS_SQD;
+	case CMDQ_MODIFY_QP_NEW_STATE_SQE:
+		return IB_QPS_SQE;
+	case CMDQ_MODIFY_QP_NEW_STATE_ERR:
+	default:
+		return IB_QPS_ERR;
+	}
+}
+
+static inline int bnxt_re_init_pow2_flag(struct bnxt_re_uctx_req *req,
+					 struct bnxt_re_uctx_resp *resp)
+{
+	resp->comp_mask |= BNXT_RE_COMP_MASK_UCNTX_POW2_DISABLED;
+	if (!(req->comp_mask & BNXT_RE_COMP_MASK_REQ_UCNTX_POW2_SUPPORT)) {
+		resp->comp_mask &= ~BNXT_RE_COMP_MASK_UCNTX_POW2_DISABLED;
+		return -EINVAL;
+	}
+	return 0;
+}
+
+static inline u32 bnxt_re_init_depth(u32 ent, struct bnxt_re_ucontext *uctx)
+{
+	return uctx ? (uctx->cmask & BNXT_RE_COMP_MASK_UCNTX_POW2_DISABLED) ?
+		       ent : roundup_pow_of_two(ent) : ent;
+}
+
+static inline int bnxt_re_init_rsvd_wqe_flag(struct bnxt_re_uctx_req *req,
+					     struct bnxt_re_uctx_resp *resp,
+					     bool genp5)
+{
+	resp->comp_mask |= BNXT_RE_COMP_MASK_UCNTX_RSVD_WQE_DISABLED;
+	if (!(req->comp_mask & BNXT_RE_COMP_MASK_REQ_UCNTX_RSVD_WQE)) {
+		resp->comp_mask &= ~BNXT_RE_COMP_MASK_UCNTX_RSVD_WQE_DISABLED;
+		return -EINVAL;
+	} else if (!genp5) {
+		resp->comp_mask &= ~BNXT_RE_COMP_MASK_UCNTX_RSVD_WQE_DISABLED;
+	}
+	return 0;
+}
+
+static inline u32 bnxt_re_get_diff(struct bnxt_re_ucontext *uctx,
+				   struct bnxt_qplib_chip_ctx *cctx)
+{
+	if (!uctx) {
+		/* return res-wqe only for gen p4 for user resource */
+		return _is_chip_gen_p5_p7(cctx) ? 0 : BNXT_QPLIB_RESERVED_QP_WRS;
+	} else if (uctx->cmask & BNXT_RE_COMP_MASK_UCNTX_RSVD_WQE_DISABLED) {
+		return 0;
+	}
+	/* old lib */
+	return BNXT_QPLIB_RESERVED_QP_WRS;
+}
+
+static inline void bnxt_re_init_qpmtu(struct bnxt_re_qp *qp, int mtu,
+				      int mask, struct ib_qp_attr *qp_attr,
+				      bool *is_qpmtu_high)
+{
+	int qpmtu, qpmtu_int;
+	int ifmtu, ifmtu_int;
+
+	ifmtu = iboe_get_mtu(mtu);
+	ifmtu_int = ib_mtu_enum_to_int(ifmtu);
+	qpmtu = ifmtu;
+	qpmtu_int = ifmtu_int;
+	if (mask & IB_QP_PATH_MTU) {
+		qpmtu = qp_attr->path_mtu;
+		qpmtu_int = ib_mtu_enum_to_int(qpmtu);
+		if (qpmtu_int > ifmtu_int) {
+			/* Trim the QP path mtu to interface mtu and update
+			 * the new mtu to user qp for retransmission psn
+			 * calculations.
+			 */
+			qpmtu = ifmtu;
+			qpmtu_int = ifmtu_int;
+			*is_qpmtu_high = true;
+		}
+	}
+	qp->qplib_qp.path_mtu = __from_ib_mtu(qpmtu);
+	qp->qplib_qp.mtu = qpmtu_int;
+	qp->qplib_qp.modify_flags |=
+		CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU;
+}
+
+inline unsigned long compare_ether_header(void *a, void *b)
+{
+	u32 *a32 = (u32 *)((u8 *)a + 2);
+	u32 *b32 = (u32 *)((u8 *)b + 2);
+
+	return (*(u16 *)a ^ *(u16 *)b) | (a32[0] ^ b32[0]) |
+	       (a32[1] ^ b32[1]) | (a32[2] ^ b32[2]);
+}
+
+struct vlan_hdr {
+	__be16	h_vlan_TCI;
+	__be16	h_vlan_encapsulated_proto;
+};
+
+inline uint16_t
+crc16(uint16_t crc, const void *buffer, unsigned int len)
+{
+	const unsigned char *cp = buffer;
+	/* CRC table for the CRC-16. The poly is 0x8005 (x16 + x15 + x2 + 1). */
+	static uint16_t const crc16_table[256] = {
+		0x0000, 0xC0C1, 0xC181, 0x0140, 0xC301, 0x03C0, 0x0280, 0xC241,
+		0xC601, 0x06C0, 0x0780, 0xC741, 0x0500, 0xC5C1, 0xC481, 0x0440,
+		0xCC01, 0x0CC0, 0x0D80, 0xCD41, 0x0F00, 0xCFC1, 0xCE81, 0x0E40,
+		0x0A00, 0xCAC1, 0xCB81, 0x0B40, 0xC901, 0x09C0, 0x0880, 0xC841,
+		0xD801, 0x18C0, 0x1980, 0xD941, 0x1B00, 0xDBC1, 0xDA81, 0x1A40,
+		0x1E00, 0xDEC1, 0xDF81, 0x1F40, 0xDD01, 0x1DC0, 0x1C80, 0xDC41,
+		0x1400, 0xD4C1, 0xD581, 0x1540, 0xD701, 0x17C0, 0x1680, 0xD641,
+		0xD201, 0x12C0, 0x1380, 0xD341, 0x1100, 0xD1C1, 0xD081, 0x1040,
+		0xF001, 0x30C0, 0x3180, 0xF141, 0x3300, 0xF3C1, 0xF281, 0x3240,
+		0x3600, 0xF6C1, 0xF781, 0x3740, 0xF501, 0x35C0, 0x3480, 0xF441,
+		0x3C00, 0xFCC1, 0xFD81, 0x3D40, 0xFF01, 0x3FC0, 0x3E80, 0xFE41,
+		0xFA01, 0x3AC0, 0x3B80, 0xFB41, 0x3900, 0xF9C1, 0xF881, 0x3840,
+		0x2800, 0xE8C1, 0xE981, 0x2940, 0xEB01, 0x2BC0, 0x2A80, 0xEA41,
+		0xEE01, 0x2EC0, 0x2F80, 0xEF41, 0x2D00, 0xEDC1, 0xEC81, 0x2C40,
+		0xE401, 0x24C0, 0x2580, 0xE541, 0x2700, 0xE7C1, 0xE681, 0x2640,
+		0x2200, 0xE2C1, 0xE381, 0x2340, 0xE101, 0x21C0, 0x2080, 0xE041,
+		0xA001, 0x60C0, 0x6180, 0xA141, 0x6300, 0xA3C1, 0xA281, 0x6240,
+		0x6600, 0xA6C1, 0xA781, 0x6740, 0xA501, 0x65C0, 0x6480, 0xA441,
+		0x6C00, 0xACC1, 0xAD81, 0x6D40, 0xAF01, 0x6FC0, 0x6E80, 0xAE41,
+		0xAA01, 0x6AC0, 0x6B80, 0xAB41, 0x6900, 0xA9C1, 0xA881, 0x6840,
+		0x7800, 0xB8C1, 0xB981, 0x7940, 0xBB01, 0x7BC0, 0x7A80, 0xBA41,
+		0xBE01, 0x7EC0, 0x7F80, 0xBF41, 0x7D00, 0xBDC1, 0xBC81, 0x7C40,
+		0xB401, 0x74C0, 0x7580, 0xB541, 0x7700, 0xB7C1, 0xB681, 0x7640,
+		0x7200, 0xB2C1, 0xB381, 0x7340, 0xB101, 0x71C0, 0x7080, 0xB041,
+		0x5000, 0x90C1, 0x9181, 0x5140, 0x9301, 0x53C0, 0x5280, 0x9241,
+		0x9601, 0x56C0, 0x5780, 0x9741, 0x5500, 0x95C1, 0x9481, 0x5440,
+		0x9C01, 0x5CC0, 0x5D80, 0x9D41, 0x5F00, 0x9FC1, 0x9E81, 0x5E40,
+		0x5A00, 0x9AC1, 0x9B81, 0x5B40, 0x9901, 0x59C0, 0x5880, 0x9841,
+		0x8801, 0x48C0, 0x4980, 0x8941, 0x4B00, 0x8BC1, 0x8A81, 0x4A40,
+		0x4E00, 0x8EC1, 0x8F81, 0x4F40, 0x8D01, 0x4DC0, 0x4C80, 0x8C41,
+		0x4400, 0x84C1, 0x8581, 0x4540, 0x8701, 0x47C0, 0x4680, 0x8641,
+		0x8201, 0x42C0, 0x4380, 0x8341, 0x4100, 0x81C1, 0x8081, 0x4040
+	};
+
+	while (len--)
+		crc = (((crc >> 8) & 0xffU) ^
+		    crc16_table[(crc ^ *cp++) & 0xffU]) & 0x0000ffffU;
+	return crc;
+}
+
+static inline int __bnxt_re_set_vma_data(void *bnxt_re_uctx,
+					 struct vm_area_struct *vma)
+{
+	return 0;
+}
+
+static inline bool bnxt_re_check_if_vlan_valid(struct bnxt_re_dev *rdev,
+					       u16 vlan_id)
+{
+	bool ret = true;
+	/*
+	 * Check if the vlan is configured in the host.
+	 * If not configured, it  can be a transparent
+	 * VLAN. So dont report the vlan id.
+	 */
+	return ret;
+}
+
+#endif
diff --git a/sys/dev/bnxt/bnxt_re/main.c b/sys/dev/bnxt/bnxt_re/main.c
new file mode 100644
index 000000000000..e6c6f754ea47
--- /dev/null
+++ b/sys/dev/bnxt/bnxt_re/main.c
@@ -0,0 +1,4467 @@
+/*
+ * Copyright (c) 2015-2024, Broadcom. All rights reserved.  The term
+ * Broadcom refers to Broadcom Limited and/or its subsidiaries.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS''
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
+ * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
+ * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * Description: Main component of the bnxt_re driver
+ */
+
+#include <linux/if_ether.h>
+#include <linux/module.h>
+#include <linux/errno.h>
+#include <linux/pci.h>
+#include <linux/dma-mapping.h>
+#include <linux/slab.h>
+#include <linux/sched.h>
+#include <linux/delay.h>
+#include <linux/fs.h>
+#include <rdma/ib_user_verbs.h>
+#include <rdma/ib_addr.h>
+#include <rdma/ib_cache.h>
+#include <dev/mlx5/port.h>
+#include <dev/mlx5/vport.h>
+#include <linux/list.h>
+#include <rdma/ib_smi.h>
+#include <rdma/ib_umem.h>
+#include <linux/in.h>
+#include <linux/etherdevice.h>
+
+#include "bnxt_re.h"
+#include "ib_verbs.h"
+#include "bnxt_re-abi.h"
+#include "bnxt.h"
+
+static char drv_version[] =
+		"Broadcom NetXtreme-C/E RoCE Driver " ROCE_DRV_MODULE_NAME \
+		" v" ROCE_DRV_MODULE_VERSION " (" ROCE_DRV_MODULE_RELDATE ")\n";
+
+#define BNXT_RE_DESC	"Broadcom NetXtreme RoCE"
+#define BNXT_ADEV_NAME "if_bnxt"
+
+MODULE_DESCRIPTION("Broadcom NetXtreme-C/E RoCE Driver");
+MODULE_LICENSE("Dual BSD/GPL");
+MODULE_DEPEND(bnxt_re, linuxkpi, 1, 1, 1);
+MODULE_DEPEND(bnxt_re, ibcore, 1, 1, 1);
+MODULE_DEPEND(bnxt_re, if_bnxt, 1, 1, 1);
+MODULE_VERSION(bnxt_re, 1);
+
+
+DEFINE_MUTEX(bnxt_re_mutex); /* mutex lock for driver */
+
+static unsigned int restrict_mrs = 0;
+module_param(restrict_mrs, uint, 0);
+MODULE_PARM_DESC(restrict_mrs, " Restrict the no. of MRs 0 = 256K , 1 = 64K");
+
+unsigned int restrict_stats = 0;
+module_param(restrict_stats, uint, 0);
+MODULE_PARM_DESC(restrict_stats, "Restrict stats query frequency to ethtool coalesce value. Disabled by default");
+
+unsigned int enable_fc = 1;
+module_param(enable_fc, uint, 0);
+MODULE_PARM_DESC(enable_fc, "Enable default PFC, CC,ETS during driver load. 1 - fc enable, 0 - fc disable - Default is 1");
+
+unsigned int min_tx_depth = 1;
+module_param(min_tx_depth, uint, 0);
+MODULE_PARM_DESC(min_tx_depth, "Minimum TX depth - Default is 1");
+
+static uint8_t max_msix_vec[BNXT_RE_MAX_DEVICES] = {0};
+static unsigned int max_msix_vec_argc;
+module_param_array(max_msix_vec, byte, &max_msix_vec_argc, 0444);
+MODULE_PARM_DESC(max_msix_vec, "Max MSI-x vectors per PF (2 - 64) - Default is 64");
+
+unsigned int cmdq_shadow_qd = RCFW_CMD_NON_BLOCKING_SHADOW_QD;
+module_param_named(cmdq_shadow_qd, cmdq_shadow_qd, uint, 0644);
+MODULE_PARM_DESC(cmdq_shadow_qd, "Perf Stat Debug: Shadow QD Range (1-64) - Default is 64");
+
+
+/* globals */
+struct list_head bnxt_re_dev_list = LINUX_LIST_HEAD_INIT(bnxt_re_dev_list);
+static int bnxt_re_probe_count;
+
+DEFINE_MUTEX(bnxt_re_dev_lock);
+static u32 gmod_exit;
+static u32 gadd_dev_inprogress;
+
+static void bnxt_re_task(struct work_struct *work_task);
+static struct workqueue_struct *bnxt_re_wq;
+static int bnxt_re_query_hwrm_intf_version(struct bnxt_re_dev *rdev);
+static int bnxt_re_hwrm_qcfg(struct bnxt_re_dev *rdev, u32 *db_len,
+			     u32 *offset);
+static int bnxt_re_ib_init(struct bnxt_re_dev *rdev);
+static void bnxt_re_ib_init_2(struct bnxt_re_dev *rdev);
+void _bnxt_re_remove(struct auxiliary_device *adev);
+void writel_fbsd(struct bnxt_softc *bp, u32, u8, u32);
+u32 readl_fbsd(struct bnxt_softc *bp, u32, u8);
+static int bnxt_re_hwrm_dbr_pacing_qcfg(struct bnxt_re_dev *rdev);
+
+int bnxt_re_register_netdevice_notifier(struct notifier_block *nb)
+{
+	int rc;
+	rc = register_netdevice_notifier(nb);
+	return rc;
+}
+
+int bnxt_re_unregister_netdevice_notifier(struct notifier_block *nb)
+{
+	int rc;
+	rc = unregister_netdevice_notifier(nb);
+	return rc;
+}
+
+void bnxt_re_set_dma_device(struct ib_device *ibdev, struct bnxt_re_dev *rdev)
+{
+	ibdev->dma_device = &rdev->en_dev->pdev->dev;
+}
+
+void bnxt_re_init_resolve_wq(struct bnxt_re_dev *rdev)
+{
+	rdev->resolve_wq = create_singlethread_workqueue("bnxt_re_resolve_wq");
+	 INIT_LIST_HEAD(&rdev->mac_wq_list);
+}
+
+void bnxt_re_uninit_resolve_wq(struct bnxt_re_dev *rdev)
+{
+	struct bnxt_re_resolve_dmac_work *tmp_work = NULL, *tmp_st;
+	if (!rdev->resolve_wq)
+		return;
+	flush_workqueue(rdev->resolve_wq);
+	list_for_each_entry_safe(tmp_work, tmp_st, &rdev->mac_wq_list, list) {
+			list_del(&tmp_work->list);
+			kfree(tmp_work);
+	}
+	destroy_workqueue(rdev->resolve_wq);
+	rdev->resolve_wq = NULL;
+}
+
+u32 readl_fbsd(struct bnxt_softc *bp, u32 reg_off, u8 bar_idx)
+{
+
+	if (bar_idx)
+		return bus_space_read_8(bp->doorbell_bar.tag, bp->doorbell_bar.handle, reg_off);
+	else
+		return bus_space_read_8(bp->hwrm_bar.tag, bp->hwrm_bar.handle, reg_off);
+}
+
+void writel_fbsd(struct bnxt_softc *bp, u32 reg_off, u8 bar_idx, u32 val)
+{
+	if (bar_idx)
+		bus_space_write_8(bp->doorbell_bar.tag, bp->doorbell_bar.handle, reg_off, htole32(val));
+	else
+		bus_space_write_8(bp->hwrm_bar.tag, bp->hwrm_bar.handle, reg_off, htole32(val));
+}
+
+static void bnxt_re_update_fifo_occup_slabs(struct bnxt_re_dev *rdev,
+					    u32 fifo_occup)
+{
+	if (fifo_occup > rdev->dbg_stats->dbq.fifo_occup_water_mark)
+		rdev->dbg_stats->dbq.fifo_occup_water_mark = fifo_occup;
+
+	if (fifo_occup > 8 * rdev->pacing_algo_th)
+		rdev->dbg_stats->dbq.fifo_occup_slab_4++;
+	else if (fifo_occup > 4 * rdev->pacing_algo_th)
+		rdev->dbg_stats->dbq.fifo_occup_slab_3++;
+	else if (fifo_occup > 2 * rdev->pacing_algo_th)
+		rdev->dbg_stats->dbq.fifo_occup_slab_2++;
+	else if (fifo_occup > rdev->pacing_algo_th)
+		rdev->dbg_stats->dbq.fifo_occup_slab_1++;
+}
+
+static void bnxt_re_update_do_pacing_slabs(struct bnxt_re_dev *rdev)
+{
+	struct bnxt_qplib_db_pacing_data *pacing_data = rdev->qplib_res.pacing_data;
+
+	if (pacing_data->do_pacing > rdev->dbg_stats->dbq.do_pacing_water_mark)
+		rdev->dbg_stats->dbq.do_pacing_water_mark = pacing_data->do_pacing;
+
+	if (pacing_data->do_pacing > 16 * rdev->dbr_def_do_pacing)
+		rdev->dbg_stats->dbq.do_pacing_slab_5++;
+	else if (pacing_data->do_pacing > 8 * rdev->dbr_def_do_pacing)
+		rdev->dbg_stats->dbq.do_pacing_slab_4++;
+	else if (pacing_data->do_pacing > 4 * rdev->dbr_def_do_pacing)
+		rdev->dbg_stats->dbq.do_pacing_slab_3++;
+	else if (pacing_data->do_pacing > 2 * rdev->dbr_def_do_pacing)
+		rdev->dbg_stats->dbq.do_pacing_slab_2++;
+	else if (pacing_data->do_pacing > rdev->dbr_def_do_pacing)
+		rdev->dbg_stats->dbq.do_pacing_slab_1++;
+}
+
+static bool bnxt_re_is_qp1_qp(struct bnxt_re_qp *qp)
+{
+	return qp->ib_qp.qp_type == IB_QPT_GSI;
+}
+
+static struct bnxt_re_qp *bnxt_re_get_qp1_qp(struct bnxt_re_dev *rdev)
+{
+	struct bnxt_re_qp *qp;
+
+	mutex_lock(&rdev->qp_lock);
+	list_for_each_entry(qp, &rdev->qp_list, list) {
+		if (bnxt_re_is_qp1_qp(qp)) {
+			mutex_unlock(&rdev->qp_lock);
+			return qp;
+		}
+	}
+	mutex_unlock(&rdev->qp_lock);
+	return NULL;
+}
+
+/* Set the maximum number of each resource that the driver actually wants
+ * to allocate. This may be up to the maximum number the firmware has
+ * reserved for the function. The driver may choose to allocate fewer
+ * resources than the firmware maximum.
+ */
+static void bnxt_re_limit_pf_res(struct bnxt_re_dev *rdev)
+{
+	struct bnxt_qplib_max_res dev_res = {};
+	struct bnxt_qplib_chip_ctx *cctx;
+	struct bnxt_qplib_dev_attr *attr;
+	struct bnxt_qplib_ctx *hctx;
+	int i;
+
+	attr = rdev->dev_attr;
+	hctx = rdev->qplib_res.hctx;
+	cctx = rdev->chip_ctx;
+
+	bnxt_qplib_max_res_supported(cctx, &rdev->qplib_res, &dev_res, false);
+	if (!_is_chip_gen_p5_p7(cctx)) {
+		hctx->qp_ctx.max = min_t(u32, dev_res.max_qp, attr->max_qp);
+		hctx->mrw_ctx.max = min_t(u32, dev_res.max_mr, attr->max_mr);
+		/* To accommodate 16k MRs and 16k AHs,
+		 * driver has to allocate 32k backing store memory
+		 */
+		hctx->mrw_ctx.max *= 2;
+		hctx->srq_ctx.max = min_t(u32, dev_res.max_srq, attr->max_srq);
+		hctx->cq_ctx.max = min_t(u32, dev_res.max_cq, attr->max_cq);
+		for (i = 0; i < MAX_TQM_ALLOC_REQ; i++)
+			hctx->tqm_ctx.qcount[i] = attr->tqm_alloc_reqs[i];
+	} else {
+		hctx->qp_ctx.max = attr->max_qp ? attr->max_qp : dev_res.max_qp;
+		hctx->mrw_ctx.max = attr->max_mr ? attr->max_mr : dev_res.max_mr;
+		hctx->srq_ctx.max = attr->max_srq ? attr->max_srq : dev_res.max_srq;
+		hctx->cq_ctx.max = attr->max_cq ? attr->max_cq : dev_res.max_cq;
+	}
+}
+
+static void bnxt_re_limit_vf_res(struct bnxt_re_dev *rdev,
+				 struct bnxt_qplib_vf_res *vf_res,
+				 u32 num_vf)
+{
+	struct bnxt_qplib_chip_ctx *cctx = rdev->chip_ctx;
+	struct bnxt_qplib_max_res dev_res = {};
+
+	bnxt_qplib_max_res_supported(cctx, &rdev->qplib_res, &dev_res, true);
+	vf_res->max_qp = dev_res.max_qp / num_vf;
+	vf_res->max_srq = dev_res.max_srq / num_vf;
+	vf_res->max_cq = dev_res.max_cq / num_vf;
+	/*
+	 * MR and AH shares the same backing store, the value specified
+	 * for max_mrw is split into half by the FW for MR and AH
+	 */
+	vf_res->max_mrw = dev_res.max_mr * 2 / num_vf;
+	vf_res->max_gid = BNXT_RE_MAX_GID_PER_VF;
+}
+
+static void bnxt_re_set_resource_limits(struct bnxt_re_dev *rdev)
+{
+	struct bnxt_qplib_ctx *hctx;
+
+	hctx = rdev->qplib_res.hctx;
+	memset(&hctx->vf_res, 0, sizeof(struct bnxt_qplib_vf_res));
+	bnxt_re_limit_pf_res(rdev);
+
+	if (rdev->num_vfs)
+		bnxt_re_limit_vf_res(rdev, &hctx->vf_res, rdev->num_vfs);
+}
+
+static void bnxt_re_dettach_irq(struct bnxt_re_dev *rdev)
+{
+	struct bnxt_qplib_rcfw *rcfw = NULL;
+	struct bnxt_qplib_nq *nq;
+	int indx;
+
+	rcfw = &rdev->rcfw;
+	for (indx = 0; indx < rdev->nqr.max_init; indx++) {
+		nq = &rdev->nqr.nq[indx];
+		mutex_lock(&nq->lock);
+		bnxt_qplib_nq_stop_irq(nq, false);
+		mutex_unlock(&nq->lock);
+	}
+
+	bnxt_qplib_rcfw_stop_irq(rcfw, false);
+}
+
+static void bnxt_re_detach_err_device(struct bnxt_re_dev *rdev)
+{
+	/* Free the MSIx vectors only so that L2 can proceed with MSIx disable */
+	bnxt_re_dettach_irq(rdev);
+
+	/* Set the state as detached to prevent sending any more commands */
+	set_bit(ERR_DEVICE_DETACHED, &rdev->rcfw.cmdq.flags);
+	set_bit(BNXT_RE_FLAG_ERR_DEVICE_DETACHED, &rdev->flags);
+	wake_up_all(&rdev->rcfw.cmdq.waitq);
+}
+
+#define MAX_DSCP_PRI_TUPLE	64
+
+struct bnxt_re_dcb_work {
+	struct work_struct work;
+	struct bnxt_re_dev *rdev;
+	struct hwrm_async_event_cmpl cmpl;
+};
+
+static void bnxt_re_init_dcb_wq(struct bnxt_re_dev *rdev)
+{
+	rdev->dcb_wq = create_singlethread_workqueue("bnxt_re_dcb_wq");
+}
+
+static void bnxt_re_uninit_dcb_wq(struct bnxt_re_dev *rdev)
+{
+	if (!rdev->dcb_wq)
+		return;
+	flush_workqueue(rdev->dcb_wq);
+	destroy_workqueue(rdev->dcb_wq);
+	rdev->dcb_wq = NULL;
+}
+
+static void bnxt_re_init_aer_wq(struct bnxt_re_dev *rdev)
+{
+	rdev->aer_wq = create_singlethread_workqueue("bnxt_re_aer_wq");
+}
+
+static void bnxt_re_uninit_aer_wq(struct bnxt_re_dev *rdev)
+{
+	if (!rdev->aer_wq)
+		return;
+	flush_workqueue(rdev->aer_wq);
+	destroy_workqueue(rdev->aer_wq);
+	rdev->aer_wq = NULL;
+}
+
+static int bnxt_re_update_qp1_tos_dscp(struct bnxt_re_dev *rdev)
+{
+	struct bnxt_re_qp *qp;
+
+	if (!_is_chip_gen_p5_p7(rdev->chip_ctx))
+		return 0;
+
+	qp = bnxt_re_get_qp1_qp(rdev);
+	if (!qp)
+		return 0;
+
+	qp->qplib_qp.modify_flags = CMDQ_MODIFY_QP_MODIFY_MASK_TOS_DSCP;
+	qp->qplib_qp.tos_dscp = rdev->cc_param.qp1_tos_dscp;
+
+	return bnxt_qplib_modify_qp(&rdev->qplib_res, &qp->qplib_qp);
+}
+
+static void bnxt_re_reconfigure_dscp(struct bnxt_re_dev *rdev)
+{
+	struct bnxt_qplib_cc_param *cc_param;
+	struct bnxt_re_tc_rec *tc_rec;
+	bool update_cc = false;
+	u8 dscp_user;
+	int rc;
+
+	cc_param = &rdev->cc_param;
+	tc_rec = &rdev->tc_rec[0];
+
+	if (!(cc_param->roce_dscp_user || cc_param->cnp_dscp_user))
+		return;
+
+	if (cc_param->cnp_dscp_user) {
+		dscp_user = (cc_param->cnp_dscp_user & 0x3f);
+		if ((tc_rec->cnp_dscp_bv & (1ul << dscp_user)) &&
+		    (cc_param->alt_tos_dscp != dscp_user)) {
+			cc_param->alt_tos_dscp = dscp_user;
+			cc_param->mask |= CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_ALT_TOS_DSCP;
+			update_cc = true;
+		}
+	}
+
+	if (cc_param->roce_dscp_user) {
+		dscp_user = (cc_param->roce_dscp_user & 0x3f);
+		if ((tc_rec->roce_dscp_bv & (1ul << dscp_user)) &&
+		    (cc_param->tos_dscp != dscp_user)) {
+			cc_param->tos_dscp = dscp_user;
+			cc_param->mask |= CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_TOS_DSCP;
+			update_cc = true;
+		}
+	}
+
+	if (update_cc) {
+		rc = bnxt_qplib_modify_cc(&rdev->qplib_res, cc_param);
+		if (rc)
+			dev_err(rdev_to_dev(rdev), "Failed to apply cc settings\n");
+	}
+}
+
+static void bnxt_re_dcb_wq_task(struct work_struct *work)
+{
+	struct bnxt_qplib_cc_param *cc_param;
+	struct bnxt_re_tc_rec *tc_rec;
+	struct bnxt_re_dev *rdev;
+	struct bnxt_re_dcb_work *dcb_work =
+			container_of(work, struct bnxt_re_dcb_work, work);
+	int rc;
+
+	rdev = dcb_work->rdev;
+	if (!rdev)
+		goto exit;
+
+	mutex_lock(&rdev->cc_lock);
+
+	cc_param = &rdev->cc_param;
+	rc = bnxt_qplib_query_cc_param(&rdev->qplib_res, cc_param);
+	if (rc) {
+		dev_err(rdev_to_dev(rdev), "Failed to query ccparam rc:%d", rc);
+		goto fail;
+	}
+	tc_rec = &rdev->tc_rec[0];
+	/*
+	 * Upon the receival of DCB Async event:
+	 *   If roce_dscp or cnp_dscp or both (which user configured using configfs)
+	 *   is in the list, re-program the value using modify_roce_cc command
+	 */
+	bnxt_re_reconfigure_dscp(rdev);
+
+	cc_param->roce_pri = tc_rec->roce_prio;
+	if (cc_param->qp1_tos_dscp != cc_param->tos_dscp) {
+		cc_param->qp1_tos_dscp = cc_param->tos_dscp;
+		rc = bnxt_re_update_qp1_tos_dscp(rdev);
+		if (rc) {
+			dev_err(rdev_to_dev(rdev), "%s:Failed to modify QP1 rc:%d",
+				__func__, rc);
+			goto fail;
+		}
+	}
+
+fail:
+	mutex_unlock(&rdev->cc_lock);
+exit:
+	kfree(dcb_work);
+}
+
+static int bnxt_re_hwrm_dbr_pacing_broadcast_event(struct bnxt_re_dev *rdev)
+{
+	struct hwrm_func_dbr_pacing_broadcast_event_output resp = {0};
+	struct hwrm_func_dbr_pacing_broadcast_event_input req = {0};
+	struct bnxt_en_dev *en_dev = rdev->en_dev;
+	struct bnxt_fw_msg fw_msg;
+	int rc;
+
+	memset(&fw_msg, 0, sizeof(fw_msg));
+	bnxt_re_init_hwrm_hdr(rdev, (void *)&req,
+			      HWRM_FUNC_DBR_PACING_BROADCAST_EVENT, -1, -1);
+	bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp,
+			    sizeof(resp), BNXT_RE_HWRM_CMD_TIMEOUT(rdev));
+	rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg);
+	if (rc) {
+		dev_dbg(rdev_to_dev(rdev),
+			"Failed to send dbr pacing broadcast event rc:%d", rc);
+		return rc;
+	}
+	return 0;
+}
+
+static int bnxt_re_hwrm_dbr_pacing_nqlist_query(struct bnxt_re_dev *rdev)
+{
+	struct hwrm_func_dbr_pacing_nqlist_query_output resp = {0};
+	struct hwrm_func_dbr_pacing_nqlist_query_input req = {0};
+	struct bnxt_dbq_nq_list *nq_list = &rdev->nq_list;
+	struct bnxt_en_dev *en_dev = rdev->en_dev;
+	bool primary_found = false;
+	struct bnxt_fw_msg fw_msg;
+	struct bnxt_qplib_nq *nq;
+	int rc, i, j = 1;
+	u16 *nql_ptr;
+
+	nq = &rdev->nqr.nq[0];
+
+	memset(&fw_msg, 0, sizeof(fw_msg));
+	bnxt_re_init_hwrm_hdr(rdev, (void *)&req,
+			      HWRM_FUNC_DBR_PACING_NQLIST_QUERY, -1, -1);
+	bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp,
+			    sizeof(resp), BNXT_RE_HWRM_CMD_TIMEOUT(rdev));
+	rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg);
+	if (rc) {
+		dev_err(rdev_to_dev(rdev), "Failed to send dbr pacing nq list query rc:%d", rc);
+		return rc;
+	}
+	nq_list->num_nql_entries = le32_to_cpu(resp.num_nqs);
+	nql_ptr = &resp.nq_ring_id0;
+	/* populate the nq_list of the primary function with list received
+	 * from FW. Fill the NQ IDs of secondary functions from index 1 to
+	 * num_nql_entries - 1. Fill the  nq_list->nq_id[0] with the
+	 * nq_id of the primary pf
+	 */
+	for (i = 0; i < nq_list->num_nql_entries; i++) {
+		u16 nq_id = *nql_ptr;
+
+		dev_dbg(rdev_to_dev(rdev),
+			"nq_list->nq_id[%d] = %d\n", i, nq_id);
+		if (nq_id != nq->ring_id) {
+			nq_list->nq_id[j] = nq_id;
+			j++;
+		} else {
+			primary_found = true;
+			nq_list->nq_id[0] = nq->ring_id;
+		}
+		nql_ptr++;
+	}
+	if (primary_found)
+		bnxt_qplib_dbr_pacing_set_primary_pf(rdev->chip_ctx, 1);
+
+	return 0;
+}
+
+static void __wait_for_fifo_occupancy_below_th(struct bnxt_re_dev *rdev)
+{
+	struct bnxt_qplib_db_pacing_data *pacing_data = rdev->qplib_res.pacing_data;
+	u32 read_val, fifo_occup;
+	bool first_read = true;
+
+	/* loop shouldn't run infintely as the occupancy usually goes
+	 * below pacing algo threshold as soon as pacing kicks in.
+	 */
+	while (1) {
+		read_val = readl_fbsd(rdev->en_dev->softc, rdev->dbr_db_fifo_reg_off, 0);
+		fifo_occup = pacing_data->fifo_max_depth -
+			     ((read_val & pacing_data->fifo_room_mask) >>
+			      pacing_data->fifo_room_shift);
+		/* Fifo occupancy cannot be greater the MAX FIFO depth */
+		if (fifo_occup > pacing_data->fifo_max_depth)
+			break;
+
+		if (first_read) {
+			bnxt_re_update_fifo_occup_slabs(rdev, fifo_occup);
+			first_read = false;
+		}
+		if (fifo_occup < pacing_data->pacing_th)
+			break;
+	}
+}
+
+static void bnxt_re_set_default_pacing_data(struct bnxt_re_dev *rdev)
+{
+	struct bnxt_qplib_db_pacing_data *pacing_data = rdev->qplib_res.pacing_data;
+
+	pacing_data->do_pacing = rdev->dbr_def_do_pacing;
+	pacing_data->pacing_th = rdev->pacing_algo_th;
+	pacing_data->alarm_th =
+		pacing_data->pacing_th * BNXT_RE_PACING_ALARM_TH_MULTIPLE(rdev->chip_ctx);
+}
+
+#define CAG_RING_MASK 0x7FF
+#define CAG_RING_SHIFT 17
+#define WATERMARK_MASK 0xFFF
+#define WATERMARK_SHIFT	0
+
+static bool bnxt_re_check_if_dbq_intr_triggered(struct bnxt_re_dev *rdev)
+{
+	u32 read_val;
+	int j;
+
+	for (j = 0; j < 10; j++) {
+		read_val = readl_fbsd(rdev->en_dev->softc, rdev->dbr_aeq_arm_reg_off, 0);
+		dev_dbg(rdev_to_dev(rdev), "AEQ ARM status = 0x%x\n",
+			read_val);
+		if (!read_val)
+			return true;
+	}
+	return false;
+}
+
+int bnxt_re_set_dbq_throttling_reg(struct bnxt_re_dev *rdev, u16 nq_id, u32 throttle)
+{
+	u32 cag_ring_water_mark = 0, read_val;
+	u32 throttle_val;
+
+	/* Convert throttle percentage to value */
+	throttle_val = (rdev->qplib_res.pacing_data->fifo_max_depth * throttle) / 100;
+
+	if (bnxt_qplib_dbr_pacing_ext_en(rdev->chip_ctx)) {
+		cag_ring_water_mark = (nq_id & CAG_RING_MASK) << CAG_RING_SHIFT |
+				      (throttle_val & WATERMARK_MASK);
+		writel_fbsd(rdev->en_dev->softc,  rdev->dbr_throttling_reg_off, 0, cag_ring_water_mark);
+		read_val = readl_fbsd(rdev->en_dev->softc , rdev->dbr_throttling_reg_off, 0);
+		dev_dbg(rdev_to_dev(rdev),
+			"%s: dbr_throttling_reg_off read_val = 0x%x\n",
+			__func__, read_val);
+		if (read_val != cag_ring_water_mark) {
+			dev_dbg(rdev_to_dev(rdev),
+				"nq_id = %d write_val=0x%x read_val=0x%x\n",
+				nq_id, cag_ring_water_mark, read_val);
+			return 1;
+		}
+	}
+	writel_fbsd(rdev->en_dev->softc,  rdev->dbr_aeq_arm_reg_off, 0, 1);
+	return 0;
+}
+
+static void bnxt_re_set_dbq_throttling_for_non_primary(struct bnxt_re_dev *rdev)
+{
+	struct bnxt_dbq_nq_list *nq_list;
+	struct bnxt_qplib_nq *nq;
+	int i;
+
+	nq_list = &rdev->nq_list;
+	/* Run a loop for other Active functions if this is primary function */
+	if (bnxt_qplib_dbr_pacing_is_primary_pf(rdev->chip_ctx)) {
+		dev_dbg(rdev_to_dev(rdev), "%s:  nq_list->num_nql_entries= %d\n",
+			__func__, nq_list->num_nql_entries);
+		nq = &rdev->nqr.nq[0];
+		for (i = nq_list->num_nql_entries - 1; i > 0; i--) {
+			u16 nq_id = nq_list->nq_id[i];
+			if (nq)
+				dev_dbg(rdev_to_dev(rdev),
+					"%s: nq_id = %d cur_fn_ring_id = %d\n",
+					__func__, nq_id, nq->ring_id);
+			if (bnxt_re_set_dbq_throttling_reg
+					(rdev, nq_id, 0))
+				break;
+			bnxt_re_check_if_dbq_intr_triggered(rdev);
+		}
+	}
+}
+
+static void bnxt_re_handle_dbr_nq_pacing_notification(struct bnxt_re_dev *rdev)
+{
+	struct bnxt_qplib_nq *nq;
+	int rc = 0;
+
+	nq = &rdev->nqr.nq[0];
+
+	/* Query the NQ list*/
+	rc = bnxt_re_hwrm_dbr_pacing_nqlist_query(rdev);
+	if (rc) {
+		dev_err(rdev_to_dev(rdev),
+			"Failed to Query NQ list rc= %d", rc);
+		return;
+	}
+	/*Configure GRC access for Throttling and aeq_arm register */
+	writel_fbsd(rdev->en_dev->softc,  BNXT_GRCPF_REG_WINDOW_BASE_OUT + 28, 0,
+		    rdev->chip_ctx->dbr_aeq_arm_reg & BNXT_GRC_BASE_MASK);
+
+	rdev->dbr_throttling_reg_off =
+		(rdev->chip_ctx->dbr_throttling_reg &
+		 BNXT_GRC_OFFSET_MASK) + 0x8000;
+	rdev->dbr_aeq_arm_reg_off =
+		(rdev->chip_ctx->dbr_aeq_arm_reg &
+		 BNXT_GRC_OFFSET_MASK) + 0x8000;
+
+	bnxt_re_set_dbq_throttling_reg(rdev, nq->ring_id, rdev->dbq_watermark);
+}
+
+static void bnxt_re_dbq_wq_task(struct work_struct *work)
+{
+	struct bnxt_re_dbq_work *dbq_work =
+			container_of(work, struct bnxt_re_dbq_work, work);
+	struct bnxt_re_dev *rdev;
+
+	rdev = dbq_work->rdev;
+
+	if (!rdev)
+		goto exit;
+	switch (dbq_work->event) {
+	case BNXT_RE_DBQ_EVENT_SCHED:
+		dev_dbg(rdev_to_dev(rdev), "%s: Handle DBQ Pacing event\n",
+			__func__);
+		if (!bnxt_qplib_dbr_pacing_ext_en(rdev->chip_ctx))
+			bnxt_re_hwrm_dbr_pacing_broadcast_event(rdev);
+		else
+			bnxt_re_pacing_alert(rdev);
+		break;
+	case BNXT_RE_DBR_PACING_EVENT:
+		dev_dbg(rdev_to_dev(rdev), "%s: Sched interrupt/pacing worker\n",
+			__func__);
+		if (_is_chip_p7(rdev->chip_ctx))
+			bnxt_re_pacing_alert(rdev);
+		else if (!rdev->chip_ctx->modes.dbr_pacing_v0)
+			bnxt_re_hwrm_dbr_pacing_qcfg(rdev);
+		break;
+	case BNXT_RE_DBR_NQ_PACING_NOTIFICATION:
+		bnxt_re_handle_dbr_nq_pacing_notification(rdev);
+		/* Issue a broadcast event to notify other functions
+		 * that primary changed
+		 */
+		bnxt_re_hwrm_dbr_pacing_broadcast_event(rdev);
+		break;
+	}
+exit:
+	kfree(dbq_work);
+}
+
+static void bnxt_re_async_notifier(void *handle, struct hwrm_async_event_cmpl *cmpl)
+{
+	struct bnxt_re_en_dev_info *en_info = auxiliary_get_drvdata(handle);
+	struct bnxt_re_dcb_work *dcb_work;
+	struct bnxt_re_dbq_work *dbq_work;
+	struct bnxt_re_dev *rdev;
+	u16 event_id;
+	u32 data1;
+	u32 data2 = 0;
+
+	if (!cmpl) {
+		pr_err("Async event, bad completion\n");
+		return;
+	}
+
+	if (!en_info || !en_info->en_dev) {
+		pr_err("Async event, bad en_info or en_dev\n");
+		return;
+	}
+	rdev = en_info->rdev;
+
+	event_id = le16_to_cpu(cmpl->event_id);
+	data1 = le32_to_cpu(cmpl->event_data1);
+	data2 = le32_to_cpu(cmpl->event_data2);
+
+	if (!rdev || !rdev_to_dev(rdev)) {
+		dev_dbg(NULL, "Async event, bad rdev or netdev\n");
+		return;
+	}
+
+	if (test_bit(BNXT_RE_FLAG_ERR_DEVICE_DETACHED, &rdev->flags) ||
+	    !test_bit(BNXT_RE_FLAG_NETDEV_REGISTERED, &rdev->flags)) {
+		dev_dbg(NULL, "Async event, device already detached\n");
+		return;
+	}
+	if (data2 >= 0)
+		dev_dbg(rdev_to_dev(rdev), "Async event_id = %d data1 = %d data2 = %d",
+			event_id, data1, data2);
+
+	switch (event_id) {
+	case HWRM_ASYNC_EVENT_CMPL_EVENT_ID_DCB_CONFIG_CHANGE:
+		/* Not handling the event in older FWs */
+		if (!is_qport_service_type_supported(rdev))
+			break;
+		if (!rdev->dcb_wq)
+			break;
+		dcb_work = kzalloc(sizeof(*dcb_work), GFP_ATOMIC);
+		if (!dcb_work)
+			break;
+
+		dcb_work->rdev = rdev;
+		memcpy(&dcb_work->cmpl, cmpl, sizeof(*cmpl));
+		INIT_WORK(&dcb_work->work, bnxt_re_dcb_wq_task);
+		queue_work(rdev->dcb_wq, &dcb_work->work);
+		break;
+	case HWRM_ASYNC_EVENT_CMPL_EVENT_ID_RESET_NOTIFY:
+		if (EVENT_DATA1_RESET_NOTIFY_FATAL(data1)) {
+			/* Set rcfw flag to control commands send to Bono */
+			set_bit(ERR_DEVICE_DETACHED, &rdev->rcfw.cmdq.flags);
+			/* Set bnxt_re flag to control commands send via L2 driver */
+			set_bit(BNXT_RE_FLAG_ERR_DEVICE_DETACHED, &rdev->flags);
+			wake_up_all(&rdev->rcfw.cmdq.waitq);
+		}
+		break;
+	case HWRM_ASYNC_EVENT_CMPL_EVENT_ID_DOORBELL_PACING_THRESHOLD:
+		if (!rdev->dbr_pacing)
+			break;
+		dbq_work = kzalloc(sizeof(*dbq_work), GFP_ATOMIC);
+		if (!dbq_work)
+			goto unlock;
+		dbq_work->rdev = rdev;
+		dbq_work->event = BNXT_RE_DBR_PACING_EVENT;
+		INIT_WORK(&dbq_work->work, bnxt_re_dbq_wq_task);
+		queue_work(rdev->dbq_wq, &dbq_work->work);
+		rdev->dbr_sw_stats->dbq_int_recv++;
+		break;
+	case HWRM_ASYNC_EVENT_CMPL_EVENT_ID_DOORBELL_PACING_NQ_UPDATE:
+		if (!rdev->dbr_pacing)
+			break;
+
+		dbq_work = kzalloc(sizeof(*dbq_work), GFP_ATOMIC);
+		if (!dbq_work)
+			goto unlock;
+		dbq_work->rdev = rdev;
+		dbq_work->event = BNXT_RE_DBR_NQ_PACING_NOTIFICATION;
+		INIT_WORK(&dbq_work->work, bnxt_re_dbq_wq_task);
+		queue_work(rdev->dbq_wq, &dbq_work->work);
+		break;
+
+	default:
+		break;
+	}
+unlock:
+	return;
+}
+
+static void bnxt_re_db_fifo_check(struct work_struct *work)
+{
+	struct bnxt_re_dev *rdev = container_of(work, struct bnxt_re_dev,
+						dbq_fifo_check_work);
+	struct bnxt_qplib_db_pacing_data *pacing_data;
+	u32 pacing_save;
+
+	if (!mutex_trylock(&rdev->dbq_lock))
+		return;
+	pacing_data = rdev->qplib_res.pacing_data;
+	pacing_save = rdev->do_pacing_save;
+	__wait_for_fifo_occupancy_below_th(rdev);
+	cancel_delayed_work_sync(&rdev->dbq_pacing_work);
+	if (rdev->dbr_recovery_on)
+		goto recovery_on;
+	if (pacing_save > rdev->dbr_def_do_pacing) {
+		/* Double the do_pacing value during the congestion */
+		pacing_save = pacing_save << 1;
+	} else {
+		/*
+		 * when a new congestion is detected increase the do_pacing
+		 * by 8 times. And also increase the pacing_th by 4 times. The
+		 * reason to increase pacing_th is to give more space for the
+		 * queue to oscillate down without getting empty, but also more
+		 * room for the queue to increase without causing another alarm.
+		 */
+		pacing_save = pacing_save << 3;
+		pacing_data->pacing_th = rdev->pacing_algo_th * 4;
+	}
+
+	if (pacing_save > BNXT_RE_MAX_DBR_DO_PACING)
+		pacing_save = BNXT_RE_MAX_DBR_DO_PACING;
+
+	pacing_data->do_pacing = pacing_save;
+	rdev->do_pacing_save = pacing_data->do_pacing;
+	pacing_data->alarm_th =
+		pacing_data->pacing_th * BNXT_RE_PACING_ALARM_TH_MULTIPLE(rdev->chip_ctx);
+recovery_on:
+	schedule_delayed_work(&rdev->dbq_pacing_work,
+			      msecs_to_jiffies(rdev->dbq_pacing_time));
+	rdev->dbr_sw_stats->dbq_pacing_alerts++;
+	mutex_unlock(&rdev->dbq_lock);
+}
+
+static void bnxt_re_pacing_timer_exp(struct work_struct *work)
+{
+	struct bnxt_re_dev *rdev = container_of(work, struct bnxt_re_dev,
+						dbq_pacing_work.work);
+	struct bnxt_qplib_db_pacing_data *pacing_data;
+	u32 read_val, fifo_occup;
+	struct bnxt_qplib_nq *nq;
+
+	if (!mutex_trylock(&rdev->dbq_lock))
+		return;
+
+	pacing_data = rdev->qplib_res.pacing_data;
+	read_val = readl_fbsd(rdev->en_dev->softc , rdev->dbr_db_fifo_reg_off, 0);
+	fifo_occup = pacing_data->fifo_max_depth -
+		     ((read_val & pacing_data->fifo_room_mask) >>
+		      pacing_data->fifo_room_shift);
+
+	if (fifo_occup > pacing_data->pacing_th)
+		goto restart_timer;
+
+	/*
+	 * Instead of immediately going back to the default do_pacing
+	 * reduce it by 1/8 times and restart the timer.
+	 */
+	pacing_data->do_pacing = pacing_data->do_pacing - (pacing_data->do_pacing >> 3);
+	pacing_data->do_pacing = max_t(u32, rdev->dbr_def_do_pacing, pacing_data->do_pacing);
+	/*
+	 * If the fifo_occup is less than the interrupt enable threshold
+	 * enable the interrupt on the primary PF.
+	 */
+	if (rdev->dbq_int_disable && fifo_occup < rdev->pacing_en_int_th) {
+		if (bnxt_qplib_dbr_pacing_is_primary_pf(rdev->chip_ctx)) {
+			if (!rdev->chip_ctx->modes.dbr_pacing_v0) {
+				nq = &rdev->nqr.nq[0];
+				bnxt_re_set_dbq_throttling_reg(rdev, nq->ring_id,
+							       rdev->dbq_watermark);
+				rdev->dbr_sw_stats->dbq_int_en++;
+				rdev->dbq_int_disable = false;
+			}
+		}
+	}
+	if (pacing_data->do_pacing <= rdev->dbr_def_do_pacing) {
+		bnxt_re_set_default_pacing_data(rdev);
+		rdev->dbr_sw_stats->dbq_pacing_complete++;
+		goto dbq_unlock;
+	}
+restart_timer:
+	schedule_delayed_work(&rdev->dbq_pacing_work,
+			      msecs_to_jiffies(rdev->dbq_pacing_time));
+	bnxt_re_update_do_pacing_slabs(rdev);
+	rdev->dbr_sw_stats->dbq_pacing_resched++;
+dbq_unlock:
+	rdev->do_pacing_save = pacing_data->do_pacing;
+	mutex_unlock(&rdev->dbq_lock);
+}
+
+void bnxt_re_pacing_alert(struct bnxt_re_dev *rdev)
+{
+	struct bnxt_qplib_db_pacing_data *pacing_data;
+
+	if (!rdev->dbr_pacing)
+		return;
+	mutex_lock(&rdev->dbq_lock);
+	pacing_data = rdev->qplib_res.pacing_data;
+
+	/*
+	 * Increase the alarm_th to max so that other user lib instances do not
+	 * keep alerting the driver.
+	 */
+	pacing_data->alarm_th = pacing_data->fifo_max_depth;
+	pacing_data->do_pacing = BNXT_RE_MAX_DBR_DO_PACING;
+	cancel_work_sync(&rdev->dbq_fifo_check_work);
+	schedule_work(&rdev->dbq_fifo_check_work);
+	mutex_unlock(&rdev->dbq_lock);
+}
+
+void bnxt_re_schedule_dbq_event(struct bnxt_qplib_res *res)
+{
+	struct bnxt_re_dbq_work *dbq_work;
+	struct bnxt_re_dev *rdev;
+
+	rdev = container_of(res, struct bnxt_re_dev, qplib_res);
+
+	atomic_set(&rdev->dbq_intr_running, 1);
+
+	if (test_bit(BNXT_RE_FLAG_ERR_DEVICE_DETACHED, &rdev->flags))
+		goto exit;
+	/* Run the loop to send dbq event to other functions
+	 * for newer FW
+	 */
+	if (bnxt_qplib_dbr_pacing_ext_en(rdev->chip_ctx) &&
+	    !rdev->chip_ctx->modes.dbr_pacing_v0)
+		bnxt_re_set_dbq_throttling_for_non_primary(rdev);
+
+	dbq_work = kzalloc(sizeof(*dbq_work), GFP_ATOMIC);
+	if (!dbq_work)
+		goto exit;
+	dbq_work->rdev = rdev;
+	dbq_work->event = BNXT_RE_DBQ_EVENT_SCHED;
+	INIT_WORK(&dbq_work->work, bnxt_re_dbq_wq_task);
+	queue_work(rdev->dbq_wq, &dbq_work->work);
+	rdev->dbr_sw_stats->dbq_int_recv++;
+	rdev->dbq_int_disable = true;
+exit:
+	atomic_set(&rdev->dbq_intr_running, 0);
+}
+
+static void bnxt_re_free_msix(struct bnxt_re_dev *rdev)
+{
+	struct bnxt_en_dev *en_dev = rdev->en_dev;
+	int rc;
+
+	rc = en_dev->en_ops->bnxt_free_msix(rdev->en_dev, BNXT_ROCE_ULP);
+	if (rc)
+		dev_err(rdev_to_dev(rdev), "netdev %p free_msix failed! rc = 0x%x",
+			rdev->netdev, rc);
+}
+
+static int bnxt_re_request_msix(struct bnxt_re_dev *rdev)
+{
+	struct bnxt_en_dev *en_dev = rdev->en_dev;
+	int rc = 0, num_msix_want, num_msix_got;
+	struct bnxt_msix_entry *entry;
+
+	/*
+	 * Request MSIx based on the function type. This is
+	 * a temporory solution to enable max VFs when NPAR is
+	 * enabled.
+	 * TODO - change the scheme with an adapter specific check
+	 * as the latest adapters can support more NQs. For now
+	 * this change satisfy all adapter versions.
+	 */
+
+	if (rdev->is_virtfn)
+		num_msix_want = BNXT_RE_MAX_MSIX_VF;
+	else if (BNXT_EN_NPAR(en_dev))
+		num_msix_want = BNXT_RE_MAX_MSIX_NPAR_PF;
+	else if (_is_chip_gen_p5_p7(rdev->chip_ctx))
+		num_msix_want = rdev->num_msix_requested ?: BNXT_RE_MAX_MSIX_GEN_P5_PF;
+	else
+		num_msix_want = BNXT_RE_MAX_MSIX_PF;
+
+	/*
+	 * Since MSIX vectors are used for both NQs and CREQ, we should try to
+	 * allocate num_online_cpus + 1 by taking into account the CREQ. This
+	 * leaves the number of MSIX vectors for NQs match the number of CPUs
+	 * and allows the system to be fully utilized
+	 */
+	num_msix_want = min_t(u32, num_msix_want, num_online_cpus() + 1);
+	num_msix_want = min_t(u32, num_msix_want, BNXT_RE_MAX_MSIX);
+	num_msix_want = max_t(u32, num_msix_want, BNXT_RE_MIN_MSIX);
+
+	entry = rdev->nqr.msix_entries;
+
+	num_msix_got = en_dev->en_ops->bnxt_request_msix(en_dev, BNXT_ROCE_ULP,
+							 entry, num_msix_want);
+	if (num_msix_got < BNXT_RE_MIN_MSIX) {
+		rc = -EINVAL;
+		goto done;
+	}
+	if (num_msix_got != num_msix_want)
+		dev_warn(rdev_to_dev(rdev),
+			 "bnxt_request_msix: wanted %d vectors, got %d\n",
+			 num_msix_want, num_msix_got);
+
+	rdev->nqr.num_msix = num_msix_got;
+	return 0;
+done:
+	if (num_msix_got)
+		bnxt_re_free_msix(rdev);
+	return rc;
+}
+
+static int  __wait_for_ib_unregister(struct bnxt_re_dev *rdev,
+				     struct bnxt_re_en_dev_info *en_info)
+{
+	u64 timeout = 0;
+	u32 cur_prod = 0, cur_cons = 0;
+	int retry = 0, rc = 0, ret = 0;
+
+	cur_prod = rdev->rcfw.cmdq.hwq.prod;
+	cur_cons = rdev->rcfw.cmdq.hwq.cons;
+	timeout = msecs_to_jiffies(BNXT_RE_RECOVERY_IB_UNINIT_WAIT_TIME_MS);
+	retry = BNXT_RE_RECOVERY_IB_UNINIT_WAIT_RETRY;
+	/* During module exit, increase timeout ten-fold to 100 mins to wait
+	 * as long as possible for ib_unregister() to complete
+	 */
+	if (rdev->mod_exit)
+		retry *= 10;
+	do {
+		/*
+		 * Since the caller of this function invokes with bnxt_re_mutex held,
+		 * release it to avoid holding a lock while in wait / sleep mode.
+		 */
+		mutex_unlock(&bnxt_re_mutex);
+		rc = wait_event_timeout(en_info->waitq,
+					en_info->ib_uninit_done,
+					timeout);
+		mutex_lock(&bnxt_re_mutex);
+
+		if (!bnxt_re_is_rdev_valid(rdev))
+			break;
+
+		if (rc)
+			break;
+
+		if (!RCFW_NO_FW_ACCESS(&rdev->rcfw)) {
+			/* No need to check for cmdq stall during module exit,
+			 * wait for ib unregister to complete
+			 */
+			if (!rdev->mod_exit)
+				ret = __check_cmdq_stall(&rdev->rcfw, &cur_prod, &cur_cons);
+			if (ret || en_info->ib_uninit_done)
+				break;
+		}
+	} while (retry--);
+
+	return rc;
+}
+
+static int bnxt_re_handle_start(struct auxiliary_device *adev)
+{
+	struct bnxt_re_en_dev_info *en_info = auxiliary_get_drvdata(adev);
+	struct bnxt_re_dev *rdev = NULL;
+	struct ifnet *real_dev;
+	struct bnxt_en_dev *en_dev;
+	struct ifnet *netdev;
+	int rc = 0;
+
+	if (!en_info || !en_info->en_dev) {
+		pr_err("Start, bad en_info or en_dev\n");
+		return -EINVAL;
+	}
+	netdev = en_info->en_dev->net;
+	if (en_info->rdev) {
+		dev_info(rdev_to_dev(en_info->rdev),
+			 "%s: Device is already added adev %p rdev: %p\n",
+			 __func__, adev, en_info->rdev);
+		return 0;
+	}
+
+	en_dev = en_info->en_dev;
+	real_dev = rdma_vlan_dev_real_dev(netdev);
+	if (!real_dev)
+		real_dev = netdev;
+	rc = bnxt_re_add_device(&rdev, real_dev,
+				en_info->gsi_mode,
+				BNXT_RE_POST_RECOVERY_INIT,
+				en_info->wqe_mode,
+				en_info->num_msix_requested, adev);
+	if (rc) {
+		/* Add device failed. Unregister the device.
+		 * This has to be done explicitly as
+		 * bnxt_re_stop would not have unregistered
+		 */
+		rtnl_lock();
+		en_dev->en_ops->bnxt_unregister_device(en_dev, BNXT_ROCE_ULP);
+		rtnl_unlock();
+		mutex_lock(&bnxt_re_dev_lock);
+		gadd_dev_inprogress--;
+		mutex_unlock(&bnxt_re_dev_lock);
+		return rc;
+	}
+	rdev->adev = adev;
+	rtnl_lock();
+	bnxt_re_get_link_speed(rdev);
+	rtnl_unlock();
+	rc = bnxt_re_ib_init(rdev);
+	if (rc) {
+		dev_err(rdev_to_dev(rdev), "Failed ib_init\n");
+		return rc;
+	}
+	bnxt_re_ib_init_2(rdev);
+
+	return rc;
+}
+
+static void bnxt_re_stop(void *handle)
+{
+	struct bnxt_re_en_dev_info *en_info = auxiliary_get_drvdata(handle);
+	struct ifnet *netdev;
+	struct bnxt_re_dev *rdev;
+	struct bnxt_en_dev *en_dev;
+	int rc = 0;
+
+	rtnl_unlock();
+	mutex_lock(&bnxt_re_mutex);
+	if (!en_info || !en_info->en_dev) {
+		pr_err("Stop, bad en_info or en_dev\n");
+		goto exit;
+	}
+	netdev = en_info->en_dev->net;
+	rdev = en_info->rdev;
+	if (!rdev)
+		goto exit;
+
+	if (!bnxt_re_is_rdev_valid(rdev))
+		goto exit;
+
+	/*
+	 * Check if fw has undergone reset or is in a fatal condition.
+	 * If so, set flags so that no further commands are sent down to FW
+	 */
+	en_dev = rdev->en_dev;
+	if (en_dev->en_state & BNXT_STATE_FW_FATAL_COND ||
+	    en_dev->en_state & BNXT_STATE_FW_RESET_DET) {
+		/* Set rcfw flag to control commands send to Bono */
+		set_bit(ERR_DEVICE_DETACHED, &rdev->rcfw.cmdq.flags);
+		/* Set bnxt_re flag to control commands send via L2 driver */
+		set_bit(BNXT_RE_FLAG_ERR_DEVICE_DETACHED, &rdev->flags);
+		wake_up_all(&rdev->rcfw.cmdq.waitq);
+	}
+
+	if (test_bit(BNXT_RE_FLAG_STOP_IN_PROGRESS, &rdev->flags))
+		goto exit;
+	set_bit(BNXT_RE_FLAG_STOP_IN_PROGRESS, &rdev->flags);
+
+	en_info->wqe_mode = rdev->chip_ctx->modes.wqe_mode;
+	en_info->gsi_mode = rdev->gsi_ctx.gsi_qp_mode;
+	en_info->num_msix_requested = rdev->num_msix_requested;
+	en_info->ib_uninit_done = false;
+
+	if (rdev->dbr_pacing)
+		bnxt_re_set_pacing_dev_state(rdev);
+
+	dev_info(rdev_to_dev(rdev), "%s: L2 driver notified to stop."
+		 "Attempting to stop and Dispatching event "
+		 "to inform the stack\n", __func__);
+	init_waitqueue_head(&en_info->waitq);
+	/* Schedule a work item to handle IB UNINIT for recovery */
+	bnxt_re_schedule_work(rdev, NETDEV_UNREGISTER,
+			      NULL, netdev, rdev->adev);
+	rc = __wait_for_ib_unregister(rdev, en_info);
+	if (!bnxt_re_is_rdev_valid(rdev))
+		goto exit;
+	if (!rc) {
+		dev_info(rdev_to_dev(rdev), "%s: Attempt to stop failed\n",
+			 __func__);
+		bnxt_re_detach_err_device(rdev);
+		goto exit;
+	}
+	bnxt_re_remove_device(rdev, BNXT_RE_PRE_RECOVERY_REMOVE, rdev->adev);
+exit:
+	mutex_unlock(&bnxt_re_mutex);
+	/* Take rtnl_lock before return, bnxt_re_stop is called with rtnl_lock */
+	rtnl_lock();
+
+	return;
+}
+
+static void bnxt_re_start(void *handle)
+{
+	rtnl_unlock();
+	mutex_lock(&bnxt_re_mutex);
+	if (bnxt_re_handle_start((struct auxiliary_device *)handle))
+		pr_err("Failed to start RoCE device");
+	mutex_unlock(&bnxt_re_mutex);
+	/* Take rtnl_lock before return, bnxt_re_start is called with rtnl_lock */
+	rtnl_lock();
+	return;
+}
+
+static void bnxt_re_shutdown(void *p)
+{
+	struct bnxt_re_en_dev_info *en_info = auxiliary_get_drvdata(p);
+	struct bnxt_re_dev *rdev;
+
+	if (!en_info) {
+		pr_err("Shutdown, bad en_info\n");
+		return;
+	}
+	rtnl_unlock();
+	mutex_lock(&bnxt_re_mutex);
+	rdev = en_info->rdev;
+	if (!rdev || !bnxt_re_is_rdev_valid(rdev))
+		goto exit;
+
+	/* rtnl_lock held by L2 before coming here */
+	bnxt_re_stopqps_and_ib_uninit(rdev);
+	bnxt_re_remove_device(rdev, BNXT_RE_COMPLETE_REMOVE, rdev->adev);
+exit:
+	mutex_unlock(&bnxt_re_mutex);
+	rtnl_lock();
+	return;
+}
+
+static void bnxt_re_stop_irq(void *handle)
+{
+	struct bnxt_re_en_dev_info *en_info = auxiliary_get_drvdata(handle);
+	struct bnxt_qplib_rcfw *rcfw = NULL;
+	struct bnxt_re_dev *rdev;
+	struct bnxt_qplib_nq *nq;
+	int indx;
+
+	if (!en_info) {
+		pr_err("Stop irq, bad en_info\n");
+		return;
+	}
+	rdev = en_info->rdev;
+
+	if (!rdev)
+		return;
+
+	rcfw = &rdev->rcfw;
+	for (indx = 0; indx < rdev->nqr.max_init; indx++) {
+		nq = &rdev->nqr.nq[indx];
+		mutex_lock(&nq->lock);
+		bnxt_qplib_nq_stop_irq(nq, false);
+		mutex_unlock(&nq->lock);
+	}
+
+	if (test_bit(BNXT_RE_FLAG_ALLOC_RCFW, &rdev->flags))
+		bnxt_qplib_rcfw_stop_irq(rcfw, false);
+}
+
+static void bnxt_re_start_irq(void *handle, struct bnxt_msix_entry *ent)
+{
+	struct bnxt_re_en_dev_info *en_info = auxiliary_get_drvdata(handle);
+	struct bnxt_msix_entry *msix_ent = NULL;
+	struct bnxt_qplib_rcfw *rcfw = NULL;
+	struct bnxt_re_dev *rdev;
+	struct bnxt_qplib_nq *nq;
+	int indx, rc, vec;
+
+	if (!en_info) {
+		pr_err("Start irq, bad en_info\n");
+		return;
+	}
+	rdev = en_info->rdev;
+	if (!rdev)
+		return;
+	if (test_bit(BNXT_RE_FLAG_ERR_DEVICE_DETACHED, &rdev->flags))
+		return;
+	msix_ent = rdev->nqr.msix_entries;
+	rcfw = &rdev->rcfw;
+
+	if (!ent) {
+		/* Not setting the f/w timeout bit in rcfw.
+		 * During the driver unload the first command
+		 * to f/w will timeout and that will set the
+		 * timeout bit.
+		 */
+		dev_err(rdev_to_dev(rdev), "Failed to re-start IRQs\n");
+		return;
+	}
+
+	/* Vectors may change after restart, so update with new vectors
+	 * in device structure.
+	 */
+	for (indx = 0; indx < rdev->nqr.num_msix; indx++)
+		rdev->nqr.msix_entries[indx].vector = ent[indx].vector;
+
+	if (test_bit(BNXT_RE_FLAG_ALLOC_RCFW, &rdev->flags)) {
+		rc = bnxt_qplib_rcfw_start_irq(rcfw, msix_ent[BNXT_RE_AEQ_IDX].vector,
+					       false);
+		if (rc) {
+			dev_warn(rdev_to_dev(rdev),
+				 "Failed to reinit CREQ\n");
+			return;
+		}
+	}
+	for (indx = 0 ; indx < rdev->nqr.max_init; indx++) {
+		nq = &rdev->nqr.nq[indx];
+		vec = indx + 1;
+		rc = bnxt_qplib_nq_start_irq(nq, indx, msix_ent[vec].vector,
+					     false);
+		if (rc) {
+			dev_warn(rdev_to_dev(rdev),
+				 "Failed to reinit NQ index %d\n", indx);
+			return;
+		}
+	}
+}
+
+/*
+ * Except for ulp_async_notifier, the remaining ulp_ops
+ * below are called with rtnl_lock held
+ */
+static struct bnxt_ulp_ops bnxt_re_ulp_ops = {
+	.ulp_async_notifier = bnxt_re_async_notifier,
+	.ulp_stop = bnxt_re_stop,
+	.ulp_start = bnxt_re_start,
+	.ulp_shutdown = bnxt_re_shutdown,
+	.ulp_irq_stop = bnxt_re_stop_irq,
+	.ulp_irq_restart = bnxt_re_start_irq,
+};
+
+static inline const char *bnxt_re_netevent(unsigned long event)
+{
+	BNXT_RE_NETDEV_EVENT(event, NETDEV_UP);
+	BNXT_RE_NETDEV_EVENT(event, NETDEV_DOWN);
+	BNXT_RE_NETDEV_EVENT(event, NETDEV_CHANGE);
+	BNXT_RE_NETDEV_EVENT(event, NETDEV_REGISTER);
+	BNXT_RE_NETDEV_EVENT(event, NETDEV_UNREGISTER);
+	BNXT_RE_NETDEV_EVENT(event, NETDEV_CHANGEADDR);
+	return "Unknown";
+}
+
+/* RoCE -> Net driver */
+
+/* Driver registration routines used to let the networking driver (bnxt_en)
+ * to know that the RoCE driver is now installed */
+static void bnxt_re_unregister_netdev(struct bnxt_re_dev *rdev)
+{
+	struct bnxt_en_dev *en_dev = rdev->en_dev;
+	int rc;
+
+	rtnl_lock();
+	rc = en_dev->en_ops->bnxt_unregister_device(rdev->en_dev,
+						    BNXT_ROCE_ULP);
+	rtnl_unlock();
+	if (rc)
+		dev_err(rdev_to_dev(rdev), "netdev %p unregister failed! rc = 0x%x",
+			rdev->en_dev->net, rc);
+
+	clear_bit(BNXT_RE_FLAG_NETDEV_REGISTERED, &rdev->flags);
+}
+
+static int bnxt_re_register_netdev(struct bnxt_re_dev *rdev)
+{
+	struct bnxt_en_dev *en_dev = rdev->en_dev;
+	int rc = 0;
+
+	rtnl_lock();
+	rc = en_dev->en_ops->bnxt_register_device(en_dev,
+						  BNXT_ROCE_ULP,
+						  &bnxt_re_ulp_ops,
+						  rdev->adev);
+	rtnl_unlock();
+	if (rc) {
+		dev_err(rdev_to_dev(rdev), "netdev %p register failed! rc = 0x%x",
+			rdev->netdev, rc);
+		return rc;
+	}
+
+	return rc;
+}
+
+static void bnxt_re_set_db_offset(struct bnxt_re_dev *rdev)
+{
+	struct bnxt_qplib_chip_ctx *cctx;
+	struct bnxt_en_dev *en_dev;
+	struct bnxt_qplib_res *res;
+	u32 l2db_len = 0;
+	u32 offset = 0;
+	u32 barlen;
+	int rc;
+
+	res = &rdev->qplib_res;
+	en_dev = rdev->en_dev;
+	cctx = rdev->chip_ctx;
+
+	/* Issue qcfg */
+	rc = bnxt_re_hwrm_qcfg(rdev, &l2db_len, &offset);
+	if (rc)
+		dev_info(rdev_to_dev(rdev),
+			 "Couldn't get DB bar size, Low latency framework is disabled\n");
+	/* set register offsets for both UC and WC */
+	if (_is_chip_p7(cctx))
+		res->dpi_tbl.ucreg.offset = offset;
+	else
+		res->dpi_tbl.ucreg.offset = res->is_vf ? BNXT_QPLIB_DBR_VF_DB_OFFSET :
+							 BNXT_QPLIB_DBR_PF_DB_OFFSET;
+	res->dpi_tbl.wcreg.offset = res->dpi_tbl.ucreg.offset;
+
+	/* If WC mapping is disabled by L2 driver then en_dev->l2_db_size
+	 * is equal to the DB-Bar actual size. This indicates that L2
+	 * is mapping entire bar as UC-. RoCE driver can't enable WC mapping
+	 * in such cases and DB-push will be disabled.
+	 */
+	barlen = pci_resource_len(res->pdev, RCFW_DBR_PCI_BAR_REGION);
+	if (cctx->modes.db_push && l2db_len && en_dev->l2_db_size != barlen) {
+		res->dpi_tbl.wcreg.offset = en_dev->l2_db_size;
+		dev_info(rdev_to_dev(rdev),
+			 "Low latency framework is enabled\n");
+	}
+
+	return;
+}
+
+static void bnxt_re_set_drv_mode(struct bnxt_re_dev *rdev, u8 mode)
+{
+	struct bnxt_qplib_chip_ctx *cctx;
+	struct bnxt_en_dev *en_dev;
+
+	en_dev = rdev->en_dev;
+	cctx = rdev->chip_ctx;
+	cctx->modes.wqe_mode = _is_chip_gen_p5_p7(rdev->chip_ctx) ?
+					mode : BNXT_QPLIB_WQE_MODE_STATIC;
+	cctx->modes.te_bypass = false;
+	if (bnxt_re_hwrm_qcaps(rdev))
+		dev_err(rdev_to_dev(rdev),
+			"Failed to query hwrm qcaps\n");
+	 /*
+	  * TODO: Need a better mechanism for spreading of the
+	  * 512 extended PPP pages in the presence of VF and
+	  * NPAR, until then not enabling push
+	  */
+	if (_is_chip_p7(rdev->chip_ctx) && cctx->modes.db_push) {
+		if (rdev->is_virtfn || BNXT_EN_NPAR(en_dev))
+			cctx->modes.db_push = false;
+	}
+
+	rdev->roce_mode = en_dev->flags & BNXT_EN_FLAG_ROCE_CAP;
+	dev_dbg(rdev_to_dev(rdev),
+		"RoCE is supported on the device - caps:0x%x",
+		rdev->roce_mode);
+	if (!_is_chip_gen_p5_p7(rdev->chip_ctx))
+		rdev->roce_mode = BNXT_RE_FLAG_ROCEV2_CAP;
+	cctx->hw_stats_size = en_dev->hw_ring_stats_size;
+}
+
+static void bnxt_re_destroy_chip_ctx(struct bnxt_re_dev *rdev)
+{
+	struct bnxt_qplib_chip_ctx *chip_ctx;
+	struct bnxt_qplib_res *res;
+
+	if (!rdev->chip_ctx)
+		return;
+
+	res = &rdev->qplib_res;
+	bnxt_qplib_unmap_db_bar(res);
+
+	kfree(res->hctx);
+	res->rcfw = NULL;
+	kfree(rdev->dev_attr);
+	rdev->dev_attr = NULL;
+
+	chip_ctx = rdev->chip_ctx;
+	rdev->chip_ctx = NULL;
+	res->cctx = NULL;
+	res->hctx = NULL;
+	res->pdev = NULL;
+	res->netdev = NULL;
+	kfree(chip_ctx);
+}
+
+static int bnxt_re_setup_chip_ctx(struct bnxt_re_dev *rdev, u8 wqe_mode)
+{
+	struct bnxt_qplib_chip_ctx *chip_ctx;
+	struct bnxt_en_dev *en_dev;
+	int rc;
+
+	en_dev = rdev->en_dev;
+	/* Supply pci device to qplib */
+	rdev->qplib_res.pdev = en_dev->pdev;
+	rdev->qplib_res.netdev = rdev->netdev;
+	rdev->qplib_res.en_dev = en_dev;
+
+	chip_ctx = kzalloc(sizeof(*chip_ctx), GFP_KERNEL);
+	if (!chip_ctx)
+		return -ENOMEM;
+	rdev->chip_ctx = chip_ctx;
+	rdev->qplib_res.cctx = chip_ctx;
+	rc = bnxt_re_query_hwrm_intf_version(rdev);
+	if (rc)
+		goto fail;
+	rdev->dev_attr = kzalloc(sizeof(*rdev->dev_attr), GFP_KERNEL);
+	if (!rdev->dev_attr) {
+		rc = -ENOMEM;
+		goto fail;
+	}
+	rdev->qplib_res.dattr = rdev->dev_attr;
+	rdev->qplib_res.rcfw = &rdev->rcfw;
+	rdev->qplib_res.is_vf = rdev->is_virtfn;
+
+	rdev->qplib_res.hctx = kzalloc(sizeof(*rdev->qplib_res.hctx),
+				       GFP_KERNEL);
+	if (!rdev->qplib_res.hctx) {
+		rc = -ENOMEM;
+		goto fail;
+	}
+	bnxt_re_set_drv_mode(rdev, wqe_mode);
+
+	bnxt_re_set_db_offset(rdev);
+	rc = bnxt_qplib_map_db_bar(&rdev->qplib_res);
+	if (rc)
+		goto fail;
+
+	rc = bnxt_qplib_enable_atomic_ops_to_root(en_dev->pdev);
+	if (rc)
+		dev_dbg(rdev_to_dev(rdev),
+			"platform doesn't support global atomics");
+
+	return 0;
+fail:
+	kfree(rdev->chip_ctx);
+	rdev->chip_ctx = NULL;
+
+	kfree(rdev->dev_attr);
+	rdev->dev_attr = NULL;
+
+	kfree(rdev->qplib_res.hctx);
+	rdev->qplib_res.hctx = NULL;
+	return rc;
+}
+
+static u16 bnxt_re_get_rtype(struct bnxt_re_dev *rdev) {
+	return _is_chip_gen_p5_p7(rdev->chip_ctx) ?
+	       HWRM_RING_ALLOC_INPUT_RING_TYPE_NQ :
+	       HWRM_RING_ALLOC_INPUT_RING_TYPE_ROCE_CMPL;
+}
+
+static int bnxt_re_net_ring_free(struct bnxt_re_dev *rdev, u16 fw_ring_id)
+{
+	int rc = -EINVAL;
+	struct hwrm_ring_free_input req = {0};
+	struct hwrm_ring_free_output resp;
+	struct bnxt_en_dev *en_dev = rdev->en_dev;
+	struct bnxt_fw_msg fw_msg;
+
+	if (!en_dev)
+		return rc;
+
+	/* To avoid unnecessary error messages during recovery.
+	 * HW is anyway in error state. So dont send down the command */
+	if (test_bit(BNXT_RE_FLAG_ERR_DEVICE_DETACHED, &rdev->flags))
+		return 0;
+
+	/* allocation had failed, no need to issue hwrm */
+	if (fw_ring_id == 0xffff)
+		return 0;
+
+	memset(&fw_msg, 0, sizeof(fw_msg));
+
+	bnxt_re_init_hwrm_hdr(rdev, (void *)&req, HWRM_RING_FREE, -1, -1);
+	req.ring_type = bnxt_re_get_rtype(rdev);
+	req.ring_id = cpu_to_le16(fw_ring_id);
+	bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp,
+			    sizeof(resp), DFLT_HWRM_CMD_TIMEOUT);
+	rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg);
+	if (rc) {
+		dev_err(rdev_to_dev(rdev),
+			"Failed to free HW ring with rc = 0x%x", rc);
+		return rc;
+	}
+	dev_dbg(rdev_to_dev(rdev), "HW ring freed with id = 0x%x\n",
+		fw_ring_id);
+
+	return rc;
+}
+
+static int bnxt_re_net_ring_alloc(struct bnxt_re_dev *rdev,
+				  struct bnxt_re_ring_attr *ring_attr,
+				  u16 *fw_ring_id)
+{
+	int rc = -EINVAL;
+	struct hwrm_ring_alloc_input req = {0};
+	struct hwrm_ring_alloc_output resp;
+	struct bnxt_en_dev *en_dev = rdev->en_dev;
+	struct bnxt_fw_msg fw_msg;
+
+	if (!en_dev)
+		return rc;
+
+	memset(&fw_msg, 0, sizeof(fw_msg));
+	bnxt_re_init_hwrm_hdr(rdev, (void *)&req, HWRM_RING_ALLOC, -1, -1);
+	req.flags = cpu_to_le16(ring_attr->flags);
+	req.enables = 0;
+	req.page_tbl_addr =  cpu_to_le64(ring_attr->dma_arr[0]);
+	if (ring_attr->pages > 1) {
+		/* Page size is in log2 units */
+		req.page_size = BNXT_PAGE_SHIFT;
+		req.page_tbl_depth = 1;
+	} else {
+		req.page_size = 4;
+		req.page_tbl_depth = 0;
+	}
+
+	req.fbo = 0;
+	/* Association of ring index with doorbell index and MSIX number */
+	req.logical_id = cpu_to_le16(ring_attr->lrid);
+	req.length = cpu_to_le32(ring_attr->depth + 1);
+	req.ring_type = ring_attr->type;
+	req.int_mode = ring_attr->mode;
+	bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp,
+			    sizeof(resp), DFLT_HWRM_CMD_TIMEOUT);
+	rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg);
+	if (rc) {
+		dev_err(rdev_to_dev(rdev),
+			"Failed to allocate HW ring with rc = 0x%x", rc);
+		return rc;
+	}
+	*fw_ring_id = le16_to_cpu(resp.ring_id);
+	dev_dbg(rdev_to_dev(rdev),
+		"HW ring allocated with id = 0x%x at slot 0x%x",
+		resp.ring_id, ring_attr->lrid);
+
+	return rc;
+}
+
+static int bnxt_re_net_stats_ctx_free(struct bnxt_re_dev *rdev,
+				      u32 fw_stats_ctx_id, u16 tid)
+{
+	struct bnxt_en_dev *en_dev = rdev->en_dev;
+	struct hwrm_stat_ctx_free_input req = {0};
+	struct hwrm_stat_ctx_free_output resp;
+	struct bnxt_fw_msg fw_msg;
+	int rc = -EINVAL;
+
+	if (!en_dev)
+		return rc;
+
+	/* To avoid unnecessary error messages during recovery.
+	 * HW is anyway in error state. So dont send down the command */
+	if (test_bit(BNXT_RE_FLAG_ERR_DEVICE_DETACHED, &rdev->flags))
+		return 0;
+	memset(&fw_msg, 0, sizeof(fw_msg));
+	bnxt_re_init_hwrm_hdr(rdev, (void *)&req, HWRM_STAT_CTX_FREE, -1, tid);
+	req.stat_ctx_id = cpu_to_le32(fw_stats_ctx_id);
+	bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp,
+			    sizeof(resp), DFLT_HWRM_CMD_TIMEOUT);
+	rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg);
+	if (rc) {
+		dev_err(rdev_to_dev(rdev),
+			"Failed to free HW stats ctx with rc = 0x%x", rc);
+		return rc;
+	}
+	dev_dbg(rdev_to_dev(rdev),
+		"HW stats ctx freed with id = 0x%x", fw_stats_ctx_id);
+
+	return rc;
+}
+
+static int bnxt_re_net_stats_ctx_alloc(struct bnxt_re_dev *rdev, u16 tid)
+{
+	struct hwrm_stat_ctx_alloc_output resp = {};
+	struct hwrm_stat_ctx_alloc_input req = {};
+	struct bnxt_en_dev *en_dev = rdev->en_dev;
+	struct bnxt_qplib_stats *stat;
+	struct bnxt_qplib_ctx *hctx;
+	struct bnxt_fw_msg fw_msg;
+	int rc = 0;
+
+	hctx = rdev->qplib_res.hctx;
+	stat = (tid == 0xffff) ? &hctx->stats : &hctx->stats2;
+	stat->fw_id = INVALID_STATS_CTX_ID;
+
+	if (!en_dev)
+		return -EINVAL;
+
+	memset(&fw_msg, 0, sizeof(fw_msg));
+	bnxt_re_init_hwrm_hdr(rdev, (void *)&req,
+			      HWRM_STAT_CTX_ALLOC, -1, tid);
+	req.update_period_ms = cpu_to_le32(1000);
+	req.stats_dma_length = rdev->chip_ctx->hw_stats_size;
+	req.stats_dma_addr = cpu_to_le64(stat->dma_map);
+	req.stat_ctx_flags = HWRM_STAT_CTX_ALLOC_INPUT_STAT_CTX_FLAGS_ROCE;
+	bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp,
+			    sizeof(resp), DFLT_HWRM_CMD_TIMEOUT);
+	rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg);
+	if (rc) {
+		dev_err(rdev_to_dev(rdev),
+			"Failed to allocate HW stats ctx, rc = 0x%x", rc);
+		return rc;
+	}
+	stat->fw_id = le32_to_cpu(resp.stat_ctx_id);
+	dev_dbg(rdev_to_dev(rdev), "HW stats ctx allocated with id = 0x%x",
+		stat->fw_id);
+
+	return rc;
+}
+
+static void bnxt_re_net_unregister_async_event(struct bnxt_re_dev *rdev)
+{
+	const struct bnxt_en_ops *en_ops;
+
+	if (rdev->is_virtfn ||
+	    test_bit(BNXT_RE_FLAG_ERR_DEVICE_DETACHED, &rdev->flags))
+		return;
+
+	memset(rdev->event_bitmap, 0, sizeof(rdev->event_bitmap));
+	en_ops = rdev->en_dev->en_ops;
+	if (en_ops->bnxt_register_fw_async_events
+	    (rdev->en_dev, BNXT_ROCE_ULP,
+	     (unsigned long *)rdev->event_bitmap,
+	      HWRM_ASYNC_EVENT_CMPL_EVENT_ID_DOORBELL_PACING_NQ_UPDATE))
+		dev_err(rdev_to_dev(rdev),
+			"Failed to unregister async event");
+}
+
+static void bnxt_re_net_register_async_event(struct bnxt_re_dev *rdev)
+{
+	const struct bnxt_en_ops *en_ops;
+
+	if (rdev->is_virtfn)
+		return;
+
+	rdev->event_bitmap[0] |=
+		BIT(HWRM_ASYNC_EVENT_CMPL_EVENT_ID_DCB_CONFIG_CHANGE) |
+		BIT(HWRM_ASYNC_EVENT_CMPL_EVENT_ID_RESET_NOTIFY);
+
+	rdev->event_bitmap[2] |=
+	   BIT(HWRM_ASYNC_EVENT_CMPL_EVENT_ID_ERROR_REPORT - 64);
+	rdev->event_bitmap[2] |=
+		BIT(HWRM_ASYNC_EVENT_CMPL_EVENT_ID_DOORBELL_PACING_THRESHOLD - 64) |
+		BIT(HWRM_ASYNC_EVENT_CMPL_EVENT_ID_DOORBELL_PACING_NQ_UPDATE - 64);
+	en_ops = rdev->en_dev->en_ops;
+	if (en_ops->bnxt_register_fw_async_events
+	    (rdev->en_dev, BNXT_ROCE_ULP,
+	     (unsigned long *)rdev->event_bitmap,
+	      HWRM_ASYNC_EVENT_CMPL_EVENT_ID_DOORBELL_PACING_NQ_UPDATE))
+		dev_err(rdev_to_dev(rdev),
+			"Failed to reg Async event");
+}
+
+static int bnxt_re_query_hwrm_intf_version(struct bnxt_re_dev *rdev)
+{
+	struct bnxt_en_dev *en_dev = rdev->en_dev;
+	struct hwrm_ver_get_output resp = {0};
+	struct hwrm_ver_get_input req = {0};
+	struct bnxt_qplib_chip_ctx *cctx;
+	struct bnxt_fw_msg fw_msg;
+	int rc = 0;
+
+	memset(&fw_msg, 0, sizeof(fw_msg));
+	bnxt_re_init_hwrm_hdr(rdev, (void *)&req,
+			      HWRM_VER_GET, -1, -1);
+	req.hwrm_intf_maj = HWRM_VERSION_MAJOR;
+	req.hwrm_intf_min = HWRM_VERSION_MINOR;
+	req.hwrm_intf_upd = HWRM_VERSION_UPDATE;
+	bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp,
+			    sizeof(resp), DFLT_HWRM_CMD_TIMEOUT);
+	rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg);
+	if (rc) {
+		dev_err(rdev_to_dev(rdev),
+			"Failed to query HW version, rc = 0x%x", rc);
+		return rc;
+	}
+	cctx = rdev->chip_ctx;
+	cctx->hwrm_intf_ver = (u64) le16_to_cpu(resp.hwrm_intf_major) << 48 |
+			      (u64) le16_to_cpu(resp.hwrm_intf_minor) << 32 |
+			      (u64) le16_to_cpu(resp.hwrm_intf_build) << 16 |
+				    le16_to_cpu(resp.hwrm_intf_patch);
+
+	cctx->hwrm_cmd_max_timeout = le16_to_cpu(resp.max_req_timeout);
+
+	if (!cctx->hwrm_cmd_max_timeout)
+		cctx->hwrm_cmd_max_timeout = RCFW_FW_STALL_MAX_TIMEOUT;
+
+	cctx->chip_num = le16_to_cpu(resp.chip_num);
+	cctx->chip_rev = resp.chip_rev;
+	cctx->chip_metal = resp.chip_metal;
+	return 0;
+}
+
+/* Query device config using common hwrm */
+static int bnxt_re_hwrm_qcfg(struct bnxt_re_dev *rdev, u32 *db_len,
+			     u32 *offset)
+{
+	struct bnxt_en_dev *en_dev = rdev->en_dev;
+	struct hwrm_func_qcfg_output resp = {0};
+	struct hwrm_func_qcfg_input req = {0};
+	struct bnxt_fw_msg fw_msg;
+	int rc;
+
+	memset(&fw_msg, 0, sizeof(fw_msg));
+	bnxt_re_init_hwrm_hdr(rdev, (void *)&req,
+			      HWRM_FUNC_QCFG, -1, -1);
+	req.fid = cpu_to_le16(0xffff);
+	bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp,
+			    sizeof(resp), DFLT_HWRM_CMD_TIMEOUT);
+	rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg);
+	if (rc) {
+		dev_err(rdev_to_dev(rdev),
+			"Failed to query config, rc = %#x", rc);
+		return rc;
+	}
+
+	*db_len = PAGE_ALIGN(le16_to_cpu(resp.l2_doorbell_bar_size_kb) * 1024);
+	*offset = PAGE_ALIGN(le16_to_cpu(resp.legacy_l2_db_size_kb) * 1024);
+	return 0;
+}
+
+/* Query function capabilities using common hwrm */
+int bnxt_re_hwrm_qcaps(struct bnxt_re_dev *rdev)
+{
+	struct bnxt_en_dev *en_dev = rdev->en_dev;
+	struct hwrm_func_qcaps_output resp = {0};
+	struct hwrm_func_qcaps_input req = {0};
+	struct bnxt_qplib_chip_ctx *cctx;
+	struct bnxt_fw_msg fw_msg;
+	u8 push_enable = false;
+	int rc;
+
+	cctx = rdev->chip_ctx;
+	memset(&fw_msg, 0, sizeof(fw_msg));
+	bnxt_re_init_hwrm_hdr(rdev, (void *)&req,
+			      HWRM_FUNC_QCAPS, -1, -1);
+	req.fid = cpu_to_le16(0xffff);
+	bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp,
+			    sizeof(resp), DFLT_HWRM_CMD_TIMEOUT);
+	rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg);
+	if (rc) {
+		dev_err(rdev_to_dev(rdev),
+			"Failed to query capabilities, rc = %#x", rc);
+		return rc;
+	}
+	if (_is_chip_p7(rdev->chip_ctx))
+		push_enable =
+			(resp.flags_ext &
+			 HWRM_FUNC_QCAPS_OUTPUT_FLAGS_EXT_PPP_PUSH_MODE_SUPPORTED) ?
+			 true : false;
+	else
+		push_enable =
+			(resp.flags & HWRM_FUNC_QCAPS_OUTPUT_FLAGS_WCB_PUSH_MODE) ?
+			 true : false;
+	cctx->modes.db_push = push_enable;
+
+	cctx->modes.dbr_pacing =
+		resp.flags_ext & HWRM_FUNC_QCAPS_OUTPUT_FLAGS_EXT_DBR_PACING_SUPPORTED ?
+			true : false;
+	cctx->modes.dbr_pacing_ext =
+		resp.flags_ext2 &
+			HWRM_FUNC_QCAPS_OUTPUT_FLAGS_EXT2_DBR_PACING_EXT_SUPPORTED ?
+			true : false;
+	cctx->modes.dbr_drop_recov =
+		(resp.flags_ext2 &
+		 HWRM_FUNC_QCAPS_OUTPUT_FLAGS_EXT2_SW_DBR_DROP_RECOVERY_SUPPORTED) ?
+			true : false;
+	cctx->modes.dbr_pacing_v0 =
+		(resp.flags_ext2 &
+		 HWRM_FUNC_QCAPS_OUTPUT_FLAGS_EXT2_DBR_PACING_V0_SUPPORTED) ?
+			true : false;
+	dev_dbg(rdev_to_dev(rdev),
+		"%s: cctx->modes.dbr_pacing = %d cctx->modes.dbr_pacing_ext = %d, dbr_drop_recov %d\n",
+		__func__, cctx->modes.dbr_pacing, cctx->modes.dbr_pacing_ext, cctx->modes.dbr_drop_recov);
+
+	return 0;
+}
+
+static int bnxt_re_hwrm_dbr_pacing_qcfg(struct bnxt_re_dev *rdev)
+{
+	struct bnxt_qplib_db_pacing_data *pacing_data = rdev->qplib_res.pacing_data;
+	struct hwrm_func_dbr_pacing_qcfg_output resp = {0};
+	struct hwrm_func_dbr_pacing_qcfg_input req = {0};
+	struct bnxt_en_dev *en_dev = rdev->en_dev;
+	struct bnxt_qplib_chip_ctx *cctx;
+	struct bnxt_fw_msg fw_msg;
+	u32 primary_nq_id;
+	int rc;
+
+	cctx = rdev->chip_ctx;
+	memset(&fw_msg, 0, sizeof(fw_msg));
+	bnxt_re_init_hwrm_hdr(rdev, (void *)&req,
+			      HWRM_FUNC_DBR_PACING_QCFG, -1, -1);
+	bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp,
+			    sizeof(resp), BNXT_RE_HWRM_CMD_TIMEOUT(rdev));
+	rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg);
+	if (rc) {
+		dev_dbg(rdev_to_dev(rdev),
+			"Failed to query dbr pacing config, rc = %#x", rc);
+		return rc;
+	}
+
+	primary_nq_id = le32_to_cpu(resp.primary_nq_id);
+	if (primary_nq_id == 0xffffffff &&
+	    !bnxt_qplib_dbr_pacing_ext_en(rdev->chip_ctx)) {
+		dev_err(rdev_to_dev(rdev), "%s:%d Invoke bnxt_qplib_dbr_pacing_set_primary_pf with 1\n",
+			__func__, __LINE__);
+		bnxt_qplib_dbr_pacing_set_primary_pf(rdev->chip_ctx, 1);
+	}
+
+	if (bnxt_qplib_dbr_pacing_ext_en(rdev->chip_ctx)) {
+		struct bnxt_qplib_nq *nq;
+
+		nq = &rdev->nqr.nq[0];
+		/* Reset the primary capability */
+		if (nq->ring_id != primary_nq_id)
+			bnxt_qplib_dbr_pacing_set_primary_pf(rdev->chip_ctx, 0);
+	}
+
+	if ((resp.dbr_stat_db_fifo_reg &
+	     HWRM_FUNC_DBR_PACING_QCFG_OUTPUT_DBR_STAT_DB_FIFO_REG_ADDR_SPACE_MASK) ==
+	    HWRM_FUNC_DBR_PACING_QCFG_OUTPUT_DBR_STAT_DB_FIFO_REG_ADDR_SPACE_GRC)
+		cctx->dbr_stat_db_fifo =
+		resp.dbr_stat_db_fifo_reg &
+		~HWRM_FUNC_DBR_PACING_QCFG_OUTPUT_DBR_STAT_DB_FIFO_REG_ADDR_SPACE_MASK;
+
+	if ((resp.dbr_throttling_aeq_arm_reg &
+	    HWRM_FUNC_DBR_PACING_QCFG_OUTPUT_DBR_THROTTLING_AEQ_ARM_REG_ADDR_SPACE_MASK)
+	    == HWRM_FUNC_DBR_PACING_QCFG_OUTPUT_DBR_THROTTLING_AEQ_ARM_REG_ADDR_SPACE_GRC) {
+		cctx->dbr_aeq_arm_reg = resp.dbr_throttling_aeq_arm_reg &
+			~HWRM_FUNC_DBR_PACING_QCFG_OUTPUT_DBR_STAT_DB_FIFO_REG_ADDR_SPACE_MASK;
+		cctx->dbr_throttling_reg = cctx->dbr_aeq_arm_reg - 4;
+	}
+	pacing_data->fifo_max_depth = le32_to_cpu(resp.dbr_stat_db_max_fifo_depth);
+	if (!pacing_data->fifo_max_depth)
+		pacing_data->fifo_max_depth = BNXT_RE_MAX_FIFO_DEPTH(cctx);
+	pacing_data->fifo_room_mask = le32_to_cpu(resp.dbr_stat_db_fifo_reg_fifo_room_mask);
+	pacing_data->fifo_room_shift = resp.dbr_stat_db_fifo_reg_fifo_room_shift;
+	dev_dbg(rdev_to_dev(rdev),
+		"%s: nq:0x%x primary_pf:%d db_fifo:0x%x aeq_arm:0x%x i"
+		"fifo_max_depth 0x%x , resp.dbr_stat_db_max_fifo_depth 0x%x);\n",
+		__func__, resp.primary_nq_id, cctx->modes.dbr_primary_pf,
+		 cctx->dbr_stat_db_fifo, cctx->dbr_aeq_arm_reg,
+		 pacing_data->fifo_max_depth,
+		le32_to_cpu(resp.dbr_stat_db_max_fifo_depth));
+	return 0;
+}
+
+static int bnxt_re_hwrm_dbr_pacing_cfg(struct bnxt_re_dev *rdev, bool enable)
+{
+	struct hwrm_func_dbr_pacing_cfg_output resp = {0};
+	struct hwrm_func_dbr_pacing_cfg_input req = {0};
+	struct bnxt_en_dev *en_dev = rdev->en_dev;
+	struct bnxt_fw_msg fw_msg;
+	int rc;
+
+	if (test_bit(BNXT_RE_FLAG_ERR_DEVICE_DETACHED, &rdev->flags))
+		return 0;
+
+	memset(&fw_msg, 0, sizeof(fw_msg));
+	bnxt_re_init_hwrm_hdr(rdev, (void *)&req,
+			      HWRM_FUNC_DBR_PACING_CFG, -1, -1);
+	if (enable) {
+		req.flags = HWRM_FUNC_DBR_PACING_CFG_INPUT_FLAGS_DBR_NQ_EVENT_ENABLE;
+		req.enables =
+		cpu_to_le32(HWRM_FUNC_DBR_PACING_CFG_INPUT_ENABLES_PRIMARY_NQ_ID_VALID |
+			    HWRM_FUNC_DBR_PACING_CFG_INPUT_ENABLES_PACING_THRESHOLD_VALID);
+	} else {
+		req.flags = HWRM_FUNC_DBR_PACING_CFG_INPUT_FLAGS_DBR_NQ_EVENT_DISABLE;
+	}
+	req.primary_nq_id = cpu_to_le32(rdev->dbq_nq_id);
+	req.pacing_threshold = cpu_to_le32(rdev->dbq_watermark);
+	dev_dbg(rdev_to_dev(rdev), "%s: nq_id = 0x%x pacing_threshold = 0x%x",
+		__func__, req.primary_nq_id, req.pacing_threshold);
+	bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp,
+			    sizeof(resp), BNXT_RE_HWRM_CMD_TIMEOUT(rdev));
+	rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg);
+	if (rc) {
+		dev_dbg(rdev_to_dev(rdev),
+			"Failed to set dbr pacing config, rc = %#x", rc);
+		return rc;
+	}
+	return 0;
+}
+
+/* Net -> RoCE driver */
+
+/* Device */
+struct bnxt_re_dev *bnxt_re_from_netdev(struct ifnet *netdev)
+{
+	struct bnxt_re_dev *rdev;
+
+	rcu_read_lock();
+	list_for_each_entry_rcu(rdev, &bnxt_re_dev_list, list) {
+		if (rdev->netdev == netdev) {
+			rcu_read_unlock();
+			dev_dbg(rdev_to_dev(rdev),
+				"netdev (%p) found, ref_count = 0x%x",
+				netdev, atomic_read(&rdev->ref_count));
+			return rdev;
+		}
+	}
+	rcu_read_unlock();
+	return NULL;
+}
+
+static ssize_t show_rev(struct device *device, struct device_attribute *attr,
+			char *buf)
+{
+	struct bnxt_re_dev *rdev = to_bnxt_re_dev(device, ibdev.dev);
+
+	return scnprintf(buf, PAGE_SIZE, "0x%x\n", rdev->en_dev->pdev->vendor);
+}
+
+
+static ssize_t show_hca(struct device *device, struct device_attribute *attr,
+			char *buf)
+{
+	struct bnxt_re_dev *rdev = to_bnxt_re_dev(device, ibdev.dev);
+
+	return scnprintf(buf, PAGE_SIZE, "%s\n", rdev->ibdev.node_desc);
+}
+
+static DEVICE_ATTR(hw_rev, 0444, show_rev, NULL);
+static DEVICE_ATTR(hca_type, 0444, show_hca, NULL);
+static struct device_attribute *bnxt_re_attributes[] = {
+	&dev_attr_hw_rev,
+	&dev_attr_hca_type
+};
+
+int ib_register_device_compat(struct bnxt_re_dev *rdev)
+{
+	struct ib_device *ibdev = &rdev->ibdev;
+	char name[IB_DEVICE_NAME_MAX];
+
+	memset(name, 0, IB_DEVICE_NAME_MAX);
+	strlcpy(name, "bnxt_re%d", IB_DEVICE_NAME_MAX);
+
+	strlcpy(ibdev->name, name, IB_DEVICE_NAME_MAX);
+
+	return ib_register_device(ibdev, NULL);
+}
+
+static int bnxt_re_register_ib(struct bnxt_re_dev *rdev)
+{
+	struct ib_device *ibdev = &rdev->ibdev;
+	int ret = 0;
+
+	/* ib device init */
+	ibdev->owner = THIS_MODULE;
+	ibdev->uverbs_abi_ver = BNXT_RE_ABI_VERSION;
+	ibdev->node_type = RDMA_NODE_IB_CA;
+	strlcpy(ibdev->node_desc, BNXT_RE_DESC " HCA",
+		strlen(BNXT_RE_DESC) + 5);
+	ibdev->phys_port_cnt = 1;
+
+	bnxt_qplib_get_guid(rdev->dev_addr, (u8 *)&ibdev->node_guid);
+
+	/* Data path irqs is one less than the max msix vectors */
+	ibdev->num_comp_vectors	= rdev->nqr.num_msix - 1;
+	bnxt_re_set_dma_device(ibdev, rdev);
+	ibdev->local_dma_lkey = BNXT_QPLIB_RSVD_LKEY;
+
+	/* User space */
+	ibdev->uverbs_cmd_mask =
+			(1ull << IB_USER_VERBS_CMD_GET_CONTEXT)		|
+			(1ull << IB_USER_VERBS_CMD_QUERY_DEVICE)	|
+			(1ull << IB_USER_VERBS_CMD_QUERY_PORT)		|
+			(1ull << IB_USER_VERBS_CMD_ALLOC_PD)		|
+			(1ull << IB_USER_VERBS_CMD_DEALLOC_PD)		|
+			(1ull << IB_USER_VERBS_CMD_REG_MR)		|
+			(1ull << IB_USER_VERBS_CMD_DEREG_MR)		|
+			(1ull << IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL) |
+			(1ull << IB_USER_VERBS_CMD_CREATE_CQ)		|
+			(1ull << IB_USER_VERBS_CMD_DESTROY_CQ)		|
+			(1ull << IB_USER_VERBS_CMD_CREATE_QP)		|
+			(1ull << IB_USER_VERBS_CMD_MODIFY_QP)		|
+			(1ull << IB_USER_VERBS_CMD_QUERY_QP)		|
+			(1ull << IB_USER_VERBS_CMD_DESTROY_QP)		|
+			(1ull << IB_USER_VERBS_CMD_REREG_MR)		|
+			(1ull << IB_USER_VERBS_CMD_RESIZE_CQ)		|
+			(1ull << IB_USER_VERBS_CMD_CREATE_SRQ)		|
+			(1ull << IB_USER_VERBS_CMD_MODIFY_SRQ)		|
+			(1ull << IB_USER_VERBS_CMD_QUERY_SRQ)		|
+			(1ull << IB_USER_VERBS_CMD_DESTROY_SRQ)		|
+			(1ull << IB_USER_VERBS_CMD_ALLOC_MW)		|
+			(1ull << IB_USER_VERBS_CMD_DEALLOC_MW)		|
+			(1ull << IB_USER_VERBS_CMD_CREATE_AH)		|
+			(1ull << IB_USER_VERBS_CMD_MODIFY_AH)		|
+			(1ull << IB_USER_VERBS_CMD_QUERY_AH)		|
+			(1ull << IB_USER_VERBS_CMD_DESTROY_AH);
+
+	ibdev->uverbs_ex_cmd_mask = (1ull << IB_USER_VERBS_EX_CMD_MODIFY_QP);
+	ibdev->uverbs_cmd_mask |= (1ull << IB_USER_VERBS_CMD_POLL_CQ);
+
+#define bnxt_re_ib_ah bnxt_re_ah
+#define bnxt_re_ib_cq bnxt_re_cq
+#define bnxt_re_ib_pd bnxt_re_pd
+#define bnxt_re_ib_srq bnxt_re_srq
+#define bnxt_re_ib_ucontext bnxt_re_ucontext
+	INIT_IB_DEVICE_OPS(&ibdev->ops, bnxt_re, BNXT_RE);
+
+	ibdev->query_device		= bnxt_re_query_device;
+	ibdev->modify_device		= bnxt_re_modify_device;
+	ibdev->query_port		= bnxt_re_query_port;
+	ibdev->modify_port		= bnxt_re_modify_port;
+	ibdev->get_port_immutable	= bnxt_re_get_port_immutable;
+	ibdev->query_pkey		= bnxt_re_query_pkey;
+	ibdev->query_gid		= bnxt_re_query_gid;
+	ibdev->get_netdev		= bnxt_re_get_netdev;
+	ibdev->add_gid			= bnxt_re_add_gid;
+	ibdev->del_gid			= bnxt_re_del_gid;
+	ibdev->get_link_layer		= bnxt_re_get_link_layer;
+	ibdev->alloc_pd			= bnxt_re_alloc_pd;
+	ibdev->dealloc_pd		= bnxt_re_dealloc_pd;
+	ibdev->create_ah		= bnxt_re_create_ah;
+	ibdev->modify_ah		= bnxt_re_modify_ah;
+	ibdev->query_ah			= bnxt_re_query_ah;
+	ibdev->destroy_ah		= bnxt_re_destroy_ah;
+	ibdev->create_srq		= bnxt_re_create_srq;
+	ibdev->modify_srq		= bnxt_re_modify_srq;
+	ibdev->query_srq		= bnxt_re_query_srq;
+	ibdev->destroy_srq		= bnxt_re_destroy_srq;
+	ibdev->post_srq_recv		= bnxt_re_post_srq_recv;
+	ibdev->create_qp		= bnxt_re_create_qp;
+	ibdev->modify_qp		= bnxt_re_modify_qp;
+	ibdev->query_qp			= bnxt_re_query_qp;
+	ibdev->destroy_qp		= bnxt_re_destroy_qp;
+	ibdev->post_send		= bnxt_re_post_send;
+	ibdev->post_recv		= bnxt_re_post_recv;
+	ibdev->create_cq		= bnxt_re_create_cq;
+	ibdev->modify_cq		= bnxt_re_modify_cq;
+	ibdev->destroy_cq		= bnxt_re_destroy_cq;
+	ibdev->resize_cq		= bnxt_re_resize_cq;
+	ibdev->poll_cq			= bnxt_re_poll_cq;
+	ibdev->req_notify_cq		= bnxt_re_req_notify_cq;
+	ibdev->get_dma_mr		= bnxt_re_get_dma_mr;
+	ibdev->get_hw_stats		= bnxt_re_get_hw_stats;
+	ibdev->alloc_hw_stats		= bnxt_re_alloc_hw_port_stats;
+	ibdev->dereg_mr			= bnxt_re_dereg_mr;
+	ibdev->alloc_mr			= bnxt_re_alloc_mr;
+	ibdev->map_mr_sg		= bnxt_re_map_mr_sg;
+	ibdev->alloc_mw			= bnxt_re_alloc_mw;
+	ibdev->dealloc_mw		= bnxt_re_dealloc_mw;
+	ibdev->reg_user_mr		= bnxt_re_reg_user_mr;
+	ibdev->rereg_user_mr		= bnxt_re_rereg_user_mr;
+	ibdev->disassociate_ucontext	= bnxt_re_disassociate_ucntx;
+	ibdev->alloc_ucontext		= bnxt_re_alloc_ucontext;
+	ibdev->dealloc_ucontext		= bnxt_re_dealloc_ucontext;
+	ibdev->mmap			= bnxt_re_mmap;
+	ibdev->process_mad		= bnxt_re_process_mad;
+
+	ret = ib_register_device_compat(rdev);
+	return ret;
+}
+
+static void bnxt_re_dev_dealloc(struct bnxt_re_dev *rdev)
+{
+	int i = BNXT_RE_REF_WAIT_COUNT;
+
+	dev_dbg(rdev_to_dev(rdev), "%s:Remove the device %p\n", __func__, rdev);
+	/* Wait for rdev refcount to come down */
+	while ((atomic_read(&rdev->ref_count) > 1) && i--)
+		msleep(100);
+
+	if (atomic_read(&rdev->ref_count) > 1)
+		dev_err(rdev_to_dev(rdev),
+			"Failed waiting for ref count to deplete %d",
+			atomic_read(&rdev->ref_count));
+
+	atomic_set(&rdev->ref_count, 0);
+	if_rele(rdev->netdev);
+	rdev->netdev = NULL;
+	synchronize_rcu();
+
+	kfree(rdev->gid_map);
+	kfree(rdev->dbg_stats);
+	ib_dealloc_device(&rdev->ibdev);
+}
+
+static struct bnxt_re_dev *bnxt_re_dev_alloc(struct ifnet *netdev,
+					   struct bnxt_en_dev *en_dev)
+{
+	struct bnxt_re_dev *rdev;
+	u32 count;
+
+	/* Allocate bnxt_re_dev instance here */
+	rdev = (struct bnxt_re_dev *)compat_ib_alloc_device(sizeof(*rdev));
+	if (!rdev) {
+		pr_err("%s: bnxt_re_dev allocation failure!",
+			ROCE_DRV_MODULE_NAME);
+		return NULL;
+	}
+	/* Default values */
+	atomic_set(&rdev->ref_count, 0);
+	rdev->netdev = netdev;
+	dev_hold(rdev->netdev);
+	rdev->en_dev = en_dev;
+	rdev->id = rdev->en_dev->pdev->devfn;
+	INIT_LIST_HEAD(&rdev->qp_list);
+	mutex_init(&rdev->qp_lock);
+	mutex_init(&rdev->cc_lock);
+	mutex_init(&rdev->dbq_lock);
+	bnxt_re_clear_rsors_stat(&rdev->stats.rsors);
+	rdev->cosq[0] = rdev->cosq[1] = 0xFFFF;
+	rdev->min_tx_depth = 1;
+	rdev->stats.stats_query_sec = 1;
+	/* Disable priority vlan as the default mode is DSCP based PFC */
+	rdev->cc_param.disable_prio_vlan_tx = 1;
+
+	/* Initialize worker for DBR Pacing */
+	INIT_WORK(&rdev->dbq_fifo_check_work, bnxt_re_db_fifo_check);
+	INIT_DELAYED_WORK(&rdev->dbq_pacing_work, bnxt_re_pacing_timer_exp);
+	rdev->gid_map = kzalloc(sizeof(*(rdev->gid_map)) *
+				  BNXT_RE_MAX_SGID_ENTRIES,
+				  GFP_KERNEL);
+	if (!rdev->gid_map) {
+		ib_dealloc_device(&rdev->ibdev);
+		return NULL;
+	}
+	for(count = 0; count < BNXT_RE_MAX_SGID_ENTRIES; count++)
+		rdev->gid_map[count] = -1;
+
+	rdev->dbg_stats = kzalloc(sizeof(*rdev->dbg_stats), GFP_KERNEL);
+	if (!rdev->dbg_stats) {
+		ib_dealloc_device(&rdev->ibdev);
+		return NULL;
+	}
+
+	return rdev;
+}
+
+static int bnxt_re_handle_unaffi_async_event(
+		struct creq_func_event *unaffi_async)
+{
+	switch (unaffi_async->event) {
+	case CREQ_FUNC_EVENT_EVENT_TX_WQE_ERROR:
+	case CREQ_FUNC_EVENT_EVENT_TX_DATA_ERROR:
+	case CREQ_FUNC_EVENT_EVENT_RX_WQE_ERROR:
+	case CREQ_FUNC_EVENT_EVENT_RX_DATA_ERROR:
+	case CREQ_FUNC_EVENT_EVENT_CQ_ERROR:
+	case CREQ_FUNC_EVENT_EVENT_TQM_ERROR:
+	case CREQ_FUNC_EVENT_EVENT_CFCQ_ERROR:
+	case CREQ_FUNC_EVENT_EVENT_CFCS_ERROR:
+	case CREQ_FUNC_EVENT_EVENT_CFCC_ERROR:
+	case CREQ_FUNC_EVENT_EVENT_CFCM_ERROR:
+	case CREQ_FUNC_EVENT_EVENT_TIM_ERROR:
+		break;
+	default:
+		return -EINVAL;
+	}
+	return 0;
+}
+
+static int bnxt_re_handle_qp_async_event(void *qp_event, struct bnxt_re_qp *qp)
+{
+	struct creq_qp_error_notification *err_event;
+	struct ib_event event;
+	unsigned int flags;
+
+	if (qp->qplib_qp.state == CMDQ_MODIFY_QP_NEW_STATE_ERR &&
+	    !qp->qplib_qp.is_user) {
+		flags = bnxt_re_lock_cqs(qp);
+		bnxt_qplib_add_flush_qp(&qp->qplib_qp);
+		bnxt_re_unlock_cqs(qp, flags);
+	}
+	memset(&event, 0, sizeof(event));
+	event.device = &qp->rdev->ibdev;
+	event.element.qp = &qp->ib_qp;
+	event.event = IB_EVENT_QP_FATAL;
+
+	err_event = qp_event;
+	switch(err_event->res_err_state_reason) {
+	case CFCQ_RES_ERR_STATE_REASON_RES_EXCEED_MAX:
+	case CFCQ_RES_ERR_STATE_REASON_RES_PAYLOAD_LENGTH_MISMATCH:
+	case CFCQ_RES_ERR_STATE_REASON_RES_OPCODE_ERROR:
+	case CFCQ_RES_ERR_STATE_REASON_RES_PSN_SEQ_ERROR_RETRY_LIMIT:
+	case CFCQ_RES_ERR_STATE_REASON_RES_RX_INVALID_R_KEY:
+	case CFCQ_RES_ERR_STATE_REASON_RES_RX_DOMAIN_ERROR:
+	case CFCQ_RES_ERR_STATE_REASON_RES_RX_NO_PERMISSION:
+	case CFCQ_RES_ERR_STATE_REASON_RES_RX_RANGE_ERROR:
+	case CFCQ_RES_ERR_STATE_REASON_RES_TX_INVALID_R_KEY:
+	case CFCQ_RES_ERR_STATE_REASON_RES_TX_DOMAIN_ERROR:
+	case CFCQ_RES_ERR_STATE_REASON_RES_TX_NO_PERMISSION:
+	case CFCQ_RES_ERR_STATE_REASON_RES_TX_RANGE_ERROR:
+	case CFCQ_RES_ERR_STATE_REASON_RES_IVALID_DUP_RKEY:
+	case CFCQ_RES_ERR_STATE_REASON_RES_UNALIGN_ATOMIC:
+		event.event = IB_EVENT_QP_ACCESS_ERR;
+		break;
+	case CFCQ_RES_ERR_STATE_REASON_RES_EXCEEDS_WQE:
+	case CFCQ_RES_ERR_STATE_REASON_RES_WQE_FORMAT_ERROR:
+	case CFCQ_RES_ERR_STATE_REASON_RES_SRQ_LOAD_ERROR:
+	case CFCQ_RES_ERR_STATE_REASON_RES_UNSUPPORTED_OPCODE:
+	case CFCQ_RES_ERR_STATE_REASON_RES_REM_INVALIDATE:
+		event.event = IB_EVENT_QP_REQ_ERR;
+		break;
+	case CFCQ_RES_ERR_STATE_REASON_RES_IRRQ_OFLOW:
+	case CFCQ_RES_ERR_STATE_REASON_RES_CMP_ERROR:
+	case CFCQ_RES_ERR_STATE_REASON_RES_CQ_LOAD_ERROR:
+	case CFCQ_RES_ERR_STATE_REASON_RES_TX_PCI_ERROR:
+	case CFCQ_RES_ERR_STATE_REASON_RES_RX_PCI_ERROR:
+	case CFCQ_RES_ERR_STATE_REASON_RES_MEMORY_ERROR:
+	case CFCQ_RES_ERR_STATE_REASON_RES_SRQ_ERROR:
+		event.event = IB_EVENT_QP_FATAL;
+		break;
+	default:
+		if (qp->qplib_qp.srq)
+			event.event = IB_EVENT_QP_LAST_WQE_REACHED;
+		break;
+	}
+
+	if (err_event->res_err_state_reason)
+		dev_err(rdev_to_dev(qp->rdev),
+			"%s %s qp_id: %d cons (%d %d) req (%d %d) res (%d %d)\n",
+			__func__,  qp->qplib_qp.is_user ? "user" : "kernel",
+			qp->qplib_qp.id,
+			err_event->sq_cons_idx,
+			err_event->rq_cons_idx,
+			err_event->req_slow_path_state,
+			err_event->req_err_state_reason,
+			err_event->res_slow_path_state,
+			err_event->res_err_state_reason);
+
+	if (event.device && qp->ib_qp.event_handler)
+		qp->ib_qp.event_handler(&event, qp->ib_qp.qp_context);
+
+	return 0;
+}
+
+static int bnxt_re_handle_cq_async_error(void *event, struct bnxt_re_cq *cq)
+{
+	struct creq_cq_error_notification *cqerr;
+	bool send = false;
+
+	cqerr = event;
+	switch (cqerr->cq_err_reason) {
+	case CREQ_CQ_ERROR_NOTIFICATION_CQ_ERR_REASON_REQ_CQ_INVALID_ERROR:
+	case CREQ_CQ_ERROR_NOTIFICATION_CQ_ERR_REASON_REQ_CQ_OVERFLOW_ERROR:
+	case CREQ_CQ_ERROR_NOTIFICATION_CQ_ERR_REASON_REQ_CQ_LOAD_ERROR:
+	case CREQ_CQ_ERROR_NOTIFICATION_CQ_ERR_REASON_RES_CQ_INVALID_ERROR:
+	case CREQ_CQ_ERROR_NOTIFICATION_CQ_ERR_REASON_RES_CQ_OVERFLOW_ERROR:
+	case CREQ_CQ_ERROR_NOTIFICATION_CQ_ERR_REASON_RES_CQ_LOAD_ERROR:
+		send = true;
+	default:
+		break;
+	}
+
+	if (send && cq->ibcq.event_handler) {
+		struct ib_event ibevent = {};
+
+		ibevent.event = IB_EVENT_CQ_ERR;
+		ibevent.element.cq = &cq->ibcq;
+		ibevent.device = &cq->rdev->ibdev;
+
+		dev_err(rdev_to_dev(cq->rdev),
+			"%s err reason %d\n", __func__, cqerr->cq_err_reason);
+		cq->ibcq.event_handler(&ibevent, cq->ibcq.cq_context);
+	}
+
+	cq->qplib_cq.is_cq_err_event = true;
+
+	return 0;
+}
+
+static int bnxt_re_handle_affi_async_event(struct creq_qp_event *affi_async,
+					   void *obj)
+{
+	struct bnxt_qplib_qp *qplqp;
+	struct bnxt_qplib_cq *qplcq;
+	struct bnxt_re_qp *qp;
+	struct bnxt_re_cq *cq;
+	int rc = 0;
+	u8 event;
+
+	if (!obj)
+		return rc; /* QP was already dead, still return success */
+
+	event = affi_async->event;
+	switch (event) {
+	case CREQ_QP_EVENT_EVENT_QP_ERROR_NOTIFICATION:
+		qplqp = obj;
+		qp = container_of(qplqp, struct bnxt_re_qp, qplib_qp);
+		rc = bnxt_re_handle_qp_async_event(affi_async, qp);
+		break;
+	case CREQ_QP_EVENT_EVENT_CQ_ERROR_NOTIFICATION:
+		qplcq = obj;
+		cq = container_of(qplcq, struct bnxt_re_cq, qplib_cq);
+		rc = bnxt_re_handle_cq_async_error(affi_async, cq);
+		break;
+	default:
+		rc = -EINVAL;
+	}
+
+	return rc;
+}
+
+static int bnxt_re_aeq_handler(struct bnxt_qplib_rcfw *rcfw,
+			       void *aeqe, void *obj)
+{
+	struct creq_func_event *unaffi_async;
+	struct creq_qp_event *affi_async;
+	u8 type;
+	int rc;
+
+	type = ((struct creq_base *)aeqe)->type;
+	if (type == CREQ_BASE_TYPE_FUNC_EVENT) {
+		unaffi_async = aeqe;
+		rc = bnxt_re_handle_unaffi_async_event(unaffi_async);
+	} else {
+		affi_async = aeqe;
+		rc = bnxt_re_handle_affi_async_event(affi_async, obj);
+	}
+
+	return rc;
+}
+
+static int bnxt_re_srqn_handler(struct bnxt_qplib_nq *nq,
+				struct bnxt_qplib_srq *handle, u8 event)
+{
+	struct bnxt_re_srq *srq = to_bnxt_re(handle, struct bnxt_re_srq,
+					     qplib_srq);
+	struct ib_event ib_event;
+
+	if (srq == NULL) {
+		pr_err("%s: SRQ is NULL, SRQN not handled",
+			ROCE_DRV_MODULE_NAME);
+		return -EINVAL;
+	}
+	ib_event.device = &srq->rdev->ibdev;
+	ib_event.element.srq = &srq->ibsrq;
+	if (event == NQ_SRQ_EVENT_EVENT_SRQ_THRESHOLD_EVENT)
+		ib_event.event = IB_EVENT_SRQ_LIMIT_REACHED;
+	else
+		ib_event.event = IB_EVENT_SRQ_ERR;
+
+	if (srq->ibsrq.event_handler) {
+		/* Lock event_handler? */
+		(*srq->ibsrq.event_handler)(&ib_event,
+					     srq->ibsrq.srq_context);
+	}
+	return 0;
+}
+
+static int bnxt_re_cqn_handler(struct bnxt_qplib_nq *nq,
+			       struct bnxt_qplib_cq *handle)
+{
+	struct bnxt_re_cq *cq = to_bnxt_re(handle, struct bnxt_re_cq,
+					   qplib_cq);
+	u32 *cq_ptr;
+
+	if (cq == NULL) {
+		pr_err("%s: CQ is NULL, CQN not handled",
+			ROCE_DRV_MODULE_NAME);
+		return -EINVAL;
+	}
+	/* CQ already in destroy path. Do not handle any more events */
+	if (handle->destroyed || !atomic_read(&cq->ibcq.usecnt)) {
+		if (!handle->destroyed)
+			dev_dbg(NULL, "%s: CQ being destroyed, CQN not handled",
+				ROCE_DRV_MODULE_NAME);
+		return 0;
+	}
+
+	if (cq->ibcq.comp_handler) {
+		if (cq->uctx_cq_page) {
+			cq_ptr = (u32 *)cq->uctx_cq_page;
+			*cq_ptr = cq->qplib_cq.toggle;
+		}
+		/* Lock comp_handler? */
+		(*cq->ibcq.comp_handler)(&cq->ibcq, cq->ibcq.cq_context);
+	}
+
+	return 0;
+}
+
+struct bnxt_qplib_nq *bnxt_re_get_nq(struct bnxt_re_dev *rdev)
+{
+	int min, indx;
+
+	mutex_lock(&rdev->nqr.load_lock);
+	for (indx = 0, min = 0; indx < (rdev->nqr.num_msix - 1); indx++) {
+		if (rdev->nqr.nq[min].load > rdev->nqr.nq[indx].load)
+			min = indx;
+	}
+	rdev->nqr.nq[min].load++;
+	mutex_unlock(&rdev->nqr.load_lock);
+
+	return &rdev->nqr.nq[min];
+}
+
+void bnxt_re_put_nq(struct bnxt_re_dev *rdev, struct bnxt_qplib_nq *nq)
+{
+	mutex_lock(&rdev->nqr.load_lock);
+	nq->load--;
+	mutex_unlock(&rdev->nqr.load_lock);
+}
+
+static bool bnxt_re_check_min_attr(struct bnxt_re_dev *rdev)
+{
+	struct bnxt_qplib_dev_attr *attr;
+	bool rc = true;
+
+	attr = rdev->dev_attr;
+
+	if (!attr->max_cq || !attr->max_qp ||
+	    !attr->max_sgid || !attr->max_mr) {
+		dev_err(rdev_to_dev(rdev),"Insufficient RoCE resources");
+		dev_dbg(rdev_to_dev(rdev),
+			"max_cq = %d, max_qp = %d, max_dpi = %d, max_sgid = %d, max_mr = %d",
+			attr->max_cq, attr->max_qp, attr->max_dpi,
+			attr->max_sgid, attr->max_mr);
+		rc = false;
+	}
+	return rc;
+}
+
+static void bnxt_re_dispatch_event(struct ib_device *ibdev, struct ib_qp *qp,
+				   u8 port_num, enum ib_event_type event)
+{
+	struct ib_event ib_event;
+
+	ib_event.device = ibdev;
+	if (qp) {
+		ib_event.element.qp = qp;
+		ib_event.event = event;
+		if (qp->event_handler)
+			qp->event_handler(&ib_event, qp->qp_context);
+	} else {
+		ib_event.element.port_num = port_num;
+		ib_event.event = event;
+		ib_dispatch_event(&ib_event);
+	}
+
+	dev_dbg(rdev_to_dev(to_bnxt_re_dev(ibdev, ibdev)),
+		"ibdev %p Event 0x%x port_num 0x%x", ibdev, event, port_num);
+}
+
+static bool bnxt_re_is_qp1_or_shadow_qp(struct bnxt_re_dev *rdev,
+					struct bnxt_re_qp *qp)
+{
+	if (rdev->gsi_ctx.gsi_qp_mode == BNXT_RE_GSI_MODE_ALL)
+		return (qp->ib_qp.qp_type == IB_QPT_GSI) ||
+			(qp == rdev->gsi_ctx.gsi_sqp);
+	else
+		return (qp->ib_qp.qp_type == IB_QPT_GSI);
+}
+
+static void bnxt_re_stop_all_nonqp1_nonshadow_qps(struct bnxt_re_dev *rdev)
+{
+	struct bnxt_qplib_qp *qpl_qp;
+	bool dev_detached = false;
+	struct ib_qp_attr qp_attr;
+	int num_qps_stopped = 0;
+	int mask = IB_QP_STATE;
+	struct bnxt_re_qp *qp;
+	unsigned long flags;
+
+	if (!rdev)
+		return;
+
+restart:
+	if (test_bit(BNXT_RE_FLAG_ERR_DEVICE_DETACHED, &rdev->flags))
+		dev_detached = true;
+
+	qp_attr.qp_state = IB_QPS_ERR;
+	mutex_lock(&rdev->qp_lock);
+	list_for_each_entry(qp, &rdev->qp_list, list) {
+		qpl_qp = &qp->qplib_qp;
+		if (dev_detached || !bnxt_re_is_qp1_or_shadow_qp(rdev, qp)) {
+			if (qpl_qp->state !=
+			    CMDQ_MODIFY_QP_NEW_STATE_RESET &&
+			    qpl_qp->state !=
+			    CMDQ_MODIFY_QP_NEW_STATE_ERR) {
+				if (dev_detached) {
+					/*
+					 * Cant actually send the command down,
+					 * marking the state for bookkeeping
+					 */
+					qpl_qp->state =
+						CMDQ_MODIFY_QP_NEW_STATE_ERR;
+					qpl_qp->cur_qp_state = qpl_qp->state;
+					if (!qpl_qp->is_user) {
+						/* Add to flush list */
+						flags = bnxt_re_lock_cqs(qp);
+						bnxt_qplib_add_flush_qp(qpl_qp);
+						bnxt_re_unlock_cqs(qp, flags);
+					}
+				} else {
+					num_qps_stopped++;
+					bnxt_re_modify_qp(&qp->ib_qp,
+							  &qp_attr, mask,
+							  NULL);
+				}
+
+				bnxt_re_dispatch_event(&rdev->ibdev, &qp->ib_qp,
+						       1, IB_EVENT_QP_FATAL);
+				/*
+				 * 1. Release qp_lock after a budget to unblock other verb
+				 *    requests (like qp_destroy) from stack.
+				 * 2. Traverse through the qp_list freshly as addition / deletion
+				 *    might have happened since qp_lock is getting released here.
+				 */
+				if (num_qps_stopped % BNXT_RE_STOP_QPS_BUDGET == 0) {
+					mutex_unlock(&rdev->qp_lock);
+					goto restart;
+				}
+			}
+		}
+	}
+
+	mutex_unlock(&rdev->qp_lock);
+}
+
+static int bnxt_re_update_gid(struct bnxt_re_dev *rdev)
+{
+	struct bnxt_qplib_sgid_tbl *sgid_tbl = &rdev->qplib_res.sgid_tbl;
+	struct bnxt_qplib_gid gid;
+	u16 gid_idx, index;
+	int rc = 0;
+
+	if (!test_bit(BNXT_RE_FLAG_IBDEV_REGISTERED, &rdev->flags))
+		return 0;
+
+	if (sgid_tbl == NULL) {
+		dev_err(rdev_to_dev(rdev), "QPLIB: SGID table not allocated");
+		return -EINVAL;
+	}
+
+	for (index = 0; index < sgid_tbl->active; index++) {
+		gid_idx = sgid_tbl->hw_id[index];
+
+		if (!memcmp(&sgid_tbl->tbl[index], &bnxt_qplib_gid_zero,
+			    sizeof(bnxt_qplib_gid_zero)))
+			continue;
+		/* Need to modify the VLAN enable setting of non VLAN GID only
+		 * as setting is done for VLAN GID while adding GID
+		 *
+		 * If disable_prio_vlan_tx is enable, then we'll need to remove the
+		 * vlan entry from the sgid_tbl.
+		 */
+		if (sgid_tbl->vlan[index] == true)
+			continue;
+
+		memcpy(&gid, &sgid_tbl->tbl[index], sizeof(gid));
+
+		rc = bnxt_qplib_update_sgid(sgid_tbl, &gid, gid_idx,
+					    rdev->dev_addr);
+	}
+
+	return rc;
+}
+
+static void bnxt_re_clear_cc(struct bnxt_re_dev *rdev)
+{
+	struct bnxt_qplib_cc_param *cc_param = &rdev->cc_param;
+
+	if (_is_chip_p7(rdev->chip_ctx)) {
+		cc_param->mask = CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_TOS_DSCP;
+	} else {
+		cc_param->mask = (CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_CC_MODE |
+				  CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_ENABLE_CC |
+				  CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_TOS_ECN);
+
+		if (!is_qport_service_type_supported(rdev))
+			cc_param->mask |=
+			(CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_ALT_VLAN_PCP |
+			 CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_ALT_TOS_DSCP |
+			 CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_TOS_DSCP);
+	}
+
+	cc_param->cur_mask  = cc_param->mask;
+
+	if (bnxt_qplib_modify_cc(&rdev->qplib_res, cc_param))
+		dev_err(rdev_to_dev(rdev), "Failed to modify cc\n");
+}
+
+static int bnxt_re_setup_cc(struct bnxt_re_dev *rdev)
+{
+	struct bnxt_qplib_cc_param *cc_param = &rdev->cc_param;
+	int rc;
+
+	if (_is_chip_p7(rdev->chip_ctx)) {
+		cc_param->enable = 0x0;
+		cc_param->mask = CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_TOS_DSCP;
+	} else {
+		cc_param->enable = 0x1;
+		cc_param->mask = (CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_CC_MODE |
+				  CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_ENABLE_CC |
+				  CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_TOS_ECN);
+
+		if (!is_qport_service_type_supported(rdev))
+			cc_param->mask |=
+			(CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_ALT_VLAN_PCP |
+			 CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_ALT_TOS_DSCP |
+			 CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_TOS_DSCP);
+	}
+
+	cc_param->cur_mask  = cc_param->mask;
+
+	rc = bnxt_qplib_modify_cc(&rdev->qplib_res, cc_param);
+	if (rc) {
+		dev_err(rdev_to_dev(rdev), "Failed to modify cc\n");
+		return rc;
+	}
+	/* Reset the programming mask */
+	cc_param->mask = 0;
+	if (cc_param->qp1_tos_dscp != cc_param->tos_dscp) {
+		cc_param->qp1_tos_dscp = cc_param->tos_dscp;
+		rc = bnxt_re_update_qp1_tos_dscp(rdev);
+		if (rc) {
+			dev_err(rdev_to_dev(rdev), "%s:Failed to modify QP1:%d",
+				__func__, rc);
+			goto clear;
+		}
+	}
+	return 0;
+
+clear:
+	bnxt_re_clear_cc(rdev);
+	return rc;
+}
+
+int bnxt_re_query_hwrm_dscp2pri(struct bnxt_re_dev *rdev,
+				struct bnxt_re_dscp2pri *d2p, u16 *count,
+				u16 target_id)
+{
+	struct bnxt_en_dev *en_dev = rdev->en_dev;
+	struct hwrm_queue_dscp2pri_qcfg_input req;
+	struct hwrm_queue_dscp2pri_qcfg_output resp;
+	struct bnxt_re_dscp2pri *dscp2pri;
+	struct bnxt_fw_msg fw_msg;
+	u16 in_count = *count;
+	dma_addr_t dma_handle;
+	int rc = 0, i;
+	u16 data_len;
+	u8 *kmem;
+
+	data_len = *count * sizeof(*dscp2pri);
+	memset(&fw_msg, 0, sizeof(fw_msg));
+	memset(&req, 0, sizeof(req));
+	bnxt_re_init_hwrm_hdr(rdev, (void *)&req,
+			      HWRM_QUEUE_DSCP2PRI_QCFG, -1, target_id);
+	req.port_id = (target_id == 0xFFFF) ? en_dev->pf_port_id : 1;
+
+	kmem = dma_zalloc_coherent(&en_dev->pdev->dev, data_len, &dma_handle,
+				   GFP_KERNEL);
+	if (!kmem) {
+		dev_err(rdev_to_dev(rdev),
+			"dma_zalloc_coherent failure, length = %u\n",
+			(unsigned)data_len);
+		return -ENOMEM;
+	}
+	req.dest_data_addr = cpu_to_le64(dma_handle);
+	req.dest_data_buffer_size = cpu_to_le16(data_len);
+	bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp,
+			    sizeof(resp), DFLT_HWRM_CMD_TIMEOUT);
+	rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg);
+	if (rc)
+		goto out;
+
+	/* Upload the DSCP-MASK-PRI tuple(s) */
+	dscp2pri = (struct bnxt_re_dscp2pri *)kmem;
+	for (i = 0; i < le16_to_cpu(resp.entry_cnt) && i < in_count; i++) {
+		d2p[i].dscp = dscp2pri->dscp;
+		d2p[i].mask = dscp2pri->mask;
+		d2p[i].pri = dscp2pri->pri;
+		dscp2pri++;
+	}
+	*count = le16_to_cpu(resp.entry_cnt);
+out:
+	dma_free_coherent(&en_dev->pdev->dev, data_len, kmem, dma_handle);
+	return rc;
+}
+
+int bnxt_re_prio_vlan_tx_update(struct bnxt_re_dev *rdev)
+{
+	/* Remove the VLAN from the GID entry */
+	if (rdev->cc_param.disable_prio_vlan_tx)
+		rdev->qplib_res.prio = false;
+	else
+		rdev->qplib_res.prio = true;
+
+	return bnxt_re_update_gid(rdev);
+}
+
+int bnxt_re_set_hwrm_dscp2pri(struct bnxt_re_dev *rdev,
+			      struct bnxt_re_dscp2pri *d2p, u16 count,
+			      u16 target_id)
+{
+	struct bnxt_en_dev *en_dev = rdev->en_dev;
+	struct hwrm_queue_dscp2pri_cfg_input req;
+	struct hwrm_queue_dscp2pri_cfg_output resp;
+	struct bnxt_fw_msg fw_msg;
+	struct bnxt_re_dscp2pri *dscp2pri;
+	int i, rc, data_len = 3 * 256;
+	dma_addr_t dma_handle;
+	u8 *kmem;
+
+	memset(&req, 0, sizeof(req));
+	memset(&fw_msg, 0, sizeof(fw_msg));
+	bnxt_re_init_hwrm_hdr(rdev, (void *)&req,
+			      HWRM_QUEUE_DSCP2PRI_CFG, -1, target_id);
+	req.port_id = (target_id == 0xFFFF) ? en_dev->pf_port_id : 1;
+
+	kmem = dma_alloc_coherent(&en_dev->pdev->dev, data_len, &dma_handle,
+				  GFP_KERNEL);
+	if (!kmem) {
+		dev_err(rdev_to_dev(rdev),
+			"dma_alloc_coherent failure, length = %u\n",
+			(unsigned)data_len);
+		return -ENOMEM;
+	}
+	req.src_data_addr = cpu_to_le64(dma_handle);
+
+	/* Download the DSCP-MASK-PRI tuple(s) */
+	dscp2pri = (struct bnxt_re_dscp2pri *)kmem;
+	for (i = 0; i < count; i++) {
+		dscp2pri->dscp = d2p[i].dscp;
+		dscp2pri->mask = d2p[i].mask;
+		dscp2pri->pri = d2p[i].pri;
+		dscp2pri++;
+	}
+
+	req.entry_cnt = cpu_to_le16(count);
+	bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp,
+			    sizeof(resp), DFLT_HWRM_CMD_TIMEOUT);
+	rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg);
+	dma_free_coherent(&en_dev->pdev->dev, data_len, kmem, dma_handle);
+	return rc;
+}
+
+int bnxt_re_query_hwrm_qportcfg(struct bnxt_re_dev *rdev,
+			struct bnxt_re_tc_rec *tc_rec, u16 tid)
+{
+	u8 max_tc, tc, *qptr, *type_ptr0, *type_ptr1;
+	struct hwrm_queue_qportcfg_output resp = {0};
+	struct hwrm_queue_qportcfg_input req = {0};
+	struct bnxt_en_dev *en_dev = rdev->en_dev;
+	struct bnxt_fw_msg fw_msg;
+	bool def_init = false;
+	u8 *tmp_type;
+	u8 cos_id;
+	int rc;
+
+	memset(&fw_msg, 0, sizeof(fw_msg));
+	bnxt_re_init_hwrm_hdr(rdev, (void *)&req, HWRM_QUEUE_QPORTCFG,
+			      -1, tid);
+	req.port_id = (tid == 0xFFFF) ? en_dev->pf_port_id : 1;
+	if (BNXT_EN_ASYM_Q(en_dev))
+		req.flags = htole32(HWRM_QUEUE_QPORTCFG_INPUT_FLAGS_PATH_RX);
+
+	bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp,
+			    sizeof(resp), DFLT_HWRM_CMD_TIMEOUT);
+	rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg);
+	if (rc)
+		return rc;
+
+	if (!resp.max_configurable_queues)
+		return -EINVAL;
+
+	max_tc = resp.max_configurable_queues;
+	tc_rec->max_tc = max_tc;
+
+	if (resp.queue_cfg_info & HWRM_QUEUE_QPORTCFG_OUTPUT_QUEUE_CFG_INFO_USE_PROFILE_TYPE)
+		tc_rec->serv_type_enabled = true;
+
+	qptr = &resp.queue_id0;
+	type_ptr0 = &resp.queue_id0_service_profile_type;
+	type_ptr1 = &resp.queue_id1_service_profile_type;
+	for (tc = 0; tc < max_tc; tc++) {
+		tmp_type = tc ? type_ptr1 + (tc - 1) : type_ptr0;
+
+		cos_id = *qptr++;
+		/* RoCE CoS queue is the first cos queue.
+		 * For MP12 and MP17 order is 405 and 141015.
+		 */
+		if (is_bnxt_roce_queue(rdev, *qptr, *tmp_type)) {
+			tc_rec->cos_id_roce = cos_id;
+			tc_rec->tc_roce = tc;
+		} else if (is_bnxt_cnp_queue(rdev, *qptr, *tmp_type)) {
+			tc_rec->cos_id_cnp = cos_id;
+			tc_rec->tc_cnp = tc;
+		} else if (!def_init) {
+			def_init = true;
+			tc_rec->tc_def = tc;
+			tc_rec->cos_id_def = cos_id;
+		}
+		qptr++;
+	}
+
+	return rc;
+}
+
+int bnxt_re_hwrm_cos2bw_qcfg(struct bnxt_re_dev *rdev, u16 target_id,
+			     struct bnxt_re_cos2bw_cfg *cfg)
+{
+	struct bnxt_en_dev *en_dev = rdev->en_dev;
+	struct hwrm_queue_cos2bw_qcfg_output resp;
+	struct hwrm_queue_cos2bw_qcfg_input req = {0};
+	struct bnxt_fw_msg fw_msg;
+	int rc, indx;
+	void *data;
+
+	memset(&fw_msg, 0, sizeof(fw_msg));
+	bnxt_re_init_hwrm_hdr(rdev, (void *)&req,
+			      HWRM_QUEUE_COS2BW_QCFG, -1, target_id);
+	req.port_id = (target_id == 0xFFFF) ? en_dev->pf_port_id : 1;
+
+	bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp,
+			    sizeof(resp), DFLT_HWRM_CMD_TIMEOUT);
+	rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg);
+	if (rc)
+		return rc;
+	data = &resp.queue_id0 + offsetof(struct bnxt_re_cos2bw_cfg,
+					  queue_id);
+	for (indx = 0; indx < 8; indx++, data += (sizeof(cfg->cfg))) {
+		memcpy(&cfg->cfg, data, sizeof(cfg->cfg));
+		if (indx == 0)
+			cfg->queue_id = resp.queue_id0;
+		cfg++;
+	}
+
+	return rc;
+}
+
+int bnxt_re_hwrm_cos2bw_cfg(struct bnxt_re_dev *rdev, u16 target_id,
+			    struct bnxt_re_cos2bw_cfg *cfg)
+{
+	struct bnxt_en_dev *en_dev = rdev->en_dev;
+	struct hwrm_queue_cos2bw_cfg_input req = {0};
+	struct hwrm_queue_cos2bw_cfg_output resp = {0};
+	struct bnxt_fw_msg fw_msg;
+	void *data;
+	int indx;
+	int rc;
+
+	memset(&fw_msg, 0, sizeof(fw_msg));
+	bnxt_re_init_hwrm_hdr(rdev, (void *)&req,
+			      HWRM_QUEUE_COS2BW_CFG, -1, target_id);
+	req.port_id = (target_id == 0xFFFF) ? en_dev->pf_port_id : 1;
+
+	/* Chimp wants enable bit to retain previous
+	 * config done by L2 driver
+	 */
+	for (indx = 0; indx < 8; indx++) {
+		if (cfg[indx].queue_id < 40) {
+			req.enables |= cpu_to_le32(
+				HWRM_QUEUE_COS2BW_CFG_INPUT_ENABLES_COS_QUEUE_ID0_VALID <<
+				indx);
+		}
+
+		data = (char *)&req.unused_0 + indx * (sizeof(*cfg) - 4);
+		memcpy(data, &cfg[indx].queue_id, sizeof(*cfg) - 4);
+		if (indx == 0) {
+			req.queue_id0 = cfg[0].queue_id;
+			req.unused_0 = 0;
+		}
+	}
+
+	memset(&resp, 0, sizeof(resp));
+	bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp,
+			    sizeof(resp), DFLT_HWRM_CMD_TIMEOUT);
+	rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg);
+	return rc;
+}
+
+int bnxt_re_host_pf_id_query(struct bnxt_re_dev *rdev,
+			     struct bnxt_qplib_query_fn_info *fn_info,
+			     u32 *pf_mask, u32 *first_pf)
+{
+	struct hwrm_func_host_pf_ids_query_output resp = {0};
+	struct hwrm_func_host_pf_ids_query_input req;
+	struct bnxt_en_dev *en_dev = rdev->en_dev;
+	struct bnxt_fw_msg fw_msg;
+	int rc;
+
+	memset(&fw_msg, 0, sizeof(fw_msg));
+	memset(&req, 0, sizeof(req));
+	bnxt_re_init_hwrm_hdr(rdev, (void *)&req,
+			      HWRM_FUNC_HOST_PF_IDS_QUERY, -1, -1);
+	/* To query the info from the host EPs */
+	switch (fn_info->host) {
+		case HWRM_FUNC_HOST_PF_IDS_QUERY_INPUT_HOST_SOC:
+		case HWRM_FUNC_HOST_PF_IDS_QUERY_INPUT_HOST_EP_0:
+		case HWRM_FUNC_HOST_PF_IDS_QUERY_INPUT_HOST_EP_1:
+		case HWRM_FUNC_HOST_PF_IDS_QUERY_INPUT_HOST_EP_2:
+		case HWRM_FUNC_HOST_PF_IDS_QUERY_INPUT_HOST_EP_3:
+			req.host = fn_info->host;
+		break;
+		default:
+			req.host = HWRM_FUNC_HOST_PF_IDS_QUERY_INPUT_HOST_EP_0;
+		break;
+	}
+
+	req.filter = fn_info->filter;
+	if (req.filter > HWRM_FUNC_HOST_PF_IDS_QUERY_INPUT_FILTER_ROCE)
+		req.filter = HWRM_FUNC_HOST_PF_IDS_QUERY_INPUT_FILTER_ALL;
+
+	bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp,
+			    sizeof(resp), DFLT_HWRM_CMD_TIMEOUT);
+	rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg);
+
+
+	*first_pf = le16_to_cpu(resp.first_pf_id);
+	*pf_mask = le16_to_cpu(resp.pf_ordinal_mask);
+
+	return rc;
+}
+
+static void bnxt_re_put_stats_ctx(struct bnxt_re_dev *rdev)
+{
+	struct bnxt_qplib_ctx *hctx;
+	struct bnxt_qplib_res *res;
+	u16 tid = 0xffff;
+
+	res = &rdev->qplib_res;
+	hctx = res->hctx;
+
+	if (test_and_clear_bit(BNXT_RE_FLAG_STATS_CTX_ALLOC, &rdev->flags)) {
+		bnxt_re_net_stats_ctx_free(rdev, hctx->stats.fw_id, tid);
+		bnxt_qplib_free_stat_mem(res, &hctx->stats);
+	}
+}
+
+static void bnxt_re_put_stats2_ctx(struct bnxt_re_dev *rdev)
+{
+	test_and_clear_bit(BNXT_RE_FLAG_STATS_CTX2_ALLOC, &rdev->flags);
+}
+
+static int bnxt_re_get_stats_ctx(struct bnxt_re_dev *rdev)
+{
+	struct bnxt_qplib_ctx *hctx;
+	struct bnxt_qplib_res *res;
+	u16 tid = 0xffff;
+	int rc;
+
+	res = &rdev->qplib_res;
+	hctx = res->hctx;
+
+	rc = bnxt_qplib_alloc_stat_mem(res->pdev, rdev->chip_ctx, &hctx->stats);
+	if (rc)
+		return -ENOMEM;
+	rc = bnxt_re_net_stats_ctx_alloc(rdev, tid);
+	if (rc)
+		goto free_stat_mem;
+	set_bit(BNXT_RE_FLAG_STATS_CTX_ALLOC, &rdev->flags);
+
+	return 0;
+
+free_stat_mem:
+	bnxt_qplib_free_stat_mem(res, &hctx->stats);
+
+	return rc;
+}
+
+static int bnxt_re_update_dev_attr(struct bnxt_re_dev *rdev)
+{
+	int rc;
+
+	rc = bnxt_qplib_get_dev_attr(&rdev->rcfw);
+	if (rc)
+		return rc;
+	if (!bnxt_re_check_min_attr(rdev))
+		return -EINVAL;
+	return 0;
+}
+
+static void bnxt_re_free_tbls(struct bnxt_re_dev *rdev)
+{
+	bnxt_qplib_clear_tbls(&rdev->qplib_res);
+	bnxt_qplib_free_tbls(&rdev->qplib_res);
+}
+
+static int bnxt_re_alloc_init_tbls(struct bnxt_re_dev *rdev)
+{
+	struct bnxt_qplib_chip_ctx *chip_ctx = rdev->chip_ctx;
+	u8 pppp_factor = 0;
+	int rc;
+
+	 /*
+	  * TODO: Need a better mechanism for spreading of the
+	  * 512 extended PPP pages. For now, spreading it
+	  * based on port_count
+	  */
+	if (_is_chip_p7(chip_ctx) && chip_ctx->modes.db_push)
+		pppp_factor = rdev->en_dev->port_count;
+	rc = bnxt_qplib_alloc_tbls(&rdev->qplib_res, pppp_factor);
+	if (rc)
+		return rc;
+	bnxt_qplib_init_tbls(&rdev->qplib_res);
+	set_bit(BNXT_RE_FLAG_TBLS_ALLOCINIT, &rdev->flags);
+
+	return 0;
+}
+
+static void bnxt_re_clean_nqs(struct bnxt_re_dev *rdev)
+{
+	struct bnxt_qplib_nq *nq;
+	int i;
+
+	if (!rdev->nqr.max_init)
+		return;
+
+	for (i = (rdev->nqr.max_init - 1) ; i >= 0; i--) {
+		nq = &rdev->nqr.nq[i];
+		bnxt_qplib_disable_nq(nq);
+		bnxt_re_net_ring_free(rdev, nq->ring_id);
+		bnxt_qplib_free_nq_mem(nq);
+	}
+	rdev->nqr.max_init = 0;
+}
+
+static int bnxt_re_setup_nqs(struct bnxt_re_dev *rdev)
+{
+	struct bnxt_re_ring_attr rattr = {};
+	struct bnxt_qplib_nq *nq;
+	int rc, i;
+	int depth;
+	u32 offt;
+	u16 vec;
+
+	mutex_init(&rdev->nqr.load_lock);
+	/*
+	 * TODO: Optimize the depth based on the
+	 * number of NQs.
+	 */
+	depth = BNXT_QPLIB_NQE_MAX_CNT;
+	for (i = 0; i < rdev->nqr.num_msix - 1; i++) {
+		nq = &rdev->nqr.nq[i];
+		vec = rdev->nqr.msix_entries[i + 1].vector;
+		offt = rdev->nqr.msix_entries[i + 1].db_offset;
+		nq->hwq.max_elements = depth;
+		rc = bnxt_qplib_alloc_nq_mem(&rdev->qplib_res, nq);
+		if (rc) {
+			dev_err(rdev_to_dev(rdev),
+				"Failed to get mem for NQ %d, rc = 0x%x",
+				i, rc);
+			goto fail_mem;
+		}
+
+		rattr.dma_arr = nq->hwq.pbl[PBL_LVL_0].pg_map_arr;
+		rattr.pages = nq->hwq.pbl[rdev->nqr.nq[i].hwq.level].pg_count;
+		rattr.type = bnxt_re_get_rtype(rdev);
+		rattr.mode = HWRM_RING_ALLOC_INPUT_INT_MODE_MSIX;
+		rattr.depth = nq->hwq.max_elements - 1;
+		rattr.lrid = rdev->nqr.msix_entries[i + 1].ring_idx;
+
+		/* Set DBR pacing capability on the first NQ ring only */
+		if (!i && bnxt_qplib_dbr_pacing_ext_en(rdev->chip_ctx))
+			rattr.flags = HWRM_RING_ALLOC_INPUT_FLAGS_NQ_DBR_PACING;
+		else
+			rattr.flags = 0;
+
+		rc = bnxt_re_net_ring_alloc(rdev, &rattr, &nq->ring_id);
+		if (rc) {
+			nq->ring_id = 0xffff; /* Invalid ring-id */
+			dev_err(rdev_to_dev(rdev),
+				"Failed to get fw id for NQ %d, rc = 0x%x",
+				i, rc);
+			goto fail_ring;
+		}
+
+		rc = bnxt_qplib_enable_nq(nq, i, vec, offt,
+					  &bnxt_re_cqn_handler,
+					  &bnxt_re_srqn_handler);
+		if (rc) {
+			dev_err(rdev_to_dev(rdev),
+				"Failed to enable NQ %d, rc = 0x%x", i, rc);
+			goto fail_en;
+		}
+	}
+
+	rdev->nqr.max_init = i;
+	return 0;
+fail_en:
+	/* *nq was i'th nq */
+	bnxt_re_net_ring_free(rdev, nq->ring_id);
+fail_ring:
+	bnxt_qplib_free_nq_mem(nq);
+fail_mem:
+	rdev->nqr.max_init = i;
+	return rc;
+}
+
+static void bnxt_re_sysfs_destroy_file(struct bnxt_re_dev *rdev)
+{
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(bnxt_re_attributes); i++)
+		device_remove_file(&rdev->ibdev.dev, bnxt_re_attributes[i]);
+}
+
+static int bnxt_re_sysfs_create_file(struct bnxt_re_dev *rdev)
+{
+	int i, j, rc = 0;
+
+	for (i = 0; i < ARRAY_SIZE(bnxt_re_attributes); i++) {
+		rc = device_create_file(&rdev->ibdev.dev,
+					bnxt_re_attributes[i]);
+		if (rc) {
+			dev_err(rdev_to_dev(rdev),
+				"Failed to create IB sysfs with rc = 0x%x", rc);
+			/* Must clean up all created device files */
+			for (j = 0; j < i; j++)
+				device_remove_file(&rdev->ibdev.dev,
+						   bnxt_re_attributes[j]);
+			clear_bit(BNXT_RE_FLAG_IBDEV_REGISTERED, &rdev->flags);
+			ib_unregister_device(&rdev->ibdev);
+			return 1;
+		}
+	}
+	return 0;
+}
+
+/* worker thread for polling periodic events. Now used for QoS programming*/
+static void bnxt_re_worker(struct work_struct *work)
+{
+	struct bnxt_re_dev *rdev = container_of(work, struct bnxt_re_dev,
+						worker.work);
+	int rc;
+
+	/* QoS is in 30s cadence for PFs*/
+	if (!rdev->is_virtfn && !rdev->worker_30s--)
+		rdev->worker_30s = 30;
+	/* Use trylock for  bnxt_re_dev_lock as this can be
+	 * held for long time by debugfs show path while issuing
+	 * HWRMS. If the debugfs name update is not done in this
+	 * iteration, the driver will check for the same in the
+	 * next schedule of the worker i.e after 1 sec.
+	 */
+	if (mutex_trylock(&bnxt_re_dev_lock))
+		mutex_unlock(&bnxt_re_dev_lock);
+
+	if (!rdev->stats.stats_query_sec)
+		goto resched;
+
+	if (test_bit(BNXT_RE_FLAG_ISSUE_CFA_FLOW_STATS, &rdev->flags) &&
+	    (rdev->is_virtfn ||
+	    !_is_ext_stats_supported(rdev->dev_attr->dev_cap_flags))) {
+		if (!(rdev->stats.stats_query_counter++ %
+		      rdev->stats.stats_query_sec)) {
+			rc = bnxt_re_get_qos_stats(rdev);
+			if (rc && rc != -ENOMEM)
+				clear_bit(BNXT_RE_FLAG_ISSUE_CFA_FLOW_STATS,
+					  &rdev->flags);
+			}
+	}
+
+resched:
+	schedule_delayed_work(&rdev->worker, msecs_to_jiffies(1000));
+}
+
+static int bnxt_re_alloc_dbr_sw_stats_mem(struct bnxt_re_dev *rdev)
+{
+	if (!(rdev->dbr_drop_recov || rdev->dbr_pacing))
+		return 0;
+
+	rdev->dbr_sw_stats = kzalloc(sizeof(*rdev->dbr_sw_stats), GFP_KERNEL);
+	if (!rdev->dbr_sw_stats)
+		return -ENOMEM;
+
+	return 0;
+}
+
+static void bnxt_re_free_dbr_sw_stats_mem(struct bnxt_re_dev *rdev)
+{
+	kfree(rdev->dbr_sw_stats);
+	rdev->dbr_sw_stats = NULL;
+}
+
+static int bnxt_re_initialize_dbr_drop_recov(struct bnxt_re_dev *rdev)
+{
+	rdev->dbr_drop_recov_wq =
+		create_singlethread_workqueue("bnxt_re_dbr_drop_recov");
+	if (!rdev->dbr_drop_recov_wq) {
+		dev_err(rdev_to_dev(rdev), "DBR Drop Revov wq alloc failed!");
+		return -EINVAL;
+	}
+	rdev->dbr_drop_recov = true;
+
+	/* Enable configfs setting dbr_drop_recov by default*/
+	rdev->user_dbr_drop_recov = true;
+
+	rdev->user_dbr_drop_recov_timeout = BNXT_RE_DBR_RECOV_USERLAND_TIMEOUT;
+	return 0;
+}
+
+static void bnxt_re_deinitialize_dbr_drop_recov(struct bnxt_re_dev *rdev)
+{
+	if (rdev->dbr_drop_recov_wq) {
+		flush_workqueue(rdev->dbr_drop_recov_wq);
+		destroy_workqueue(rdev->dbr_drop_recov_wq);
+		rdev->dbr_drop_recov_wq = NULL;
+	}
+	rdev->dbr_drop_recov = false;
+}
+
+static int bnxt_re_initialize_dbr_pacing(struct bnxt_re_dev *rdev)
+{
+	int rc;
+
+	/* Allocate a page for app use */
+	rdev->dbr_page = (void *)__get_free_page(GFP_KERNEL);
+	if (!rdev->dbr_page) {
+		dev_err(rdev_to_dev(rdev), "DBR page allocation failed!");
+		return -ENOMEM;
+	}
+	memset((u8 *)rdev->dbr_page, 0, PAGE_SIZE);
+	rdev->qplib_res.pacing_data = (struct bnxt_qplib_db_pacing_data *)rdev->dbr_page;
+	rc = bnxt_re_hwrm_dbr_pacing_qcfg(rdev);
+	if (rc) {
+		dev_err(rdev_to_dev(rdev),
+			"Failed to query dbr pacing config %d\n", rc);
+		goto fail;
+	}
+	/* Create a work queue for scheduling dbq event */
+	rdev->dbq_wq = create_singlethread_workqueue("bnxt_re_dbq");
+	if (!rdev->dbq_wq) {
+		dev_err(rdev_to_dev(rdev), "DBQ wq alloc failed!");
+		rc = -ENOMEM;
+		goto fail;
+	}
+	/* MAP grc window 2 for reading db fifo depth */
+	writel_fbsd(rdev->en_dev->softc,  BNXT_GRCPF_REG_WINDOW_BASE_OUT + 4, 0,
+			rdev->chip_ctx->dbr_stat_db_fifo & BNXT_GRC_BASE_MASK);
+	rdev->dbr_db_fifo_reg_off =
+		(rdev->chip_ctx->dbr_stat_db_fifo & BNXT_GRC_OFFSET_MASK) +
+		0x2000;
+	rdev->qplib_res.pacing_data->grc_reg_offset = rdev->dbr_db_fifo_reg_off;
+
+	rdev->dbr_bar_addr =
+		pci_resource_start(rdev->qplib_res.pdev, 0) +
+		rdev->dbr_db_fifo_reg_off;
+
+	/* Percentage of DB FIFO */
+	rdev->dbq_watermark = BNXT_RE_PACING_DBQ_THRESHOLD;
+	rdev->pacing_en_int_th = BNXT_RE_PACING_EN_INT_THRESHOLD;
+	rdev->pacing_algo_th = BNXT_RE_PACING_ALGO_THRESHOLD;
+	rdev->dbq_pacing_time = BNXT_RE_DBR_INT_TIME;
+	rdev->dbr_def_do_pacing = BNXT_RE_DBR_DO_PACING_NO_CONGESTION;
+	rdev->do_pacing_save = rdev->dbr_def_do_pacing;
+	bnxt_re_set_default_pacing_data(rdev);
+	dev_dbg(rdev_to_dev(rdev), "Initialized db pacing\n");
+
+	return 0;
+fail:
+	free_page((u64)rdev->dbr_page);
+	rdev->dbr_page = NULL;
+	return rc;
+}
+
+static void bnxt_re_deinitialize_dbr_pacing(struct bnxt_re_dev *rdev)
+{
+	if (rdev->dbq_wq)
+		flush_workqueue(rdev->dbq_wq);
+
+	cancel_work_sync(&rdev->dbq_fifo_check_work);
+	cancel_delayed_work_sync(&rdev->dbq_pacing_work);
+
+	if (rdev->dbq_wq) {
+		destroy_workqueue(rdev->dbq_wq);
+		rdev->dbq_wq = NULL;
+	}
+
+	if (rdev->dbr_page)
+		free_page((u64)rdev->dbr_page);
+	rdev->dbr_page = NULL;
+	rdev->dbr_pacing = false;
+}
+
+/* enable_dbr_pacing needs to be done only for older FWs
+ * where host selects primary function. ie. pacing_ext
+ * flags is not set.
+ */
+int bnxt_re_enable_dbr_pacing(struct bnxt_re_dev *rdev)
+{
+	struct bnxt_qplib_nq *nq;
+
+	nq = &rdev->nqr.nq[0];
+	rdev->dbq_nq_id = nq->ring_id;
+
+	if (!bnxt_qplib_dbr_pacing_ext_en(rdev->chip_ctx) &&
+	    bnxt_qplib_dbr_pacing_is_primary_pf(rdev->chip_ctx)) {
+		if (bnxt_re_hwrm_dbr_pacing_cfg(rdev, true)) {
+			dev_err(rdev_to_dev(rdev),
+					"Failed to set dbr pacing config\n");
+			return -EIO;
+		}
+		/* MAP grc window 8 for ARMing the NQ DBQ */
+		writel_fbsd(rdev->en_dev->softc, BNXT_GRCPF_REG_WINDOW_BASE_OUT + 28 , 0,
+			    rdev->chip_ctx->dbr_aeq_arm_reg & BNXT_GRC_BASE_MASK);
+		rdev->dbr_aeq_arm_reg_off =
+			(rdev->chip_ctx->dbr_aeq_arm_reg &
+			 BNXT_GRC_OFFSET_MASK) + 0x8000;
+		writel_fbsd(rdev->en_dev->softc, rdev->dbr_aeq_arm_reg_off , 0, 1);
+	}
+
+	return 0;
+}
+
+/* disable_dbr_pacing needs to be done only for older FWs
+ * where host selects primary function. ie. pacing_ext
+ * flags is not set.
+ */
+
+int bnxt_re_disable_dbr_pacing(struct bnxt_re_dev *rdev)
+{
+	int rc = 0;
+
+	if (!bnxt_qplib_dbr_pacing_ext_en(rdev->chip_ctx) &&
+	    bnxt_qplib_dbr_pacing_is_primary_pf(rdev->chip_ctx))
+		rc = bnxt_re_hwrm_dbr_pacing_cfg(rdev, false);
+
+	return rc;
+}
+
+static void bnxt_re_ib_uninit(struct bnxt_re_dev *rdev)
+{
+	if (test_bit(BNXT_RE_FLAG_IBDEV_REGISTERED, &rdev->flags)) {
+		bnxt_re_sysfs_destroy_file(rdev);
+		/* Cleanup ib dev */
+		ib_unregister_device(&rdev->ibdev);
+		clear_bit(BNXT_RE_FLAG_IBDEV_REGISTERED, &rdev->flags);
+		return;
+	}
+}
+
+static void bnxt_re_dev_uninit(struct bnxt_re_dev *rdev, u8 op_type)
+{
+	struct bnxt_qplib_dpi *kdpi;
+	int rc, wait_count = BNXT_RE_RES_FREE_WAIT_COUNT;
+
+	bnxt_re_net_unregister_async_event(rdev);
+
+	bnxt_re_put_stats2_ctx(rdev);
+	if (test_and_clear_bit(BNXT_RE_FLAG_DEV_LIST_INITIALIZED,
+			       &rdev->flags)) {
+		/* did the caller hold the lock? */
+		mutex_lock(&bnxt_re_dev_lock);
+		list_del_rcu(&rdev->list);
+		mutex_unlock(&bnxt_re_dev_lock);
+	}
+
+	bnxt_re_uninit_resolve_wq(rdev);
+	bnxt_re_uninit_dcb_wq(rdev);
+	bnxt_re_uninit_aer_wq(rdev);
+
+	bnxt_re_deinitialize_dbr_drop_recov(rdev);
+
+	if (bnxt_qplib_dbr_pacing_en(rdev->chip_ctx))
+		(void)bnxt_re_disable_dbr_pacing(rdev);
+
+	if (test_and_clear_bit(BNXT_RE_FLAG_WORKER_REG, &rdev->flags)) {
+		cancel_delayed_work_sync(&rdev->worker);
+	}
+
+	/* Wait for ULPs to release references */
+	while (atomic_read(&rdev->stats.rsors.cq_count) && --wait_count)
+		usleep_range(500, 1000);
+	if (!wait_count)
+		dev_err(rdev_to_dev(rdev),
+			"CQ resources not freed by stack, count = 0x%x",
+			atomic_read(&rdev->stats.rsors.cq_count));
+
+	kdpi = &rdev->dpi_privileged;
+	if (kdpi->umdbr) { /* kernel DPI was allocated with success */
+		(void)bnxt_qplib_dealloc_dpi(&rdev->qplib_res, kdpi);
+		/*
+		 * Driver just need to know no command had failed
+		 * during driver load sequence and below command is
+		 * required indeed. Piggybacking dpi allocation status.
+		 */
+	}
+
+	/* Protect the device uninitialization and start_irq/stop_irq L2
+	 * callbacks with rtnl lock to avoid race condition between these calls
+	 */
+	rtnl_lock();
+	if (test_and_clear_bit(BNXT_RE_FLAG_SETUP_NQ, &rdev->flags))
+		bnxt_re_clean_nqs(rdev);
+	rtnl_unlock();
+
+	if (test_and_clear_bit(BNXT_RE_FLAG_TBLS_ALLOCINIT, &rdev->flags))
+		bnxt_re_free_tbls(rdev);
+	if (test_and_clear_bit(BNXT_RE_FLAG_RCFW_CHANNEL_INIT, &rdev->flags)) {
+		rc = bnxt_qplib_deinit_rcfw(&rdev->rcfw);
+		if (rc)
+			dev_warn(rdev_to_dev(rdev),
+				 "Failed to deinitialize fw, rc = 0x%x", rc);
+	}
+
+	bnxt_re_put_stats_ctx(rdev);
+
+	if (test_and_clear_bit(BNXT_RE_FLAG_ALLOC_CTX, &rdev->flags))
+		bnxt_qplib_free_hwctx(&rdev->qplib_res);
+
+	rtnl_lock();
+	if (test_and_clear_bit(BNXT_RE_FLAG_RCFW_CHANNEL_EN, &rdev->flags))
+		bnxt_qplib_disable_rcfw_channel(&rdev->rcfw);
+
+	if (rdev->dbr_pacing)
+		bnxt_re_deinitialize_dbr_pacing(rdev);
+
+	bnxt_re_free_dbr_sw_stats_mem(rdev);
+
+	if (test_and_clear_bit(BNXT_RE_FLAG_NET_RING_ALLOC, &rdev->flags))
+		bnxt_re_net_ring_free(rdev, rdev->rcfw.creq.ring_id);
+
+	if (test_and_clear_bit(BNXT_RE_FLAG_ALLOC_RCFW, &rdev->flags))
+		bnxt_qplib_free_rcfw_channel(&rdev->qplib_res);
+
+	if (test_and_clear_bit(BNXT_RE_FLAG_GOT_MSIX, &rdev->flags))
+		bnxt_re_free_msix(rdev);
+	rtnl_unlock();
+
+	bnxt_re_destroy_chip_ctx(rdev);
+
+	if (op_type != BNXT_RE_PRE_RECOVERY_REMOVE) {
+		if (test_and_clear_bit(BNXT_RE_FLAG_NETDEV_REGISTERED,
+				       &rdev->flags))
+			bnxt_re_unregister_netdev(rdev);
+	}
+}
+
+static int bnxt_re_dev_init(struct bnxt_re_dev *rdev, u8 op_type, u8 wqe_mode)
+{
+	struct bnxt_re_ring_attr rattr = {};
+	struct bnxt_qplib_creq_ctx *creq;
+	int vec, offset;
+	int rc = 0;
+
+	if (op_type != BNXT_RE_POST_RECOVERY_INIT) {
+		/* Registered a new RoCE device instance to netdev */
+		rc = bnxt_re_register_netdev(rdev);
+		if (rc)
+			return -EINVAL;
+	}
+	set_bit(BNXT_RE_FLAG_NETDEV_REGISTERED, &rdev->flags);
+
+	rc = bnxt_re_setup_chip_ctx(rdev, wqe_mode);
+	if (rc) {
+		dev_err(rdev_to_dev(rdev), "Failed to get chip context rc 0x%x", rc);
+		bnxt_re_unregister_netdev(rdev);
+		clear_bit(BNXT_RE_FLAG_NETDEV_REGISTERED, &rdev->flags);
+		rc = -EINVAL;
+		return rc;
+	}
+
+	/* Protect the device initialization and start_irq/stop_irq L2 callbacks
+	 * with rtnl lock to avoid race condition between these calls
+	 */
+	rtnl_lock();
+	rc = bnxt_re_request_msix(rdev);
+	if (rc) {
+		dev_err(rdev_to_dev(rdev),
+			"Requesting MSI-X vectors failed with rc = 0x%x", rc);
+		rc = -EINVAL;
+		goto release_rtnl;
+	}
+	set_bit(BNXT_RE_FLAG_GOT_MSIX, &rdev->flags);
+
+	/* Establish RCFW Communication Channel to initialize the context
+	   memory for the function and all child VFs */
+	rc = bnxt_qplib_alloc_rcfw_channel(&rdev->qplib_res);
+	if (rc) {
+		dev_err(rdev_to_dev(rdev),
+			"Failed to alloc mem for rcfw, rc = %#x\n", rc);
+		goto release_rtnl;
+	}
+	set_bit(BNXT_RE_FLAG_ALLOC_RCFW, &rdev->flags);
+
+	creq = &rdev->rcfw.creq;
+	rattr.dma_arr = creq->hwq.pbl[PBL_LVL_0].pg_map_arr;
+	rattr.pages = creq->hwq.pbl[creq->hwq.level].pg_count;
+	rattr.type = bnxt_re_get_rtype(rdev);
+	rattr.mode = HWRM_RING_ALLOC_INPUT_INT_MODE_MSIX;
+	rattr.depth = BNXT_QPLIB_CREQE_MAX_CNT - 1;
+	rattr.lrid = rdev->nqr.msix_entries[BNXT_RE_AEQ_IDX].ring_idx;
+	rc = bnxt_re_net_ring_alloc(rdev, &rattr, &creq->ring_id);
+	if (rc) {
+		creq->ring_id = 0xffff;
+		dev_err(rdev_to_dev(rdev),
+			"Failed to allocate CREQ fw id with rc = 0x%x", rc);
+		goto release_rtnl;
+	}
+
+	if (!rdev->chip_ctx)
+		goto release_rtnl;
+	/* Program the NQ ID for DBQ notification */
+	if (rdev->chip_ctx->modes.dbr_pacing_v0 ||
+	    bnxt_qplib_dbr_pacing_en(rdev->chip_ctx) ||
+	    bnxt_qplib_dbr_pacing_ext_en(rdev->chip_ctx)) {
+		rc = bnxt_re_initialize_dbr_pacing(rdev);
+		if (!rc)
+			rdev->dbr_pacing = true;
+		else
+			rdev->dbr_pacing = false;
+		dev_dbg(rdev_to_dev(rdev), "%s: initialize db pacing ret %d\n",
+			__func__, rc);
+	}
+
+	vec = rdev->nqr.msix_entries[BNXT_RE_AEQ_IDX].vector;
+	offset = rdev->nqr.msix_entries[BNXT_RE_AEQ_IDX].db_offset;
+	rc = bnxt_qplib_enable_rcfw_channel(&rdev->rcfw, vec, offset,
+					    &bnxt_re_aeq_handler);
+	if (rc) {
+		dev_err(rdev_to_dev(rdev),
+			"Failed to enable RCFW channel with rc = 0x%x", rc);
+		goto release_rtnl;
+	}
+	set_bit(BNXT_RE_FLAG_RCFW_CHANNEL_EN, &rdev->flags);
+
+	rc = bnxt_re_update_dev_attr(rdev);
+	if (rc)
+		goto release_rtnl;
+	bnxt_re_set_resource_limits(rdev);
+	if (!rdev->is_virtfn && !_is_chip_gen_p5_p7(rdev->chip_ctx)) {
+		rc = bnxt_qplib_alloc_hwctx(&rdev->qplib_res);
+		if (rc) {
+			dev_err(rdev_to_dev(rdev),
+				"Failed to alloc hw contexts, rc = 0x%x", rc);
+			goto release_rtnl;
+		}
+		set_bit(BNXT_RE_FLAG_ALLOC_CTX, &rdev->flags);
+	}
+
+	rc = bnxt_re_get_stats_ctx(rdev);
+	if (rc)
+		goto release_rtnl;
+
+	rc = bnxt_qplib_init_rcfw(&rdev->rcfw, rdev->is_virtfn);
+	if (rc) {
+		dev_err(rdev_to_dev(rdev),
+			"Failed to initialize fw with rc = 0x%x", rc);
+		goto release_rtnl;
+	}
+	set_bit(BNXT_RE_FLAG_RCFW_CHANNEL_INIT, &rdev->flags);
+
+	/* Based resource count on the 'new' device caps */
+	rc = bnxt_re_update_dev_attr(rdev);
+	if (rc)
+		goto release_rtnl;
+	rc = bnxt_re_alloc_init_tbls(rdev);
+	if (rc) {
+		dev_err(rdev_to_dev(rdev), "tbls alloc-init failed rc = %#x",
+			rc);
+		goto release_rtnl;
+	}
+	rc = bnxt_re_setup_nqs(rdev);
+	if (rc) {
+		dev_err(rdev_to_dev(rdev), "NQs alloc-init failed rc = %#x\n",
+			rc);
+		if (rdev->nqr.max_init == 0)
+			goto release_rtnl;
+
+		dev_warn(rdev_to_dev(rdev),
+			"expected nqs %d available nqs %d\n",
+			rdev->nqr.num_msix, rdev->nqr.max_init);
+	}
+	set_bit(BNXT_RE_FLAG_SETUP_NQ, &rdev->flags);
+	rtnl_unlock();
+
+	rc = bnxt_qplib_alloc_dpi(&rdev->qplib_res, &rdev->dpi_privileged,
+				  rdev, BNXT_QPLIB_DPI_TYPE_KERNEL);
+	if (rc)
+		goto fail;
+
+	if (rdev->dbr_pacing)
+		bnxt_re_enable_dbr_pacing(rdev);
+
+	if (rdev->chip_ctx->modes.dbr_drop_recov)
+		bnxt_re_initialize_dbr_drop_recov(rdev);
+
+	rc = bnxt_re_alloc_dbr_sw_stats_mem(rdev);
+	if (rc)
+		goto fail;
+
+	/* This block of code is needed for error recovery support */
+	if (!rdev->is_virtfn) {
+		struct bnxt_re_tc_rec *tc_rec;
+
+		tc_rec = &rdev->tc_rec[0];
+		rc =  bnxt_re_query_hwrm_qportcfg(rdev, tc_rec, 0xFFFF);
+		if (rc) {
+			dev_err(rdev_to_dev(rdev),
+				"Failed to query port config rc:%d", rc);
+			return rc;
+		}
+
+		/* Query f/w defaults of CC params */
+		rc = bnxt_qplib_query_cc_param(&rdev->qplib_res, &rdev->cc_param);
+		if (rc)
+			dev_warn(rdev_to_dev(rdev),
+				"Failed to query CC defaults\n");
+		if (1) {
+			rdev->num_vfs = pci_num_vf(rdev->en_dev->pdev);
+			if (rdev->num_vfs) {
+				bnxt_re_set_resource_limits(rdev);
+				bnxt_qplib_set_func_resources(&rdev->qplib_res);
+			}
+		}
+	}
+	INIT_DELAYED_WORK(&rdev->worker, bnxt_re_worker);
+	set_bit(BNXT_RE_FLAG_WORKER_REG, &rdev->flags);
+	schedule_delayed_work(&rdev->worker, msecs_to_jiffies(1000));
+
+	bnxt_re_init_dcb_wq(rdev);
+	bnxt_re_init_aer_wq(rdev);
+	bnxt_re_init_resolve_wq(rdev);
+	mutex_lock(&bnxt_re_dev_lock);
+	list_add_tail_rcu(&rdev->list, &bnxt_re_dev_list);
+	/* Added to the list, not in progress anymore */
+	gadd_dev_inprogress--;
+	set_bit(BNXT_RE_FLAG_DEV_LIST_INITIALIZED, &rdev->flags);
+	mutex_unlock(&bnxt_re_dev_lock);
+
+
+	return rc;
+release_rtnl:
+	rtnl_unlock();
+fail:
+	bnxt_re_dev_uninit(rdev, BNXT_RE_COMPLETE_REMOVE);
+
+	return rc;
+}
+
+static int bnxt_re_ib_init(struct bnxt_re_dev *rdev)
+{
+	int rc = 0;
+
+	rc = bnxt_re_register_ib(rdev);
+	if (rc) {
+		dev_err(rdev_to_dev(rdev),
+			"Register IB failed with rc = 0x%x", rc);
+		goto fail;
+	}
+	if (bnxt_re_sysfs_create_file(rdev)) {
+		bnxt_re_stopqps_and_ib_uninit(rdev);
+		goto fail;
+	}
+
+	set_bit(BNXT_RE_FLAG_IBDEV_REGISTERED, &rdev->flags);
+	set_bit(BNXT_RE_FLAG_ISSUE_ROCE_STATS, &rdev->flags);
+	set_bit(BNXT_RE_FLAG_ISSUE_CFA_FLOW_STATS, &rdev->flags);
+	bnxt_re_dispatch_event(&rdev->ibdev, NULL, 1, IB_EVENT_PORT_ACTIVE);
+	bnxt_re_dispatch_event(&rdev->ibdev, NULL, 1, IB_EVENT_GID_CHANGE);
+
+	return rc;
+fail:
+	bnxt_re_dev_uninit(rdev, BNXT_RE_COMPLETE_REMOVE);
+	return rc;
+}
+
+/* wrapper for ib_init funcs */
+int _bnxt_re_ib_init(struct bnxt_re_dev *rdev)
+{
+	return bnxt_re_ib_init(rdev);
+}
+
+/* wrapper for aux init funcs */
+int _bnxt_re_ib_init2(struct bnxt_re_dev *rdev)
+{
+	bnxt_re_ib_init_2(rdev);
+	return 0; /* add return for future proof */
+}
+
+static void bnxt_re_dev_unreg(struct bnxt_re_dev *rdev)
+{
+	bnxt_re_dev_dealloc(rdev);
+}
+
+
+static int bnxt_re_dev_reg(struct bnxt_re_dev **rdev, struct ifnet *netdev,
+			   struct bnxt_en_dev *en_dev)
+{
+	struct ifnet *realdev = NULL;
+
+	realdev = netdev;
+	if (realdev)
+		dev_dbg(NULL, "%s: realdev = %p netdev = %p\n", __func__,
+			realdev, netdev);
+	/*
+	 * Note:
+	 * The first argument to bnxt_re_dev_alloc() is 'netdev' and
+	 * not 'realdev', since in the case of bonding we want to
+	 * register the bonded virtual netdev (master) to the ib stack.
+	 * And 'en_dev' (for L2/PCI communication) is the first slave
+	 * device (PF0 on the card).
+	 * In the case of a regular netdev, both netdev and the en_dev
+	 * correspond to the same device.
+	 */
+	*rdev = bnxt_re_dev_alloc(netdev, en_dev);
+	if (!*rdev) {
+		pr_err("%s: netdev %p not handled",
+			ROCE_DRV_MODULE_NAME, netdev);
+		return -ENOMEM;
+	}
+	bnxt_re_hold(*rdev);
+
+	return 0;
+}
+
+void bnxt_re_get_link_speed(struct bnxt_re_dev *rdev)
+{
+	rdev->espeed = rdev->en_dev->espeed;
+	return;
+}
+
+void bnxt_re_stopqps_and_ib_uninit(struct bnxt_re_dev *rdev)
+{
+	dev_dbg(rdev_to_dev(rdev), "%s: Stopping QPs, IB uninit on rdev: %p\n",
+		__func__, rdev);
+	bnxt_re_stop_all_nonqp1_nonshadow_qps(rdev);
+	bnxt_re_ib_uninit(rdev);
+}
+
+void bnxt_re_remove_device(struct bnxt_re_dev *rdev, u8 op_type,
+			   struct auxiliary_device *aux_dev)
+{
+	struct bnxt_re_en_dev_info *en_info;
+	struct bnxt_qplib_cmdq_ctx *cmdq;
+	struct bnxt_qplib_rcfw *rcfw;
+
+	rcfw = &rdev->rcfw;
+	cmdq = &rcfw->cmdq;
+	if (test_bit(FIRMWARE_STALL_DETECTED, &cmdq->flags))
+		set_bit(BNXT_RE_FLAG_ERR_DEVICE_DETACHED, &rdev->flags);
+
+	dev_dbg(rdev_to_dev(rdev), "%s: Removing rdev: %p\n", __func__, rdev);
+	bnxt_re_dev_uninit(rdev, op_type);
+	en_info = auxiliary_get_drvdata(aux_dev);
+	if (en_info) {
+		rtnl_lock();
+		en_info->rdev = NULL;
+		rtnl_unlock();
+		if (op_type != BNXT_RE_PRE_RECOVERY_REMOVE) {
+			clear_bit(BNXT_RE_FLAG_EN_DEV_PRIMARY_DEV, &en_info->flags);
+			clear_bit(BNXT_RE_FLAG_EN_DEV_SECONDARY_DEV, &en_info->flags);
+			clear_bit(BNXT_RE_FLAG_EN_DEV_NETDEV_REG, &en_info->flags);
+		}
+	}
+	bnxt_re_dev_unreg(rdev);
+}
+
+int bnxt_re_add_device(struct bnxt_re_dev **rdev,
+		       struct ifnet *netdev,
+		       u8 qp_mode, u8 op_type, u8 wqe_mode,
+		       u32 num_msix_requested,
+		       struct auxiliary_device *aux_dev)
+{
+	struct bnxt_re_en_dev_info *en_info;
+	struct bnxt_en_dev *en_dev;
+	int rc = 0;
+
+	en_info = auxiliary_get_drvdata(aux_dev);
+	en_dev = en_info->en_dev;
+
+	mutex_lock(&bnxt_re_dev_lock);
+	/* Check if driver already in mod exit and aux_dev is valid */
+	if (gmod_exit || !aux_dev) {
+		mutex_unlock(&bnxt_re_dev_lock);
+		return -ENODEV;
+	}
+	/* Add device in progress */
+	gadd_dev_inprogress++;
+	mutex_unlock(&bnxt_re_dev_lock);
+
+	rc = bnxt_re_dev_reg(rdev, netdev, en_dev);
+	if (rc) {
+		dev_dbg(NULL, "Failed to create add device for netdev %p\n",
+			netdev);
+		/*
+		 * For BNXT_RE_POST_RECOVERY_INIT special case
+		 * called from bnxt_re_start, the work is
+		 * complete only after, bnxt_re_start completes
+		 * bnxt_unregister_device in case of failure.
+		 * So bnxt_re_start will decrement gadd_dev_inprogress
+		 * in case of failure.
+		 */
+		if (op_type != BNXT_RE_POST_RECOVERY_INIT) {
+			mutex_lock(&bnxt_re_dev_lock);
+			gadd_dev_inprogress--;
+			mutex_unlock(&bnxt_re_dev_lock);
+		}
+		return rc;
+	}
+
+	if (rc != 0)
+		goto ref_error;
+
+	/*
+	 *  num_msix_requested = BNXT_RE_MSIX_FROM_MOD_PARAM indicates fresh driver load.
+	 *  Otherwaise, this invocation can be the result of lag create / destroy,
+	 *  err revovery, hot fw upgrade, etc..
+	 */
+	if (num_msix_requested == BNXT_RE_MSIX_FROM_MOD_PARAM) {
+		if (bnxt_re_probe_count < BNXT_RE_MAX_DEVICES)
+			num_msix_requested = max_msix_vec[bnxt_re_probe_count++];
+		else
+			/* Consider as default when probe_count exceeds its limit */
+			num_msix_requested = 0;
+
+		/* if user specifies only one value, use the same for all PFs */
+		if (max_msix_vec_argc == 1)
+			num_msix_requested = max_msix_vec[0];
+	}
+
+	(*rdev)->num_msix_requested = num_msix_requested;
+	(*rdev)->gsi_ctx.gsi_qp_mode = qp_mode;
+	(*rdev)->adev = aux_dev;
+	(*rdev)->dev_addr = en_dev->softc->func.mac_addr;
+	/* Before updating the rdev pointer in bnxt_re_en_dev_info structure,
+	 * take the rtnl lock to avoid accessing invalid rdev pointer from
+	 * L2 ULP callbacks. This is applicable in all the places where rdev
+	 * pointer is updated in bnxt_re_en_dev_info.
+	 */
+	rtnl_lock();
+	en_info->rdev = *rdev;
+	rtnl_unlock();
+	rc = bnxt_re_dev_init(*rdev, op_type, wqe_mode);
+	if (rc) {
+ref_error:
+		bnxt_re_dev_unreg(*rdev);
+		*rdev = NULL;
+		/*
+		 * For BNXT_RE_POST_RECOVERY_INIT special case
+		 * called from bnxt_re_start, the work is
+		 * complete only after, bnxt_re_start completes
+		 * bnxt_unregister_device in case of failure.
+		 * So bnxt_re_start will decrement gadd_dev_inprogress
+		 * in case of failure.
+		 */
+		if (op_type != BNXT_RE_POST_RECOVERY_INIT) {
+			mutex_lock(&bnxt_re_dev_lock);
+			gadd_dev_inprogress--;
+			mutex_unlock(&bnxt_re_dev_lock);
+		}
+	}
+	dev_dbg(rdev_to_dev(*rdev), "%s: Adding rdev: %p\n", __func__, *rdev);
+	if (!rc) {
+		set_bit(BNXT_RE_FLAG_EN_DEV_NETDEV_REG, &en_info->flags);
+	}
+	return rc;
+}
+
+struct bnxt_re_dev *bnxt_re_get_peer_pf(struct bnxt_re_dev *rdev)
+{
+	struct pci_dev *pdev_in = rdev->en_dev->pdev;
+	int tmp_bus_num, bus_num = pdev_in->bus->number;
+	int tmp_dev_num, dev_num = PCI_SLOT(pdev_in->devfn);
+	int tmp_func_num, func_num = PCI_FUNC(pdev_in->devfn);
+	struct bnxt_re_dev *tmp_rdev;
+
+	rcu_read_lock();
+	list_for_each_entry_rcu(tmp_rdev, &bnxt_re_dev_list, list) {
+		tmp_bus_num = tmp_rdev->en_dev->pdev->bus->number;
+		tmp_dev_num = PCI_SLOT(tmp_rdev->en_dev->pdev->devfn);
+		tmp_func_num = PCI_FUNC(tmp_rdev->en_dev->pdev->devfn);
+
+		if (bus_num == tmp_bus_num && dev_num == tmp_dev_num &&
+		    func_num != tmp_func_num) {
+			rcu_read_unlock();
+			return tmp_rdev;
+		}
+	}
+	rcu_read_unlock();
+	return NULL;
+}
+
+
+int bnxt_re_schedule_work(struct bnxt_re_dev *rdev, unsigned long event,
+			  struct ifnet *vlan_dev,
+			  struct ifnet *netdev,
+			  struct auxiliary_device *adev)
+{
+	struct bnxt_re_work *re_work;
+
+	/* Allocate for the deferred task */
+	re_work = kzalloc(sizeof(*re_work), GFP_KERNEL);
+	if (!re_work)
+		return -ENOMEM;
+
+	re_work->rdev = rdev;
+	re_work->event = event;
+	re_work->vlan_dev = vlan_dev;
+	re_work->adev = adev;
+	INIT_WORK(&re_work->work, bnxt_re_task);
+	if (rdev)
+		atomic_inc(&rdev->sched_count);
+	re_work->netdev = netdev;
+	queue_work(bnxt_re_wq, &re_work->work);
+
+	return 0;
+}
+
+
+int bnxt_re_get_slot_pf_count(struct bnxt_re_dev *rdev)
+{
+	struct pci_dev *pdev_in = rdev->en_dev->pdev;
+	int tmp_bus_num, bus_num = pdev_in->bus->number;
+	int tmp_dev_num, dev_num = PCI_SLOT(pdev_in->devfn);
+	struct bnxt_re_dev *tmp_rdev;
+	int pf_cnt = 0;
+
+	rcu_read_lock();
+	list_for_each_entry_rcu(tmp_rdev, &bnxt_re_dev_list, list) {
+		tmp_bus_num = tmp_rdev->en_dev->pdev->bus->number;
+		tmp_dev_num = PCI_SLOT(tmp_rdev->en_dev->pdev->devfn);
+
+		if (bus_num == tmp_bus_num && dev_num == tmp_dev_num)
+			pf_cnt++;
+	}
+	rcu_read_unlock();
+	return pf_cnt;
+}
+
+/* Handle all deferred netevents tasks */
+static void bnxt_re_task(struct work_struct *work)
+{
+	struct bnxt_re_en_dev_info *en_info;
+	struct auxiliary_device *aux_dev;
+	struct bnxt_re_work *re_work;
+	struct bnxt_re_dev *rdev;
+
+	re_work = container_of(work, struct bnxt_re_work, work);
+
+	mutex_lock(&bnxt_re_mutex);
+	rdev = re_work->rdev;
+
+	/*
+	 * If the previous rdev is deleted due to bond creation
+	 * do not handle the event
+	 */
+	if (!bnxt_re_is_rdev_valid(rdev))
+		goto exit;
+
+	/* Ignore the event, if the device is not registred with IB stack. This
+	 * is to avoid handling any event while the device is added/removed.
+	 */
+	if (rdev && !test_bit(BNXT_RE_FLAG_IBDEV_REGISTERED, &rdev->flags)) {
+		dev_dbg(rdev_to_dev(rdev), "%s: Ignoring netdev event 0x%lx",
+			__func__, re_work->event);
+		goto done;
+	}
+
+	/* Extra check to silence coverity. We shouldn't handle any event
+	 * when rdev is NULL.
+	 */
+	if (!rdev)
+		goto exit;
+
+	dev_dbg(rdev_to_dev(rdev), "Scheduled work for event 0x%lx",
+		re_work->event);
+
+	switch (re_work->event) {
+	case NETDEV_UP:
+		bnxt_re_dispatch_event(&rdev->ibdev, NULL, 1,
+				       IB_EVENT_PORT_ACTIVE);
+		bnxt_re_net_register_async_event(rdev);
+		break;
+
+	case NETDEV_DOWN:
+		bnxt_qplib_dbr_pacing_set_primary_pf(rdev->chip_ctx, 0);
+		bnxt_re_stop_all_nonqp1_nonshadow_qps(rdev);
+		bnxt_re_dispatch_event(&rdev->ibdev, NULL, 1,
+				       IB_EVENT_PORT_ERR);
+		break;
+
+	case NETDEV_CHANGE:
+		if (bnxt_re_get_link_state(rdev) == IB_PORT_DOWN) {
+			bnxt_re_stop_all_nonqp1_nonshadow_qps(rdev);
+			bnxt_re_dispatch_event(&rdev->ibdev, NULL, 1,
+					       IB_EVENT_PORT_ERR);
+			break;
+		} else if (bnxt_re_get_link_state(rdev) == IB_PORT_ACTIVE) {
+			bnxt_re_dispatch_event(&rdev->ibdev, NULL, 1,
+					       IB_EVENT_PORT_ACTIVE);
+		}
+
+		/* temporarily disable the check for SR2 */
+		if (!bnxt_qplib_query_cc_param(&rdev->qplib_res,
+					       &rdev->cc_param) &&
+		    !_is_chip_p7(rdev->chip_ctx)) {
+			/*
+			 *  Disable CC for 10G speed
+			 * for non p5 devices
+			 */
+			if (rdev->sl_espeed == SPEED_10000 &&
+			    !_is_chip_gen_p5_p7(rdev->chip_ctx)) {
+				if (rdev->cc_param.enable)
+					bnxt_re_clear_cc(rdev);
+			} else {
+				if (!rdev->cc_param.enable &&
+				    rdev->cc_param.admin_enable)
+					bnxt_re_setup_cc(rdev);
+			}
+		}
+		break;
+
+	case NETDEV_UNREGISTER:
+		bnxt_re_stopqps_and_ib_uninit(rdev);
+		aux_dev = rdev->adev;
+		if (re_work->adev)
+			goto done;
+
+		bnxt_re_remove_device(rdev, BNXT_RE_COMPLETE_REMOVE, aux_dev);
+
+		break;
+
+	default:
+		break;
+	}
+done:
+	if (rdev) {
+		/* memory barrier to guarantee task completion
+		 * before decrementing sched count
+		 */
+		mmiowb();
+		atomic_dec(&rdev->sched_count);
+	}
+exit:
+	if (re_work->adev && re_work->event == NETDEV_UNREGISTER) {
+		en_info = auxiliary_get_drvdata(re_work->adev);
+		en_info->ib_uninit_done = true;
+		wake_up(&en_info->waitq);
+	}
+	kfree(re_work);
+	mutex_unlock(&bnxt_re_mutex);
+}
+
+/*
+    "Notifier chain callback can be invoked for the same chain from
+    different CPUs at the same time".
+
+    For cases when the netdev is already present, our call to the
+    register_netdevice_notifier() will actually get the rtnl_lock()
+    before sending NETDEV_REGISTER and (if up) NETDEV_UP
+    events.
+
+    But for cases when the netdev is not already present, the notifier
+    chain is subjected to be invoked from different CPUs simultaneously.
+
+    This is protected by the netdev_mutex.
+*/
+static int bnxt_re_netdev_event(struct notifier_block *notifier,
+				unsigned long event, void *ptr)
+{
+	struct ifnet *real_dev, *netdev;
+	struct bnxt_re_dev *rdev = NULL;
+
+	netdev = netdev_notifier_info_to_ifp(ptr);
+	real_dev = rdma_vlan_dev_real_dev(netdev);
+	if (!real_dev)
+		real_dev = netdev;
+	/* In case of bonding,this will be bond's rdev */
+	rdev = bnxt_re_from_netdev(real_dev);
+
+	if (!rdev)
+		goto exit;
+
+	dev_info(rdev_to_dev(rdev), "%s: Event = %s (0x%lx), rdev %s (real_dev %s)\n",
+		 __func__, bnxt_re_netevent(event), event,
+		 rdev ? rdev->netdev ? rdev->netdev->if_dname : "->netdev = NULL" : "= NULL",
+		 (real_dev == netdev) ? "= netdev" : real_dev->if_dname);
+
+	if (!test_bit(BNXT_RE_FLAG_IBDEV_REGISTERED, &rdev->flags))
+		goto exit;
+
+	bnxt_re_hold(rdev);
+
+	if (real_dev != netdev) {
+		switch (event) {
+		case NETDEV_UP:
+			bnxt_re_schedule_work(rdev, event, netdev,
+					      NULL, NULL);
+			break;
+		case NETDEV_DOWN:
+			break;
+		default:
+			break;
+		}
+		goto done;
+	}
+
+	switch (event) {
+	case NETDEV_CHANGEADDR:
+		if (!_is_chip_gen_p5_p7(rdev->chip_ctx))
+			bnxt_re_update_shadow_ah(rdev);
+		bnxt_qplib_get_guid(rdev->dev_addr,
+				    (u8 *)&rdev->ibdev.node_guid);
+		break;
+
+	case NETDEV_CHANGE:
+		bnxt_re_get_link_speed(rdev);
+		bnxt_re_schedule_work(rdev, event, NULL, NULL, NULL);
+		break;
+	case NETDEV_UNREGISTER:
+		/* netdev notifier will call NETDEV_UNREGISTER again later since
+		 * we are still holding the reference to the netdev
+		 */
+
+		/*
+		 *  Workaround to avoid ib_unregister hang. Check for module
+		 *  reference and dont free up the device if the reference
+		 *  is non zero. Checking only for PF functions.
+		 */
+
+		if (rdev) {
+			dev_info(rdev_to_dev(rdev),
+				 "bnxt_re:Unreg recvd when module refcnt > 0");
+			dev_info(rdev_to_dev(rdev),
+				 "bnxt_re:Close all apps using bnxt_re devs");
+			dev_info(rdev_to_dev(rdev),
+				 "bnxt_re:Remove the configfs entry created for the device");
+			dev_info(rdev_to_dev(rdev),
+				 "bnxt_re:Refer documentation for details");
+			goto done;
+		}
+
+		if (atomic_read(&rdev->sched_count) > 0)
+			goto done;
+		if (!rdev->unreg_sched) {
+			bnxt_re_schedule_work(rdev, NETDEV_UNREGISTER,
+					      NULL, NULL, NULL);
+			rdev->unreg_sched = true;
+			goto done;
+		}
+
+		break;
+	default:
+		break;
+	}
+done:
+	if (rdev)
+		bnxt_re_put(rdev);
+exit:
+	return NOTIFY_DONE;
+}
+
+static struct notifier_block bnxt_re_netdev_notifier = {
+	.notifier_call = bnxt_re_netdev_event
+};
+
+static void bnxt_re_remove_base_interface(struct bnxt_re_dev *rdev,
+					  struct auxiliary_device *adev)
+{
+	bnxt_re_stopqps_and_ib_uninit(rdev);
+	bnxt_re_remove_device(rdev, BNXT_RE_COMPLETE_REMOVE, adev);
+	auxiliary_set_drvdata(adev, NULL);
+}
+
+/*
+ *  bnxt_re_remove  -	Removes the roce aux device
+ *  @adev  -  aux device pointer
+ *
+ * This function removes the roce device. This gets
+ * called in the mod exit path and pci unbind path.
+ * If the rdev is bond interace, destroys the lag
+ * in module exit path, and in pci unbind case
+ * destroys the lag and recreates other base interface.
+ * If the device is already removed in error recovery
+ * path, it just unregister with the L2.
+ */
+static void bnxt_re_remove(struct auxiliary_device *adev)
+{
+	struct bnxt_re_en_dev_info *en_info = auxiliary_get_drvdata(adev);
+	struct bnxt_en_dev *en_dev;
+	struct bnxt_re_dev *rdev;
+	bool primary_dev = false;
+	bool secondary_dev = false;
+
+	if (!en_info)
+		return;
+
+	mutex_lock(&bnxt_re_mutex);
+	en_dev = en_info->en_dev;
+
+	rdev = en_info->rdev;
+
+	if (rdev && bnxt_re_is_rdev_valid(rdev)) {
+		if (pci_channel_offline(rdev->rcfw.pdev))
+			set_bit(ERR_DEVICE_DETACHED, &rdev->rcfw.cmdq.flags);
+
+		if (test_bit(BNXT_RE_FLAG_EN_DEV_PRIMARY_DEV, &en_info->flags))
+			primary_dev = true;
+		if (test_bit(BNXT_RE_FLAG_EN_DEV_SECONDARY_DEV, &en_info->flags))
+			secondary_dev = true;
+
+		/*
+		 * en_dev_info of primary device and secondary device have the
+		 * same rdev pointer when LAG is configured. This rdev pointer
+		 * is rdev of bond interface.
+		 */
+		if (!primary_dev && !secondary_dev) {
+			/* removal of non bond interface */
+			bnxt_re_remove_base_interface(rdev, adev);
+		} else {
+			/*
+			 * removal of bond primary/secondary interface. In this
+			 * case bond device is already removed, so rdev->binfo
+			 * is NULL.
+			 */
+			auxiliary_set_drvdata(adev, NULL);
+		}
+	} else {
+		/* device is removed from ulp stop, unregister the net dev */
+		if (test_bit(BNXT_RE_FLAG_EN_DEV_NETDEV_REG, &en_info->flags)) {
+			rtnl_lock();
+			en_dev->en_ops->bnxt_unregister_device(en_dev,
+							       BNXT_ROCE_ULP);
+			rtnl_unlock();
+		}
+	}
+	mutex_unlock(&bnxt_re_mutex);
+	return;
+}
+
+/* wrapper for all external user context callers */
+void _bnxt_re_remove(struct auxiliary_device *adev)
+{
+	bnxt_re_remove(adev);
+}
+
+static void bnxt_re_ib_init_2(struct bnxt_re_dev *rdev)
+{
+	int rc;
+
+	rc = bnxt_re_get_device_stats(rdev);
+	if (rc)
+		dev_err(rdev_to_dev(rdev),
+			"Failed initial device stat query");
+
+	bnxt_re_net_register_async_event(rdev);
+}
+
+static int bnxt_re_probe(struct auxiliary_device *adev,
+			 const struct auxiliary_device_id *id)
+{
+	struct bnxt_aux_dev *aux_dev =
+		container_of(adev, struct bnxt_aux_dev, aux_dev);
+	struct bnxt_re_en_dev_info *en_info;
+	struct bnxt_en_dev *en_dev = NULL;
+	struct bnxt_re_dev *rdev;
+	int rc = -ENODEV;
+
+	if (aux_dev)
+		en_dev = aux_dev->edev;
+
+	if (!en_dev)
+		return rc;
+
+	if (en_dev->ulp_version != BNXT_ULP_VERSION) {
+		pr_err("%s: probe error: bnxt_en ulp version magic %x is not compatible!\n",
+			ROCE_DRV_MODULE_NAME, en_dev->ulp_version);
+		return -EINVAL;
+	}
+
+	en_info = kzalloc(sizeof(*en_info), GFP_KERNEL);
+	if (!en_info)
+		return -ENOMEM;
+	memset(en_info, 0, sizeof(struct bnxt_re_en_dev_info));
+	en_info->en_dev = en_dev;
+	auxiliary_set_drvdata(adev, en_info);
+
+	mutex_lock(&bnxt_re_mutex);
+	rc = bnxt_re_add_device(&rdev, en_dev->net,
+				BNXT_RE_GSI_MODE_ALL,
+				BNXT_RE_COMPLETE_INIT,
+				BNXT_QPLIB_WQE_MODE_STATIC,
+				BNXT_RE_MSIX_FROM_MOD_PARAM, adev);
+	if (rc) {
+		mutex_unlock(&bnxt_re_mutex);
+		return rc;
+	}
+
+	rc = bnxt_re_ib_init(rdev);
+	if (rc)
+		goto err;
+
+	bnxt_re_ib_init_2(rdev);
+
+	dev_dbg(rdev_to_dev(rdev), "%s: adev: %p\n", __func__, adev);
+	rdev->adev = adev;
+
+	mutex_unlock(&bnxt_re_mutex);
+
+	return 0;
+
+err:
+	mutex_unlock(&bnxt_re_mutex);
+	bnxt_re_remove(adev);
+
+	return rc;
+}
+
+static const struct auxiliary_device_id bnxt_re_id_table[] = {
+	{ .name = BNXT_ADEV_NAME ".rdma", },
+	{},
+};
+
+MODULE_DEVICE_TABLE(auxiliary, bnxt_re_id_table);
+
+static struct auxiliary_driver bnxt_re_driver = {
+	.name = "rdma",
+	.probe = bnxt_re_probe,
+	.remove = bnxt_re_remove,
+	.id_table = bnxt_re_id_table,
+};
+
+static int __init bnxt_re_mod_init(void)
+{
+	int rc = 0;
+
+	pr_info("%s: %s", ROCE_DRV_MODULE_NAME, drv_version);
+
+	bnxt_re_wq = create_singlethread_workqueue("bnxt_re");
+	if (!bnxt_re_wq)
+		return -ENOMEM;
+
+	rc = bnxt_re_register_netdevice_notifier(&bnxt_re_netdev_notifier);
+	if (rc) {
+		pr_err("%s: Cannot register to netdevice_notifier",
+			ROCE_DRV_MODULE_NAME);
+		goto err_netdev;
+	}
+
+	INIT_LIST_HEAD(&bnxt_re_dev_list);
+
+	rc = auxiliary_driver_register(&bnxt_re_driver);
+	if (rc) {
+		pr_err("%s: Failed to register auxiliary driver\n",
+		       ROCE_DRV_MODULE_NAME);
+		goto err_auxdrv;
+	}
+
+	return 0;
+
+err_auxdrv:
+	bnxt_re_unregister_netdevice_notifier(&bnxt_re_netdev_notifier);
+
+err_netdev:
+	destroy_workqueue(bnxt_re_wq);
+
+	return rc;
+}
+
+static void __exit bnxt_re_mod_exit(void)
+{
+	gmod_exit = 1;
+	auxiliary_driver_unregister(&bnxt_re_driver);
+
+	bnxt_re_unregister_netdevice_notifier(&bnxt_re_netdev_notifier);
+
+	if (bnxt_re_wq)
+		destroy_workqueue(bnxt_re_wq);
+}
+
+module_init(bnxt_re_mod_init);
+module_exit(bnxt_re_mod_exit);
diff --git a/sys/dev/bnxt/bnxt_re/qplib_fp.c b/sys/dev/bnxt/bnxt_re/qplib_fp.c
new file mode 100644
index 000000000000..438f7ddd468e
--- /dev/null
+++ b/sys/dev/bnxt/bnxt_re/qplib_fp.c
@@ -0,0 +1,3544 @@
+/*
+ * Copyright (c) 2015-2024, Broadcom. All rights reserved.  The term
+ * Broadcom refers to Broadcom Limited and/or its subsidiaries.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS''
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
+ * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
+ * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * Description: Fast Path Operators
+ */
+
+#include <linux/interrupt.h>
+#include <linux/spinlock.h>
+#include <linux/sched.h>
+#include <linux/slab.h>
+#include <linux/pci.h>
+#include <linux/delay.h>
+#include <linux/if_ether.h>
+#include <linux/hardirq.h>
+#include <rdma/ib_mad.h>
+
+#include "hsi_struct_def.h"
+#include "qplib_tlv.h"
+#include "qplib_res.h"
+#include "qplib_rcfw.h"
+#include "qplib_sp.h"
+#include "qplib_fp.h"
+#include "ib_verbs.h"
+
+static void __clean_cq(struct bnxt_qplib_cq *cq, u64 qp);
+
+static void bnxt_re_legacy_cancel_phantom_processing(struct bnxt_qplib_qp *qp)
+{
+	qp->sq.condition = false;
+	qp->sq.legacy_send_phantom = false;
+	qp->sq.single = false;
+}
+
+static void __bnxt_qplib_add_flush_qp(struct bnxt_qplib_qp *qp)
+{
+	struct bnxt_qplib_cq *scq, *rcq;
+
+	scq = qp->scq;
+	rcq = qp->rcq;
+
+	if (!qp->sq.flushed) {
+		dev_dbg(&scq->hwq.pdev->dev,
+			"QPLIB: FP: Adding to SQ Flush list = %p\n",
+			qp);
+		bnxt_re_legacy_cancel_phantom_processing(qp);
+		list_add_tail(&qp->sq_flush, &scq->sqf_head);
+		qp->sq.flushed = true;
+	}
+	if (!qp->srq) {
+		if (!qp->rq.flushed) {
+			dev_dbg(&rcq->hwq.pdev->dev,
+				"QPLIB: FP: Adding to RQ Flush list = %p\n",
+				qp);
+			list_add_tail(&qp->rq_flush, &rcq->rqf_head);
+			qp->rq.flushed = true;
+		}
+	}
+}
+
+static void bnxt_qplib_acquire_cq_flush_locks(struct bnxt_qplib_qp *qp)
+	__acquires(&qp->scq->flush_lock) __acquires(&qp->rcq->flush_lock)
+{
+	/* Interrupts are already disabled in calling functions */
+	spin_lock(&qp->scq->flush_lock);
+	if (qp->scq == qp->rcq)
+		__acquire(&qp->rcq->flush_lock);
+	else
+		spin_lock(&qp->rcq->flush_lock);
+}
+
+static void bnxt_qplib_release_cq_flush_locks(struct bnxt_qplib_qp *qp)
+	__releases(&qp->scq->flush_lock) __releases(&qp->rcq->flush_lock)
+{
+	if (qp->scq == qp->rcq)
+		__release(&qp->rcq->flush_lock);
+	else
+		spin_unlock(&qp->rcq->flush_lock);
+	spin_unlock(&qp->scq->flush_lock);
+}
+
+void bnxt_qplib_add_flush_qp(struct bnxt_qplib_qp *qp)
+{
+
+	bnxt_qplib_acquire_cq_flush_locks(qp);
+	__bnxt_qplib_add_flush_qp(qp);
+	bnxt_qplib_release_cq_flush_locks(qp);
+}
+
+static void __bnxt_qplib_del_flush_qp(struct bnxt_qplib_qp *qp)
+{
+	if (qp->sq.flushed) {
+		qp->sq.flushed = false;
+		list_del(&qp->sq_flush);
+	}
+	if (!qp->srq) {
+		if (qp->rq.flushed) {
+			qp->rq.flushed = false;
+			list_del(&qp->rq_flush);
+		}
+	}
+}
+
+void bnxt_qplib_clean_qp(struct bnxt_qplib_qp *qp)
+{
+
+	bnxt_qplib_acquire_cq_flush_locks(qp);
+	__clean_cq(qp->scq, (u64)(unsigned long)qp);
+	qp->sq.hwq.prod = 0;
+	qp->sq.hwq.cons = 0;
+	qp->sq.swq_start = 0;
+	qp->sq.swq_last = 0;
+	__clean_cq(qp->rcq, (u64)(unsigned long)qp);
+	qp->rq.hwq.prod = 0;
+	qp->rq.hwq.cons = 0;
+	qp->rq.swq_start = 0;
+	qp->rq.swq_last = 0;
+
+	__bnxt_qplib_del_flush_qp(qp);
+	bnxt_qplib_release_cq_flush_locks(qp);
+}
+
+static void bnxt_qpn_cqn_sched_task(struct work_struct *work)
+{
+	struct bnxt_qplib_nq_work *nq_work =
+			container_of(work, struct bnxt_qplib_nq_work, work);
+
+	struct bnxt_qplib_cq *cq = nq_work->cq;
+	struct bnxt_qplib_nq *nq = nq_work->nq;
+
+	if (cq && nq) {
+		spin_lock_bh(&cq->compl_lock);
+		if (nq->cqn_handler) {
+			dev_dbg(&nq->res->pdev->dev,
+				"%s:Trigger cq  = %p event nq = %p\n",
+				__func__, cq, nq);
+			nq->cqn_handler(nq, cq);
+		}
+		spin_unlock_bh(&cq->compl_lock);
+	}
+	kfree(nq_work);
+}
+
+static void bnxt_qplib_put_hdr_buf(struct pci_dev *pdev,
+				   struct bnxt_qplib_hdrbuf *buf)
+{
+	dma_free_coherent(&pdev->dev, buf->len, buf->va, buf->dma_map);
+	kfree(buf);
+}
+
+static void *bnxt_qplib_get_hdr_buf(struct pci_dev *pdev,  u32 step, u32 cnt)
+{
+	struct bnxt_qplib_hdrbuf *hdrbuf;
+	u32 len;
+
+	hdrbuf = kmalloc(sizeof(*hdrbuf), GFP_KERNEL);
+	if (!hdrbuf)
+		return NULL;
+
+	len = ALIGN((step * cnt), PAGE_SIZE);
+	hdrbuf->va = dma_alloc_coherent(&pdev->dev, len,
+					&hdrbuf->dma_map, GFP_KERNEL);
+	if (!hdrbuf->va)
+		goto out;
+
+	hdrbuf->len = len;
+	hdrbuf->step = step;
+	return hdrbuf;
+out:
+	kfree(hdrbuf);
+	return NULL;
+}
+
+void bnxt_qplib_free_hdr_buf(struct bnxt_qplib_res *res,
+			     struct bnxt_qplib_qp *qp)
+{
+	if (qp->rq_hdr_buf) {
+		bnxt_qplib_put_hdr_buf(res->pdev, qp->rq_hdr_buf);
+		qp->rq_hdr_buf = NULL;
+	}
+
+	if (qp->sq_hdr_buf) {
+		bnxt_qplib_put_hdr_buf(res->pdev, qp->sq_hdr_buf);
+		qp->sq_hdr_buf = NULL;
+	}
+}
+
+int bnxt_qplib_alloc_hdr_buf(struct bnxt_qplib_res *res,
+			     struct bnxt_qplib_qp *qp, u32 sstep, u32 rstep)
+{
+	struct pci_dev *pdev;
+	int rc = 0;
+
+	pdev = res->pdev;
+	if (sstep) {
+		qp->sq_hdr_buf = bnxt_qplib_get_hdr_buf(pdev, sstep,
+							qp->sq.max_wqe);
+		if (!qp->sq_hdr_buf) {
+			dev_err(&pdev->dev, "QPLIB: Failed to get sq_hdr_buf\n");
+			return -ENOMEM;
+		}
+	}
+
+	if (rstep) {
+		qp->rq_hdr_buf = bnxt_qplib_get_hdr_buf(pdev, rstep,
+							qp->rq.max_wqe);
+		if (!qp->rq_hdr_buf) {
+			rc = -ENOMEM;
+			dev_err(&pdev->dev, "QPLIB: Failed to get rq_hdr_buf\n");
+			goto fail;
+		}
+	}
+
+	return 0;
+fail:
+	bnxt_qplib_free_hdr_buf(res, qp);
+	return rc;
+}
+
+/*
+ * clean_nq -	Invalidate cqe from given nq.
+ * @cq      -	Completion queue
+ *
+ * Traverse whole notification queue and invalidate any completion
+ * associated cq handler provided by caller.
+ * Note - This function traverse the hardware queue but do not update
+ * consumer index. Invalidated cqe(marked from this function) will be
+ * ignored from actual completion of notification queue.
+ */
+static void clean_nq(struct bnxt_qplib_cq *cq)
+{
+	struct bnxt_qplib_hwq *nq_hwq = NULL;
+	struct bnxt_qplib_nq *nq = NULL;
+	struct nq_base *hw_nqe = NULL;
+	struct nq_cn *nqcne = NULL;
+	u32 peek_flags, peek_cons;
+	u64 q_handle;
+	u32 type;
+	int i;
+
+	nq = cq->nq;
+	nq_hwq = &nq->hwq;
+
+	spin_lock_bh(&nq_hwq->lock);
+	peek_flags = nq->nq_db.dbinfo.flags;
+	peek_cons = nq_hwq->cons;
+	for (i = 0; i < nq_hwq->max_elements; i++) {
+		hw_nqe = bnxt_qplib_get_qe(nq_hwq, peek_cons, NULL);
+		if (!NQE_CMP_VALID(hw_nqe, peek_flags))
+			break;
+
+		/* The valid test of the entry must be done first
+		 * before reading any further.
+		 */
+		dma_rmb();
+		type = le16_to_cpu(hw_nqe->info10_type) &
+				   NQ_BASE_TYPE_MASK;
+
+		/* Processing only NQ_BASE_TYPE_CQ_NOTIFICATION */
+		if (type == NQ_BASE_TYPE_CQ_NOTIFICATION) {
+			nqcne = (struct nq_cn *)hw_nqe;
+
+			q_handle = le32_to_cpu(nqcne->cq_handle_low);
+			q_handle |= (u64)le32_to_cpu(nqcne->cq_handle_high) << 32;
+			if (q_handle == (u64)cq) {
+				nqcne->cq_handle_low = 0;
+				nqcne->cq_handle_high = 0;
+				cq->cnq_events++;
+			}
+		}
+		bnxt_qplib_hwq_incr_cons(nq_hwq->max_elements, &peek_cons,
+					 1, &peek_flags);
+	}
+	spin_unlock_bh(&nq_hwq->lock);
+}
+
+/*
+ * Wait for receiving all NQEs for this CQ.
+ * clean_nq is tried 100 times, each time clean_cq
+ * loops upto budget times. budget is based on the
+ * number of CQs shared by that NQ. So any NQE from
+ * CQ would be already in the NQ.
+ */
+static void __wait_for_all_nqes(struct bnxt_qplib_cq *cq, u16 cnq_events)
+{
+	u32 retry_cnt = 100;
+	u16 total_events;
+
+	if (!cnq_events) {
+		clean_nq(cq);
+		return;
+	}
+	while (retry_cnt--) {
+		total_events = cq->cnq_events;
+
+		/* Increment total_events by 1 if any CREQ event received with CQ notification */
+		if (cq->is_cq_err_event)
+			total_events++;
+
+		if (cnq_events == total_events) {
+			dev_dbg(&cq->nq->res->pdev->dev,
+				"QPLIB: NQ cleanup - Received all NQ events\n");
+			return;
+		}
+		msleep(1);
+		clean_nq(cq);
+	}
+}
+
+static void bnxt_qplib_service_nq(unsigned long data)
+{
+	struct bnxt_qplib_nq *nq = (struct bnxt_qplib_nq *)data;
+	struct bnxt_qplib_hwq *nq_hwq = &nq->hwq;
+	int budget = nq->budget;
+	struct bnxt_qplib_res *res;
+	struct bnxt_qplib_cq *cq;
+	struct pci_dev *pdev;
+	struct nq_base *nqe;
+	u32 hw_polled = 0;
+	u64 q_handle;
+	u32 type;
+
+	res = nq->res;
+	pdev = res->pdev;
+
+	spin_lock_bh(&nq_hwq->lock);
+	/* Service the NQ until empty or budget expired */
+	while (budget--) {
+		nqe = bnxt_qplib_get_qe(nq_hwq, nq_hwq->cons, NULL);
+		if (!NQE_CMP_VALID(nqe, nq->nq_db.dbinfo.flags))
+			break;
+		/* The valid test of the entry must be done first before
+		 * reading any further.
+		 */
+		dma_rmb();
+		type = le16_to_cpu(nqe->info10_type) & NQ_BASE_TYPE_MASK;
+		switch (type) {
+		case NQ_BASE_TYPE_CQ_NOTIFICATION:
+		{
+			struct nq_cn *nqcne = (struct nq_cn *)nqe;
+
+			q_handle = le32_to_cpu(nqcne->cq_handle_low);
+			q_handle |= (u64)le32_to_cpu(nqcne->cq_handle_high) << 32;
+			cq = (struct bnxt_qplib_cq *)q_handle;
+			if (!cq)
+				break;
+			cq->toggle = (le16_to_cpu(nqe->info10_type) & NQ_CN_TOGGLE_MASK) >> NQ_CN_TOGGLE_SFT;
+			cq->dbinfo.toggle = cq->toggle;
+			bnxt_qplib_armen_db(&cq->dbinfo,
+					    DBC_DBC_TYPE_CQ_ARMENA);
+			spin_lock_bh(&cq->compl_lock);
+			atomic_set(&cq->arm_state, 0) ;
+			if (!nq->cqn_handler(nq, (cq)))
+				nq->stats.num_cqne_processed++;
+			else
+				dev_warn(&pdev->dev,
+					 "QPLIB: cqn - type 0x%x not handled\n",
+					 type);
+			cq->cnq_events++;
+			spin_unlock_bh(&cq->compl_lock);
+			break;
+		}
+		case NQ_BASE_TYPE_SRQ_EVENT:
+		{
+			struct bnxt_qplib_srq *srq;
+			struct nq_srq_event *nqsrqe =
+						(struct nq_srq_event *)nqe;
+
+			q_handle = le32_to_cpu(nqsrqe->srq_handle_low);
+			q_handle |= (u64)le32_to_cpu(nqsrqe->srq_handle_high) << 32;
+			srq = (struct bnxt_qplib_srq *)q_handle;
+			bnxt_qplib_armen_db(&srq->dbinfo,
+					    DBC_DBC_TYPE_SRQ_ARMENA);
+			if (!nq->srqn_handler(nq,
+					      (struct bnxt_qplib_srq *)q_handle,
+					      nqsrqe->event))
+				nq->stats.num_srqne_processed++;
+			else
+				dev_warn(&pdev->dev,
+					 "QPLIB: SRQ event 0x%x not handled\n",
+					 nqsrqe->event);
+			break;
+		}
+		default:
+			dev_warn(&pdev->dev,
+				 "QPLIB: nqe with opcode = 0x%x not handled\n",
+				 type);
+			break;
+		}
+		hw_polled++;
+		bnxt_qplib_hwq_incr_cons(nq_hwq->max_elements, &nq_hwq->cons,
+					 1, &nq->nq_db.dbinfo.flags);
+	}
+	nqe = bnxt_qplib_get_qe(nq_hwq, nq_hwq->cons, NULL);
+	if (!NQE_CMP_VALID(nqe, nq->nq_db.dbinfo.flags)) {
+		nq->stats.num_nq_rearm++;
+		bnxt_qplib_ring_nq_db(&nq->nq_db.dbinfo, res->cctx, true);
+	} else if (nq->requested) {
+		bnxt_qplib_ring_nq_db(&nq->nq_db.dbinfo, res->cctx, true);
+		nq->stats.num_tasklet_resched++;
+	}
+	dev_dbg(&pdev->dev, "QPLIB: cqn/srqn/dbqn \n");
+	if (hw_polled >= 0)
+		dev_dbg(&pdev->dev,
+			"QPLIB: serviced %llu/%llu/%llu budget 0x%x reaped 0x%x\n",
+			nq->stats.num_cqne_processed, nq->stats.num_srqne_processed,
+			nq->stats.num_dbqne_processed, budget, hw_polled);
+	dev_dbg(&pdev->dev,
+		"QPLIB: resched_cnt  = %llu arm_count = %llu\n",
+		nq->stats.num_tasklet_resched, nq->stats.num_nq_rearm);
+	spin_unlock_bh(&nq_hwq->lock);
+}
+
+static irqreturn_t bnxt_qplib_nq_irq(int irq, void *dev_instance)
+{
+	struct bnxt_qplib_nq *nq = dev_instance;
+	struct bnxt_qplib_hwq *nq_hwq = &nq->hwq;
+	u32 sw_cons;
+
+	/* Prefetch the NQ element */
+	sw_cons = HWQ_CMP(nq_hwq->cons, nq_hwq);
+	if (sw_cons >= 0)
+		prefetch(bnxt_qplib_get_qe(nq_hwq, sw_cons, NULL));
+
+	bnxt_qplib_service_nq((unsigned long)nq);
+
+	return IRQ_HANDLED;
+}
+
+void bnxt_qplib_nq_stop_irq(struct bnxt_qplib_nq *nq, bool kill)
+{
+	struct bnxt_qplib_res *res;
+
+	if (!nq->requested)
+		return;
+
+	nq->requested = false;
+	res = nq->res;
+	/* Mask h/w interrupt */
+	bnxt_qplib_ring_nq_db(&nq->nq_db.dbinfo, res->cctx, false);
+	/* Sync with last running IRQ handler */
+	synchronize_irq(nq->msix_vec);
+	free_irq(nq->msix_vec, nq);
+	kfree(nq->name);
+	nq->name = NULL;
+}
+
+void bnxt_qplib_disable_nq(struct bnxt_qplib_nq *nq)
+{
+	if (nq->cqn_wq) {
+		destroy_workqueue(nq->cqn_wq);
+		nq->cqn_wq = NULL;
+	}
+	/* Make sure the HW is stopped! */
+	bnxt_qplib_nq_stop_irq(nq, true);
+
+	nq->nq_db.reg.bar_reg = NULL;
+	nq->nq_db.db = NULL;
+
+	nq->cqn_handler = NULL;
+	nq->srqn_handler = NULL;
+	nq->msix_vec = 0;
+}
+
+int bnxt_qplib_nq_start_irq(struct bnxt_qplib_nq *nq, int nq_indx,
+			    int msix_vector, bool need_init)
+{
+	struct bnxt_qplib_res *res;
+	int rc;
+
+	res = nq->res;
+	if (nq->requested)
+		return -EFAULT;
+
+	nq->msix_vec = msix_vector;
+	nq->name = kasprintf(GFP_KERNEL, "bnxt_re-nq-%d@pci:%s\n",
+			     nq_indx, pci_name(res->pdev));
+	if (!nq->name)
+		return -ENOMEM;
+	rc = request_irq(nq->msix_vec, bnxt_qplib_nq_irq, 0, nq->name, nq);
+	if (rc) {
+		kfree(nq->name);
+		nq->name = NULL;
+		return rc;
+	}
+	nq->requested = true;
+	bnxt_qplib_ring_nq_db(&nq->nq_db.dbinfo, res->cctx, true);
+
+	return rc;
+}
+
+static void bnxt_qplib_map_nq_db(struct bnxt_qplib_nq *nq,  u32 reg_offt)
+{
+	struct bnxt_qplib_reg_desc *dbreg;
+	struct bnxt_qplib_nq_db *nq_db;
+	struct bnxt_qplib_res *res;
+
+	nq_db = &nq->nq_db;
+	res = nq->res;
+	dbreg = &res->dpi_tbl.ucreg;
+
+	nq_db->reg.bar_id = dbreg->bar_id;
+	nq_db->reg.bar_base = dbreg->bar_base;
+	nq_db->reg.bar_reg = dbreg->bar_reg + reg_offt;
+	nq_db->reg.len = _is_chip_gen_p5_p7(res->cctx) ? sizeof(u64) :
+						      sizeof(u32);
+
+	nq_db->dbinfo.db = nq_db->reg.bar_reg;
+	nq_db->dbinfo.hwq = &nq->hwq;
+	nq_db->dbinfo.xid = nq->ring_id;
+	nq_db->dbinfo.seed = nq->ring_id;
+	nq_db->dbinfo.flags = 0;
+	spin_lock_init(&nq_db->dbinfo.lock);
+	nq_db->dbinfo.shadow_key = BNXT_QPLIB_DBR_KEY_INVALID;
+	nq_db->dbinfo.res = nq->res;
+
+	return;
+}
+
+int bnxt_qplib_enable_nq(struct bnxt_qplib_nq *nq, int nq_idx,
+			 int msix_vector, int bar_reg_offset,
+			 cqn_handler_t cqn_handler,
+			 srqn_handler_t srqn_handler)
+{
+	struct pci_dev *pdev;
+	int rc;
+
+	pdev = nq->res->pdev;
+	nq->cqn_handler = cqn_handler;
+	nq->srqn_handler = srqn_handler;
+	nq->load = 0;
+	mutex_init(&nq->lock);
+
+	/* Have a task to schedule CQ notifiers in post send case */
+	nq->cqn_wq  = create_singlethread_workqueue("bnxt_qplib_nq\n");
+	if (!nq->cqn_wq)
+		return -ENOMEM;
+
+	bnxt_qplib_map_nq_db(nq, bar_reg_offset);
+	rc = bnxt_qplib_nq_start_irq(nq, nq_idx, msix_vector, true);
+	if (rc) {
+		dev_err(&pdev->dev,
+			"QPLIB: Failed to request irq for nq-idx %d\n", nq_idx);
+		goto fail;
+	}
+	dev_dbg(&pdev->dev, "QPLIB: NQ max = 0x%x\n", nq->hwq.max_elements);
+
+	return 0;
+fail:
+	bnxt_qplib_disable_nq(nq);
+	return rc;
+}
+
+void bnxt_qplib_free_nq_mem(struct bnxt_qplib_nq *nq)
+{
+	if (nq->hwq.max_elements) {
+		bnxt_qplib_free_hwq(nq->res, &nq->hwq);
+		nq->hwq.max_elements = 0;
+	}
+}
+
+int bnxt_qplib_alloc_nq_mem(struct bnxt_qplib_res *res,
+			    struct bnxt_qplib_nq *nq)
+{
+	struct bnxt_qplib_hwq_attr hwq_attr = {};
+	struct bnxt_qplib_sg_info sginfo = {};
+
+	nq->res = res;
+	if (!nq->hwq.max_elements ||
+	    nq->hwq.max_elements > BNXT_QPLIB_NQE_MAX_CNT)
+		nq->hwq.max_elements = BNXT_QPLIB_NQE_MAX_CNT;
+
+	sginfo.pgsize = PAGE_SIZE;
+	sginfo.pgshft = PAGE_SHIFT;
+	hwq_attr.res = res;
+	hwq_attr.sginfo = &sginfo;
+	hwq_attr.depth = nq->hwq.max_elements;
+	hwq_attr.stride = sizeof(struct nq_base);
+	hwq_attr.type = _get_hwq_type(res);
+	if (bnxt_qplib_alloc_init_hwq(&nq->hwq, &hwq_attr)) {
+		dev_err(&res->pdev->dev, "QPLIB: FP NQ allocation failed\n");
+		return -ENOMEM;
+	}
+	nq->budget = 8;
+	return 0;
+}
+
+/* SRQ */
+static int __qplib_destroy_srq(struct bnxt_qplib_rcfw *rcfw,
+			       struct bnxt_qplib_srq *srq)
+{
+	struct creq_destroy_srq_resp resp = {};
+	struct bnxt_qplib_cmdqmsg msg = {};
+	struct cmdq_destroy_srq req = {};
+	/* Configure the request */
+	req.srq_cid = cpu_to_le32(srq->id);
+	bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_DESTROY_SRQ,
+				 sizeof(req));
+	bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL, sizeof(req),
+				sizeof(resp), 0);
+	return bnxt_qplib_rcfw_send_message(rcfw, &msg);
+}
+
+int bnxt_qplib_destroy_srq(struct bnxt_qplib_res *res,
+			   struct bnxt_qplib_srq *srq)
+{
+	struct bnxt_qplib_rcfw *rcfw = res->rcfw;
+	int rc;
+
+	rc = __qplib_destroy_srq(rcfw, srq);
+	if (rc)
+		return rc;
+	bnxt_qplib_free_hwq(res, &srq->hwq);
+	kfree(srq->swq);
+	return 0;
+}
+
+int bnxt_qplib_create_srq(struct bnxt_qplib_res *res,
+			  struct bnxt_qplib_srq *srq)
+{
+	struct bnxt_qplib_hwq_attr hwq_attr = {};
+	struct bnxt_qplib_rcfw *rcfw = res->rcfw;
+	struct creq_create_srq_resp resp = {};
+	struct bnxt_qplib_cmdqmsg msg = {};
+	struct cmdq_create_srq req = {};
+	u16 pg_sz_lvl = 0;
+	u16 srq_size;
+	int rc, idx;
+
+	hwq_attr.res = res;
+	hwq_attr.sginfo = &srq->sginfo;
+	hwq_attr.depth = srq->max_wqe;
+	hwq_attr.stride = srq->wqe_size;
+	hwq_attr.type = HWQ_TYPE_QUEUE;
+	rc = bnxt_qplib_alloc_init_hwq(&srq->hwq, &hwq_attr);
+	if (rc)
+		goto exit;
+	/* Configure the request */
+	req.dpi = cpu_to_le32(srq->dpi->dpi);
+	req.srq_handle = cpu_to_le64(srq);
+	srq_size = min_t(u32, srq->hwq.depth, U16_MAX);
+	req.srq_size = cpu_to_le16(srq_size);
+	pg_sz_lvl |= (_get_base_pg_size(&srq->hwq) <<
+		      CMDQ_CREATE_SRQ_PG_SIZE_SFT);
+	pg_sz_lvl |= (srq->hwq.level & CMDQ_CREATE_SRQ_LVL_MASK);
+	req.pg_size_lvl = cpu_to_le16(pg_sz_lvl);
+	req.pbl = cpu_to_le64(_get_base_addr(&srq->hwq));
+	req.pd_id = cpu_to_le32(srq->pd->id);
+	req.eventq_id = cpu_to_le16(srq->eventq_hw_ring_id);
+	bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_CREATE_SRQ,
+				 sizeof(req));
+	bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL, sizeof(req),
+				sizeof(resp), 0);
+	rc = bnxt_qplib_rcfw_send_message(rcfw, &msg);
+	if (rc)
+		goto fail;
+	if (!srq->is_user) {
+		srq->swq = kcalloc(srq->hwq.depth, sizeof(*srq->swq),
+				   GFP_KERNEL);
+		if (!srq->swq)
+			goto srq_fail;
+		srq->start_idx = 0;
+		srq->last_idx = srq->hwq.depth - 1;
+		for (idx = 0; idx < srq->hwq.depth; idx++)
+			srq->swq[idx].next_idx = idx + 1;
+		srq->swq[srq->last_idx].next_idx = -1;
+	}
+
+	spin_lock_init(&srq->lock);
+	srq->id = le32_to_cpu(resp.xid);
+	srq->cctx = res->cctx;
+	srq->dbinfo.hwq = &srq->hwq;
+	srq->dbinfo.xid = srq->id;
+	srq->dbinfo.db = srq->dpi->dbr;
+	srq->dbinfo.max_slot = 1;
+	srq->dbinfo.priv_db = res->dpi_tbl.priv_db;
+	srq->dbinfo.flags = 0;
+	spin_lock_init(&srq->dbinfo.lock);
+	srq->dbinfo.shadow_key = BNXT_QPLIB_DBR_KEY_INVALID;
+	srq->dbinfo.shadow_key_arm_ena = BNXT_QPLIB_DBR_KEY_INVALID;
+	srq->dbinfo.res = res;
+	srq->dbinfo.seed = srq->id;
+	if (srq->threshold)
+		bnxt_qplib_armen_db(&srq->dbinfo, DBC_DBC_TYPE_SRQ_ARMENA);
+	srq->arm_req = false;
+	return 0;
+srq_fail:
+	__qplib_destroy_srq(rcfw, srq);
+fail:
+	bnxt_qplib_free_hwq(res, &srq->hwq);
+exit:
+	return rc;
+}
+
+int bnxt_qplib_modify_srq(struct bnxt_qplib_res *res,
+			  struct bnxt_qplib_srq *srq)
+{
+	struct bnxt_qplib_hwq *srq_hwq = &srq->hwq;
+	u32 avail = 0;
+
+	avail = __bnxt_qplib_get_avail(srq_hwq);
+	if (avail <= srq->threshold) {
+		srq->arm_req = false;
+		bnxt_qplib_srq_arm_db(&srq->dbinfo);
+	} else {
+		/* Deferred arming */
+		srq->arm_req = true;
+	}
+	return 0;
+}
+
+int bnxt_qplib_query_srq(struct bnxt_qplib_res *res,
+			 struct bnxt_qplib_srq *srq)
+{
+	struct bnxt_qplib_rcfw *rcfw = res->rcfw;
+	struct creq_query_srq_resp resp = {};
+	struct bnxt_qplib_cmdqmsg msg = {};
+	struct creq_query_srq_resp_sb *sb;
+	struct bnxt_qplib_rcfw_sbuf sbuf;
+	struct cmdq_query_srq req = {};
+	int rc = 0;
+
+	bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_QUERY_SRQ,
+				 sizeof(req));
+	sbuf.size = ALIGN(sizeof(*sb), BNXT_QPLIB_CMDQE_UNITS);
+	sbuf.sb = dma_zalloc_coherent(&rcfw->pdev->dev, sbuf.size,
+				       &sbuf.dma_addr, GFP_KERNEL);
+	if (!sbuf.sb)
+		return -ENOMEM;
+	req.resp_size = sbuf.size / BNXT_QPLIB_CMDQE_UNITS;
+	req.srq_cid = cpu_to_le32(srq->id);
+	bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, &sbuf, sizeof(req),
+				sizeof(resp), 0);
+	rc = bnxt_qplib_rcfw_send_message(rcfw, &msg);
+	/* TODO: What to do with the query? */
+	dma_free_coherent(&rcfw->pdev->dev, sbuf.size,
+				  sbuf.sb, sbuf.dma_addr);
+
+	return rc;
+}
+
+int bnxt_qplib_post_srq_recv(struct bnxt_qplib_srq *srq,
+			     struct bnxt_qplib_swqe *wqe)
+{
+	struct bnxt_qplib_hwq *srq_hwq = &srq->hwq;
+	struct sq_sge *hw_sge;
+	struct rq_wqe *srqe;
+	int i, rc = 0, next;
+	u32 avail;
+
+	spin_lock(&srq_hwq->lock);
+	if (srq->start_idx == srq->last_idx) {
+		dev_err(&srq_hwq->pdev->dev, "QPLIB: FP: SRQ (0x%x) is full!\n",
+			srq->id);
+		rc = -EINVAL;
+		spin_unlock(&srq_hwq->lock);
+		goto done;
+	}
+	next = srq->start_idx;
+	srq->start_idx = srq->swq[next].next_idx;
+	spin_unlock(&srq_hwq->lock);
+
+	srqe = bnxt_qplib_get_qe(srq_hwq, srq_hwq->prod, NULL);
+	memset(srqe, 0, srq->wqe_size);
+	/* Calculate wqe_size and data_len */
+	for (i = 0, hw_sge = (struct sq_sge *)srqe->data;
+	     i < wqe->num_sge; i++, hw_sge++) {
+		hw_sge->va_or_pa = cpu_to_le64(wqe->sg_list[i].addr);
+		hw_sge->l_key = cpu_to_le32(wqe->sg_list[i].lkey);
+		hw_sge->size = cpu_to_le32(wqe->sg_list[i].size);
+	}
+	srqe->wqe_type = wqe->type;
+	srqe->flags = wqe->flags;
+	srqe->wqe_size = wqe->num_sge +
+			((offsetof(typeof(*srqe), data) + 15) >> 4);
+	if (!wqe->num_sge)
+		srqe->wqe_size++;
+	srqe->wr_id |= cpu_to_le32((u32)next);
+	srq->swq[next].wr_id = wqe->wr_id;
+	bnxt_qplib_hwq_incr_prod(&srq->dbinfo, srq_hwq, srq->dbinfo.max_slot);
+	/* retaining srq_hwq->cons for this logic actually the lock is only
+	 * required to read srq_hwq->cons.
+	 */
+	spin_lock(&srq_hwq->lock);
+	avail = __bnxt_qplib_get_avail(srq_hwq);
+	spin_unlock(&srq_hwq->lock);
+	/* Ring DB */
+	bnxt_qplib_ring_prod_db(&srq->dbinfo, DBC_DBC_TYPE_SRQ);
+	if (srq->arm_req && avail <= srq->threshold) {
+		srq->arm_req = false;
+		bnxt_qplib_srq_arm_db(&srq->dbinfo);
+	}
+done:
+	return rc;
+}
+
+/* QP */
+static int __qplib_destroy_qp(struct bnxt_qplib_rcfw *rcfw,
+			      struct bnxt_qplib_qp *qp)
+{
+	struct creq_destroy_qp_resp resp = {};
+	struct bnxt_qplib_cmdqmsg msg = {};
+	struct cmdq_destroy_qp req = {};
+
+	req.qp_cid = cpu_to_le32(qp->id);
+	bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_DESTROY_QP,
+				 sizeof(req));
+	bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL, sizeof(req),
+				sizeof(resp), 0);
+	return bnxt_qplib_rcfw_send_message(rcfw, &msg);
+}
+
+static int bnxt_qplib_alloc_init_swq(struct bnxt_qplib_q *que)
+{
+	int rc = 0;
+	int indx;
+
+	que->swq = kcalloc(que->max_wqe, sizeof(*que->swq), GFP_KERNEL);
+	if (!que->swq) {
+		rc = -ENOMEM;
+		goto out;
+	}
+
+	que->swq_start = 0;
+	que->swq_last = que->max_wqe - 1;
+	for (indx = 0; indx < que->max_wqe; indx++)
+		que->swq[indx].next_idx = indx + 1;
+	que->swq[que->swq_last].next_idx = 0; /* Make it circular */
+	que->swq_last = 0;
+out:
+	return rc;
+}
+
+static struct bnxt_qplib_swq *bnxt_qplib_get_swqe(struct bnxt_qplib_q *que,
+						  u32 *swq_idx)
+{
+	u32 idx;
+
+	idx = que->swq_start;
+	if (swq_idx)
+		*swq_idx = idx;
+	return &que->swq[idx];
+}
+
+static void bnxt_qplib_swq_mod_start(struct bnxt_qplib_q *que, u32 idx)
+{
+	que->swq_start = que->swq[idx].next_idx;
+}
+
+static u32 bnxt_qplib_get_stride(void)
+{
+	return sizeof(struct sq_sge);
+}
+
+static u32 bnxt_qplib_get_depth(struct bnxt_qplib_q *que)
+{
+	u8 stride;
+
+	stride = bnxt_qplib_get_stride();
+	return (que->wqe_size * que->max_wqe) / stride;
+}
+
+static u32 _set_sq_size(struct bnxt_qplib_q *que, u8 wqe_mode)
+{
+	/* For Variable mode supply number of 16B slots */
+	return (wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC) ?
+		que->max_wqe : bnxt_qplib_get_depth(que);
+}
+
+static u32 _set_sq_max_slot(u8 wqe_mode)
+{
+	/* for static mode index divisor is 8 */
+	return (wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC) ?
+		sizeof(struct sq_send) / sizeof(struct sq_sge) : 1;
+}
+
+static u32 _set_rq_max_slot(struct bnxt_qplib_q *que)
+{
+	return (que->wqe_size / sizeof(struct sq_sge));
+}
+
+int bnxt_qplib_create_qp1(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp)
+{
+	struct bnxt_qplib_hwq_attr hwq_attr = {};
+	struct bnxt_qplib_rcfw *rcfw = res->rcfw;
+	struct creq_create_qp1_resp resp = {};
+	struct bnxt_qplib_cmdqmsg msg = {};
+	struct bnxt_qplib_q *sq = &qp->sq;
+	struct bnxt_qplib_q *rq = &qp->rq;
+	struct cmdq_create_qp1 req = {};
+	struct bnxt_qplib_reftbl *tbl;
+	unsigned long flag;
+	u8 pg_sz_lvl = 0;
+	u32 qp_flags = 0;
+	int rc;
+
+	/* General */
+	req.type = qp->type;
+	req.dpi = cpu_to_le32(qp->dpi->dpi);
+	req.qp_handle = cpu_to_le64(qp->qp_handle);
+	/* SQ */
+	hwq_attr.res = res;
+	hwq_attr.sginfo = &sq->sginfo;
+	hwq_attr.stride = bnxt_qplib_get_stride();
+	hwq_attr.depth = bnxt_qplib_get_depth(sq);
+	hwq_attr.type = HWQ_TYPE_QUEUE;
+	rc = bnxt_qplib_alloc_init_hwq(&sq->hwq, &hwq_attr);
+	if (rc)
+		goto exit;
+
+	req.sq_size = cpu_to_le32(_set_sq_size(sq, qp->wqe_mode));
+	req.sq_pbl = cpu_to_le64(_get_base_addr(&sq->hwq));
+	pg_sz_lvl = _get_base_pg_size(&sq->hwq) <<
+		    CMDQ_CREATE_QP1_SQ_PG_SIZE_SFT;
+	pg_sz_lvl |= ((sq->hwq.level & CMDQ_CREATE_QP1_SQ_LVL_MASK) <<
+		       CMDQ_CREATE_QP1_SQ_LVL_SFT);
+	req.sq_pg_size_sq_lvl = pg_sz_lvl;
+	req.sq_fwo_sq_sge = cpu_to_le16(((0 << CMDQ_CREATE_QP1_SQ_FWO_SFT) &
+					 CMDQ_CREATE_QP1_SQ_FWO_MASK) |
+					 (sq->max_sge &
+					 CMDQ_CREATE_QP1_SQ_SGE_MASK));
+	req.scq_cid = cpu_to_le32(qp->scq->id);
+
+	/* RQ */
+	if (!qp->srq) {
+		hwq_attr.res = res;
+		hwq_attr.sginfo = &rq->sginfo;
+		hwq_attr.stride = bnxt_qplib_get_stride();
+		hwq_attr.depth = bnxt_qplib_get_depth(rq);
+		hwq_attr.type = HWQ_TYPE_QUEUE;
+		rc = bnxt_qplib_alloc_init_hwq(&rq->hwq, &hwq_attr);
+		if (rc)
+			goto fail_sq;
+		req.rq_size = cpu_to_le32(rq->max_wqe);
+		req.rq_pbl = cpu_to_le64(_get_base_addr(&rq->hwq));
+		pg_sz_lvl = _get_base_pg_size(&rq->hwq) <<
+					      CMDQ_CREATE_QP1_RQ_PG_SIZE_SFT;
+		pg_sz_lvl |= ((rq->hwq.level & CMDQ_CREATE_QP1_RQ_LVL_MASK) <<
+			      CMDQ_CREATE_QP1_RQ_LVL_SFT);
+		req.rq_pg_size_rq_lvl = pg_sz_lvl;
+		req.rq_fwo_rq_sge =
+			cpu_to_le16(((0 << CMDQ_CREATE_QP1_RQ_FWO_SFT) &
+				     CMDQ_CREATE_QP1_RQ_FWO_MASK) |
+				     (rq->max_sge &
+				     CMDQ_CREATE_QP1_RQ_SGE_MASK));
+	} else {
+		/* SRQ */
+		qp_flags |= CMDQ_CREATE_QP1_QP_FLAGS_SRQ_USED;
+		req.srq_cid = cpu_to_le32(qp->srq->id);
+	}
+	req.rcq_cid = cpu_to_le32(qp->rcq->id);
+
+	qp_flags |= CMDQ_CREATE_QP1_QP_FLAGS_RESERVED_LKEY_ENABLE;
+	req.qp_flags = cpu_to_le32(qp_flags);
+	req.pd_id = cpu_to_le32(qp->pd->id);
+
+	bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_CREATE_QP1,
+				 sizeof(req));
+	bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL, sizeof(req),
+				sizeof(resp), 0);
+	rc = bnxt_qplib_rcfw_send_message(rcfw, &msg);
+	if (rc)
+		goto fail_rq;
+
+	rc = bnxt_qplib_alloc_init_swq(sq);
+	if (rc)
+		goto sq_swq;
+
+	if (!qp->srq) {
+		rc = bnxt_qplib_alloc_init_swq(rq);
+		if (rc)
+			goto rq_swq;
+	}
+
+	qp->id = le32_to_cpu(resp.xid);
+	qp->cur_qp_state = CMDQ_MODIFY_QP_NEW_STATE_RESET;
+	qp->cctx = res->cctx;
+	sq->dbinfo.hwq = &sq->hwq;
+	sq->dbinfo.xid = qp->id;
+	sq->dbinfo.db = qp->dpi->dbr;
+	sq->dbinfo.max_slot = _set_sq_max_slot(qp->wqe_mode);
+	sq->dbinfo.flags = 0;
+	spin_lock_init(&sq->dbinfo.lock);
+	sq->dbinfo.shadow_key = BNXT_QPLIB_DBR_KEY_INVALID;
+	sq->dbinfo.res = res;
+	if (rq->max_wqe) {
+		rq->dbinfo.hwq = &rq->hwq;
+		rq->dbinfo.xid = qp->id;
+		rq->dbinfo.db = qp->dpi->dbr;
+		rq->dbinfo.max_slot = _set_rq_max_slot(rq);
+		rq->dbinfo.flags = 0;
+		spin_lock_init(&rq->dbinfo.lock);
+		rq->dbinfo.shadow_key = BNXT_QPLIB_DBR_KEY_INVALID;
+		rq->dbinfo.res = res;
+	}
+
+	tbl = &res->reftbl.qpref;
+	spin_lock_irqsave(&tbl->lock, flag);
+	tbl->rec[tbl->max].xid = qp->id;
+	tbl->rec[tbl->max].handle = qp;
+	spin_unlock_irqrestore(&tbl->lock, flag);
+
+	return 0;
+rq_swq:
+	kfree(sq->swq);
+sq_swq:
+	__qplib_destroy_qp(rcfw, qp);
+fail_rq:
+	bnxt_qplib_free_hwq(res, &rq->hwq);
+fail_sq:
+	bnxt_qplib_free_hwq(res, &sq->hwq);
+exit:
+	return rc;
+}
+
+static void bnxt_qplib_init_psn_ptr(struct bnxt_qplib_qp *qp, int size)
+{
+	struct bnxt_qplib_hwq *sq_hwq;
+	struct bnxt_qplib_q *sq;
+	u64 fpsne, psn_pg;
+	u16 indx_pad = 0;
+
+	sq = &qp->sq;
+	sq_hwq = &sq->hwq;
+	/* First psn entry */
+	fpsne = (u64)bnxt_qplib_get_qe(sq_hwq, sq_hwq->depth, &psn_pg);
+	if (!IS_ALIGNED(fpsne, PAGE_SIZE))
+		indx_pad = (fpsne & ~PAGE_MASK) / size;
+	sq_hwq->pad_pgofft = indx_pad;
+	sq_hwq->pad_pg = (u64 *)psn_pg;
+	sq_hwq->pad_stride = size;
+}
+
+int bnxt_qplib_create_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp)
+{
+	struct bnxt_qplib_hwq_attr hwq_attr = {};
+	struct bnxt_qplib_rcfw *rcfw = res->rcfw;
+	struct bnxt_qplib_sg_info sginfo = {};
+	struct creq_create_qp_resp resp = {};
+	struct bnxt_qplib_cmdqmsg msg = {};
+	struct bnxt_qplib_q *sq = &qp->sq;
+	struct bnxt_qplib_q *rq = &qp->rq;
+	struct cmdq_create_qp req = {};
+	struct bnxt_qplib_reftbl *tbl;
+	struct bnxt_qplib_hwq *xrrq;
+	int rc, req_size, psn_sz;
+	unsigned long flag;
+	u8 pg_sz_lvl = 0;
+	u32 qp_flags = 0;
+	u32 qp_idx;
+	u16 nsge;
+	u32 sqsz;
+
+	qp->cctx = res->cctx;
+	if (res->dattr)
+		qp->dev_cap_flags = res->dattr->dev_cap_flags;
+	/* General */
+	req.type = qp->type;
+	req.dpi = cpu_to_le32(qp->dpi->dpi);
+	req.qp_handle = cpu_to_le64(qp->qp_handle);
+
+	/* SQ */
+	if (qp->type == CMDQ_CREATE_QP_TYPE_RC) {
+		psn_sz = _is_chip_gen_p5_p7(qp->cctx) ?
+			 sizeof(struct sq_psn_search_ext) :
+			 sizeof(struct sq_psn_search);
+		if (BNXT_RE_HW_RETX(qp->dev_cap_flags)) {
+			psn_sz = sizeof(struct sq_msn_search);
+			qp->msn = 0;
+		}
+	} else {
+		psn_sz = 0;
+	}
+
+	hwq_attr.res = res;
+	hwq_attr.sginfo = &sq->sginfo;
+	hwq_attr.stride = bnxt_qplib_get_stride();
+	hwq_attr.depth = bnxt_qplib_get_depth(sq);
+	hwq_attr.aux_stride = psn_sz;
+	hwq_attr.aux_depth = (psn_sz) ?
+				 _set_sq_size(sq, qp->wqe_mode) : 0;
+	/* Update msn tbl size */
+	if (BNXT_RE_HW_RETX(qp->dev_cap_flags) && psn_sz) {
+		if (qp->wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC)
+			hwq_attr.aux_depth = roundup_pow_of_two(_set_sq_size(sq, qp->wqe_mode));
+		else
+			hwq_attr.aux_depth = roundup_pow_of_two(_set_sq_size(sq, qp->wqe_mode)) / 2;
+		qp->msn_tbl_sz = hwq_attr.aux_depth;
+		qp->msn = 0;
+	}
+	hwq_attr.type = HWQ_TYPE_QUEUE;
+	rc = bnxt_qplib_alloc_init_hwq(&sq->hwq, &hwq_attr);
+	if (rc)
+		goto exit;
+
+	sqsz = _set_sq_size(sq, qp->wqe_mode);
+	/* 0xffff is the max sq size hw limits to */
+	if (sqsz > BNXT_QPLIB_MAX_SQSZ) {
+		pr_err("QPLIB: FP: QP (0x%x) exceeds sq size %d\n", qp->id, sqsz);
+		goto fail_sq;
+	}
+	req.sq_size = cpu_to_le32(sqsz);
+	req.sq_pbl = cpu_to_le64(_get_base_addr(&sq->hwq));
+	pg_sz_lvl = _get_base_pg_size(&sq->hwq) <<
+		    CMDQ_CREATE_QP_SQ_PG_SIZE_SFT;
+	pg_sz_lvl |= ((sq->hwq.level & CMDQ_CREATE_QP_SQ_LVL_MASK) <<
+		       CMDQ_CREATE_QP_SQ_LVL_SFT);
+	req.sq_pg_size_sq_lvl = pg_sz_lvl;
+	req.sq_fwo_sq_sge = cpu_to_le16(((0 << CMDQ_CREATE_QP_SQ_FWO_SFT) &
+					 CMDQ_CREATE_QP_SQ_FWO_MASK) |
+					 ((BNXT_RE_HW_RETX(qp->dev_cap_flags)) ?
+					  BNXT_MSN_TBLE_SGE : sq->max_sge &
+					 CMDQ_CREATE_QP_SQ_SGE_MASK));
+	req.scq_cid = cpu_to_le32(qp->scq->id);
+
+	/* RQ/SRQ */
+	if (!qp->srq) {
+		hwq_attr.res = res;
+		hwq_attr.sginfo = &rq->sginfo;
+		hwq_attr.stride = bnxt_qplib_get_stride();
+		hwq_attr.depth = bnxt_qplib_get_depth(rq);
+		hwq_attr.aux_stride = 0;
+		hwq_attr.aux_depth = 0;
+		hwq_attr.type = HWQ_TYPE_QUEUE;
+		rc = bnxt_qplib_alloc_init_hwq(&rq->hwq, &hwq_attr);
+		if (rc)
+			goto fail_sq;
+		req.rq_size = cpu_to_le32(rq->max_wqe);
+		req.rq_pbl = cpu_to_le64(_get_base_addr(&rq->hwq));
+		pg_sz_lvl = _get_base_pg_size(&rq->hwq) <<
+				CMDQ_CREATE_QP_RQ_PG_SIZE_SFT;
+		pg_sz_lvl |= ((rq->hwq.level & CMDQ_CREATE_QP_RQ_LVL_MASK) <<
+				CMDQ_CREATE_QP_RQ_LVL_SFT);
+		req.rq_pg_size_rq_lvl = pg_sz_lvl;
+		nsge = (qp->wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC) ?
+			res->dattr->max_qp_sges : rq->max_sge;
+		req.rq_fwo_rq_sge =
+			cpu_to_le16(((0 << CMDQ_CREATE_QP_RQ_FWO_SFT) &
+				      CMDQ_CREATE_QP_RQ_FWO_MASK) |
+				     (nsge & CMDQ_CREATE_QP_RQ_SGE_MASK));
+	} else {
+		qp_flags |= CMDQ_CREATE_QP_QP_FLAGS_SRQ_USED;
+		req.srq_cid = cpu_to_le32(qp->srq->id);
+	}
+	req.rcq_cid = cpu_to_le32(qp->rcq->id);
+
+	qp_flags |= CMDQ_CREATE_QP_QP_FLAGS_RESERVED_LKEY_ENABLE;
+	qp_flags |= CMDQ_CREATE_QP_QP_FLAGS_FR_PMR_ENABLED;
+	if (qp->sig_type)
+		qp_flags |= CMDQ_CREATE_QP_QP_FLAGS_FORCE_COMPLETION;
+	if (qp->wqe_mode == BNXT_QPLIB_WQE_MODE_VARIABLE)
+		qp_flags |= CMDQ_CREATE_QP_QP_FLAGS_VARIABLE_SIZED_WQE_ENABLED;
+	if (res->cctx->modes.te_bypass)
+		qp_flags |= CMDQ_CREATE_QP_QP_FLAGS_OPTIMIZED_TRANSMIT_ENABLED;
+	if (res->dattr &&
+	    bnxt_ext_stats_supported(qp->cctx, res->dattr->dev_cap_flags, res->is_vf))
+		qp_flags |= CMDQ_CREATE_QP_QP_FLAGS_EXT_STATS_ENABLED;
+	req.qp_flags = cpu_to_le32(qp_flags);
+
+	/* ORRQ and IRRQ */
+	if (psn_sz) {
+		xrrq = &qp->orrq;
+		xrrq->max_elements =
+			ORD_LIMIT_TO_ORRQ_SLOTS(qp->max_rd_atomic);
+		req_size = xrrq->max_elements *
+			   BNXT_QPLIB_MAX_ORRQE_ENTRY_SIZE + PAGE_SIZE - 1;
+		req_size &= ~(PAGE_SIZE - 1);
+		sginfo.pgsize = req_size;
+		sginfo.pgshft = PAGE_SHIFT;
+
+		hwq_attr.res = res;
+		hwq_attr.sginfo = &sginfo;
+		hwq_attr.depth = xrrq->max_elements;
+		hwq_attr.stride = BNXT_QPLIB_MAX_ORRQE_ENTRY_SIZE;
+		hwq_attr.aux_stride = 0;
+		hwq_attr.aux_depth = 0;
+		hwq_attr.type = HWQ_TYPE_CTX;
+		rc = bnxt_qplib_alloc_init_hwq(xrrq, &hwq_attr);
+		if (rc)
+			goto fail_rq;
+		req.orrq_addr = cpu_to_le64(_get_base_addr(xrrq));
+
+		xrrq = &qp->irrq;
+		xrrq->max_elements = IRD_LIMIT_TO_IRRQ_SLOTS(
+						qp->max_dest_rd_atomic);
+		req_size = xrrq->max_elements *
+			   BNXT_QPLIB_MAX_IRRQE_ENTRY_SIZE + PAGE_SIZE - 1;
+		req_size &= ~(PAGE_SIZE - 1);
+		sginfo.pgsize = req_size;
+		hwq_attr.depth =  xrrq->max_elements;
+		hwq_attr.stride = BNXT_QPLIB_MAX_IRRQE_ENTRY_SIZE;
+		rc = bnxt_qplib_alloc_init_hwq(xrrq, &hwq_attr);
+		if (rc)
+			goto fail_orrq;
+		req.irrq_addr = cpu_to_le64(_get_base_addr(xrrq));
+	}
+	req.pd_id = cpu_to_le32(qp->pd->id);
+
+	bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_CREATE_QP,
+				 sizeof(req));
+	bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL, sizeof(req),
+				sizeof(resp), 0);
+	rc = bnxt_qplib_rcfw_send_message(rcfw, &msg);
+	if (rc)
+		goto fail;
+
+	if (!qp->is_user) {
+		rc = bnxt_qplib_alloc_init_swq(sq);
+		if (rc)
+			goto swq_sq;
+		if (!qp->srq) {
+			rc = bnxt_qplib_alloc_init_swq(rq);
+			if (rc)
+				goto swq_rq;
+		}
+		if (psn_sz)
+			bnxt_qplib_init_psn_ptr(qp, psn_sz);
+	}
+	qp->id = le32_to_cpu(resp.xid);
+	qp->cur_qp_state = CMDQ_MODIFY_QP_NEW_STATE_RESET;
+	INIT_LIST_HEAD(&qp->sq_flush);
+	INIT_LIST_HEAD(&qp->rq_flush);
+
+	sq->dbinfo.hwq = &sq->hwq;
+	sq->dbinfo.xid = qp->id;
+	sq->dbinfo.db = qp->dpi->dbr;
+	sq->dbinfo.max_slot = _set_sq_max_slot(qp->wqe_mode);
+	sq->dbinfo.flags = 0;
+	spin_lock_init(&sq->dbinfo.lock);
+	sq->dbinfo.shadow_key = BNXT_QPLIB_DBR_KEY_INVALID;
+	sq->dbinfo.res = res;
+	sq->dbinfo.seed = qp->id;
+	if (rq->max_wqe) {
+		rq->dbinfo.hwq = &rq->hwq;
+		rq->dbinfo.xid = qp->id;
+		rq->dbinfo.db = qp->dpi->dbr;
+		rq->dbinfo.max_slot = _set_rq_max_slot(rq);
+		rq->dbinfo.flags = 0;
+		spin_lock_init(&rq->dbinfo.lock);
+		rq->dbinfo.shadow_key = BNXT_QPLIB_DBR_KEY_INVALID;
+		rq->dbinfo.res = res;
+		rq->dbinfo.seed = qp->id;
+	}
+
+	tbl = &res->reftbl.qpref;
+	qp_idx = map_qp_id_to_tbl_indx(qp->id, tbl);
+	spin_lock_irqsave(&tbl->lock, flag);
+	tbl->rec[qp_idx].xid = qp->id;
+	tbl->rec[qp_idx].handle = qp;
+	spin_unlock_irqrestore(&tbl->lock, flag);
+
+	return 0;
+swq_rq:
+	kfree(sq->swq);
+swq_sq:
+	__qplib_destroy_qp(rcfw, qp);
+fail:
+	bnxt_qplib_free_hwq(res, &qp->irrq);
+fail_orrq:
+	bnxt_qplib_free_hwq(res, &qp->orrq);
+fail_rq:
+	bnxt_qplib_free_hwq(res, &rq->hwq);
+fail_sq:
+	bnxt_qplib_free_hwq(res, &sq->hwq);
+exit:
+	return rc;
+}
+
+static void __filter_modify_flags(struct bnxt_qplib_qp *qp)
+{
+	switch (qp->cur_qp_state) {
+	case CMDQ_MODIFY_QP_NEW_STATE_RESET:
+		switch (qp->state) {
+		case CMDQ_MODIFY_QP_NEW_STATE_INIT:
+			break;
+		default:
+			break;
+		}
+		break;
+	case CMDQ_MODIFY_QP_NEW_STATE_INIT:
+		switch (qp->state) {
+		case CMDQ_MODIFY_QP_NEW_STATE_RTR:
+			if (!(qp->modify_flags &
+			      CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU)) {
+				qp->modify_flags |=
+					CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU;
+				qp->path_mtu = CMDQ_MODIFY_QP_PATH_MTU_MTU_2048;
+			}
+			qp->modify_flags &=
+				~CMDQ_MODIFY_QP_MODIFY_MASK_VLAN_ID;
+			/* Bono FW requires the max_dest_rd_atomic to be >= 1 */
+			if (qp->max_dest_rd_atomic < 1)
+				qp->max_dest_rd_atomic = 1;
+			qp->modify_flags &= ~CMDQ_MODIFY_QP_MODIFY_MASK_SRC_MAC;
+			/* Bono FW 20.6.5 requires SGID_INDEX to be configured */
+			if (!(qp->modify_flags &
+			      CMDQ_MODIFY_QP_MODIFY_MASK_SGID_INDEX)) {
+				qp->modify_flags |=
+					CMDQ_MODIFY_QP_MODIFY_MASK_SGID_INDEX;
+				qp->ah.sgid_index = 0;
+			}
+			break;
+		default:
+			break;
+		}
+		break;
+	case CMDQ_MODIFY_QP_NEW_STATE_RTR:
+		switch (qp->state) {
+		case CMDQ_MODIFY_QP_NEW_STATE_RTS:
+			/* Bono FW requires the max_rd_atomic to be >= 1 */
+			if (qp->max_rd_atomic < 1)
+				qp->max_rd_atomic = 1;
+			qp->modify_flags &=
+				~(CMDQ_MODIFY_QP_MODIFY_MASK_PKEY |
+				  CMDQ_MODIFY_QP_MODIFY_MASK_DGID |
+				  CMDQ_MODIFY_QP_MODIFY_MASK_FLOW_LABEL |
+				  CMDQ_MODIFY_QP_MODIFY_MASK_SGID_INDEX |
+				  CMDQ_MODIFY_QP_MODIFY_MASK_HOP_LIMIT |
+				  CMDQ_MODIFY_QP_MODIFY_MASK_TRAFFIC_CLASS |
+				  CMDQ_MODIFY_QP_MODIFY_MASK_DEST_MAC |
+				  CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU |
+				  CMDQ_MODIFY_QP_MODIFY_MASK_RQ_PSN |
+				  CMDQ_MODIFY_QP_MODIFY_MASK_MIN_RNR_TIMER |
+				  CMDQ_MODIFY_QP_MODIFY_MASK_MAX_DEST_RD_ATOMIC |
+				  CMDQ_MODIFY_QP_MODIFY_MASK_DEST_QP_ID);
+			break;
+		default:
+			break;
+		}
+		break;
+	case CMDQ_MODIFY_QP_NEW_STATE_RTS:
+		break;
+	case CMDQ_MODIFY_QP_NEW_STATE_SQD:
+		break;
+	case CMDQ_MODIFY_QP_NEW_STATE_SQE:
+		break;
+	case CMDQ_MODIFY_QP_NEW_STATE_ERR:
+		break;
+	default:
+		break;
+	}
+}
+
+int bnxt_qplib_modify_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp)
+{
+	struct bnxt_qplib_rcfw *rcfw = res->rcfw;
+	struct creq_modify_qp_resp resp = {};
+	struct bnxt_qplib_cmdqmsg msg = {};
+	struct cmdq_modify_qp req = {};
+	bool ppp_requested = false;
+	u32 temp32[4];
+	u32 bmask;
+	int rc;
+
+	bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_MODIFY_QP,
+				 sizeof(req));
+
+	/* Filter out the qp_attr_mask based on the state->new transition */
+	__filter_modify_flags(qp);
+	bmask = qp->modify_flags;
+	req.modify_mask = cpu_to_le32(qp->modify_flags);
+	req.qp_cid = cpu_to_le32(qp->id);
+	if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_STATE) {
+		req.network_type_en_sqd_async_notify_new_state =
+				(qp->state & CMDQ_MODIFY_QP_NEW_STATE_MASK) |
+				(qp->en_sqd_async_notify == true ?
+					CMDQ_MODIFY_QP_EN_SQD_ASYNC_NOTIFY : 0);
+		if (__can_request_ppp(qp)) {
+			req.path_mtu_pingpong_push_enable =
+				CMDQ_MODIFY_QP_PINGPONG_PUSH_ENABLE;
+			req.pingpong_push_dpi = qp->ppp.dpi;
+			ppp_requested = true;
+		}
+	}
+	req.network_type_en_sqd_async_notify_new_state |= qp->nw_type;
+
+	if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_ACCESS) {
+		req.access = qp->access;
+	}
+	if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_PKEY)
+		req.pkey = IB_DEFAULT_PKEY_FULL;
+
+	if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_QKEY) {
+		req.qkey = cpu_to_le32(qp->qkey);
+	}
+	if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_DGID) {
+		memcpy(temp32, qp->ah.dgid.data, sizeof(struct bnxt_qplib_gid));
+		req.dgid[0] = cpu_to_le32(temp32[0]);
+		req.dgid[1] = cpu_to_le32(temp32[1]);
+		req.dgid[2] = cpu_to_le32(temp32[2]);
+		req.dgid[3] = cpu_to_le32(temp32[3]);
+	}
+	if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_FLOW_LABEL) {
+		req.flow_label = cpu_to_le32(qp->ah.flow_label);
+	}
+	if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_SGID_INDEX) {
+		req.sgid_index = cpu_to_le16(res->sgid_tbl.hw_id[qp->ah.sgid_index]);
+	}
+	if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_HOP_LIMIT) {
+		req.hop_limit = qp->ah.hop_limit;
+	}
+	if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_TRAFFIC_CLASS) {
+		req.traffic_class = qp->ah.traffic_class;
+	}
+	if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_DEST_MAC) {
+		memcpy(req.dest_mac, qp->ah.dmac, 6);
+	}
+	if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU) {
+		req.path_mtu_pingpong_push_enable = qp->path_mtu;
+	}
+	if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_TIMEOUT) {
+		req.timeout = qp->timeout;
+	}
+	if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_RETRY_CNT) {
+		req.retry_cnt = qp->retry_cnt;
+	}
+	if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_RNR_RETRY) {
+		req.rnr_retry = qp->rnr_retry;
+	}
+	if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_MIN_RNR_TIMER) {
+		req.min_rnr_timer = qp->min_rnr_timer;
+	}
+	if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_RQ_PSN) {
+		req.rq_psn = cpu_to_le32(qp->rq.psn);
+	}
+	if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_SQ_PSN) {
+		req.sq_psn = cpu_to_le32(qp->sq.psn);
+	}
+	if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_MAX_RD_ATOMIC) {
+		req.max_rd_atomic =
+			ORD_LIMIT_TO_ORRQ_SLOTS(qp->max_rd_atomic);
+	}
+	if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_MAX_DEST_RD_ATOMIC) {
+		req.max_dest_rd_atomic =
+			IRD_LIMIT_TO_IRRQ_SLOTS(qp->max_dest_rd_atomic);
+	}
+	req.sq_size = cpu_to_le32(qp->sq.hwq.max_elements);
+	req.rq_size = cpu_to_le32(qp->rq.hwq.max_elements);
+	req.sq_sge = cpu_to_le16(qp->sq.max_sge);
+	req.rq_sge = cpu_to_le16(qp->rq.max_sge);
+	req.max_inline_data = cpu_to_le32(qp->max_inline_data);
+	if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_DEST_QP_ID)
+		req.dest_qp_id = cpu_to_le32(qp->dest_qpn);
+	if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_ENABLE_CC)
+		req.enable_cc = cpu_to_le16(CMDQ_MODIFY_QP_ENABLE_CC);
+	if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_TOS_ECN)
+		req.tos_dscp_tos_ecn =
+			((qp->tos_ecn << CMDQ_MODIFY_QP_TOS_ECN_SFT) &
+			 CMDQ_MODIFY_QP_TOS_ECN_MASK);
+	if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_TOS_DSCP)
+		req.tos_dscp_tos_ecn |=
+			((qp->tos_dscp << CMDQ_MODIFY_QP_TOS_DSCP_SFT) &
+			 CMDQ_MODIFY_QP_TOS_DSCP_MASK);
+	req.vlan_pcp_vlan_dei_vlan_id = cpu_to_le16(qp->vlan_id);
+	bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL, sizeof(req),
+				sizeof(resp), 0);
+	msg.qp_state = qp->state;
+
+	rc = bnxt_qplib_rcfw_send_message(rcfw, &msg);
+	if (rc == -ETIMEDOUT && (qp->state == CMDQ_MODIFY_QP_NEW_STATE_ERR)) {
+		qp->cur_qp_state = qp->state;
+		return 0;
+	} else if (rc) {
+		return rc;
+	}
+	if (qp->state == CMDQ_MODIFY_QP_NEW_STATE_RTR)
+		qp->lag_src_mac = be32_to_cpu(resp.lag_src_mac);
+
+	if (ppp_requested)
+		qp->ppp.st_idx_en = resp.pingpong_push_state_index_enabled;
+
+	qp->cur_qp_state = qp->state;
+	return 0;
+}
+
+int bnxt_qplib_query_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp)
+{
+	struct bnxt_qplib_rcfw *rcfw = res->rcfw;
+	struct creq_query_qp_resp resp = {};
+	struct bnxt_qplib_cmdqmsg msg = {};
+	struct bnxt_qplib_rcfw_sbuf sbuf;
+	struct creq_query_qp_resp_sb *sb;
+	struct cmdq_query_qp req = {};
+	u32 temp32[4];
+	int i, rc;
+
+	sbuf.size = ALIGN(sizeof(*sb), BNXT_QPLIB_CMDQE_UNITS);
+	sbuf.sb = dma_zalloc_coherent(&rcfw->pdev->dev, sbuf.size,
+				       &sbuf.dma_addr, GFP_KERNEL);
+	if (!sbuf.sb)
+		return -ENOMEM;
+	sb = sbuf.sb;
+
+	bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_QUERY_QP,
+				 sizeof(req));
+	req.qp_cid = cpu_to_le32(qp->id);
+	req.resp_size = sbuf.size / BNXT_QPLIB_CMDQE_UNITS;
+	bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, &sbuf, sizeof(req),
+				sizeof(resp), 0);
+	rc = bnxt_qplib_rcfw_send_message(rcfw, &msg);
+	if (rc)
+		goto bail;
+
+	/* Extract the context from the side buffer */
+	qp->state = sb->en_sqd_async_notify_state &
+			CREQ_QUERY_QP_RESP_SB_STATE_MASK;
+	qp->cur_qp_state = qp->state;
+	qp->en_sqd_async_notify = sb->en_sqd_async_notify_state &
+				  CREQ_QUERY_QP_RESP_SB_EN_SQD_ASYNC_NOTIFY ?
+				  true : false;
+	qp->access = sb->access;
+	qp->pkey_index = le16_to_cpu(sb->pkey);
+	qp->qkey = le32_to_cpu(sb->qkey);
+
+	temp32[0] = le32_to_cpu(sb->dgid[0]);
+	temp32[1] = le32_to_cpu(sb->dgid[1]);
+	temp32[2] = le32_to_cpu(sb->dgid[2]);
+	temp32[3] = le32_to_cpu(sb->dgid[3]);
+	memcpy(qp->ah.dgid.data, temp32, sizeof(qp->ah.dgid.data));
+
+	qp->ah.flow_label = le32_to_cpu(sb->flow_label);
+
+	qp->ah.sgid_index = 0;
+	for (i = 0; i < res->sgid_tbl.max; i++) {
+		if (res->sgid_tbl.hw_id[i] == le16_to_cpu(sb->sgid_index)) {
+			qp->ah.sgid_index = i;
+			break;
+		}
+	}
+	if (i == res->sgid_tbl.max)
+		dev_dbg(&res->pdev->dev,
+			"QPLIB: SGID not found qp->id = 0x%x sgid_index = 0x%x\n",
+			qp->id, le16_to_cpu(sb->sgid_index));
+
+	qp->ah.hop_limit = sb->hop_limit;
+	qp->ah.traffic_class = sb->traffic_class;
+	memcpy(qp->ah.dmac, sb->dest_mac, ETH_ALEN);
+	qp->ah.vlan_id = le16_to_cpu(sb->path_mtu_dest_vlan_id) &
+				CREQ_QUERY_QP_RESP_SB_VLAN_ID_MASK >>
+				CREQ_QUERY_QP_RESP_SB_VLAN_ID_SFT;
+	qp->path_mtu = le16_to_cpu(sb->path_mtu_dest_vlan_id) &
+				    CREQ_QUERY_QP_RESP_SB_PATH_MTU_MASK;
+	qp->timeout = sb->timeout;
+	qp->retry_cnt = sb->retry_cnt;
+	qp->rnr_retry = sb->rnr_retry;
+	qp->min_rnr_timer = sb->min_rnr_timer;
+	qp->rq.psn = le32_to_cpu(sb->rq_psn);
+	qp->max_rd_atomic = ORRQ_SLOTS_TO_ORD_LIMIT(sb->max_rd_atomic);
+	qp->sq.psn = le32_to_cpu(sb->sq_psn);
+	qp->max_dest_rd_atomic =
+			IRRQ_SLOTS_TO_IRD_LIMIT(sb->max_dest_rd_atomic);
+	qp->sq.max_wqe = qp->sq.hwq.max_elements;
+	qp->rq.max_wqe = qp->rq.hwq.max_elements;
+	qp->sq.max_sge = le16_to_cpu(sb->sq_sge);
+	qp->rq.max_sge = le16_to_cpu(sb->rq_sge);
+	qp->max_inline_data = le32_to_cpu(sb->max_inline_data);
+	qp->dest_qpn = le32_to_cpu(sb->dest_qp_id);
+	memcpy(qp->smac, sb->src_mac, ETH_ALEN);
+	qp->vlan_id = le16_to_cpu(sb->vlan_pcp_vlan_dei_vlan_id);
+	qp->port_id = le16_to_cpu(sb->port_id);
+bail:
+	dma_free_coherent(&rcfw->pdev->dev, sbuf.size,
+				  sbuf.sb, sbuf.dma_addr);
+	return rc;
+}
+
+
+static void __clean_cq(struct bnxt_qplib_cq *cq, u64 qp)
+{
+	struct bnxt_qplib_hwq *cq_hwq = &cq->hwq;
+	u32 peek_flags, peek_cons;
+	struct cq_base *hw_cqe;
+	int i;
+
+	peek_flags = cq->dbinfo.flags;
+	peek_cons = cq_hwq->cons;
+	for (i = 0; i < cq_hwq->depth; i++) {
+		hw_cqe = bnxt_qplib_get_qe(cq_hwq, peek_cons, NULL);
+		if (CQE_CMP_VALID(hw_cqe, peek_flags)) {
+			dma_rmb();
+			switch (hw_cqe->cqe_type_toggle & CQ_BASE_CQE_TYPE_MASK) {
+			case CQ_BASE_CQE_TYPE_REQ:
+			case CQ_BASE_CQE_TYPE_TERMINAL:
+			{
+				struct cq_req *cqe = (struct cq_req *)hw_cqe;
+
+				if (qp == le64_to_cpu(cqe->qp_handle))
+					cqe->qp_handle = 0;
+				break;
+			}
+			case CQ_BASE_CQE_TYPE_RES_RC:
+			case CQ_BASE_CQE_TYPE_RES_UD:
+			case CQ_BASE_CQE_TYPE_RES_RAWETH_QP1:
+			{
+				struct cq_res_rc *cqe = (struct cq_res_rc *)hw_cqe;
+
+				if (qp == le64_to_cpu(cqe->qp_handle))
+					cqe->qp_handle = 0;
+				break;
+			}
+			default:
+				break;
+			}
+		}
+		bnxt_qplib_hwq_incr_cons(cq_hwq->depth, &peek_cons,
+					 1, &peek_flags);
+	}
+}
+
+int bnxt_qplib_destroy_qp(struct bnxt_qplib_res *res,
+			  struct bnxt_qplib_qp *qp)
+{
+	struct bnxt_qplib_rcfw *rcfw = res->rcfw;
+	struct bnxt_qplib_reftbl *tbl;
+	unsigned long flags;
+	u32 qp_idx;
+	int rc;
+
+	tbl = &res->reftbl.qpref;
+	qp_idx = map_qp_id_to_tbl_indx(qp->id, tbl);
+	spin_lock_irqsave(&tbl->lock, flags);
+	tbl->rec[qp_idx].xid = BNXT_QPLIB_QP_ID_INVALID;
+	tbl->rec[qp_idx].handle = NULL;
+	spin_unlock_irqrestore(&tbl->lock, flags);
+
+	rc = __qplib_destroy_qp(rcfw, qp);
+	if (rc) {
+		spin_lock_irqsave(&tbl->lock, flags);
+		tbl->rec[qp_idx].xid = qp->id;
+		tbl->rec[qp_idx].handle = qp;
+		spin_unlock_irqrestore(&tbl->lock, flags);
+		return rc;
+	}
+
+	return 0;
+}
+
+void bnxt_qplib_free_qp_res(struct bnxt_qplib_res *res,
+			    struct bnxt_qplib_qp *qp)
+{
+	if (qp->irrq.max_elements)
+		bnxt_qplib_free_hwq(res, &qp->irrq);
+	if (qp->orrq.max_elements)
+		bnxt_qplib_free_hwq(res, &qp->orrq);
+
+	if (!qp->is_user)
+		kfree(qp->rq.swq);
+	bnxt_qplib_free_hwq(res, &qp->rq.hwq);
+
+	if (!qp->is_user)
+		kfree(qp->sq.swq);
+	bnxt_qplib_free_hwq(res, &qp->sq.hwq);
+}
+
+void *bnxt_qplib_get_qp1_sq_buf(struct bnxt_qplib_qp *qp,
+				struct bnxt_qplib_sge *sge)
+{
+	struct bnxt_qplib_q *sq = &qp->sq;
+	struct bnxt_qplib_hdrbuf *buf;
+	u32 sw_prod;
+
+	memset(sge, 0, sizeof(*sge));
+
+	buf = qp->sq_hdr_buf;
+	if (buf) {
+		sw_prod = sq->swq_start;
+		sge->addr = (dma_addr_t)(buf->dma_map + sw_prod * buf->step);
+		sge->lkey = 0xFFFFFFFF;
+		sge->size = buf->step;
+		return buf->va + sw_prod * sge->size;
+	}
+	return NULL;
+}
+
+u32 bnxt_qplib_get_rq_prod_index(struct bnxt_qplib_qp *qp)
+{
+	struct bnxt_qplib_q *rq = &qp->rq;
+
+	return rq->swq_start;
+}
+
+void *bnxt_qplib_get_qp1_rq_buf(struct bnxt_qplib_qp *qp,
+				struct bnxt_qplib_sge *sge)
+{
+	struct bnxt_qplib_q *rq = &qp->rq;
+	struct bnxt_qplib_hdrbuf *buf;
+	u32 sw_prod;
+
+	memset(sge, 0, sizeof(*sge));
+
+	buf = qp->rq_hdr_buf;
+	if (buf) {
+		sw_prod = rq->swq_start;
+		sge->addr = (dma_addr_t)(buf->dma_map + sw_prod * buf->step);
+		sge->lkey = 0xFFFFFFFF;
+		sge->size = buf->step;
+		return buf->va + sw_prod * sge->size;
+	}
+	return NULL;
+}
+
+/* Fil the MSN table into the next psn row */
+static void bnxt_qplib_fill_msn_search(struct bnxt_qplib_qp *qp,
+				       struct bnxt_qplib_swqe *wqe,
+				       struct bnxt_qplib_swq *swq)
+{
+	struct sq_msn_search *msns;
+	u32 start_psn, next_psn;
+	u16 start_idx;
+
+	msns = (struct sq_msn_search *)swq->psn_search;
+	msns->start_idx_next_psn_start_psn = 0;
+
+	start_psn = swq->start_psn;
+	next_psn = swq->next_psn;
+	start_idx = swq->slot_idx;
+	msns->start_idx_next_psn_start_psn |=
+		bnxt_re_update_msn_tbl(start_idx, next_psn, start_psn);
+	pr_debug("QP_LIB MSN %d START_IDX %u NEXT_PSN %u START_PSN %u\n",
+		 qp->msn,
+		 (u16)
+		 cpu_to_le16(BNXT_RE_MSN_IDX(msns->start_idx_next_psn_start_psn)),
+		 (u32)
+		 cpu_to_le32(BNXT_RE_MSN_NPSN(msns->start_idx_next_psn_start_psn)),
+		 (u32)
+		 cpu_to_le32(BNXT_RE_MSN_SPSN(msns->start_idx_next_psn_start_psn)));
+	qp->msn++;
+	qp->msn %= qp->msn_tbl_sz;
+}
+
+static void bnxt_qplib_fill_psn_search(struct bnxt_qplib_qp *qp,
+				       struct bnxt_qplib_swqe *wqe,
+				       struct bnxt_qplib_swq *swq)
+{
+	struct sq_psn_search_ext *psns_ext;
+	struct sq_psn_search *psns;
+	u32 flg_npsn;
+	u32 op_spsn;
+
+	if (!swq->psn_search)
+		return;
+
+	/* Handle MSN differently on cap flags  */
+	if (BNXT_RE_HW_RETX(qp->dev_cap_flags)) {
+		bnxt_qplib_fill_msn_search(qp, wqe, swq);
+		return;
+	}
+	psns = (struct sq_psn_search *)swq->psn_search;
+	psns_ext = (struct sq_psn_search_ext *)swq->psn_search;
+
+	op_spsn = ((swq->start_psn << SQ_PSN_SEARCH_START_PSN_SFT) &
+		   SQ_PSN_SEARCH_START_PSN_MASK);
+	op_spsn |= ((wqe->type << SQ_PSN_SEARCH_OPCODE_SFT) &
+		    SQ_PSN_SEARCH_OPCODE_MASK);
+	flg_npsn = ((swq->next_psn << SQ_PSN_SEARCH_NEXT_PSN_SFT) &
+		    SQ_PSN_SEARCH_NEXT_PSN_MASK);
+
+	if (_is_chip_gen_p5_p7(qp->cctx)) {
+		psns_ext->opcode_start_psn = cpu_to_le32(op_spsn);
+		psns_ext->flags_next_psn = cpu_to_le32(flg_npsn);
+		psns_ext->start_slot_idx = cpu_to_le16(swq->slot_idx);
+	} else {
+		psns->opcode_start_psn = cpu_to_le32(op_spsn);
+		psns->flags_next_psn = cpu_to_le32(flg_npsn);
+	}
+}
+
+static u16 _calc_ilsize(struct bnxt_qplib_swqe *wqe)
+{
+	u16 size = 0;
+	int indx;
+
+	for (indx = 0; indx < wqe->num_sge; indx++)
+		size += wqe->sg_list[indx].size;
+	return size;
+}
+
+static int bnxt_qplib_put_inline(struct bnxt_qplib_qp *qp,
+				 struct bnxt_qplib_swqe *wqe,
+				 u32 *sw_prod)
+{
+	struct bnxt_qplib_hwq *sq_hwq;
+	int len, t_len, offt = 0;
+	int t_cplen = 0, cplen;
+	bool pull_dst = true;
+	void *il_dst = NULL;
+	void *il_src = NULL;
+	int indx;
+
+	sq_hwq = &qp->sq.hwq;
+	t_len = 0;
+	for (indx = 0; indx < wqe->num_sge; indx++) {
+		len = wqe->sg_list[indx].size;
+		il_src = (void *)wqe->sg_list[indx].addr;
+		t_len += len;
+		if (t_len > qp->max_inline_data)
+			goto bad;
+		while (len) {
+			if (pull_dst) {
+				pull_dst = false;
+				il_dst = bnxt_qplib_get_qe(sq_hwq, ((*sw_prod) %
+							   sq_hwq->depth), NULL);
+				(*sw_prod)++;
+				t_cplen = 0;
+				offt = 0;
+			}
+			cplen = min_t(int, len, sizeof(struct sq_sge));
+			cplen = min_t(int, cplen,
+				      (sizeof(struct sq_sge) - offt));
+			memcpy(il_dst, il_src, cplen);
+			t_cplen += cplen;
+			il_src += cplen;
+			il_dst += cplen;
+			offt += cplen;
+			len -= cplen;
+			if (t_cplen == sizeof(struct sq_sge))
+				pull_dst = true;
+		}
+	}
+
+	return t_len;
+bad:
+	return -ENOMEM;
+}
+
+static int bnxt_qplib_put_sges(struct bnxt_qplib_hwq *sq_hwq,
+			       struct bnxt_qplib_sge *ssge,
+			       u32 nsge, u32 *sw_prod)
+{
+	struct sq_sge *dsge;
+	int indx, len = 0;
+
+	for (indx = 0; indx < nsge; indx++, (*sw_prod)++) {
+		dsge = bnxt_qplib_get_qe(sq_hwq, ((*sw_prod) % sq_hwq->depth), NULL);
+		dsge->va_or_pa = cpu_to_le64(ssge[indx].addr);
+		dsge->l_key = cpu_to_le32(ssge[indx].lkey);
+		dsge->size = cpu_to_le32(ssge[indx].size);
+		len += ssge[indx].size;
+	}
+	return len;
+}
+
+static u16 _calculate_wqe_byte(struct bnxt_qplib_qp *qp,
+			       struct bnxt_qplib_swqe *wqe, u16 *wqe_byte)
+{
+	u16 wqe_size;
+	u32 ilsize;
+	u16 nsge;
+
+	nsge = wqe->num_sge;
+	if (wqe->flags & BNXT_QPLIB_SWQE_FLAGS_INLINE) {
+		ilsize = _calc_ilsize(wqe);
+		wqe_size = (ilsize > qp->max_inline_data) ?
+			    qp->max_inline_data : ilsize;
+		wqe_size = ALIGN(wqe_size, sizeof(struct sq_sge));
+	} else {
+		wqe_size = nsge * sizeof(struct sq_sge);
+	}
+	/* Adding sq_send_hdr is a misnomer, for rq also hdr size is same. */
+	wqe_size += sizeof(struct sq_send_hdr);
+	if (wqe_byte)
+		*wqe_byte = wqe_size;
+	return wqe_size / sizeof(struct sq_sge);
+}
+
+static u16 _translate_q_full_delta(struct bnxt_qplib_q *que, u16 wqe_bytes)
+{
+	/* For Cu/Wh delta = 128, stride = 16, wqe_bytes = 128
+	 * For Gen-p5 B/C mode delta = 0, stride = 16, wqe_bytes = 128.
+	 * For Gen-p5 delta = 0, stride = 16, 32 <= wqe_bytes <= 512.
+	 * when 8916 is disabled.
+	 */
+	return (que->q_full_delta * wqe_bytes) / que->hwq.element_size;
+}
+
+static void bnxt_qplib_pull_psn_buff(struct bnxt_qplib_qp *qp, struct bnxt_qplib_q *sq,
+				     struct bnxt_qplib_swq *swq, bool hw_retx)
+{
+	struct bnxt_qplib_hwq *sq_hwq;
+	u32 pg_num, pg_indx;
+	void *buff;
+	u32 tail;
+
+	sq_hwq = &sq->hwq;
+	if (!sq_hwq->pad_pg)
+		return;
+
+	tail = swq->slot_idx / sq->dbinfo.max_slot;
+	if (hw_retx)
+		tail %= qp->msn_tbl_sz;
+	pg_num = (tail + sq_hwq->pad_pgofft) / (PAGE_SIZE / sq_hwq->pad_stride);
+	pg_indx = (tail + sq_hwq->pad_pgofft) % (PAGE_SIZE / sq_hwq->pad_stride);
+	buff = (void *)(sq_hwq->pad_pg[pg_num] + pg_indx * sq_hwq->pad_stride);
+	/* the start ptr for buff is same ie after the SQ */
+	swq->psn_search = buff;
+}
+
+void bnxt_qplib_post_send_db(struct bnxt_qplib_qp *qp)
+{
+	struct bnxt_qplib_q *sq = &qp->sq;
+
+	bnxt_qplib_ring_prod_db(&sq->dbinfo, DBC_DBC_TYPE_SQ);
+}
+
+int bnxt_qplib_post_send(struct bnxt_qplib_qp *qp,
+			 struct bnxt_qplib_swqe *wqe)
+{
+	struct bnxt_qplib_nq_work *nq_work = NULL;
+	int i, rc = 0, data_len = 0, pkt_num = 0;
+	struct bnxt_qplib_q *sq = &qp->sq;
+	struct bnxt_qplib_hwq *sq_hwq;
+	struct bnxt_qplib_swq *swq;
+	bool sch_handler = false;
+	u16 slots_needed;
+	void *base_hdr;
+	void *ext_hdr;
+	__le32 temp32;
+	u16 qfd_slots;
+	u8 wqe_slots;
+	u16 wqe_size;
+	u32 sw_prod;
+	u32 wqe_idx;
+
+	sq_hwq = &sq->hwq;
+	if (qp->state != CMDQ_MODIFY_QP_NEW_STATE_RTS &&
+	    qp->state != CMDQ_MODIFY_QP_NEW_STATE_ERR) {
+		dev_err(&sq_hwq->pdev->dev,
+			"QPLIB: FP: QP (0x%x) is in the 0x%x state\n",
+			qp->id, qp->state);
+		rc = -EINVAL;
+		goto done;
+	}
+
+	wqe_slots = _calculate_wqe_byte(qp, wqe, &wqe_size);
+	slots_needed = (qp->wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC) ?
+			sq->dbinfo.max_slot : wqe_slots;
+	qfd_slots = _translate_q_full_delta(sq, wqe_size);
+	if (bnxt_qplib_queue_full(sq_hwq, (slots_needed + qfd_slots))) {
+		dev_err(&sq_hwq->pdev->dev,
+			"QPLIB: FP: QP (0x%x) SQ is full!\n", qp->id);
+		dev_err(&sq_hwq->pdev->dev,
+			"QPLIB: prod = %#x cons = %#x qdepth = %#x delta = %#x slots = %#x\n",
+			HWQ_CMP(sq_hwq->prod, sq_hwq),
+			HWQ_CMP(sq_hwq->cons, sq_hwq),
+			sq_hwq->max_elements, qfd_slots, slots_needed);
+		dev_err(&sq_hwq->pdev->dev,
+			"QPLIB: phantom_wqe_cnt: %d phantom_cqe_cnt: %d\n",
+			sq->phantom_wqe_cnt, sq->phantom_cqe_cnt);
+		rc = -ENOMEM;
+		goto done;
+	}
+
+	sw_prod = sq_hwq->prod;
+	swq = bnxt_qplib_get_swqe(sq, &wqe_idx);
+	swq->slot_idx = sw_prod;
+	bnxt_qplib_pull_psn_buff(qp, sq, swq, BNXT_RE_HW_RETX(qp->dev_cap_flags));
+
+	swq->wr_id = wqe->wr_id;
+	swq->type = wqe->type;
+	swq->flags = wqe->flags;
+	swq->slots = slots_needed;
+	swq->start_psn = sq->psn & BTH_PSN_MASK;
+	if (qp->sig_type || wqe->flags & BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP)
+		swq->flags |= SQ_SEND_FLAGS_SIGNAL_COMP;
+
+	dev_dbg(&sq_hwq->pdev->dev,
+		"QPLIB: FP: QP(0x%x) post SQ wr_id[%d] = 0x%llx\n",
+		qp->id, wqe_idx, swq->wr_id);
+	if (qp->cur_qp_state == CMDQ_MODIFY_QP_NEW_STATE_ERR) {
+		sch_handler = true;
+		dev_dbg(&sq_hwq->pdev->dev,
+			"%s Error QP. Scheduling for poll_cq\n", __func__);
+		goto queue_err;
+	}
+
+	base_hdr = bnxt_qplib_get_qe(sq_hwq, sw_prod, NULL);
+	sw_prod++;
+	ext_hdr = bnxt_qplib_get_qe(sq_hwq, (sw_prod % sq_hwq->depth), NULL);
+	sw_prod++;
+	memset(base_hdr, 0, sizeof(struct sq_sge));
+	memset(ext_hdr, 0, sizeof(struct sq_sge));
+
+	if (wqe->flags & BNXT_QPLIB_SWQE_FLAGS_INLINE)
+		data_len = bnxt_qplib_put_inline(qp, wqe, &sw_prod);
+	else
+		data_len = bnxt_qplib_put_sges(sq_hwq, wqe->sg_list,
+					       wqe->num_sge, &sw_prod);
+	if (data_len < 0)
+		goto queue_err;
+	/* Specifics */
+	switch (wqe->type) {
+	case BNXT_QPLIB_SWQE_TYPE_SEND:
+		if (qp->type == CMDQ_CREATE_QP_TYPE_RAW_ETHERTYPE ||
+		    qp->type == CMDQ_CREATE_QP1_TYPE_GSI) {
+			/* Assemble info for Raw Ethertype QPs */
+			struct sq_send_raweth_qp1_hdr *sqe = base_hdr;
+			struct sq_raw_ext_hdr *ext_sqe = ext_hdr;
+
+			sqe->wqe_type = wqe->type;
+			sqe->flags = wqe->flags;
+			sqe->wqe_size = wqe_slots;
+			sqe->cfa_action = cpu_to_le16(wqe->rawqp1.cfa_action);
+			sqe->lflags = cpu_to_le16(wqe->rawqp1.lflags);
+			sqe->length = cpu_to_le32(data_len);
+			ext_sqe->cfa_meta = cpu_to_le32((wqe->rawqp1.cfa_meta &
+				SQ_SEND_RAWETH_QP1_CFA_META_VLAN_VID_MASK) <<
+				SQ_SEND_RAWETH_QP1_CFA_META_VLAN_VID_SFT);
+
+			dev_dbg(&sq_hwq->pdev->dev,
+				"QPLIB: FP: RAW/QP1 Send WQE:\n"
+				"\twqe_type = 0x%x\n"
+				"\tflags = 0x%x\n"
+				"\twqe_size = 0x%x\n"
+				"\tlflags = 0x%x\n"
+				"\tcfa_action = 0x%x\n"
+				"\tlength = 0x%x\n"
+				"\tcfa_meta = 0x%x\n",
+				sqe->wqe_type, sqe->flags, sqe->wqe_size,
+				sqe->lflags, sqe->cfa_action,
+				sqe->length, ext_sqe->cfa_meta);
+			break;
+		}
+		fallthrough;
+	case BNXT_QPLIB_SWQE_TYPE_SEND_WITH_IMM:
+		fallthrough;
+	case BNXT_QPLIB_SWQE_TYPE_SEND_WITH_INV:
+	{
+		struct sq_send_hdr *sqe = base_hdr;
+		struct sq_ud_ext_hdr *ext_sqe = ext_hdr;
+
+		sqe->wqe_type = wqe->type;
+		sqe->flags = wqe->flags;
+		sqe->wqe_size = wqe_slots;
+		sqe->inv_key_or_imm_data = cpu_to_le32(wqe->send.inv_key);
+		if (qp->type == CMDQ_CREATE_QP_TYPE_UD ||
+		    qp->type == CMDQ_CREATE_QP_TYPE_GSI) {
+			sqe->q_key = cpu_to_le32(wqe->send.q_key);
+			sqe->length = cpu_to_le32(data_len);
+			ext_sqe->dst_qp = cpu_to_le32(
+					wqe->send.dst_qp & SQ_SEND_DST_QP_MASK);
+			ext_sqe->avid = cpu_to_le32(wqe->send.avid &
+						SQ_SEND_AVID_MASK);
+			sq->psn = (sq->psn + 1) & BTH_PSN_MASK;
+		} else {
+			sqe->length = cpu_to_le32(data_len);
+			if (qp->mtu)
+				pkt_num = (data_len + qp->mtu - 1) / qp->mtu;
+			if (!pkt_num)
+				pkt_num = 1;
+			sq->psn = (sq->psn + pkt_num) & BTH_PSN_MASK;
+		}
+		dev_dbg(&sq_hwq->pdev->dev,
+			"QPLIB: FP: Send WQE:\n"
+			"\twqe_type = 0x%x\n"
+			"\tflags = 0x%x\n"
+			"\twqe_size = 0x%x\n"
+			"\tinv_key/immdata = 0x%x\n"
+			"\tq_key = 0x%x\n"
+			"\tdst_qp = 0x%x\n"
+			"\tlength = 0x%x\n"
+			"\tavid = 0x%x\n",
+			sqe->wqe_type, sqe->flags, sqe->wqe_size,
+			sqe->inv_key_or_imm_data, sqe->q_key, ext_sqe->dst_qp,
+			sqe->length, ext_sqe->avid);
+		break;
+	}
+	case BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE:
+		/* fall-thru */
+	case BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE_WITH_IMM:
+		/* fall-thru */
+	case BNXT_QPLIB_SWQE_TYPE_RDMA_READ:
+	{
+		struct sq_rdma_hdr *sqe = base_hdr;
+		struct sq_rdma_ext_hdr *ext_sqe = ext_hdr;
+
+		sqe->wqe_type = wqe->type;
+		sqe->flags = wqe->flags;
+		sqe->wqe_size = wqe_slots;
+		sqe->imm_data = cpu_to_le32(wqe->rdma.inv_key);
+		sqe->length = cpu_to_le32((u32)data_len);
+		ext_sqe->remote_va = cpu_to_le64(wqe->rdma.remote_va);
+		ext_sqe->remote_key = cpu_to_le32(wqe->rdma.r_key);
+		if (qp->mtu)
+			pkt_num = (data_len + qp->mtu - 1) / qp->mtu;
+		if (!pkt_num)
+			pkt_num = 1;
+		sq->psn = (sq->psn + pkt_num) & BTH_PSN_MASK;
+
+		dev_dbg(&sq_hwq->pdev->dev,
+			"QPLIB: FP: RDMA WQE:\n"
+			"\twqe_type = 0x%x\n"
+			"\tflags = 0x%x\n"
+			"\twqe_size = 0x%x\n"
+			"\timmdata = 0x%x\n"
+			"\tlength = 0x%x\n"
+			"\tremote_va = 0x%llx\n"
+			"\tremote_key = 0x%x\n",
+			sqe->wqe_type, sqe->flags, sqe->wqe_size,
+			sqe->imm_data, sqe->length, ext_sqe->remote_va,
+			ext_sqe->remote_key);
+		break;
+	}
+	case BNXT_QPLIB_SWQE_TYPE_ATOMIC_CMP_AND_SWP:
+		/* fall-thru */
+	case BNXT_QPLIB_SWQE_TYPE_ATOMIC_FETCH_AND_ADD:
+	{
+		struct sq_atomic_hdr *sqe = base_hdr;
+		struct sq_atomic_ext_hdr *ext_sqe = ext_hdr;
+
+		sqe->wqe_type = wqe->type;
+		sqe->flags = wqe->flags;
+		sqe->remote_key = cpu_to_le32(wqe->atomic.r_key);
+		sqe->remote_va = cpu_to_le64(wqe->atomic.remote_va);
+		ext_sqe->swap_data = cpu_to_le64(wqe->atomic.swap_data);
+		ext_sqe->cmp_data = cpu_to_le64(wqe->atomic.cmp_data);
+		if (qp->mtu)
+			pkt_num = (data_len + qp->mtu - 1) / qp->mtu;
+		if (!pkt_num)
+			pkt_num = 1;
+		sq->psn = (sq->psn + pkt_num) & BTH_PSN_MASK;
+		break;
+	}
+	case BNXT_QPLIB_SWQE_TYPE_LOCAL_INV:
+	{
+		struct sq_localinvalidate_hdr *sqe = base_hdr;
+
+		sqe->wqe_type = wqe->type;
+		sqe->flags = wqe->flags;
+		sqe->inv_l_key = cpu_to_le32(wqe->local_inv.inv_l_key);
+
+		dev_dbg(&sq_hwq->pdev->dev,
+			"QPLIB: FP: LOCAL INV WQE:\n"
+			"\twqe_type = 0x%x\n"
+			"\tflags = 0x%x\n"
+			"\tinv_l_key = 0x%x\n",
+			sqe->wqe_type, sqe->flags, sqe->inv_l_key);
+		break;
+	}
+	case BNXT_QPLIB_SWQE_TYPE_FAST_REG_MR:
+	{
+		struct sq_fr_pmr_hdr *sqe = base_hdr;
+		struct sq_fr_pmr_ext_hdr *ext_sqe = ext_hdr;
+
+		sqe->wqe_type = wqe->type;
+		sqe->flags = wqe->flags;
+		sqe->access_cntl = wqe->frmr.access_cntl |
+				   SQ_FR_PMR_ACCESS_CNTL_LOCAL_WRITE;
+		sqe->zero_based_page_size_log =
+			(wqe->frmr.pg_sz_log & SQ_FR_PMR_PAGE_SIZE_LOG_MASK) <<
+			SQ_FR_PMR_PAGE_SIZE_LOG_SFT |
+			(wqe->frmr.zero_based == true ? SQ_FR_PMR_ZERO_BASED : 0);
+		sqe->l_key = cpu_to_le32(wqe->frmr.l_key);
+		/* TODO: OFED only provides length of MR up to 32-bits for FRMR */
+		temp32 = cpu_to_le32(wqe->frmr.length);
+		memcpy(sqe->length, &temp32, sizeof(wqe->frmr.length));
+		sqe->numlevels_pbl_page_size_log =
+			((wqe->frmr.pbl_pg_sz_log <<
+					SQ_FR_PMR_PBL_PAGE_SIZE_LOG_SFT) &
+					SQ_FR_PMR_PBL_PAGE_SIZE_LOG_MASK) |
+			((wqe->frmr.levels << SQ_FR_PMR_NUMLEVELS_SFT) &
+					SQ_FR_PMR_NUMLEVELS_MASK);
+		if (!wqe->frmr.levels && !wqe->frmr.pbl_ptr) {
+			ext_sqe->pblptr = cpu_to_le64(wqe->frmr.page_list[0]);
+		} else {
+			for (i = 0; i < wqe->frmr.page_list_len; i++)
+				wqe->frmr.pbl_ptr[i] = cpu_to_le64(
+						wqe->frmr.page_list[i] |
+						PTU_PTE_VALID);
+			ext_sqe->pblptr = cpu_to_le64(wqe->frmr.pbl_dma_ptr);
+		}
+		ext_sqe->va = cpu_to_le64(wqe->frmr.va);
+		dev_dbg(&sq_hwq->pdev->dev,
+			"QPLIB: FP: FRMR WQE:\n"
+			"\twqe_type = 0x%x\n"
+			"\tflags = 0x%x\n"
+			"\taccess_cntl = 0x%x\n"
+			"\tzero_based_page_size_log = 0x%x\n"
+			"\tl_key = 0x%x\n"
+			"\tlength = 0x%x\n"
+			"\tnumlevels_pbl_page_size_log = 0x%x\n"
+			"\tpblptr = 0x%llx\n"
+			"\tva = 0x%llx\n",
+			sqe->wqe_type, sqe->flags, sqe->access_cntl,
+			sqe->zero_based_page_size_log, sqe->l_key,
+			*(u32 *)sqe->length, sqe->numlevels_pbl_page_size_log,
+			ext_sqe->pblptr, ext_sqe->va);
+		break;
+	}
+	case BNXT_QPLIB_SWQE_TYPE_BIND_MW:
+	{
+		struct sq_bind_hdr *sqe = base_hdr;
+		struct sq_bind_ext_hdr *ext_sqe = ext_hdr;
+
+		sqe->wqe_type = wqe->type;
+		sqe->flags = wqe->flags;
+		sqe->access_cntl = wqe->bind.access_cntl;
+		sqe->mw_type_zero_based = wqe->bind.mw_type |
+			(wqe->bind.zero_based == true ? SQ_BIND_ZERO_BASED : 0);
+		sqe->parent_l_key = cpu_to_le32(wqe->bind.parent_l_key);
+		sqe->l_key = cpu_to_le32(wqe->bind.r_key);
+		ext_sqe->va = cpu_to_le64(wqe->bind.va);
+		ext_sqe->length_lo = cpu_to_le32(wqe->bind.length);
+		dev_dbg(&sq_hwq->pdev->dev,
+			"QPLIB: FP: BIND WQE:\n"
+			"\twqe_type = 0x%x\n"
+			"\tflags = 0x%x\n"
+			"\taccess_cntl = 0x%x\n"
+			"\tmw_type_zero_based = 0x%x\n"
+			"\tparent_l_key = 0x%x\n"
+			"\tl_key = 0x%x\n"
+			"\tva = 0x%llx\n"
+			"\tlength = 0x%x\n",
+			sqe->wqe_type, sqe->flags, sqe->access_cntl,
+			sqe->mw_type_zero_based, sqe->parent_l_key,
+			sqe->l_key, sqe->va, ext_sqe->length_lo);
+		break;
+	}
+	default:
+		/* Bad wqe, return error */
+		rc = -EINVAL;
+		goto done;
+	}
+	swq->next_psn = sq->psn & BTH_PSN_MASK;
+	bnxt_qplib_fill_psn_search(qp, wqe, swq);
+
+queue_err:
+	bnxt_qplib_swq_mod_start(sq, wqe_idx);
+	bnxt_qplib_hwq_incr_prod(&sq->dbinfo, sq_hwq, swq->slots);
+	qp->wqe_cnt++;
+done:
+	if (sch_handler) {
+		nq_work = kzalloc(sizeof(*nq_work), GFP_ATOMIC);
+		if (nq_work) {
+			nq_work->cq = qp->scq;
+			nq_work->nq = qp->scq->nq;
+			INIT_WORK(&nq_work->work, bnxt_qpn_cqn_sched_task);
+			queue_work(qp->scq->nq->cqn_wq, &nq_work->work);
+		} else {
+			dev_err(&sq->hwq.pdev->dev,
+				"QPLIB: FP: Failed to allocate SQ nq_work!\n");
+			rc = -ENOMEM;
+		}
+	}
+	return rc;
+}
+
+void bnxt_qplib_post_recv_db(struct bnxt_qplib_qp *qp)
+{
+	struct bnxt_qplib_q *rq = &qp->rq;
+
+	bnxt_qplib_ring_prod_db(&rq->dbinfo, DBC_DBC_TYPE_RQ);
+}
+
+void bnxt_re_handle_cqn(struct bnxt_qplib_cq *cq)
+{
+	struct bnxt_qplib_nq *nq;
+
+	if (!(cq && cq->nq))
+		return;
+
+	nq = cq->nq;
+	spin_lock_bh(&cq->compl_lock);
+	if (nq->cqn_handler) {
+		dev_dbg(&nq->res->pdev->dev,
+			"%s:Trigger cq  = %p event nq = %p\n",
+			__func__, cq, nq);
+		nq->cqn_handler(nq, cq);
+	}
+	spin_unlock_bh(&cq->compl_lock);
+}
+
+int bnxt_qplib_post_recv(struct bnxt_qplib_qp *qp,
+			 struct bnxt_qplib_swqe *wqe)
+{
+	struct bnxt_qplib_nq_work *nq_work = NULL;
+	struct bnxt_qplib_q *rq = &qp->rq;
+	struct bnxt_qplib_hwq *rq_hwq;
+	struct bnxt_qplib_swq *swq;
+	bool sch_handler = false;
+	struct rq_wqe_hdr *base_hdr;
+	struct rq_ext_hdr *ext_hdr;
+	struct sq_sge *dsge;
+	u8 wqe_slots;
+	u32 wqe_idx;
+	u32 sw_prod;
+	int rc = 0;
+
+	rq_hwq = &rq->hwq;
+	if (qp->state == CMDQ_MODIFY_QP_NEW_STATE_RESET) {
+		dev_err(&rq_hwq->pdev->dev,
+			"QPLIB: FP: QP (0x%x) is in the 0x%x state\n",
+			qp->id, qp->state);
+		rc = -EINVAL;
+		goto done;
+	}
+
+	wqe_slots = _calculate_wqe_byte(qp, wqe, NULL);
+	if (bnxt_qplib_queue_full(rq_hwq, rq->dbinfo.max_slot)) {
+		dev_err(&rq_hwq->pdev->dev,
+			"QPLIB: FP: QP (0x%x) RQ is full!\n", qp->id);
+		rc = -EINVAL;
+		goto done;
+	}
+
+	swq = bnxt_qplib_get_swqe(rq, &wqe_idx);
+	swq->wr_id = wqe->wr_id;
+	swq->slots = rq->dbinfo.max_slot;
+	dev_dbg(&rq_hwq->pdev->dev,
+		"QPLIB: FP: post RQ wr_id[%d] = 0x%llx\n",
+		wqe_idx, swq->wr_id);
+	if (qp->cur_qp_state == CMDQ_MODIFY_QP_NEW_STATE_ERR) {
+		sch_handler = true;
+		dev_dbg(&rq_hwq->pdev->dev, "%s Error QP. Sched a flushed cmpl\n",
+			__func__);
+		goto queue_err;
+	}
+
+	sw_prod = rq_hwq->prod;
+	base_hdr = bnxt_qplib_get_qe(rq_hwq, sw_prod, NULL);
+	sw_prod++;
+	ext_hdr = bnxt_qplib_get_qe(rq_hwq, (sw_prod % rq_hwq->depth), NULL);
+	sw_prod++;
+	memset(base_hdr, 0, sizeof(struct sq_sge));
+	memset(ext_hdr, 0, sizeof(struct sq_sge));
+
+	if (!wqe->num_sge) {
+		dsge = bnxt_qplib_get_qe(rq_hwq, (sw_prod % rq_hwq->depth), NULL);
+		dsge->size = 0;
+		wqe_slots++;
+	} else {
+		bnxt_qplib_put_sges(rq_hwq, wqe->sg_list, wqe->num_sge, &sw_prod);
+	}
+	base_hdr->wqe_type = wqe->type;
+	base_hdr->flags = wqe->flags;
+	base_hdr->wqe_size = wqe_slots;
+	base_hdr->wr_id |= cpu_to_le32(wqe_idx);
+queue_err:
+	bnxt_qplib_swq_mod_start(rq, wqe_idx);
+	bnxt_qplib_hwq_incr_prod(&rq->dbinfo, &rq->hwq, swq->slots);
+done:
+	if (sch_handler) {
+		nq_work = kzalloc(sizeof(*nq_work), GFP_ATOMIC);
+		if (nq_work) {
+			nq_work->cq = qp->rcq;
+			nq_work->nq = qp->rcq->nq;
+			INIT_WORK(&nq_work->work, bnxt_qpn_cqn_sched_task);
+			queue_work(qp->rcq->nq->cqn_wq, &nq_work->work);
+		} else {
+			dev_err(&rq->hwq.pdev->dev,
+				"QPLIB: FP: Failed to allocate RQ nq_work!\n");
+			rc = -ENOMEM;
+		}
+	}
+	return rc;
+}
+
+/* CQ */
+int bnxt_qplib_create_cq(struct bnxt_qplib_res *res, struct bnxt_qplib_cq *cq)
+{
+	struct bnxt_qplib_hwq_attr hwq_attr = {};
+	struct bnxt_qplib_rcfw *rcfw = res->rcfw;
+	struct creq_create_cq_resp resp = {};
+	struct bnxt_qplib_cmdqmsg msg = {};
+	struct cmdq_create_cq req = {};
+	struct bnxt_qplib_reftbl *tbl;
+	unsigned long flag;
+	u32 pg_sz_lvl = 0;
+	int rc;
+
+	hwq_attr.res = res;
+	hwq_attr.depth = cq->max_wqe;
+	hwq_attr.stride = sizeof(struct cq_base);
+	hwq_attr.type = HWQ_TYPE_QUEUE;
+	hwq_attr.sginfo = &cq->sginfo;
+	rc = bnxt_qplib_alloc_init_hwq(&cq->hwq, &hwq_attr);
+	if (rc)
+		goto exit;
+
+	bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_CREATE_CQ,
+				 sizeof(req));
+
+	if (!cq->dpi) {
+		dev_err(&rcfw->pdev->dev,
+			"QPLIB: FP: CREATE_CQ failed due to NULL DPI\n");
+		return -EINVAL;
+	}
+	req.dpi = cpu_to_le32(cq->dpi->dpi);
+	req.cq_handle = cpu_to_le64(cq->cq_handle);
+
+	req.cq_size = cpu_to_le32(cq->max_wqe);
+	req.pbl = cpu_to_le64(_get_base_addr(&cq->hwq));
+	pg_sz_lvl = _get_base_pg_size(&cq->hwq) << CMDQ_CREATE_CQ_PG_SIZE_SFT;
+	pg_sz_lvl |= ((cq->hwq.level & CMDQ_CREATE_CQ_LVL_MASK) <<
+		       CMDQ_CREATE_CQ_LVL_SFT);
+	req.pg_size_lvl = cpu_to_le32(pg_sz_lvl);
+
+	req.cq_fco_cnq_id = cpu_to_le32(
+			(cq->cnq_hw_ring_id & CMDQ_CREATE_CQ_CNQ_ID_MASK) <<
+			 CMDQ_CREATE_CQ_CNQ_ID_SFT);
+	bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL, sizeof(req),
+				sizeof(resp), 0);
+	rc = bnxt_qplib_rcfw_send_message(rcfw, &msg);
+	if (rc)
+		goto fail;
+	cq->id = le32_to_cpu(resp.xid);
+	cq->period = BNXT_QPLIB_QUEUE_START_PERIOD;
+	init_waitqueue_head(&cq->waitq);
+	INIT_LIST_HEAD(&cq->sqf_head);
+	INIT_LIST_HEAD(&cq->rqf_head);
+	spin_lock_init(&cq->flush_lock);
+	spin_lock_init(&cq->compl_lock);
+
+	/* init dbinfo */
+	cq->cctx = res->cctx;
+	cq->dbinfo.hwq = &cq->hwq;
+	cq->dbinfo.xid = cq->id;
+	cq->dbinfo.db = cq->dpi->dbr;
+	cq->dbinfo.priv_db = res->dpi_tbl.priv_db;
+	cq->dbinfo.flags = 0;
+	cq->dbinfo.toggle = 0;
+	cq->dbinfo.res = res;
+	cq->dbinfo.seed = cq->id;
+	spin_lock_init(&cq->dbinfo.lock);
+	cq->dbinfo.shadow_key = BNXT_QPLIB_DBR_KEY_INVALID;
+	cq->dbinfo.shadow_key_arm_ena = BNXT_QPLIB_DBR_KEY_INVALID;
+
+	tbl = &res->reftbl.cqref;
+	spin_lock_irqsave(&tbl->lock, flag);
+	tbl->rec[GET_TBL_INDEX(cq->id, tbl)].xid = cq->id;
+	tbl->rec[GET_TBL_INDEX(cq->id, tbl)].handle = cq;
+	spin_unlock_irqrestore(&tbl->lock, flag);
+
+	bnxt_qplib_armen_db(&cq->dbinfo, DBC_DBC_TYPE_CQ_ARMENA);
+	return 0;
+
+fail:
+	bnxt_qplib_free_hwq(res, &cq->hwq);
+exit:
+	return rc;
+}
+
+int bnxt_qplib_modify_cq(struct bnxt_qplib_res *res, struct bnxt_qplib_cq *cq)
+{
+	/* TODO: Modify CQ threshold are passed to the HW via DBR */
+	return 0;
+}
+
+void bnxt_qplib_resize_cq_complete(struct bnxt_qplib_res *res,
+				   struct bnxt_qplib_cq *cq)
+{
+	bnxt_qplib_free_hwq(res, &cq->hwq);
+	memcpy(&cq->hwq, &cq->resize_hwq, sizeof(cq->hwq));
+	/* Reset only the cons bit in the flags */
+	cq->dbinfo.flags &= ~(1UL << BNXT_QPLIB_FLAG_EPOCH_CONS_SHIFT);
+
+	/* Tell HW to switch over to the new CQ */
+	if (!cq->resize_hwq.is_user)
+		bnxt_qplib_cq_coffack_db(&cq->dbinfo);
+}
+
+int bnxt_qplib_resize_cq(struct bnxt_qplib_res *res, struct bnxt_qplib_cq *cq,
+			 int new_cqes)
+{
+	struct bnxt_qplib_hwq_attr hwq_attr = {};
+	struct bnxt_qplib_rcfw *rcfw = res->rcfw;
+	struct creq_resize_cq_resp resp = {};
+	struct bnxt_qplib_cmdqmsg msg = {};
+	struct cmdq_resize_cq req = {};
+	u32 pgsz = 0, lvl = 0, nsz = 0;
+	struct bnxt_qplib_pbl *pbl;
+	u16 count = -1;
+	int rc;
+
+	bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_RESIZE_CQ,
+				 sizeof(req));
+
+	hwq_attr.sginfo = &cq->sginfo;
+	hwq_attr.res = res;
+	hwq_attr.depth = new_cqes;
+	hwq_attr.stride = sizeof(struct cq_base);
+	hwq_attr.type = HWQ_TYPE_QUEUE;
+	rc = bnxt_qplib_alloc_init_hwq(&cq->resize_hwq, &hwq_attr);
+	if (rc)
+		return rc;
+
+	dev_dbg(&rcfw->pdev->dev, "QPLIB: FP: %s: pbl_lvl: %d\n", __func__,
+		cq->resize_hwq.level);
+	req.cq_cid = cpu_to_le32(cq->id);
+	pbl = &cq->resize_hwq.pbl[PBL_LVL_0];
+	pgsz = ((pbl->pg_size == ROCE_PG_SIZE_4K ? CMDQ_RESIZE_CQ_PG_SIZE_PG_4K :
+		pbl->pg_size == ROCE_PG_SIZE_8K ? CMDQ_RESIZE_CQ_PG_SIZE_PG_8K :
+		pbl->pg_size == ROCE_PG_SIZE_64K ? CMDQ_RESIZE_CQ_PG_SIZE_PG_64K :
+		pbl->pg_size == ROCE_PG_SIZE_2M ? CMDQ_RESIZE_CQ_PG_SIZE_PG_2M :
+		pbl->pg_size == ROCE_PG_SIZE_8M ? CMDQ_RESIZE_CQ_PG_SIZE_PG_8M :
+		pbl->pg_size == ROCE_PG_SIZE_1G ? CMDQ_RESIZE_CQ_PG_SIZE_PG_1G :
+		CMDQ_RESIZE_CQ_PG_SIZE_PG_4K) & CMDQ_RESIZE_CQ_PG_SIZE_MASK);
+	lvl = (cq->resize_hwq.level << CMDQ_RESIZE_CQ_LVL_SFT) &
+				       CMDQ_RESIZE_CQ_LVL_MASK;
+	nsz = (new_cqes << CMDQ_RESIZE_CQ_NEW_CQ_SIZE_SFT) &
+	       CMDQ_RESIZE_CQ_NEW_CQ_SIZE_MASK;
+	req.new_cq_size_pg_size_lvl = cpu_to_le32(nsz|pgsz|lvl);
+	req.new_pbl = cpu_to_le64(pbl->pg_map_arr[0]);
+
+	if (!cq->resize_hwq.is_user)
+		set_bit(CQ_FLAGS_RESIZE_IN_PROG, &cq->flags);
+
+	bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL, sizeof(req),
+				sizeof(resp), 0);
+	rc = bnxt_qplib_rcfw_send_message(rcfw, &msg);
+	if (rc)
+		goto fail;
+
+	if (!cq->resize_hwq.is_user) {
+wait:
+		/* Wait here for the HW to switch the CQ over */
+		if (wait_event_interruptible_timeout(cq->waitq,
+		    !test_bit(CQ_FLAGS_RESIZE_IN_PROG, &cq->flags),
+		    msecs_to_jiffies(CQ_RESIZE_WAIT_TIME_MS)) ==
+		    -ERESTARTSYS && count--)
+			goto wait;
+
+		if (test_bit(CQ_FLAGS_RESIZE_IN_PROG, &cq->flags)) {
+			dev_err(&rcfw->pdev->dev,
+				"QPLIB: FP: RESIZE_CQ timed out\n");
+			rc = -ETIMEDOUT;
+			goto fail;
+		}
+
+		bnxt_qplib_resize_cq_complete(res, cq);
+	}
+
+	return 0;
+fail:
+	if (!cq->resize_hwq.is_user) {
+		bnxt_qplib_free_hwq(res, &cq->resize_hwq);
+		clear_bit(CQ_FLAGS_RESIZE_IN_PROG, &cq->flags);
+	}
+	return rc;
+}
+
+void bnxt_qplib_free_cq(struct bnxt_qplib_res *res, struct bnxt_qplib_cq *cq)
+{
+	bnxt_qplib_free_hwq(res, &cq->hwq);
+}
+
+static void bnxt_qplib_sync_cq(struct bnxt_qplib_cq *cq)
+{
+	struct bnxt_qplib_nq *nq = cq->nq;
+	/* Flush any pending work and synchronize irq */
+	flush_workqueue(cq->nq->cqn_wq);
+	mutex_lock(&nq->lock);
+	if (nq->requested)
+		synchronize_irq(nq->msix_vec);
+	mutex_unlock(&nq->lock);
+}
+
+int bnxt_qplib_destroy_cq(struct bnxt_qplib_res *res, struct bnxt_qplib_cq *cq)
+{
+	struct bnxt_qplib_rcfw *rcfw = res->rcfw;
+	struct creq_destroy_cq_resp resp = {};
+	struct bnxt_qplib_cmdqmsg msg = {};
+	struct cmdq_destroy_cq req = {};
+	struct bnxt_qplib_reftbl *tbl;
+	u16 total_cnq_events;
+	unsigned long flag;
+	int rc;
+
+	tbl = &res->reftbl.cqref;
+	spin_lock_irqsave(&tbl->lock, flag);
+	tbl->rec[GET_TBL_INDEX(cq->id, tbl)].handle = NULL;
+	tbl->rec[GET_TBL_INDEX(cq->id, tbl)].xid = 0;
+	spin_unlock_irqrestore(&tbl->lock, flag);
+
+	bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_DESTROY_CQ,
+				 sizeof(req));
+
+	req.cq_cid = cpu_to_le32(cq->id);
+	bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL, sizeof(req),
+				sizeof(resp), 0);
+	rc = bnxt_qplib_rcfw_send_message(rcfw, &msg);
+	if (rc)
+		return rc;
+
+	total_cnq_events = le16_to_cpu(resp.total_cnq_events);
+	if (total_cnq_events >= 0)
+		dev_dbg(&rcfw->pdev->dev,
+			"%s: cq_id = 0x%x cq = 0x%p resp.total_cnq_events = 0x%x\n",
+			__func__, cq->id, cq, total_cnq_events);
+	__wait_for_all_nqes(cq, total_cnq_events);
+	bnxt_qplib_sync_cq(cq);
+	bnxt_qplib_free_hwq(res, &cq->hwq);
+	return 0;
+}
+
+static int __flush_sq(struct bnxt_qplib_q *sq, struct bnxt_qplib_qp *qp,
+		      struct bnxt_qplib_cqe **pcqe, int *budget)
+{
+	struct bnxt_qplib_cqe *cqe;
+	u32 start, last;
+	int rc = 0;
+
+	/* Now complete all outstanding SQEs with FLUSHED_ERR */
+	start = sq->swq_start;
+	cqe = *pcqe;
+	while (*budget) {
+		last = sq->swq_last;
+		if (start == last) {
+			break;
+		}
+		/* Skip the FENCE WQE completions */
+		if (sq->swq[last].wr_id == BNXT_QPLIB_FENCE_WRID) {
+			bnxt_re_legacy_cancel_phantom_processing(qp);
+			goto skip_compl;
+		}
+
+		memset(cqe, 0, sizeof(*cqe));
+		cqe->status = CQ_REQ_STATUS_WORK_REQUEST_FLUSHED_ERR;
+		cqe->opcode = CQ_BASE_CQE_TYPE_REQ;
+		cqe->qp_handle = (u64)qp;
+		cqe->wr_id = sq->swq[last].wr_id;
+		cqe->src_qp = qp->id;
+		cqe->type = sq->swq[last].type;
+		dev_dbg(&sq->hwq.pdev->dev,
+			"QPLIB: FP: CQ Processed terminal Req \n");
+		dev_dbg(&sq->hwq.pdev->dev,
+			"QPLIB: wr_id[%d] = 0x%llx with status 0x%x\n",
+			last, cqe->wr_id, cqe->status);
+		cqe++;
+		(*budget)--;
+skip_compl:
+		bnxt_qplib_hwq_incr_cons(sq->hwq.depth,
+					 &sq->hwq.cons,
+					 sq->swq[last].slots,
+					 &sq->dbinfo.flags);
+		sq->swq_last = sq->swq[last].next_idx;
+	}
+	*pcqe = cqe;
+	if (!*budget && sq->swq_last != start)
+		/* Out of budget */
+		rc = -EAGAIN;
+	dev_dbg(&sq->hwq.pdev->dev, "QPLIB: FP: Flush SQ rc = 0x%x\n", rc);
+
+	return rc;
+}
+
+static int __flush_rq(struct bnxt_qplib_q *rq, struct bnxt_qplib_qp *qp,
+		      struct bnxt_qplib_cqe **pcqe, int *budget)
+{
+	struct bnxt_qplib_cqe *cqe;
+	u32 start, last;
+	int opcode = 0;
+	int rc = 0;
+
+	switch (qp->type) {
+	case CMDQ_CREATE_QP1_TYPE_GSI:
+		opcode = CQ_BASE_CQE_TYPE_RES_RAWETH_QP1;
+		break;
+	case CMDQ_CREATE_QP_TYPE_RC:
+		opcode = CQ_BASE_CQE_TYPE_RES_RC;
+		break;
+	case CMDQ_CREATE_QP_TYPE_UD:
+		opcode = CQ_BASE_CQE_TYPE_RES_UD;
+		break;
+	}
+
+	/* Flush the rest of the RQ */
+	start = rq->swq_start;
+	cqe = *pcqe;
+	while (*budget) {
+		last = rq->swq_last;
+		if (last == start)
+			break;
+		memset(cqe, 0, sizeof(*cqe));
+		cqe->status =
+		    CQ_RES_RC_STATUS_WORK_REQUEST_FLUSHED_ERR;
+		cqe->opcode = opcode;
+		cqe->qp_handle = (u64)qp;
+		cqe->wr_id = rq->swq[last].wr_id;
+		dev_dbg(&rq->hwq.pdev->dev, "QPLIB: FP: CQ Processed Res RC \n");
+		dev_dbg(&rq->hwq.pdev->dev,
+			"QPLIB: rq[%d] = 0x%llx with status 0x%x\n",
+			last, cqe->wr_id, cqe->status);
+		cqe++;
+		(*budget)--;
+		bnxt_qplib_hwq_incr_cons(rq->hwq.depth,
+					 &rq->hwq.cons,
+					 rq->swq[last].slots,
+					 &rq->dbinfo.flags);
+		rq->swq_last = rq->swq[last].next_idx;
+	}
+	*pcqe = cqe;
+	if (!*budget && rq->swq_last != start)
+		/* Out of budget */
+		rc = -EAGAIN;
+
+	dev_dbg(&rq->hwq.pdev->dev, "QPLIB: FP: Flush RQ rc = 0x%x\n", rc);
+	return rc;
+}
+
+void bnxt_qplib_mark_qp_error(void *qp_handle)
+{
+	struct bnxt_qplib_qp *qp = qp_handle;
+
+	if (!qp)
+		return;
+
+	/* Must block new posting of SQ and RQ */
+	qp->cur_qp_state = CMDQ_MODIFY_QP_NEW_STATE_ERR;
+	qp->state = qp->cur_qp_state;
+
+	/* Add qp to flush list of the CQ */
+	if (!qp->is_user)
+		bnxt_qplib_add_flush_qp(qp);
+}
+
+/* Note: SQE is valid from sw_sq_cons up to cqe_sq_cons (exclusive)
+ *       CQE is track from sw_cq_cons to max_element but valid only if VALID=1
+ */
+static int bnxt_re_legacy_do_wa9060(struct bnxt_qplib_qp *qp,
+				 struct bnxt_qplib_cq *cq,
+				 u32 cq_cons, u32 swq_last,
+				 u32 cqe_sq_cons)
+{
+	struct bnxt_qplib_q *sq = &qp->sq;
+	struct bnxt_qplib_swq *swq;
+	u32 peek_sw_cq_cons, peek_sq_cons_idx, peek_flags;
+	struct cq_terminal *peek_term_hwcqe;
+	struct cq_req *peek_req_hwcqe;
+	struct bnxt_qplib_qp *peek_qp;
+	struct bnxt_qplib_q *peek_sq;
+	struct cq_base *peek_hwcqe;
+	int i, rc = 0;
+
+	/* Check for the psn_search marking before completing */
+	swq = &sq->swq[swq_last];
+	if (swq->psn_search &&
+	    le32_to_cpu(swq->psn_search->flags_next_psn) & 0x80000000) {
+		/* Unmark */
+		swq->psn_search->flags_next_psn = cpu_to_le32
+				(le32_to_cpu(swq->psn_search->flags_next_psn)
+				 & ~0x80000000);
+		dev_dbg(&cq->hwq.pdev->dev,
+			"FP: Process Req cq_cons=0x%x qp=0x%x sq cons sw=0x%x cqe=0x%x marked!\n",
+			cq_cons, qp->id, swq_last, cqe_sq_cons);
+		sq->condition = true;
+		sq->legacy_send_phantom = true;
+
+		/* TODO: Only ARM if the previous SQE is ARMALL */
+		bnxt_qplib_ring_db(&cq->dbinfo, DBC_DBC_TYPE_CQ_ARMALL);
+
+		rc = -EAGAIN;
+		goto out;
+	}
+	if (sq->condition == true) {
+		/* Peek at the completions */
+		peek_flags = cq->dbinfo.flags;
+		peek_sw_cq_cons = cq_cons;
+		i = cq->hwq.depth;
+		while (i--) {
+			peek_hwcqe = bnxt_qplib_get_qe(&cq->hwq,
+						       peek_sw_cq_cons, NULL);
+			/* If the next hwcqe is VALID */
+			if (CQE_CMP_VALID(peek_hwcqe, peek_flags)) {
+				/* If the next hwcqe is a REQ */
+				dma_rmb();
+				switch (peek_hwcqe->cqe_type_toggle &
+					CQ_BASE_CQE_TYPE_MASK) {
+				case CQ_BASE_CQE_TYPE_REQ:
+					peek_req_hwcqe = (struct cq_req *)
+							 peek_hwcqe;
+					peek_qp = (struct bnxt_qplib_qp *)
+						le64_to_cpu(
+						peek_req_hwcqe->qp_handle);
+					peek_sq = &peek_qp->sq;
+					peek_sq_cons_idx =
+						((le16_to_cpu(
+						  peek_req_hwcqe->sq_cons_idx)
+						  - 1) % sq->max_wqe);
+					/* If the hwcqe's sq's wr_id matches */
+					if (peek_sq == sq &&
+					    sq->swq[peek_sq_cons_idx].wr_id ==
+					    BNXT_QPLIB_FENCE_WRID) {
+						/* Unbreak only if the phantom
+						   comes back */
+						dev_dbg(&cq->hwq.pdev->dev,
+							"FP: Process Req qp=0x%x current sq cons sw=0x%x cqe=0x%x\n",
+							qp->id, swq_last,
+							cqe_sq_cons);
+						sq->condition = false;
+						sq->single = true;
+						sq->phantom_cqe_cnt++;
+						dev_dbg(&cq->hwq.pdev->dev,
+							"qp %#x condition restored at peek cq_cons=%#x sq_cons_idx %#x, phantom_cqe_cnt: %d unmark\n",
+							peek_qp->id,
+							peek_sw_cq_cons,
+							peek_sq_cons_idx,
+							sq->phantom_cqe_cnt);
+						rc = 0;
+						goto out;
+					}
+					break;
+
+				case CQ_BASE_CQE_TYPE_TERMINAL:
+					/* In case the QP has gone into the
+					   error state */
+					peek_term_hwcqe = (struct cq_terminal *)
+							  peek_hwcqe;
+					peek_qp = (struct bnxt_qplib_qp *)
+						le64_to_cpu(
+						peek_term_hwcqe->qp_handle);
+					if (peek_qp == qp) {
+						sq->condition = false;
+						rc = 0;
+						goto out;
+					}
+					break;
+				default:
+					break;
+				}
+				/* Valid but not the phantom, so keep looping */
+			} else {
+				/* Not valid yet, just exit and wait */
+				rc = -EINVAL;
+				goto out;
+			}
+			bnxt_qplib_hwq_incr_cons(cq->hwq.depth,
+						 &peek_sw_cq_cons,
+						 1, &peek_flags);
+		}
+		dev_err(&cq->hwq.pdev->dev,
+			"Should not have come here! cq_cons=0x%x qp=0x%x sq cons sw=0x%x hw=0x%x\n",
+			cq_cons, qp->id, swq_last, cqe_sq_cons);
+		rc = -EINVAL;
+	}
+out:
+	return rc;
+}
+
+static int bnxt_qplib_cq_process_req(struct bnxt_qplib_cq *cq,
+				     struct cq_req *hwcqe,
+				     struct bnxt_qplib_cqe **pcqe, int *budget,
+				     u32 cq_cons, struct bnxt_qplib_qp **lib_qp)
+{
+	struct bnxt_qplib_qp *qp;
+	struct bnxt_qplib_q *sq;
+	struct bnxt_qplib_cqe *cqe;
+	u32 cqe_sq_cons;
+	struct bnxt_qplib_swq *swq;
+	int rc = 0;
+
+	qp = (struct bnxt_qplib_qp *)le64_to_cpu(hwcqe->qp_handle);
+	dev_dbg(&cq->hwq.pdev->dev, "FP: Process Req qp=0x%p\n", qp);
+	if (!qp) {
+		dev_err(&cq->hwq.pdev->dev,
+			"QPLIB: FP: Process Req qp is NULL\n");
+		return -EINVAL;
+	}
+	sq = &qp->sq;
+
+	cqe_sq_cons = le16_to_cpu(hwcqe->sq_cons_idx) % sq->max_wqe;
+	if (qp->sq.flushed) {
+		dev_dbg(&cq->hwq.pdev->dev,
+			"%s: QPLIB: QP in Flush QP = %p\n", __func__, qp);
+		goto done;
+	}
+
+	/* Require to walk the sq's swq to fabricate CQEs for all previously
+	 * signaled SWQEs due to CQE aggregation from the current sq cons
+	 * to the cqe_sq_cons
+	 */
+	cqe = *pcqe;
+	while (*budget) {
+		if (sq->swq_last == cqe_sq_cons)
+			/* Done */
+			break;
+
+		swq = &sq->swq[sq->swq_last];
+		memset(cqe, 0, sizeof(*cqe));
+		cqe->opcode = CQ_BASE_CQE_TYPE_REQ;
+		cqe->qp_handle = (u64)qp;
+		cqe->src_qp = qp->id;
+		cqe->wr_id = swq->wr_id;
+
+		if (cqe->wr_id == BNXT_QPLIB_FENCE_WRID)
+			goto skip;
+
+		cqe->type = swq->type;
+
+		/* For the last CQE, check for status.  For errors, regardless
+		 * of the request being signaled or not, it must complete with
+		 * the hwcqe error status
+		 */
+		if (swq->next_idx == cqe_sq_cons &&
+		    hwcqe->status != CQ_REQ_STATUS_OK) {
+			cqe->status = hwcqe->status;
+			dev_err(&cq->hwq.pdev->dev,
+				"QPLIB: FP: CQ Processed Req \n");
+			dev_err(&cq->hwq.pdev->dev,
+				"QPLIB: QP 0x%x wr_id[%d] = 0x%lx vendor type 0x%x with vendor status 0x%x\n",
+				cqe->src_qp, sq->swq_last, cqe->wr_id, cqe->type, cqe->status);
+			cqe++;
+			(*budget)--;
+			bnxt_qplib_mark_qp_error(qp);
+		} else {
+			/* Before we complete, do WA 9060 */
+			if (!_is_chip_gen_p5_p7(qp->cctx)) {
+				if (bnxt_re_legacy_do_wa9060(qp, cq, cq_cons,
+					      sq->swq_last,
+					      cqe_sq_cons)) {
+					*lib_qp = qp;
+					goto out;
+				}
+			}
+			if (swq->flags & SQ_SEND_FLAGS_SIGNAL_COMP) {
+
+				dev_dbg(&cq->hwq.pdev->dev,
+					"QPLIB: FP: CQ Processed Req \n");
+				dev_dbg(&cq->hwq.pdev->dev,
+					"QPLIB: wr_id[%d] = 0x%llx \n",
+					sq->swq_last, cqe->wr_id);
+				dev_dbg(&cq->hwq.pdev->dev,
+					"QPLIB: with status 0x%x\n", cqe->status);
+				cqe->status = CQ_REQ_STATUS_OK;
+				cqe++;
+				(*budget)--;
+			}
+		}
+skip:
+		bnxt_qplib_hwq_incr_cons(sq->hwq.depth, &sq->hwq.cons,
+					 swq->slots, &sq->dbinfo.flags);
+		sq->swq_last = swq->next_idx;
+		if (sq->single == true)
+			break;
+	}
+out:
+	*pcqe = cqe;
+	if (sq->swq_last != cqe_sq_cons) {
+		/* Out of budget */
+		rc = -EAGAIN;
+		goto done;
+	}
+	/* Back to normal completion mode only after it has completed all of
+	   the WC for this CQE */
+	sq->single = false;
+done:
+	return rc;
+}
+
+static void bnxt_qplib_release_srqe(struct bnxt_qplib_srq *srq, u32 tag)
+{
+	spin_lock(&srq->hwq.lock);
+	srq->swq[srq->last_idx].next_idx = (int)tag;
+	srq->last_idx = (int)tag;
+	srq->swq[srq->last_idx].next_idx = -1;
+	bnxt_qplib_hwq_incr_cons(srq->hwq.depth, &srq->hwq.cons,
+				 srq->dbinfo.max_slot, &srq->dbinfo.flags);
+	spin_unlock(&srq->hwq.lock);
+}
+
+static int bnxt_qplib_cq_process_res_rc(struct bnxt_qplib_cq *cq,
+					struct cq_res_rc *hwcqe,
+					struct bnxt_qplib_cqe **pcqe,
+					int *budget)
+{
+	struct bnxt_qplib_srq *srq;
+	struct bnxt_qplib_cqe *cqe;
+	struct bnxt_qplib_qp *qp;
+	struct bnxt_qplib_q *rq;
+	u32 wr_id_idx;
+	int rc = 0;
+
+	qp = (struct bnxt_qplib_qp *)le64_to_cpu(hwcqe->qp_handle);
+	if (!qp) {
+		dev_err(&cq->hwq.pdev->dev, "QPLIB: process_cq RC qp is NULL\n");
+		return -EINVAL;
+	}
+	if (qp->rq.flushed) {
+		dev_dbg(&cq->hwq.pdev->dev,
+			"%s: QPLIB: QP in Flush QP = %p\n", __func__, qp);
+		goto done;
+	}
+
+	cqe = *pcqe;
+	cqe->opcode = hwcqe->cqe_type_toggle & CQ_BASE_CQE_TYPE_MASK;
+	cqe->length = le32_to_cpu(hwcqe->length);
+	cqe->invrkey = le32_to_cpu(hwcqe->imm_data_or_inv_r_key);
+	cqe->mr_handle = le64_to_cpu(hwcqe->mr_handle);
+	cqe->flags = le16_to_cpu(hwcqe->flags);
+	cqe->status = hwcqe->status;
+	cqe->qp_handle = (u64)(unsigned long)qp;
+
+	wr_id_idx = le32_to_cpu(hwcqe->srq_or_rq_wr_id) &
+				CQ_RES_RC_SRQ_OR_RQ_WR_ID_MASK;
+	if (cqe->flags & CQ_RES_RC_FLAGS_SRQ_SRQ) {
+		srq = qp->srq;
+		if (!srq) {
+			dev_err(&cq->hwq.pdev->dev,
+				"QPLIB: FP: SRQ used but not defined??\n");
+			return -EINVAL;
+		}
+		if (wr_id_idx > srq->hwq.depth - 1) {
+			dev_err(&cq->hwq.pdev->dev,
+				"QPLIB: FP: CQ Process RC \n");
+			dev_err(&cq->hwq.pdev->dev,
+				"QPLIB: wr_id idx 0x%x exceeded SRQ max 0x%x\n",
+				wr_id_idx, srq->hwq.depth);
+			return -EINVAL;
+		}
+		cqe->wr_id = srq->swq[wr_id_idx].wr_id;
+		bnxt_qplib_release_srqe(srq, wr_id_idx);
+		dev_dbg(&srq->hwq.pdev->dev,
+			"QPLIB: FP: CQ Processed RC SRQ wr_id[%d] = 0x%llx\n",
+			wr_id_idx, cqe->wr_id);
+		cqe++;
+		(*budget)--;
+		*pcqe = cqe;
+	} else {
+		rq = &qp->rq;
+		if (wr_id_idx > (rq->max_wqe - 1)) {
+			dev_err(&cq->hwq.pdev->dev,
+				"QPLIB: FP: CQ Process RC \n");
+			dev_err(&cq->hwq.pdev->dev,
+				"QPLIB: wr_id idx 0x%x exceeded RQ max 0x%x\n",
+				wr_id_idx, rq->hwq.depth);
+			return -EINVAL;
+		}
+		if (wr_id_idx != rq->swq_last)
+			return -EINVAL;
+		cqe->wr_id = rq->swq[rq->swq_last].wr_id;
+		dev_dbg(&cq->hwq.pdev->dev,
+			"QPLIB: FP: CQ Processed RC RQ wr_id[%d] = 0x%llx\n",
+			rq->swq_last, cqe->wr_id);
+		cqe++;
+		(*budget)--;
+		bnxt_qplib_hwq_incr_cons(rq->hwq.depth, &rq->hwq.cons,
+					 rq->swq[rq->swq_last].slots,
+					 &rq->dbinfo.flags);
+		rq->swq_last = rq->swq[rq->swq_last].next_idx;
+		*pcqe = cqe;
+
+		if (hwcqe->status != CQ_RES_RC_STATUS_OK)
+			bnxt_qplib_mark_qp_error(qp);
+	}
+done:
+	return rc;
+}
+
+static int bnxt_qplib_cq_process_res_ud(struct bnxt_qplib_cq *cq,
+					struct cq_res_ud_v2 *hwcqe,
+					struct bnxt_qplib_cqe **pcqe,
+					int *budget)
+{
+	struct bnxt_qplib_srq *srq;
+	struct bnxt_qplib_cqe *cqe;
+	struct bnxt_qplib_qp *qp;
+	struct bnxt_qplib_q *rq;
+	u32 wr_id_idx;
+	int rc = 0;
+	u16 *smac;
+
+	qp = (struct bnxt_qplib_qp *)le64_to_cpu(hwcqe->qp_handle);
+	if (!qp) {
+		dev_err(&cq->hwq.pdev->dev, "QPLIB: process_cq UD qp is NULL\n");
+		return -EINVAL;
+	}
+	if (qp->rq.flushed) {
+		dev_dbg(&cq->hwq.pdev->dev,
+			"%s: QPLIB: QP in Flush QP = %p\n", __func__, qp);
+		goto done;
+	}
+	cqe = *pcqe;
+	cqe->opcode = hwcqe->cqe_type_toggle & CQ_RES_UD_V2_CQE_TYPE_MASK;
+	cqe->length = le32_to_cpu((hwcqe->length & CQ_RES_UD_V2_LENGTH_MASK));
+	cqe->cfa_meta = le16_to_cpu(hwcqe->cfa_metadata0);
+	/* V2 format has metadata1 */
+	cqe->cfa_meta |= (((le32_to_cpu(hwcqe->src_qp_high_srq_or_rq_wr_id) &
+			   CQ_RES_UD_V2_CFA_METADATA1_MASK) >>
+			  CQ_RES_UD_V2_CFA_METADATA1_SFT) <<
+			 BNXT_QPLIB_META1_SHIFT);
+	cqe->invrkey = le32_to_cpu(hwcqe->imm_data);
+	cqe->flags = le16_to_cpu(hwcqe->flags);
+	cqe->status = hwcqe->status;
+	cqe->qp_handle = (u64)(unsigned long)qp;
+	smac = (u16 *)cqe->smac;
+	smac[2] = ntohs(le16_to_cpu(hwcqe->src_mac[0]));
+	smac[1] = ntohs(le16_to_cpu(hwcqe->src_mac[1]));
+	smac[0] = ntohs(le16_to_cpu(hwcqe->src_mac[2]));
+	wr_id_idx = le32_to_cpu(hwcqe->src_qp_high_srq_or_rq_wr_id)
+				& CQ_RES_UD_V2_SRQ_OR_RQ_WR_ID_MASK;
+	cqe->src_qp = le16_to_cpu(hwcqe->src_qp_low) |
+				  ((le32_to_cpu(
+				    hwcqe->src_qp_high_srq_or_rq_wr_id) &
+				    CQ_RES_UD_V2_SRC_QP_HIGH_MASK) >> 8);
+
+	if (cqe->flags & CQ_RES_UD_V2_FLAGS_SRQ) {
+		srq = qp->srq;
+		if (!srq) {
+			dev_err(&cq->hwq.pdev->dev,
+				"QPLIB: FP: SRQ used but not defined??\n");
+			return -EINVAL;
+		}
+		if (wr_id_idx > srq->hwq.depth - 1) {
+			dev_err(&cq->hwq.pdev->dev,
+				"QPLIB: FP: CQ Process UD \n");
+			dev_err(&cq->hwq.pdev->dev,
+				"QPLIB: wr_id idx 0x%x exceeded SRQ max 0x%x\n",
+				wr_id_idx, srq->hwq.depth);
+			return -EINVAL;
+		}
+		cqe->wr_id = srq->swq[wr_id_idx].wr_id;
+		bnxt_qplib_release_srqe(srq, wr_id_idx);
+		dev_dbg(&cq->hwq.pdev->dev,
+			"QPLIB: FP: CQ Processed UD SRQ wr_id[%d] = 0x%llx\n",
+			wr_id_idx, cqe->wr_id);
+		cqe++;
+		(*budget)--;
+		*pcqe = cqe;
+	} else {
+		rq = &qp->rq;
+		if (wr_id_idx > (rq->max_wqe - 1)) {
+			dev_err(&cq->hwq.pdev->dev,
+				"QPLIB: FP: CQ Process UD \n");
+			dev_err(&cq->hwq.pdev->dev,
+				"QPLIB: wr_id idx 0x%x exceeded RQ max 0x%x\n",
+				wr_id_idx, rq->hwq.depth);
+			return -EINVAL;
+		}
+		if (rq->swq_last != wr_id_idx)
+			return -EINVAL;
+
+		cqe->wr_id = rq->swq[rq->swq_last].wr_id;
+		dev_dbg(&cq->hwq.pdev->dev,
+			"QPLIB: FP: CQ Processed UD RQ wr_id[%d] = 0x%llx\n",
+			 rq->swq_last, cqe->wr_id);
+		cqe++;
+		(*budget)--;
+		bnxt_qplib_hwq_incr_cons(rq->hwq.depth, &rq->hwq.cons,
+					 rq->swq[rq->swq_last].slots,
+					 &rq->dbinfo.flags);
+		rq->swq_last = rq->swq[rq->swq_last].next_idx;
+		*pcqe = cqe;
+
+		if (hwcqe->status != CQ_RES_UD_V2_STATUS_OK)
+			bnxt_qplib_mark_qp_error(qp);
+	}
+done:
+	return rc;
+}
+
+bool bnxt_qplib_is_cq_empty(struct bnxt_qplib_cq *cq)
+{
+
+	struct cq_base *hw_cqe;
+	unsigned long flags;
+	bool rc = true;
+
+	spin_lock_irqsave(&cq->hwq.lock, flags);
+	hw_cqe = bnxt_qplib_get_qe(&cq->hwq, cq->hwq.cons, NULL);
+
+	 /* Check for Valid bit. If the CQE is valid, return false */
+	rc = !CQE_CMP_VALID(hw_cqe, cq->dbinfo.flags);
+	spin_unlock_irqrestore(&cq->hwq.lock, flags);
+	return rc;
+}
+
+static int bnxt_qplib_cq_process_res_raweth_qp1(struct bnxt_qplib_cq *cq,
+						struct cq_res_raweth_qp1 *hwcqe,
+						struct bnxt_qplib_cqe **pcqe,
+						int *budget)
+{
+	struct bnxt_qplib_qp *qp;
+	struct bnxt_qplib_q *rq;
+	struct bnxt_qplib_srq *srq;
+	struct bnxt_qplib_cqe *cqe;
+	u32 wr_id_idx;
+	int rc = 0;
+
+	qp = (struct bnxt_qplib_qp *)le64_to_cpu(hwcqe->qp_handle);
+	if (!qp) {
+		dev_err(&cq->hwq.pdev->dev,
+			"QPLIB: process_cq Raw/QP1 qp is NULL\n");
+		return -EINVAL;
+	}
+	if (qp->rq.flushed) {
+		dev_dbg(&cq->hwq.pdev->dev,
+			"%s: QPLIB: QP in Flush QP = %p\n", __func__, qp);
+		goto done;
+	}
+	cqe = *pcqe;
+	cqe->opcode = hwcqe->cqe_type_toggle & CQ_BASE_CQE_TYPE_MASK;
+	cqe->flags = le16_to_cpu(hwcqe->flags);
+	cqe->qp_handle = (u64)(unsigned long)qp;
+
+	wr_id_idx = le32_to_cpu(hwcqe->raweth_qp1_payload_offset_srq_or_rq_wr_id)
+				& CQ_RES_RAWETH_QP1_SRQ_OR_RQ_WR_ID_MASK;
+	cqe->src_qp = qp->id;
+	if (qp->id == 1 && !cqe->length) {
+		/* Add workaround for the length misdetection */
+		cqe->length = 296;
+	} else {
+		cqe->length = le16_to_cpu(hwcqe->length);
+	}
+	cqe->pkey_index = qp->pkey_index;
+	memcpy(cqe->smac, qp->smac, 6);
+
+	cqe->raweth_qp1_flags = le16_to_cpu(hwcqe->raweth_qp1_flags);
+	cqe->raweth_qp1_flags2 = le32_to_cpu(hwcqe->raweth_qp1_flags2);
+	cqe->raweth_qp1_metadata = le32_to_cpu(hwcqe->raweth_qp1_metadata);
+
+	dev_dbg(&cq->hwq.pdev->dev,
+		 "QPLIB: raweth_qp1_flags = 0x%x raweth_qp1_flags2 = 0x%x\n",
+		 cqe->raweth_qp1_flags, cqe->raweth_qp1_flags2);
+
+	if (cqe->flags & CQ_RES_RAWETH_QP1_FLAGS_SRQ_SRQ) {
+		srq = qp->srq;
+		if (!srq) {
+			dev_err(&cq->hwq.pdev->dev,
+				"QPLIB: FP: SRQ used but not defined??\n");
+			return -EINVAL;
+		}
+		if (wr_id_idx > srq->hwq.depth - 1) {
+			dev_err(&cq->hwq.pdev->dev,
+				"QPLIB: FP: CQ Process Raw/QP1 \n");
+			dev_err(&cq->hwq.pdev->dev,
+				"QPLIB: wr_id idx 0x%x exceeded SRQ max 0x%x\n",
+				wr_id_idx, srq->hwq.depth);
+			return -EINVAL;
+		}
+		cqe->wr_id = srq->swq[wr_id_idx].wr_id;
+		dev_dbg(&cq->hwq.pdev->dev,
+			"QPLIB: FP: CQ Processed Raw/QP1 SRQ \n");
+		dev_dbg(&cq->hwq.pdev->dev,
+			"QPLIB: wr_id[%d] = 0x%llx with status = 0x%x\n",
+			wr_id_idx, cqe->wr_id, hwcqe->status);
+		cqe++;
+		(*budget)--;
+		srq->hwq.cons++;
+		*pcqe = cqe;
+	} else {
+		rq = &qp->rq;
+		if (wr_id_idx > (rq->max_wqe - 1)) {
+			dev_err(&cq->hwq.pdev->dev,
+				"QPLIB: FP: CQ Process Raw/QP1 RQ wr_id \n");
+			dev_err(&cq->hwq.pdev->dev,
+				"QPLIB: ix 0x%x exceeded RQ max 0x%x\n",
+				wr_id_idx, rq->max_wqe);
+			return -EINVAL;
+		}
+		if (wr_id_idx != rq->swq_last)
+			return -EINVAL;
+		cqe->wr_id = rq->swq[rq->swq_last].wr_id;
+		dev_dbg(&cq->hwq.pdev->dev,
+			"QPLIB: FP: CQ Processed Raw/QP1 RQ \n");
+		dev_dbg(&cq->hwq.pdev->dev,
+			"QPLIB: wr_id[%d] = 0x%llx with status = 0x%x\n",
+			wr_id_idx, cqe->wr_id, hwcqe->status);
+		cqe++;
+		(*budget)--;
+		bnxt_qplib_hwq_incr_cons(rq->hwq.depth, &rq->hwq.cons,
+					 rq->swq[wr_id_idx].slots,
+					 &rq->dbinfo.flags);
+		rq->swq_last = rq->swq[rq->swq_last].next_idx;
+		*pcqe = cqe;
+
+		if (hwcqe->status != CQ_RES_RC_STATUS_OK)
+			bnxt_qplib_mark_qp_error(qp);
+	}
+done:
+	return rc;
+}
+
+static int bnxt_qplib_cq_process_terminal(struct bnxt_qplib_cq *cq,
+					  struct cq_terminal *hwcqe,
+					  struct bnxt_qplib_cqe **pcqe,
+					  int *budget)
+{
+	struct bnxt_qplib_q *sq, *rq;
+	struct bnxt_qplib_cqe *cqe;
+	struct bnxt_qplib_qp *qp;
+	u32 swq_last;
+	u32 cqe_cons;
+	int rc = 0;
+
+	/* Check the Status */
+	if (hwcqe->status != CQ_TERMINAL_STATUS_OK)
+		dev_warn(&cq->hwq.pdev->dev,
+			 "QPLIB: FP: CQ Process Terminal Error status = 0x%x\n",
+			 hwcqe->status);
+
+	qp = (struct bnxt_qplib_qp *)le64_to_cpu(hwcqe->qp_handle);
+	if (!qp)
+		return -EINVAL;
+	dev_dbg(&cq->hwq.pdev->dev,
+		"QPLIB: FP: CQ Process terminal for qp (0x%x)\n", qp->id);
+
+	/* Terminal CQE requires all posted RQEs to complete with FLUSHED_ERR
+	 * from the current rq->cons to the rq->prod regardless what the
+	 * rq->cons the terminal CQE indicates.
+	 */
+	bnxt_qplib_mark_qp_error(qp);
+
+	sq = &qp->sq;
+	rq = &qp->rq;
+
+	cqe_cons = le16_to_cpu(hwcqe->sq_cons_idx);
+	if (cqe_cons == 0xFFFF)
+		goto do_rq;
+
+	cqe_cons %= sq->max_wqe;
+	if (qp->sq.flushed) {
+		dev_dbg(&cq->hwq.pdev->dev,
+			"%s: QPLIB: QP in Flush QP = %p\n", __func__, qp);
+		goto sq_done;
+	}
+
+	/* Terminal CQE can also include aggregated successful CQEs prior.
+	   So we must complete all CQEs from the current sq's cons to the
+	   cq_cons with status OK */
+	cqe = *pcqe;
+	while (*budget) {
+		/*sw_cons = HWQ_CMP(sq->hwq.cons, &sq->hwq);*/
+		swq_last = sq->swq_last;
+		if (swq_last == cqe_cons)
+			break;
+		if (sq->swq[swq_last].flags & SQ_SEND_FLAGS_SIGNAL_COMP) {
+			memset(cqe, 0, sizeof(*cqe));
+			cqe->status = CQ_REQ_STATUS_OK;
+			cqe->opcode = CQ_BASE_CQE_TYPE_REQ;
+			cqe->qp_handle = (u64)qp;
+			cqe->src_qp = qp->id;
+			cqe->wr_id = sq->swq[swq_last].wr_id;
+			cqe->type = sq->swq[swq_last].type;
+			dev_dbg(&cq->hwq.pdev->dev,
+				"QPLIB: FP: CQ Processed terminal Req \n");
+			dev_dbg(&cq->hwq.pdev->dev,
+				"QPLIB: wr_id[%d] = 0x%llx with status 0x%x\n",
+				swq_last, cqe->wr_id, cqe->status);
+			cqe++;
+			(*budget)--;
+		}
+		bnxt_qplib_hwq_incr_cons(sq->hwq.depth, &sq->hwq.cons,
+					 sq->swq[swq_last].slots,
+					 &sq->dbinfo.flags);
+		sq->swq_last = sq->swq[swq_last].next_idx;
+	}
+	*pcqe = cqe;
+	if (!*budget && swq_last != cqe_cons) {
+		/* Out of budget */
+		rc = -EAGAIN;
+		goto sq_done;
+	}
+sq_done:
+	if (rc)
+		return rc;
+do_rq:
+	cqe_cons = le16_to_cpu(hwcqe->rq_cons_idx);
+	if (cqe_cons == 0xFFFF) {
+		goto done;
+	} else if (cqe_cons > (rq->max_wqe - 1)) {
+		dev_err(&cq->hwq.pdev->dev,
+			"QPLIB: FP: CQ Processed terminal \n");
+		dev_err(&cq->hwq.pdev->dev,
+			"QPLIB: reported rq_cons_idx 0x%x exceeds max 0x%x\n",
+			cqe_cons, rq->hwq.depth);
+		goto done;
+	}
+	if (qp->rq.flushed) {
+		dev_dbg(&cq->hwq.pdev->dev,
+			"%s: QPLIB: QP in Flush QP = %p\n", __func__, qp);
+		rc = 0;
+		goto rq_done;
+	}
+
+rq_done:
+done:
+	return rc;
+}
+
+static int bnxt_qplib_cq_process_cutoff(struct bnxt_qplib_cq *cq,
+					struct cq_cutoff *hwcqe)
+{
+	/* Check the Status */
+	if (hwcqe->status != CQ_CUTOFF_STATUS_OK) {
+		dev_err(&cq->hwq.pdev->dev,
+			"QPLIB: FP: CQ Process Cutoff Error status = 0x%x\n",
+			hwcqe->status);
+		return -EINVAL;
+	}
+	clear_bit(CQ_FLAGS_RESIZE_IN_PROG, &cq->flags);
+	wake_up_interruptible(&cq->waitq);
+
+	dev_dbg(&cq->hwq.pdev->dev, "QPLIB: FP: CQ Processed Cutoff\n");
+	return 0;
+}
+
+int bnxt_qplib_process_flush_list(struct bnxt_qplib_cq *cq,
+				struct bnxt_qplib_cqe *cqe,
+				int num_cqes)
+{
+	struct bnxt_qplib_qp *qp = NULL;
+	u32 budget = num_cqes;
+	unsigned long flags;
+
+	spin_lock_irqsave(&cq->flush_lock, flags);
+	list_for_each_entry(qp, &cq->sqf_head, sq_flush) {
+		dev_dbg(&cq->hwq.pdev->dev,
+			"QPLIB: FP: Flushing SQ QP= %p\n",
+			qp);
+		__flush_sq(&qp->sq, qp, &cqe, &budget);
+	}
+
+	list_for_each_entry(qp, &cq->rqf_head, rq_flush) {
+		dev_dbg(&cq->hwq.pdev->dev,
+			"QPLIB: FP: Flushing RQ QP= %p\n",
+			qp);
+		__flush_rq(&qp->rq, qp, &cqe, &budget);
+	}
+	spin_unlock_irqrestore(&cq->flush_lock, flags);
+
+	return num_cqes - budget;
+}
+
+int bnxt_qplib_poll_cq(struct bnxt_qplib_cq *cq, struct bnxt_qplib_cqe *cqe,
+		       int num_cqes, struct bnxt_qplib_qp **lib_qp)
+{
+	struct cq_base *hw_cqe;
+	u32 hw_polled = 0;
+	int budget, rc = 0;
+	u8 type;
+
+	budget = num_cqes;
+
+	while (budget) {
+		hw_cqe = bnxt_qplib_get_qe(&cq->hwq, cq->hwq.cons, NULL);
+
+		/* Check for Valid bit */
+		if (!CQE_CMP_VALID(hw_cqe, cq->dbinfo.flags))
+			break;
+
+		/* The valid test of the entry must be done first before
+		 * reading any further.
+		 */
+		dma_rmb();
+		/* From the device's respective CQE format to qplib_wc*/
+		type = hw_cqe->cqe_type_toggle & CQ_BASE_CQE_TYPE_MASK;
+		switch (type) {
+		case CQ_BASE_CQE_TYPE_REQ:
+			rc = bnxt_qplib_cq_process_req(cq,
+					(struct cq_req *)hw_cqe, &cqe, &budget,
+					cq->hwq.cons, lib_qp);
+			break;
+		case CQ_BASE_CQE_TYPE_RES_RC:
+			rc = bnxt_qplib_cq_process_res_rc(cq,
+						(struct cq_res_rc *)hw_cqe, &cqe,
+						&budget);
+			break;
+		case CQ_BASE_CQE_TYPE_RES_UD:
+			rc = bnxt_qplib_cq_process_res_ud(cq,
+						(struct cq_res_ud_v2 *)hw_cqe,
+						&cqe, &budget);
+			break;
+		case CQ_BASE_CQE_TYPE_RES_RAWETH_QP1:
+			rc = bnxt_qplib_cq_process_res_raweth_qp1(cq,
+						(struct cq_res_raweth_qp1 *)
+						hw_cqe, &cqe, &budget);
+			break;
+		case CQ_BASE_CQE_TYPE_TERMINAL:
+			rc = bnxt_qplib_cq_process_terminal(cq,
+						(struct cq_terminal *)hw_cqe,
+						&cqe, &budget);
+			break;
+		case CQ_BASE_CQE_TYPE_CUT_OFF:
+			bnxt_qplib_cq_process_cutoff(cq,
+						(struct cq_cutoff *)hw_cqe);
+			/* Done processing this CQ */
+			goto exit;
+		default:
+			dev_err(&cq->hwq.pdev->dev,
+				"QPLIB: process_cq unknown type 0x%x\n",
+				hw_cqe->cqe_type_toggle &
+				CQ_BASE_CQE_TYPE_MASK);
+			rc = -EINVAL;
+			break;
+		}
+		if (rc < 0) {
+			dev_dbg(&cq->hwq.pdev->dev,
+				"QPLIB: process_cqe rc = 0x%x\n", rc);
+			if (rc == -EAGAIN)
+				break;
+			/* Error while processing the CQE, just skip to the
+			   next one */
+			if (type != CQ_BASE_CQE_TYPE_TERMINAL)
+				dev_err(&cq->hwq.pdev->dev,
+					"QPLIB: process_cqe error rc = 0x%x\n",
+					rc);
+		}
+		hw_polled++;
+		bnxt_qplib_hwq_incr_cons(cq->hwq.depth, &cq->hwq.cons,
+					 1, &cq->dbinfo.flags);
+	}
+	if (hw_polled)
+		bnxt_qplib_ring_db(&cq->dbinfo, DBC_DBC_TYPE_CQ);
+exit:
+	return num_cqes - budget;
+}
+
+void bnxt_qplib_req_notify_cq(struct bnxt_qplib_cq *cq, u32 arm_type)
+{
+	cq->dbinfo.toggle = cq->toggle;
+	if (arm_type)
+		bnxt_qplib_ring_db(&cq->dbinfo, arm_type);
+	/* Using cq->arm_state variable to track whether to issue cq handler */
+	atomic_set(&cq->arm_state, 1);
+}
+
+void bnxt_qplib_flush_cqn_wq(struct bnxt_qplib_qp *qp)
+{
+	flush_workqueue(qp->scq->nq->cqn_wq);
+	if (qp->scq != qp->rcq)
+		flush_workqueue(qp->rcq->nq->cqn_wq);
+}
diff --git a/sys/dev/bnxt/bnxt_re/qplib_fp.h b/sys/dev/bnxt/bnxt_re/qplib_fp.h
new file mode 100644
index 000000000000..527c377f0aa5
--- /dev/null
+++ b/sys/dev/bnxt/bnxt_re/qplib_fp.h
@@ -0,0 +1,638 @@
+/*
+ * Copyright (c) 2015-2024, Broadcom. All rights reserved.  The term
+ * Broadcom refers to Broadcom Limited and/or its subsidiaries.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS''
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
+ * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
+ * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * Description: Fast Path Operators (header)
+ */
+
+#ifndef __BNXT_QPLIB_FP_H__
+#define __BNXT_QPLIB_FP_H__
+
+/* Temp header structures for SQ */
+struct sq_ud_ext_hdr {
+	__le32	dst_qp;
+	__le32	avid;
+	__le64	rsvd;
+};
+
+struct sq_raw_ext_hdr {
+	__le32	cfa_meta;
+	__le32	rsvd0;
+	__le64	rsvd1;
+};
+
+struct sq_rdma_ext_hdr {
+	__le64	remote_va;
+	__le32	remote_key;
+	__le32	rsvd;
+};
+
+struct sq_atomic_ext_hdr {
+	__le64	swap_data;
+	__le64	cmp_data;
+};
+
+struct sq_fr_pmr_ext_hdr {
+	__le64	pblptr;
+	__le64	va;
+};
+
+struct sq_bind_ext_hdr {
+	__le64	va;
+	__le32	length_lo;
+	__le32	length_hi;
+};
+
+struct rq_ext_hdr {
+	__le64	rsvd1;
+	__le64	rsvd2;
+};
+
+#define BNXT_QPLIB_ETHTYPE_ROCEV1	0x8915
+
+struct bnxt_qplib_srq {
+	struct bnxt_qplib_pd		*pd;
+	struct bnxt_qplib_dpi		*dpi;
+	struct bnxt_qplib_chip_ctx	*cctx;
+	struct bnxt_qplib_cq		*cq;
+	struct bnxt_qplib_swq		*swq;
+	struct bnxt_qplib_hwq		hwq;
+	struct bnxt_qplib_db_info	dbinfo;
+	struct bnxt_qplib_sg_info	sginfo;
+	u64				srq_handle;
+	u32				id;
+	u16				wqe_size;
+	u32				max_wqe;
+	u32				max_sge;
+	u32				threshold;
+	bool				arm_req;
+	int				start_idx;
+	int				last_idx;
+	u16				eventq_hw_ring_id;
+	bool				is_user;
+	spinlock_t			lock;
+};
+
+struct bnxt_qplib_sge {
+	u64				addr;
+	u32				size;
+	u32				lkey;
+};
+
+/*
+ * Buffer space for ETH(14), IP or GRH(40), UDP header(8)
+ * and ib_bth + ib_deth (20).
+ * Max required is 82 when RoCE V2 is enabled
+ */
+
+/*
+ *		RoCE V1 (38 bytes needed)
+ * +------------+----------+--------+--------+-------+
+ * |Eth-hdr(14B)| GRH (40B)|bth+deth|  Mad   | iCRC  |
+ * |		| supplied |  20B   |payload |  4B   |
+ * |		| by user  |supplied| 256B   |	     |
+ * |		| mad      |        |by user |	     |
+ * |		|	   |	    |	     |       |
+ * |    sge 1	|  sge 2   | sge 3  | sge 4  | sge 5 |
+ * +------------+----------+--------+--------+-------+
+ */
+
+/*
+ *		RoCE V2-IPv4 (46 Bytes needed)
+ * +------------+----------+--------+--------+-------+
+ * |Eth-hdr(14B)| IP-hdr   |UDP-hdr |  Mad   | iCRC  |
+ * |		| supplied |   8B   |payload |  4B   |
+ * |		| by user  |bth+deth|  256B  |	     |
+ * |		| mad lower|   20B  |supplied|       |
+ * |		| 20B out  | (sge 3)|by user |       |
+ * |		| of 40B   |	    |	     |	     |
+ * |		| grh space|	    |	     |	     |
+ * |	sge 1	|  sge 2   |  sge 3 |  sge 4 | sge 5 |
+ * +------------+----------+--------+--------+-------+
+ */
+
+/*
+ *		RoCE V2-IPv6 (46 Bytes needed)
+ * +------------+----------+--------+--------+-------+
+ * |Eth-hdr(14B)| IPv6     |UDP-hdr |  Mad   | iCRC  |
+ * |		| supplied |   8B   |payload |  4B   |
+ * |		| by user  |bth+deth| 256B   |       |
+ * |		| mad lower|   20B  |supplied|       |
+ * |		| 40 bytes |        |by user |       |
+ * |		| grh space|	    |        | 	     |
+ * |		|	   |	    |	     |	     |
+ * |	sge 1	|  sge 2   |  sge 3 |  sge 4 | sge 5 |
+ * +------------+----------+--------+--------+-------+
+ */
+
+#define BNXT_QPLIB_MAX_QP1_SQ_HDR_SIZE		74
+#define BNXT_QPLIB_MAX_QP1_SQ_HDR_SIZE_V2	86
+#define BNXT_QPLIB_MAX_QP1_RQ_HDR_SIZE		46
+#define BNXT_QPLIB_MAX_QP1_RQ_ETH_HDR_SIZE	14
+#define BNXT_QPLIB_MAX_QP1_RQ_HDR_SIZE_V2	512
+#define BNXT_QPLIB_MAX_GRH_HDR_SIZE_IPV4	20
+#define BNXT_QPLIB_MAX_GRH_HDR_SIZE_IPV6	40
+#define BNXT_QPLIB_MAX_QP1_RQ_BDETH_HDR_SIZE	20
+#define BNXT_QPLIB_MAX_SQSZ			0xFFFF
+
+struct bnxt_qplib_hdrbuf {
+	dma_addr_t	dma_map;
+	void		*va;
+	u32		len;
+	u32		step;
+};
+
+struct bnxt_qplib_swq {
+	u64				wr_id;
+	int				next_idx;
+	u8				type;
+	u8				flags;
+	u32				start_psn;
+	u32				next_psn;
+	u32				slot_idx;
+	u8				slots;
+	/* WIP: make it void * to handle legacy also */
+	struct sq_psn_search		*psn_search;
+	void				*inline_data;
+};
+
+struct bnxt_qplib_swqe {
+	/* General */
+#define	BNXT_QPLIB_FENCE_WRID	0x46454E43	/* "FENC" */
+#define	BNXT_QPLIB_QP1_DUMMY_WRID 0x44554D59 /* "DUMY" */
+	u64				wr_id;
+	u8				reqs_type;
+	u8				type;
+#define BNXT_QPLIB_SWQE_TYPE_SEND			0
+#define BNXT_QPLIB_SWQE_TYPE_SEND_WITH_IMM		1
+#define BNXT_QPLIB_SWQE_TYPE_SEND_WITH_INV		2
+#define BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE			4
+#define BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE_WITH_IMM	5
+#define BNXT_QPLIB_SWQE_TYPE_RDMA_READ			6
+#define BNXT_QPLIB_SWQE_TYPE_ATOMIC_CMP_AND_SWP		8
+#define BNXT_QPLIB_SWQE_TYPE_ATOMIC_FETCH_AND_ADD	11
+#define BNXT_QPLIB_SWQE_TYPE_LOCAL_INV			12
+#define BNXT_QPLIB_SWQE_TYPE_FAST_REG_MR		13
+#define BNXT_QPLIB_SWQE_TYPE_REG_MR			13
+#define BNXT_QPLIB_SWQE_TYPE_BIND_MW			14
+#define BNXT_QPLIB_SWQE_TYPE_RECV			128
+#define BNXT_QPLIB_SWQE_TYPE_RECV_RDMA_IMM		129
+	u8				flags;
+#define BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP		(1 << 0)
+#define BNXT_QPLIB_SWQE_FLAGS_RD_ATOMIC_FENCE		(1 << 1)
+#define BNXT_QPLIB_SWQE_FLAGS_UC_FENCE			(1 << 2)
+#define BNXT_QPLIB_SWQE_FLAGS_SOLICIT_EVENT		(1 << 3)
+#define BNXT_QPLIB_SWQE_FLAGS_INLINE			(1 << 4)
+	struct bnxt_qplib_sge		*sg_list;
+	int				num_sge;
+
+	union {
+		/* Send, with imm, inval key */
+		struct {
+			union {
+				__be32  imm_data;
+				u32     inv_key;
+			};
+			u32		q_key;
+			u32		dst_qp;
+			u16		avid;
+		} send;
+
+		/* Send Raw Ethernet and QP1 */
+		struct {
+			u16		lflags;
+			u16		cfa_action;
+			u32		cfa_meta;
+		} rawqp1;
+
+		/* RDMA write, with imm, read */
+		struct {
+			union {
+				__be32  imm_data;
+				u32     inv_key;
+			};
+			u64		remote_va;
+			u32		r_key;
+		} rdma;
+
+		/* Atomic cmp/swap, fetch/add */
+		struct {
+			u64		remote_va;
+			u32		r_key;
+			u64		swap_data;
+			u64		cmp_data;
+		} atomic;
+
+		/* Local Invalidate */
+		struct {
+			u32		inv_l_key;
+		} local_inv;
+
+		/* FR-PMR */
+		struct {
+			u8		access_cntl;
+			u8		pg_sz_log;
+			bool		zero_based;
+			u32		l_key;
+			u32		length;
+			u8		pbl_pg_sz_log;
+#define BNXT_QPLIB_SWQE_PAGE_SIZE_4K			0
+#define BNXT_QPLIB_SWQE_PAGE_SIZE_8K			1
+#define BNXT_QPLIB_SWQE_PAGE_SIZE_64K			4
+#define BNXT_QPLIB_SWQE_PAGE_SIZE_256K			6
+#define BNXT_QPLIB_SWQE_PAGE_SIZE_1M			8
+#define BNXT_QPLIB_SWQE_PAGE_SIZE_2M			9
+#define BNXT_QPLIB_SWQE_PAGE_SIZE_4M			10
+#define BNXT_QPLIB_SWQE_PAGE_SIZE_1G			18
+			u8		levels;
+#define PAGE_SHIFT_4K	12
+			__le64		*pbl_ptr;
+			dma_addr_t	pbl_dma_ptr;
+			u64		*page_list;
+			u16		page_list_len;
+			u64		va;
+		} frmr;
+
+		/* Bind */
+		struct {
+			u8		access_cntl;
+#define BNXT_QPLIB_BIND_SWQE_ACCESS_LOCAL_WRITE		(1 << 0)
+#define BNXT_QPLIB_BIND_SWQE_ACCESS_REMOTE_READ		(1 << 1)
+#define BNXT_QPLIB_BIND_SWQE_ACCESS_REMOTE_WRITE	(1 << 2)
+#define BNXT_QPLIB_BIND_SWQE_ACCESS_REMOTE_ATOMIC	(1 << 3)
+#define BNXT_QPLIB_BIND_SWQE_ACCESS_WINDOW_BIND		(1 << 4)
+			bool		zero_based;
+			u8		mw_type;
+			u32		parent_l_key;
+			u32		r_key;
+			u64		va;
+			u32		length;
+		} bind;
+	};
+};
+
+struct bnxt_qplib_q {
+	struct bnxt_qplib_swq		*swq;
+	struct bnxt_qplib_db_info	dbinfo;
+	struct bnxt_qplib_sg_info	sginfo;
+	struct bnxt_qplib_hwq		hwq;
+	u32				max_wqe;
+	u16				max_sge;
+	u16				wqe_size;
+	u16				q_full_delta;
+	u32				psn;
+	bool				condition;
+	bool				single;
+	bool				legacy_send_phantom;
+	u32				phantom_wqe_cnt;
+	u32				phantom_cqe_cnt;
+	u32				next_cq_cons;
+	bool				flushed;
+	u32				swq_start;
+	u32				swq_last;
+};
+
+#define BNXT_QPLIB_PPP_REQ		0x1
+#define BNXT_QPLIB_PPP_ST_IDX_SHIFT	0x1
+
+struct bnxt_qplib_ppp {
+	u32 dpi;
+	u8 req;
+	u8 st_idx_en;
+};
+
+struct bnxt_qplib_qp {
+	struct bnxt_qplib_pd		*pd;
+	struct bnxt_qplib_dpi		*dpi;
+	struct bnxt_qplib_chip_ctx	*cctx;
+	u64				qp_handle;
+#define	BNXT_QPLIB_QP_ID_INVALID	0xFFFFFFFF
+	u32				id;
+	u8				type;
+	u8				sig_type;
+	u8				wqe_mode;
+	u8				state;
+	u8				cur_qp_state;
+	u8				is_user;
+	u64				modify_flags;
+	u32				max_inline_data;
+	u32				mtu;
+	u32				path_mtu;
+	bool				en_sqd_async_notify;
+	u16				pkey_index;
+	u32				qkey;
+	u32				dest_qp_id;
+	u8				access;
+	u8				timeout;
+	u8				retry_cnt;
+	u8				rnr_retry;
+	u64				wqe_cnt;
+	u32				min_rnr_timer;
+	u32				max_rd_atomic;
+	u32				max_dest_rd_atomic;
+	u32				dest_qpn;
+	u8				smac[6];
+	u16				vlan_id;
+	u8				nw_type;
+	u16				port_id;
+	struct bnxt_qplib_ah		ah;
+	struct bnxt_qplib_ppp		ppp;
+
+#define BTH_PSN_MASK			((1 << 24) - 1)
+	/* SQ */
+	struct bnxt_qplib_q		sq;
+	/* RQ */
+	struct bnxt_qplib_q		rq;
+	/* SRQ */
+	struct bnxt_qplib_srq		*srq;
+	/* CQ */
+	struct bnxt_qplib_cq		*scq;
+	struct bnxt_qplib_cq		*rcq;
+	/* IRRQ and ORRQ */
+	struct bnxt_qplib_hwq		irrq;
+	struct bnxt_qplib_hwq		orrq;
+	/* Header buffer for QP1 */
+	struct bnxt_qplib_hdrbuf	*sq_hdr_buf;
+	struct bnxt_qplib_hdrbuf	*rq_hdr_buf;
+
+	/* ToS */
+	u8				tos_ecn;
+	u8				tos_dscp;
+	/* To track the SQ and RQ flush list */
+	struct list_head		sq_flush;
+	struct list_head		rq_flush;
+	/* 4 bytes of QP's scrabled mac received from FW */
+	u32				lag_src_mac;
+	u32				msn;
+	u32				msn_tbl_sz;
+	/* get devflags in PI code */
+	u16				dev_cap_flags;
+};
+
+
+#define CQE_CMP_VALID(hdr, pass)				\
+	(!!((hdr)->cqe_type_toggle & CQ_BASE_TOGGLE) ==		\
+	   !(pass & BNXT_QPLIB_FLAG_EPOCH_CONS_MASK))
+
+static inline u32 __bnxt_qplib_get_avail(struct bnxt_qplib_hwq *hwq)
+{
+	int cons, prod, avail;
+
+	/* False full is possible retrying post-send makes sense */
+	cons = hwq->cons;
+	prod = hwq->prod;
+	avail = cons - prod;
+	if (cons <= prod)
+		avail += hwq->depth;
+	return avail;
+}
+
+static inline bool bnxt_qplib_queue_full(struct bnxt_qplib_hwq *hwq, u8 slots)
+{
+	return __bnxt_qplib_get_avail(hwq) <= slots;
+}
+
+struct bnxt_qplib_cqe {
+	u8				status;
+	u8				type;
+	u8				opcode;
+	u32				length;
+	/* Lower 16 is cfa_metadata0, Upper 16 is cfa_metadata1 */
+	u32				cfa_meta;
+#define BNXT_QPLIB_META1_SHIFT		16
+#define	BNXT_QPLIB_CQE_CFA_META1_VALID  0x80000UL
+	u64				wr_id;
+	union {
+		__be32                  immdata;
+		u32                     invrkey;
+	};
+	u64				qp_handle;
+	u64				mr_handle;
+	u16				flags;
+	u8				smac[6];
+	u32				src_qp;
+	u16				raweth_qp1_flags;
+	u16				raweth_qp1_errors;
+	u16				raweth_qp1_cfa_code;
+	u32				raweth_qp1_flags2;
+	u32				raweth_qp1_metadata;
+	u8				raweth_qp1_payload_offset;
+	u16				pkey_index;
+};
+
+#define BNXT_QPLIB_QUEUE_START_PERIOD		0x01
+struct bnxt_qplib_cq {
+	struct bnxt_qplib_dpi		*dpi;
+	struct bnxt_qplib_chip_ctx	*cctx;
+	struct bnxt_qplib_nq		*nq;
+	struct bnxt_qplib_db_info	dbinfo;
+	struct bnxt_qplib_sg_info	sginfo;
+	struct bnxt_qplib_hwq		hwq;
+	struct bnxt_qplib_hwq		resize_hwq;
+	struct list_head		sqf_head;
+	struct list_head		rqf_head;
+	u32				max_wqe;
+	u32				id;
+	u16				count;
+	u16				period;
+	u32				cnq_hw_ring_id;
+	u64				cq_handle;
+	atomic_t			arm_state;
+#define CQ_RESIZE_WAIT_TIME_MS		500
+	unsigned long			flags;
+#define CQ_FLAGS_RESIZE_IN_PROG		1
+	wait_queue_head_t		waitq;
+	spinlock_t			flush_lock; /* lock flush queue list */
+	spinlock_t			compl_lock; /* synch CQ handlers */
+	u16				cnq_events;
+	bool				is_cq_err_event;
+	bool				destroyed;
+	u8				toggle;
+};
+
+#define BNXT_QPLIB_MAX_IRRQE_ENTRY_SIZE	sizeof(struct xrrq_irrq)
+#define BNXT_QPLIB_MAX_ORRQE_ENTRY_SIZE	sizeof(struct xrrq_orrq)
+#define IRD_LIMIT_TO_IRRQ_SLOTS(x)	(2 * x + 2)
+#define IRRQ_SLOTS_TO_IRD_LIMIT(s)	((s >> 1) - 1)
+#define ORD_LIMIT_TO_ORRQ_SLOTS(x)	(x + 1)
+#define ORRQ_SLOTS_TO_ORD_LIMIT(s)	(s - 1)
+
+#define NQE_CMP_VALID(hdr, pass)				\
+	(!!(le32_to_cpu((hdr)->info63_v & 0xffffffff) & NQ_BASE_V) ==	\
+	   !(pass & BNXT_QPLIB_FLAG_EPOCH_CONS_MASK))
+
+#define BNXT_QPLIB_NQE_MAX_CNT		(128 * 1024)
+
+/* MSN table print macros for debugging */
+#define BNXT_RE_MSN_IDX(m) (((m) & SQ_MSN_SEARCH_START_IDX_MASK) >> \
+		SQ_MSN_SEARCH_START_IDX_SFT)
+#define BNXT_RE_MSN_NPSN(m) (((m) & SQ_MSN_SEARCH_NEXT_PSN_MASK) >> \
+		SQ_MSN_SEARCH_NEXT_PSN_SFT)
+#define BNXT_RE_MSN_SPSN(m) (((m) & SQ_MSN_SEARCH_START_PSN_MASK) >> \
+		SQ_MSN_SEARCH_START_PSN_SFT)
+#define BNXT_MSN_TBLE_SGE 6
+
+struct bnxt_qplib_nq_stats {
+	u64	num_dbqne_processed;
+	u64	num_srqne_processed;
+	u64	num_cqne_processed;
+	u64	num_tasklet_resched;
+	u64	num_nq_rearm;
+};
+
+struct bnxt_qplib_nq_db {
+	struct bnxt_qplib_reg_desc	reg;
+	void __iomem			*db;
+	struct bnxt_qplib_db_info	dbinfo;
+};
+
+typedef int (*cqn_handler_t)(struct bnxt_qplib_nq *nq,
+			     struct bnxt_qplib_cq *cq);
+typedef int (*srqn_handler_t)(struct bnxt_qplib_nq *nq,
+			      struct bnxt_qplib_srq *srq, u8 event);
+
+struct bnxt_qplib_nq {
+	struct bnxt_qplib_res		*res;
+	struct bnxt_qplib_hwq		hwq;
+	struct bnxt_qplib_nq_db		nq_db;
+
+	char				*name;
+	u16				ring_id;
+	int				msix_vec;
+	bool				requested;
+	int				budget;
+	u32				load;
+	struct mutex			lock;
+
+	cqn_handler_t			cqn_handler;
+	srqn_handler_t			srqn_handler;
+	struct workqueue_struct		*cqn_wq;
+	struct bnxt_qplib_nq_stats	stats;
+};
+
+struct bnxt_qplib_nq_work {
+	struct work_struct      work;
+	struct bnxt_qplib_nq    *nq;
+	struct bnxt_qplib_cq	*cq;
+};
+
+static inline dma_addr_t
+bnxt_qplib_get_qp_buf_from_index(struct bnxt_qplib_qp *qp, u32 index)
+{
+	struct bnxt_qplib_hdrbuf *buf;
+
+	buf = qp->rq_hdr_buf;
+	return (buf->dma_map + index * buf->step);
+}
+
+void bnxt_qplib_nq_stop_irq(struct bnxt_qplib_nq *nq, bool kill);
+void bnxt_qplib_disable_nq(struct bnxt_qplib_nq *nq);
+int bnxt_qplib_nq_start_irq(struct bnxt_qplib_nq *nq, int nq_indx,
+			    int msix_vector, bool need_init);
+int bnxt_qplib_enable_nq(struct bnxt_qplib_nq *nq, int nq_idx,
+			 int msix_vector, int bar_reg_offset,
+			 cqn_handler_t cqn_handler,
+			 srqn_handler_t srq_handler);
+int bnxt_qplib_create_srq(struct bnxt_qplib_res *res,
+			  struct bnxt_qplib_srq *srq);
+int bnxt_qplib_modify_srq(struct bnxt_qplib_res *res,
+			  struct bnxt_qplib_srq *srq);
+int bnxt_qplib_query_srq(struct bnxt_qplib_res *res,
+			 struct bnxt_qplib_srq *srq);
+int bnxt_qplib_destroy_srq(struct bnxt_qplib_res *res,
+			   struct bnxt_qplib_srq *srq);
+int bnxt_qplib_post_srq_recv(struct bnxt_qplib_srq *srq,
+			     struct bnxt_qplib_swqe *wqe);
+int bnxt_qplib_create_qp1(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp);
+int bnxt_qplib_create_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp);
+int bnxt_qplib_modify_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp);
+int bnxt_qplib_query_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp);
+int bnxt_qplib_destroy_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp);
+void bnxt_qplib_clean_qp(struct bnxt_qplib_qp *qp);
+void bnxt_qplib_free_qp_res(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp);
+void *bnxt_qplib_get_qp1_sq_buf(struct bnxt_qplib_qp *qp,
+				struct bnxt_qplib_sge *sge);
+void *bnxt_qplib_get_qp1_rq_buf(struct bnxt_qplib_qp *qp,
+				struct bnxt_qplib_sge *sge);
+u32 bnxt_qplib_get_rq_prod_index(struct bnxt_qplib_qp *qp);
+void bnxt_qplib_post_send_db(struct bnxt_qplib_qp *qp);
+int bnxt_qplib_post_send(struct bnxt_qplib_qp *qp,
+			 struct bnxt_qplib_swqe *wqe);
+void bnxt_qplib_post_recv_db(struct bnxt_qplib_qp *qp);
+int bnxt_qplib_post_recv(struct bnxt_qplib_qp *qp,
+			 struct bnxt_qplib_swqe *wqe);
+int bnxt_qplib_create_cq(struct bnxt_qplib_res *res, struct bnxt_qplib_cq *cq);
+int bnxt_qplib_modify_cq(struct bnxt_qplib_res *res, struct bnxt_qplib_cq *cq);
+int bnxt_qplib_resize_cq(struct bnxt_qplib_res *res, struct bnxt_qplib_cq *cq,
+			 int new_cqes);
+void bnxt_qplib_resize_cq_complete(struct bnxt_qplib_res *res,
+				   struct bnxt_qplib_cq *cq);
+int bnxt_qplib_destroy_cq(struct bnxt_qplib_res *res, struct bnxt_qplib_cq *cq);
+void bnxt_qplib_free_cq(struct bnxt_qplib_res *res, struct bnxt_qplib_cq *cq);
+int bnxt_qplib_poll_cq(struct bnxt_qplib_cq *cq, struct bnxt_qplib_cqe *cqe,
+		       int num, struct bnxt_qplib_qp **qp);
+bool bnxt_qplib_is_cq_empty(struct bnxt_qplib_cq *cq);
+void bnxt_qplib_req_notify_cq(struct bnxt_qplib_cq *cq, u32 arm_type);
+void bnxt_qplib_free_nq_mem(struct bnxt_qplib_nq *nq);
+int bnxt_qplib_alloc_nq_mem(struct bnxt_qplib_res *res,
+			    struct bnxt_qplib_nq *nq);
+void bnxt_qplib_add_flush_qp(struct bnxt_qplib_qp *qp);
+void bnxt_qplib_del_flush_qp(struct bnxt_qplib_qp *qp);
+int bnxt_qplib_process_flush_list(struct bnxt_qplib_cq *cq,
+				struct bnxt_qplib_cqe *cqe,
+				int num_cqes);
+void bnxt_qplib_flush_cqn_wq(struct bnxt_qplib_qp *qp);
+void bnxt_qplib_free_hdr_buf(struct bnxt_qplib_res *res,
+			     struct bnxt_qplib_qp *qp);
+int bnxt_qplib_alloc_hdr_buf(struct bnxt_qplib_res *res,
+			     struct bnxt_qplib_qp *qp, u32 slen, u32 rlen);
+
+static inline bool __can_request_ppp(struct bnxt_qplib_qp *qp)
+{
+	bool can_request = false;
+
+	if (qp->cur_qp_state == CMDQ_MODIFY_QP_NEW_STATE_RESET &&
+	    qp->state ==  CMDQ_MODIFY_QP_NEW_STATE_INIT &&
+	    qp->ppp.req &&
+	    !(qp->ppp.st_idx_en &
+		    CREQ_MODIFY_QP_RESP_PINGPONG_PUSH_ENABLED))
+		can_request = true;
+	return can_request;
+}
+
+/* MSN table update inlin */
+static inline uint64_t bnxt_re_update_msn_tbl(uint32_t st_idx, uint32_t npsn, uint32_t start_psn)
+{
+	return cpu_to_le64((((u64)(st_idx) << SQ_MSN_SEARCH_START_IDX_SFT) &
+		SQ_MSN_SEARCH_START_IDX_MASK) |
+		(((u64)(npsn) << SQ_MSN_SEARCH_NEXT_PSN_SFT) &
+		SQ_MSN_SEARCH_NEXT_PSN_MASK) |
+		(((start_psn) << SQ_MSN_SEARCH_START_PSN_SFT) &
+		SQ_MSN_SEARCH_START_PSN_MASK));
+}
+
+void bnxt_re_schedule_dbq_event(struct bnxt_qplib_res *res);
+#endif
diff --git a/sys/dev/bnxt/bnxt_re/qplib_rcfw.c b/sys/dev/bnxt/bnxt_re/qplib_rcfw.c
new file mode 100644
index 000000000000..7e3453a1e044
--- /dev/null
+++ b/sys/dev/bnxt/bnxt_re/qplib_rcfw.c
@@ -0,0 +1,1338 @@
+/*
+ * Copyright (c) 2015-2024, Broadcom. All rights reserved.  The term
+ * Broadcom refers to Broadcom Limited and/or its subsidiaries.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS''
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
+ * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
+ * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * Description: RDMA Controller HW interface
+ */
+
+#include <linux/interrupt.h>
+#include <linux/spinlock.h>
+#include <linux/dma-mapping.h>
+#include <linux/sched.h>
+#include <linux/pci.h>
+#include <linux/delay.h>
+#include <linux/hardirq.h>
+#include <linux/device.h>
+
+#include "hsi_struct_def.h"
+#include "qplib_tlv.h"
+#include "qplib_res.h"
+#include "qplib_sp.h"
+#include "qplib_rcfw.h"
+#include "bnxt_re.h"
+
+static void bnxt_qplib_service_creq(unsigned long data);
+
+int __check_cmdq_stall(struct bnxt_qplib_rcfw *rcfw,
+			      u32 *cur_prod, u32 *cur_cons)
+{
+	struct bnxt_qplib_cmdq_ctx *cmdq;
+
+	cmdq = &rcfw->cmdq;
+
+	if (*cur_prod == cmdq->hwq.prod &&
+	    *cur_cons == cmdq->hwq.cons)
+		/* No activity on CMDQ or CREQ. FW down */
+		return -ETIMEDOUT;
+
+	*cur_prod = cmdq->hwq.prod;
+	*cur_cons = cmdq->hwq.cons;
+	return 0;
+}
+
+static int bnxt_qplib_map_rc(u8 opcode)
+{
+	switch (opcode) {
+	case CMDQ_BASE_OPCODE_DESTROY_QP:
+	case CMDQ_BASE_OPCODE_DESTROY_SRQ:
+	case CMDQ_BASE_OPCODE_DESTROY_CQ:
+	case CMDQ_BASE_OPCODE_DEALLOCATE_KEY:
+	case CMDQ_BASE_OPCODE_DEREGISTER_MR:
+	case CMDQ_BASE_OPCODE_DELETE_GID:
+	case CMDQ_BASE_OPCODE_DESTROY_QP1:
+	case CMDQ_BASE_OPCODE_DESTROY_AH:
+	case CMDQ_BASE_OPCODE_DEINITIALIZE_FW:
+	case CMDQ_BASE_OPCODE_MODIFY_ROCE_CC:
+	case CMDQ_BASE_OPCODE_SET_LINK_AGGR_MODE:
+		return 0;
+	default:
+		return -ETIMEDOUT;
+	}
+}
+
+/**
+ * bnxt_re_is_fw_stalled   -	Check firmware health
+ * @rcfw      -   rcfw channel instance of rdev
+ * @cookie    -   cookie to track the command
+ *
+ * If firmware has not responded any rcfw command within
+ * rcfw->max_timeout, consider firmware as stalled.
+ *
+ * Returns:
+ * 0 if firmware is responding
+ * -ENODEV if firmware is not responding
+ */
+static int bnxt_re_is_fw_stalled(struct bnxt_qplib_rcfw *rcfw, u16 cookie)
+{
+	struct bnxt_qplib_cmdq_ctx *cmdq;
+	struct bnxt_qplib_crsqe *crsqe;
+
+	crsqe = &rcfw->crsqe_tbl[cookie];
+	cmdq = &rcfw->cmdq;
+
+	if (time_after(jiffies, cmdq->last_seen +
+		      (rcfw->max_timeout * HZ))) {
+		dev_warn_ratelimited(&rcfw->pdev->dev,
+				     "%s: FW STALL Detected. cmdq[%#x]=%#x waited (%ld > %d) msec active %d\n",
+				     __func__, cookie, crsqe->opcode,
+				     (long)jiffies_to_msecs(jiffies - cmdq->last_seen),
+				     rcfw->max_timeout * 1000,
+				     crsqe->is_in_used);
+		return -ENODEV;
+	}
+
+	return 0;
+}
+/**
+ * __wait_for_resp   -	Don't hold the cpu context and wait for response
+ * @rcfw      -   rcfw channel instance of rdev
+ * @cookie    -   cookie to track the command
+ *
+ * Wait for command completion in sleepable context.
+ *
+ * Returns:
+ * 0 if command is completed by firmware.
+ * Non zero error code for rest of the case.
+ */
+static int __wait_for_resp(struct bnxt_qplib_rcfw *rcfw, u16 cookie)
+{
+	struct bnxt_qplib_cmdq_ctx *cmdq;
+	struct bnxt_qplib_crsqe *crsqe;
+	unsigned long issue_time;
+	int ret;
+
+	cmdq = &rcfw->cmdq;
+	issue_time = jiffies;
+	crsqe = &rcfw->crsqe_tbl[cookie];
+
+	do {
+		if (RCFW_NO_FW_ACCESS(rcfw))
+			return bnxt_qplib_map_rc(crsqe->opcode);
+		if (test_bit(FIRMWARE_STALL_DETECTED, &cmdq->flags))
+			return -ETIMEDOUT;
+
+		/* Non zero means command completed */
+		ret = wait_event_timeout(cmdq->waitq,
+					 !crsqe->is_in_used ||
+					 RCFW_NO_FW_ACCESS(rcfw),
+					 msecs_to_jiffies(rcfw->max_timeout * 1000));
+
+		if (!crsqe->is_in_used)
+			return 0;
+		/*
+		 * Take care if interrupt miss or other cases like DBR drop
+		 */
+		bnxt_qplib_service_creq((unsigned long)rcfw);
+		dev_warn_ratelimited(&rcfw->pdev->dev,
+			"Non-Blocking QPLIB: cmdq[%#x]=%#x waited (%lu) msec bit %d\n",
+			cookie, crsqe->opcode,
+			(long)jiffies_to_msecs(jiffies - issue_time),
+			crsqe->is_in_used);
+
+		if (!crsqe->is_in_used)
+			return 0;
+
+		ret = bnxt_re_is_fw_stalled(rcfw, cookie);
+		if (ret)
+			return ret;
+
+	} while (true);
+};
+
+/**
+ * __block_for_resp   -	hold the cpu context and wait for response
+ * @rcfw      -   rcfw channel instance of rdev
+ * @cookie    -   cookie to track the command
+ *
+ * This function will hold the cpu (non-sleepable context) and
+ * wait for command completion. Maximum holding interval is 8 second.
+ *
+ * Returns:
+ * -ETIMEOUT if command is not completed in specific time interval.
+ * 0 if command is completed by firmware.
+ */
+static int __block_for_resp(struct bnxt_qplib_rcfw *rcfw, u16 cookie)
+{
+	struct bnxt_qplib_cmdq_ctx *cmdq = &rcfw->cmdq;
+	struct bnxt_qplib_crsqe *crsqe;
+	unsigned long issue_time = 0;
+
+	issue_time = jiffies;
+	crsqe = &rcfw->crsqe_tbl[cookie];
+
+	do {
+		if (RCFW_NO_FW_ACCESS(rcfw))
+			return bnxt_qplib_map_rc(crsqe->opcode);
+		if (test_bit(FIRMWARE_STALL_DETECTED, &cmdq->flags))
+			return -ETIMEDOUT;
+
+		udelay(1);
+
+		/* Below call is must since there can be a deadlock
+		 * if interrupt is mapped to the same cpu
+		 */
+		bnxt_qplib_service_creq((unsigned long)rcfw);
+		if (!crsqe->is_in_used)
+			return 0;
+
+	} while (time_before(jiffies, issue_time + (8 * HZ)));
+
+	dev_warn_ratelimited(&rcfw->pdev->dev,
+		"Blocking QPLIB: cmdq[%#x]=%#x taken (%lu) msec",
+		cookie, crsqe->opcode,
+		(long)jiffies_to_msecs(jiffies - issue_time));
+
+	return -ETIMEDOUT;
+};
+
+/*  __send_message_no_waiter -	get cookie and post the message.
+ * @rcfw      -   rcfw channel instance of rdev
+ * @msg      -    qplib message internal
+ *
+ * This function will just post and don't bother about completion.
+ * Current design of this function is -
+ * user must hold the completion queue hwq->lock.
+ * user must have used existing completion and free the resources.
+ * this function will not check queue full condition.
+ * this function will explicitly set is_waiter_alive=false.
+ * current use case is - send destroy_ah if create_ah is return
+ * after waiter of create_ah is lost. It can be extended for other
+ * use case as well.
+ *
+ * Returns: Nothing
+ *
+ */
+static  void __send_message_no_waiter(struct bnxt_qplib_rcfw *rcfw,
+			  struct bnxt_qplib_cmdqmsg *msg)
+{
+	struct bnxt_qplib_cmdq_ctx *cmdq = &rcfw->cmdq;
+	struct bnxt_qplib_hwq *cmdq_hwq = &cmdq->hwq;
+	struct bnxt_qplib_crsqe *crsqe;
+	struct bnxt_qplib_cmdqe *cmdqe;
+	u32 sw_prod, cmdq_prod, bsize;
+	u16 cookie;
+	u8 *preq;
+
+	cookie = cmdq->seq_num & RCFW_MAX_COOKIE_VALUE;
+	__set_cmdq_base_cookie(msg->req, msg->req_sz, cpu_to_le16(cookie));
+	crsqe = &rcfw->crsqe_tbl[cookie];
+
+	/* Set cmd_size in terms of 16B slots in req. */
+	bsize = bnxt_qplib_set_cmd_slots(msg->req);
+	/* GET_CMD_SIZE would return number of slots in either case of tlv
+	 * and non-tlv commands after call to bnxt_qplib_set_cmd_slots()
+	 */
+	crsqe->send_timestamp = jiffies;
+	crsqe->is_internal_cmd = true;
+	crsqe->is_waiter_alive = false;
+	crsqe->is_in_used = true;
+	crsqe->req_size = __get_cmdq_base_cmd_size(msg->req, msg->req_sz);
+
+	preq = (u8 *)msg->req;
+	do {
+		/* Locate the next cmdq slot */
+		sw_prod = HWQ_CMP(cmdq_hwq->prod, cmdq_hwq);
+		cmdqe = bnxt_qplib_get_qe(cmdq_hwq, sw_prod, NULL);
+		/* Copy a segment of the req cmd to the cmdq */
+		memset(cmdqe, 0, sizeof(*cmdqe));
+		memcpy(cmdqe, preq, min_t(u32, bsize, sizeof(*cmdqe)));
+		preq += min_t(u32, bsize, sizeof(*cmdqe));
+		bsize -= min_t(u32, bsize, sizeof(*cmdqe));
+		cmdq_hwq->prod++;
+	} while (bsize > 0);
+	cmdq->seq_num++;
+
+	cmdq_prod = cmdq_hwq->prod & 0xFFFF;
+	atomic_inc(&rcfw->timeout_send);
+	/* ring CMDQ DB */
+	wmb();
+	writel(cmdq_prod, cmdq->cmdq_mbox.prod);
+	writel(RCFW_CMDQ_TRIG_VAL, cmdq->cmdq_mbox.db);
+}
+
+static int __send_message(struct bnxt_qplib_rcfw *rcfw,
+			  struct bnxt_qplib_cmdqmsg *msg)
+{
+	u32 bsize, free_slots, required_slots;
+	struct bnxt_qplib_cmdq_ctx *cmdq;
+	struct bnxt_qplib_crsqe *crsqe;
+	struct bnxt_qplib_cmdqe *cmdqe;
+	struct bnxt_qplib_hwq *cmdq_hwq;
+	u32 sw_prod, cmdq_prod;
+	struct pci_dev *pdev;
+	unsigned long flags;
+	u16 cookie;
+	u8 opcode;
+	u8 *preq;
+
+	cmdq = &rcfw->cmdq;
+	cmdq_hwq = &cmdq->hwq;
+	pdev = rcfw->pdev;
+	opcode = __get_cmdq_base_opcode(msg->req, msg->req_sz);
+
+	/* Cmdq are in 16-byte units, each request can consume 1 or more
+	   cmdqe */
+	spin_lock_irqsave(&cmdq_hwq->lock, flags);
+	required_slots = bnxt_qplib_get_cmd_slots(msg->req);
+	free_slots = HWQ_FREE_SLOTS(cmdq_hwq);
+	cookie = cmdq->seq_num & RCFW_MAX_COOKIE_VALUE;
+	crsqe = &rcfw->crsqe_tbl[cookie];
+
+	if (required_slots >= free_slots) {
+		dev_warn_ratelimited(&pdev->dev,
+				"QPLIB: RCFW: CMDQ is full req/free %d/%d!\n",
+				required_slots, free_slots);
+		rcfw->cmdq_full_dbg++;
+		spin_unlock_irqrestore(&cmdq_hwq->lock, flags);
+		return -EAGAIN;
+	}
+
+	if (crsqe->is_in_used)
+		panic("QPLIB: Cookie was not requested %d\n",
+				cookie);
+
+	if (msg->block)
+		cookie |= RCFW_CMD_IS_BLOCKING;
+	__set_cmdq_base_cookie(msg->req, msg->req_sz, cpu_to_le16(cookie));
+
+	/* Set cmd_size in terms of 16B slots in req. */
+	bsize = bnxt_qplib_set_cmd_slots(msg->req);
+	/* GET_CMD_SIZE would return number of slots in either case of tlv
+	 * and non-tlv commands after call to bnxt_qplib_set_cmd_slots()
+	 */
+	crsqe->send_timestamp = jiffies;
+	crsqe->free_slots = free_slots;
+	crsqe->resp = (struct creq_qp_event *)msg->resp;
+	crsqe->resp->cookie = cpu_to_le16(cookie);
+	crsqe->is_internal_cmd = false;
+	crsqe->is_waiter_alive = true;
+	crsqe->is_in_used = true;
+	crsqe->opcode = opcode;
+	crsqe->requested_qp_state = msg->qp_state;
+
+	crsqe->req_size = __get_cmdq_base_cmd_size(msg->req, msg->req_sz);
+	if (__get_cmdq_base_resp_size(msg->req, msg->req_sz) && msg->sb) {
+		struct bnxt_qplib_rcfw_sbuf *sbuf = msg->sb;
+
+		__set_cmdq_base_resp_addr(msg->req, msg->req_sz,
+					  cpu_to_le64(sbuf->dma_addr));
+		__set_cmdq_base_resp_size(msg->req, msg->req_sz,
+					  ALIGN(sbuf->size, BNXT_QPLIB_CMDQE_UNITS) /
+					   BNXT_QPLIB_CMDQE_UNITS);
+	}
+
+	preq = (u8 *)msg->req;
+	do {
+		/* Locate the next cmdq slot */
+		sw_prod = HWQ_CMP(cmdq_hwq->prod, cmdq_hwq);
+		cmdqe = bnxt_qplib_get_qe(cmdq_hwq, sw_prod, NULL);
+		/* Copy a segment of the req cmd to the cmdq */
+		memset(cmdqe, 0, sizeof(*cmdqe));
+		memcpy(cmdqe, preq, min_t(u32, bsize, sizeof(*cmdqe)));
+		preq += min_t(u32, bsize, sizeof(*cmdqe));
+		bsize -= min_t(u32, bsize, sizeof(*cmdqe));
+		cmdq_hwq->prod++;
+	} while (bsize > 0);
+	cmdq->seq_num++;
+
+	cmdq_prod = cmdq_hwq->prod & 0xFFFF;
+	if (test_bit(FIRMWARE_FIRST_FLAG, &cmdq->flags)) {
+		/* The very first doorbell write
+		 * is required to set this flag
+		 * which prompts the FW to reset
+		 * its internal pointers
+		 */
+		cmdq_prod |= BIT(FIRMWARE_FIRST_FLAG);
+		clear_bit(FIRMWARE_FIRST_FLAG, &cmdq->flags);
+	}
+	/* ring CMDQ DB */
+	wmb();
+	writel(cmdq_prod, cmdq->cmdq_mbox.prod);
+	writel(RCFW_CMDQ_TRIG_VAL, cmdq->cmdq_mbox.db);
+
+	dev_dbg(&pdev->dev, "QPLIB: RCFW sent request with 0x%x 0x%x 0x%x\n",
+			cmdq_prod, cmdq_hwq->prod, crsqe->req_size);
+	dev_dbg(&pdev->dev,
+		"QPLIB: opcode 0x%x with cookie 0x%x at cmdq/crsq 0x%p/0x%p\n",
+		opcode,
+		__get_cmdq_base_cookie(msg->req, msg->req_sz),
+		cmdqe, crsqe);
+	spin_unlock_irqrestore(&cmdq_hwq->lock, flags);
+	/* Return the CREQ response pointer */
+	return 0;
+}
+
+/**
+ * __poll_for_resp   -	self poll completion for rcfw command
+ * @rcfw      -   rcfw channel instance of rdev
+ * @cookie    -   cookie to track the command
+ *
+ * It works same as __wait_for_resp except this function will
+ * do self polling in sort interval since interrupt is disabled.
+ * This function can not be called from non-sleepable context.
+ *
+ * Returns:
+ * -ETIMEOUT if command is not completed in specific time interval.
+ * 0 if command is completed by firmware.
+ */
+static int __poll_for_resp(struct bnxt_qplib_rcfw *rcfw, u16 cookie)
+{
+	struct bnxt_qplib_cmdq_ctx *cmdq = &rcfw->cmdq;
+	struct bnxt_qplib_crsqe *crsqe;
+	unsigned long issue_time;
+	int ret;
+
+	issue_time = jiffies;
+	crsqe = &rcfw->crsqe_tbl[cookie];
+
+	do {
+		if (RCFW_NO_FW_ACCESS(rcfw))
+			return bnxt_qplib_map_rc(crsqe->opcode);
+		if (test_bit(FIRMWARE_STALL_DETECTED, &cmdq->flags))
+			return -ETIMEDOUT;
+
+		usleep_range(1000, 1001);
+
+		bnxt_qplib_service_creq((unsigned long)rcfw);
+		if (!crsqe->is_in_used)
+			return 0;
+
+		if (jiffies_to_msecs(jiffies - issue_time) >
+		    (rcfw->max_timeout * 1000)) {
+			dev_warn_ratelimited(&rcfw->pdev->dev,
+				"Self Polling QPLIB: cmdq[%#x]=%#x taken (%lu) msec",
+				cookie, crsqe->opcode,
+				(long)jiffies_to_msecs(jiffies - issue_time));
+			ret = bnxt_re_is_fw_stalled(rcfw, cookie);
+			if (ret)
+				return ret;
+		}
+	} while (true);
+
+};
+
+static int __send_message_basic_sanity(struct bnxt_qplib_rcfw *rcfw,
+			  struct bnxt_qplib_cmdqmsg *msg, u8 opcode)
+{
+	struct bnxt_qplib_cmdq_ctx *cmdq;
+
+	cmdq = &rcfw->cmdq;
+
+	/* Prevent posting if f/w is not in a state to process */
+	if (RCFW_NO_FW_ACCESS(rcfw))
+		return -ENXIO;
+
+	if (test_bit(FIRMWARE_STALL_DETECTED, &cmdq->flags))
+		return -ETIMEDOUT;
+
+	if (test_bit(FIRMWARE_INITIALIZED_FLAG, &cmdq->flags) &&
+	    opcode == CMDQ_BASE_OPCODE_INITIALIZE_FW) {
+		dev_err(&rcfw->pdev->dev, "QPLIB: RCFW already initialized!\n");
+		return -EINVAL;
+	}
+
+	if (!test_bit(FIRMWARE_INITIALIZED_FLAG, &cmdq->flags) &&
+	    (opcode != CMDQ_BASE_OPCODE_QUERY_FUNC &&
+	     opcode != CMDQ_BASE_OPCODE_INITIALIZE_FW &&
+	     opcode != CMDQ_BASE_OPCODE_QUERY_VERSION)) {
+		dev_err(&rcfw->pdev->dev,
+			"QPLIB: RCFW not initialized, reject opcode 0x%x\n",
+			opcode);
+		return -ENOTSUPP;
+	}
+
+	return 0;
+}
+
+/* This function will just post and do not bother about completion */
+static  void __destroy_timedout_ah(struct bnxt_qplib_rcfw *rcfw,
+			  struct creq_create_ah_resp *create_ah_resp)
+{
+	struct bnxt_qplib_cmdqmsg msg = {};
+	struct cmdq_destroy_ah req = {};
+
+	bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_DESTROY_AH,
+				 sizeof(req));
+	req.ah_cid = create_ah_resp->xid;
+	msg.req = (struct cmdq_base *)&req;
+	msg.req_sz = sizeof(req);
+	__send_message_no_waiter(rcfw, &msg);
+	dev_warn_ratelimited(&rcfw->pdev->dev,
+		"From %s: ah_cid = %d timeout_send %d\n", __func__,
+		req.ah_cid,
+		atomic_read(&rcfw->timeout_send));
+}
+
+/**
+ * __bnxt_qplib_rcfw_send_message   -	qplib interface to send
+ * and complete rcfw command.
+ * @rcfw      -   rcfw channel instance of rdev
+ * @msg      -    qplib message internal
+ *
+ * This function does not account shadow queue depth. It will send
+ * all the command unconditionally as long as send queue is not full.
+ *
+ * Returns:
+ * 0 if command completed by firmware.
+ * Non zero if the command is not completed by firmware.
+ */
+static int __bnxt_qplib_rcfw_send_message(struct bnxt_qplib_rcfw *rcfw,
+				   struct bnxt_qplib_cmdqmsg *msg)
+{
+	struct bnxt_qplib_crsqe *crsqe;
+	struct creq_qp_event *event;
+	unsigned long flags;
+	u16 cookie;
+	int rc = 0;
+	u8 opcode;
+
+	opcode = __get_cmdq_base_opcode(msg->req, msg->req_sz);
+
+	rc = __send_message_basic_sanity(rcfw, msg, opcode);
+	if (rc)
+		return rc == -ENXIO ? bnxt_qplib_map_rc(opcode) : rc;
+
+	rc = __send_message(rcfw, msg);
+	if (rc)
+		return rc;
+
+	cookie = le16_to_cpu(__get_cmdq_base_cookie(msg->req,
+				msg->req_sz)) & RCFW_MAX_COOKIE_VALUE;
+
+
+	if (msg->block)
+		rc = __block_for_resp(rcfw, cookie);
+	else if (atomic_read(&rcfw->rcfw_intr_enabled))
+		rc = __wait_for_resp(rcfw, cookie);
+	else
+		rc = __poll_for_resp(rcfw, cookie);
+
+	if (rc) {
+		/* First check if it is FW stall.
+		 * Use hwq.lock to avoid race with actual completion.
+		 */
+		spin_lock_irqsave(&rcfw->cmdq.hwq.lock, flags);
+		crsqe = &rcfw->crsqe_tbl[cookie];
+		crsqe->is_waiter_alive = false;
+		if (rc == -ENODEV)
+			set_bit(FIRMWARE_STALL_DETECTED, &rcfw->cmdq.flags);
+		spin_unlock_irqrestore(&rcfw->cmdq.hwq.lock, flags);
+
+		return -ETIMEDOUT;
+	}
+
+	event = (struct creq_qp_event *)msg->resp;
+	if (event->status) {
+		/* failed with status */
+		dev_err(&rcfw->pdev->dev, "QPLIB: cmdq[%#x]=%#x (%s) status %d\n",
+			cookie, opcode, GET_OPCODE_TYPE(opcode), event->status);
+		rc = -EFAULT;
+		/*
+		 * Workaround to avoid errors in the stack during bond
+		 * creation and deletion.
+		 * Disable error returned for  ADD_GID/DEL_GID
+		 */
+		if (opcode == CMDQ_BASE_OPCODE_ADD_GID ||
+		    opcode == CMDQ_BASE_OPCODE_DELETE_GID)
+			rc = 0;
+	}
+
+	dev_dbg(&pdev->dev, "QPLIB: %s:%d - op 0x%x (%s), cookie 0x%x -- Return: e->status 0x%x, rc = 0x%x\n",
+		__func__, __LINE__, opcode, GET_OPCODE_TYPE(opcode), cookie, event->status, rc);
+	return rc;
+}
+
+/**
+ * bnxt_qplib_rcfw_send_message   -	qplib interface to send
+ * and complete rcfw command.
+ * @rcfw      -   rcfw channel instance of rdev
+ * @msg      -    qplib message internal
+ *
+ * Driver interact with Firmware through rcfw channel/slow path in two ways.
+ * a. Blocking rcfw command send. In this path, driver cannot hold
+ * the context for longer period since it is holding cpu until
+ * command is not completed.
+ * b. Non-blocking rcfw command send. In this path, driver can hold the
+ * context for longer period. There may be many pending command waiting
+ * for completion because of non-blocking nature.
+ *
+ * Driver will use shadow queue depth. Current queue depth of 8K
+ * (due to size of rcfw message it can be actual ~4K rcfw outstanding)
+ * is not optimal for rcfw command processing in firmware.
+ * RCFW_CMD_NON_BLOCKING_SHADOW_QD is defined as 64.
+ * Restrict at max 64 Non-Blocking rcfw commands.
+ * Do not allow more than 64 non-blocking command to the Firmware.
+ * Allow all blocking commands until there is no queue full.
+ *
+ * Returns:
+ * 0 if command completed by firmware.
+ * Non zero if the command is not completed by firmware.
+ */
+int bnxt_qplib_rcfw_send_message(struct bnxt_qplib_rcfw *rcfw,
+				 struct bnxt_qplib_cmdqmsg *msg)
+{
+	int ret;
+
+	if (!msg->block) {
+		down(&rcfw->rcfw_inflight);
+		ret = __bnxt_qplib_rcfw_send_message(rcfw, msg);
+		up(&rcfw->rcfw_inflight);
+	} else {
+		ret = __bnxt_qplib_rcfw_send_message(rcfw, msg);
+	}
+
+	return ret;
+}
+
+static void bnxt_re_add_perf_stats(struct bnxt_qplib_rcfw *rcfw,
+		struct bnxt_qplib_crsqe *crsqe)
+{
+	u32 latency_msec, dest_stats_id;
+	u64 *dest_stats_ptr = NULL;
+
+	latency_msec = jiffies_to_msecs(rcfw->cmdq.last_seen -
+				crsqe->send_timestamp);
+	if (latency_msec/1000 < RCFW_MAX_LATENCY_SEC_SLAB_INDEX)
+		rcfw->rcfw_lat_slab_sec[latency_msec/1000]++;
+
+	if (!rcfw->sp_perf_stats_enabled)
+		return;
+
+	if (latency_msec < RCFW_MAX_LATENCY_MSEC_SLAB_INDEX)
+		rcfw->rcfw_lat_slab_msec[latency_msec]++;
+
+	switch (crsqe->opcode) {
+	case CMDQ_BASE_OPCODE_CREATE_QP:
+		dest_stats_id = rcfw->qp_create_stats_id++;
+		dest_stats_id = dest_stats_id % RCFW_MAX_STAT_INDEX;
+		dest_stats_ptr = &rcfw->qp_create_stats[dest_stats_id];
+		break;
+	case CMDQ_BASE_OPCODE_DESTROY_QP:
+		dest_stats_id = rcfw->qp_destroy_stats_id++;
+		dest_stats_id = dest_stats_id % RCFW_MAX_STAT_INDEX;
+		dest_stats_ptr = &rcfw->qp_destroy_stats[dest_stats_id];
+		break;
+	case CMDQ_BASE_OPCODE_REGISTER_MR:
+		dest_stats_id = rcfw->mr_create_stats_id++;
+		dest_stats_id = dest_stats_id % RCFW_MAX_STAT_INDEX;
+		dest_stats_ptr = &rcfw->mr_create_stats[dest_stats_id];
+		break;
+	case CMDQ_BASE_OPCODE_DEREGISTER_MR:
+	case CMDQ_BASE_OPCODE_DEALLOCATE_KEY:
+		dest_stats_id = rcfw->mr_destroy_stats_id++;
+		dest_stats_id = dest_stats_id % RCFW_MAX_STAT_INDEX;
+		dest_stats_ptr = &rcfw->mr_destroy_stats[dest_stats_id];
+		break;
+	case CMDQ_BASE_OPCODE_MODIFY_QP:
+		if (crsqe->requested_qp_state != IB_QPS_ERR)
+			break;
+		dest_stats_id = rcfw->qp_modify_stats_id++;
+		dest_stats_id = dest_stats_id % RCFW_MAX_STAT_INDEX;
+		dest_stats_ptr = &rcfw->qp_modify_stats[dest_stats_id];
+		break;
+	default:
+		break;
+	}
+	if (dest_stats_ptr)
+		*dest_stats_ptr = max_t(unsigned long,
+				(rcfw->cmdq.last_seen - crsqe->send_timestamp), 1);
+
+}
+
+/* Completions */
+static int bnxt_qplib_process_qp_event(struct bnxt_qplib_rcfw *rcfw,
+				       struct creq_qp_event *event,
+				       u32 *num_wait)
+{
+	struct bnxt_qplib_hwq *cmdq_hwq = &rcfw->cmdq.hwq;
+	struct creq_cq_error_notification *cqerr;
+	struct creq_qp_error_notification *qperr;
+	struct bnxt_qplib_crsqe *crsqe;
+	struct bnxt_qplib_reftbl *tbl;
+	struct bnxt_qplib_qp *qp;
+	struct bnxt_qplib_cq *cq;
+	u16 cookie, blocked = 0;
+	struct pci_dev *pdev;
+	bool is_waiter_alive;
+	unsigned long flags;
+	u32 wait_cmds = 0;
+	u32 xid, qp_idx;
+	u32 req_size;
+	int rc = 0;
+
+	pdev = rcfw->pdev;
+	switch (event->event) {
+	case CREQ_QP_EVENT_EVENT_QP_ERROR_NOTIFICATION:
+		tbl = &rcfw->res->reftbl.qpref;
+		qperr = (struct creq_qp_error_notification *)event;
+		xid = le32_to_cpu(qperr->xid);
+		qp_idx = map_qp_id_to_tbl_indx(xid, tbl);
+		spin_lock(&tbl->lock);
+		qp = tbl->rec[qp_idx].handle;
+		if (!qp) {
+			spin_unlock(&tbl->lock);
+			break;
+		}
+		bnxt_qplib_mark_qp_error(qp);
+		rc = rcfw->creq.aeq_handler(rcfw, event, qp);
+		spin_unlock(&tbl->lock);
+		/*
+		 * Keeping these prints as debug to avoid flooding of log
+		 * messages during modify QP to error state by applications
+		 */
+		dev_dbg(&pdev->dev, "QPLIB: QP Error encountered!\n");
+		dev_dbg(&pdev->dev,
+			"QPLIB: qpid 0x%x, req_err=0x%x, resp_err=0x%x\n",
+			xid, qperr->req_err_state_reason,
+			qperr->res_err_state_reason);
+		break;
+	case CREQ_QP_EVENT_EVENT_CQ_ERROR_NOTIFICATION:
+		tbl = &rcfw->res->reftbl.cqref;
+		cqerr = (struct creq_cq_error_notification *)event;
+		xid = le32_to_cpu(cqerr->xid);
+		spin_lock(&tbl->lock);
+		cq = tbl->rec[GET_TBL_INDEX(xid, tbl)].handle;
+		if (!cq) {
+			spin_unlock(&tbl->lock);
+			break;
+		}
+		rc = rcfw->creq.aeq_handler(rcfw, event, cq);
+		spin_unlock(&tbl->lock);
+		dev_dbg(&pdev->dev, "QPLIB: CQ error encountered!\n");
+		break;
+	default:
+		/*
+		 * Command Response
+		 * cmdq hwq lock needs to be acquired to synchronize
+		 * the command send and completion reaping. This function
+		 * is always called with creq hwq lock held. So there is no
+		 * chance of deadlock here as the locking is in correct sequence.
+		 * Using  the nested variant of spin_lock to annotate
+		 */
+		spin_lock_irqsave_nested(&cmdq_hwq->lock, flags,
+					 SINGLE_DEPTH_NESTING);
+		cookie = le16_to_cpu(event->cookie);
+		blocked = cookie & RCFW_CMD_IS_BLOCKING;
+		cookie &= RCFW_MAX_COOKIE_VALUE;
+
+		crsqe = &rcfw->crsqe_tbl[cookie];
+
+		bnxt_re_add_perf_stats(rcfw, crsqe);
+
+		if (WARN_ONCE(test_bit(FIRMWARE_STALL_DETECTED,
+				       &rcfw->cmdq.flags),
+		    "QPLIB: Unreponsive rcfw channel detected.!!")) {
+			dev_info(&pdev->dev, "rcfw timedout: cookie = %#x,"
+				" latency_msec = %ld free_slots = %d\n", cookie,
+				(long)jiffies_to_msecs(rcfw->cmdq.last_seen -
+						 crsqe->send_timestamp),
+				crsqe->free_slots);
+			spin_unlock_irqrestore(&cmdq_hwq->lock, flags);
+			return rc;
+		}
+
+		if (crsqe->is_internal_cmd && !event->status)
+			atomic_dec(&rcfw->timeout_send);
+
+		if (crsqe->is_waiter_alive) {
+			if (crsqe->resp)
+				memcpy(crsqe->resp, event, sizeof(*event));
+			if (!blocked)
+				wait_cmds++;
+		}
+
+		req_size = crsqe->req_size;
+		is_waiter_alive = crsqe->is_waiter_alive;
+
+		crsqe->req_size = 0;
+		if (!crsqe->is_waiter_alive)
+			crsqe->resp = NULL;
+		crsqe->is_in_used = false;
+		/* Consumer is updated so that __send_message_no_waiter
+		 * can never see queue full.
+		 * It is safe since we are still holding cmdq_hwq->lock.
+		 */
+		cmdq_hwq->cons += req_size;
+
+		/* This is a case to handle below scenario -
+		 * Create AH is completed successfully by firmware,
+		 * but completion took more time and driver already lost
+		 * the context of create_ah from caller.
+		 * We have already return failure for create_ah verbs,
+		 * so let's destroy the same address vector since it is
+		 * no more used in stack. We don't care about completion
+		 * in __send_message_no_waiter.
+		 * If destroy_ah is failued by firmware, there will be AH
+		 * resource leak and relatively not critical +  unlikely
+		 * scenario. Current design is not to handle such case.
+		 */
+		if (!is_waiter_alive && !event->status &&
+		    event->event == CREQ_QP_EVENT_EVENT_CREATE_AH)
+			__destroy_timedout_ah(rcfw,
+					      (struct creq_create_ah_resp *)
+					      event);
+
+		spin_unlock_irqrestore(&cmdq_hwq->lock, flags);
+	}
+	*num_wait += wait_cmds;
+	return rc;
+}
+
+/* SP - CREQ Completion handlers */
+static void bnxt_qplib_service_creq(unsigned long data)
+{
+	struct bnxt_qplib_rcfw *rcfw = (struct bnxt_qplib_rcfw *)data;
+	struct bnxt_qplib_creq_ctx *creq = &rcfw->creq;
+	struct bnxt_qplib_res *res;
+	u32 type, budget = CREQ_ENTRY_POLL_BUDGET;
+	struct bnxt_qplib_hwq *creq_hwq = &creq->hwq;
+	struct creq_base *creqe;
+	struct pci_dev *pdev;
+	unsigned long flags;
+	u32 num_wakeup = 0;
+	int rc;
+
+	pdev = rcfw->pdev;
+	res = rcfw->res;
+	/* Service the CREQ until empty */
+	spin_lock_irqsave(&creq_hwq->lock, flags);
+	while (budget > 0) {
+		if (RCFW_NO_FW_ACCESS(rcfw)) {
+			spin_unlock_irqrestore(&creq_hwq->lock, flags);
+			return;
+		}
+		creqe = bnxt_qplib_get_qe(creq_hwq, creq_hwq->cons, NULL);
+		if (!CREQ_CMP_VALID(creqe, creq->creq_db.dbinfo.flags))
+			break;
+		/* The valid test of the entry must be done first before
+		 * reading any further.
+		 */
+		dma_rmb();
+		type = creqe->type & CREQ_BASE_TYPE_MASK;
+		rcfw->cmdq.last_seen = jiffies;
+
+		switch (type) {
+		case CREQ_BASE_TYPE_QP_EVENT:
+			bnxt_qplib_process_qp_event
+				(rcfw,(struct creq_qp_event *)creqe,
+				 &num_wakeup);
+			creq->stats.creq_qp_event_processed++;
+			break;
+		case CREQ_BASE_TYPE_FUNC_EVENT:
+			rc = rcfw->creq.aeq_handler(rcfw, creqe, NULL);
+			if (rc)
+				dev_warn(&pdev->dev,
+					 "QPLIB: async event type = 0x%x not handled",
+					 type);
+			creq->stats.creq_func_event_processed++;
+			break;
+		default:
+			if (type != HWRM_ASYNC_EVENT_CMPL_TYPE_HWRM_ASYNC_EVENT) {
+				dev_warn(&pdev->dev,
+					 "QPLIB: op_event = 0x%x not handled\n",
+					 type);
+			}
+			break;
+		}
+		budget--;
+		bnxt_qplib_hwq_incr_cons(creq_hwq->max_elements, &creq_hwq->cons,
+					 1, &creq->creq_db.dbinfo.flags);
+	}
+	if (budget == CREQ_ENTRY_POLL_BUDGET &&
+	    !CREQ_CMP_VALID(creqe, creq->creq_db.dbinfo.flags)) {
+		/* No completions received during this poll. Enable interrupt now */
+		bnxt_qplib_ring_nq_db(&creq->creq_db.dbinfo, res->cctx, true);
+		creq->stats.creq_arm_count++;
+		dev_dbg(&pdev->dev, "QPLIB: Num of Func (0x%llx) \n",
+			creq->stats.creq_func_event_processed);
+		dev_dbg(&pdev->dev, "QPLIB: QP (0x%llx) events processed\n",
+			creq->stats.creq_qp_event_processed);
+		dev_dbg(&pdev->dev, "QPLIB: Armed:%#llx resched:%#llx \n",
+			creq->stats.creq_arm_count,
+			creq->stats.creq_tasklet_schedule_count);
+	} else if (creq->requested) {
+		/*
+		 * Currently there is no bottom half implementation to process
+		 * completions, all completions are processed in interrupt context
+		 * only. So enable interrupts.
+		 */
+		bnxt_qplib_ring_nq_db(&creq->creq_db.dbinfo, res->cctx, true);
+		creq->stats.creq_tasklet_schedule_count++;
+	}
+	spin_unlock_irqrestore(&creq_hwq->lock, flags);
+	if (num_wakeup)
+		wake_up_all(&rcfw->cmdq.waitq);
+}
+
+static irqreturn_t bnxt_qplib_creq_irq(int irq, void *dev_instance)
+{
+	struct bnxt_qplib_rcfw *rcfw = dev_instance;
+
+	bnxt_qplib_service_creq((unsigned long)rcfw);
+	return IRQ_HANDLED;
+}
+
+/* RCFW */
+int bnxt_qplib_deinit_rcfw(struct bnxt_qplib_rcfw *rcfw)
+{
+	struct creq_deinitialize_fw_resp resp = {};
+	struct cmdq_deinitialize_fw req = {};
+	struct bnxt_qplib_cmdqmsg msg = {};
+	int rc;
+
+	bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_DEINITIALIZE_FW,
+				 sizeof(req));
+	bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL,
+				sizeof(req), sizeof(resp), 0);
+	rc = bnxt_qplib_rcfw_send_message(rcfw, &msg);
+	if (rc)
+		return rc;
+	clear_bit(FIRMWARE_INITIALIZED_FLAG, &rcfw->cmdq.flags);
+	return 0;
+}
+
+int bnxt_qplib_init_rcfw(struct bnxt_qplib_rcfw *rcfw, int is_virtfn)
+{
+	struct creq_initialize_fw_resp resp = {};
+	struct cmdq_initialize_fw req = {};
+	struct bnxt_qplib_cmdqmsg msg = {};
+	struct bnxt_qplib_chip_ctx *cctx;
+	struct bnxt_qplib_ctx *hctx;
+	struct bnxt_qplib_res *res;
+	struct bnxt_qplib_hwq *hwq;
+	int rc;
+
+	res = rcfw->res;
+	cctx = res->cctx;
+	hctx = res->hctx;
+
+	bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_INITIALIZE_FW,
+				 sizeof(req));
+	/* Supply (log-base-2-of-host-page-size - base-page-shift)
+	 * to bono to adjust the doorbell page sizes.
+	 */
+	req.log2_dbr_pg_size = cpu_to_le16(PAGE_SHIFT -
+					   RCFW_DBR_BASE_PAGE_SHIFT);
+	/*
+	 * VFs need not setup the HW context area, PF
+	 * shall setup this area for VF. Skipping the
+	 * HW programming
+	 */
+	if (is_virtfn || _is_chip_gen_p5_p7(cctx))
+		goto skip_ctx_setup;
+
+	hwq = &hctx->qp_ctx.hwq;
+	req.qpc_page_dir = cpu_to_le64(_get_base_addr(hwq));
+	req.number_of_qp = cpu_to_le32(hwq->max_elements);
+	req.qpc_pg_size_qpc_lvl = (_get_pte_pg_size(hwq) <<
+				   CMDQ_INITIALIZE_FW_QPC_PG_SIZE_SFT) |
+				   (u8)hwq->level;
+
+	hwq = &hctx->mrw_ctx.hwq;
+	req.mrw_page_dir = cpu_to_le64(_get_base_addr(hwq));
+	req.number_of_mrw = cpu_to_le32(hwq->max_elements);
+	req.mrw_pg_size_mrw_lvl = (_get_pte_pg_size(hwq) <<
+				   CMDQ_INITIALIZE_FW_MRW_PG_SIZE_SFT) |
+				   (u8)hwq->level;
+
+	hwq = &hctx->srq_ctx.hwq;
+	req.srq_page_dir = cpu_to_le64(_get_base_addr(hwq));
+	req.number_of_srq = cpu_to_le32(hwq->max_elements);
+	req.srq_pg_size_srq_lvl = (_get_pte_pg_size(hwq) <<
+				   CMDQ_INITIALIZE_FW_SRQ_PG_SIZE_SFT) |
+				   (u8)hwq->level;
+
+	hwq = &hctx->cq_ctx.hwq;
+	req.cq_page_dir = cpu_to_le64(_get_base_addr(hwq));
+	req.number_of_cq = cpu_to_le32(hwq->max_elements);
+	req.cq_pg_size_cq_lvl = (_get_pte_pg_size(hwq) <<
+				 CMDQ_INITIALIZE_FW_CQ_PG_SIZE_SFT) |
+				 (u8)hwq->level;
+
+	hwq = &hctx->tim_ctx.hwq;
+	req.tim_page_dir = cpu_to_le64(_get_base_addr(hwq));
+	req.tim_pg_size_tim_lvl = (_get_pte_pg_size(hwq) <<
+				   CMDQ_INITIALIZE_FW_TIM_PG_SIZE_SFT) |
+				   (u8)hwq->level;
+	hwq = &hctx->tqm_ctx.pde;
+	req.tqm_page_dir = cpu_to_le64(_get_base_addr(hwq));
+	req.tqm_pg_size_tqm_lvl = (_get_pte_pg_size(hwq) <<
+				   CMDQ_INITIALIZE_FW_TQM_PG_SIZE_SFT) |
+				   (u8)hwq->level;
+skip_ctx_setup:
+	if (BNXT_RE_HW_RETX(res->dattr->dev_cap_flags))
+		req.flags |= CMDQ_INITIALIZE_FW_FLAGS_HW_REQUESTER_RETX_SUPPORTED;
+	req.stat_ctx_id = cpu_to_le32(hctx->stats.fw_id);
+	bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL,
+				sizeof(req), sizeof(resp), 0);
+	rc = bnxt_qplib_rcfw_send_message(rcfw, &msg);
+	if (rc)
+		return rc;
+	set_bit(FIRMWARE_INITIALIZED_FLAG, &rcfw->cmdq.flags);
+
+	return 0;
+}
+
+void bnxt_qplib_free_rcfw_channel(struct bnxt_qplib_res *res)
+{
+	struct bnxt_qplib_rcfw *rcfw = res->rcfw;
+
+	vfree(rcfw->rcfw_lat_slab_msec);
+	rcfw->rcfw_lat_slab_msec = NULL;
+	vfree(rcfw->qp_create_stats);
+	rcfw->qp_create_stats = NULL;
+	vfree(rcfw->qp_destroy_stats);
+	rcfw->qp_destroy_stats = NULL;
+	vfree(rcfw->mr_create_stats);
+	rcfw->mr_create_stats = NULL;
+	vfree(rcfw->mr_destroy_stats);
+	rcfw->mr_destroy_stats = NULL;
+	vfree(rcfw->qp_modify_stats);
+	rcfw->qp_modify_stats = NULL;
+	rcfw->sp_perf_stats_enabled = false;
+
+	kfree(rcfw->crsqe_tbl);
+	rcfw->crsqe_tbl = NULL;
+
+	bnxt_qplib_free_hwq(res, &rcfw->cmdq.hwq);
+	bnxt_qplib_free_hwq(res, &rcfw->creq.hwq);
+	rcfw->pdev = NULL;
+}
+
+int bnxt_qplib_alloc_rcfw_channel(struct bnxt_qplib_res *res)
+{
+	struct bnxt_qplib_hwq_attr hwq_attr = {};
+	struct bnxt_qplib_rcfw *rcfw = res->rcfw;
+	struct bnxt_qplib_sg_info sginfo = {};
+	struct bnxt_qplib_cmdq_ctx *cmdq;
+	struct bnxt_qplib_creq_ctx *creq;
+
+	rcfw->pdev = res->pdev;
+	rcfw->res = res;
+	cmdq = &rcfw->cmdq;
+	creq = &rcfw->creq;
+
+	sginfo.pgsize = PAGE_SIZE;
+	sginfo.pgshft = PAGE_SHIFT;
+
+	hwq_attr.sginfo = &sginfo;
+	hwq_attr.res = rcfw->res;
+	hwq_attr.depth = BNXT_QPLIB_CREQE_MAX_CNT;
+	hwq_attr.stride = BNXT_QPLIB_CREQE_UNITS;
+	hwq_attr.type = _get_hwq_type(res);
+
+	if (bnxt_qplib_alloc_init_hwq(&creq->hwq, &hwq_attr)) {
+		dev_err(&rcfw->pdev->dev,
+			"QPLIB: HW channel CREQ allocation failed\n");
+		return -ENOMEM;
+	}
+
+	sginfo.pgsize = BNXT_QPLIB_CMDQE_PAGE_SIZE;
+	hwq_attr.depth = BNXT_QPLIB_CMDQE_MAX_CNT & 0x7FFFFFFF;
+	hwq_attr.stride = BNXT_QPLIB_CMDQE_UNITS;
+	hwq_attr.type = HWQ_TYPE_CTX;
+	if (bnxt_qplib_alloc_init_hwq(&cmdq->hwq, &hwq_attr)) {
+		dev_err(&rcfw->pdev->dev,
+			"QPLIB: HW channel CMDQ allocation failed\n");
+		goto fail_free_creq_hwq;
+	}
+
+	rcfw->crsqe_tbl = kcalloc(cmdq->hwq.max_elements,
+			sizeof(*rcfw->crsqe_tbl), GFP_KERNEL);
+	if (!rcfw->crsqe_tbl) {
+		dev_err(&rcfw->pdev->dev,
+			"QPLIB: HW channel CRSQ allocation failed\n");
+		goto fail_free_cmdq_hwq;
+	}
+
+	rcfw->max_timeout = res->cctx->hwrm_cmd_max_timeout;
+
+	rcfw->sp_perf_stats_enabled = false;
+	rcfw->rcfw_lat_slab_msec = vzalloc(sizeof(u32) *
+					   RCFW_MAX_LATENCY_MSEC_SLAB_INDEX);
+	rcfw->qp_create_stats = vzalloc(sizeof(u64) * RCFW_MAX_STAT_INDEX);
+	rcfw->qp_destroy_stats = vzalloc(sizeof(u64) * RCFW_MAX_STAT_INDEX);
+	rcfw->mr_create_stats = vzalloc(sizeof(u64) * RCFW_MAX_STAT_INDEX);
+	rcfw->mr_destroy_stats = vzalloc(sizeof(u64) * RCFW_MAX_STAT_INDEX);
+	rcfw->qp_modify_stats = vzalloc(sizeof(u64) * RCFW_MAX_STAT_INDEX);
+
+	if (rcfw->rcfw_lat_slab_msec &&
+	    rcfw->qp_create_stats &&
+	    rcfw->qp_destroy_stats &&
+	    rcfw->mr_create_stats &&
+	    rcfw->mr_destroy_stats &&
+	    rcfw->qp_modify_stats)
+		rcfw->sp_perf_stats_enabled = true;
+
+	return 0;
+fail_free_cmdq_hwq:
+	bnxt_qplib_free_hwq(res, &rcfw->cmdq.hwq);
+fail_free_creq_hwq:
+	bnxt_qplib_free_hwq(res, &rcfw->creq.hwq);
+	return -ENOMEM;
+}
+
+void bnxt_qplib_rcfw_stop_irq(struct bnxt_qplib_rcfw *rcfw, bool kill)
+{
+	struct bnxt_qplib_creq_ctx *creq;
+	struct bnxt_qplib_res *res;
+
+	creq = &rcfw->creq;
+	res = rcfw->res;
+
+	if (!creq->requested)
+		return;
+
+	creq->requested = false;
+	/* Mask h/w interrupts */
+	bnxt_qplib_ring_nq_db(&creq->creq_db.dbinfo, res->cctx, false);
+	/* Sync with last running IRQ-handler */
+	synchronize_irq(creq->msix_vec);
+	free_irq(creq->msix_vec, rcfw);
+	kfree(creq->irq_name);
+	creq->irq_name = NULL;
+	/* rcfw_intr_enabled should not be greater than 1. Debug
+	 * print to check if that is the case
+	 */
+	if (atomic_read(&rcfw->rcfw_intr_enabled) > 1) {
+		dev_err(&rcfw->pdev->dev,
+			"%s: rcfw->rcfw_intr_enabled = 0x%x\n", __func__,
+			atomic_read(&rcfw->rcfw_intr_enabled));
+	}
+	atomic_set(&rcfw->rcfw_intr_enabled, 0);
+	rcfw->num_irq_stopped++;
+}
+
+void bnxt_qplib_disable_rcfw_channel(struct bnxt_qplib_rcfw *rcfw)
+{
+	struct bnxt_qplib_creq_ctx *creq;
+	struct bnxt_qplib_cmdq_ctx *cmdq;
+
+	creq = &rcfw->creq;
+	cmdq = &rcfw->cmdq;
+	/* Make sure the HW channel is stopped! */
+	bnxt_qplib_rcfw_stop_irq(rcfw, true);
+
+	creq->creq_db.reg.bar_reg = NULL;
+	creq->creq_db.db = NULL;
+
+	if (cmdq->cmdq_mbox.reg.bar_reg) {
+		iounmap(cmdq->cmdq_mbox.reg.bar_reg);
+		cmdq->cmdq_mbox.reg.bar_reg = NULL;
+		cmdq->cmdq_mbox.prod = NULL;
+		cmdq->cmdq_mbox.db = NULL;
+	}
+
+	creq->aeq_handler = NULL;
+	creq->msix_vec = 0;
+}
+
+int bnxt_qplib_rcfw_start_irq(struct bnxt_qplib_rcfw *rcfw, int msix_vector,
+			      bool need_init)
+{
+	struct bnxt_qplib_creq_ctx *creq;
+	struct bnxt_qplib_res *res;
+	int rc;
+
+	creq = &rcfw->creq;
+	res = rcfw->res;
+
+	if (creq->requested)
+		return -EFAULT;
+
+	creq->msix_vec = msix_vector;
+
+	creq->irq_name = kasprintf(GFP_KERNEL, "bnxt_re-creq@pci:%s\n",
+				   pci_name(res->pdev));
+	if (!creq->irq_name)
+		return -ENOMEM;
+
+	rc = request_irq(creq->msix_vec, bnxt_qplib_creq_irq, 0,
+			 creq->irq_name, rcfw);
+	if (rc) {
+		kfree(creq->irq_name);
+		creq->irq_name = NULL;
+		return rc;
+	}
+	creq->requested = true;
+
+	bnxt_qplib_ring_nq_db(&creq->creq_db.dbinfo, res->cctx, true);
+
+	rcfw->num_irq_started++;
+	/* Debug print to check rcfw interrupt enable/disable is invoked
+	 * out of sequence
+	 */
+	if (atomic_read(&rcfw->rcfw_intr_enabled) > 0) {
+		dev_err(&rcfw->pdev->dev,
+			"%s: rcfw->rcfw_intr_enabled = 0x%x\n", __func__,
+			atomic_read(&rcfw->rcfw_intr_enabled));
+	}
+	atomic_inc(&rcfw->rcfw_intr_enabled);
+	return 0;
+}
+
+static int bnxt_qplib_map_cmdq_mbox(struct bnxt_qplib_rcfw *rcfw)
+{
+	struct bnxt_qplib_cmdq_mbox *mbox;
+	resource_size_t bar_reg;
+	struct pci_dev *pdev;
+
+	pdev = rcfw->pdev;
+	mbox = &rcfw->cmdq.cmdq_mbox;
+
+	mbox->reg.bar_id = RCFW_COMM_PCI_BAR_REGION;
+	mbox->reg.len = RCFW_COMM_SIZE;
+	mbox->reg.bar_base = pci_resource_start(pdev, mbox->reg.bar_id);
+	if (!mbox->reg.bar_base) {
+		dev_err(&pdev->dev,
+			"QPLIB: CMDQ BAR region %d resc start is 0!\n",
+			mbox->reg.bar_id);
+		return -ENOMEM;
+	}
+
+	bar_reg = mbox->reg.bar_base + RCFW_COMM_BASE_OFFSET;
+	mbox->reg.len = RCFW_COMM_SIZE;
+	mbox->reg.bar_reg = ioremap(bar_reg, mbox->reg.len);
+	if (!mbox->reg.bar_reg) {
+		dev_err(&pdev->dev,
+			"QPLIB: CMDQ BAR region %d mapping failed\n",
+			mbox->reg.bar_id);
+		return -ENOMEM;
+	}
+
+	mbox->prod = (void  __iomem *)((char *)mbox->reg.bar_reg +
+					RCFW_PF_VF_COMM_PROD_OFFSET);
+	mbox->db = (void __iomem *)((char *)mbox->reg.bar_reg +
+				     RCFW_COMM_TRIG_OFFSET);
+	return 0;
+}
+
+static int bnxt_qplib_map_creq_db(struct bnxt_qplib_rcfw *rcfw, u32 reg_offt)
+{
+	struct bnxt_qplib_creq_db *creq_db;
+	struct bnxt_qplib_reg_desc *dbreg;
+	struct bnxt_qplib_res *res;
+
+	res = rcfw->res;
+	creq_db = &rcfw->creq.creq_db;
+	dbreg = &res->dpi_tbl.ucreg;
+
+	creq_db->reg.bar_id = dbreg->bar_id;
+	creq_db->reg.bar_base = dbreg->bar_base;
+	creq_db->reg.bar_reg = dbreg->bar_reg + reg_offt;
+	creq_db->reg.len = _is_chip_gen_p5_p7(res->cctx) ? sizeof(u64) :
+							sizeof(u32);
+
+	creq_db->dbinfo.db = creq_db->reg.bar_reg;
+	creq_db->dbinfo.hwq = &rcfw->creq.hwq;
+	creq_db->dbinfo.xid = rcfw->creq.ring_id;
+	creq_db->dbinfo.seed = rcfw->creq.ring_id;
+	creq_db->dbinfo.flags = 0;
+	spin_lock_init(&creq_db->dbinfo.lock);
+	creq_db->dbinfo.shadow_key = BNXT_QPLIB_DBR_KEY_INVALID;
+	creq_db->dbinfo.res = rcfw->res;
+
+	return 0;
+}
+
+static void bnxt_qplib_start_rcfw(struct bnxt_qplib_rcfw *rcfw)
+{
+	struct bnxt_qplib_cmdq_ctx *cmdq;
+	struct bnxt_qplib_creq_ctx *creq;
+	struct bnxt_qplib_cmdq_mbox *mbox;
+	struct cmdq_init init = {0};
+
+	cmdq = &rcfw->cmdq;
+	creq = &rcfw->creq;
+	mbox = &cmdq->cmdq_mbox;
+
+	init.cmdq_pbl = cpu_to_le64(cmdq->hwq.pbl[PBL_LVL_0].pg_map_arr[0]);
+	init.cmdq_size_cmdq_lvl = cpu_to_le16(
+			((BNXT_QPLIB_CMDQE_MAX_CNT << CMDQ_INIT_CMDQ_SIZE_SFT) &
+			 CMDQ_INIT_CMDQ_SIZE_MASK) |
+			((cmdq->hwq.level << CMDQ_INIT_CMDQ_LVL_SFT) &
+			 CMDQ_INIT_CMDQ_LVL_MASK));
+	init.creq_ring_id = cpu_to_le16(creq->ring_id);
+	/* Write to the Bono mailbox register */
+	__iowrite32_copy(mbox->reg.bar_reg, &init, sizeof(init) / 4);
+}
+
+int bnxt_qplib_enable_rcfw_channel(struct bnxt_qplib_rcfw *rcfw,
+				   int msix_vector,
+				   int cp_bar_reg_off,
+				   aeq_handler_t aeq_handler)
+{
+	struct bnxt_qplib_cmdq_ctx *cmdq;
+	struct bnxt_qplib_creq_ctx *creq;
+	int rc;
+
+	cmdq = &rcfw->cmdq;
+	creq = &rcfw->creq;
+
+	/* Clear to defaults */
+	cmdq->seq_num = 0;
+	set_bit(FIRMWARE_FIRST_FLAG, &cmdq->flags);
+	init_waitqueue_head(&cmdq->waitq);
+
+	creq->stats.creq_qp_event_processed = 0;
+	creq->stats.creq_func_event_processed = 0;
+	creq->aeq_handler = aeq_handler;
+
+	rc = bnxt_qplib_map_cmdq_mbox(rcfw);
+	if (rc)
+		return rc;
+
+	rc = bnxt_qplib_map_creq_db(rcfw, cp_bar_reg_off);
+	if (rc)
+		return rc;
+
+	rc = bnxt_qplib_rcfw_start_irq(rcfw, msix_vector, true);
+	if (rc) {
+		dev_err(&rcfw->pdev->dev,
+			"QPLIB: Failed to request IRQ for CREQ rc = 0x%x\n", rc);
+		bnxt_qplib_disable_rcfw_channel(rcfw);
+		return rc;
+	}
+
+	rcfw->curr_shadow_qd = min_not_zero(cmdq_shadow_qd,
+					    (unsigned int)RCFW_CMD_NON_BLOCKING_SHADOW_QD);
+	sema_init(&rcfw->rcfw_inflight, rcfw->curr_shadow_qd);
+	dev_dbg(&rcfw->pdev->dev,
+		"Perf Debug: shadow qd %d\n", rcfw->curr_shadow_qd);
+	bnxt_qplib_start_rcfw(rcfw);
+
+	return 0;
+}
diff --git a/sys/dev/bnxt/bnxt_re/qplib_rcfw.h b/sys/dev/bnxt/bnxt_re/qplib_rcfw.h
new file mode 100644
index 000000000000..f117525daacb
--- /dev/null
+++ b/sys/dev/bnxt/bnxt_re/qplib_rcfw.h
@@ -0,0 +1,354 @@
+/*
+ * Copyright (c) 2015-2024, Broadcom. All rights reserved.  The term
+ * Broadcom refers to Broadcom Limited and/or its subsidiaries.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS''
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
+ * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
+ * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * Description: RDMA Controller HW interface (header)
+ */
+
+#ifndef __BNXT_QPLIB_RCFW_H__
+#define __BNXT_QPLIB_RCFW_H__
+
+#include <linux/semaphore.h>
+#include <linux/module.h>
+#include <linux/netdevice.h>
+#include <linux/mutex.h>
+#include <linux/list.h>
+#include <linux/rculist.h>
+#include <linux/spinlock.h>
+#include <linux/pci.h>
+#include <net/ipv6.h>
+#include <linux/if_ether.h>
+#include <linux/debugfs.h>
+#include <linux/seq_file.h>
+#include <linux/interrupt.h>
+#include <linux/vmalloc.h>
+#include <linux/delay.h>
+
+#include "qplib_tlv.h"
+
+#define RCFW_CMDQ_TRIG_VAL		1
+#define RCFW_COMM_PCI_BAR_REGION	0
+#define RCFW_COMM_CONS_PCI_BAR_REGION	2
+#define RCFW_COMM_BASE_OFFSET		0x600
+#define RCFW_PF_VF_COMM_PROD_OFFSET	0xc
+#define RCFW_COMM_TRIG_OFFSET		0x100
+#define RCFW_COMM_SIZE			0x104
+
+#define RCFW_DBR_PCI_BAR_REGION		2
+#define RCFW_DBR_BASE_PAGE_SHIFT	12
+#define RCFW_MAX_LATENCY_SEC_SLAB_INDEX	128
+#define RCFW_MAX_LATENCY_MSEC_SLAB_INDEX	3000
+#define RCFW_MAX_STAT_INDEX	0xFFFF
+#define	RCFW_FW_STALL_MAX_TIMEOUT	40
+
+#define GET_OPCODE_TYPE(x) \
+	((x) == 0x1 ? "CREATE_QP": \
+	((x) == 0x2 ? "DESTROY_QP": \
+	((x) == 0x3 ? "MODIFY_QP": \
+	((x) == 0x4 ? "QUERY_QP": \
+	((x) == 0x5 ? "CREATE_SRQ": \
+	((x) == 0x6 ? "DESTROY_SRQ": \
+	((x) == 0x8 ? "QUERY_SRQ": \
+	((x) == 0x9 ? "CREATE_CQ": \
+	((x) == 0xa ? "DESTROY_CQ": \
+	((x) == 0xc ? "RESIZE_CQ": \
+	((x) == 0xd ? "ALLOCATE_MRW": \
+	((x) == 0xe ? "DEALLOCATE_KEY": \
+	((x) == 0xf ? "REGISTER_MR": \
+	((x) == 0x10 ? "DEREGISTER_MR": \
+	((x) == 0x11 ? "ADD_GID": \
+	((x) == 0x12 ? "DELETE_GID": \
+	((x) == 0x17 ? "MODIFY_GID": \
+	((x) == 0x18 ? "QUERY_GID": \
+	((x) == 0x13 ? "CREATE_QP1": \
+	((x) == 0x14 ? "DESTROY_QP1": \
+	((x) == 0x15 ? "CREATE_AH": \
+	((x) == 0x16 ? "DESTROY_AH": \
+	((x) == 0x80 ? "INITIALIZE_FW": \
+	((x) == 0x81 ? "DEINITIALIZE_FW": \
+	((x) == 0x82 ? "STOP_FUNC": \
+	((x) == 0x83 ? "QUERY_FUNC": \
+	((x) == 0x84 ? "SET_FUNC_RESOURCES": \
+	((x) == 0x85 ? "READ_CONTEXT": \
+	((x) == 0x86 ? "VF_BACKCHANNEL_REQUEST": \
+	((x) == 0x87 ? "READ_VF_MEMORY": \
+	((x) == 0x88 ? "COMPLETE_VF_REQUEST": \
+	((x) == 0x89 ? "EXTEND_CONTEXT_ARRRAY": \
+	((x) == 0x8a ? "MAP_TC_TO_COS": \
+	((x) == 0x8b ? "QUERY_VERSION": \
+	((x) == 0x8c ? "MODIFY_ROCE_CC": \
+	((x) == 0x8d ? "QUERY_ROCE_CC": \
+	((x) == 0x8e ? "QUERY_ROCE_STATS": \
+	((x) == 0x8f ? "SET_LINK_AGGR_MODE": \
+	((x) == 0x90 ? "MODIFY_CQ": \
+	((x) == 0x91 ? "QUERY_QP_EXTEND": \
+	((x) == 0x92 ? "QUERY_ROCE_STATS_EXT": \
+	"Unknown OPCODE"	\
+	)))))))))))))))))))))))))))))))))))))))))
+
+extern unsigned int cmdq_shadow_qd;
+/* Cmdq contains a fix number of a 16-Byte slots */
+struct bnxt_qplib_cmdqe {
+	u8		data[16];
+};
+#define BNXT_QPLIB_CMDQE_UNITS		sizeof(struct bnxt_qplib_cmdqe)
+
+static inline void bnxt_qplib_rcfw_cmd_prep(void *r, u8 opcode, u8 cmd_size)
+{
+	struct cmdq_base *req = r;
+
+	req->opcode = opcode;
+	req->cmd_size = cmd_size;
+}
+
+/* Shadow queue depth for non blocking command */
+#define RCFW_CMD_NON_BLOCKING_SHADOW_QD	64
+#define RCFW_CMD_DEV_ERR_CHECK_TIME_MS	1000 /* 1 Second time out*/
+#define RCFW_ERR_RETRY_COUNT		(RCFW_CMD_WAIT_TIME_MS / RCFW_CMD_DEV_ERR_CHECK_TIME_MS)
+
+/* CMDQ elements */
+#define BNXT_QPLIB_CMDQE_MAX_CNT	8192
+#define BNXT_QPLIB_CMDQE_BYTES		(BNXT_QPLIB_CMDQE_MAX_CNT * 	\
+					 BNXT_QPLIB_CMDQE_UNITS)
+#define BNXT_QPLIB_CMDQE_NPAGES		((BNXT_QPLIB_CMDQE_BYTES %	\
+					  PAGE_SIZE) ?			\
+					  ((BNXT_QPLIB_CMDQE_BYTES /	\
+					    PAGE_SIZE) + 1) :		\
+					  (BNXT_QPLIB_CMDQE_BYTES /	\
+					   PAGE_SIZE))
+#define BNXT_QPLIB_CMDQE_PAGE_SIZE	(BNXT_QPLIB_CMDQE_NPAGES *	\
+					 PAGE_SIZE)
+
+#define RCFW_MAX_OUTSTANDING_CMD	BNXT_QPLIB_CMDQE_MAX_CNT
+#define RCFW_MAX_COOKIE_VALUE		(BNXT_QPLIB_CMDQE_MAX_CNT - 1)
+#define RCFW_CMD_IS_BLOCKING		0x8000
+#define RCFW_NO_FW_ACCESS(rcfw)						\
+	(test_bit(ERR_DEVICE_DETACHED, &(rcfw)->cmdq.flags) ||		\
+	 pci_channel_offline((rcfw)->pdev))
+
+/* Crsq buf is 1024-Byte */
+struct bnxt_qplib_crsbe {
+	u8			data[1024];
+};
+
+/* Get the number of command units required for the req. The
+ * function returns correct value only if called before
+ * setting using bnxt_qplib_set_cmd_slots
+ */
+static inline u32 bnxt_qplib_get_cmd_slots(struct cmdq_base *req)
+{
+	u32 cmd_units = 0;
+
+	if (HAS_TLV_HEADER(req)) {
+		struct roce_tlv *tlv_req = (struct roce_tlv *)req;
+		cmd_units = tlv_req->total_size;
+	} else {
+		cmd_units = (req->cmd_size + BNXT_QPLIB_CMDQE_UNITS - 1) /
+			    BNXT_QPLIB_CMDQE_UNITS;
+	}
+	return cmd_units;
+}
+
+/* Set the cmd_size to a factor of CMDQE unit */
+static inline u32 bnxt_qplib_set_cmd_slots(struct cmdq_base *req)
+{
+	u32 cmd_byte = 0;
+
+	if (HAS_TLV_HEADER(req)) {
+		struct roce_tlv *tlv_req = (struct roce_tlv *)req;
+		cmd_byte = tlv_req->total_size * BNXT_QPLIB_CMDQE_UNITS;
+	} else {
+		cmd_byte = req->cmd_size;
+		req->cmd_size = (req->cmd_size + BNXT_QPLIB_CMDQE_UNITS - 1) /
+				 BNXT_QPLIB_CMDQE_UNITS;
+	}
+
+	return cmd_byte;
+}
+
+/* CREQ */
+/* Allocate 1 per QP for async error notification for now */
+#define BNXT_QPLIB_CREQE_MAX_CNT	(64 * 1024)
+#define BNXT_QPLIB_CREQE_UNITS		16	/* 16-Bytes per prod unit */
+
+#define CREQ_CMP_VALID(hdr, pass)				\
+	(!!((hdr)->v & CREQ_BASE_V) ==				\
+	   !(pass & BNXT_QPLIB_FLAG_EPOCH_CONS_MASK))
+
+#define CREQ_ENTRY_POLL_BUDGET		8
+
+typedef int (*aeq_handler_t)(struct bnxt_qplib_rcfw *, void *, void *);
+
+struct bnxt_qplib_crsqe {
+	struct creq_qp_event	*resp;
+	u32			req_size;
+	bool			is_waiter_alive;
+	bool			is_internal_cmd;
+	bool			is_in_used;
+
+	/* Free slots at the time of submission */
+	u32			free_slots;
+	unsigned long		send_timestamp;
+	u8			opcode;
+	u8			requested_qp_state;
+};
+
+struct bnxt_qplib_rcfw_sbuf {
+	void *sb;
+	dma_addr_t dma_addr;
+	u32 size;
+};
+
+#define BNXT_QPLIB_OOS_COUNT_MASK 0xFFFFFFFF
+
+#define FIRMWARE_INITIALIZED_FLAG	(0)
+#define FIRMWARE_FIRST_FLAG		(31)
+#define FIRMWARE_STALL_DETECTED		(3)
+#define ERR_DEVICE_DETACHED		(4)
+struct bnxt_qplib_cmdq_mbox {
+	struct bnxt_qplib_reg_desc	reg;
+	void __iomem			*prod;
+	void __iomem			*db;
+};
+
+struct bnxt_qplib_cmdq_ctx {
+	struct bnxt_qplib_hwq		hwq;
+	struct bnxt_qplib_cmdq_mbox	cmdq_mbox;
+	wait_queue_head_t		waitq;
+	unsigned long			flags;
+	unsigned long			last_seen;
+	u32				seq_num;
+};
+
+struct bnxt_qplib_creq_db {
+	struct bnxt_qplib_reg_desc	reg;
+	void __iomem			*db;
+	struct bnxt_qplib_db_info	dbinfo;
+};
+
+struct bnxt_qplib_creq_stat {
+	u64	creq_arm_count;
+	u64	creq_tasklet_schedule_count;
+	u64	creq_qp_event_processed;
+	u64	creq_func_event_processed;
+};
+
+struct bnxt_qplib_creq_ctx {
+	struct bnxt_qplib_hwq		hwq;
+	struct bnxt_qplib_creq_db	creq_db;
+	struct bnxt_qplib_creq_stat	stats;
+	aeq_handler_t			aeq_handler;
+	u16				ring_id;
+	int				msix_vec;
+	bool				requested;
+	char				*irq_name;
+};
+
+/* RCFW Communication Channels */
+#define BNXT_QPLIB_RCFW_SEND_RETRY_COUNT 4000
+struct bnxt_qplib_rcfw {
+	struct pci_dev			*pdev;
+	struct bnxt_qplib_res		*res;
+	struct bnxt_qplib_cmdq_ctx	cmdq;
+	struct bnxt_qplib_creq_ctx	creq;
+	struct bnxt_qplib_crsqe		*crsqe_tbl;
+	u32	rcfw_lat_slab_sec[RCFW_MAX_LATENCY_SEC_SLAB_INDEX];
+
+	/* Slow path Perf Stats */
+	bool	sp_perf_stats_enabled;
+	u32	*rcfw_lat_slab_msec;
+	u64	*qp_create_stats;
+	u64	*qp_destroy_stats;
+	u64	*qp_modify_stats;
+	u64	*mr_create_stats;
+	u64	*mr_destroy_stats;
+	u32	qp_create_stats_id;
+	u32	qp_destroy_stats_id;
+	u32	qp_modify_stats_id;
+	u32	mr_create_stats_id;
+	u32	mr_destroy_stats_id;
+	bool init_oos_stats;
+	u64 oos_prev;
+	u32 num_irq_stopped;
+	u32 num_irq_started;
+	u32 poll_in_intr_en;
+	u32 poll_in_intr_dis;
+	atomic_t rcfw_intr_enabled;
+	u32 cmdq_full_dbg;
+	struct semaphore rcfw_inflight;
+	unsigned int	curr_shadow_qd;
+	atomic_t timeout_send;
+	/* cached from chip cctx for quick reference in slow path */
+	u16 max_timeout;
+};
+
+struct bnxt_qplib_cmdqmsg {
+	struct cmdq_base	*req;
+	struct creq_base	*resp;
+	void			*sb;
+	u32			req_sz;
+	u32			res_sz;
+	u8			block;
+	u8			qp_state;
+};
+
+static inline void bnxt_qplib_fill_cmdqmsg(struct bnxt_qplib_cmdqmsg *msg,
+					   void *req, void *resp, void *sb,
+					   u32 req_sz, u32 res_sz, u8 block)
+{
+	msg->req = req;
+	msg->resp = resp;
+	msg->sb = sb;
+	msg->req_sz = req_sz;
+	msg->res_sz = res_sz;
+	msg->block = block;
+}
+
+void bnxt_qplib_free_rcfw_channel(struct bnxt_qplib_res *res);
+int bnxt_qplib_alloc_rcfw_channel(struct bnxt_qplib_res *res);
+void bnxt_qplib_rcfw_stop_irq(struct bnxt_qplib_rcfw *rcfw, bool kill);
+void bnxt_qplib_disable_rcfw_channel(struct bnxt_qplib_rcfw *rcfw);
+int bnxt_qplib_rcfw_start_irq(struct bnxt_qplib_rcfw *rcfw, int msix_vector,
+			      bool need_init);
+int bnxt_qplib_enable_rcfw_channel(struct bnxt_qplib_rcfw *rcfw,
+				   int msix_vector,
+				   int cp_bar_reg_off,
+				   aeq_handler_t aeq_handler);
+
+struct bnxt_qplib_rcfw_sbuf *bnxt_qplib_rcfw_alloc_sbuf(
+				struct bnxt_qplib_rcfw *rcfw,
+				u32 size);
+void bnxt_qplib_rcfw_free_sbuf(struct bnxt_qplib_rcfw *rcfw,
+				struct bnxt_qplib_rcfw_sbuf *sbuf);
+int bnxt_qplib_rcfw_send_message(struct bnxt_qplib_rcfw *rcfw,
+				 struct bnxt_qplib_cmdqmsg *msg);
+
+int bnxt_qplib_deinit_rcfw(struct bnxt_qplib_rcfw *rcfw);
+int bnxt_qplib_init_rcfw(struct bnxt_qplib_rcfw *rcfw, int is_virtfn);
+void bnxt_qplib_mark_qp_error(void *qp_handle);
+int __check_cmdq_stall(struct bnxt_qplib_rcfw *rcfw,
+			u32 *cur_prod, u32 *cur_cons);
+#endif
diff --git a/sys/dev/bnxt/bnxt_re/qplib_res.c b/sys/dev/bnxt/bnxt_re/qplib_res.c
new file mode 100644
index 000000000000..69661c67708c
--- /dev/null
+++ b/sys/dev/bnxt/bnxt_re/qplib_res.c
@@ -0,0 +1,1226 @@
+/*
+ * Copyright (c) 2015-2024, Broadcom. All rights reserved.  The term
+ * Broadcom refers to Broadcom Limited and/or its subsidiaries.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS''
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
+ * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
+ * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * Description: QPLib resource manager
+ */
+
+#include <linux/spinlock.h>
+#include <linux/pci.h>
+#include <linux/interrupt.h>
+#include <linux/inetdevice.h>
+#include <linux/dma-mapping.h>
+#include <linux/if_vlan.h>
+#include <linux/vmalloc.h>
+
+#include <net/ipv6.h>
+#include <rdma/ib_verbs.h>
+
+#include "hsi_struct_def.h"
+#include "qplib_res.h"
+#include "qplib_sp.h"
+#include "qplib_rcfw.h"
+#include "bnxt.h"
+#include "bnxt_ulp.h"
+
+uint8_t _get_chip_gen_p5_type(struct bnxt_qplib_chip_ctx *cctx)
+{
+       /* Extend this for granular type */
+	return(BNXT_RE_DEFAULT);
+}
+
+inline bool _is_alloc_mr_unified(struct bnxt_qplib_dev_attr *dattr)
+{
+	return dattr->dev_cap_flags &
+	       CREQ_QUERY_FUNC_RESP_SB_MR_REGISTER_ALLOC;
+}
+
+/* PBL */
+static void __free_pbl(struct bnxt_qplib_res *res,
+		       struct bnxt_qplib_pbl *pbl, bool is_umem)
+{
+	struct pci_dev *pdev;
+	int i;
+
+	pdev = res->pdev;
+	if (is_umem == false) {
+		for (i = 0; i < pbl->pg_count; i++) {
+			if (pbl->pg_arr[i]) {
+				dma_free_coherent(&pdev->dev, pbl->pg_size,
+					(void *)((u64)pbl->pg_arr[i] &
+						 PAGE_MASK),
+					pbl->pg_map_arr[i]);
+			}
+			else
+				dev_warn(&pdev->dev,
+					 "QPLIB: PBL free pg_arr[%d] empty?!\n",
+					 i);
+			pbl->pg_arr[i] = NULL;
+		}
+	}
+
+	if (pbl->pg_arr) {
+		vfree(pbl->pg_arr);
+		pbl->pg_arr = NULL;
+	}
+	if (pbl->pg_map_arr) {
+		vfree(pbl->pg_map_arr);
+		pbl->pg_map_arr = NULL;
+	}
+	pbl->pg_count = 0;
+	pbl->pg_size = 0;
+}
+
+struct qplib_sg {
+	dma_addr_t 	pg_map_arr;
+	u32		size;
+};
+
+static int __fill_user_dma_pages(struct bnxt_qplib_pbl *pbl,
+				 struct bnxt_qplib_sg_info *sginfo)
+{
+	int sg_indx, pg_indx, tmp_size, offset;
+	struct qplib_sg *tmp_sg = NULL;
+	struct scatterlist *sg;
+	u64 pmask, addr;
+
+	tmp_sg = vzalloc(sginfo->nmap * sizeof(struct qplib_sg));
+	if (!tmp_sg)
+		return -ENOMEM;
+
+	pmask = BIT_ULL(sginfo->pgshft) - 1;
+	sg_indx = 0;
+	for_each_sg(sginfo->sghead, sg, sginfo->nmap, sg_indx) {
+		tmp_sg[sg_indx].pg_map_arr = sg_dma_address(sg);
+		tmp_sg[sg_indx].size = sg_dma_len(sg);
+	}
+	pg_indx = 0;
+	for (sg_indx = 0; sg_indx < sginfo->nmap; sg_indx++) {
+		tmp_size = tmp_sg[sg_indx].size;
+		offset = 0;
+		while (tmp_size > 0) {
+			addr = tmp_sg[sg_indx].pg_map_arr + offset;
+			if ((!sg_indx && !pg_indx) || !(addr & pmask)) {
+				pbl->pg_map_arr[pg_indx] = addr &(~pmask);
+				pbl->pg_count++;
+				pg_indx++;
+			}
+			offset += sginfo->pgsize;
+			tmp_size -= sginfo->pgsize;
+		}
+	}
+
+	vfree(tmp_sg);
+	return 0;
+}
+
+static int bnxt_qplib_fill_user_dma_pages(struct bnxt_qplib_pbl *pbl,
+					  struct bnxt_qplib_sg_info *sginfo)
+{
+	int rc = 0;
+
+	rc =  __fill_user_dma_pages(pbl, sginfo);
+
+	return rc;
+}
+
+static int __alloc_pbl(struct bnxt_qplib_res *res, struct bnxt_qplib_pbl *pbl,
+		       struct bnxt_qplib_sg_info *sginfo)
+{
+	struct pci_dev *pdev;
+	bool is_umem = false;
+	int i;
+
+	if (sginfo->nopte)
+		return 0;
+
+	pdev = res->pdev;
+	/* page ptr arrays */
+	pbl->pg_arr = vmalloc(sginfo->npages * sizeof(void *));
+	if (!pbl->pg_arr)
+		return -ENOMEM;
+
+	pbl->pg_map_arr = vmalloc(sginfo->npages * sizeof(dma_addr_t));
+	if (!pbl->pg_map_arr) {
+		vfree(pbl->pg_arr);
+		return -ENOMEM;
+	}
+	pbl->pg_count = 0;
+	pbl->pg_size = sginfo->pgsize;
+	if (!sginfo->sghead) {
+		for (i = 0; i < sginfo->npages; i++) {
+			pbl->pg_arr[i] = dma_zalloc_coherent(&pdev->dev,
+							     pbl->pg_size,
+							     &pbl->pg_map_arr[i],
+							     GFP_KERNEL);
+			if (!pbl->pg_arr[i])
+				goto fail;
+			pbl->pg_count++;
+		}
+	} else {
+		is_umem = true;
+		if (bnxt_qplib_fill_user_dma_pages(pbl, sginfo))
+			goto fail;
+	}
+
+	return 0;
+fail:
+	__free_pbl(res, pbl, is_umem);
+	return -ENOMEM;
+}
+
+/* HWQ */
+void bnxt_qplib_free_hwq(struct bnxt_qplib_res *res,
+			 struct bnxt_qplib_hwq *hwq)
+{
+	int i;
+
+	if (!hwq->max_elements)
+		return;
+	if (hwq->level >= PBL_LVL_MAX)
+		return;
+
+	for (i = 0; i < hwq->level + 1; i++) {
+		if (i == hwq->level)
+			__free_pbl(res, &hwq->pbl[i], hwq->is_user);
+		else
+			__free_pbl(res, &hwq->pbl[i], false);
+	}
+
+	hwq->level = PBL_LVL_MAX;
+	hwq->max_elements = 0;
+	hwq->element_size = 0;
+	hwq->prod = hwq->cons = 0;
+	hwq->cp_bit = 0;
+}
+
+/* All HWQs are power of 2 in size */
+int bnxt_qplib_alloc_init_hwq(struct bnxt_qplib_hwq *hwq,
+			      struct bnxt_qplib_hwq_attr *hwq_attr)
+{
+	u32 npages = 0, depth, stride, aux_pages = 0;
+	dma_addr_t *src_phys_ptr, **dst_virt_ptr;
+	struct bnxt_qplib_sg_info sginfo = {};
+	u32 aux_size = 0, npbl, npde;
+	void *umem;
+	struct bnxt_qplib_res *res;
+	u32 aux_slots, pg_size;
+	struct pci_dev *pdev;
+	int i, rc, lvl;
+
+	res = hwq_attr->res;
+	pdev = res->pdev;
+	umem = hwq_attr->sginfo->sghead;
+	pg_size = hwq_attr->sginfo->pgsize;
+	hwq->level = PBL_LVL_MAX;
+
+	depth = roundup_pow_of_two(hwq_attr->depth);
+	stride = roundup_pow_of_two(hwq_attr->stride);
+	if (hwq_attr->aux_depth) {
+		aux_slots = hwq_attr->aux_depth;
+		aux_size = roundup_pow_of_two(hwq_attr->aux_stride);
+		aux_pages = (aux_slots * aux_size) / pg_size;
+		if ((aux_slots * aux_size) % pg_size)
+			aux_pages++;
+	}
+
+	if (!umem) {
+		hwq->is_user = false;
+		npages = (depth * stride) / pg_size + aux_pages;
+		if ((depth * stride) % pg_size)
+			npages++;
+		if (!npages)
+			return -EINVAL;
+		hwq_attr->sginfo->npages = npages;
+	} else {
+		hwq->is_user = true;
+		npages = hwq_attr->sginfo->npages;
+		npages = (npages * (u64)pg_size) /
+			  BIT_ULL(hwq_attr->sginfo->pgshft);
+		if ((hwq_attr->sginfo->npages * (u64)pg_size) %
+		     BIT_ULL(hwq_attr->sginfo->pgshft))
+			npages++;
+	}
+	if (npages == MAX_PBL_LVL_0_PGS && !hwq_attr->sginfo->nopte) {
+		/* This request is Level 0, map PTE */
+		rc = __alloc_pbl(res, &hwq->pbl[PBL_LVL_0], hwq_attr->sginfo);
+		if (rc)
+			goto fail;
+		hwq->level = PBL_LVL_0;
+		goto done;
+	}
+
+	if (npages >= MAX_PBL_LVL_0_PGS) {
+		if (npages > MAX_PBL_LVL_1_PGS) {
+			u32 flag = (hwq_attr->type == HWQ_TYPE_L2_CMPL) ?
+				    0 : PTU_PTE_VALID;
+			/* 2 levels of indirection */
+			npbl = npages >> MAX_PBL_LVL_1_PGS_SHIFT;
+			if (npages % BIT(MAX_PBL_LVL_1_PGS_SHIFT))
+				npbl++;
+			npde = npbl >> MAX_PDL_LVL_SHIFT;
+			if(npbl % BIT(MAX_PDL_LVL_SHIFT))
+				npde++;
+			/* Alloc PDE pages */
+			sginfo.pgsize = npde * PAGE_SIZE;
+			sginfo.npages = 1;
+			rc = __alloc_pbl(res, &hwq->pbl[PBL_LVL_0], &sginfo);
+
+			/* Alloc PBL pages */
+			sginfo.npages = npbl;
+			sginfo.pgsize = PAGE_SIZE;
+			rc = __alloc_pbl(res, &hwq->pbl[PBL_LVL_1], &sginfo);
+			if (rc)
+				goto fail;
+			/* Fill PDL with PBL page pointers */
+			dst_virt_ptr =
+				(dma_addr_t **)hwq->pbl[PBL_LVL_0].pg_arr;
+			src_phys_ptr = hwq->pbl[PBL_LVL_1].pg_map_arr;
+			if (hwq_attr->type == HWQ_TYPE_MR) {
+			/* For MR it is expected that we supply only 1 contigous
+			 * page i.e only 1 entry in the PDL that will contain
+			 * all the PBLs for the user supplied memory region
+			 */
+				for (i = 0; i < hwq->pbl[PBL_LVL_1].pg_count; i++)
+					dst_virt_ptr[0][i] = src_phys_ptr[i] |
+						flag;
+			} else {
+				for (i = 0; i < hwq->pbl[PBL_LVL_1].pg_count; i++)
+					dst_virt_ptr[PTR_PG(i)][PTR_IDX(i)] =
+						src_phys_ptr[i] | PTU_PDE_VALID;
+			}
+			/* Alloc or init PTEs */
+			rc = __alloc_pbl(res, &hwq->pbl[PBL_LVL_2],
+					 hwq_attr->sginfo);
+			if (rc)
+				goto fail;
+			hwq->level = PBL_LVL_2;
+			if (hwq_attr->sginfo->nopte)
+				goto done;
+			/* Fill PBLs with PTE pointers */
+			dst_virt_ptr =
+				(dma_addr_t **)hwq->pbl[PBL_LVL_1].pg_arr;
+			src_phys_ptr = hwq->pbl[PBL_LVL_2].pg_map_arr;
+			for (i = 0; i < hwq->pbl[PBL_LVL_2].pg_count; i++) {
+				dst_virt_ptr[PTR_PG(i)][PTR_IDX(i)] =
+					src_phys_ptr[i] | PTU_PTE_VALID;
+			}
+			if (hwq_attr->type == HWQ_TYPE_QUEUE) {
+				/* Find the last pg of the size */
+				i = hwq->pbl[PBL_LVL_2].pg_count;
+				dst_virt_ptr[PTR_PG(i - 1)][PTR_IDX(i - 1)] |=
+								  PTU_PTE_LAST;
+				if (i > 1)
+					dst_virt_ptr[PTR_PG(i - 2)]
+						    [PTR_IDX(i - 2)] |=
+						    PTU_PTE_NEXT_TO_LAST;
+			}
+		} else { /* pages < 512 npbl = 1, npde = 0 */
+			u32 flag = (hwq_attr->type == HWQ_TYPE_L2_CMPL) ?
+				    0 : PTU_PTE_VALID;
+
+			/* 1 level of indirection */
+			npbl = npages >> MAX_PBL_LVL_1_PGS_SHIFT;
+			if (npages % BIT(MAX_PBL_LVL_1_PGS_SHIFT))
+				npbl++;
+			sginfo.npages = npbl;
+			sginfo.pgsize = PAGE_SIZE;
+			/* Alloc PBL page */
+			rc = __alloc_pbl(res, &hwq->pbl[PBL_LVL_0], &sginfo);
+			if (rc)
+				goto fail;
+			/* Alloc or init  PTEs */
+			rc = __alloc_pbl(res, &hwq->pbl[PBL_LVL_1],
+					 hwq_attr->sginfo);
+			if (rc)
+				goto fail;
+			hwq->level = PBL_LVL_1;
+			if (hwq_attr->sginfo->nopte)
+				goto done;
+			/* Fill PBL with PTE pointers */
+			dst_virt_ptr =
+				(dma_addr_t **)hwq->pbl[PBL_LVL_0].pg_arr;
+			src_phys_ptr = hwq->pbl[PBL_LVL_1].pg_map_arr;
+			for (i = 0; i < hwq->pbl[PBL_LVL_1].pg_count; i++)
+				dst_virt_ptr[PTR_PG(i)][PTR_IDX(i)] =
+					src_phys_ptr[i] | flag;
+			if (hwq_attr->type == HWQ_TYPE_QUEUE) {
+				/* Find the last pg of the size */
+				i = hwq->pbl[PBL_LVL_1].pg_count;
+				dst_virt_ptr[PTR_PG(i - 1)][PTR_IDX(i - 1)] |=
+								  PTU_PTE_LAST;
+				if (i > 1)
+					dst_virt_ptr[PTR_PG(i - 2)]
+						    [PTR_IDX(i - 2)] |=
+						    PTU_PTE_NEXT_TO_LAST;
+			}
+		}
+	}
+done:
+	hwq->prod = 0;
+	hwq->cons = 0;
+	hwq->pdev = pdev;
+	hwq->depth = hwq_attr->depth;
+	hwq->max_elements = depth;
+	hwq->element_size = stride;
+	hwq->qe_ppg = (pg_size/stride);
+
+	if (hwq->level >= PBL_LVL_MAX)
+		goto fail;
+	/* For direct access to the elements */
+	lvl = hwq->level;
+	if (hwq_attr->sginfo->nopte && hwq->level)
+		lvl = hwq->level - 1;
+	hwq->pbl_ptr = hwq->pbl[lvl].pg_arr;
+	hwq->pbl_dma_ptr = hwq->pbl[lvl].pg_map_arr;
+	spin_lock_init(&hwq->lock);
+
+	return 0;
+fail:
+	bnxt_qplib_free_hwq(res, hwq);
+	return -ENOMEM;
+}
+
+/* Context Tables */
+void bnxt_qplib_free_hwctx(struct bnxt_qplib_res *res)
+{
+	struct bnxt_qplib_ctx *hctx;
+	int i;
+
+	hctx = res->hctx;
+	bnxt_qplib_free_hwq(res, &hctx->qp_ctx.hwq);
+	bnxt_qplib_free_hwq(res, &hctx->mrw_ctx.hwq);
+	bnxt_qplib_free_hwq(res, &hctx->srq_ctx.hwq);
+	bnxt_qplib_free_hwq(res, &hctx->cq_ctx.hwq);
+	bnxt_qplib_free_hwq(res, &hctx->tim_ctx.hwq);
+	for (i = 0; i < MAX_TQM_ALLOC_REQ; i++)
+		bnxt_qplib_free_hwq(res, &hctx->tqm_ctx.qtbl[i]);
+	/* restor original pde level before destroy */
+	hctx->tqm_ctx.pde.level = hctx->tqm_ctx.pde_level;
+	bnxt_qplib_free_hwq(res, &hctx->tqm_ctx.pde);
+}
+
+static int bnxt_qplib_alloc_tqm_rings(struct bnxt_qplib_res *res,
+				      struct bnxt_qplib_ctx *hctx)
+{
+	struct bnxt_qplib_hwq_attr hwq_attr = {};
+	struct bnxt_qplib_sg_info sginfo = {};
+	struct bnxt_qplib_tqm_ctx *tqmctx;
+	int rc = 0;
+	int i;
+
+	tqmctx = &hctx->tqm_ctx;
+
+	sginfo.pgsize = PAGE_SIZE;
+	sginfo.pgshft = PAGE_SHIFT;
+	hwq_attr.sginfo = &sginfo;
+	hwq_attr.res = res;
+	hwq_attr.type = HWQ_TYPE_CTX;
+	hwq_attr.depth = 512;
+	hwq_attr.stride = sizeof(u64);
+	/* Alloc pdl buffer */
+	rc = bnxt_qplib_alloc_init_hwq(&tqmctx->pde, &hwq_attr);
+	if (rc)
+		goto out;
+	/* Save original pdl level */
+	tqmctx->pde_level = tqmctx->pde.level;
+
+	hwq_attr.stride = 1;
+	for (i = 0; i < MAX_TQM_ALLOC_REQ; i++) {
+		if (!tqmctx->qcount[i])
+			continue;
+		hwq_attr.depth = hctx->qp_ctx.max * tqmctx->qcount[i];
+		rc = bnxt_qplib_alloc_init_hwq(&tqmctx->qtbl[i], &hwq_attr);
+		if (rc)
+			goto out;
+	}
+out:
+	return rc;
+}
+
+static void bnxt_qplib_map_tqm_pgtbl(struct bnxt_qplib_tqm_ctx *ctx)
+{
+	struct bnxt_qplib_hwq *qtbl_hwq;
+	dma_addr_t *dma_ptr;
+	__le64 **pbl_ptr, *ptr;
+	int i, j, k;
+	int fnz_idx = -1;
+	int pg_count;
+
+	pbl_ptr = (__le64 **)ctx->pde.pbl_ptr;
+
+	for (i = 0, j = 0; i < MAX_TQM_ALLOC_REQ;
+	     i++, j += MAX_TQM_ALLOC_BLK_SIZE) {
+		qtbl_hwq = &ctx->qtbl[i];
+		if (!qtbl_hwq->max_elements)
+			continue;
+		if (fnz_idx == -1)
+			fnz_idx = i; /* first non-zero index */
+		switch (qtbl_hwq->level) {
+			case PBL_LVL_2:
+			pg_count = qtbl_hwq->pbl[PBL_LVL_1].pg_count;
+			for (k = 0; k < pg_count; k++) {
+				ptr = &pbl_ptr[PTR_PG(j + k)][PTR_IDX(j + k)];
+				dma_ptr = &qtbl_hwq->pbl[PBL_LVL_1].pg_map_arr[k];
+				*ptr = cpu_to_le64(*dma_ptr | PTU_PTE_VALID);
+			}
+			break;
+			case PBL_LVL_1:
+			case PBL_LVL_0:
+			default:
+			ptr = &pbl_ptr[PTR_PG(j)][PTR_IDX(j)];
+			*ptr = cpu_to_le64(qtbl_hwq->pbl[PBL_LVL_0].pg_map_arr[0] |
+					   PTU_PTE_VALID);
+			break;
+		}
+	}
+	if (fnz_idx == -1)
+		fnz_idx = 0;
+	/* update pde level as per page table programming */
+	ctx->pde.level = (ctx->qtbl[fnz_idx].level == PBL_LVL_2) ? PBL_LVL_2 :
+			  ctx->qtbl[fnz_idx].level + 1;
+}
+
+static int bnxt_qplib_setup_tqm_rings(struct bnxt_qplib_res *res,
+				      struct bnxt_qplib_ctx *hctx)
+{
+	int rc = 0;
+
+	rc = bnxt_qplib_alloc_tqm_rings(res, hctx);
+	if (rc)
+		goto fail;
+
+	bnxt_qplib_map_tqm_pgtbl(&hctx->tqm_ctx);
+fail:
+	return rc;
+}
+
+/*
+ * Routine: bnxt_qplib_alloc_hwctx
+ * Description:
+ *     Context tables are memories which are used by the chip.
+ *     The 6 tables defined are:
+ *             QPC ctx - holds QP states
+ *             MRW ctx - holds memory region and window
+ *             SRQ ctx - holds shared RQ states
+ *             CQ ctx - holds completion queue states
+ *             TQM ctx - holds Tx Queue Manager context
+ *             TIM ctx - holds timer context
+ *     Depending on the size of the tbl requested, either a 1 Page Buffer List
+ *     or a 1-to-2-stage indirection Page Directory List + 1 PBL is used
+ *     instead.
+ *     Table might be employed as follows:
+ *             For 0      < ctx size <= 1 PAGE, 0 level of ind is used
+ *             For 1 PAGE < ctx size <= 512 entries size, 1 level of ind is used
+ *             For 512    < ctx size <= MAX, 2 levels of ind is used
+ * Returns:
+ *     0 if success, else -ERRORS
+ */
+int bnxt_qplib_alloc_hwctx(struct bnxt_qplib_res *res)
+{
+	struct bnxt_qplib_hwq_attr hwq_attr = {};
+	struct bnxt_qplib_sg_info sginfo = {};
+	struct bnxt_qplib_ctx *hctx;
+	struct bnxt_qplib_hwq *hwq;
+	int rc = 0;
+
+	hctx = res->hctx;
+	/* QPC Tables */
+	sginfo.pgsize = PAGE_SIZE;
+	sginfo.pgshft = PAGE_SHIFT;
+	hwq_attr.sginfo = &sginfo;
+
+	hwq_attr.res = res;
+	hwq_attr.depth = hctx->qp_ctx.max;
+	hwq_attr.stride = BNXT_QPLIB_MAX_QP_CTX_ENTRY_SIZE;
+	hwq_attr.type = HWQ_TYPE_CTX;
+	hwq = &hctx->qp_ctx.hwq;
+	rc = bnxt_qplib_alloc_init_hwq(hwq, &hwq_attr);
+	if (rc)
+		goto fail;
+
+	/* MRW Tables */
+	hwq_attr.depth = hctx->mrw_ctx.max;
+	hwq_attr.stride = BNXT_QPLIB_MAX_MRW_CTX_ENTRY_SIZE;
+	hwq = &hctx->mrw_ctx.hwq;
+	rc = bnxt_qplib_alloc_init_hwq(hwq, &hwq_attr);
+	if (rc)
+		goto fail;
+
+	/* SRQ Tables */
+	hwq_attr.depth = hctx->srq_ctx.max;
+	hwq_attr.stride = BNXT_QPLIB_MAX_SRQ_CTX_ENTRY_SIZE;
+	hwq = &hctx->srq_ctx.hwq;
+	rc = bnxt_qplib_alloc_init_hwq(hwq, &hwq_attr);
+	if (rc)
+		goto fail;
+
+	/* CQ Tables */
+	hwq_attr.depth = hctx->cq_ctx.max;
+	hwq_attr.stride = BNXT_QPLIB_MAX_CQ_CTX_ENTRY_SIZE;
+	hwq = &hctx->cq_ctx.hwq;
+	rc = bnxt_qplib_alloc_init_hwq(hwq, &hwq_attr);
+	if (rc)
+		goto fail;
+
+	/* TQM Buffer */
+	rc = bnxt_qplib_setup_tqm_rings(res, hctx);
+	if (rc)
+		goto fail;
+	/* TIM Buffer */
+	hwq_attr.depth = hctx->qp_ctx.max * 16;
+	hwq_attr.stride = 1;
+	hwq = &hctx->tim_ctx.hwq;
+	rc = bnxt_qplib_alloc_init_hwq(hwq, &hwq_attr);
+	if (rc)
+		goto fail;
+
+	return 0;
+fail:
+	bnxt_qplib_free_hwctx(res);
+	return rc;
+}
+
+/* GUID */
+void bnxt_qplib_get_guid(const u8 *dev_addr, u8 *guid)
+{
+	u8 mac[ETH_ALEN];
+
+	/* MAC-48 to EUI-64 mapping */
+	memcpy(mac, dev_addr, ETH_ALEN);
+	guid[0] = mac[0] ^ 2;
+	guid[1] = mac[1];
+	guid[2] = mac[2];
+	guid[3] = 0xff;
+	guid[4] = 0xfe;
+	guid[5] = mac[3];
+	guid[6] = mac[4];
+	guid[7] = mac[5];
+}
+
+static void bnxt_qplib_free_sgid_tbl(struct bnxt_qplib_res *res)
+{
+	struct bnxt_qplib_sgid_tbl *sgid_tbl;
+
+	sgid_tbl = &res->sgid_tbl;
+
+	if (sgid_tbl->tbl) {
+		kfree(sgid_tbl->tbl);
+		sgid_tbl->tbl = NULL;
+		kfree(sgid_tbl->hw_id);
+		sgid_tbl->hw_id = NULL;
+		kfree(sgid_tbl->ctx);
+		sgid_tbl->ctx = NULL;
+		kfree(sgid_tbl->vlan);
+		sgid_tbl->vlan = NULL;
+	} else {
+		dev_dbg(&res->pdev->dev, "QPLIB: SGID tbl not present");
+	}
+	sgid_tbl->max = 0;
+	sgid_tbl->active = 0;
+}
+
+static void bnxt_qplib_free_reftbls(struct bnxt_qplib_res *res)
+{
+	struct bnxt_qplib_reftbl *tbl;
+
+	tbl = &res->reftbl.srqref;
+	vfree(tbl->rec);
+
+	tbl = &res->reftbl.cqref;
+	vfree(tbl->rec);
+
+	tbl = &res->reftbl.qpref;
+	vfree(tbl->rec);
+}
+
+static int bnxt_qplib_alloc_reftbl(struct bnxt_qplib_reftbl *tbl, u32 max)
+{
+	tbl->max = max;
+	tbl->rec = vzalloc(sizeof(*tbl->rec) * max);
+	if (!tbl->rec)
+		return -ENOMEM;
+	spin_lock_init(&tbl->lock);
+	return 0;
+}
+
+static int bnxt_qplib_alloc_reftbls(struct bnxt_qplib_res *res,
+				    struct bnxt_qplib_dev_attr *dattr)
+{
+	u32 max_cq = BNXT_QPLIB_MAX_CQ_COUNT;
+	struct bnxt_qplib_reftbl *tbl;
+	u32 res_cnt;
+	int rc;
+
+	/*
+	 * Allocating one extra entry  to hold QP1 info.
+	 * Store QP1 info at the last entry of the table.
+	 * Decrement the tbl->max by one so that modulo
+	 * operation to get the qp table index from qp id
+	 * returns any value between 0 and max_qp-1
+	 */
+	res_cnt = max_t(u32, BNXT_QPLIB_MAX_QPC_COUNT + 1, dattr->max_qp);
+	tbl = &res->reftbl.qpref;
+	rc = bnxt_qplib_alloc_reftbl(tbl, res_cnt);
+	if (rc)
+		goto fail;
+	tbl->max--;
+
+	if (_is_chip_gen_p5_p7(res->cctx))
+		max_cq = BNXT_QPLIB_MAX_CQ_COUNT_P5;
+	res_cnt = max_t(u32, max_cq, dattr->max_cq);
+	tbl = &res->reftbl.cqref;
+	rc = bnxt_qplib_alloc_reftbl(tbl, res_cnt);
+	if (rc)
+		goto fail;
+
+	res_cnt = max_t(u32, BNXT_QPLIB_MAX_SRQC_COUNT, dattr->max_cq);
+	tbl = &res->reftbl.srqref;
+	rc = bnxt_qplib_alloc_reftbl(tbl, BNXT_QPLIB_MAX_SRQC_COUNT);
+	if (rc)
+		goto fail;
+
+	return 0;
+fail:
+	return rc;
+}
+
+static int bnxt_qplib_alloc_sgid_tbl(struct bnxt_qplib_res *res, u16 max)
+{
+	struct bnxt_qplib_sgid_tbl *sgid_tbl;
+
+	sgid_tbl = &res->sgid_tbl;
+
+	sgid_tbl->tbl = kcalloc(max, sizeof(*sgid_tbl->tbl), GFP_KERNEL);
+	if (!sgid_tbl->tbl)
+		return -ENOMEM;
+
+	sgid_tbl->hw_id = kcalloc(max, sizeof(u32), GFP_KERNEL);
+	if (!sgid_tbl->hw_id)
+		goto free_tbl;
+
+	sgid_tbl->ctx = kcalloc(max, sizeof(void *), GFP_KERNEL);
+	if (!sgid_tbl->ctx)
+		goto free_hw_id;
+
+	sgid_tbl->vlan = kcalloc(max, sizeof(u8), GFP_KERNEL);
+	if (!sgid_tbl->vlan)
+		goto free_ctx;
+
+	sgid_tbl->max = max;
+	return 0;
+free_ctx:
+	kfree(sgid_tbl->ctx);
+free_hw_id:
+	kfree(sgid_tbl->hw_id);
+free_tbl:
+	kfree(sgid_tbl->tbl);
+	return -ENOMEM;
+};
+
+static void bnxt_qplib_cleanup_sgid_tbl(struct bnxt_qplib_res *res,
+					struct bnxt_qplib_sgid_tbl *sgid_tbl)
+{
+	int i;
+
+	for (i = 0; i < sgid_tbl->max; i++) {
+		if (memcmp(&sgid_tbl->tbl[i], &bnxt_qplib_gid_zero,
+			   sizeof(bnxt_qplib_gid_zero)))
+			bnxt_qplib_del_sgid(sgid_tbl, &sgid_tbl->tbl[i].gid,
+					    sgid_tbl->tbl[i].vlan_id, true);
+	}
+	memset(sgid_tbl->tbl, 0, sizeof(*sgid_tbl->tbl) * sgid_tbl->max);
+	memset(sgid_tbl->hw_id, -1, sizeof(u16) * sgid_tbl->max);
+	memset(sgid_tbl->vlan, 0, sizeof(u8) * sgid_tbl->max);
+	sgid_tbl->active = 0;
+}
+
+static void bnxt_qplib_init_sgid_tbl(struct bnxt_qplib_sgid_tbl *sgid_tbl,
+				     struct ifnet *netdev)
+{
+	u32 i;
+
+	for (i = 0; i < sgid_tbl->max; i++)
+		sgid_tbl->tbl[i].vlan_id = 0xffff;
+	memset(sgid_tbl->hw_id, -1, sizeof(u16) * sgid_tbl->max);
+}
+
+/* PDs */
+int bnxt_qplib_alloc_pd(struct bnxt_qplib_res *res, struct bnxt_qplib_pd *pd)
+{
+	u32 bit_num;
+	int rc = 0;
+	struct bnxt_qplib_pd_tbl *pdt = &res->pd_tbl;
+
+	mutex_lock(&res->pd_tbl_lock);
+	bit_num = find_first_bit(pdt->tbl, pdt->max);
+	if (bit_num == pdt->max - 1) {/* Last bit is reserved */
+		rc = -ENOMEM;
+		goto fail;
+	}
+
+	/* Found unused PD */
+	clear_bit(bit_num, pdt->tbl);
+	pd->id = bit_num;
+fail:
+	mutex_unlock(&res->pd_tbl_lock);
+	return rc;
+}
+
+int bnxt_qplib_dealloc_pd(struct bnxt_qplib_res *res,
+			  struct bnxt_qplib_pd_tbl *pdt,
+			  struct bnxt_qplib_pd *pd)
+{
+	mutex_lock(&res->pd_tbl_lock);
+	if (test_and_set_bit(pd->id, pdt->tbl)) {
+		dev_warn(&res->pdev->dev, "Freeing an unused PD? pdn = %d\n",
+			 pd->id);
+		mutex_unlock(&res->pd_tbl_lock);
+		return -EINVAL;
+	}
+	/* Reset to reserved pdid. */
+	pd->id = pdt->max - 1;
+
+	mutex_unlock(&res->pd_tbl_lock);
+	return 0;
+}
+
+static void bnxt_qplib_free_pd_tbl(struct bnxt_qplib_pd_tbl *pdt)
+{
+	if (pdt->tbl) {
+		kfree(pdt->tbl);
+		pdt->tbl = NULL;
+	}
+	pdt->max = 0;
+}
+
+static int bnxt_qplib_alloc_pd_tbl(struct bnxt_qplib_res *res, u32 max)
+{
+	struct bnxt_qplib_pd_tbl *pdt;
+	u32 bytes;
+
+	pdt = &res->pd_tbl;
+
+	max++; /* One extra for reserved pdid. */
+	bytes = DIV_ROUND_UP(max, 8);
+
+	if (!bytes)
+		bytes = 1;
+	pdt->tbl = kmalloc(bytes, GFP_KERNEL);
+	if (!pdt->tbl) {
+		dev_err(&res->pdev->dev,
+			"QPLIB: PD tbl allocation failed for size = %d\n", bytes);
+		return -ENOMEM;
+	}
+	pdt->max = max;
+	memset((u8 *)pdt->tbl, 0xFF, bytes);
+	mutex_init(&res->pd_tbl_lock);
+
+	return 0;
+}
+
+/* DPIs */
+int bnxt_qplib_alloc_dpi(struct bnxt_qplib_res	*res,
+			 struct bnxt_qplib_dpi	*dpi,
+			 void *app, u8 type)
+{
+	struct bnxt_qplib_dpi_tbl *dpit = &res->dpi_tbl;
+	struct bnxt_qplib_reg_desc *reg;
+	u32 bit_num;
+	u64 umaddr;
+	int rc = 0;
+
+	reg = &dpit->wcreg;
+	mutex_lock(&res->dpi_tbl_lock);
+	if (type == BNXT_QPLIB_DPI_TYPE_WC && _is_chip_p7(res->cctx) &&
+	    !dpit->avail_ppp) {
+		rc = -ENOMEM;
+		goto fail;
+	}
+	bit_num = find_first_bit(dpit->tbl, dpit->max);
+	if (bit_num == dpit->max) {
+		rc = -ENOMEM;
+		goto fail;
+	}
+	/* Found unused DPI */
+	clear_bit(bit_num, dpit->tbl);
+	dpit->app_tbl[bit_num] = app;
+	dpi->bit = bit_num;
+	dpi->dpi = bit_num + (reg->offset - dpit->ucreg.offset) / PAGE_SIZE;
+
+	umaddr = reg->bar_base + reg->offset + bit_num * PAGE_SIZE;
+	dpi->umdbr = umaddr;
+	switch (type) {
+	case BNXT_QPLIB_DPI_TYPE_KERNEL:
+		/* priviledged dbr was already mapped just initialize it. */
+		dpi->umdbr = dpit->ucreg.bar_base +
+			     dpit->ucreg.offset + bit_num * PAGE_SIZE;
+		dpi->dbr = dpit->priv_db;
+		dpi->dpi = dpi->bit;
+		break;
+	case BNXT_QPLIB_DPI_TYPE_WC:
+		dpi->dbr = ioremap_wc(umaddr, PAGE_SIZE);
+		if (_is_chip_p7(res->cctx) && dpi->dbr)
+			dpit->avail_ppp--;
+		break;
+	default:
+		dpi->dbr = ioremap(umaddr, PAGE_SIZE);
+	}
+	if (!dpi->dbr) {
+		dev_err(&res->pdev->dev, "QPLIB: DB remap failed, type = %d\n",
+			type);
+		rc = -ENOMEM;
+	}
+	dpi->type = type;
+fail:
+	mutex_unlock(&res->dpi_tbl_lock);
+	return rc;
+}
+
+int bnxt_qplib_dealloc_dpi(struct bnxt_qplib_res *res,
+			   struct bnxt_qplib_dpi *dpi)
+{
+	struct bnxt_qplib_dpi_tbl *dpit = &res->dpi_tbl;
+	int rc = 0;
+
+	mutex_lock(&res->dpi_tbl_lock);
+	if (dpi->bit >= dpit->max) {
+		dev_warn(&res->pdev->dev,
+			 "Invalid DPI? dpi = %d, bit = %d\n",
+			 dpi->dpi, dpi->bit);
+		rc = -EINVAL;
+		goto fail;
+	}
+
+	if (dpi->dpi && dpi->type != BNXT_QPLIB_DPI_TYPE_KERNEL) {
+		if (dpi->type == BNXT_QPLIB_DPI_TYPE_WC &&
+		    _is_chip_p7(res->cctx) && dpi->dbr)
+			dpit->avail_ppp++;
+		pci_iounmap(res->pdev, dpi->dbr);
+	}
+
+	if (test_and_set_bit(dpi->bit, dpit->tbl)) {
+		dev_warn(&res->pdev->dev,
+			 "Freeing an unused DPI? dpi = %d, bit = %d\n",
+			 dpi->dpi, dpi->bit);
+		rc = -EINVAL;
+		goto fail;
+	}
+	if (dpit->app_tbl)
+		dpit->app_tbl[dpi->bit] = NULL;
+	memset(dpi, 0, sizeof(*dpi));
+fail:
+	mutex_unlock(&res->dpi_tbl_lock);
+	return rc;
+}
+
+static void bnxt_qplib_free_dpi_tbl(struct bnxt_qplib_dpi_tbl *dpit)
+{
+	kfree(dpit->tbl);
+	kfree(dpit->app_tbl);
+	dpit->tbl = NULL;
+	dpit->app_tbl = NULL;
+	dpit->max = 0;
+}
+
+static int bnxt_qplib_alloc_dpi_tbl(struct bnxt_qplib_res *res,
+				    struct bnxt_qplib_dev_attr *dev_attr,
+				    u8 pppp_factor)
+{
+	struct bnxt_qplib_dpi_tbl *dpit;
+	struct bnxt_qplib_reg_desc *reg;
+	unsigned long bar_len;
+	u32 dbr_offset;
+	u32 bytes;
+
+	dpit = &res->dpi_tbl;
+	reg = &dpit->wcreg;
+
+	if (!_is_chip_gen_p5_p7(res->cctx)) {
+		/* Offest should come from L2 driver */
+		dbr_offset = dev_attr->l2_db_size;
+		dpit->ucreg.offset = dbr_offset;
+		dpit->wcreg.offset = dbr_offset;
+	}
+
+	bar_len = pci_resource_len(res->pdev, reg->bar_id);
+	dpit->max = (bar_len - reg->offset) / PAGE_SIZE;
+	if (dev_attr->max_dpi)
+		dpit->max = min_t(u32, dpit->max, dev_attr->max_dpi);
+
+	dpit->app_tbl = kzalloc(dpit->max * sizeof(void*), GFP_KERNEL);
+	if (!dpit->app_tbl) {
+		dev_err(&res->pdev->dev,
+			"QPLIB: DPI app tbl allocation failed");
+		return -ENOMEM;
+	}
+
+	bytes = dpit->max >> 3;
+	if (!bytes)
+		bytes = 1;
+
+	dpit->tbl = kmalloc(bytes, GFP_KERNEL);
+	if (!dpit->tbl) {
+		kfree(dpit->app_tbl);
+		dev_err(&res->pdev->dev,
+			"QPLIB: DPI tbl allocation failed for size = %d\n",
+			bytes);
+		return -ENOMEM;
+	}
+
+	memset((u8 *)dpit->tbl, 0xFF, bytes);
+	/*
+	 * On SR2, 2nd doorbell page of each function
+	 * is reserved for L2 PPP. Now that the tbl is
+	 * initialized, mark it as unavailable. By default
+	 * RoCE can make use of the 512 extended pages for
+	 * PPP.
+	 */
+	if (_is_chip_p7(res->cctx)) {
+		clear_bit(1, dpit->tbl);
+		if (pppp_factor)
+			dpit->avail_ppp =
+				BNXT_QPLIB_MAX_EXTENDED_PPP_PAGES / pppp_factor;
+	}
+	mutex_init(&res->dpi_tbl_lock);
+	dpit->priv_db = dpit->ucreg.bar_reg + dpit->ucreg.offset;
+
+	return 0;
+}
+
+/* Stats */
+void bnxt_qplib_free_stat_mem(struct bnxt_qplib_res *res,
+			      struct bnxt_qplib_stats *stats)
+{
+	struct pci_dev *pdev;
+
+	pdev = res->pdev;
+	if (stats->dma)
+		dma_free_coherent(&pdev->dev, stats->size,
+				  stats->dma, stats->dma_map);
+
+	memset(stats, 0, sizeof(*stats));
+	stats->fw_id = -1;
+}
+
+int bnxt_qplib_alloc_stat_mem(struct pci_dev *pdev,
+			      struct bnxt_qplib_chip_ctx *cctx,
+			      struct bnxt_qplib_stats *stats)
+{
+	cctx->hw_stats_size = 168;
+
+	memset(stats, 0, sizeof(*stats));
+	stats->fw_id = -1;
+	stats->size = cctx->hw_stats_size;
+	stats->dma = dma_alloc_coherent(&pdev->dev, stats->size,
+					&stats->dma_map, GFP_KERNEL);
+	if (!stats->dma) {
+		dev_err(&pdev->dev, "QPLIB: Stats DMA allocation failed");
+		return -ENOMEM;
+	}
+	return 0;
+}
+
+/* Resource */
+int bnxt_qplib_stop_res(struct bnxt_qplib_res *res)
+{
+	struct bnxt_qplib_rcfw *rcfw = res->rcfw;
+	struct creq_stop_func_resp resp = {};
+	struct bnxt_qplib_cmdqmsg msg = {};
+	struct cmdq_stop_func req = {};
+	int rc;
+
+	bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_STOP_FUNC,
+				 sizeof(req));
+	bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL, sizeof(req),
+				sizeof(resp), 0);
+	rc = bnxt_qplib_rcfw_send_message(rcfw, &msg);
+	return rc;
+}
+
+void bnxt_qplib_clear_tbls(struct bnxt_qplib_res *res)
+{
+	bnxt_qplib_cleanup_sgid_tbl(res, &res->sgid_tbl);
+}
+
+int bnxt_qplib_init_tbls(struct bnxt_qplib_res *res)
+{
+	bnxt_qplib_init_sgid_tbl(&res->sgid_tbl, res->netdev);
+
+	return 0;
+}
+
+void bnxt_qplib_free_tbls(struct bnxt_qplib_res *res)
+{
+	bnxt_qplib_free_sgid_tbl(res);
+	bnxt_qplib_free_pd_tbl(&res->pd_tbl);
+	bnxt_qplib_free_dpi_tbl(&res->dpi_tbl);
+	bnxt_qplib_free_reftbls(res);
+}
+
+int bnxt_qplib_alloc_tbls(struct bnxt_qplib_res *res, u8 pppp_factor)
+{
+	struct bnxt_qplib_dev_attr *dev_attr;
+	int rc = 0;
+
+	dev_attr = res->dattr;
+
+	rc = bnxt_qplib_alloc_reftbls(res, dev_attr);
+	if (rc)
+		goto fail;
+
+	rc = bnxt_qplib_alloc_sgid_tbl(res, dev_attr->max_sgid);
+	if (rc)
+		goto fail;
+
+	rc = bnxt_qplib_alloc_pd_tbl(res, dev_attr->max_pd);
+	if (rc)
+		goto fail;
+
+	rc = bnxt_qplib_alloc_dpi_tbl(res, dev_attr, pppp_factor);
+	if (rc)
+		goto fail;
+
+	return 0;
+fail:
+	bnxt_qplib_free_tbls(res);
+	return rc;
+}
+
+void bnxt_qplib_unmap_db_bar(struct bnxt_qplib_res *res)
+{
+	struct bnxt_qplib_reg_desc *reg;
+
+	reg = &res->dpi_tbl.ucreg;
+	if (reg->bar_reg)
+		pci_iounmap(res->pdev, reg->bar_reg);
+	reg->bar_reg = NULL;
+	reg->bar_base = 0;
+	reg->len = 0;
+	reg->bar_id = 0; /* Zero? or ff */
+}
+
+int bnxt_qplib_map_db_bar(struct bnxt_qplib_res *res)
+{
+	struct bnxt_qplib_reg_desc *ucreg;
+	struct bnxt_qplib_reg_desc *wcreg;
+
+	wcreg = &res->dpi_tbl.wcreg;
+	wcreg->bar_id = RCFW_DBR_PCI_BAR_REGION;
+	if (!res || !res->pdev || !wcreg)
+		return -1;
+	wcreg->bar_base = pci_resource_start(res->pdev, wcreg->bar_id);
+	/* No need to set the wcreg->len here */
+
+	ucreg = &res->dpi_tbl.ucreg;
+	ucreg->bar_id = RCFW_DBR_PCI_BAR_REGION;
+	ucreg->bar_base = pci_resource_start(res->pdev, ucreg->bar_id);
+
+	ucreg->offset = 65536;
+
+	ucreg->len = ucreg->offset + PAGE_SIZE;
+
+	if (!ucreg->len || ((ucreg->len & (PAGE_SIZE - 1)) != 0)) {
+		dev_err(&res->pdev->dev, "QPLIB: invalid dbr length %d\n",
+			(int)ucreg->len);
+		return -EINVAL;
+	}
+	ucreg->bar_reg = ioremap(ucreg->bar_base, ucreg->len);
+	if (!ucreg->bar_reg) {
+		dev_err(&res->pdev->dev, "priviledged dpi map failed!\n");
+		return -ENOMEM;
+	}
+
+	return 0;
+}
+
+/**
+ * pci_enable_atomic_ops_to_root - enable AtomicOp requests to root port
+ * @dev: the PCI device
+ * @cap_mask: mask of desired AtomicOp sizes, including one or more of:
+ *	PCI_EXP_DEVCAP2_ATOMIC_COMP32
+ *	PCI_EXP_DEVCAP2_ATOMIC_COMP64
+ *	PCI_EXP_DEVCAP2_ATOMIC_COMP128
+ *
+ * Return 0 if all upstream bridges support AtomicOp routing, egress
+ * blocking is disabled on all upstream ports, and the root port supports
+ * the requested completion capabilities (32-bit, 64-bit and/or 128-bit
+ * AtomicOp completion), or negative otherwise.
+ */
+int pci_enable_atomic_ops_to_root(struct pci_dev *dev, u32 cap_mask)
+{
+	struct pci_bus *bus = dev->bus;
+	struct pci_dev *bridge;
+	u32 cap;
+
+	if (!pci_is_pcie(dev))
+		return -EINVAL;
+
+	/*
+	 * Per PCIe r4.0, sec 6.15, endpoints and root ports may be
+	 * AtomicOp requesters.  For now, we only support endpoints as
+	 * requesters and root ports as completers.  No endpoints as
+	 * completers, and no peer-to-peer.
+	 */
+
+	switch (pci_pcie_type(dev)) {
+	case PCI_EXP_TYPE_ENDPOINT:
+	case PCI_EXP_TYPE_LEG_END:
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	bridge = bus->self;
+
+	pcie_capability_read_dword(bridge, PCI_EXP_DEVCAP2, &cap);
+
+	switch (pci_pcie_type(bridge)) {
+	case PCI_EXP_TYPE_DOWNSTREAM:
+		if (!(cap & PCI_EXP_DEVCAP2_ATOMIC_ROUTE))
+			return -EINVAL;
+		break;
+
+	/* Ensure root port supports all the sizes we care about */
+	case PCI_EXP_TYPE_ROOT_PORT:
+		if ((cap & cap_mask) != cap_mask)
+			return -EINVAL;
+		break;
+	}
+	return 0;
+}
+
+int bnxt_qplib_enable_atomic_ops_to_root(struct pci_dev *dev)
+{
+	u16 ctl2;
+
+	if(pci_enable_atomic_ops_to_root(dev, PCI_EXP_DEVCAP2_ATOMIC_COMP32) &&
+	   pci_enable_atomic_ops_to_root(dev, PCI_EXP_DEVCAP2_ATOMIC_COMP64))
+		return -EOPNOTSUPP;
+
+	pcie_capability_read_word(dev, PCI_EXP_DEVCTL2, &ctl2);
+	return !(ctl2 & PCI_EXP_DEVCTL2_ATOMIC_REQ);
+}
diff --git a/sys/dev/bnxt/bnxt_re/qplib_res.h b/sys/dev/bnxt/bnxt_re/qplib_res.h
new file mode 100644
index 000000000000..6468207a49aa
--- /dev/null
+++ b/sys/dev/bnxt/bnxt_re/qplib_res.h
@@ -0,0 +1,840 @@
+/*
+ * Copyright (c) 2015-2024, Broadcom. All rights reserved.  The term
+ * Broadcom refers to Broadcom Limited and/or its subsidiaries.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS''
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
+ * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
+ * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * Description: QPLib resource manager (header)
+ */
+
+#ifndef __BNXT_QPLIB_RES_H__
+#define __BNXT_QPLIB_RES_H__
+
+#include "hsi_struct_def.h"
+
+extern const struct bnxt_qplib_gid bnxt_qplib_gid_zero;
+
+#define CHIP_NUM_57508		0x1750
+#define CHIP_NUM_57504		0x1751
+#define CHIP_NUM_57502		0x1752
+#define CHIP_NUM_58818          0xd818
+#define CHIP_NUM_57608		0x1760
+
+#define BNXT_QPLIB_MAX_QPC_COUNT	(64 * 1024)
+#define BNXT_QPLIB_MAX_SRQC_COUNT	(64 * 1024)
+#define BNXT_QPLIB_MAX_CQ_COUNT		(64 * 1024)
+#define BNXT_QPLIB_MAX_CQ_COUNT_P5	(128 * 1024)
+
+#define BNXT_QPLIB_DBR_VALID (0x1UL << 26)
+#define BNXT_QPLIB_DBR_EPOCH_SHIFT   24
+#define BNXT_QPLIB_DBR_TOGGLE_SHIFT  25
+
+#define BNXT_QPLIB_DBR_PF_DB_OFFSET	0x10000
+#define BNXT_QPLIB_DBR_VF_DB_OFFSET	0x4000
+
+#define BNXT_QPLIB_DBR_KEY_INVALID	-1
+
+/* chip gen type */
+#define BNXT_RE_DEFAULT 0xf
+
+enum bnxt_qplib_wqe_mode {
+	BNXT_QPLIB_WQE_MODE_STATIC	= 0x00,
+	BNXT_QPLIB_WQE_MODE_VARIABLE	= 0x01,
+	BNXT_QPLIB_WQE_MODE_INVALID	= 0x02
+};
+
+#define BNXT_RE_PUSH_MODE_NONE	0
+#define BNXT_RE_PUSH_MODE_WCB	1
+#define BNXT_RE_PUSH_MODE_PPP	2
+#define BNXT_RE_PUSH_ENABLED(mode) ((mode) == BNXT_RE_PUSH_MODE_WCB ||\
+				    (mode) == BNXT_RE_PUSH_MODE_PPP)
+#define BNXT_RE_PPP_ENABLED(cctx) ((cctx)->modes.db_push_mode ==\
+				   BNXT_RE_PUSH_MODE_PPP)
+#define	PCI_EXP_DEVCAP2_ATOMIC_ROUTE	0x00000040 /* Atomic Op routing */
+#define	PCI_EXP_DEVCAP2_ATOMIC_COMP32	0x00000080 /* 32b AtomicOp completion */
+#define	PCI_EXP_DEVCAP2_ATOMIC_COMP64	0x00000100 /* 64b AtomicOp completion */
+#define	PCI_EXP_DEVCTL2_ATOMIC_EGRESS_BLOCK 0x0080 /* Block atomic egress */
+#define PCI_EXP_DEVCTL2_ATOMIC_REQ	0x0040	/* Set Atomic requests */
+
+int pci_enable_atomic_ops_to_root(struct pci_dev *dev, u32 cap_mask);
+
+struct bnxt_qplib_drv_modes {
+	u8	wqe_mode;
+	u8	te_bypass;
+	u8	db_push;
+	/* To control advanced cc params display in configfs */
+	u8	cc_pr_mode;
+	/* Other modes to follow here e.g. GSI QP mode */
+	u8	dbr_pacing;
+	u8	dbr_pacing_ext;
+	u8	dbr_drop_recov;
+	u8	dbr_primary_pf;
+	u8	dbr_pacing_v0;
+};
+
+struct bnxt_qplib_chip_ctx {
+	u16     chip_num;
+	u8      chip_rev;
+	u8      chip_metal;
+	u64	hwrm_intf_ver;
+	struct bnxt_qplib_drv_modes	modes;
+	u32	dbr_stat_db_fifo;
+	u32	dbr_aeq_arm_reg;
+	u32	dbr_throttling_reg;
+	u16	hw_stats_size;
+	u16	hwrm_cmd_max_timeout;
+};
+
+static inline bool _is_chip_num_p7(u16 chip_num)
+{
+	return (chip_num == CHIP_NUM_58818 ||
+		chip_num == CHIP_NUM_57608);
+}
+
+static inline bool _is_chip_p7(struct bnxt_qplib_chip_ctx *cctx)
+{
+	return _is_chip_num_p7(cctx->chip_num);
+}
+
+/* SR2 is Gen P5 */
+static inline bool _is_chip_gen_p5(struct bnxt_qplib_chip_ctx *cctx)
+{
+	return (cctx->chip_num == CHIP_NUM_57508 ||
+		cctx->chip_num == CHIP_NUM_57504 ||
+		cctx->chip_num == CHIP_NUM_57502);
+}
+
+static inline bool _is_chip_gen_p5_p7(struct bnxt_qplib_chip_ctx *cctx)
+{
+	return (_is_chip_gen_p5(cctx) || _is_chip_p7(cctx));
+}
+
+static inline bool _is_wqe_mode_variable(struct bnxt_qplib_chip_ctx *cctx)
+{
+	return cctx->modes.wqe_mode == BNXT_QPLIB_WQE_MODE_VARIABLE;
+}
+
+struct bnxt_qplib_db_pacing_data {
+	u32 do_pacing;
+	u32 pacing_th;
+	u32 dev_err_state;
+	u32 alarm_th;
+	u32 grc_reg_offset;
+	u32 fifo_max_depth;
+	u32 fifo_room_mask;
+	u8  fifo_room_shift;
+};
+
+static inline u8 bnxt_qplib_dbr_pacing_en(struct bnxt_qplib_chip_ctx *cctx)
+{
+	return cctx->modes.dbr_pacing;
+}
+
+static inline u8 bnxt_qplib_dbr_pacing_ext_en(struct bnxt_qplib_chip_ctx *cctx)
+{
+	return cctx->modes.dbr_pacing_ext;
+}
+
+static inline u8 bnxt_qplib_dbr_pacing_is_primary_pf(struct bnxt_qplib_chip_ctx *cctx)
+{
+	return cctx->modes.dbr_primary_pf;
+}
+
+static inline void bnxt_qplib_dbr_pacing_set_primary_pf
+		(struct bnxt_qplib_chip_ctx *cctx, u8 val)
+{
+	cctx->modes.dbr_primary_pf = val;
+}
+
+/* Defines for handling the HWRM version check */
+#define HWRM_VERSION_DEV_ATTR_MAX_DPI	0x1000A0000000D
+#define HWRM_VERSION_ROCE_STATS_FN_ID	0x1000A00000045
+
+#define PTR_CNT_PER_PG		(PAGE_SIZE / sizeof(void *))
+#define PTR_MAX_IDX_PER_PG	(PTR_CNT_PER_PG - 1)
+#define PTR_PG(x)		(((x) & ~PTR_MAX_IDX_PER_PG) / PTR_CNT_PER_PG)
+#define PTR_IDX(x)		((x) & PTR_MAX_IDX_PER_PG)
+
+#define HWQ_CMP(idx, hwq)	((idx) & ((hwq)->max_elements - 1))
+#define HWQ_FREE_SLOTS(hwq)	(hwq->max_elements - \
+				((HWQ_CMP(hwq->prod, hwq)\
+				- HWQ_CMP(hwq->cons, hwq))\
+				& (hwq->max_elements - 1)))
+enum bnxt_qplib_hwq_type {
+	HWQ_TYPE_CTX,
+	HWQ_TYPE_QUEUE,
+	HWQ_TYPE_L2_CMPL,
+	HWQ_TYPE_MR
+};
+
+#define MAX_PBL_LVL_0_PGS		1
+#define MAX_PBL_LVL_1_PGS		512
+#define MAX_PBL_LVL_1_PGS_SHIFT		9
+#define MAX_PDL_LVL_SHIFT		9
+
+enum bnxt_qplib_pbl_lvl {
+	PBL_LVL_0,
+	PBL_LVL_1,
+	PBL_LVL_2,
+	PBL_LVL_MAX
+};
+
+#define ROCE_PG_SIZE_4K		(4 * 1024)
+#define ROCE_PG_SIZE_8K		(8 * 1024)
+#define ROCE_PG_SIZE_64K	(64 * 1024)
+#define ROCE_PG_SIZE_2M		(2 * 1024 * 1024)
+#define ROCE_PG_SIZE_8M		(8 * 1024 * 1024)
+#define ROCE_PG_SIZE_1G		(1024 * 1024 * 1024)
+enum bnxt_qplib_hwrm_pg_size {
+	BNXT_QPLIB_HWRM_PG_SIZE_4K	= 0,
+	BNXT_QPLIB_HWRM_PG_SIZE_8K	= 1,
+	BNXT_QPLIB_HWRM_PG_SIZE_64K	= 2,
+	BNXT_QPLIB_HWRM_PG_SIZE_2M	= 3,
+	BNXT_QPLIB_HWRM_PG_SIZE_8M	= 4,
+	BNXT_QPLIB_HWRM_PG_SIZE_1G	= 5,
+};
+
+struct bnxt_qplib_reg_desc {
+	u8		bar_id;
+	resource_size_t	bar_base;
+	unsigned long	offset;
+	void __iomem	*bar_reg;
+	size_t		len;
+};
+
+struct bnxt_qplib_pbl {
+	u32				pg_count;
+	u32				pg_size;
+	void				**pg_arr;
+	dma_addr_t			*pg_map_arr;
+};
+
+struct bnxt_qplib_sg_info {
+	struct scatterlist              *sghead;
+	u32                             nmap;
+	u32                             npages;
+	u32				pgshft;
+	u32				pgsize;
+	bool				nopte;
+};
+
+struct bnxt_qplib_hwq_attr {
+	struct bnxt_qplib_res		*res;
+	struct bnxt_qplib_sg_info	*sginfo;
+	enum bnxt_qplib_hwq_type	type;
+	u32				depth;
+	u32				stride;
+	u32				aux_stride;
+	u32				aux_depth;
+};
+
+struct bnxt_qplib_hwq {
+	struct pci_dev			*pdev;
+	spinlock_t			lock;
+	struct bnxt_qplib_pbl		pbl[PBL_LVL_MAX];
+	enum bnxt_qplib_pbl_lvl		level;		/* 0, 1, or 2 */
+	void				**pbl_ptr;	/* ptr for easy access
+							   to the PBL entries */
+	dma_addr_t			*pbl_dma_ptr;	/* ptr for easy access
+							   to the dma_addr */
+	u32				max_elements;
+	u32				depth;	/* original requested depth */
+	u16				element_size;	/* Size of each entry */
+	u16				qe_ppg;		/* queue entry per page */
+
+	u32				prod;		/* raw */
+	u32				cons;		/* raw */
+	u8				cp_bit;
+	u8				is_user;
+	u64				*pad_pg;
+	u32				pad_stride;
+	u32				pad_pgofft;
+};
+
+struct bnxt_qplib_db_info {
+	void __iomem		*db;
+	void __iomem		*priv_db;
+	struct bnxt_qplib_hwq	*hwq;
+	struct bnxt_qplib_res   *res;
+	u32			xid;
+	u32			max_slot;
+	u32			flags;
+	u8			toggle;
+	spinlock_t		lock;
+	u64			shadow_key;
+	u64			shadow_key_arm_ena;
+	u32			seed; /* For DB pacing */
+};
+
+enum bnxt_qplib_db_info_flags_mask {
+	BNXT_QPLIB_FLAG_EPOCH_CONS_SHIFT	= 0x0UL,
+	BNXT_QPLIB_FLAG_EPOCH_PROD_SHIFT	= 0x1UL,
+	BNXT_QPLIB_FLAG_EPOCH_CONS_MASK		= 0x1UL,
+	BNXT_QPLIB_FLAG_EPOCH_PROD_MASK		= 0x2UL,
+};
+
+enum bnxt_qplib_db_epoch_flag_shift {
+	BNXT_QPLIB_DB_EPOCH_CONS_SHIFT	= BNXT_QPLIB_DBR_EPOCH_SHIFT,
+	BNXT_QPLIB_DB_EPOCH_PROD_SHIFT	= (BNXT_QPLIB_DBR_EPOCH_SHIFT - 1)
+};
+
+/* Tables */
+struct bnxt_qplib_pd_tbl {
+	unsigned long			*tbl;
+	u32				max;
+};
+
+struct bnxt_qplib_sgid_tbl {
+	struct bnxt_qplib_gid_info	*tbl;
+	u16				*hw_id;
+	u16				max;
+	u16				active;
+	void				*ctx;
+	bool                            *vlan;
+};
+
+enum {
+	BNXT_QPLIB_DPI_TYPE_KERNEL	= 0,
+	BNXT_QPLIB_DPI_TYPE_UC		= 1,
+	BNXT_QPLIB_DPI_TYPE_WC		= 2
+};
+
+struct bnxt_qplib_dpi {
+	u32				dpi;
+	u32				bit;
+	void __iomem			*dbr;
+	u64				umdbr;
+	u8				type;
+};
+
+#define BNXT_QPLIB_MAX_EXTENDED_PPP_PAGES	512
+struct bnxt_qplib_dpi_tbl {
+	void				**app_tbl;
+	unsigned long			*tbl;
+	u16				max;
+	u16				avail_ppp;
+	struct bnxt_qplib_reg_desc	ucreg; /* Hold entire DB bar. */
+	struct bnxt_qplib_reg_desc	wcreg;
+	void __iomem			*priv_db;
+};
+
+struct bnxt_qplib_stats {
+	dma_addr_t			dma_map;
+	void				*dma;
+	u32				size;
+	u32				fw_id;
+};
+
+struct bnxt_qplib_vf_res {
+	u32 max_qp;
+	u32 max_mrw;
+	u32 max_srq;
+	u32 max_cq;
+	u32 max_gid;
+};
+
+#define BNXT_QPLIB_MAX_QP_CTX_ENTRY_SIZE	448
+#define BNXT_QPLIB_MAX_SRQ_CTX_ENTRY_SIZE	64
+#define BNXT_QPLIB_MAX_CQ_CTX_ENTRY_SIZE	64
+#define BNXT_QPLIB_MAX_MRW_CTX_ENTRY_SIZE	128
+
+#define MAX_TQM_ALLOC_REQ		48
+#define MAX_TQM_ALLOC_BLK_SIZE		8
+struct bnxt_qplib_tqm_ctx {
+	struct bnxt_qplib_hwq		pde;
+	enum bnxt_qplib_pbl_lvl		pde_level; /* Original level */
+	struct bnxt_qplib_hwq		qtbl[MAX_TQM_ALLOC_REQ];
+	u8				qcount[MAX_TQM_ALLOC_REQ];
+};
+
+struct bnxt_qplib_hctx {
+	struct bnxt_qplib_hwq	hwq;
+	u32			max;
+};
+
+struct bnxt_qplib_refrec {
+	void *handle;
+	u32 xid;
+};
+
+struct bnxt_qplib_reftbl {
+	struct bnxt_qplib_refrec *rec;
+	u32 max;
+	spinlock_t lock; /* reftbl lock */
+};
+
+struct bnxt_qplib_reftbls {
+	struct bnxt_qplib_reftbl qpref;
+	struct bnxt_qplib_reftbl cqref;
+	struct bnxt_qplib_reftbl srqref;
+};
+
+#define GET_TBL_INDEX(id, tbl) ((id) % (((tbl)->max) - 1))
+static inline u32 map_qp_id_to_tbl_indx(u32 qid, struct bnxt_qplib_reftbl *tbl)
+{
+	return (qid == 1) ? tbl->max : GET_TBL_INDEX(qid, tbl);
+}
+
+/*
+ * This structure includes the number of various roce resource table sizes
+ * actually allocated by the driver. May be less than the maximums the firmware
+ * allows if the driver imposes lower limits than the firmware.
+ */
+struct bnxt_qplib_ctx {
+	struct bnxt_qplib_hctx		qp_ctx;
+	struct bnxt_qplib_hctx		mrw_ctx;
+	struct bnxt_qplib_hctx		srq_ctx;
+	struct bnxt_qplib_hctx		cq_ctx;
+	struct bnxt_qplib_hctx		tim_ctx;
+	struct bnxt_qplib_tqm_ctx	tqm_ctx;
+
+	struct bnxt_qplib_stats		stats;
+	struct bnxt_qplib_stats		stats2;
+	struct bnxt_qplib_vf_res	vf_res;
+};
+
+struct bnxt_qplib_res {
+	struct pci_dev			*pdev;
+	struct bnxt_qplib_chip_ctx	*cctx;
+	struct bnxt_qplib_dev_attr      *dattr;
+	struct bnxt_qplib_ctx		*hctx;
+	struct ifnet			*netdev;
+	struct bnxt_en_dev		*en_dev;
+
+	struct bnxt_qplib_rcfw		*rcfw;
+
+	struct bnxt_qplib_pd_tbl	pd_tbl;
+	struct mutex			pd_tbl_lock;
+	struct bnxt_qplib_sgid_tbl	sgid_tbl;
+	struct bnxt_qplib_dpi_tbl	dpi_tbl;
+	struct mutex			dpi_tbl_lock;
+	struct bnxt_qplib_reftbls	reftbl;
+	bool				prio;
+	bool				is_vf;
+	struct bnxt_qplib_db_pacing_data *pacing_data;
+};
+
+struct bnxt_qplib_query_stats_info {
+	u32 function_id;
+	u8 collection_id;
+	bool vf_valid;
+};
+
+struct bnxt_qplib_query_qp_info {
+	u32 function_id;
+	u32 num_qps;
+	u32 start_index;
+	bool vf_valid;
+};
+
+struct bnxt_qplib_query_fn_info {
+	bool vf_valid;
+	u32 host;
+	u32 filter;
+};
+
+
+#define to_bnxt_qplib(ptr, type, member)	\
+	container_of(ptr, type, member)
+
+struct bnxt_qplib_pd;
+struct bnxt_qplib_dev_attr;
+
+bool _is_chip_gen_p5(struct bnxt_qplib_chip_ctx *cctx);
+bool _is_chip_gen_p5_p7(struct bnxt_qplib_chip_ctx *cctx);
+bool _is_chip_a0(struct bnxt_qplib_chip_ctx *cctx);
+bool _is_chip_p7(struct bnxt_qplib_chip_ctx *cctx);
+bool _is_alloc_mr_unified(struct bnxt_qplib_dev_attr *dattr);
+void bnxt_qplib_free_hwq(struct bnxt_qplib_res *res,
+			 struct bnxt_qplib_hwq *hwq);
+int bnxt_qplib_alloc_init_hwq(struct bnxt_qplib_hwq *hwq,
+			      struct bnxt_qplib_hwq_attr *hwq_attr);
+void bnxt_qplib_get_guid(const u8 *dev_addr, u8 *guid);
+int bnxt_qplib_alloc_pd(struct bnxt_qplib_res *res,
+			struct bnxt_qplib_pd *pd);
+int bnxt_qplib_dealloc_pd(struct bnxt_qplib_res *res,
+			  struct bnxt_qplib_pd_tbl *pd_tbl,
+			  struct bnxt_qplib_pd *pd);
+int bnxt_qplib_alloc_dpi(struct bnxt_qplib_res *res,
+			 struct bnxt_qplib_dpi *dpi,
+			 void *app, u8 type);
+int bnxt_qplib_dealloc_dpi(struct bnxt_qplib_res *res,
+			   struct bnxt_qplib_dpi *dpi);
+int bnxt_qplib_stop_res(struct bnxt_qplib_res *res);
+void bnxt_qplib_clear_tbls(struct bnxt_qplib_res *res);
+int bnxt_qplib_init_tbls(struct bnxt_qplib_res *res);
+void bnxt_qplib_free_tbls(struct bnxt_qplib_res *res);
+int bnxt_qplib_alloc_tbls(struct bnxt_qplib_res *res, u8 pppp_factor);
+void bnxt_qplib_free_hwctx(struct bnxt_qplib_res *res);
+int bnxt_qplib_alloc_hwctx(struct bnxt_qplib_res *res);
+int bnxt_qplib_alloc_stat_mem(struct pci_dev *pdev,
+			      struct bnxt_qplib_chip_ctx *cctx,
+			      struct bnxt_qplib_stats *stats);
+void bnxt_qplib_free_stat_mem(struct bnxt_qplib_res *res,
+			      struct bnxt_qplib_stats *stats);
+
+int bnxt_qplib_map_db_bar(struct bnxt_qplib_res *res);
+void bnxt_qplib_unmap_db_bar(struct bnxt_qplib_res *res);
+int bnxt_qplib_enable_atomic_ops_to_root(struct pci_dev *dev);
+u8 _get_chip_gen_p5_type(struct bnxt_qplib_chip_ctx *cctx);
+
+static inline void *bnxt_qplib_get_qe(struct bnxt_qplib_hwq *hwq,
+				      u32 indx, u64 *pg)
+{
+	u32 pg_num, pg_idx;
+
+	pg_num = (indx / hwq->qe_ppg);
+	pg_idx = (indx % hwq->qe_ppg);
+	if (pg)
+		*pg = (u64)&hwq->pbl_ptr[pg_num];
+	return (void *)((u8 *)hwq->pbl_ptr[pg_num] + hwq->element_size * pg_idx);
+}
+
+static inline void bnxt_qplib_hwq_incr_prod(struct bnxt_qplib_db_info *dbinfo,
+					    struct bnxt_qplib_hwq *hwq, u32 cnt)
+{
+	/* move prod and update toggle/epoch if wrap around */
+	hwq->prod += cnt;
+	if (hwq->prod >= hwq->depth) {
+		hwq->prod %= hwq->depth;
+		dbinfo->flags ^= 1UL << BNXT_QPLIB_FLAG_EPOCH_PROD_SHIFT;
+	}
+}
+
+static inline void bnxt_qplib_hwq_incr_cons(u32 max_elements, u32 *cons,
+					    u32 cnt, u32 *dbinfo_flags)
+{
+	/* move cons and update toggle/epoch if wrap around */
+	*cons += cnt;
+	if (*cons >= max_elements) {
+		*cons %= max_elements;
+		*dbinfo_flags ^= 1UL << BNXT_QPLIB_FLAG_EPOCH_CONS_SHIFT;
+	}
+}
+
+static inline u8 _get_pte_pg_size(struct bnxt_qplib_hwq *hwq)
+{
+	u8 pg_size = BNXT_QPLIB_HWRM_PG_SIZE_4K;
+	struct bnxt_qplib_pbl *pbl;
+
+	pbl = &hwq->pbl[hwq->level];
+	switch (pbl->pg_size) {
+		case ROCE_PG_SIZE_4K: pg_size = BNXT_QPLIB_HWRM_PG_SIZE_4K;
+		break;
+		case ROCE_PG_SIZE_8K: pg_size = BNXT_QPLIB_HWRM_PG_SIZE_8K;
+		break;
+		case ROCE_PG_SIZE_64K: pg_size = BNXT_QPLIB_HWRM_PG_SIZE_64K;
+		break;
+		case ROCE_PG_SIZE_2M: pg_size = BNXT_QPLIB_HWRM_PG_SIZE_2M;
+		break;
+		case ROCE_PG_SIZE_8M: pg_size = BNXT_QPLIB_HWRM_PG_SIZE_8M;
+		break;
+		case ROCE_PG_SIZE_1G: pg_size = BNXT_QPLIB_HWRM_PG_SIZE_1G;
+		break;
+		default:
+		break;
+	}
+	return pg_size;
+}
+
+static inline u64 _get_base_addr(struct bnxt_qplib_hwq *hwq)
+{
+	return hwq->pbl[PBL_LVL_0].pg_map_arr[0];
+}
+
+static inline u8 _get_base_pg_size(struct bnxt_qplib_hwq *hwq)
+{
+	u8 pg_size = BNXT_QPLIB_HWRM_PG_SIZE_4K;
+	struct bnxt_qplib_pbl *pbl;
+
+	pbl = &hwq->pbl[PBL_LVL_0];
+	switch (pbl->pg_size) {
+		case ROCE_PG_SIZE_4K: pg_size = BNXT_QPLIB_HWRM_PG_SIZE_4K;
+		break;
+		case ROCE_PG_SIZE_8K: pg_size = BNXT_QPLIB_HWRM_PG_SIZE_8K;
+		break;
+		case ROCE_PG_SIZE_64K: pg_size = BNXT_QPLIB_HWRM_PG_SIZE_64K;
+		break;
+		case ROCE_PG_SIZE_2M: pg_size = BNXT_QPLIB_HWRM_PG_SIZE_2M;
+		break;
+		case ROCE_PG_SIZE_8M: pg_size = BNXT_QPLIB_HWRM_PG_SIZE_8M;
+		break;
+		case ROCE_PG_SIZE_1G: pg_size = BNXT_QPLIB_HWRM_PG_SIZE_1G;
+		break;
+		default:
+		break;
+	}
+	return pg_size;
+}
+
+static inline enum bnxt_qplib_hwq_type _get_hwq_type(struct bnxt_qplib_res *res)
+{
+	return _is_chip_gen_p5_p7(res->cctx) ? HWQ_TYPE_QUEUE : HWQ_TYPE_L2_CMPL;
+}
+
+static inline bool _is_ext_stats_supported(u16 dev_cap_flags)
+{
+	return dev_cap_flags &
+		CREQ_QUERY_FUNC_RESP_SB_EXT_STATS;
+}
+
+static inline int bnxt_ext_stats_supported(struct bnxt_qplib_chip_ctx *ctx,
+					   u16 flags, bool virtfn)
+{
+	return (_is_ext_stats_supported(flags) &&
+		((virtfn && _is_chip_p7(ctx)) || (!virtfn)));
+}
+
+static inline bool _is_hw_retx_supported(u16 dev_cap_flags)
+{
+	return dev_cap_flags &
+		(CREQ_QUERY_FUNC_RESP_SB_HW_REQUESTER_RETX_ENABLED |
+		 CREQ_QUERY_FUNC_RESP_SB_HW_RESPONDER_RETX_ENABLED);
+}
+
+/* Disable HW_RETX */
+#define BNXT_RE_HW_RETX(a) _is_hw_retx_supported((a))
+
+static inline bool _is_cqe_v2_supported(u16 dev_cap_flags)
+{
+	return dev_cap_flags &
+		CREQ_QUERY_FUNC_RESP_SB_CQE_V2;
+}
+
+#define BNXT_DB_FIFO_ROOM_MASK      0x1fff8000
+#define BNXT_DB_FIFO_ROOM_SHIFT     15
+#define BNXT_MAX_FIFO_DEPTH         0x2c00
+
+#define BNXT_DB_PACING_ALGO_THRESHOLD	250
+#define BNXT_DEFAULT_PACING_PROBABILITY 0xFFFF
+
+#define BNXT_DBR_PACING_WIN_BASE	0x2000
+#define BNXT_DBR_PACING_WIN_MAP_OFF	4
+#define BNXT_DBR_PACING_WIN_OFF(reg)	(BNXT_DBR_PACING_WIN_BASE +	\
+
+static inline void bnxt_qplib_ring_db32(struct bnxt_qplib_db_info *info,
+					bool arm)
+{
+	u32 key = 0;
+
+	key = info->hwq->cons | (CMPL_DOORBELL_IDX_VALID |
+		(CMPL_DOORBELL_KEY_CMPL & CMPL_DOORBELL_KEY_MASK));
+	if (!arm)
+		key |= CMPL_DOORBELL_MASK;
+	/* memory barrier */
+	wmb();
+	writel(key, info->db);
+}
+
+#define BNXT_QPLIB_INIT_DBHDR(xid, type, indx, toggle)			\
+	(((u64)(((xid) & DBC_DBC_XID_MASK) | DBC_DBC_PATH_ROCE |	\
+	    (type) | BNXT_QPLIB_DBR_VALID) << 32) | (indx) |		\
+	    ((toggle) << (BNXT_QPLIB_DBR_TOGGLE_SHIFT)))
+
+static inline void bnxt_qplib_write_db(struct bnxt_qplib_db_info *info,
+				       u64 key, void __iomem *db,
+				       u64 *shadow_key)
+{
+	unsigned long flags;
+
+	spin_lock_irqsave(&info->lock, flags);
+	*shadow_key = key;
+	writeq(key, db);
+	spin_unlock_irqrestore(&info->lock, flags);
+}
+
+static inline void __replay_writeq(u64 key, void __iomem *db)
+{
+	/* No need to replay uninitialised shadow_keys */
+	if (key != BNXT_QPLIB_DBR_KEY_INVALID)
+		writeq(key, db);
+}
+
+static inline void bnxt_qplib_replay_db(struct bnxt_qplib_db_info *info,
+					bool is_arm_ena)
+
+{
+	if (!spin_trylock_irq(&info->lock))
+		return;
+
+	if (is_arm_ena)
+		__replay_writeq(info->shadow_key_arm_ena, info->priv_db);
+	else
+		__replay_writeq(info->shadow_key, info->db);
+
+	spin_unlock_irq(&info->lock);
+}
+
+static inline void bnxt_qplib_ring_db(struct bnxt_qplib_db_info *info,
+				      u32 type)
+{
+	u64 key = 0;
+	u32 indx;
+	u8 toggle = 0;
+
+	if (type == DBC_DBC_TYPE_CQ_ARMALL ||
+	    type == DBC_DBC_TYPE_CQ_ARMSE)
+		toggle = info->toggle;
+
+	indx = ((info->hwq->cons & DBC_DBC_INDEX_MASK) |
+		((info->flags & BNXT_QPLIB_FLAG_EPOCH_CONS_MASK) <<
+		 BNXT_QPLIB_DB_EPOCH_CONS_SHIFT));
+
+	key = BNXT_QPLIB_INIT_DBHDR(info->xid, type, indx, toggle);
+	bnxt_qplib_write_db(info, key, info->db, &info->shadow_key);
+}
+
+static inline void bnxt_qplib_ring_prod_db(struct bnxt_qplib_db_info *info,
+					   u32 type)
+{
+	u64 key = 0;
+	u32 indx;
+
+	indx = (((info->hwq->prod / info->max_slot) & DBC_DBC_INDEX_MASK) |
+		((info->flags & BNXT_QPLIB_FLAG_EPOCH_PROD_MASK) <<
+		 BNXT_QPLIB_DB_EPOCH_PROD_SHIFT));
+	key = BNXT_QPLIB_INIT_DBHDR(info->xid, type, indx, 0);
+	bnxt_qplib_write_db(info, key, info->db, &info->shadow_key);
+}
+
+static inline void bnxt_qplib_armen_db(struct bnxt_qplib_db_info *info,
+				       u32 type)
+{
+	u64 key = 0;
+	u8 toggle = 0;
+
+	if (type == DBC_DBC_TYPE_CQ_ARMENA)
+		toggle = info->toggle;
+	/* Index always at 0 */
+	key = BNXT_QPLIB_INIT_DBHDR(info->xid, type, 0, toggle);
+	bnxt_qplib_write_db(info, key, info->priv_db,
+			    &info->shadow_key_arm_ena);
+}
+
+static inline void bnxt_qplib_cq_coffack_db(struct bnxt_qplib_db_info *info)
+{
+	u64 key = 0;
+
+	/* Index always at 0 */
+	key = BNXT_QPLIB_INIT_DBHDR(info->xid, DBC_DBC_TYPE_CQ_CUTOFF_ACK, 0, 0);
+	bnxt_qplib_write_db(info, key, info->priv_db, &info->shadow_key);
+}
+
+static inline void bnxt_qplib_srq_arm_db(struct bnxt_qplib_db_info *info)
+{
+	u64 key = 0;
+
+	/* Index always at 0 */
+	key = BNXT_QPLIB_INIT_DBHDR(info->xid, DBC_DBC_TYPE_SRQ_ARM, 0, 0);
+	bnxt_qplib_write_db(info, key, info->priv_db, &info->shadow_key);
+}
+
+static inline void bnxt_qplib_ring_nq_db(struct bnxt_qplib_db_info *info,
+					 struct bnxt_qplib_chip_ctx *cctx,
+					 bool arm)
+{
+	u32 type;
+
+	type = arm ? DBC_DBC_TYPE_NQ_ARM : DBC_DBC_TYPE_NQ;
+	if (_is_chip_gen_p5_p7(cctx))
+		bnxt_qplib_ring_db(info, type);
+	else
+		bnxt_qplib_ring_db32(info, arm);
+}
+
+struct bnxt_qplib_max_res {
+	u32 max_qp;
+	u32 max_mr;
+	u32 max_cq;
+	u32 max_srq;
+	u32 max_ah;
+	u32 max_pd;
+};
+
+/*
+ * Defines for maximum resources supported for chip revisions
+ * Maximum PDs supported are restricted to Max QPs
+ * GENP4 - Wh+
+ * DEFAULT - Thor
+ */
+#define BNXT_QPLIB_GENP4_PF_MAX_QP	(16 * 1024)
+#define BNXT_QPLIB_GENP4_PF_MAX_MRW	(16 * 1024)
+#define BNXT_QPLIB_GENP4_PF_MAX_CQ	(16 * 1024)
+#define BNXT_QPLIB_GENP4_PF_MAX_SRQ	(1 * 1024)
+#define BNXT_QPLIB_GENP4_PF_MAX_AH	(16 * 1024)
+#define BNXT_QPLIB_GENP4_PF_MAX_PD	 BNXT_QPLIB_GENP4_PF_MAX_QP
+
+#define BNXT_QPLIB_DEFAULT_PF_MAX_QP	(64 * 1024)
+#define BNXT_QPLIB_DEFAULT_PF_MAX_MRW	(256 * 1024)
+#define BNXT_QPLIB_DEFAULT_PF_MAX_CQ	(64 * 1024)
+#define BNXT_QPLIB_DEFAULT_PF_MAX_SRQ	(4 * 1024)
+#define BNXT_QPLIB_DEFAULT_PF_MAX_AH	(64 * 1024)
+#define BNXT_QPLIB_DEFAULT_PF_MAX_PD	BNXT_QPLIB_DEFAULT_PF_MAX_QP
+
+#define BNXT_QPLIB_DEFAULT_VF_MAX_QP	(6 * 1024)
+#define BNXT_QPLIB_DEFAULT_VF_MAX_MRW	(6 * 1024)
+#define BNXT_QPLIB_DEFAULT_VF_MAX_CQ	(6 * 1024)
+#define BNXT_QPLIB_DEFAULT_VF_MAX_SRQ	(4 * 1024)
+#define BNXT_QPLIB_DEFAULT_VF_MAX_AH	(6 * 1024)
+#define BNXT_QPLIB_DEFAULT_VF_MAX_PD	BNXT_QPLIB_DEFAULT_VF_MAX_QP
+
+static inline void bnxt_qplib_max_res_supported(struct bnxt_qplib_chip_ctx *cctx,
+						struct bnxt_qplib_res *qpl_res,
+						struct bnxt_qplib_max_res *max_res,
+						bool vf_res_limit)
+{
+	switch (cctx->chip_num) {
+	case CHIP_NUM_57608:
+	case CHIP_NUM_58818:
+	case CHIP_NUM_57504:
+	case CHIP_NUM_57502:
+	case CHIP_NUM_57508:
+		if (!qpl_res->is_vf) {
+			max_res->max_qp = BNXT_QPLIB_DEFAULT_PF_MAX_QP;
+			max_res->max_mr = BNXT_QPLIB_DEFAULT_PF_MAX_MRW;
+			max_res->max_cq = BNXT_QPLIB_DEFAULT_PF_MAX_CQ;
+			max_res->max_srq = BNXT_QPLIB_DEFAULT_PF_MAX_SRQ;
+			max_res->max_ah = BNXT_QPLIB_DEFAULT_PF_MAX_AH;
+			max_res->max_pd = BNXT_QPLIB_DEFAULT_PF_MAX_PD;
+		} else {
+			max_res->max_qp = BNXT_QPLIB_DEFAULT_VF_MAX_QP;
+			max_res->max_mr = BNXT_QPLIB_DEFAULT_VF_MAX_MRW;
+			max_res->max_cq = BNXT_QPLIB_DEFAULT_VF_MAX_CQ;
+			max_res->max_srq = BNXT_QPLIB_DEFAULT_VF_MAX_SRQ;
+			max_res->max_ah = BNXT_QPLIB_DEFAULT_VF_MAX_AH;
+			max_res->max_pd = BNXT_QPLIB_DEFAULT_VF_MAX_PD;
+		}
+		break;
+	default:
+		/* Wh+/Stratus max resources */
+		max_res->max_qp = BNXT_QPLIB_GENP4_PF_MAX_QP;
+		max_res->max_mr = BNXT_QPLIB_GENP4_PF_MAX_MRW;
+		max_res->max_cq = BNXT_QPLIB_GENP4_PF_MAX_CQ;
+		max_res->max_srq = BNXT_QPLIB_GENP4_PF_MAX_SRQ;
+		max_res->max_ah = BNXT_QPLIB_GENP4_PF_MAX_AH;
+		max_res->max_pd = BNXT_QPLIB_GENP4_PF_MAX_PD;
+		break;
+	}
+}
+#endif
diff --git a/sys/dev/bnxt/bnxt_re/qplib_sp.c b/sys/dev/bnxt/bnxt_re/qplib_sp.c
new file mode 100644
index 000000000000..8faa3cd9390c
--- /dev/null
+++ b/sys/dev/bnxt/bnxt_re/qplib_sp.c
@@ -0,0 +1,1234 @@
+/*
+ * Copyright (c) 2015-2024, Broadcom. All rights reserved.  The term
+ * Broadcom refers to Broadcom Limited and/or its subsidiaries.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS''
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
+ * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
+ * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * Description: Slow Path Operators
+ */
+
+#include <linux/interrupt.h>
+#include <linux/spinlock.h>
+#include <linux/sched.h>
+#include <linux/pci.h>
+#include <linux/if_ether.h>
+#include <linux/printk.h>
+
+#include "hsi_struct_def.h"
+#include "qplib_tlv.h"
+#include "qplib_res.h"
+#include "qplib_rcfw.h"
+#include "qplib_sp.h"
+
+const struct bnxt_qplib_gid bnxt_qplib_gid_zero = {{ 0, 0, 0, 0, 0, 0, 0, 0,
+						     0, 0, 0, 0, 0, 0, 0, 0 }};
+
+/* Device */
+static u8 bnxt_qplib_is_atomic_cap(struct bnxt_qplib_rcfw *rcfw)
+{
+	u16 pcie_ctl2 = 0;
+
+	if (!_is_chip_gen_p5_p7(rcfw->res->cctx))
+		return false;
+	pcie_capability_read_word(rcfw->pdev, PCI_EXP_DEVCTL2, &pcie_ctl2);
+	return (pcie_ctl2 & PCI_EXP_DEVCTL2_ATOMIC_REQ);
+}
+
+static void bnxt_qplib_query_version(struct bnxt_qplib_rcfw *rcfw, char *fw_ver)
+{
+	struct creq_query_version_resp resp = {};
+	struct bnxt_qplib_cmdqmsg msg = {};
+	struct cmdq_query_version req = {};
+	int rc = 0;
+
+	bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_QUERY_VERSION,
+				 sizeof(req));
+	bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL, sizeof(req),
+				sizeof(resp), 0);
+	rc = bnxt_qplib_rcfw_send_message(rcfw, &msg);
+	if (rc) {
+		dev_err(&rcfw->pdev->dev, "QPLIB: Failed to query version\n");
+		return;
+	}
+	fw_ver[0] = resp.fw_maj;
+	fw_ver[1] = resp.fw_minor;
+	fw_ver[2] = resp.fw_bld;
+	fw_ver[3] = resp.fw_rsvd;
+}
+
+int bnxt_qplib_get_dev_attr(struct bnxt_qplib_rcfw *rcfw)
+{
+	struct creq_query_func_resp resp = {};
+	struct bnxt_qplib_cmdqmsg msg = {};
+	struct creq_query_func_resp_sb *sb;
+	struct bnxt_qplib_rcfw_sbuf sbuf;
+	struct bnxt_qplib_dev_attr *attr;
+	struct bnxt_qplib_chip_ctx *cctx;
+	struct cmdq_query_func req = {};
+	u8 *tqm_alloc;
+	int i, rc = 0;
+	u32 temp;
+	u8 chip_gen = BNXT_RE_DEFAULT;
+
+	cctx = rcfw->res->cctx;
+	attr = rcfw->res->dattr;
+
+	bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_QUERY_FUNC,
+				 sizeof(req));
+
+	sbuf.size = sizeof(*sb);
+	sbuf.sb = dma_zalloc_coherent(&rcfw->pdev->dev, sbuf.size,
+				       &sbuf.dma_addr, GFP_KERNEL);
+	if (!sbuf.sb)
+		return -ENOMEM;
+
+	sb = sbuf.sb;
+	req.resp_size = sbuf.size / BNXT_QPLIB_CMDQE_UNITS;
+	bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, &sbuf, sizeof(req),
+				sizeof(resp), 0);
+	rc = bnxt_qplib_rcfw_send_message(rcfw, &msg);
+	if (rc)
+		goto bail;
+	/* Extract the context from the side buffer */
+	chip_gen = _get_chip_gen_p5_type(cctx);
+	attr->max_qp = le32_to_cpu(sb->max_qp);
+	attr->max_qp = min_t(u32, attr->max_qp, BNXT_RE_MAX_QP_SUPPORTED(chip_gen));
+	/* max_qp value reported by FW does not include the QP1 */
+	attr->max_qp += 1;
+	attr->max_qp_rd_atom =
+		sb->max_qp_rd_atom > BNXT_QPLIB_MAX_OUT_RD_ATOM ?
+		BNXT_QPLIB_MAX_OUT_RD_ATOM : sb->max_qp_rd_atom;
+	attr->max_qp_init_rd_atom =
+		sb->max_qp_init_rd_atom > BNXT_QPLIB_MAX_OUT_RD_ATOM ?
+		BNXT_QPLIB_MAX_OUT_RD_ATOM : sb->max_qp_init_rd_atom;
+	/* Report 1 less than the max_qp_wqes reported by FW as driver adds
+	 * one extra entry while creating the qp
+	 */
+	attr->max_qp_wqes = le16_to_cpu(sb->max_qp_wr) - 1;
+	/* Adjust for max_qp_wqes for variable wqe */
+	if (cctx->modes.wqe_mode == BNXT_QPLIB_WQE_MODE_VARIABLE) {
+		attr->max_qp_wqes = (BNXT_MAX_SQ_SIZE) /
+			(BNXT_MAX_VAR_WQE_SIZE / BNXT_SGE_SIZE) - 1;
+	}
+	if (!_is_chip_gen_p5_p7(cctx)) {
+		/*
+		 * 128 WQEs needs to be reserved for the HW (8916). Prevent
+		 * reporting the max number for gen-p4 only.
+		 */
+		attr->max_qp_wqes -= BNXT_QPLIB_RESERVED_QP_WRS;
+	}
+	attr->max_qp_sges = sb->max_sge;
+	if (_is_chip_gen_p5_p7(cctx) &&
+	    cctx->modes.wqe_mode == BNXT_QPLIB_WQE_MODE_VARIABLE)
+		attr->max_qp_sges = sb->max_sge_var_wqe;
+	attr->max_cq = le32_to_cpu(sb->max_cq);
+	attr->max_cq = min_t(u32, attr->max_cq, BNXT_RE_MAX_CQ_SUPPORTED(chip_gen));
+
+	attr->max_cq_wqes = le32_to_cpu(sb->max_cqe);
+	attr->max_cq_wqes = min_t(u32, BNXT_QPLIB_MAX_CQ_WQES, attr->max_cq_wqes);
+
+	attr->max_cq_sges = attr->max_qp_sges;
+	attr->max_mr = le32_to_cpu(sb->max_mr);
+	attr->max_mr = min_t(u32, attr->max_mr, BNXT_RE_MAX_MRW_SUPPORTED(chip_gen));
+	attr->max_mw = le32_to_cpu(sb->max_mw);
+	attr->max_mw = min_t(u32, attr->max_mw, BNXT_RE_MAX_MRW_SUPPORTED(chip_gen));
+
+	attr->max_mr_size = le64_to_cpu(sb->max_mr_size);
+	attr->max_pd = BNXT_QPLIB_MAX_PD;
+	attr->max_raw_ethy_qp = le32_to_cpu(sb->max_raw_eth_qp);
+	attr->max_ah = le32_to_cpu(sb->max_ah);
+	attr->max_ah = min_t(u32, attr->max_ah, BNXT_RE_MAX_AH_SUPPORTED(chip_gen));
+
+	attr->max_fmr = le32_to_cpu(sb->max_fmr);
+	attr->max_map_per_fmr = sb->max_map_per_fmr;
+
+	attr->max_srq = le16_to_cpu(sb->max_srq);
+	attr->max_srq = min_t(u32, attr->max_srq, BNXT_RE_MAX_SRQ_SUPPORTED(chip_gen));
+	attr->max_srq_wqes = le32_to_cpu(sb->max_srq_wr) - 1;
+	attr->max_srq_sges = sb->max_srq_sge;
+	attr->max_pkey = 1;
+
+	attr->max_inline_data = !cctx->modes.wqe_mode ?
+				le32_to_cpu(sb->max_inline_data) :
+				le16_to_cpu(sb->max_inline_data_var_wqe);
+	if (!_is_chip_p7(cctx)) {
+		attr->l2_db_size = (sb->l2_db_space_size + 1) *
+				    (0x01 << RCFW_DBR_BASE_PAGE_SHIFT);
+	}
+	attr->max_sgid = le32_to_cpu(sb->max_gid);
+
+	/* TODO: remove this hack for statically allocated gid_map */
+	bnxt_re_set_max_gid(&attr->max_sgid);
+
+	attr->dev_cap_flags = le16_to_cpu(sb->dev_cap_flags);
+	attr->page_size_cap = BIT_ULL(28) | BIT_ULL(21) | BIT_ULL(12);
+
+	bnxt_qplib_query_version(rcfw, attr->fw_ver);
+
+	for (i = 0; i < MAX_TQM_ALLOC_REQ / 4; i++) {
+		temp = le32_to_cpu(sb->tqm_alloc_reqs[i]);
+		tqm_alloc = (u8 *)&temp;
+		attr->tqm_alloc_reqs[i * 4] = *tqm_alloc;
+		attr->tqm_alloc_reqs[i * 4 + 1] = *(++tqm_alloc);
+		attr->tqm_alloc_reqs[i * 4 + 2] = *(++tqm_alloc);
+		attr->tqm_alloc_reqs[i * 4 + 3] = *(++tqm_alloc);
+	}
+
+	if (rcfw->res->cctx->hwrm_intf_ver >= HWRM_VERSION_DEV_ATTR_MAX_DPI)
+		attr->max_dpi = le32_to_cpu(sb->max_dpi);
+
+	attr->is_atomic = bnxt_qplib_is_atomic_cap(rcfw);
+bail:
+	dma_free_coherent(&rcfw->pdev->dev, sbuf.size,
+				  sbuf.sb, sbuf.dma_addr);
+	return rc;
+}
+
+int bnxt_qplib_set_func_resources(struct bnxt_qplib_res *res)
+{
+	struct creq_set_func_resources_resp resp = {};
+	struct cmdq_set_func_resources req = {};
+	struct bnxt_qplib_cmdqmsg msg = {};
+	struct bnxt_qplib_rcfw *rcfw;
+	struct bnxt_qplib_ctx *hctx;
+	int rc = 0;
+
+	rcfw = res->rcfw;
+	hctx = res->hctx;
+	bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_SET_FUNC_RESOURCES,
+				 sizeof(req));
+
+	req.number_of_qp = cpu_to_le32(hctx->qp_ctx.max);
+	req.number_of_mrw = cpu_to_le32(hctx->mrw_ctx.max);
+	req.number_of_srq =  cpu_to_le32(hctx->srq_ctx.max);
+	req.number_of_cq = cpu_to_le32(hctx->cq_ctx.max);
+
+	req.max_qp_per_vf = cpu_to_le32(hctx->vf_res.max_qp);
+	req.max_mrw_per_vf = cpu_to_le32(hctx->vf_res.max_mrw);
+	req.max_srq_per_vf = cpu_to_le32(hctx->vf_res.max_srq);
+	req.max_cq_per_vf = cpu_to_le32(hctx->vf_res.max_cq);
+	req.max_gid_per_vf = cpu_to_le32(hctx->vf_res.max_gid);
+
+	/* Keep the old stats context id of PF */
+	req.stat_ctx_id = cpu_to_le32(hctx->stats.fw_id);
+
+	bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL, sizeof(req),
+				sizeof(resp), 0);
+	rc = bnxt_qplib_rcfw_send_message(rcfw, &msg);
+	if (rc)
+		dev_err(&res->pdev->dev,
+			"QPLIB: Failed to set function resources\n");
+
+	return rc;
+}
+
+int bnxt_qplib_update_sgid(struct bnxt_qplib_sgid_tbl *sgid_tbl,
+			   struct bnxt_qplib_gid *gid, u16 gid_idx, const u8 *smac)
+{
+	struct bnxt_qplib_res *res = to_bnxt_qplib(sgid_tbl,
+						   struct bnxt_qplib_res,
+						   sgid_tbl);
+	struct bnxt_qplib_rcfw *rcfw = res->rcfw;
+	struct creq_modify_gid_resp resp = {};
+	struct bnxt_qplib_cmdqmsg msg = {};
+	struct cmdq_modify_gid req = {};
+	int rc;
+
+	bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_MODIFY_GID,
+				 sizeof(req));
+
+	req.gid[0] = cpu_to_be32(((u32 *)gid->data)[3]);
+	req.gid[1] = cpu_to_be32(((u32 *)gid->data)[2]);
+	req.gid[2] = cpu_to_be32(((u32 *)gid->data)[1]);
+	req.gid[3] = cpu_to_be32(((u32 *)gid->data)[0]);
+	if (res->prio) {
+		req.vlan |= cpu_to_le16(CMDQ_ADD_GID_VLAN_TPID_TPID_8100 |
+			CMDQ_ADD_GID_VLAN_VLAN_EN);
+	}
+
+	/* MAC in network format */
+	req.src_mac[0] = cpu_to_be16(((u16 *)smac)[0]);
+	req.src_mac[1] = cpu_to_be16(((u16 *)smac)[1]);
+	req.src_mac[2] = cpu_to_be16(((u16 *)smac)[2]);
+	req.gid_index = cpu_to_le16(gid_idx);
+
+	bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL, sizeof(req),
+				sizeof(resp), 0);
+	rc = bnxt_qplib_rcfw_send_message(rcfw, &msg);
+	if (rc) {
+		dev_err(&res->pdev->dev,
+			"QPLIB: update SGID table failed\n");
+		return rc;
+	}
+	return 0;
+}
+
+/* SGID */
+int bnxt_qplib_get_sgid(struct bnxt_qplib_res *res,
+			struct bnxt_qplib_sgid_tbl *sgid_tbl, int index,
+			struct bnxt_qplib_gid *gid)
+{
+	if (index > sgid_tbl->max) {
+		dev_err(&res->pdev->dev,
+			"QPLIB: Index %d exceeded SGID table max (%d)\n",
+			index, sgid_tbl->max);
+		return -EINVAL;
+	}
+	memcpy(gid, &sgid_tbl->tbl[index].gid, sizeof(*gid));
+	return 0;
+}
+
+int bnxt_qplib_del_sgid(struct bnxt_qplib_sgid_tbl *sgid_tbl,
+			struct bnxt_qplib_gid *gid,
+			u16 vlan_id, bool update)
+{
+	struct bnxt_qplib_res *res = to_bnxt_qplib(sgid_tbl,
+						   struct bnxt_qplib_res,
+						   sgid_tbl);
+	struct bnxt_qplib_rcfw *rcfw = res->rcfw;
+	int index;
+
+	if (sgid_tbl == NULL) {
+		dev_err(&res->pdev->dev, "QPLIB: SGID table not allocated\n");
+		return -EINVAL;
+	}
+	/* Do we need a sgid_lock here? */
+	if (!sgid_tbl->active) {
+		dev_err(&res->pdev->dev,
+			"QPLIB: SGID table has no active entries\n");
+		return -ENOMEM;
+	}
+	for (index = 0; index < sgid_tbl->max; index++) {
+		if (!memcmp(&sgid_tbl->tbl[index].gid, gid, sizeof(*gid)) &&
+		    vlan_id == sgid_tbl->tbl[index].vlan_id)
+			break;
+	}
+	if (index == sgid_tbl->max) {
+		dev_warn(&res->pdev->dev, "GID not found in the SGID table\n");
+		return 0;
+	}
+
+	if (update) {
+		struct creq_delete_gid_resp resp = {};
+		struct bnxt_qplib_cmdqmsg msg = {};
+		struct cmdq_delete_gid req = {};
+		int rc;
+
+		bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_DELETE_GID,
+					 sizeof(req));
+		if (sgid_tbl->hw_id[index] == 0xFFFF) {
+			dev_err(&res->pdev->dev,
+				"QPLIB: GID entry contains an invalid HW id");
+			return -EINVAL;
+		}
+		req.gid_index = cpu_to_le16(sgid_tbl->hw_id[index]);
+		bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL, sizeof(req),
+					sizeof(resp), 0);
+		rc = bnxt_qplib_rcfw_send_message(rcfw, &msg);
+		if (rc)
+			return rc;
+	}
+	memcpy(&sgid_tbl->tbl[index].gid, &bnxt_qplib_gid_zero,
+	       sizeof(bnxt_qplib_gid_zero));
+	sgid_tbl->tbl[index].vlan_id = 0xFFFF;
+	sgid_tbl->vlan[index] = false;
+	sgid_tbl->active--;
+	dev_dbg(&res->pdev->dev,
+		 "QPLIB: SGID deleted hw_id[0x%x] = 0x%x active = 0x%x\n",
+		 index, sgid_tbl->hw_id[index], sgid_tbl->active);
+	sgid_tbl->hw_id[index] = (u16)-1;
+
+	return 0;
+}
+
+int bnxt_qplib_add_sgid(struct bnxt_qplib_sgid_tbl *sgid_tbl,
+			const union ib_gid *gid, const u8 *smac, u16 vlan_id,
+			bool update, u32 *index)
+{
+	struct bnxt_qplib_res *res = to_bnxt_qplib(sgid_tbl,
+						   struct bnxt_qplib_res,
+						   sgid_tbl);
+	struct bnxt_qplib_rcfw *rcfw = res->rcfw;
+	int i, free_idx;
+
+	if (sgid_tbl == NULL) {
+		dev_err(&res->pdev->dev, "QPLIB: SGID table not allocated\n");
+		return -EINVAL;
+	}
+	/* Do we need a sgid_lock here? */
+	if (sgid_tbl->active == sgid_tbl->max) {
+		dev_err(&res->pdev->dev, "QPLIB: SGID table is full\n");
+		return -ENOMEM;
+	}
+	free_idx = sgid_tbl->max;
+	for (i = 0; i < sgid_tbl->max; i++) {
+		if (!memcmp(&sgid_tbl->tbl[i], gid, sizeof(*gid)) &&
+		    sgid_tbl->tbl[i].vlan_id == vlan_id) {
+			dev_dbg(&res->pdev->dev,
+				"QPLIB: SGID entry already exist in entry %d!\n",
+				i);
+			*index = i;
+			return -EALREADY;
+		} else if (!memcmp(&sgid_tbl->tbl[i], &bnxt_qplib_gid_zero,
+				   sizeof(bnxt_qplib_gid_zero)) &&
+			   free_idx == sgid_tbl->max) {
+			free_idx = i;
+		}
+	}
+	if (free_idx == sgid_tbl->max) {
+		dev_err(&res->pdev->dev,
+			"QPLIB: SGID table is FULL but count is not MAX??\n");
+		return -ENOMEM;
+	}
+	if (update) {
+		struct creq_add_gid_resp resp = {};
+		struct bnxt_qplib_cmdqmsg msg = {};
+		struct cmdq_add_gid req = {};
+		int rc;
+
+		bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_ADD_GID,
+					 sizeof(req));
+
+		req.gid[0] = cpu_to_be32(((u32 *)gid->raw)[3]);
+		req.gid[1] = cpu_to_be32(((u32 *)gid->raw)[2]);
+		req.gid[2] = cpu_to_be32(((u32 *)gid->raw)[1]);
+		req.gid[3] = cpu_to_be32(((u32 *)gid->raw)[0]);
+		/* driver should ensure that all RoCE traffic is always VLAN tagged
+		 * if RoCE traffic is running on non-zero VLAN ID or
+		 * RoCE traffic is running on non-zero Priority.
+		 */
+		if ((vlan_id != 0xFFFF) || res->prio) {
+			if (vlan_id != 0xFFFF)
+				req.vlan = cpu_to_le16(vlan_id &
+						CMDQ_ADD_GID_VLAN_VLAN_ID_MASK);
+			req.vlan |=
+				cpu_to_le16(CMDQ_ADD_GID_VLAN_TPID_TPID_8100 |
+					    CMDQ_ADD_GID_VLAN_VLAN_EN);
+		}
+
+		/* MAC in network format */
+		req.src_mac[0] = cpu_to_be16(((u16 *)smac)[0]);
+		req.src_mac[1] = cpu_to_be16(((u16 *)smac)[1]);
+		req.src_mac[2] = cpu_to_be16(((u16 *)smac)[2]);
+
+		bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL, sizeof(req),
+					sizeof(resp), 0);
+		rc = bnxt_qplib_rcfw_send_message(rcfw, &msg);
+		if (rc)
+			return rc;
+		sgid_tbl->hw_id[free_idx] = le32_to_cpu(resp.xid);
+	}
+
+	if (vlan_id != 0xFFFF)
+		sgid_tbl->vlan[free_idx] = true;
+
+	memcpy(&sgid_tbl->tbl[free_idx], gid, sizeof(*gid));
+	sgid_tbl->tbl[free_idx].vlan_id = vlan_id;
+	sgid_tbl->active++;
+	dev_dbg(&res->pdev->dev,
+		 "QPLIB: SGID added hw_id[0x%x] = 0x%x active = 0x%x\n",
+		 free_idx, sgid_tbl->hw_id[free_idx], sgid_tbl->active);
+
+	*index = free_idx;
+	/* unlock */
+	return 0;
+}
+
+/* AH */
+int bnxt_qplib_create_ah(struct bnxt_qplib_res *res, struct bnxt_qplib_ah *ah,
+			 bool block)
+{
+	struct bnxt_qplib_rcfw *rcfw = res->rcfw;
+	struct creq_create_ah_resp resp = {};
+	struct bnxt_qplib_cmdqmsg msg = {};
+	struct cmdq_create_ah req = {};
+	u32 temp32[4];
+	u16 temp16[3];
+	int rc;
+
+	bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_CREATE_AH,
+				 sizeof(req));
+
+	memcpy(temp32, ah->dgid.data, sizeof(struct bnxt_qplib_gid));
+	req.dgid[0] = cpu_to_le32(temp32[0]);
+	req.dgid[1] = cpu_to_le32(temp32[1]);
+	req.dgid[2] = cpu_to_le32(temp32[2]);
+	req.dgid[3] = cpu_to_le32(temp32[3]);
+
+	req.type = ah->nw_type;
+	req.hop_limit = ah->hop_limit;
+	req.sgid_index = cpu_to_le16(res->sgid_tbl.hw_id[ah->sgid_index]);
+	req.dest_vlan_id_flow_label = cpu_to_le32((ah->flow_label &
+					CMDQ_CREATE_AH_FLOW_LABEL_MASK) |
+					CMDQ_CREATE_AH_DEST_VLAN_ID_MASK);
+	req.pd_id = cpu_to_le32(ah->pd->id);
+	req.traffic_class = ah->traffic_class;
+
+	/* MAC in network format */
+	memcpy(temp16, ah->dmac, ETH_ALEN);
+	req.dest_mac[0] = cpu_to_le16(temp16[0]);
+	req.dest_mac[1] = cpu_to_le16(temp16[1]);
+	req.dest_mac[2] = cpu_to_le16(temp16[2]);
+
+	bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL, sizeof(req),
+				sizeof(resp), block);
+	rc = bnxt_qplib_rcfw_send_message(rcfw, &msg);
+	if (rc)
+		return rc;
+
+	ah->id = le32_to_cpu(resp.xid);
+	/* for Cu/Wh AHID 0 is not valid */
+	if (!_is_chip_gen_p5_p7(res->cctx) && !ah->id)
+		rc = -EINVAL;
+
+	return rc;
+}
+
+int bnxt_qplib_destroy_ah(struct bnxt_qplib_res *res, struct bnxt_qplib_ah *ah,
+			  bool block)
+{
+	struct bnxt_qplib_rcfw *rcfw = res->rcfw;
+	struct creq_destroy_ah_resp resp = {};
+	struct bnxt_qplib_cmdqmsg msg = {};
+	struct cmdq_destroy_ah req = {};
+	int rc;
+
+	bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_DESTROY_AH,
+				 sizeof(req));
+
+	req.ah_cid = cpu_to_le32(ah->id);
+
+	bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL, sizeof(req),
+				sizeof(resp), block);
+	rc = bnxt_qplib_rcfw_send_message(rcfw, &msg);
+	return rc;
+}
+
+/* MRW */
+int bnxt_qplib_free_mrw(struct bnxt_qplib_res *res, struct bnxt_qplib_mrw *mrw)
+{
+	struct creq_deallocate_key_resp resp = {};
+	struct bnxt_qplib_rcfw *rcfw = res->rcfw;
+	struct cmdq_deallocate_key req = {};
+	struct bnxt_qplib_cmdqmsg msg = {};
+	int rc;
+
+	if (mrw->lkey == 0xFFFFFFFF) {
+		dev_info(&res->pdev->dev,
+			 "QPLIB: SP: Free a reserved lkey MRW\n");
+		return 0;
+	}
+
+	bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_DEALLOCATE_KEY,
+				 sizeof(req));
+
+	req.mrw_flags = mrw->type;
+
+	if ((mrw->type == CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE1)  ||
+	    (mrw->type == CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE2A) ||
+	    (mrw->type == CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE2B))
+		req.key = cpu_to_le32(mrw->rkey);
+	else
+		req.key = cpu_to_le32(mrw->lkey);
+
+	bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL, sizeof(req),
+				sizeof(resp), 0);
+	rc = bnxt_qplib_rcfw_send_message(rcfw, &msg);
+	if (rc)
+		return rc;
+
+	if (mrw->hwq.max_elements)
+		bnxt_qplib_free_hwq(res, &mrw->hwq);
+
+	return 0;
+}
+
+int bnxt_qplib_alloc_mrw(struct bnxt_qplib_res *res, struct bnxt_qplib_mrw *mrw)
+{
+	struct bnxt_qplib_rcfw *rcfw = res->rcfw;
+	struct creq_allocate_mrw_resp resp = {};
+	struct bnxt_qplib_cmdqmsg msg = {};
+	struct cmdq_allocate_mrw req = {};
+	int rc;
+
+	bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_ALLOCATE_MRW,
+				 sizeof(req));
+
+	req.pd_id = cpu_to_le32(mrw->pd->id);
+	req.mrw_flags = mrw->type;
+	if ((mrw->type == CMDQ_ALLOCATE_MRW_MRW_FLAGS_PMR &&
+	     mrw->flags & BNXT_QPLIB_FR_PMR) ||
+	    mrw->type == CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE2A ||
+	    mrw->type == CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE2B)
+		req.access = CMDQ_ALLOCATE_MRW_ACCESS_CONSUMER_OWNED_KEY;
+	req.mrw_handle = cpu_to_le64(mrw);
+
+	bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL, sizeof(req),
+				sizeof(resp), 0);
+	rc = bnxt_qplib_rcfw_send_message(rcfw, &msg);
+	if (rc)
+		return rc;
+	if ((mrw->type == CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE1)  ||
+	    (mrw->type == CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE2A) ||
+	    (mrw->type == CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE2B))
+		mrw->rkey = le32_to_cpu(resp.xid);
+	else
+		mrw->lkey = le32_to_cpu(resp.xid);
+
+	return 0;
+}
+
+int bnxt_qplib_dereg_mrw(struct bnxt_qplib_res *res, struct bnxt_qplib_mrw *mrw,
+			 bool block)
+{
+	struct bnxt_qplib_rcfw *rcfw = res->rcfw;
+	struct creq_deregister_mr_resp resp = {};
+	struct bnxt_qplib_cmdqmsg msg = {};
+	struct cmdq_deregister_mr req = {};
+	int rc;
+
+	bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_DEREGISTER_MR,
+				 sizeof(req));
+
+	req.lkey = cpu_to_le32(mrw->lkey);
+	bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL, sizeof(req),
+				sizeof(resp), block);
+	rc = bnxt_qplib_rcfw_send_message(rcfw, &msg);
+	if (rc)
+		return rc;
+
+	if (mrw->hwq.max_elements) {
+		mrw->va = 0;
+		mrw->total_size = 0;
+		bnxt_qplib_free_hwq(res, &mrw->hwq);
+	}
+
+	return 0;
+}
+
+int bnxt_qplib_reg_mr(struct bnxt_qplib_res *res,
+		      struct bnxt_qplib_mrinfo *mrinfo,
+		      bool block)
+{
+	struct bnxt_qplib_hwq_attr hwq_attr = {};
+	struct bnxt_qplib_rcfw *rcfw = res->rcfw;
+	struct creq_register_mr_resp resp = {};
+	struct bnxt_qplib_cmdqmsg msg = {};
+	struct cmdq_register_mr req = {};
+	struct bnxt_qplib_mrw *mr;
+	u32 buf_pg_size;
+	u32 pg_size;
+	u16 level;
+	u16 flags;
+	int rc;
+
+	mr = mrinfo->mrw;
+	buf_pg_size = 0x01ULL << mrinfo->sg.pgshft;
+	if (mrinfo->sg.npages) {
+		/* Free the hwq if it already exist, must be a rereg */
+		if (mr->hwq.max_elements)
+			bnxt_qplib_free_hwq(res, &mr->hwq);
+		/* Use system PAGE_SIZE */
+		hwq_attr.res = res;
+		hwq_attr.depth = mrinfo->sg.npages;
+		hwq_attr.stride = PAGE_SIZE;
+		hwq_attr.type = HWQ_TYPE_MR;
+		hwq_attr.sginfo = &mrinfo->sg;
+		rc = bnxt_qplib_alloc_init_hwq(&mr->hwq, &hwq_attr);
+		if (rc) {
+			dev_err(&res->pdev->dev,
+				"SP: Reg MR memory allocation failed\n");
+			return -ENOMEM;
+		}
+	}
+
+	bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_REGISTER_MR,
+				 sizeof(req));
+	/* Configure the request */
+	if (mrinfo->is_dma) {
+		/* No PBL provided, just use system PAGE_SIZE */
+		level = 0;
+		req.pbl = 0;
+		pg_size = PAGE_SIZE;
+	} else {
+		level = mr->hwq.level;
+		req.pbl = cpu_to_le64(mr->hwq.pbl[PBL_LVL_0].pg_map_arr[0]);
+	}
+
+	pg_size = buf_pg_size ? buf_pg_size : PAGE_SIZE;
+	req.log2_pg_size_lvl = (level << CMDQ_REGISTER_MR_LVL_SFT) |
+			       ((ilog2(pg_size) <<
+				 CMDQ_REGISTER_MR_LOG2_PG_SIZE_SFT) &
+				CMDQ_REGISTER_MR_LOG2_PG_SIZE_MASK);
+	req.log2_pbl_pg_size = cpu_to_le16(((ilog2(PAGE_SIZE) <<
+				 CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_SFT) &
+				CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_MASK));
+	req.access = (mr->flags & 0xFFFF);
+	req.va = cpu_to_le64(mr->va);
+	req.key = cpu_to_le32(mr->lkey);
+	if (_is_alloc_mr_unified(res->dattr)) {
+		flags = 0;
+		req.key = cpu_to_le32(mr->pd->id);
+		flags |= CMDQ_REGISTER_MR_FLAGS_ALLOC_MR;
+		req.flags = cpu_to_le16(flags);
+	}
+	req.mr_size = cpu_to_le64(mr->total_size);
+
+	bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL, sizeof(req),
+				sizeof(resp), block);
+	rc = bnxt_qplib_rcfw_send_message(rcfw, &msg);
+	if (rc)
+		goto fail;
+
+	if (_is_alloc_mr_unified(res->dattr)) {
+		mr->lkey = le32_to_cpu(resp.xid);
+		mr->rkey = mr->lkey;
+	}
+
+	return 0;
+fail:
+	if (mr->hwq.max_elements)
+		bnxt_qplib_free_hwq(res, &mr->hwq);
+	return rc;
+}
+
+int bnxt_qplib_alloc_fast_reg_page_list(struct bnxt_qplib_res *res,
+					struct bnxt_qplib_frpl *frpl,
+					int max_pg_ptrs)
+{
+	struct bnxt_qplib_hwq_attr hwq_attr = {};
+	struct bnxt_qplib_sg_info sginfo = {};
+	int pg_ptrs, rc;
+
+	/* Re-calculate the max to fit the HWQ allocation model */
+	pg_ptrs = roundup_pow_of_two(max_pg_ptrs);
+
+	sginfo.pgsize = PAGE_SIZE;
+	sginfo.nopte = true;
+
+	hwq_attr.res = res;
+	hwq_attr.depth = pg_ptrs;
+	hwq_attr.stride = PAGE_SIZE;
+	hwq_attr.sginfo = &sginfo;
+	hwq_attr.type = HWQ_TYPE_CTX;
+	rc = bnxt_qplib_alloc_init_hwq(&frpl->hwq, &hwq_attr);
+	if (!rc)
+		frpl->max_pg_ptrs = pg_ptrs;
+
+	return rc;
+}
+
+void bnxt_qplib_free_fast_reg_page_list(struct bnxt_qplib_res *res,
+					struct bnxt_qplib_frpl *frpl)
+{
+	bnxt_qplib_free_hwq(res, &frpl->hwq);
+}
+
+int bnxt_qplib_map_tc2cos(struct bnxt_qplib_res *res, u16 *cids)
+{
+	struct bnxt_qplib_rcfw *rcfw = res->rcfw;
+	struct creq_map_tc_to_cos_resp resp = {};
+	struct bnxt_qplib_cmdqmsg msg = {};
+	struct cmdq_map_tc_to_cos req = {};
+	int rc;
+
+	bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_MAP_TC_TO_COS,
+				 sizeof(req));
+	req.cos0 = cpu_to_le16(cids[0]);
+	req.cos1 = cpu_to_le16(cids[1]);
+
+	bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL, sizeof(req),
+				sizeof(resp), 0);
+	rc = bnxt_qplib_rcfw_send_message(rcfw, &msg);
+	return rc;
+}
+
+static void bnxt_qplib_fill_cc_gen1(struct cmdq_modify_roce_cc_gen1_tlv *ext_req,
+			     struct bnxt_qplib_cc_param_ext *cc_ext)
+{
+	ext_req->modify_mask = cpu_to_le64(cc_ext->ext_mask);
+	cc_ext->ext_mask = 0;
+	ext_req->inactivity_th_hi = cpu_to_le16(cc_ext->inact_th_hi);
+	ext_req->min_time_between_cnps = cpu_to_le16(cc_ext->min_delta_cnp);
+	ext_req->init_cp = cpu_to_le16(cc_ext->init_cp);
+	ext_req->tr_update_mode = cc_ext->tr_update_mode;
+	ext_req->tr_update_cycles = cc_ext->tr_update_cyls;
+	ext_req->fr_num_rtts = cc_ext->fr_rtt;
+	ext_req->ai_rate_increase = cc_ext->ai_rate_incr;
+	ext_req->reduction_relax_rtts_th = cpu_to_le16(cc_ext->rr_rtt_th);
+	ext_req->additional_relax_cr_th = cpu_to_le16(cc_ext->ar_cr_th);
+	ext_req->cr_min_th = cpu_to_le16(cc_ext->cr_min_th);
+	ext_req->bw_avg_weight = cc_ext->bw_avg_weight;
+	ext_req->actual_cr_factor = cc_ext->cr_factor;
+	ext_req->max_cp_cr_th = cpu_to_le16(cc_ext->cr_th_max_cp);
+	ext_req->cp_bias_en = cc_ext->cp_bias_en;
+	ext_req->cp_bias = cc_ext->cp_bias;
+	ext_req->cnp_ecn = cc_ext->cnp_ecn;
+	ext_req->rtt_jitter_en = cc_ext->rtt_jitter_en;
+	ext_req->link_bytes_per_usec = cpu_to_le16(cc_ext->bytes_per_usec);
+	ext_req->reset_cc_cr_th = cpu_to_le16(cc_ext->cc_cr_reset_th);
+	ext_req->cr_width = cc_ext->cr_width;
+	ext_req->quota_period_min = cc_ext->min_quota;
+	ext_req->quota_period_max = cc_ext->max_quota;
+	ext_req->quota_period_abs_max = cc_ext->abs_max_quota;
+	ext_req->tr_lower_bound = cpu_to_le16(cc_ext->tr_lb);
+	ext_req->cr_prob_factor = cc_ext->cr_prob_fac;
+	ext_req->tr_prob_factor = cc_ext->tr_prob_fac;
+	ext_req->fairness_cr_th = cpu_to_le16(cc_ext->fair_cr_th);
+	ext_req->red_div = cc_ext->red_div;
+	ext_req->cnp_ratio_th = cc_ext->cnp_ratio_th;
+	ext_req->exp_ai_rtts = cpu_to_le16(cc_ext->ai_ext_rtt);
+	ext_req->exp_ai_cr_cp_ratio = cc_ext->exp_crcp_ratio;
+	ext_req->use_rate_table = cc_ext->low_rate_en;
+	ext_req->cp_exp_update_th = cpu_to_le16(cc_ext->cpcr_update_th);
+	ext_req->high_exp_ai_rtts_th1 = cpu_to_le16(cc_ext->ai_rtt_th1);
+	ext_req->high_exp_ai_rtts_th2 = cpu_to_le16(cc_ext->ai_rtt_th2);
+	ext_req->actual_cr_cong_free_rtts_th = cpu_to_le16(cc_ext->cf_rtt_th);
+	ext_req->severe_cong_cr_th1 = cpu_to_le16(cc_ext->sc_cr_th1);
+	ext_req->severe_cong_cr_th2 = cpu_to_le16(cc_ext->sc_cr_th2);
+	ext_req->link64B_per_rtt = cpu_to_le32(cc_ext->l64B_per_rtt);
+	ext_req->cc_ack_bytes = cc_ext->cc_ack_bytes;
+	ext_req->reduce_init_cong_free_rtts_th = cpu_to_le16(cc_ext->reduce_cf_rtt_th);
+}
+
+int bnxt_qplib_modify_cc(struct bnxt_qplib_res *res,
+			 struct bnxt_qplib_cc_param *cc_param)
+{
+	struct bnxt_qplib_tlv_modify_cc_req tlv_req = {};
+	struct creq_modify_roce_cc_resp resp = {};
+	struct bnxt_qplib_cmdqmsg msg = {};
+	struct cmdq_modify_roce_cc *req;
+	int req_size;
+	void *cmd;
+	int rc;
+
+	/* Prepare the older base command */
+	req = &tlv_req.base_req;
+	cmd = req;
+	req_size = sizeof(*req);
+	bnxt_qplib_rcfw_cmd_prep(req, CMDQ_BASE_OPCODE_MODIFY_ROCE_CC,
+				 sizeof(*req));
+	req->modify_mask = cpu_to_le32(cc_param->mask);
+	req->enable_cc = cc_param->enable;
+	req->g = cc_param->g;
+	req->num_phases_per_state = cc_param->nph_per_state;
+	req->time_per_phase = cc_param->time_pph;
+	req->pkts_per_phase = cc_param->pkts_pph;
+	req->init_cr = cpu_to_le16(cc_param->init_cr);
+	req->init_tr = cpu_to_le16(cc_param->init_tr);
+	req->tos_dscp_tos_ecn = (cc_param->tos_dscp <<
+				 CMDQ_MODIFY_ROCE_CC_TOS_DSCP_SFT) |
+				 (cc_param->tos_ecn &
+				 CMDQ_MODIFY_ROCE_CC_TOS_ECN_MASK);
+	req->alt_vlan_pcp = cc_param->alt_vlan_pcp;
+	req->alt_tos_dscp = cpu_to_le16(cc_param->alt_tos_dscp);
+	req->rtt = cpu_to_le16(cc_param->rtt);
+	req->tcp_cp = cpu_to_le16(cc_param->tcp_cp);
+	req->cc_mode = cc_param->cc_mode;
+	req->inactivity_th = cpu_to_le16(cc_param->inact_th);
+
+	/* For chip gen P5 onwards fill extended cmd and header */
+	if (_is_chip_gen_p5_p7(res->cctx)) {
+		struct roce_tlv *hdr;
+		u32 payload;
+		u32 chunks;
+
+		cmd = &tlv_req;
+		req_size = sizeof(tlv_req);
+		/* Prepare primary tlv header */
+		hdr = &tlv_req.tlv_hdr;
+		chunks = CHUNKS(sizeof(struct bnxt_qplib_tlv_modify_cc_req));
+		payload = sizeof(struct cmdq_modify_roce_cc);
+		ROCE_1ST_TLV_PREP(hdr, chunks, payload, true);
+		/* Prepare secondary tlv header */
+		hdr = (struct roce_tlv *)&tlv_req.ext_req;
+		payload = sizeof(struct cmdq_modify_roce_cc_gen1_tlv) -
+			  sizeof(struct roce_tlv);
+		ROCE_EXT_TLV_PREP(hdr, TLV_TYPE_MODIFY_ROCE_CC_GEN1, payload,
+				  false, true);
+		bnxt_qplib_fill_cc_gen1(&tlv_req.ext_req, &cc_param->cc_ext);
+	}
+
+	bnxt_qplib_fill_cmdqmsg(&msg, cmd, &resp, NULL, req_size,
+				sizeof(resp), 0);
+	rc = bnxt_qplib_rcfw_send_message(res->rcfw, &msg);
+	return rc;
+}
+
+static void bnxt_qplib_read_cc_gen1(struct bnxt_qplib_cc_param_ext *cc_ext,
+			     struct creq_query_roce_cc_gen1_resp_sb_tlv *sb)
+{
+	cc_ext->inact_th_hi = le16_to_cpu(sb->inactivity_th_hi);
+	cc_ext->min_delta_cnp = le16_to_cpu(sb->min_time_between_cnps);
+	cc_ext->init_cp = le16_to_cpu(sb->init_cp);
+	cc_ext->tr_update_mode = sb->tr_update_mode;
+	cc_ext->tr_update_cyls = sb->tr_update_cycles;
+	cc_ext->fr_rtt = sb->fr_num_rtts;
+	cc_ext->ai_rate_incr = sb->ai_rate_increase;
+	cc_ext->rr_rtt_th = le16_to_cpu(sb->reduction_relax_rtts_th);
+	cc_ext->ar_cr_th = le16_to_cpu(sb->additional_relax_cr_th);
+	cc_ext->cr_min_th = le16_to_cpu(sb->cr_min_th);
+	cc_ext->bw_avg_weight = sb->bw_avg_weight;
+	cc_ext->cr_factor = sb->actual_cr_factor;
+	cc_ext->cr_th_max_cp = le16_to_cpu(sb->max_cp_cr_th);
+	cc_ext->cp_bias_en = sb->cp_bias_en;
+	cc_ext->cp_bias = sb->cp_bias;
+	cc_ext->cnp_ecn = sb->cnp_ecn;
+	cc_ext->rtt_jitter_en = sb->rtt_jitter_en;
+	cc_ext->bytes_per_usec = le16_to_cpu(sb->link_bytes_per_usec);
+	cc_ext->cc_cr_reset_th = le16_to_cpu(sb->reset_cc_cr_th);
+	cc_ext->cr_width = sb->cr_width;
+	cc_ext->min_quota = sb->quota_period_min;
+	cc_ext->max_quota = sb->quota_period_max;
+	cc_ext->abs_max_quota = sb->quota_period_abs_max;
+	cc_ext->tr_lb = le16_to_cpu(sb->tr_lower_bound);
+	cc_ext->cr_prob_fac = sb->cr_prob_factor;
+	cc_ext->tr_prob_fac = sb->tr_prob_factor;
+	cc_ext->fair_cr_th = le16_to_cpu(sb->fairness_cr_th);
+	cc_ext->red_div = sb->red_div;
+	cc_ext->cnp_ratio_th = sb->cnp_ratio_th;
+	cc_ext->ai_ext_rtt = le16_to_cpu(sb->exp_ai_rtts);
+	cc_ext->exp_crcp_ratio = sb->exp_ai_cr_cp_ratio;
+	cc_ext->low_rate_en = sb->use_rate_table;
+	cc_ext->cpcr_update_th = le16_to_cpu(sb->cp_exp_update_th);
+	cc_ext->ai_rtt_th1 = le16_to_cpu(sb->high_exp_ai_rtts_th1);
+	cc_ext->ai_rtt_th2 = le16_to_cpu(sb->high_exp_ai_rtts_th2);
+	cc_ext->cf_rtt_th = le16_to_cpu(sb->actual_cr_cong_free_rtts_th);
+	cc_ext->sc_cr_th1 = le16_to_cpu(sb->severe_cong_cr_th1);
+	cc_ext->sc_cr_th2 = le16_to_cpu(sb->severe_cong_cr_th2);
+	cc_ext->l64B_per_rtt = le32_to_cpu(sb->link64B_per_rtt);
+	cc_ext->cc_ack_bytes = sb->cc_ack_bytes;
+	cc_ext->reduce_cf_rtt_th = le16_to_cpu(sb->reduce_init_cong_free_rtts_th);
+}
+
+int bnxt_qplib_query_cc_param(struct bnxt_qplib_res *res,
+			      struct bnxt_qplib_cc_param *cc_param)
+{
+	struct creq_query_roce_cc_gen1_resp_sb_tlv *gen1_sb;
+	struct bnxt_qplib_tlv_query_rcc_sb *ext_sb;
+	struct bnxt_qplib_rcfw *rcfw = res->rcfw;
+	struct creq_query_roce_cc_resp resp = {};
+	struct creq_query_roce_cc_resp_sb *sb;
+	struct bnxt_qplib_cmdqmsg msg = {};
+	struct cmdq_query_roce_cc req = {};
+	struct bnxt_qplib_rcfw_sbuf sbuf;
+	size_t resp_size;
+	int rc;
+
+	/* Query the parameters from chip */
+	bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_QUERY_ROCE_CC,
+				 sizeof(req));
+	if (_is_chip_gen_p5_p7(res->cctx))
+		resp_size = sizeof(*ext_sb);
+	else
+		resp_size = sizeof(*sb);
+	sbuf.size = ALIGN(resp_size, BNXT_QPLIB_CMDQE_UNITS);
+	sbuf.sb = dma_zalloc_coherent(&rcfw->pdev->dev, sbuf.size,
+				       &sbuf.dma_addr, GFP_KERNEL);
+	if (!sbuf.sb)
+		return -ENOMEM;
+
+	req.resp_size = sbuf.size / BNXT_QPLIB_CMDQE_UNITS;
+	bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, &sbuf, sizeof(req),
+				sizeof(resp), 0);
+	rc = bnxt_qplib_rcfw_send_message(res->rcfw, &msg);
+	if (rc) {
+		dev_dbg(&res->pdev->dev, "%s:Query CC param failed:0x%x\n",
+			__func__, rc);
+		goto out;
+	}
+
+	ext_sb = sbuf.sb;
+	gen1_sb = &ext_sb->gen1_sb;
+	sb = _is_chip_gen_p5_p7(res->cctx) ? &ext_sb->base_sb :
+		(struct creq_query_roce_cc_resp_sb *)ext_sb;
+
+	cc_param->enable = sb->enable_cc & CREQ_QUERY_ROCE_CC_RESP_SB_ENABLE_CC;
+	cc_param->tos_ecn = (sb->tos_dscp_tos_ecn &
+			     CREQ_QUERY_ROCE_CC_RESP_SB_TOS_ECN_MASK) >>
+			     CREQ_QUERY_ROCE_CC_RESP_SB_TOS_ECN_SFT;
+	cc_param->tos_dscp = (sb->tos_dscp_tos_ecn &
+			      CREQ_QUERY_ROCE_CC_RESP_SB_TOS_DSCP_MASK) >>
+			      CREQ_QUERY_ROCE_CC_RESP_SB_TOS_DSCP_SFT;
+	cc_param->alt_tos_dscp = sb->alt_tos_dscp;
+	cc_param->alt_vlan_pcp = sb->alt_vlan_pcp;
+
+	cc_param->g = sb->g;
+	cc_param->nph_per_state = sb->num_phases_per_state;
+	cc_param->init_cr = le16_to_cpu(sb->init_cr);
+	cc_param->init_tr = le16_to_cpu(sb->init_tr);
+	cc_param->cc_mode = sb->cc_mode;
+	cc_param->inact_th = le16_to_cpu(sb->inactivity_th);
+	cc_param->rtt = le16_to_cpu(sb->rtt);
+	cc_param->tcp_cp = le16_to_cpu(sb->tcp_cp);
+	cc_param->time_pph = sb->time_per_phase;
+	cc_param->pkts_pph = sb->pkts_per_phase;
+	if (_is_chip_gen_p5_p7(res->cctx))
+		bnxt_qplib_read_cc_gen1(&cc_param->cc_ext, gen1_sb);
+out:
+	dma_free_coherent(&rcfw->pdev->dev, sbuf.size,
+				  sbuf.sb, sbuf.dma_addr);
+	return rc;
+}
+
+
+int bnxt_qplib_get_roce_error_stats(struct bnxt_qplib_rcfw *rcfw,
+				    struct bnxt_qplib_roce_stats *stats,
+				    struct bnxt_qplib_query_stats_info *sinfo)
+{
+	struct creq_query_roce_stats_resp resp = {};
+	struct creq_query_roce_stats_resp_sb *sb;
+	struct cmdq_query_roce_stats req = {};
+	struct bnxt_qplib_cmdqmsg msg = {};
+	struct bnxt_qplib_rcfw_sbuf sbuf;
+	u16 cmd_flags = 0;
+	u32 fn_id = 0;
+	int rc = 0;
+
+	bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_QUERY_ROCE_STATS,
+				 sizeof(req));
+
+	sbuf.size = sizeof(*sb);
+	sbuf.sb = dma_zalloc_coherent(&rcfw->pdev->dev, sbuf.size,
+				       &sbuf.dma_addr, GFP_KERNEL);
+	if (!sbuf.sb)
+		return -ENOMEM;
+	sb = sbuf.sb;
+
+	if (rcfw->res->cctx->hwrm_intf_ver >= HWRM_VERSION_ROCE_STATS_FN_ID) {
+		if (sinfo->function_id != 0xFFFFFFFF) {
+			cmd_flags = CMDQ_QUERY_ROCE_STATS_FLAGS_FUNCTION_ID;
+			if (sinfo->vf_valid) {
+				fn_id = CMDQ_QUERY_ROCE_STATS_VF_VALID;
+				fn_id |= (sinfo->function_id <<
+					  CMDQ_QUERY_ROCE_STATS_VF_NUM_SFT) &
+					  CMDQ_QUERY_ROCE_STATS_VF_NUM_MASK;
+			} else {
+				fn_id = sinfo->function_id &
+					CMDQ_QUERY_ROCE_STATS_PF_NUM_MASK;
+			}
+		}
+
+		req.flags = cpu_to_le16(cmd_flags);
+		req.function_id = cpu_to_le32(fn_id);
+
+		if (sinfo->collection_id != 0xFF) {
+			cmd_flags |= CMDQ_QUERY_ROCE_STATS_FLAGS_COLLECTION_ID;
+			req.collection_id = sinfo->collection_id;
+		}
+	} else {
+		/* For older HWRM version, the command length has to be
+		 * adjusted. 8 bytes are more in the newer command.
+		 * So subtract these 8 bytes for older HWRM version.
+		 * command units are adjusted inside
+		 * bnxt_qplib_rcfw_send_message.
+		 */
+		req.cmd_size -= 8;
+	}
+
+	req.resp_size = sbuf.size / BNXT_QPLIB_CMDQE_UNITS;
+	bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, &sbuf, sizeof(req),
+				sizeof(resp), 0);
+	rc = bnxt_qplib_rcfw_send_message(rcfw, &msg);
+	if (rc)
+		goto bail;
+	/* Extract the context from the side buffer */
+	stats->to_retransmits = le64_to_cpu(sb->to_retransmits);
+	stats->seq_err_naks_rcvd = le64_to_cpu(sb->seq_err_naks_rcvd);
+	stats->max_retry_exceeded = le64_to_cpu(sb->max_retry_exceeded);
+	stats->rnr_naks_rcvd = le64_to_cpu(sb->rnr_naks_rcvd);
+	stats->missing_resp = le64_to_cpu(sb->missing_resp);
+	stats->unrecoverable_err = le64_to_cpu(sb->unrecoverable_err);
+	stats->bad_resp_err = le64_to_cpu(sb->bad_resp_err);
+	stats->local_qp_op_err = le64_to_cpu(sb->local_qp_op_err);
+	stats->local_protection_err = le64_to_cpu(sb->local_protection_err);
+	stats->mem_mgmt_op_err = le64_to_cpu(sb->mem_mgmt_op_err);
+	stats->remote_invalid_req_err = le64_to_cpu(sb->remote_invalid_req_err);
+	stats->remote_access_err = le64_to_cpu(sb->remote_access_err);
+	stats->remote_op_err = le64_to_cpu(sb->remote_op_err);
+	stats->dup_req = le64_to_cpu(sb->dup_req);
+	stats->res_exceed_max = le64_to_cpu(sb->res_exceed_max);
+	stats->res_length_mismatch = le64_to_cpu(sb->res_length_mismatch);
+	stats->res_exceeds_wqe = le64_to_cpu(sb->res_exceeds_wqe);
+	stats->res_opcode_err = le64_to_cpu(sb->res_opcode_err);
+	stats->res_rx_invalid_rkey = le64_to_cpu(sb->res_rx_invalid_rkey);
+	stats->res_rx_domain_err = le64_to_cpu(sb->res_rx_domain_err);
+	stats->res_rx_no_perm = le64_to_cpu(sb->res_rx_no_perm);
+	stats->res_rx_range_err = le64_to_cpu(sb->res_rx_range_err);
+	stats->res_tx_invalid_rkey = le64_to_cpu(sb->res_tx_invalid_rkey);
+	stats->res_tx_domain_err = le64_to_cpu(sb->res_tx_domain_err);
+	stats->res_tx_no_perm = le64_to_cpu(sb->res_tx_no_perm);
+	stats->res_tx_range_err = le64_to_cpu(sb->res_tx_range_err);
+	stats->res_irrq_oflow = le64_to_cpu(sb->res_irrq_oflow);
+	stats->res_unsup_opcode = le64_to_cpu(sb->res_unsup_opcode);
+	stats->res_unaligned_atomic = le64_to_cpu(sb->res_unaligned_atomic);
+	stats->res_rem_inv_err = le64_to_cpu(sb->res_rem_inv_err);
+	stats->res_mem_error = le64_to_cpu(sb->res_mem_error);
+	stats->res_srq_err = le64_to_cpu(sb->res_srq_err);
+	stats->res_cmp_err = le64_to_cpu(sb->res_cmp_err);
+	stats->res_invalid_dup_rkey = le64_to_cpu(sb->res_invalid_dup_rkey);
+	stats->res_wqe_format_err = le64_to_cpu(sb->res_wqe_format_err);
+	stats->res_cq_load_err = le64_to_cpu(sb->res_cq_load_err);
+	stats->res_srq_load_err = le64_to_cpu(sb->res_srq_load_err);
+	stats->res_tx_pci_err = le64_to_cpu(sb->res_tx_pci_err);
+	stats->res_rx_pci_err = le64_to_cpu(sb->res_rx_pci_err);
+
+	if (!rcfw->init_oos_stats) {
+		rcfw->oos_prev = le64_to_cpu(sb->res_oos_drop_count);
+		rcfw->init_oos_stats = true;
+	} else {
+		stats->res_oos_drop_count += (le64_to_cpu(sb->res_oos_drop_count) -
+					      rcfw->oos_prev) &
+					     BNXT_QPLIB_OOS_COUNT_MASK;
+		rcfw->oos_prev = le64_to_cpu(sb->res_oos_drop_count);
+	}
+
+	stats->active_qp_count_p0 = le64_to_cpu(sb->active_qp_count_p0);
+	stats->active_qp_count_p1 = le64_to_cpu(sb->active_qp_count_p1);
+	stats->active_qp_count_p2 = le64_to_cpu(sb->active_qp_count_p2);
+	stats->active_qp_count_p3 = le64_to_cpu(sb->active_qp_count_p3);
+bail:
+	dma_free_coherent(&rcfw->pdev->dev, sbuf.size,
+				  sbuf.sb, sbuf.dma_addr);
+	return rc;
+}
+
+int bnxt_qplib_set_link_aggr_mode(struct bnxt_qplib_res *res,
+				  u8 aggr_mode, u8 member_port_map,
+				  u8 active_port_map, bool aggr_en,
+				  u32 stats_fw_id)
+{
+	struct creq_set_link_aggr_mode_resources_resp resp = {};
+	struct cmdq_set_link_aggr_mode_cc req = {};
+	struct bnxt_qplib_rcfw *rcfw = res->rcfw;
+	struct bnxt_qplib_cmdqmsg msg = {};
+	int rc = 0;
+
+	bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_SET_LINK_AGGR_MODE,
+				 sizeof(req));
+
+	req.aggr_enable = aggr_en;
+	req.active_port_map = active_port_map;
+	req.member_port_map = member_port_map;
+	req.link_aggr_mode = aggr_mode;
+
+	/* need to specify only second port stats ctx id for now */
+	req.stat_ctx_id[1] = cpu_to_le16((u16)(stats_fw_id));
+
+	req.modify_mask =
+		cpu_to_le32(CMDQ_SET_LINK_AGGR_MODE_MODIFY_MASK_AGGR_EN |
+			    CMDQ_SET_LINK_AGGR_MODE_MODIFY_MASK_ACTIVE_PORT_MAP |
+			    CMDQ_SET_LINK_AGGR_MODE_MODIFY_MASK_MEMBER_PORT_MAP |
+			    CMDQ_SET_LINK_AGGR_MODE_MODIFY_MASK_AGGR_MODE |
+			    CMDQ_SET_LINK_AGGR_MODE_MODIFY_MASK_STAT_CTX_ID);
+
+	bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL, sizeof(req),
+				sizeof(resp), 0);
+	rc = bnxt_qplib_rcfw_send_message(rcfw, &msg);
+	if (rc)
+		dev_err(&res->pdev->dev,
+			"QPLIB: Failed to set link aggr mode, %#x\n", rc);
+
+	return rc;
+}
+
+int bnxt_qplib_qext_stat(struct bnxt_qplib_rcfw *rcfw, u32 fid,
+			 struct bnxt_qplib_ext_stat *estat,
+			 struct bnxt_qplib_query_stats_info *sinfo)
+{
+	struct creq_query_roce_stats_ext_resp resp = {};
+	struct creq_query_roce_stats_ext_resp_sb *sb;
+	struct cmdq_query_roce_stats_ext req = {};
+	struct bnxt_qplib_cmdqmsg msg = {};
+	struct bnxt_qplib_rcfw_sbuf sbuf;
+	int rc;
+
+	sbuf.size = sizeof(*sb);
+	sbuf.sb = dma_zalloc_coherent(&rcfw->pdev->dev, sbuf.size,
+				       &sbuf.dma_addr, GFP_KERNEL);
+	if (!sbuf.sb) {
+		dev_err(&rcfw->pdev->dev,
+			"QPLIB: SP: QUERY_ROCE_STATS_EXT alloc sb failed\n");
+		return -ENOMEM;
+	}
+	sb = sbuf.sb;
+
+	bnxt_qplib_rcfw_cmd_prep(&req,
+			CMDQ_QUERY_ROCE_STATS_EXT_OPCODE_QUERY_ROCE_STATS,
+			sizeof(req));
+	req.resp_size = sbuf.size;
+	req.resp_addr = cpu_to_le64(sbuf.dma_addr);
+	req.flags = cpu_to_le16(CMDQ_QUERY_ROCE_STATS_EXT_FLAGS_FUNCTION_ID);
+	if (_is_chip_p7(rcfw->res->cctx) && rcfw->res->is_vf) {
+		if (sinfo->vf_valid)
+			req.function_id =
+				cpu_to_le32(CMDQ_QUERY_ROCE_STATS_EXT_VF_VALID |
+					    (fid << CMDQ_QUERY_ROCE_STATS_EXT_VF_NUM_SFT));
+		else
+			req.flags = cpu_to_le16(0);
+	} else {
+		req.function_id = cpu_to_le32(fid);
+	}
+
+	bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, &sbuf, sizeof(req),
+				sizeof(resp), 0);
+	rc = bnxt_qplib_rcfw_send_message(rcfw, &msg);
+	if (rc)
+		goto bail;
+
+	/* dump when dyndbg is enabled */
+	print_hex_dump_bytes("", DUMP_PREFIX_OFFSET, sb, sizeof(*sb));
+	estat->tx_atomic_req = le64_to_cpu(sb->tx_atomic_req_pkts);
+	estat->tx_read_req = le64_to_cpu(sb->tx_read_req_pkts);
+	estat->tx_read_res = le64_to_cpu(sb->tx_read_res_pkts);
+	estat->tx_write_req = le64_to_cpu(sb->tx_write_req_pkts);
+	estat->tx_send_req = le64_to_cpu(sb->tx_send_req_pkts);
+	estat->tx_roce_pkts = le64_to_cpu(sb->tx_roce_pkts);
+	estat->tx_roce_bytes = le64_to_cpu(sb->tx_roce_bytes);
+	estat->rx_atomic_req = le64_to_cpu(sb->rx_atomic_req_pkts);
+	estat->rx_read_req = le64_to_cpu(sb->rx_read_req_pkts);
+	estat->rx_read_res = le64_to_cpu(sb->rx_read_res_pkts);
+	estat->rx_write_req = le64_to_cpu(sb->rx_write_req_pkts);
+	estat->rx_send_req = le64_to_cpu(sb->rx_send_req_pkts);
+	estat->rx_roce_pkts = le64_to_cpu(sb->rx_roce_pkts);
+	estat->rx_roce_bytes = le64_to_cpu(sb->rx_roce_bytes);
+	estat->rx_roce_good_pkts = le64_to_cpu(sb->rx_roce_good_pkts);
+	estat->rx_roce_good_bytes = le64_to_cpu(sb->rx_roce_good_bytes);
+	estat->rx_out_of_buffer = le64_to_cpu(sb->rx_out_of_buffer_pkts);
+	estat->rx_out_of_sequence = le64_to_cpu(sb->rx_out_of_sequence_pkts);
+	estat->tx_cnp = le64_to_cpu(sb->tx_cnp_pkts);
+	estat->rx_cnp = le64_to_cpu(sb->rx_cnp_pkts);
+	estat->rx_ecn_marked = le64_to_cpu(sb->rx_ecn_marked_pkts);
+	estat->seq_err_naks_rcvd = le64_to_cpu(sb->seq_err_naks_rcvd);
+	estat->rnr_naks_rcvd = le64_to_cpu(sb->rnr_naks_rcvd);
+	estat->missing_resp = le64_to_cpu(sb->missing_resp);
+	estat->to_retransmits = le64_to_cpu(sb->to_retransmit);
+	estat->dup_req = le64_to_cpu(sb->dup_req);
+	estat->rx_dcn_payload_cut = le64_to_cpu(sb->rx_dcn_payload_cut);
+	estat->te_bypassed = le64_to_cpu(sb->te_bypassed);
+bail:
+	dma_free_coherent(&rcfw->pdev->dev, sbuf.size,
+				  sbuf.sb, sbuf.dma_addr);
+	return rc;
+}
diff --git a/sys/dev/bnxt/bnxt_re/qplib_sp.h b/sys/dev/bnxt/bnxt_re/qplib_sp.h
new file mode 100644
index 000000000000..e306db3b9d8e
--- /dev/null
+++ b/sys/dev/bnxt/bnxt_re/qplib_sp.h
@@ -0,0 +1,432 @@
+/*
+ * Copyright (c) 2015-2024, Broadcom. All rights reserved.  The term
+ * Broadcom refers to Broadcom Limited and/or its subsidiaries.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS''
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
+ * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
+ * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * Description: Slow Path Operators (header)
+ */
+
+#ifndef __BNXT_QPLIB_SP_H__
+#define __BNXT_QPLIB_SP_H__
+
+#include <rdma/ib_verbs.h>
+
+#define BNXT_QPLIB_RESERVED_QP_WRS	128
+
+/* Resource maximums reported by the firmware */
+struct bnxt_qplib_dev_attr {
+#define FW_VER_ARR_LEN			4
+	u8				fw_ver[FW_VER_ARR_LEN];
+	u16				max_sgid;
+	u16				max_mrw;
+	u32				max_qp;
+#define BNXT_QPLIB_MAX_OUT_RD_ATOM	126
+	u32				max_qp_rd_atom;
+	u32				max_qp_init_rd_atom;
+	u32				max_qp_wqes;
+	u32				max_qp_sges;
+	u32				max_cq;
+	/* HW supports only 8K entries in PBL.
+	 * So max CQEs that can be supported per CQ is 1M.
+	 */
+#define BNXT_QPLIB_MAX_CQ_WQES		0xfffff
+	u32				max_cq_wqes;
+	u32				max_cq_sges;
+	u32				max_mr;
+	u64				max_mr_size;
+#define BNXT_QPLIB_MAX_PD		(64 * 1024)
+	u32				max_pd;
+	u32				max_mw;
+	u32				max_raw_ethy_qp;
+	u32				max_ah;
+	u32				max_fmr;
+	u32				max_map_per_fmr;
+	u32				max_srq;
+	u32				max_srq_wqes;
+	u32				max_srq_sges;
+	u32				max_pkey;
+	u32				max_inline_data;
+	u32				l2_db_size;
+	u8				tqm_alloc_reqs[MAX_TQM_ALLOC_REQ];
+	u8				is_atomic;
+	u16				dev_cap_flags;
+	u64				page_size_cap;
+	u32				max_dpi;
+};
+
+struct bnxt_qplib_pd {
+	u32				id;
+};
+
+struct bnxt_qplib_gid {
+	u8				data[16];
+};
+
+struct bnxt_qplib_gid_info {
+	struct bnxt_qplib_gid gid;
+	u16 vlan_id;
+};
+
+struct bnxt_qplib_ah {
+	struct bnxt_qplib_gid		dgid;
+	struct bnxt_qplib_pd		*pd;
+	u32				id;
+	u8				sgid_index;
+	u8				host_sgid_index; /* For Query AH if the hw table and SW table are differnt */
+	u8				traffic_class;
+	u32				flow_label;
+	u8				hop_limit;
+	u8				sl;
+	u8				dmac[6];
+	u16				vlan_id;
+	u8				nw_type;
+	u8				enable_cc;
+};
+
+struct bnxt_qplib_mrw {
+	struct bnxt_qplib_pd		*pd;
+	int				type;
+	u32				flags;
+#define BNXT_QPLIB_FR_PMR		0x80000000
+	u32				lkey;
+	u32				rkey;
+#define BNXT_QPLIB_RSVD_LKEY		0xFFFFFFFF
+	u64				va;
+	u64				total_size;
+	u32				npages;
+	u64				mr_handle;
+	struct bnxt_qplib_hwq		hwq;
+};
+
+struct bnxt_qplib_mrinfo {
+	struct bnxt_qplib_mrw		*mrw;
+	struct bnxt_qplib_sg_info	sg;
+	u64				*ptes;
+	bool				is_dma;
+};
+
+struct bnxt_qplib_frpl {
+	int				max_pg_ptrs;
+	struct bnxt_qplib_hwq		hwq;
+};
+
+struct bnxt_qplib_cc_param_ext {
+	u64 ext_mask;
+	u16 inact_th_hi;
+	u16 min_delta_cnp;
+	u16 init_cp;
+	u8 tr_update_mode;
+	u8 tr_update_cyls;
+	u8 fr_rtt;
+	u8 ai_rate_incr;
+	u16 rr_rtt_th;
+	u16 ar_cr_th;
+	u16 cr_min_th;
+	u8 bw_avg_weight;
+	u8 cr_factor;
+	u16 cr_th_max_cp;
+	u8 cp_bias_en;
+	u8 cp_bias;
+	u8 cnp_ecn;
+	u8 rtt_jitter_en;
+	u16 bytes_per_usec;
+	u16 cc_cr_reset_th;
+	u8 cr_width;
+	u8 min_quota;
+	u8 max_quota;
+	u8 abs_max_quota;
+	u16 tr_lb;
+	u8 cr_prob_fac;
+	u8 tr_prob_fac;
+	u16 fair_cr_th;
+	u8 red_div;
+	u8 cnp_ratio_th;
+	u16 ai_ext_rtt;
+	u8 exp_crcp_ratio;
+	u8 low_rate_en;
+	u16 cpcr_update_th;
+	u16 ai_rtt_th1;
+	u16 ai_rtt_th2;
+	u16 cf_rtt_th;
+	u16 sc_cr_th1; /* severe congestion cr threshold 1 */
+	u16 sc_cr_th2; /* severe congestion cr threshold 2 */
+	u32 l64B_per_rtt;
+	u8 cc_ack_bytes;
+	u16 reduce_cf_rtt_th;
+};
+
+struct bnxt_qplib_cc_param {
+	u8 alt_vlan_pcp;
+	u16 alt_tos_dscp;
+#define BNXT_QPLIB_USER_DSCP_VALID			0x80
+	u8 cnp_dscp_user;
+	u8 roce_dscp_user;
+	u8 cc_mode;
+	u8 enable;
+	u16 inact_th;
+	u16 init_cr;
+	u16 init_tr;
+	u16 rtt;
+	u8 g;
+	u8 nph_per_state;
+	u8 time_pph;
+	u8 pkts_pph;
+	u8 tos_ecn;
+	u8 tos_dscp;
+	u8 qp1_tos_dscp;
+	u16 tcp_cp;
+	struct bnxt_qplib_cc_param_ext cc_ext;
+	u8 disable_prio_vlan_tx;
+	/* Mask used while programming the configfs values */
+	u32 mask;
+	/* Mask used while displaying the configfs values */
+	u32 cur_mask;
+	u8 roce_pri;
+#define BNXT_QPLIB_CC_PARAM_MASK_VLAN_TX_DISABLE	0x40000
+#define BNXT_QPLIB_CC_PARAM_MASK_ROCE_PRI		0x80000
+	/* prev value to clear dscp table */
+	u8 prev_roce_pri;
+	u8 prev_alt_vlan_pcp;
+	u8 prev_tos_dscp;
+	u16 prev_alt_tos_dscp;
+	/* To track if admin has enabled ECN explicitly */
+	u8 admin_enable;
+};
+
+struct bnxt_qplib_roce_stats {
+	u64 to_retransmits;
+	u64 seq_err_naks_rcvd;
+	/* seq_err_naks_rcvd is 64 b */
+	u64 max_retry_exceeded;
+	/* max_retry_exceeded is 64 b */
+	u64 rnr_naks_rcvd;
+	/* rnr_naks_rcvd is 64 b */
+	u64 missing_resp;
+	u64 unrecoverable_err;
+	/* unrecoverable_err is 64 b */
+	u64 bad_resp_err;
+	/* bad_resp_err is 64 b */
+	u64 local_qp_op_err;
+	/* local_qp_op_err is 64 b */
+	u64 local_protection_err;
+	/* local_protection_err is 64 b */
+	u64 mem_mgmt_op_err;
+	/* mem_mgmt_op_err is 64 b */
+	u64 remote_invalid_req_err;
+	/* remote_invalid_req_err is 64 b */
+	u64 remote_access_err;
+	/* remote_access_err is 64 b */
+	u64 remote_op_err;
+	/* remote_op_err is 64 b */
+	u64 dup_req;
+	/* dup_req is 64 b */
+	u64 res_exceed_max;
+	/* res_exceed_max is 64 b */
+	u64 res_length_mismatch;
+	/* res_length_mismatch is 64 b */
+	u64 res_exceeds_wqe;
+	/* res_exceeds_wqe is 64 b */
+	u64 res_opcode_err;
+	/* res_opcode_err is 64 b */
+	u64 res_rx_invalid_rkey;
+	/* res_rx_invalid_rkey is 64 b */
+	u64 res_rx_domain_err;
+	/* res_rx_domain_err is 64 b */
+	u64 res_rx_no_perm;
+	/* res_rx_no_perm is 64 b */
+	u64 res_rx_range_err;
+	/* res_rx_range_err is 64 b */
+	u64 res_tx_invalid_rkey;
+	/* res_tx_invalid_rkey is 64 b */
+	u64 res_tx_domain_err;
+	/* res_tx_domain_err is 64 b */
+	u64 res_tx_no_perm;
+	/* res_tx_no_perm is 64 b */
+	u64 res_tx_range_err;
+	/* res_tx_range_err is 64 b */
+	u64 res_irrq_oflow;
+	/* res_irrq_oflow is 64 b */
+	u64 res_unsup_opcode;
+	/* res_unsup_opcode is 64 b */
+	u64 res_unaligned_atomic;
+	/* res_unaligned_atomic is 64 b */
+	u64 res_rem_inv_err;
+	/* res_rem_inv_err is 64 b */
+	u64 res_mem_error;
+	/* res_mem_error is 64 b */
+	u64 res_srq_err;
+	/* res_srq_err is 64 b */
+	u64 res_cmp_err;
+	/* res_cmp_err is 64 b */
+	u64 res_invalid_dup_rkey;
+	/* res_invalid_dup_rkey is 64 b */
+	u64 res_wqe_format_err;
+	/* res_wqe_format_err is 64 b */
+	u64 res_cq_load_err;
+	/* res_cq_load_err is 64 b */
+	u64 res_srq_load_err;
+	/* res_srq_load_err is 64 b */
+	u64 res_tx_pci_err;
+	/* res_tx_pci_err is 64 b */
+	u64 res_rx_pci_err;
+	/* res_rx_pci_err is 64 b */
+	u64 res_oos_drop_count;
+	/* res_oos_drop_count */
+	u64	active_qp_count_p0;
+	/* port 0 active qps */
+	u64	active_qp_count_p1;
+	/* port 1 active qps */
+	u64	active_qp_count_p2;
+	/* port 2 active qps */
+	u64	active_qp_count_p3;
+	/* port 3 active qps */
+};
+
+struct bnxt_qplib_ext_stat {
+	u64  tx_atomic_req;
+	u64  tx_read_req;
+	u64  tx_read_res;
+	u64  tx_write_req;
+	u64  tx_send_req;
+	u64  tx_roce_pkts;
+	u64  tx_roce_bytes;
+	u64  rx_atomic_req;
+	u64  rx_read_req;
+	u64  rx_read_res;
+	u64  rx_write_req;
+	u64  rx_send_req;
+	u64  rx_roce_pkts;
+	u64  rx_roce_bytes;
+	u64  rx_roce_good_pkts;
+	u64  rx_roce_good_bytes;
+	u64  rx_out_of_buffer;
+	u64  rx_out_of_sequence;
+	u64  tx_cnp;
+	u64  rx_cnp;
+	u64  rx_ecn_marked;
+	u64  seq_err_naks_rcvd;
+	u64  rnr_naks_rcvd;
+	u64  missing_resp;
+	u64  to_retransmits;
+	u64  dup_req;
+	u64  rx_dcn_payload_cut;
+	u64  te_bypassed;
+};
+
+#define BNXT_QPLIB_ACCESS_LOCAL_WRITE	(1 << 0)
+#define BNXT_QPLIB_ACCESS_REMOTE_READ	(1 << 1)
+#define BNXT_QPLIB_ACCESS_REMOTE_WRITE	(1 << 2)
+#define BNXT_QPLIB_ACCESS_REMOTE_ATOMIC	(1 << 3)
+#define BNXT_QPLIB_ACCESS_MW_BIND	(1 << 4)
+#define BNXT_QPLIB_ACCESS_ZERO_BASED	(1 << 5)
+#define BNXT_QPLIB_ACCESS_ON_DEMAND	(1 << 6)
+
+int bnxt_qplib_get_sgid(struct bnxt_qplib_res *res,
+			struct bnxt_qplib_sgid_tbl *sgid_tbl, int index,
+			struct bnxt_qplib_gid *gid);
+int bnxt_qplib_del_sgid(struct bnxt_qplib_sgid_tbl *sgid_tbl,
+			struct bnxt_qplib_gid *gid, u16 vlan_id, bool update);
+int bnxt_qplib_add_sgid(struct bnxt_qplib_sgid_tbl *sgid_tbl,
+			const union ib_gid *gid, const u8 *mac, u16 vlan_id,
+			bool update, u32 *index);
+int bnxt_qplib_update_sgid(struct bnxt_qplib_sgid_tbl *sgid_tbl,
+			   struct bnxt_qplib_gid *gid, u16 gid_idx, const u8 *smac);
+int bnxt_qplib_get_dev_attr(struct bnxt_qplib_rcfw *rcfw);
+int bnxt_qplib_set_func_resources(struct bnxt_qplib_res *res);
+int bnxt_qplib_create_ah(struct bnxt_qplib_res *res, struct bnxt_qplib_ah *ah,
+			 bool block);
+int bnxt_qplib_destroy_ah(struct bnxt_qplib_res *res, struct bnxt_qplib_ah *ah,
+			  bool block);
+int bnxt_qplib_alloc_mrw(struct bnxt_qplib_res *res, struct bnxt_qplib_mrw *mrw);
+int bnxt_qplib_dereg_mrw(struct bnxt_qplib_res *res, struct bnxt_qplib_mrw *mrw,
+			 bool block);
+int bnxt_qplib_reg_mr(struct bnxt_qplib_res *res,
+		      struct bnxt_qplib_mrinfo *mrinfo, bool block);
+int bnxt_qplib_free_mrw(struct bnxt_qplib_res *res, struct bnxt_qplib_mrw *mr);
+int bnxt_qplib_alloc_fast_reg_mr(struct bnxt_qplib_res *res,
+				 struct bnxt_qplib_mrw *mr, int max);
+int bnxt_qplib_alloc_fast_reg_page_list(struct bnxt_qplib_res *res,
+					struct bnxt_qplib_frpl *frpl, int max);
+void bnxt_qplib_free_fast_reg_page_list(struct bnxt_qplib_res *res,
+					struct bnxt_qplib_frpl *frpl);
+int bnxt_qplib_map_tc2cos(struct bnxt_qplib_res *res, u16 *cids);
+int bnxt_qplib_modify_cc(struct bnxt_qplib_res *res,
+			 struct bnxt_qplib_cc_param *cc_param);
+int bnxt_qplib_query_cc_param(struct bnxt_qplib_res *res,
+			      struct bnxt_qplib_cc_param *cc_param);
+int bnxt_qplib_set_link_aggr_mode(struct bnxt_qplib_res *res,
+				  u8 aggr_mode, u8 member_port_map,
+				  u8 active_port_map, bool aggr_en,
+				  u32 stats_fw_id);
+int bnxt_qplib_get_roce_error_stats(struct bnxt_qplib_rcfw *rcfw,
+				    struct bnxt_qplib_roce_stats *stats,
+				    struct bnxt_qplib_query_stats_info *sinfo);
+int bnxt_qplib_qext_stat(struct bnxt_qplib_rcfw *rcfw, u32 fid,
+			 struct bnxt_qplib_ext_stat *estat,
+			 struct bnxt_qplib_query_stats_info *sinfo);
+static inline void bnxt_re_set_max_gid(u16 *max_sgid);
+bool ib_modify_qp_is_ok_compat(enum ib_qp_state cur_state, enum ib_qp_state next_state,
+			       enum ib_qp_type type, enum ib_qp_attr_mask mask);
+
+#define BNXT_MAX_SQ_SIZE 0xFFFF
+#define BNXT_MAX_VAR_WQE_SIZE 512
+#define BNXT_SGE_SIZE 16
+
+/* PF defines */
+#define BNXT_RE_MAX_QP_SUPPORTED(chip_gen) \
+	chip_gen == BNXT_RE_DEFAULT ? (64 * 1024) : 0
+
+#define BNXT_RE_MAX_MRW_SUPPORTED(chip_gen) \
+	chip_gen == BNXT_RE_DEFAULT ? (256 * 1024) : 0
+
+#define BNXT_RE_MAX_CQ_SUPPORTED(chip_gen) \
+	chip_gen == BNXT_RE_DEFAULT ? (64 * 1024) : 0
+
+#define BNXT_RE_MAX_SRQ_SUPPORTED(chip_gen) \
+	chip_gen == BNXT_RE_DEFAULT ? (4 * 1024) : 0
+
+#define BNXT_RE_MAX_AH_SUPPORTED(chip_gen) \
+	chip_gen == BNXT_RE_DEFAULT ? (64 * 1024) : 0
+
+/* VF defines */
+#define BNXT_RE_VF_MAX_QP_SUPPORTED(chip_gen) \
+	chip_gen == BNXT_RE_DEFAULT ? (6 * 1024) : 0
+
+#define BNXT_RE_VF_MAX_MRW_SUPPORTED(chip_gen) \
+	chip_gen == BNXT_RE_DEFAULT ? (6 * 1024) : 0
+
+#define BNXT_RE_VF_MAX_CQ_SUPPORTED(chip_gen) \
+	chip_gen == BNXT_RE_DEFAULT ? (6 * 1024) : 0
+
+#define BNXT_RE_VF_MAX_SRQ_SUPPORTED(chip_gen) \
+	chip_gen == BNXT_RE_DEFAULT ? (4 * 1024) : 0
+
+static inline void bnxt_re_set_max_gid(u16 *max_sgid)
+{
+	*max_sgid = max_t(u32, 256, *max_sgid);
+	*max_sgid = min_t(u32, 256, *max_sgid);
+}
+
+#endif
diff --git a/sys/dev/bnxt/bnxt_re/qplib_tlv.h b/sys/dev/bnxt/bnxt_re/qplib_tlv.h
new file mode 100644
index 000000000000..eeaea41a37e6
--- /dev/null
+++ b/sys/dev/bnxt/bnxt_re/qplib_tlv.h
@@ -0,0 +1,187 @@
+/*
+ * Copyright (c) 2017 - 2024, Broadcom. All rights reserved.  The term
+ * Broadcom refers to Broadcom Limited and/or its subsidiaries.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS''
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
+ * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
+ * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#ifndef __QPLIB_TLV_H__
+#define __QPLIB_TLV_H__
+
+struct roce_tlv {
+	struct tlv tlv;
+	u8 total_size;
+	u8 unused[7];
+};
+
+#define CHUNK_SIZE 16
+#define CHUNKS(x) (((x) + CHUNK_SIZE - 1) / CHUNK_SIZE)
+
+#define ROCE_1ST_TLV_PREP(rtlv, tot_chunks, content_bytes, more)     \
+    do {                                                             \
+	(rtlv)->tlv.cmd_discr = CMD_DISCR_TLV_ENCAP;                 \
+	(rtlv)->tlv.tlv_type = TLV_TYPE_ROCE_SP_COMMAND;             \
+	(rtlv)->tlv.length = (content_bytes);                        \
+	(rtlv)->tlv.flags = TLV_FLAGS_REQUIRED;                      \
+	(rtlv)->tlv.flags |= (more) ? TLV_FLAGS_MORE : 0;            \
+	(rtlv)->total_size = (tot_chunks);                           \
+    } while (0)
+
+#define ROCE_EXT_TLV_PREP(rtlv, ext_type, content_bytes, more, reqd) \
+    do {                                                             \
+	(rtlv)->tlv.cmd_discr = CMD_DISCR_TLV_ENCAP;                 \
+	(rtlv)->tlv.tlv_type = (ext_type);                           \
+	(rtlv)->tlv.length = (content_bytes);                        \
+	(rtlv)->tlv.flags |= (more) ? TLV_FLAGS_MORE : 0;            \
+	(rtlv)->tlv.flags |= (reqd) ? TLV_FLAGS_REQUIRED : 0;        \
+    } while (0)
+
+/*
+ * TLV size in units of 16 byte chunks
+ */
+#define TLV_SIZE ((sizeof(struct roce_tlv) + 15) / 16)
+/*
+ * TLV length in bytes
+ */
+#define TLV_BYTES (TLV_SIZE * 16)
+
+#define HAS_TLV_HEADER(msg) (((struct tlv *)(msg))->cmd_discr == CMD_DISCR_TLV_ENCAP)
+#define GET_TLV_DATA(tlv)   ((void *)&((uint8_t *)(tlv))[TLV_BYTES])
+
+static inline u8 __get_cmdq_base_opcode(struct cmdq_base *req, u32 size)
+{
+	if (HAS_TLV_HEADER(req) && size > TLV_BYTES)
+		return ((struct cmdq_base *)GET_TLV_DATA(req))->opcode;
+	else
+		return req->opcode;
+}
+
+static inline void __set_cmdq_base_opcode(struct cmdq_base *req,
+					  u32 size, u8 val)
+{
+	if (HAS_TLV_HEADER(req) && size > TLV_BYTES)
+		((struct cmdq_base *)GET_TLV_DATA(req))->opcode = val;
+	else
+		req->opcode = val;
+}
+
+static inline __le16 __get_cmdq_base_cookie(struct cmdq_base *req, u32 size)
+{
+	if (HAS_TLV_HEADER(req) && size > TLV_BYTES)
+		return ((struct cmdq_base *)GET_TLV_DATA(req))->cookie;
+	else
+		return req->cookie;
+}
+
+static inline void __set_cmdq_base_cookie(struct cmdq_base *req,
+					  u32 size, __le16 val)
+{
+	if (HAS_TLV_HEADER(req) && size > TLV_BYTES)
+		((struct cmdq_base *)GET_TLV_DATA(req))->cookie = val;
+	else
+		req->cookie = val;
+}
+
+static inline __le64 __get_cmdq_base_resp_addr(struct cmdq_base *req, u32 size)
+{
+	if (HAS_TLV_HEADER(req) && size > TLV_BYTES)
+		return ((struct cmdq_base *)GET_TLV_DATA(req))->resp_addr;
+	else
+		return req->resp_addr;
+}
+
+static inline void __set_cmdq_base_resp_addr(struct cmdq_base *req,
+					     u32 size, __le64 val)
+{
+	if (HAS_TLV_HEADER(req) && size > TLV_BYTES)
+		((struct cmdq_base *)GET_TLV_DATA(req))->resp_addr = val;
+	else
+		req->resp_addr = val;
+}
+
+static inline u8 __get_cmdq_base_resp_size(struct cmdq_base *req, u32 size)
+{
+	if (HAS_TLV_HEADER(req) && size > TLV_BYTES)
+		return ((struct cmdq_base *)GET_TLV_DATA(req))->resp_size;
+	else
+		return req->resp_size;
+}
+
+static inline void __set_cmdq_base_resp_size(struct cmdq_base *req,
+					     u32 size, u8 val)
+{
+	if (HAS_TLV_HEADER(req) && size > TLV_BYTES)
+		((struct cmdq_base *)GET_TLV_DATA(req))->resp_size = val;
+	else
+		req->resp_size = val;
+}
+
+static inline u8 __get_cmdq_base_cmd_size(struct cmdq_base *req, u32 size)
+{
+	if (HAS_TLV_HEADER(req) && size > TLV_BYTES)
+		return ((struct roce_tlv *)(req))->total_size;
+	else
+		return req->cmd_size;
+}
+
+static inline void __set_cmdq_base_cmd_size(struct cmdq_base *req,
+					    u32 size, u8 val)
+{
+	if (HAS_TLV_HEADER(req) && size > TLV_BYTES)
+		((struct cmdq_base *)GET_TLV_DATA(req))->cmd_size = val;
+	else
+		req->cmd_size = val;
+}
+
+static inline __le16 __get_cmdq_base_flags(struct cmdq_base *req, u32 size)
+{
+	if (HAS_TLV_HEADER(req) && size > TLV_BYTES)
+		return ((struct cmdq_base *)GET_TLV_DATA(req))->flags;
+	else
+		return req->flags;
+}
+
+static inline void __set_cmdq_base_flags(struct cmdq_base *req,
+					 u32 size, __le16 val)
+{
+	if (HAS_TLV_HEADER(req) && size > TLV_BYTES)
+		((struct cmdq_base *)GET_TLV_DATA(req))->flags = val;
+	else
+		req->flags = val;
+}
+
+struct bnxt_qplib_tlv_modify_cc_req {
+	struct roce_tlv				tlv_hdr;
+	struct cmdq_modify_roce_cc		base_req;
+	__le64					tlvpad;
+	struct cmdq_modify_roce_cc_gen1_tlv	ext_req;
+};
+
+struct bnxt_qplib_tlv_query_rcc_sb {
+	struct roce_tlv					tlv_hdr;
+	struct creq_query_roce_cc_resp_sb		base_sb;
+	struct creq_query_roce_cc_gen1_resp_sb_tlv	gen1_sb;
+};
+#endif
diff --git a/sys/dev/bnxt/bnxt_re/stats.c b/sys/dev/bnxt/bnxt_re/stats.c
new file mode 100644
index 000000000000..7b0e6097aae6
--- /dev/null
+++ b/sys/dev/bnxt/bnxt_re/stats.c
@@ -0,0 +1,773 @@
+/*
+ * Copyright (c) 2015-2024, Broadcom. All rights reserved.  The term
+ * Broadcom refers to Broadcom Limited and/or its subsidiaries.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS''
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
+ * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
+ * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * Description: statistics related functions
+ */
+
+#include "bnxt_re.h"
+#include "bnxt.h"
+
+int bnxt_re_get_flow_stats_from_service_pf(struct bnxt_re_dev *rdev,
+					   struct bnxt_re_flow_counters *stats,
+					   struct bnxt_qplib_query_stats_info *sinfo)
+{
+	struct hwrm_cfa_flow_stats_output resp = {};
+	struct hwrm_cfa_flow_stats_input req = {};
+	struct bnxt_en_dev *en_dev = rdev->en_dev;
+	struct bnxt_fw_msg fw_msg = {};
+	u16 target_id;
+	int rc = 0;
+
+	if (sinfo->function_id == 0xFFFFFFFF)
+		target_id = -1;
+	else
+		target_id = sinfo->function_id + 1;
+
+	/* Issue HWRM cmd to read flow counters for CNP tx and rx */
+	bnxt_re_init_hwrm_hdr(rdev, (void *)&req, HWRM_CFA_FLOW_STATS, -1, target_id);
+	req.num_flows = cpu_to_le16(6);
+	req.flow_handle_0 = cpu_to_le16(HWRM_CFA_FLOW_INFO_INPUT_FLOW_HANDLE_CNP_CNT);
+	req.flow_handle_1 = cpu_to_le16(HWRM_CFA_FLOW_INFO_INPUT_FLOW_HANDLE_CNP_CNT |
+					HWRM_CFA_FLOW_INFO_INPUT_FLOW_HANDLE_DIR_RX);
+	req.flow_handle_2 = cpu_to_le16(HWRM_CFA_FLOW_INFO_INPUT_FLOW_HANDLE_ROCEV1_CNT);
+	req.flow_handle_3 = cpu_to_le16(HWRM_CFA_FLOW_INFO_INPUT_FLOW_HANDLE_ROCEV1_CNT |
+					HWRM_CFA_FLOW_INFO_INPUT_FLOW_HANDLE_DIR_RX);
+	req.flow_handle_4 = cpu_to_le16(HWRM_CFA_FLOW_INFO_INPUT_FLOW_HANDLE_ROCEV2_CNT);
+	req.flow_handle_5 = cpu_to_le16(HWRM_CFA_FLOW_INFO_INPUT_FLOW_HANDLE_ROCEV2_CNT |
+					HWRM_CFA_FLOW_INFO_INPUT_FLOW_HANDLE_DIR_RX);
+	bnxt_re_fill_fw_msg(&fw_msg, &req, sizeof(req), &resp,
+			    sizeof(resp), BNXT_RE_HWRM_CMD_TIMEOUT(rdev));
+	rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg);
+	if (rc) {
+		dev_err(rdev_to_dev(rdev),
+			"Failed to get CFA Flow stats : rc = 0x%x\n", rc);
+		return rc;
+	}
+
+	stats->cnp_stats.cnp_tx_pkts = le64_to_cpu(resp.packet_0);
+	stats->cnp_stats.cnp_tx_bytes = le64_to_cpu(resp.byte_0);
+	stats->cnp_stats.cnp_rx_pkts = le64_to_cpu(resp.packet_1);
+	stats->cnp_stats.cnp_rx_bytes = le64_to_cpu(resp.byte_1);
+
+	stats->ro_stats.tx_pkts = le64_to_cpu(resp.packet_2) +
+		le64_to_cpu(resp.packet_4);
+	stats->ro_stats.tx_bytes = le64_to_cpu(resp.byte_2) +
+		le64_to_cpu(resp.byte_4);
+	stats->ro_stats.rx_pkts = le64_to_cpu(resp.packet_3) +
+		le64_to_cpu(resp.packet_5);
+	stats->ro_stats.rx_bytes = le64_to_cpu(resp.byte_3) +
+		le64_to_cpu(resp.byte_5);
+
+	return 0;
+}
+
+int bnxt_re_get_qos_stats(struct bnxt_re_dev *rdev)
+{
+	struct bnxt_re_ro_counters roce_only_tmp[2] = {{}, {}};
+	struct bnxt_re_cnp_counters tmp_counters[2] = {{}, {}};
+	struct hwrm_cfa_flow_stats_output resp = {};
+	struct hwrm_cfa_flow_stats_input req = {};
+	struct bnxt_en_dev *en_dev = rdev->en_dev;
+	struct bnxt_fw_msg fw_msg = {};
+	struct bnxt_re_cc_stat *cnps;
+	struct bnxt_re_rstat *dstat;
+	int rc = 0;
+	u64 bytes;
+	u64 pkts;
+
+	/* Issue HWRM cmd to read flow counters for CNP tx and rx */
+	bnxt_re_init_hwrm_hdr(rdev, (void *)&req, HWRM_CFA_FLOW_STATS, -1, -1);
+	req.num_flows = cpu_to_le16(6);
+	req.flow_handle_0 = cpu_to_le16(HWRM_CFA_FLOW_INFO_INPUT_FLOW_HANDLE_CNP_CNT);
+	req.flow_handle_1 = cpu_to_le16(HWRM_CFA_FLOW_INFO_INPUT_FLOW_HANDLE_CNP_CNT |
+					HWRM_CFA_FLOW_INFO_INPUT_FLOW_HANDLE_DIR_RX);
+	req.flow_handle_2 = cpu_to_le16(HWRM_CFA_FLOW_INFO_INPUT_FLOW_HANDLE_ROCEV1_CNT);
+	req.flow_handle_3 = cpu_to_le16(HWRM_CFA_FLOW_INFO_INPUT_FLOW_HANDLE_ROCEV1_CNT |
+					HWRM_CFA_FLOW_INFO_INPUT_FLOW_HANDLE_DIR_RX);
+	req.flow_handle_4 = cpu_to_le16(HWRM_CFA_FLOW_INFO_INPUT_FLOW_HANDLE_ROCEV2_CNT);
+	req.flow_handle_5 = cpu_to_le16(HWRM_CFA_FLOW_INFO_INPUT_FLOW_HANDLE_ROCEV2_CNT |
+				       HWRM_CFA_FLOW_INFO_INPUT_FLOW_HANDLE_DIR_RX);
+	bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp,
+			    sizeof(resp), BNXT_RE_HWRM_CMD_TIMEOUT(rdev));
+	rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg);
+	if (rc) {
+		dev_err(rdev_to_dev(rdev),
+			"Failed to get CFA Flow stats : rc = 0x%x\n", rc);
+		goto done;
+	}
+
+	tmp_counters[0].cnp_tx_pkts = le64_to_cpu(resp.packet_0);
+	tmp_counters[0].cnp_tx_bytes = le64_to_cpu(resp.byte_0);
+	tmp_counters[0].cnp_rx_pkts = le64_to_cpu(resp.packet_1);
+	tmp_counters[0].cnp_rx_bytes = le64_to_cpu(resp.byte_1);
+
+	roce_only_tmp[0].tx_pkts = le64_to_cpu(resp.packet_2) +
+				   le64_to_cpu(resp.packet_4);
+	roce_only_tmp[0].tx_bytes = le64_to_cpu(resp.byte_2) +
+				    le64_to_cpu(resp.byte_4);
+	roce_only_tmp[0].rx_pkts = le64_to_cpu(resp.packet_3) +
+				   le64_to_cpu(resp.packet_5);
+	roce_only_tmp[0].rx_bytes = le64_to_cpu(resp.byte_3) +
+				    le64_to_cpu(resp.byte_5);
+
+	cnps = &rdev->stats.cnps;
+	dstat = &rdev->stats.dstat;
+	if (!cnps->is_first) {
+		/* First query done.. */
+		cnps->is_first = true;
+		cnps->prev[0].cnp_tx_pkts = tmp_counters[0].cnp_tx_pkts;
+		cnps->prev[0].cnp_tx_bytes = tmp_counters[0].cnp_tx_bytes;
+		cnps->prev[0].cnp_rx_pkts = tmp_counters[0].cnp_rx_pkts;
+		cnps->prev[0].cnp_rx_bytes = tmp_counters[0].cnp_rx_bytes;
+
+		cnps->prev[1].cnp_tx_pkts = tmp_counters[1].cnp_tx_pkts;
+		cnps->prev[1].cnp_tx_bytes = tmp_counters[1].cnp_tx_bytes;
+		cnps->prev[1].cnp_rx_pkts = tmp_counters[1].cnp_rx_pkts;
+		cnps->prev[1].cnp_rx_bytes = tmp_counters[1].cnp_rx_bytes;
+
+		dstat->prev[0].tx_pkts = roce_only_tmp[0].tx_pkts;
+		dstat->prev[0].tx_bytes = roce_only_tmp[0].tx_bytes;
+		dstat->prev[0].rx_pkts = roce_only_tmp[0].rx_pkts;
+		dstat->prev[0].rx_bytes = roce_only_tmp[0].rx_bytes;
+
+		dstat->prev[1].tx_pkts = roce_only_tmp[1].tx_pkts;
+		dstat->prev[1].tx_bytes = roce_only_tmp[1].tx_bytes;
+		dstat->prev[1].rx_pkts = roce_only_tmp[1].rx_pkts;
+		dstat->prev[1].rx_bytes = roce_only_tmp[1].rx_bytes;
+	} else {
+		u64 byte_mask, pkts_mask;
+		u64 diff;
+
+		byte_mask = bnxt_re_get_cfa_stat_mask(rdev->chip_ctx,
+						      BYTE_MASK);
+		pkts_mask = bnxt_re_get_cfa_stat_mask(rdev->chip_ctx,
+						      PKTS_MASK);
+		/*
+		 * Calculate the number of cnp packets and use
+		 * the value to calculate the CRC bytes.
+		 * Multply pkts with 4 and add it to total bytes
+		 */
+		pkts = bnxt_re_stat_diff(tmp_counters[0].cnp_tx_pkts,
+					 &cnps->prev[0].cnp_tx_pkts,
+					 pkts_mask);
+		cnps->cur[0].cnp_tx_pkts += pkts;
+		diff = bnxt_re_stat_diff(tmp_counters[0].cnp_tx_bytes,
+					 &cnps->prev[0].cnp_tx_bytes,
+					 byte_mask);
+		bytes = diff + pkts * 4;
+		cnps->cur[0].cnp_tx_bytes += bytes;
+		pkts = bnxt_re_stat_diff(tmp_counters[0].cnp_rx_pkts,
+					 &cnps->prev[0].cnp_rx_pkts,
+					 pkts_mask);
+		cnps->cur[0].cnp_rx_pkts += pkts;
+		bytes = bnxt_re_stat_diff(tmp_counters[0].cnp_rx_bytes,
+					  &cnps->prev[0].cnp_rx_bytes,
+					  byte_mask);
+		cnps->cur[0].cnp_rx_bytes += bytes;
+
+		/*
+		 * Calculate the number of cnp packets and use
+		 * the value to calculate the CRC bytes.
+		 * Multply pkts with 4 and add it to total bytes
+		 */
+		pkts = bnxt_re_stat_diff(tmp_counters[1].cnp_tx_pkts,
+					 &cnps->prev[1].cnp_tx_pkts,
+					 pkts_mask);
+		cnps->cur[1].cnp_tx_pkts += pkts;
+		diff = bnxt_re_stat_diff(tmp_counters[1].cnp_tx_bytes,
+					 &cnps->prev[1].cnp_tx_bytes,
+					 byte_mask);
+		cnps->cur[1].cnp_tx_bytes += diff + pkts * 4;
+		pkts = bnxt_re_stat_diff(tmp_counters[1].cnp_rx_pkts,
+					 &cnps->prev[1].cnp_rx_pkts,
+					 pkts_mask);
+		cnps->cur[1].cnp_rx_pkts += pkts;
+		bytes = bnxt_re_stat_diff(tmp_counters[1].cnp_rx_bytes,
+					  &cnps->prev[1].cnp_rx_bytes,
+					  byte_mask);
+		cnps->cur[1].cnp_rx_bytes += bytes;
+
+		pkts = bnxt_re_stat_diff(roce_only_tmp[0].tx_pkts,
+					 &dstat->prev[0].tx_pkts,
+					 pkts_mask);
+		dstat->cur[0].tx_pkts += pkts;
+		diff = bnxt_re_stat_diff(roce_only_tmp[0].tx_bytes,
+					 &dstat->prev[0].tx_bytes,
+					 byte_mask);
+		dstat->cur[0].tx_bytes += diff + pkts * 4;
+		pkts = bnxt_re_stat_diff(roce_only_tmp[0].rx_pkts,
+					 &dstat->prev[0].rx_pkts,
+					 pkts_mask);
+		dstat->cur[0].rx_pkts += pkts;
+
+		bytes = bnxt_re_stat_diff(roce_only_tmp[0].rx_bytes,
+					  &dstat->prev[0].rx_bytes,
+					  byte_mask);
+		dstat->cur[0].rx_bytes += bytes;
+		pkts = bnxt_re_stat_diff(roce_only_tmp[1].tx_pkts,
+					 &dstat->prev[1].tx_pkts,
+					 pkts_mask);
+		dstat->cur[1].tx_pkts += pkts;
+		diff = bnxt_re_stat_diff(roce_only_tmp[1].tx_bytes,
+					 &dstat->prev[1].tx_bytes,
+					 byte_mask);
+		dstat->cur[1].tx_bytes += diff + pkts * 4;
+		pkts = bnxt_re_stat_diff(roce_only_tmp[1].rx_pkts,
+					 &dstat->prev[1].rx_pkts,
+					 pkts_mask);
+		dstat->cur[1].rx_pkts += pkts;
+		bytes = bnxt_re_stat_diff(roce_only_tmp[1].rx_bytes,
+					  &dstat->prev[1].rx_bytes,
+					  byte_mask);
+		dstat->cur[1].rx_bytes += bytes;
+	}
+done:
+	return rc;
+}
+
+static void bnxt_re_copy_ext_stats(struct bnxt_re_dev *rdev,
+				   u8 indx, struct bnxt_qplib_ext_stat *s)
+{
+	struct bnxt_re_ext_roce_stats *e_errs;
+	struct bnxt_re_cnp_counters *cnp;
+	struct bnxt_re_ext_rstat *ext_d;
+	struct bnxt_re_ro_counters *ro;
+
+	cnp = &rdev->stats.cnps.cur[indx];
+	ro = &rdev->stats.dstat.cur[indx];
+	ext_d = &rdev->stats.dstat.ext_rstat[indx];
+	e_errs = &rdev->stats.dstat.e_errs;
+
+	cnp->cnp_tx_pkts = s->tx_cnp;
+	cnp->cnp_rx_pkts = s->rx_cnp;
+	/* In bonding mode do not duplicate other stats */
+	if (indx)
+		return;
+	cnp->ecn_marked = s->rx_ecn_marked;
+
+	ro->tx_pkts = s->tx_roce_pkts;
+	ro->tx_bytes = s->tx_roce_bytes;
+	ro->rx_pkts = s->rx_roce_pkts;
+	ro->rx_bytes = s->rx_roce_bytes;
+
+	ext_d->tx.atomic_req = s->tx_atomic_req;
+	ext_d->tx.read_req = s->tx_read_req;
+	ext_d->tx.read_resp = s->tx_read_res;
+	ext_d->tx.write_req = s->tx_write_req;
+	ext_d->tx.send_req = s->tx_send_req;
+	ext_d->rx.atomic_req = s->rx_atomic_req;
+	ext_d->rx.read_req = s->rx_read_req;
+	ext_d->rx.read_resp = s->rx_read_res;
+	ext_d->rx.write_req = s->rx_write_req;
+	ext_d->rx.send_req = s->rx_send_req;
+	ext_d->grx.rx_pkts = s->rx_roce_good_pkts;
+	ext_d->grx.rx_bytes = s->rx_roce_good_bytes;
+	ext_d->rx_dcn_payload_cut = s->rx_dcn_payload_cut;
+	ext_d->te_bypassed = s->te_bypassed;
+	e_errs->oob = s->rx_out_of_buffer;
+	e_errs->oos = s->rx_out_of_sequence;
+	e_errs->seq_err_naks_rcvd = s->seq_err_naks_rcvd;
+	e_errs->rnr_naks_rcvd = s->rnr_naks_rcvd;
+	e_errs->missing_resp = s->missing_resp;
+	e_errs->to_retransmits = s->to_retransmits;
+	e_errs->dup_req = s->dup_req;
+}
+
+static int bnxt_re_get_ext_stat(struct bnxt_re_dev *rdev)
+{
+	struct bnxt_qplib_ext_stat estat[2] = {{}, {}};
+	struct bnxt_qplib_query_stats_info sinfo;
+	u32 fid;
+	int rc;
+
+	fid = PCI_FUNC(rdev->en_dev->pdev->devfn);
+	/* Set default values for sinfo */
+	sinfo.function_id = 0xFFFFFFFF;
+	sinfo.collection_id = 0xFF;
+	sinfo.vf_valid  = false;
+	rc = bnxt_qplib_qext_stat(&rdev->rcfw, fid, &estat[0], &sinfo);
+	if (rc)
+		goto done;
+	bnxt_re_copy_ext_stats(rdev, 0, &estat[0]);
+
+done:
+	return rc;
+}
+
+static void bnxt_re_copy_rstat(struct bnxt_re_rdata_counters *d,
+			       struct ctx_hw_stats_ext *s,
+			       bool is_thor)
+{
+	d->tx_ucast_pkts = le64_to_cpu(s->tx_ucast_pkts);
+	d->tx_mcast_pkts = le64_to_cpu(s->tx_mcast_pkts);
+	d->tx_bcast_pkts = le64_to_cpu(s->tx_bcast_pkts);
+	d->tx_discard_pkts = le64_to_cpu(s->tx_discard_pkts);
+	d->tx_error_pkts = le64_to_cpu(s->tx_error_pkts);
+	d->tx_ucast_bytes = le64_to_cpu(s->tx_ucast_bytes);
+	/* Add four bytes of CRC bytes per packet */
+	d->tx_ucast_bytes +=  d->tx_ucast_pkts * 4;
+	d->tx_mcast_bytes = le64_to_cpu(s->tx_mcast_bytes);
+	d->tx_bcast_bytes = le64_to_cpu(s->tx_bcast_bytes);
+	d->rx_ucast_pkts = le64_to_cpu(s->rx_ucast_pkts);
+	d->rx_mcast_pkts = le64_to_cpu(s->rx_mcast_pkts);
+	d->rx_bcast_pkts = le64_to_cpu(s->rx_bcast_pkts);
+	d->rx_discard_pkts = le64_to_cpu(s->rx_discard_pkts);
+	d->rx_error_pkts = le64_to_cpu(s->rx_error_pkts);
+	d->rx_ucast_bytes = le64_to_cpu(s->rx_ucast_bytes);
+	d->rx_mcast_bytes = le64_to_cpu(s->rx_mcast_bytes);
+	d->rx_bcast_bytes = le64_to_cpu(s->rx_bcast_bytes);
+	if (is_thor) {
+		d->rx_agg_pkts = le64_to_cpu(s->rx_tpa_pkt);
+		d->rx_agg_bytes = le64_to_cpu(s->rx_tpa_bytes);
+		d->rx_agg_events = le64_to_cpu(s->rx_tpa_events);
+		d->rx_agg_aborts = le64_to_cpu(s->rx_tpa_errors);
+	}
+}
+
+static void bnxt_re_get_roce_data_stats(struct bnxt_re_dev *rdev)
+{
+	bool is_thor = _is_chip_gen_p5_p7(rdev->chip_ctx);
+	struct bnxt_re_rdata_counters *rstat;
+
+	rstat = &rdev->stats.dstat.rstat[0];
+	bnxt_re_copy_rstat(rstat, rdev->qplib_res.hctx->stats.dma, is_thor);
+
+}
+
+int bnxt_re_get_device_stats(struct bnxt_re_dev *rdev)
+{
+	struct bnxt_qplib_query_stats_info sinfo;
+	int rc = 0;
+
+	/* Stats are in 1s cadence */
+	if (test_bit(BNXT_RE_FLAG_ISSUE_CFA_FLOW_STATS, &rdev->flags)) {
+		if (bnxt_ext_stats_supported(rdev->chip_ctx, rdev->dev_attr->dev_cap_flags,
+					     rdev->is_virtfn))
+			rc = bnxt_re_get_ext_stat(rdev);
+		else
+			rc = bnxt_re_get_qos_stats(rdev);
+
+		if (rc && rc != -ENOMEM)
+			clear_bit(BNXT_RE_FLAG_ISSUE_CFA_FLOW_STATS,
+				  &rdev->flags);
+	}
+
+	if (test_bit(BNXT_RE_FLAG_ISSUE_ROCE_STATS, &rdev->flags)) {
+		bnxt_re_get_roce_data_stats(rdev);
+
+		/* Set default values for sinfo */
+		sinfo.function_id = 0xFFFFFFFF;
+		sinfo.collection_id = 0xFF;
+		sinfo.vf_valid  = false;
+		rc = bnxt_qplib_get_roce_error_stats(&rdev->rcfw,
+						     &rdev->stats.dstat.errs,
+						     &sinfo);
+		if (rc && rc != -ENOMEM)
+			clear_bit(BNXT_RE_FLAG_ISSUE_ROCE_STATS,
+				  &rdev->flags);
+	}
+
+	return rc;
+}
+
+static const char * const bnxt_re_stat_descs[] = {
+	"link_state",
+	"max_qp",
+	"max_srq",
+	"max_cq",
+	"max_mr",
+	"max_mw",
+	"max_ah",
+	"max_pd",
+	"active_qp",
+	"active_rc_qp",
+	"active_ud_qp",
+	"active_srq",
+	"active_cq",
+	"active_mr",
+	"active_mw",
+	"active_ah",
+	"active_pd",
+	"qp_watermark",
+	"rc_qp_watermark",
+	"ud_qp_watermark",
+	"srq_watermark",
+	"cq_watermark",
+	"mr_watermark",
+	"mw_watermark",
+	"ah_watermark",
+	"pd_watermark",
+	"resize_cq_count",
+	"hw_retransmission",
+	"recoverable_errors",
+	"rx_pkts",
+	"rx_bytes",
+	"tx_pkts",
+	"tx_bytes",
+	"cnp_tx_pkts",
+	"cnp_tx_bytes",
+	"cnp_rx_pkts",
+	"cnp_rx_bytes",
+	"roce_only_rx_pkts",
+	"roce_only_rx_bytes",
+	"roce_only_tx_pkts",
+	"roce_only_tx_bytes",
+	"rx_roce_error_pkts",
+	"rx_roce_discard_pkts",
+	"tx_roce_error_pkts",
+	"tx_roce_discards_pkts",
+	"res_oob_drop_count",
+	"tx_atomic_req",
+	"rx_atomic_req",
+	"tx_read_req",
+	"tx_read_resp",
+	"rx_read_req",
+	"rx_read_resp",
+	"tx_write_req",
+	"rx_write_req",
+	"tx_send_req",
+	"rx_send_req",
+	"rx_good_pkts",
+	"rx_good_bytes",
+	"rx_dcn_payload_cut",
+	"te_bypassed",
+	"rx_ecn_marked_pkts",
+	"max_retry_exceeded",
+	"to_retransmits",
+	"seq_err_naks_rcvd",
+	"rnr_naks_rcvd",
+	"missing_resp",
+	"dup_reqs",
+	"unrecoverable_err",
+	"bad_resp_err",
+	"local_qp_op_err",
+	"local_protection_err",
+	"mem_mgmt_op_err",
+	"remote_invalid_req_err",
+	"remote_access_err",
+	"remote_op_err",
+	"res_exceed_max",
+	"res_length_mismatch",
+	"res_exceeds_wqe",
+	"res_opcode_err",
+	"res_rx_invalid_rkey",
+	"res_rx_domain_err",
+	"res_rx_no_perm",
+	"res_rx_range_err",
+	"res_tx_invalid_rkey",
+	"res_tx_domain_err",
+	"res_tx_no_perm",
+	"res_tx_range_err",
+	"res_irrq_oflow",
+	"res_unsup_opcode",
+	"res_unaligned_atomic",
+	"res_rem_inv_err",
+	"res_mem_error64",
+	"res_srq_err",
+	"res_cmp_err",
+	"res_invalid_dup_rkey",
+	"res_wqe_format_err",
+	"res_cq_load_err",
+	"res_srq_load_err",
+	"res_tx_pci_err",
+	"res_rx_pci_err",
+	"res_oos_drop_count",
+	"num_irq_started",
+	"num_irq_stopped",
+	"poll_in_intr_en",
+	"poll_in_intr_dis",
+	"cmdq_full_dbg_cnt",
+	"fw_service_prof_type_sup",
+	"dbq_int_recv",
+	"dbq_int_en",
+	"dbq_pacing_resched",
+	"dbq_pacing_complete",
+	"dbq_pacing_alerts",
+	"dbq_dbr_fifo_reg"
+
+};
+
+static void bnxt_re_print_ext_stat(struct bnxt_re_dev *rdev,
+				   struct rdma_hw_stats *stats)
+{
+	struct bnxt_re_cnp_counters *cnp;
+	struct bnxt_re_ext_rstat *ext_s;
+
+	ext_s = &rdev->stats.dstat.ext_rstat[0];
+	cnp = &rdev->stats.cnps.cur[0];
+
+	stats->value[BNXT_RE_TX_ATOMIC_REQ] = ext_s->tx.atomic_req;
+	stats->value[BNXT_RE_RX_ATOMIC_REQ] = ext_s->rx.atomic_req;
+	stats->value[BNXT_RE_TX_READ_REQ] = ext_s->tx.read_req;
+	stats->value[BNXT_RE_TX_READ_RESP] = ext_s->tx.read_resp;
+	stats->value[BNXT_RE_RX_READ_REQ] = ext_s->rx.read_req;
+	stats->value[BNXT_RE_RX_READ_RESP] = ext_s->rx.read_resp;
+	stats->value[BNXT_RE_TX_WRITE_REQ] = ext_s->tx.write_req;
+	stats->value[BNXT_RE_RX_WRITE_REQ] = ext_s->rx.write_req;
+	stats->value[BNXT_RE_TX_SEND_REQ] = ext_s->tx.send_req;
+	stats->value[BNXT_RE_RX_SEND_REQ] = ext_s->rx.send_req;
+	stats->value[BNXT_RE_RX_GOOD_PKTS] = ext_s->grx.rx_pkts;
+	stats->value[BNXT_RE_RX_GOOD_BYTES] = ext_s->grx.rx_bytes;
+	if (_is_chip_p7(rdev->chip_ctx)) {
+		stats->value[BNXT_RE_RX_DCN_PAYLOAD_CUT] = ext_s->rx_dcn_payload_cut;
+		stats->value[BNXT_RE_TE_BYPASSED] = ext_s->te_bypassed;
+	}
+	stats->value[BNXT_RE_RX_ECN_MARKED_PKTS] = cnp->ecn_marked;
+}
+
+static void bnxt_re_print_roce_only_counters(struct bnxt_re_dev *rdev,
+					     struct rdma_hw_stats *stats)
+{
+	struct bnxt_re_ro_counters *roce_only = &rdev->stats.dstat.cur[0];
+
+	stats->value[BNXT_RE_ROCE_ONLY_RX_PKTS] = roce_only->rx_pkts;
+	stats->value[BNXT_RE_ROCE_ONLY_RX_BYTES] = roce_only->rx_bytes;
+	stats->value[BNXT_RE_ROCE_ONLY_TX_PKTS] = roce_only->tx_pkts;
+	stats->value[BNXT_RE_ROCE_ONLY_TX_BYTES] = roce_only->tx_bytes;
+}
+
+static void bnxt_re_print_normal_total_counters(struct bnxt_re_dev *rdev,
+						struct rdma_hw_stats *stats)
+{
+	struct bnxt_re_ro_counters *roce_only;
+	struct bnxt_re_cc_stat *cnps;
+
+	cnps = &rdev->stats.cnps;
+	roce_only = &rdev->stats.dstat.cur[0];
+
+	stats->value[BNXT_RE_RX_PKTS] = cnps->cur[0].cnp_rx_pkts + roce_only->rx_pkts;
+	stats->value[BNXT_RE_RX_BYTES] = cnps->cur[0].cnp_rx_bytes + roce_only->rx_bytes;
+	stats->value[BNXT_RE_TX_PKTS] = cnps->cur[0].cnp_tx_pkts + roce_only->tx_pkts;
+	stats->value[BNXT_RE_TX_BYTES] = cnps->cur[0].cnp_tx_bytes + roce_only->tx_bytes;
+}
+
+static void bnxt_re_print_normal_counters(struct bnxt_re_dev *rdev,
+					  struct rdma_hw_stats *rstats)
+{
+	struct bnxt_re_rdata_counters *stats;
+	struct bnxt_re_cc_stat *cnps;
+	bool en_disp;
+
+	stats = &rdev->stats.dstat.rstat[0];
+	cnps = &rdev->stats.cnps;
+	en_disp = !_is_chip_gen_p5_p7(rdev->chip_ctx);
+
+	bnxt_re_print_normal_total_counters(rdev, rstats);
+	if (!rdev->is_virtfn) {
+		rstats->value[BNXT_RE_CNP_TX_PKTS] = cnps->cur[0].cnp_tx_pkts;
+		if (en_disp)
+			rstats->value[BNXT_RE_CNP_TX_BYTES] = cnps->cur[0].cnp_tx_bytes;
+		rstats->value[BNXT_RE_CNP_RX_PKTS] = cnps->cur[0].cnp_rx_pkts;
+		if (en_disp)
+			rstats->value[BNXT_RE_CNP_RX_BYTES] = cnps->cur[0].cnp_rx_bytes;
+	}
+	/* Print RoCE only bytes.. CNP counters include RoCE packets also */
+	bnxt_re_print_roce_only_counters(rdev, rstats);
+
+	rstats->value[BNXT_RE_RX_ROCE_ERROR_PKTS] = stats ? stats->rx_error_pkts : 0;
+	rstats->value[BNXT_RE_RX_ROCE_DISCARD_PKTS] = stats ? stats->rx_discard_pkts : 0;
+	if (!en_disp) {
+		rstats->value[BNXT_RE_TX_ROCE_ERROR_PKTS] = stats ? stats->tx_error_pkts : 0;
+		rstats->value[BNXT_RE_TX_ROCE_DISCARDS_PKTS] = stats ? stats->tx_discard_pkts : 0;
+	}
+
+	if (bnxt_ext_stats_supported(rdev->chip_ctx, rdev->dev_attr->dev_cap_flags,
+				     rdev->is_virtfn)) {
+		rstats->value[BNXT_RE_RES_OOB_DROP_COUNT] = rdev->stats.dstat.e_errs.oob;
+		bnxt_re_print_ext_stat(rdev, rstats);
+	}
+}
+
+static void bnxt_re_copy_db_pacing_stats(struct bnxt_re_dev *rdev,
+					 struct rdma_hw_stats *stats)
+{
+	struct bnxt_re_dbr_sw_stats *dbr_sw_stats = rdev->dbr_sw_stats;
+
+	stats->value[BNXT_RE_DBQ_PACING_RESCHED] = dbr_sw_stats->dbq_pacing_resched;
+	stats->value[BNXT_RE_DBQ_PACING_CMPL] = dbr_sw_stats->dbq_pacing_complete;
+	stats->value[BNXT_RE_DBQ_PACING_ALERT] = dbr_sw_stats->dbq_pacing_alerts;
+	stats->value[BNXT_RE_DBQ_DBR_FIFO_REG] = readl_fbsd(rdev->en_dev->softc,
+						       rdev->dbr_db_fifo_reg_off, 0);
+}
+
+int bnxt_re_get_hw_stats(struct ib_device *ibdev,
+			    struct rdma_hw_stats *stats,
+			    u8 port, int index)
+{
+	struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev);
+	struct bnxt_re_ext_roce_stats *e_errs;
+	struct bnxt_re_rdata_counters *rstat;
+	struct bnxt_qplib_roce_stats *errs;
+	unsigned long tstamp_diff;
+	struct pci_dev *pdev;
+	int sched_msec;
+	int rc = 0;
+
+	if (!port || !stats)
+		return -EINVAL;
+
+	if (!rdev)
+		return -ENODEV;
+
+	if (!__bnxt_re_is_rdev_valid(rdev)) {
+		return -ENODEV;
+	}
+
+	pdev = rdev->en_dev->pdev;
+	errs = &rdev->stats.dstat.errs;
+	rstat = &rdev->stats.dstat.rstat[0];
+	e_errs = &rdev->stats.dstat.e_errs;
+#define BNXT_RE_STATS_CTX_UPDATE_TIMER	250
+	sched_msec = BNXT_RE_STATS_CTX_UPDATE_TIMER;
+	tstamp_diff = jiffies - rdev->stats.read_tstamp;
+	if (test_bit(BNXT_RE_FLAG_IBDEV_REGISTERED, &rdev->flags)) {
+		if (/* restrict_stats && */ tstamp_diff < msecs_to_jiffies(sched_msec))
+			goto skip_query;
+		rc = bnxt_re_get_device_stats(rdev);
+		if (rc)
+			dev_err(rdev_to_dev(rdev),
+				"Failed to query device stats\n");
+		rdev->stats.read_tstamp = jiffies;
+	}
+
+	if (rdev->dbr_pacing)
+		bnxt_re_copy_db_pacing_stats(rdev, stats);
+
+skip_query:
+
+	if (rdev->netdev)
+		stats->value[BNXT_RE_LINK_STATE] = bnxt_re_link_state(rdev);
+	stats->value[BNXT_RE_MAX_QP] = rdev->dev_attr->max_qp;
+	stats->value[BNXT_RE_MAX_SRQ] = rdev->dev_attr->max_srq;
+	stats->value[BNXT_RE_MAX_CQ] = rdev->dev_attr->max_cq;
+	stats->value[BNXT_RE_MAX_MR] = rdev->dev_attr->max_mr;
+	stats->value[BNXT_RE_MAX_MW] = rdev->dev_attr->max_mw;
+	stats->value[BNXT_RE_MAX_AH] = rdev->dev_attr->max_ah;
+	stats->value[BNXT_RE_MAX_PD] = rdev->dev_attr->max_pd;
+	stats->value[BNXT_RE_ACTIVE_QP] = atomic_read(&rdev->stats.rsors.qp_count);
+	stats->value[BNXT_RE_ACTIVE_RC_QP] = atomic_read(&rdev->stats.rsors.rc_qp_count);
+	stats->value[BNXT_RE_ACTIVE_UD_QP] = atomic_read(&rdev->stats.rsors.ud_qp_count);
+	stats->value[BNXT_RE_ACTIVE_SRQ] = atomic_read(&rdev->stats.rsors.srq_count);
+	stats->value[BNXT_RE_ACTIVE_CQ] = atomic_read(&rdev->stats.rsors.cq_count);
+	stats->value[BNXT_RE_ACTIVE_MR] = atomic_read(&rdev->stats.rsors.mr_count);
+	stats->value[BNXT_RE_ACTIVE_MW] = atomic_read(&rdev->stats.rsors.mw_count);
+	stats->value[BNXT_RE_ACTIVE_AH] = atomic_read(&rdev->stats.rsors.ah_count);
+	stats->value[BNXT_RE_ACTIVE_PD] = atomic_read(&rdev->stats.rsors.pd_count);
+	stats->value[BNXT_RE_QP_WATERMARK] = atomic_read(&rdev->stats.rsors.max_qp_count);
+	stats->value[BNXT_RE_RC_QP_WATERMARK] = atomic_read(&rdev->stats.rsors.max_rc_qp_count);
+	stats->value[BNXT_RE_UD_QP_WATERMARK] = atomic_read(&rdev->stats.rsors.max_ud_qp_count);
+	stats->value[BNXT_RE_SRQ_WATERMARK] = atomic_read(&rdev->stats.rsors.max_srq_count);
+	stats->value[BNXT_RE_CQ_WATERMARK] = atomic_read(&rdev->stats.rsors.max_cq_count);
+	stats->value[BNXT_RE_MR_WATERMARK] = atomic_read(&rdev->stats.rsors.max_mr_count);
+	stats->value[BNXT_RE_MW_WATERMARK] = atomic_read(&rdev->stats.rsors.max_mw_count);
+	stats->value[BNXT_RE_AH_WATERMARK] = atomic_read(&rdev->stats.rsors.max_ah_count);
+	stats->value[BNXT_RE_PD_WATERMARK] = atomic_read(&rdev->stats.rsors.max_pd_count);
+	stats->value[BNXT_RE_RESIZE_CQ_COUNT] = atomic_read(&rdev->stats.rsors.resize_count);
+	stats->value[BNXT_RE_HW_RETRANSMISSION] = BNXT_RE_HW_RETX(rdev->dev_attr->dev_cap_flags) ?  1 : 0;
+	stats->value[BNXT_RE_RECOVERABLE_ERRORS] = rstat ? rstat->tx_bcast_pkts : 0;
+
+	bnxt_re_print_normal_counters(rdev, stats);
+
+
+	stats->value[BNXT_RE_MAX_RETRY_EXCEEDED] = errs->max_retry_exceeded;
+	if (bnxt_ext_stats_supported(rdev->chip_ctx, rdev->dev_attr->dev_cap_flags,
+				     rdev->is_virtfn) &&
+	    _is_hw_retx_supported(rdev->dev_attr->dev_cap_flags)) {
+		stats->value[BNXT_RE_TO_RETRANSMITS] = e_errs->to_retransmits;
+		stats->value[BNXT_RE_SEQ_ERR_NAKS_RCVD] = e_errs->seq_err_naks_rcvd;
+		stats->value[BNXT_RE_RNR_NAKS_RCVD] = e_errs->rnr_naks_rcvd;
+		stats->value[BNXT_RE_MISSING_RESP] = e_errs->missing_resp;
+		stats->value[BNXT_RE_DUP_REQS] = e_errs->dup_req;
+	} else {
+		stats->value[BNXT_RE_TO_RETRANSMITS] = errs->to_retransmits;
+		stats->value[BNXT_RE_SEQ_ERR_NAKS_RCVD] = errs->seq_err_naks_rcvd;
+		stats->value[BNXT_RE_RNR_NAKS_RCVD] = errs->rnr_naks_rcvd;
+		stats->value[BNXT_RE_MISSING_RESP] = errs->missing_resp;
+		stats->value[BNXT_RE_DUP_REQS] = errs->dup_req;
+	}
+
+	stats->value[BNXT_RE_UNRECOVERABLE_ERR] = errs->unrecoverable_err;
+	stats->value[BNXT_RE_BAD_RESP_ERR] = errs->bad_resp_err;
+	stats->value[BNXT_RE_LOCAL_QP_OP_ERR] = errs->local_qp_op_err;
+	stats->value[BNXT_RE_LOCAL_PROTECTION_ERR] = errs->local_protection_err;
+	stats->value[BNXT_RE_MEM_MGMT_OP_ERR] = errs->mem_mgmt_op_err;
+	stats->value[BNXT_RE_REMOTE_INVALID_REQ_ERR] = errs->remote_invalid_req_err;
+	stats->value[BNXT_RE_REMOTE_ACCESS_ERR] = errs->remote_access_err;
+	stats->value[BNXT_RE_REMOTE_OP_ERR] = errs->remote_op_err;
+	stats->value[BNXT_RE_RES_EXCEED_MAX] = errs->res_exceed_max;
+	stats->value[BNXT_RE_RES_LENGTH_MISMATCH] = errs->res_length_mismatch;
+	stats->value[BNXT_RE_RES_EXCEEDS_WQE] = errs->res_exceeds_wqe;
+	stats->value[BNXT_RE_RES_OPCODE_ERR] = errs->res_opcode_err;
+	stats->value[BNXT_RE_RES_RX_INVALID_RKEY] = errs->res_rx_invalid_rkey;
+	stats->value[BNXT_RE_RES_RX_DOMAIN_ERR] = errs->res_rx_domain_err;
+	stats->value[BNXT_RE_RES_RX_NO_PERM] = errs->res_rx_no_perm;
+	stats->value[BNXT_RE_RES_RX_RANGE_ERR] = errs->res_rx_range_err;
+	stats->value[BNXT_RE_RES_TX_INVALID_RKEY] = errs->res_tx_invalid_rkey;
+	stats->value[BNXT_RE_RES_TX_DOMAIN_ERR] = errs->res_tx_domain_err;
+	stats->value[BNXT_RE_RES_TX_NO_PERM] = errs->res_tx_no_perm;
+	stats->value[BNXT_RE_RES_TX_RANGE_ERR] = errs->res_tx_range_err;
+	stats->value[BNXT_RE_RES_IRRQ_OFLOW] = errs->res_irrq_oflow;
+	stats->value[BNXT_RE_RES_UNSUP_OPCODE] = errs->res_unsup_opcode;
+	stats->value[BNXT_RE_RES_UNALIGNED_ATOMIC] = errs->res_unaligned_atomic;
+	stats->value[BNXT_RE_RES_REM_INV_ERR] = errs->res_rem_inv_err;
+	stats->value[BNXT_RE_RES_MEM_ERROR64] = errs->res_mem_error;
+	stats->value[BNXT_RE_RES_SRQ_ERR] = errs->res_srq_err;
+	stats->value[BNXT_RE_RES_CMP_ERR] = errs->res_cmp_err;
+	stats->value[BNXT_RE_RES_INVALID_DUP_RKEY] = errs->res_invalid_dup_rkey;
+	stats->value[BNXT_RE_RES_WQE_FORMAT_ERR] = errs->res_wqe_format_err;
+	stats->value[BNXT_RE_RES_CQ_LOAD_ERR] = errs->res_cq_load_err;
+	stats->value[BNXT_RE_RES_SRQ_LOAD_ERR] = errs->res_srq_load_err;
+	stats->value[BNXT_RE_RES_TX_PCI_ERR] = errs->res_tx_pci_err;
+	stats->value[BNXT_RE_RES_RX_PCI_ERR] = errs->res_rx_pci_err;
+
+
+	if (bnxt_ext_stats_supported(rdev->chip_ctx, rdev->dev_attr->dev_cap_flags,
+				     rdev->is_virtfn)) {
+	stats->value[BNXT_RE_RES_OOS_DROP_COUNT] = e_errs->oos;
+	} else {
+		/* Display on function 0 as OOS counters are chip-wide */
+		if (PCI_FUNC(pdev->devfn) == 0)
+			stats->value[BNXT_RE_RES_OOS_DROP_COUNT] = errs->res_oos_drop_count;
+	}
+	stats->value[BNXT_RE_NUM_IRQ_STARTED] = rdev->rcfw.num_irq_started;
+	stats->value[BNXT_RE_NUM_IRQ_STOPPED] = rdev->rcfw.num_irq_stopped;
+	stats->value[BNXT_RE_POLL_IN_INTR_EN] = rdev->rcfw.poll_in_intr_en;
+	stats->value[BNXT_RE_POLL_IN_INTR_DIS] = rdev->rcfw.poll_in_intr_dis;
+	stats->value[BNXT_RE_CMDQ_FULL_DBG_CNT] = rdev->rcfw.cmdq_full_dbg;
+	if (!rdev->is_virtfn)
+		stats->value[BNXT_RE_FW_SERVICE_PROF_TYPE_SUP] = is_qport_service_type_supported(rdev);
+
+	return ARRAY_SIZE(bnxt_re_stat_descs);
+}
+
+struct rdma_hw_stats *bnxt_re_alloc_hw_port_stats(struct ib_device *ibdev,
+						     u8 port_num)
+{
+	return rdma_alloc_hw_stats_struct(bnxt_re_stat_descs,
+					  ARRAY_SIZE(bnxt_re_stat_descs),
+					  RDMA_HW_STATS_DEFAULT_LIFESPAN);
+}
diff --git a/sys/dev/bnxt/bnxt_re/stats.h b/sys/dev/bnxt/bnxt_re/stats.h
new file mode 100644
index 000000000000..748d8165947b
--- /dev/null
+++ b/sys/dev/bnxt/bnxt_re/stats.h
@@ -0,0 +1,353 @@
+/*
+ * Copyright (c) 2015-2024, Broadcom. All rights reserved.  The term
+ * Broadcom refers to Broadcom Limited and/or its subsidiaries.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS''
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
+ * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
+ * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * Description: statistics related data structures
+ */
+
+#ifndef __STATS_H__
+#define __STATS_H__
+
+#define BNXT_RE_CFA_STAT_BYTES_MASK 0xFFFFFFFFF
+#define BNXT_RE_CFA_STAT_PKTS_MASK 0xFFFFFFF
+enum {
+	BYTE_MASK = 0,
+	PKTS_MASK = 1
+};
+
+struct bnxt_re_cnp_counters {
+	u64	cnp_tx_pkts;
+	u64	cnp_tx_bytes;
+	u64	cnp_rx_pkts;
+	u64	cnp_rx_bytes;
+	u64	ecn_marked;
+};
+
+struct bnxt_re_ro_counters {
+	u64	tx_pkts;
+	u64	tx_bytes;
+	u64	rx_pkts;
+	u64	rx_bytes;
+};
+
+struct bnxt_re_flow_counters {
+	struct bnxt_re_ro_counters ro_stats;
+	struct bnxt_re_cnp_counters cnp_stats;
+};
+
+struct bnxt_re_ext_cntr {
+	u64	atomic_req;
+	u64	read_req;
+	u64	read_resp;
+	u64	write_req;
+	u64	send_req;
+};
+
+struct bnxt_re_ext_good {
+	u64	rx_pkts;
+	u64	rx_bytes;
+};
+
+struct bnxt_re_ext_rstat {
+	struct bnxt_re_ext_cntr tx;
+	struct bnxt_re_ext_cntr rx;
+	struct bnxt_re_ext_good	grx;
+	u64  rx_dcn_payload_cut;
+	u64  te_bypassed;
+};
+
+struct bnxt_re_rdata_counters {
+	u64  tx_ucast_pkts;
+	u64  tx_mcast_pkts;
+	u64  tx_bcast_pkts;
+	u64  tx_discard_pkts;
+	u64  tx_error_pkts;
+	u64  tx_ucast_bytes;
+	u64  tx_mcast_bytes;
+	u64  tx_bcast_bytes;
+	u64  rx_ucast_pkts;
+	u64  rx_mcast_pkts;
+	u64  rx_bcast_pkts;
+	u64  rx_discard_pkts;
+	u64  rx_error_pkts;
+	u64  rx_ucast_bytes;
+	u64  rx_mcast_bytes;
+	u64  rx_bcast_bytes;
+	u64  rx_agg_pkts;
+	u64  rx_agg_bytes;
+	u64  rx_agg_events;
+	u64  rx_agg_aborts;
+};
+
+struct bnxt_re_cc_stat {
+	struct bnxt_re_cnp_counters prev[2];
+	struct bnxt_re_cnp_counters cur[2];
+	bool is_first;
+};
+
+struct bnxt_re_ext_roce_stats {
+	u64	oob;
+	u64	oos;
+	u64	seq_err_naks_rcvd;
+	u64	rnr_naks_rcvd;
+	u64	missing_resp;
+	u64	to_retransmits;
+	u64	dup_req;
+};
+
+struct bnxt_re_rstat {
+	struct bnxt_re_ro_counters	prev[2];
+	struct bnxt_re_ro_counters	cur[2];
+	struct bnxt_re_rdata_counters	rstat[2];
+	struct bnxt_re_ext_rstat	ext_rstat[2];
+	struct bnxt_re_ext_roce_stats	e_errs;
+	struct bnxt_qplib_roce_stats	errs;
+	unsigned long long		prev_oob;
+};
+
+struct bnxt_re_res_cntrs {
+	atomic_t qp_count;
+	atomic_t rc_qp_count;
+	atomic_t ud_qp_count;
+	atomic_t cq_count;
+	atomic_t srq_count;
+	atomic_t mr_count;
+	atomic_t mw_count;
+	atomic_t ah_count;
+	atomic_t pd_count;
+	atomic_t resize_count;
+	atomic_t max_qp_count;
+	atomic_t max_rc_qp_count;
+	atomic_t max_ud_qp_count;
+	atomic_t max_cq_count;
+	atomic_t max_srq_count;
+	atomic_t max_mr_count;
+	atomic_t max_mw_count;
+	atomic_t max_ah_count;
+	atomic_t max_pd_count;
+};
+
+struct bnxt_re_device_stats {
+	struct bnxt_re_rstat            dstat;
+	struct bnxt_re_res_cntrs        rsors;
+	struct bnxt_re_cc_stat          cnps;
+	unsigned long                   read_tstamp;
+	/* To be used in case to disable stats query from worker or change
+	 * query interval. 0 means stats_query disabled.
+	 */
+	u32				stats_query_sec;
+	/* A free running counter to be used along with stats_query_sec to
+	 * decide whether to issue the command to FW.
+	 */
+	u32				stats_query_counter;
+};
+
+static inline u64 bnxt_re_get_cfa_stat_mask(struct bnxt_qplib_chip_ctx *cctx,
+					    bool type)
+{
+	u64 mask;
+
+	if (type == BYTE_MASK) {
+		mask = BNXT_RE_CFA_STAT_BYTES_MASK; /* 36 bits */
+		if (_is_chip_gen_p5_p7(cctx))
+			mask >>= 0x01; /* 35 bits */
+	} else {
+		mask = BNXT_RE_CFA_STAT_PKTS_MASK; /* 28 bits */
+		if (_is_chip_gen_p5_p7(cctx))
+			mask |= (0x10000000ULL); /* 29 bits */
+	}
+
+	return mask;
+}
+
+static inline u64 bnxt_re_stat_diff(u64 cur, u64 *prev, u64 mask)
+{
+	u64 diff;
+
+	if (!cur)
+		return 0;
+	diff = (cur - *prev) & mask;
+	if (diff)
+		*prev = cur;
+	return diff;
+}
+
+static inline void bnxt_re_clear_rsors_stat(struct bnxt_re_res_cntrs *rsors)
+{
+	atomic_set(&rsors->qp_count, 0);
+	atomic_set(&rsors->cq_count, 0);
+	atomic_set(&rsors->srq_count, 0);
+	atomic_set(&rsors->mr_count, 0);
+	atomic_set(&rsors->mw_count, 0);
+	atomic_set(&rsors->ah_count, 0);
+	atomic_set(&rsors->pd_count, 0);
+	atomic_set(&rsors->resize_count, 0);
+	atomic_set(&rsors->max_qp_count, 0);
+	atomic_set(&rsors->max_cq_count, 0);
+	atomic_set(&rsors->max_srq_count, 0);
+	atomic_set(&rsors->max_mr_count, 0);
+	atomic_set(&rsors->max_mw_count, 0);
+	atomic_set(&rsors->max_ah_count, 0);
+	atomic_set(&rsors->max_pd_count, 0);
+	atomic_set(&rsors->max_rc_qp_count, 0);
+	atomic_set(&rsors->max_ud_qp_count, 0);
+}
+
+enum bnxt_re_hw_stats {
+	BNXT_RE_LINK_STATE,
+	BNXT_RE_MAX_QP,
+	BNXT_RE_MAX_SRQ,
+	BNXT_RE_MAX_CQ,
+	BNXT_RE_MAX_MR,
+	BNXT_RE_MAX_MW,
+	BNXT_RE_MAX_AH,
+	BNXT_RE_MAX_PD,
+	BNXT_RE_ACTIVE_QP,
+	BNXT_RE_ACTIVE_RC_QP,
+	BNXT_RE_ACTIVE_UD_QP,
+	BNXT_RE_ACTIVE_SRQ,
+	BNXT_RE_ACTIVE_CQ,
+	BNXT_RE_ACTIVE_MR,
+	BNXT_RE_ACTIVE_MW,
+	BNXT_RE_ACTIVE_AH,
+	BNXT_RE_ACTIVE_PD,
+	BNXT_RE_QP_WATERMARK,
+	BNXT_RE_RC_QP_WATERMARK,
+	BNXT_RE_UD_QP_WATERMARK,
+	BNXT_RE_SRQ_WATERMARK,
+	BNXT_RE_CQ_WATERMARK,
+	BNXT_RE_MR_WATERMARK,
+	BNXT_RE_MW_WATERMARK,
+	BNXT_RE_AH_WATERMARK,
+	BNXT_RE_PD_WATERMARK,
+	BNXT_RE_RESIZE_CQ_COUNT,
+	BNXT_RE_HW_RETRANSMISSION,
+	BNXT_RE_RECOVERABLE_ERRORS,
+	BNXT_RE_RX_PKTS,
+	BNXT_RE_RX_BYTES,
+	BNXT_RE_TX_PKTS,
+	BNXT_RE_TX_BYTES,
+	BNXT_RE_CNP_TX_PKTS,
+	BNXT_RE_CNP_TX_BYTES,
+	BNXT_RE_CNP_RX_PKTS,
+	BNXT_RE_CNP_RX_BYTES,
+	BNXT_RE_ROCE_ONLY_RX_PKTS,
+	BNXT_RE_ROCE_ONLY_RX_BYTES,
+	BNXT_RE_ROCE_ONLY_TX_PKTS,
+	BNXT_RE_ROCE_ONLY_TX_BYTES,
+	BNXT_RE_RX_ROCE_ERROR_PKTS,
+	BNXT_RE_RX_ROCE_DISCARD_PKTS,
+	BNXT_RE_TX_ROCE_ERROR_PKTS,
+	BNXT_RE_TX_ROCE_DISCARDS_PKTS,
+	BNXT_RE_RES_OOB_DROP_COUNT,
+	BNXT_RE_TX_ATOMIC_REQ,
+	BNXT_RE_RX_ATOMIC_REQ,
+	BNXT_RE_TX_READ_REQ,
+	BNXT_RE_TX_READ_RESP,
+	BNXT_RE_RX_READ_REQ,
+	BNXT_RE_RX_READ_RESP,
+	BNXT_RE_TX_WRITE_REQ,
+	BNXT_RE_RX_WRITE_REQ,
+	BNXT_RE_TX_SEND_REQ,
+	BNXT_RE_RX_SEND_REQ,
+	BNXT_RE_RX_GOOD_PKTS,
+	BNXT_RE_RX_GOOD_BYTES,
+	BNXT_RE_RX_DCN_PAYLOAD_CUT,
+	BNXT_RE_TE_BYPASSED,
+	BNXT_RE_RX_ECN_MARKED_PKTS,
+	BNXT_RE_MAX_RETRY_EXCEEDED,
+	BNXT_RE_TO_RETRANSMITS,
+	BNXT_RE_SEQ_ERR_NAKS_RCVD,
+	BNXT_RE_RNR_NAKS_RCVD,
+	BNXT_RE_MISSING_RESP,
+	BNXT_RE_DUP_REQS,
+	BNXT_RE_UNRECOVERABLE_ERR,
+	BNXT_RE_BAD_RESP_ERR,
+	BNXT_RE_LOCAL_QP_OP_ERR,
+	BNXT_RE_LOCAL_PROTECTION_ERR,
+	BNXT_RE_MEM_MGMT_OP_ERR,
+	BNXT_RE_REMOTE_INVALID_REQ_ERR,
+	BNXT_RE_REMOTE_ACCESS_ERR,
+	BNXT_RE_REMOTE_OP_ERR,
+	BNXT_RE_RES_EXCEED_MAX,
+	BNXT_RE_RES_LENGTH_MISMATCH,
+	BNXT_RE_RES_EXCEEDS_WQE,
+	BNXT_RE_RES_OPCODE_ERR,
+	BNXT_RE_RES_RX_INVALID_RKEY,
+	BNXT_RE_RES_RX_DOMAIN_ERR,
+	BNXT_RE_RES_RX_NO_PERM,
+	BNXT_RE_RES_RX_RANGE_ERR,
+	BNXT_RE_RES_TX_INVALID_RKEY,
+	BNXT_RE_RES_TX_DOMAIN_ERR,
+	BNXT_RE_RES_TX_NO_PERM,
+	BNXT_RE_RES_TX_RANGE_ERR,
+	BNXT_RE_RES_IRRQ_OFLOW,
+	BNXT_RE_RES_UNSUP_OPCODE,
+	BNXT_RE_RES_UNALIGNED_ATOMIC,
+	BNXT_RE_RES_REM_INV_ERR,
+	BNXT_RE_RES_MEM_ERROR64,
+	BNXT_RE_RES_SRQ_ERR,
+	BNXT_RE_RES_CMP_ERR,
+	BNXT_RE_RES_INVALID_DUP_RKEY,
+	BNXT_RE_RES_WQE_FORMAT_ERR,
+	BNXT_RE_RES_CQ_LOAD_ERR,
+	BNXT_RE_RES_SRQ_LOAD_ERR,
+	BNXT_RE_RES_TX_PCI_ERR,
+	BNXT_RE_RES_RX_PCI_ERR,
+	BNXT_RE_RES_OOS_DROP_COUNT,
+	BNXT_RE_NUM_IRQ_STARTED,
+	BNXT_RE_NUM_IRQ_STOPPED,
+	BNXT_RE_POLL_IN_INTR_EN,
+	BNXT_RE_POLL_IN_INTR_DIS,
+	BNXT_RE_CMDQ_FULL_DBG_CNT,
+	BNXT_RE_FW_SERVICE_PROF_TYPE_SUP,
+	BNXT_RE_DBQ_INT_RECV,
+	BNXT_RE_DBQ_INT_EN,
+	BNXT_RE_DBQ_PACING_RESCHED,
+	BNXT_RE_DBQ_PACING_CMPL,
+	BNXT_RE_DBQ_PACING_ALERT,
+	BNXT_RE_DBQ_DBR_FIFO_REG,
+	BNXT_RE_DBQ_NUM_EXT_COUNTERS
+};
+
+#define BNXT_RE_NUM_STD_COUNTERS (BNXT_RE_OUT_OF_SEQ_ERR + 1)
+
+struct bnxt_re_stats {
+	struct bnxt_qplib_roce_stats    errs;
+	struct bnxt_qplib_ext_stat      ext_stat;
+};
+
+struct rdma_hw_stats *bnxt_re_alloc_hw_port_stats(struct ib_device *ibdev,
+						     u8 port_num);
+int bnxt_re_get_hw_stats(struct ib_device *ibdev,
+			    struct rdma_hw_stats *stats,
+			    u8 port, int index);
+int bnxt_re_get_device_stats(struct bnxt_re_dev *rdev);
+int bnxt_re_get_flow_stats_from_service_pf(struct bnxt_re_dev *rdev,
+				struct bnxt_re_flow_counters *stats,
+				struct bnxt_qplib_query_stats_info *sinfo);
+int bnxt_re_get_qos_stats(struct bnxt_re_dev *rdev);
+#endif /* __STATS_H__ */
diff --git a/sys/modules/bnxt/bnxt_re/Makefile b/sys/modules/bnxt/bnxt_re/Makefile
new file mode 100644
index 000000000000..d6aa701dbae6
--- /dev/null
+++ b/sys/modules/bnxt/bnxt_re/Makefile
@@ -0,0 +1,22 @@
+.PATH:	${SRCTOP}/sys/dev/bnxt/bnxt_re
+
+KMOD=bnxt_re
+SRCS	+= ib_verbs.c ib_verbs.h
+SRCS	+= qplib_fp.c qplib_fp.h
+SRCS	+= qplib_sp.c qplib_sp.h
+SRCS	+= qplib_res.c qplib_res.h
+SRCS	+= qplib_rcfw.c qplib_rcfw.h
+SRCS	+= stats.c stats.h
+SRCS	+= main.c bnxt_re.h
+SRCS	+= opt_inet.h opt_inet6.h opt_ratelimit.h
+SRCS	+= ${LINUXKPI_GENSRCS}
+
+CFLAGS+= -I${SRCTOP}/sys/dev/bnxt/bnxt_en
+CFLAGS+= -I${SRCTOP}/sys/ofed/include
+CFLAGS+= -I${SRCTOP}/sys/ofed/include/uapi
+CFLAGS+= ${LINUXKPI_INCLUDES}
+CFLAGS+= -DCONFIG_INFINIBAND_USER_MEM
+
+.include <bsd.kmod.mk>
+
+CFLAGS+= -Wno-cast-qual -Wno-pointer-arith ${GCC_MS_EXTENSIONS}