diff --git a/sys/dev/bnxt/bnxt_re/bnxt_re-abi.h b/sys/dev/bnxt/bnxt_re/bnxt_re-abi.h new file mode 100644 index 000000000000..8f48609e7f6f --- /dev/null +++ b/sys/dev/bnxt/bnxt_re/bnxt_re-abi.h @@ -0,0 +1,177 @@ +/* + * Copyright (c) 2015-2024, Broadcom. All rights reserved. The term + * Broadcom refers to Broadcom Limited and/or its subsidiaries. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR + * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE + * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN + * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * Description: Uverbs ABI header file + */ + +#ifndef __BNXT_RE_UVERBS_ABI_H__ +#define __BNXT_RE_UVERBS_ABI_H__ + +#include +#include + +#define BNXT_RE_ABI_VERSION 6 + +enum { + BNXT_RE_COMP_MASK_UCNTX_WC_DPI_ENABLED = 0x01, + BNXT_RE_COMP_MASK_UCNTX_POW2_DISABLED = 0x02, + BNXT_RE_COMP_MASK_UCNTX_RSVD_WQE_DISABLED = 0x04, + BNXT_RE_COMP_MASK_UCNTX_MQP_EX_SUPPORTED = 0x08, + BNXT_RE_COMP_MASK_UCNTX_DBR_PACING_ENABLED = 0x10, + BNXT_RE_COMP_MASK_UCNTX_DBR_RECOVERY_ENABLED = 0x20, + BNXT_RE_COMP_MASK_UCNTX_HW_RETX_ENABLED = 0x40 +}; + +enum { + BNXT_RE_COMP_MASK_REQ_UCNTX_POW2_SUPPORT = 0x01, + BNXT_RE_COMP_MASK_REQ_UCNTX_RSVD_WQE = 0x02 +}; + +struct bnxt_re_uctx_req { + __aligned_u64 comp_mask; +}; + +#define BNXT_RE_CHIP_ID0_CHIP_NUM_SFT 0x00 +#define BNXT_RE_CHIP_ID0_CHIP_REV_SFT 0x10 +#define BNXT_RE_CHIP_ID0_CHIP_MET_SFT 0x18 +struct bnxt_re_uctx_resp { + __u32 dev_id; + __u32 max_qp; + __u32 pg_size; + __u32 cqe_sz; + __u32 max_cqd; + __u32 chip_id0; + __u32 chip_id1; + __u32 modes; + __aligned_u64 comp_mask; +} __attribute__((packed)); + +enum { + BNXT_RE_COMP_MASK_PD_HAS_WC_DPI = 0x01, + BNXT_RE_COMP_MASK_PD_HAS_DBR_BAR_ADDR = 0x02, +}; + +struct bnxt_re_pd_resp { + __u32 pdid; + __u32 dpi; + __u64 dbr; + __u64 comp_mask; + __u32 wcdpi; + __u64 dbr_bar_addr; +} __attribute__((packed)); + +enum { + BNXT_RE_COMP_MASK_CQ_HAS_DB_INFO = 0x01, + BNXT_RE_COMP_MASK_CQ_HAS_WC_DPI = 0x02, + BNXT_RE_COMP_MASK_CQ_HAS_CQ_PAGE = 0x04, +}; + +enum { + BNXT_RE_COMP_MASK_CQ_REQ_HAS_CAP_MASK = 0x1 +}; + +enum { + BNXT_RE_COMP_MASK_CQ_REQ_CAP_DBR_RECOVERY = 0x1, + BNXT_RE_COMP_MASK_CQ_REQ_CAP_DBR_PACING_NOTIFY = 0x2 +}; + +#define BNXT_RE_IS_DBR_PACING_NOTIFY_CQ(_req) \ + (_req.comp_mask & BNXT_RE_COMP_MASK_CQ_REQ_HAS_CAP_MASK && \ + _req.cq_capability & BNXT_RE_COMP_MASK_CQ_REQ_CAP_DBR_PACING_NOTIFY) + +#define BNXT_RE_IS_DBR_RECOV_CQ(_req) \ + (_req.comp_mask & BNXT_RE_COMP_MASK_CQ_REQ_HAS_CAP_MASK && \ + _req.cq_capability & BNXT_RE_COMP_MASK_CQ_REQ_CAP_DBR_RECOVERY) + +struct bnxt_re_cq_req { + __u64 cq_va; + __u64 cq_handle; + __aligned_u64 comp_mask; + __u16 cq_capability; +} __attribute__((packed)); + +struct bnxt_re_cq_resp { + __u32 cqid; + __u32 tail; + __u32 phase; + __u32 rsvd; + __aligned_u64 comp_mask; + __u32 dpi; + __u64 dbr; + __u32 wcdpi; + __u64 uctx_cq_page; +} __attribute__((packed)); + +struct bnxt_re_resize_cq_req { + __u64 cq_va; +} __attribute__((packed)); + +struct bnxt_re_qp_req { + __u64 qpsva; + __u64 qprva; + __u64 qp_handle; +} __attribute__((packed)); + +struct bnxt_re_qp_resp { + __u32 qpid; +} __attribute__((packed)); + +struct bnxt_re_srq_req { + __u64 srqva; + __u64 srq_handle; +} __attribute__((packed)); + +struct bnxt_re_srq_resp { + __u32 srqid; +} __attribute__((packed)); + +/* Modify QP */ +enum { + BNXT_RE_COMP_MASK_MQP_EX_PPP_REQ_EN_MASK = 0x1, + BNXT_RE_COMP_MASK_MQP_EX_PPP_REQ_EN = 0x1, + BNXT_RE_COMP_MASK_MQP_EX_PATH_MTU_MASK = 0x2 +}; + +struct bnxt_re_modify_qp_ex_req { + __aligned_u64 comp_mask; + __u32 dpi; + __u32 rsvd; +} __packed; + +struct bnxt_re_modify_qp_ex_resp { + __aligned_u64 comp_mask; + __u32 ppp_st_idx; + __u32 path_mtu; +} __packed; + +enum bnxt_re_shpg_offt { + BNXT_RE_BEG_RESV_OFFT = 0x00, + BNXT_RE_AVID_OFFT = 0x10, + BNXT_RE_AVID_SIZE = 0x04, + BNXT_RE_END_RESV_OFFT = 0xFF0 +}; +#endif diff --git a/sys/dev/bnxt/bnxt_re/bnxt_re.h b/sys/dev/bnxt/bnxt_re/bnxt_re.h new file mode 100644 index 000000000000..3bf0bbeb1061 --- /dev/null +++ b/sys/dev/bnxt/bnxt_re/bnxt_re.h @@ -0,0 +1,1075 @@ +/* + * Copyright (c) 2015-2024, Broadcom. All rights reserved. The term + * Broadcom refers to Broadcom Limited and/or its subsidiaries. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR + * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE + * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN + * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * Description: main (header) + */ + +#ifndef __BNXT_RE_H__ +#define __BNXT_RE_H__ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "bnxt.h" +#include "bnxt_ulp.h" +#include "hsi_struct_def.h" +#include "qplib_res.h" +#include "qplib_sp.h" +#include "qplib_fp.h" +#include "qplib_rcfw.h" +#include "ib_verbs.h" +#include "stats.h" + +#define ROCE_DRV_MODULE_NAME "bnxt_re" +#define ROCE_DRV_MODULE_VERSION "230.0.133.0" +#define ROCE_DRV_MODULE_RELDATE "April 22, 2024" + +#define BNXT_RE_REF_WAIT_COUNT 20 +#define BNXT_RE_ROCE_V1_ETH_TYPE 0x8915 +#define BNXT_RE_ROCE_V2_PORT_NO 4791 +#define BNXT_RE_RES_FREE_WAIT_COUNT 1000 + +#define BNXT_RE_PAGE_SHIFT_4K (12) +#define BNXT_RE_PAGE_SHIFT_8K (13) +#define BNXT_RE_PAGE_SHIFT_64K (16) +#define BNXT_RE_PAGE_SHIFT_2M (21) +#define BNXT_RE_PAGE_SHIFT_8M (23) +#define BNXT_RE_PAGE_SHIFT_1G (30) + +#define BNXT_RE_PAGE_SIZE_4K BIT(BNXT_RE_PAGE_SHIFT_4K) +#define BNXT_RE_PAGE_SIZE_8K BIT(BNXT_RE_PAGE_SHIFT_8K) +#define BNXT_RE_PAGE_SIZE_64K BIT(BNXT_RE_PAGE_SHIFT_64K) +#define BNXT_RE_PAGE_SIZE_2M BIT(BNXT_RE_PAGE_SHIFT_2M) +#define BNXT_RE_PAGE_SIZE_8M BIT(BNXT_RE_PAGE_SHIFT_8M) +#define BNXT_RE_PAGE_SIZE_1G BIT(BNXT_RE_PAGE_SHIFT_1G) + +#define BNXT_RE_MAX_MR_SIZE_LOW BIT(BNXT_RE_PAGE_SHIFT_1G) +#define BNXT_RE_MAX_MR_SIZE_HIGH BIT(39) +#define BNXT_RE_MAX_MR_SIZE BNXT_RE_MAX_MR_SIZE_HIGH + +/* Number of MRs to reserve for PF, leaving remainder for VFs */ +#define BNXT_RE_RESVD_MR_FOR_PF (32 * 1024) +#define BNXT_RE_MAX_GID_PER_VF 128 + +#define BNXT_RE_MAX_VF_QPS_PER_PF (6 * 1024) + +/** + * min_not_zero - return the minimum that is _not_ zero, unless both are zero + * @x: value1 + * @y: value2 + */ +#define min_not_zero(x, y) ({ \ + typeof(x) __x = (x); \ + typeof(y) __y = (y); \ + __x == 0 ? __y : ((__y == 0) ? __x : min(__x, __y)); }) + +struct ib_mr_init_attr { + int max_reg_descriptors; + u32 flags; +}; + +struct bnxt_re_dev; + +int bnxt_re_register_netdevice_notifier(struct notifier_block *nb); +int bnxt_re_unregister_netdevice_notifier(struct notifier_block *nb); +int ib_register_device_compat(struct bnxt_re_dev *rdev); + +#ifndef __struct_group +#define __struct_group(TAG, NAME, ATTRS, MEMBERS...) \ + union { \ + struct { MEMBERS } ATTRS; \ + struct TAG { MEMBERS } ATTRS NAME; \ + } +#endif /* __struct_group */ +#ifndef struct_group_attr +#define struct_group_attr(NAME, ATTRS, MEMBERS...) \ + __struct_group(/* no tag */, NAME, ATTRS, MEMBERS) +#endif /* struct_group_attr */ +/* + * Percentage of resources of each type reserved for PF. + * Remaining resources are divided equally among VFs. + * [0, 100] + */ + +#define BNXT_RE_RQ_WQE_THRESHOLD 32 +#define BNXT_RE_UD_QP_HW_STALL 0x400000 + +/* + * Setting the default ack delay value to 16, which means + * the default timeout is approx. 260ms(4 usec * 2 ^(timeout)) + */ + +#define BNXT_RE_DEFAULT_ACK_DELAY 16 +#define BNXT_RE_BOND_PF_MAX 2 + +#define BNXT_RE_STATS_CTX_UPDATE_TIMER 250 +#define BNXT_RE_30SEC_MSEC (30 * 1000) + +#define BNXT_RE_BOND_RESCHED_CNT 10 + +#define BNXT_RE_CHIP_NUM_57454 0xC454 +#define BNXT_RE_CHIP_NUM_57452 0xC452 + +#define BNXT_RE_CHIP_NUM_5745X(chip_num) \ + ((chip_num) == BNXT_RE_CHIP_NUM_57454 || \ + (chip_num) == BNXT_RE_CHIP_NUM_57452) + +#define BNXT_RE_MIN_KERNEL_QP_TX_DEPTH 4096 +#define BNXT_RE_STOP_QPS_BUDGET 200 + +#define BNXT_RE_HWRM_CMD_TIMEOUT(rdev) \ + ((rdev)->chip_ctx->hwrm_cmd_max_timeout * 1000) + +extern unsigned int min_tx_depth; +extern struct mutex bnxt_re_dev_lock; +extern struct mutex bnxt_re_mutex; +extern struct list_head bnxt_re_dev_list; + +struct bnxt_re_ring_attr { + dma_addr_t *dma_arr; + int pages; + int type; + u32 depth; + u32 lrid; /* Logical ring id */ + u16 flags; + u8 mode; + u8 rsvd; +}; + +#define BNXT_RE_MAX_DEVICES 256 +#define BNXT_RE_MSIX_FROM_MOD_PARAM -1 +#define BNXT_RE_MIN_MSIX 2 +#define BNXT_RE_MAX_MSIX_VF 2 +#define BNXT_RE_MAX_MSIX_PF 9 +#define BNXT_RE_MAX_MSIX_NPAR_PF 5 +#define BNXT_RE_MAX_MSIX 64 +#define BNXT_RE_MAX_MSIX_GEN_P5_PF BNXT_RE_MAX_MSIX +#define BNXT_RE_GEN_P5_MAX_VF 64 + +struct bnxt_re_nq_record { + struct bnxt_msix_entry msix_entries[BNXT_RE_MAX_MSIX]; + /* FP Notification Queue (CQ & SRQ) */ + struct bnxt_qplib_nq nq[BNXT_RE_MAX_MSIX]; + int num_msix; + int max_init; + struct mutex load_lock; +}; + +struct bnxt_re_work { + struct work_struct work; + unsigned long event; + struct bnxt_re_dev *rdev; + struct ifnet *vlan_dev; + bool do_lag; + + /* netdev where we received the event */ + struct ifnet *netdev; + struct auxiliary_device *adev; +}; + +/* + * Data structure and defines to handle + * recovery + */ +#define BNXT_RE_RECOVERY_IB_UNINIT_WAIT_RETRY 20 +#define BNXT_RE_RECOVERY_IB_UNINIT_WAIT_TIME_MS 30000 /* 30sec timeout */ +#define BNXT_RE_PRE_RECOVERY_REMOVE 0x1 +#define BNXT_RE_COMPLETE_REMOVE 0x2 +#define BNXT_RE_POST_RECOVERY_INIT 0x4 +#define BNXT_RE_COMPLETE_INIT 0x8 +#define BNXT_RE_COMPLETE_SHUTDOWN 0x10 + +/* QP1 SQ entry data strucutre */ +struct bnxt_re_sqp_entries { + u64 wrid; + struct bnxt_qplib_sge sge; + /* For storing the actual qp1 cqe */ + struct bnxt_qplib_cqe cqe; + struct bnxt_re_qp *qp1_qp; +}; + +/* GSI QP mode enum */ +enum bnxt_re_gsi_mode { + BNXT_RE_GSI_MODE_INVALID = 0, + BNXT_RE_GSI_MODE_ALL = 1, + BNXT_RE_GSI_MODE_ROCE_V1, + BNXT_RE_GSI_MODE_ROCE_V2_IPV4, + BNXT_RE_GSI_MODE_ROCE_V2_IPV6, + BNXT_RE_GSI_MODE_UD +}; + +enum bnxt_re_roce_cap { + BNXT_RE_FLAG_ROCEV1_CAP = 1, + BNXT_RE_FLAG_ROCEV2_CAP, + BNXT_RE_FLAG_ROCEV1_V2_CAP, +}; + +#define BNXT_RE_MAX_GSI_SQP_ENTRIES 1024 +struct bnxt_re_gsi_context { + u8 gsi_qp_mode; + bool first_cq_created; + /* Start: used only in gsi_mode_all */ + struct bnxt_re_qp *gsi_qp; + struct bnxt_re_qp *gsi_sqp; + struct bnxt_re_ah *gsi_sah; + struct bnxt_re_sqp_entries *sqp_tbl; + /* End: used only in gsi_mode_all */ +}; + +struct bnxt_re_tc_rec { + u8 cos_id_roce; + u8 tc_roce; + u8 cos_id_cnp; + u8 tc_cnp; + u8 tc_def; + u8 cos_id_def; + u8 max_tc; + u8 roce_prio; + u8 cnp_prio; + u8 roce_dscp; + u8 cnp_dscp; + u8 prio_valid; + u8 dscp_valid; + bool ecn_enabled; + bool serv_type_enabled; + u64 cnp_dscp_bv; + u64 roce_dscp_bv; +}; + +struct bnxt_re_dscp2pri { + u8 dscp; + u8 mask; + u8 pri; +}; + +struct bnxt_re_cos2bw_cfg { + u8 pad[3]; + struct_group_attr(cfg, __packed, + u8 queue_id; + __le32 min_bw; + __le32 max_bw; + u8 tsa; + u8 pri_lvl; + u8 bw_weight; + ); + u8 unused; +}; + +#define BNXT_RE_AEQ_IDX 0 +#define BNXT_RE_MAX_SGID_ENTRIES 256 + +#define BNXT_RE_DBGFS_FILE_MEM 65536 +enum { + BNXT_RE_STATS_QUERY = 1, + BNXT_RE_QP_QUERY = 2, + BNXT_RE_SERVICE_FN_QUERY = 3, +}; + +struct bnxt_re_dbg_file { + struct bnxt_re_dev *rdev; + u32 type; + union { + struct bnxt_qplib_query_stats_info sinfo; + struct bnxt_qplib_query_fn_info fninfo; + }params; + char dbg_buf[BNXT_RE_DBGFS_FILE_MEM]; +}; + +struct bnxt_re_debug_entries { + /* Dir entries */ + struct dentry *qpinfo_dir; + struct dentry *service_fn_dir; + /* file entries */ + struct dentry *stat_query; + struct bnxt_re_dbg_file stat_file; + struct dentry *qplist_query; + struct bnxt_re_dbg_file qp_file; + struct dentry *service_fn_query; + struct bnxt_re_dbg_file service_fn_file; +}; + +struct bnxt_re_en_dev_info { + struct list_head en_list; + struct bnxt_en_dev *en_dev; + struct bnxt_re_dev *rdev; + unsigned long flags; +#define BNXT_RE_FLAG_EN_DEV_NETDEV_REG 0 +#define BNXT_RE_FLAG_EN_DEV_PRIMARY_DEV 1 +#define BNXT_RE_FLAG_EN_DEV_SECONDARY_DEV 2 + u8 wqe_mode; + u8 gsi_mode; + bool te_bypass; + bool ib_uninit_done; + u32 num_msix_requested; + wait_queue_head_t waitq; +}; + +#define BNXT_RE_DB_FIFO_ROOM_MASK_P5 0x1FFF8000 +#define BNXT_RE_MAX_FIFO_DEPTH_P5 0x2c00 +#define BNXT_RE_DB_FIFO_ROOM_SHIFT 15 + +#define BNXT_RE_DB_FIFO_ROOM_MASK_P7 0x3FFF8000 +#define BNXT_RE_MAX_FIFO_DEPTH_P7 0x8000 + +#define BNXT_RE_DB_FIFO_ROOM_MASK(ctx) \ + (_is_chip_p7((ctx)) ? \ + BNXT_RE_DB_FIFO_ROOM_MASK_P7 :\ + BNXT_RE_DB_FIFO_ROOM_MASK_P5) +#define BNXT_RE_MAX_FIFO_DEPTH(ctx) \ + (_is_chip_p7((ctx)) ? \ + BNXT_RE_MAX_FIFO_DEPTH_P7 :\ + BNXT_RE_MAX_FIFO_DEPTH_P5) + +struct bnxt_dbq_nq_list { + int num_nql_entries; + u16 nq_id[16]; +}; + +#define BNXT_RE_ASYNC_ERR_REP_BASE(_type) \ + (ASYNC_EVENT_CMPL_ERROR_REPORT_BASE_EVENT_DATA1_ERROR_##_type) + +#define BNXT_RE_ASYNC_ERR_DBR_TRESH(_type) \ + (ASYNC_EVENT_CMPL_ERROR_REPORT_DOORBELL_DROP_THRESHOLD_##_type) + +#define BNXT_RE_EVENT_DBR_EPOCH(data) \ + (((data) & \ + BNXT_RE_ASYNC_ERR_DBR_TRESH(EVENT_DATA1_EPOCH_MASK)) >> \ + BNXT_RE_ASYNC_ERR_DBR_TRESH(EVENT_DATA1_EPOCH_SFT)) + +#define BNXT_RE_EVENT_ERROR_REPORT_TYPE(data1) \ + (((data1) & \ + BNXT_RE_ASYNC_ERR_REP_BASE(TYPE_MASK)) >> \ + BNXT_RE_ASYNC_ERR_REP_BASE(TYPE_SFT)) + +#define BNXT_RE_DBR_LIST_ADD(_rdev, _res, _type) \ +{ \ + spin_lock(&(_rdev)->res_list[_type].lock); \ + list_add_tail(&(_res)->dbr_list, \ + &(_rdev)->res_list[_type].head); \ + spin_unlock(&(_rdev)->res_list[_type].lock); \ +} + +#define BNXT_RE_DBR_LIST_DEL(_rdev, _res, _type) \ +{ \ + spin_lock(&(_rdev)->res_list[_type].lock); \ + list_del(&(_res)->dbr_list); \ + spin_unlock(&(_rdev)->res_list[_type].lock); \ +} + +#define BNXT_RE_CQ_PAGE_LIST_ADD(_uctx, _cq) \ +{ \ + mutex_lock(&(_uctx)->cq_lock); \ + list_add_tail(&(_cq)->cq_list, &(_uctx)->cq_list); \ + mutex_unlock(&(_uctx)->cq_lock); \ +} + +#define BNXT_RE_CQ_PAGE_LIST_DEL(_uctx, _cq) \ +{ \ + mutex_lock(&(_uctx)->cq_lock); \ + list_del(&(_cq)->cq_list); \ + mutex_unlock(&(_uctx)->cq_lock); \ +} + +#define BNXT_RE_NETDEV_EVENT(event, x) \ + do { \ + if ((event) == (x)) \ + return #x; \ + } while (0) + +/* Do not change the seq of this enum which is followed by dbr recov */ +enum { + BNXT_RE_RES_TYPE_CQ = 0, + BNXT_RE_RES_TYPE_UCTX, + BNXT_RE_RES_TYPE_QP, + BNXT_RE_RES_TYPE_SRQ, + BNXT_RE_RES_TYPE_MAX +}; + +struct bnxt_re_dbr_res_list { + struct list_head head; + spinlock_t lock; +}; + +struct bnxt_re_dbr_drop_recov_work { + struct work_struct work; + struct bnxt_re_dev *rdev; + u32 curr_epoch; +}; + +struct bnxt_re_aer_work { + struct work_struct work; + struct bnxt_re_dev *rdev; +}; + +struct bnxt_re_dbq_stats { + u64 fifo_occup_slab_1; + u64 fifo_occup_slab_2; + u64 fifo_occup_slab_3; + u64 fifo_occup_slab_4; + u64 fifo_occup_water_mark; + u64 do_pacing_slab_1; + u64 do_pacing_slab_2; + u64 do_pacing_slab_3; + u64 do_pacing_slab_4; + u64 do_pacing_slab_5; + u64 do_pacing_water_mark; +}; + +/* Device debug statistics */ +struct bnxt_re_drv_dbg_stats { + struct bnxt_re_dbq_stats dbq; +}; + +/* DB pacing counters */ +struct bnxt_re_dbr_sw_stats { + u64 dbq_int_recv; + u64 dbq_int_en; + u64 dbq_pacing_resched; + u64 dbq_pacing_complete; + u64 dbq_pacing_alerts; + u64 dbr_drop_recov_events; + u64 dbr_drop_recov_timeouts; + u64 dbr_drop_recov_timeout_users; + u64 dbr_drop_recov_event_skips; +}; + +struct bnxt_re_dev { + struct ib_device ibdev; + struct list_head list; + atomic_t ref_count; + atomic_t sched_count; + unsigned long flags; +#define BNXT_RE_FLAG_NETDEV_REGISTERED 0 +#define BNXT_RE_FLAG_IBDEV_REGISTERED 1 +#define BNXT_RE_FLAG_GOT_MSIX 2 +#define BNXT_RE_FLAG_HAVE_L2_REF 3 +#define BNXT_RE_FLAG_ALLOC_RCFW 4 +#define BNXT_RE_FLAG_NET_RING_ALLOC 5 +#define BNXT_RE_FLAG_RCFW_CHANNEL_EN 6 +#define BNXT_RE_FLAG_ALLOC_CTX 7 +#define BNXT_RE_FLAG_STATS_CTX_ALLOC 8 +#define BNXT_RE_FLAG_STATS_CTX2_ALLOC 9 +#define BNXT_RE_FLAG_RCFW_CHANNEL_INIT 10 +#define BNXT_RE_FLAG_WORKER_REG 11 +#define BNXT_RE_FLAG_TBLS_ALLOCINIT 12 +#define BNXT_RE_FLAG_SETUP_NQ 13 +#define BNXT_RE_FLAG_BOND_DEV_REGISTERED 14 +#define BNXT_RE_FLAG_PER_PORT_DEBUG_INFO 15 +#define BNXT_RE_FLAG_DEV_LIST_INITIALIZED 16 +#define BNXT_RE_FLAG_ERR_DEVICE_DETACHED 17 +#define BNXT_RE_FLAG_INIT_DCBX_CC_PARAM 18 +#define BNXT_RE_FLAG_STOP_IN_PROGRESS 20 +#define BNXT_RE_FLAG_ISSUE_ROCE_STATS 29 +#define BNXT_RE_FLAG_ISSUE_CFA_FLOW_STATS 30 + struct ifnet *netdev; + struct auxiliary_device *adev; + struct bnxt_qplib_chip_ctx *chip_ctx; + struct bnxt_en_dev *en_dev; + struct bnxt_re_nq_record nqr; + int id; + struct delayed_work worker; + u16 worker_30s; + struct bnxt_re_tc_rec tc_rec[2]; + u8 cur_prio_map; + /* RCFW Channel */ + struct bnxt_qplib_rcfw rcfw; + /* Device Resources */ + struct bnxt_qplib_dev_attr *dev_attr; + struct bnxt_qplib_res qplib_res; + struct bnxt_qplib_dpi dpi_privileged; + struct bnxt_qplib_cc_param cc_param; + struct mutex cc_lock; + struct mutex qp_lock; + struct list_head qp_list; + u8 roce_mode; + + /* Max of 2 lossless traffic class supported per port */ + u16 cosq[2]; + /* Start: QP for handling QP1 packets */ + struct bnxt_re_gsi_context gsi_ctx; + /* End: QP for handling QP1 packets */ + bool is_virtfn; + u32 num_vfs; + u32 espeed; + /* + * For storing the speed of slave interfaces. + * Same as espeed when bond is not configured + */ + u32 sl_espeed; + /* To be used for a workaround for ISER stack */ + u32 min_tx_depth; + /* To enable qp debug info. Disabled during driver load */ + u32 en_qp_dbg; + /* Array to handle gid mapping */ + char *gid_map; + + struct bnxt_re_device_stats stats; + struct bnxt_re_drv_dbg_stats *dbg_stats; + /* debugfs to expose per port information*/ + struct dentry *port_debug_dir; + struct dentry *info; + struct dentry *drv_dbg_stats; + struct dentry *sp_perf_stats; + struct dentry *pdev_debug_dir; + struct dentry *pdev_qpinfo_dir; + struct bnxt_re_debug_entries *dbg_ent; + struct workqueue_struct *resolve_wq; + struct list_head mac_wq_list; + struct workqueue_struct *dcb_wq; + struct workqueue_struct *aer_wq; + u32 event_bitmap[3]; + bool unreg_sched; + u64 dbr_throttling_reg_off; + u64 dbr_aeq_arm_reg_off; + u64 dbr_db_fifo_reg_off; + void *dbr_page; + u64 dbr_bar_addr; + u32 pacing_algo_th; + u32 pacing_en_int_th; + u32 do_pacing_save; + struct workqueue_struct *dbq_wq; + struct workqueue_struct *dbr_drop_recov_wq; + struct work_struct dbq_fifo_check_work; + struct delayed_work dbq_pacing_work; + /* protect DB pacing */ + struct mutex dbq_lock; + /* Control DBR pacing feature. Set if enabled */ + bool dbr_pacing; + /* Control DBR recovery feature. Set if enabled */ + bool dbr_drop_recov; + bool user_dbr_drop_recov; + /* DBR recovery feature. Set if running */ + bool dbr_recovery_on; + u32 user_dbr_drop_recov_timeout; + /* + * Value used for pacing algo when pacing is active + */ +#define BNXT_RE_MAX_DBR_DO_PACING 0xFFFF + u32 dbr_do_pacing; + u32 dbq_watermark; /* Current watermark set in HW registers */ + u32 dbq_nq_id; /* Current NQ ID for DBQ events */ + u32 dbq_pacing_time; /* ms */ + u32 dbr_def_do_pacing; /* do_pacing when no congestion */ + u32 dbr_evt_curr_epoch; + bool dbq_int_disable; + + bool mod_exit; + struct bnxt_re_dbr_sw_stats *dbr_sw_stats; + struct bnxt_re_dbr_res_list res_list[BNXT_RE_RES_TYPE_MAX]; + struct bnxt_dbq_nq_list nq_list; + char dev_name[IB_DEVICE_NAME_MAX]; + atomic_t dbq_intr_running; + u32 num_msix_requested; + unsigned char *dev_addr; /* For netdev->dev_addr */ +}; + +#define BNXT_RE_RESOLVE_RETRY_COUNT_US 5000000 /* 5 sec */ +struct bnxt_re_resolve_dmac_work{ + struct work_struct work; + struct list_head list; + struct bnxt_re_dev *rdev; + struct ib_ah_attr *ah_attr; + struct bnxt_re_ah_info *ah_info; + atomic_t status_wait; +}; + +static inline u8 bnxt_re_get_prio(u8 prio_map) +{ + u8 prio = 0xFF; + + for (prio = 0; prio < 8; prio++) + if (prio_map & (1UL << prio)) + break; + return prio; +} + +/* This should be called with bnxt_re_dev_lock mutex held */ +static inline bool __bnxt_re_is_rdev_valid(struct bnxt_re_dev *rdev) +{ + struct bnxt_re_dev *tmp_rdev; + + list_for_each_entry(tmp_rdev, &bnxt_re_dev_list, list) { + if (rdev == tmp_rdev) + return true; + } + return false; +} + +static inline bool bnxt_re_is_rdev_valid(struct bnxt_re_dev *rdev) +{ + struct bnxt_re_dev *tmp_rdev; + + mutex_lock(&bnxt_re_dev_lock); + list_for_each_entry(tmp_rdev, &bnxt_re_dev_list, list) { + if (rdev == tmp_rdev) { + mutex_unlock(&bnxt_re_dev_lock); + return true; + } + } + mutex_unlock(&bnxt_re_dev_lock); + + pr_debug("bnxt_re: %s : Invalid rdev received rdev = %p\n", + __func__, rdev); + return false; +} + +int bnxt_re_send_hwrm_cmd(struct bnxt_re_dev *rdev, void *cmd, + int cmdlen); +void bnxt_re_stopqps_and_ib_uninit(struct bnxt_re_dev *rdev); +int bnxt_re_set_hwrm_dscp2pri(struct bnxt_re_dev *rdev, + struct bnxt_re_dscp2pri *d2p, u16 count, + u16 target_id); +int bnxt_re_query_hwrm_dscp2pri(struct bnxt_re_dev *rdev, + struct bnxt_re_dscp2pri *d2p, u16 *count, + u16 target_id); +int bnxt_re_query_hwrm_qportcfg(struct bnxt_re_dev *rdev, + struct bnxt_re_tc_rec *cnprec, u16 tid); +int bnxt_re_hwrm_cos2bw_qcfg(struct bnxt_re_dev *rdev, u16 target_id, + struct bnxt_re_cos2bw_cfg *cfg); +int bnxt_re_hwrm_cos2bw_cfg(struct bnxt_re_dev *rdev, u16 target_id, + struct bnxt_re_cos2bw_cfg *cfg); +int bnxt_re_hwrm_pri2cos_cfg(struct bnxt_re_dev *rdev, + u16 target_id, u16 port_id, + u8 *cos_id_map, u8 pri_map); +int bnxt_re_prio_vlan_tx_update(struct bnxt_re_dev *rdev); +int bnxt_re_get_slot_pf_count(struct bnxt_re_dev *rdev); +struct bnxt_re_dev *bnxt_re_get_peer_pf(struct bnxt_re_dev *rdev); +struct bnxt_re_dev *bnxt_re_from_netdev(struct ifnet *netdev); +u8 bnxt_re_get_priority_mask(struct bnxt_re_dev *rdev, u8 selector); +struct bnxt_qplib_nq * bnxt_re_get_nq(struct bnxt_re_dev *rdev); +void bnxt_re_put_nq(struct bnxt_re_dev *rdev, struct bnxt_qplib_nq *nq); + +#define to_bnxt_re(ptr, type, member) \ + container_of(ptr, type, member) + +#define to_bnxt_re_dev(ptr, member) \ + container_of((ptr), struct bnxt_re_dev, member) + +/* Even number functions from port 0 and odd number from port 1 */ +#define BNXT_RE_IS_PORT0(rdev) (!(rdev->en_dev->pdev->devfn & 1)) + +#define BNXT_RE_ROCE_V1_PACKET 0 +#define BNXT_RE_ROCEV2_IPV4_PACKET 2 +#define BNXT_RE_ROCEV2_IPV6_PACKET 3 +#define BNXT_RE_ACTIVE_MAP_PORT1 0x1 /*port-1 active */ +#define BNXT_RE_ACTIVE_MAP_PORT2 0x2 /*port-2 active */ + +#define BNXT_RE_MEMBER_PORT_MAP (BNXT_RE_ACTIVE_MAP_PORT1 | \ + BNXT_RE_ACTIVE_MAP_PORT2) + +#define rdev_to_dev(rdev) ((rdev) ? (&(rdev)->ibdev.dev) : NULL) + +void bnxt_re_set_dma_device(struct ib_device *ibdev, struct bnxt_re_dev *rdev); +bool bnxt_re_is_rdev_valid(struct bnxt_re_dev *rdev); + +#define bnxt_re_rdev_ready(rdev) (bnxt_re_is_rdev_valid(rdev) && \ + (test_bit(BNXT_RE_FLAG_IBDEV_REGISTERED, &rdev->flags))) +#define BNXT_RE_SRIOV_CFG_TIMEOUT 6 + +int bnxt_re_get_device_stats(struct bnxt_re_dev *rdev); +void bnxt_re_remove_device(struct bnxt_re_dev *rdev, u8 removal_type, + struct auxiliary_device *aux_dev); +void bnxt_re_destroy_lag(struct bnxt_re_dev **rdev); +int bnxt_re_add_device(struct bnxt_re_dev **rdev, + struct ifnet *netdev, + u8 qp_mode, u8 op_type, u8 wqe_mode, u32 num_msix_requested, + struct auxiliary_device *aux_dev); +void bnxt_re_create_base_interface(bool primary); +int bnxt_re_schedule_work(struct bnxt_re_dev *rdev, unsigned long event, + struct ifnet *vlan_dev, + struct ifnet *netdev, + struct auxiliary_device *aux_dev); +void bnxt_re_get_link_speed(struct bnxt_re_dev *rdev); +int _bnxt_re_ib_init(struct bnxt_re_dev *rdev); +int _bnxt_re_ib_init2(struct bnxt_re_dev *rdev); +void bnxt_re_init_resolve_wq(struct bnxt_re_dev *rdev); +void bnxt_re_uninit_resolve_wq(struct bnxt_re_dev *rdev); + +/* The rdev ref_count is to protect immature removal of the device */ +static inline void bnxt_re_hold(struct bnxt_re_dev *rdev) +{ + atomic_inc(&rdev->ref_count); + dev_dbg(rdev_to_dev(rdev), + "Hold ref_count = 0x%x", atomic_read(&rdev->ref_count)); +} + +static inline void bnxt_re_put(struct bnxt_re_dev *rdev) +{ + atomic_dec(&rdev->ref_count); + dev_dbg(rdev_to_dev(rdev), + "Put ref_count = 0x%x", atomic_read(&rdev->ref_count)); +} + +/* +* Responder Error reason codes +* FIXME: Remove these when the defs +* are properly included in hsi header +*/ +enum res_err_state_reason { + /* No error. */ + CFCQ_RES_ERR_STATE_REASON_NO_ERROR = 0, + /* + * Incoming Send, RDMA write, or RDMA read exceeds the maximum + * transfer length. Detected on RX first and only packets for + * write. Detected on RX request for read. This is an RX + * Detected Error. + */ + CFCQ_RES_ERR_STATE_REASON_RES_EXCEED_MAX, + /* + * RDMA write payload size does not match write length. Detected + * when total write payload is not equal to the RDMA write + * length that was given in the first or only packet of the + * request. This is an RX Detected Error. + */ + CFCQ_RES_ERR_STATE_REASON_RES_PAYLOAD_LENGTH_MISMATCH, + /* + * Send payload exceeds RQ/SRQ WQE buffer capacity. The total + * send payload that arrived is more than the size of the WQE + * buffer that was fetched from the RQ/SRQ. This is an RX + * Detected Error. + */ + CFCQ_RES_ERR_STATE_REASON_RES_EXCEEDS_WQE, + /* + * Responder detected opcode error. * First, only, middle, last + * for incoming requests are improperly ordered with respect to + * previous (PSN) packet. * First or middle packet is not full + * MTU size. This is an RX Detected Error. + */ + CFCQ_RES_ERR_STATE_REASON_RES_OPCODE_ERROR, + /* + * PSN sequence error retry limit exceeded. The responder + * encountered a PSN sequence error for the same PSN too many + * times. This can occur via implicit or explicit NAK. This is + * an RX Detected Error. + */ + CFCQ_RES_ERR_STATE_REASON_RES_PSN_SEQ_ERROR_RETRY_LIMIT, + /* + * Invalid R_Key. An incoming request contained an R_Key that + * did not reference a valid MR/MW. This error may be detected + * by the RX engine for RDMA write or by the TX engine for RDMA + * read (detected while servicing IRRQ). This is an RX Detected + * Error. + */ + CFCQ_RES_ERR_STATE_REASON_RES_RX_INVALID_R_KEY, + /* + * Domain error. An incoming request specified an R_Key which + * referenced a MR/MW that was not in the same PD as the QP on + * which the request arrived. This is an RX Detected Error. + */ + CFCQ_RES_ERR_STATE_REASON_RES_RX_DOMAIN_ERROR, + /* + * No permission. An incoming request contained an R_Key that + * referenced a MR/MW which did not have the access permission + * needed for the operation. This is an RX Detected Error. + */ + CFCQ_RES_ERR_STATE_REASON_RES_RX_NO_PERMISSION, + /* + * Range error. An incoming request had a combination of R_Key, + * VA, and length that was out of bounds of the associated + * MR/MW. This is an RX Detected Error. + */ + CFCQ_RES_ERR_STATE_REASON_RES_RX_RANGE_ERROR, + /* + * Invalid R_Key. An incoming request contained an R_Key that + * did not reference a valid MR/MW. This error may be detected + * by the RX engine for RDMA write or by the TX engine for RDMA + * read (detected while servicing IRRQ). This is a TX Detected + * Error. + */ + CFCQ_RES_ERR_STATE_REASON_RES_TX_INVALID_R_KEY, + /* + * Domain error. An incoming request specified an R_Key which + * referenced a MR/MW that was not in the same PD as the QP on + * which the request arrived. This is a TX Detected Error. + */ + CFCQ_RES_ERR_STATE_REASON_RES_TX_DOMAIN_ERROR, + /* + * No permission. An incoming request contained an R_Key that + * referenced a MR/MW which did not have the access permission + * needed for the operation. This is a TX Detected Error. + */ + CFCQ_RES_ERR_STATE_REASON_RES_TX_NO_PERMISSION, + /* + * Range error. An incoming request had a combination of R_Key, + * VA, and length that was out of bounds of the associated + * MR/MW. This is a TX Detected Error. + */ + CFCQ_RES_ERR_STATE_REASON_RES_TX_RANGE_ERROR, + /* + * IRRQ overflow. The peer sent us more RDMA read or atomic + * requests than the negotiated maximum. This is an RX Detected + * Error. + */ + CFCQ_RES_ERR_STATE_REASON_RES_IRRQ_OFLOW, + /* + * Unsupported opcode. The peer sent us a request with an opcode + * for a request type that is not supported on this QP. This is + * an RX Detected Error. + */ + CFCQ_RES_ERR_STATE_REASON_RES_UNSUPPORTED_OPCODE, + /* + * Unaligned atomic operation. The VA of an atomic request is on + * a memory boundary that prevents atomic execution. This is an + * RX Detected Error. + */ + CFCQ_RES_ERR_STATE_REASON_RES_UNALIGN_ATOMIC, + /* + * Remote invalidate error. A send with invalidate request + * arrived in which the R_Key to invalidate did not describe a + * MR/MW which could be invalidated. RQ WQE completes with error + * status. This error is only reported if the send operation did + * not fail. If the send operation failed then the remote + * invalidate error is not reported. This is an RX Detected + * Error. + */ + CFCQ_RES_ERR_STATE_REASON_RES_REM_INVALIDATE, + /* + * Local memory error. An RQ/SRQ SGE described an inaccessible + * memory. This is an RX Detected Error. + */ + CFCQ_RES_ERR_STATE_REASON_RES_MEMORY_ERROR, + /* + * SRQ in error. The QP is moving to error state because it + * found SRQ it uses in error. This is an RX Detected Error. + */ + CFCQ_RES_ERR_STATE_REASON_RES_SRQ_ERROR, + /* + * Completion error. No CQE space available on queue or CQ not + * in VALID state. This is a Completion Detected Error. + */ + CFCQ_RES_ERR_STATE_REASON_RES_CMP_ERROR, + /* + * Invalid R_Key while resending responses to duplicate request. + * This is a TX Detected Error. + */ + CFCQ_RES_ERR_STATE_REASON_RES_IVALID_DUP_RKEY, + /* + * Problem was found in the format of a WQE in the RQ/SRQ. This + * is an RX Detected Error. + */ + CFCQ_RES_ERR_STATE_REASON_RES_WQE_FORMAT_ERROR, + /* + * A load error occurred on an attempt to load the CQ Context. + * This is a Completion Detected Error. + */ + CFCQ_RES_ERR_STATE_REASON_RES_CQ_LOAD_ERROR = 0x18, + /* + * A load error occurred on an attempt to load the SRQ Context. + * This is an RX Detected Error. + */ + CFCQ_RES_ERR_STATE_REASON_RES_SRQ_LOAD_ERROR, + /* + * A fatal error was detected on an attempt to read from or + * write to PCIe on the transmit side. This error is detected by + * the TX side, but has the priority of a Completion Detected + * Error. + */ + CFCQ_RES_ERR_STATE_REASON_RES_TX_PCI_ERROR = 0x1b, + /* + * A fatal error was detected on an attempt to read from or + * write to PCIe on the receive side. This error is detected by + * the RX side (or CAGR), but has the priority of a Completion + * Detected Error. + */ + CFCQ_RES_ERR_STATE_REASON_RES_RX_PCI_ERROR = 0x1c +}; + +int bnxt_re_host_pf_id_query(struct bnxt_re_dev *rdev, + struct bnxt_qplib_query_fn_info *fn_info, + u32 *pf_mask, u32 *first_pf); + +/* Default DCBx and CC values */ +#define BNXT_RE_DEFAULT_CNP_DSCP 48 +#define BNXT_RE_DEFAULT_CNP_PRI 7 +#define BNXT_RE_DEFAULT_ROCE_DSCP 26 +#define BNXT_RE_DEFAULT_ROCE_PRI 3 + +#define BNXT_RE_DEFAULT_L2_BW 50 +#define BNXT_RE_DEFAULT_ROCE_BW 50 + +#define ROCE_PRIO_VALID 0x0 +#define CNP_PRIO_VALID 0x1 +#define ROCE_DSCP_VALID 0x0 +#define CNP_DSCP_VALID 0x1 + +int bnxt_re_get_pri_dscp_settings(struct bnxt_re_dev *rdev, + u16 target_id, + struct bnxt_re_tc_rec *tc_rec); + +int bnxt_re_setup_dscp(struct bnxt_re_dev *rdev); +int bnxt_re_clear_dscp(struct bnxt_re_dev *rdev); +int bnxt_re_setup_cnp_cos(struct bnxt_re_dev *rdev, bool reset); + +static inline enum ib_port_state bnxt_re_get_link_state(struct bnxt_re_dev *rdev) +{ + if (rdev->netdev->if_drv_flags & IFF_DRV_RUNNING && + rdev->netdev->if_link_state == LINK_STATE_UP) + return IB_PORT_ACTIVE; + return IB_PORT_DOWN; +} + +static inline int bnxt_re_link_state(struct bnxt_re_dev *rdev) +{ + return bnxt_re_get_link_state(rdev) == IB_PORT_ACTIVE ? 1:0; +} + +static inline int is_cc_enabled(struct bnxt_re_dev *rdev) +{ + return rdev->cc_param.enable; +} + +static inline void bnxt_re_init_hwrm_hdr(struct bnxt_re_dev *rdev, + struct input *hdr, u16 opcd, + u16 crid, u16 trid) +{ + hdr->req_type = cpu_to_le16(opcd); + hdr->cmpl_ring = cpu_to_le16(crid); + hdr->target_id = cpu_to_le16(trid); +} + +static inline void bnxt_re_fill_fw_msg(struct bnxt_fw_msg *fw_msg, + void *msg, int msg_len, void *resp, + int resp_max_len, int timeout) +{ + fw_msg->msg = msg; + fw_msg->msg_len = msg_len; + fw_msg->resp = resp; + fw_msg->resp_max_len = resp_max_len; + fw_msg->timeout = timeout; +} + +static inline bool is_qport_service_type_supported(struct bnxt_re_dev *rdev) +{ + return rdev->tc_rec[0].serv_type_enabled; +} + +static inline bool is_bnxt_roce_queue(struct bnxt_re_dev *rdev, u8 ser_prof, u8 prof_type) +{ + if (is_qport_service_type_supported(rdev)) + return (prof_type & HWRM_QUEUE_QPORTCFG_OUTPUT_QUEUE_ID1_SERVICE_PROFILE_TYPE_ROCE); + else + return (ser_prof == HWRM_QUEUE_QPORTCFG_OUTPUT_QUEUE_ID0_SERVICE_PROFILE_LOSSLESS_ROCE); +} + +static inline bool is_bnxt_cnp_queue(struct bnxt_re_dev *rdev, u8 ser_prof, u8 prof_type) +{ + if (is_qport_service_type_supported(rdev)) + return (prof_type & HWRM_QUEUE_QPORTCFG_OUTPUT_QUEUE_ID1_SERVICE_PROFILE_TYPE_CNP); + else + return (ser_prof == HWRM_QUEUE_QPORTCFG_OUTPUT_QUEUE_ID0_SERVICE_PROFILE_LOSSY_ROCE_CNP); +} + +#define BNXT_RE_MAP_SH_PAGE 0x0 +#define BNXT_RE_MAP_WC 0x1 +#define BNXT_RE_DBR_PAGE 0x2 +#define BNXT_RE_MAP_DB_RECOVERY_PAGE 0x3 + +#define BNXT_RE_DBR_RECOV_USERLAND_TIMEOUT (20) /* 20 ms */ +#define BNXT_RE_DBR_INT_TIME 5 /* ms */ +#define BNXT_RE_PACING_EN_INT_THRESHOLD 50 /* Entries in DB FIFO */ +#define BNXT_RE_PACING_ALGO_THRESHOLD 250 /* Entries in DB FIFO */ +/* Percentage of DB FIFO depth */ +#define BNXT_RE_PACING_DBQ_THRESHOLD BNXT_RE_PACING_DBQ_HIGH_WATERMARK + +#define BNXT_RE_PACING_ALARM_TH_MULTIPLE(ctx) (_is_chip_p7(ctx) ? 0 : 2) + +/* + * Maximum Percentage of configurable DB FIFO depth. + * The Doorbell FIFO depth is 0x2c00. But the DBR_REG_DB_THROTTLING register has only 12 bits + * to program the high watermark. This means user can configure maximum 36% only(4095/11264). + */ +#define BNXT_RE_PACING_DBQ_HIGH_WATERMARK 36 + +/* Default do_pacing value when there is no congestion */ +#define BNXT_RE_DBR_DO_PACING_NO_CONGESTION 0x7F /* 1 in 512 probability */ + +enum { + BNXT_RE_DBQ_EVENT_SCHED = 0, + BNXT_RE_DBR_PACING_EVENT = 1, + BNXT_RE_DBR_NQ_PACING_NOTIFICATION = 2, +}; + +struct bnxt_re_dbq_work { + struct work_struct work; + struct bnxt_re_dev *rdev; + struct hwrm_async_event_cmpl cmpl; + u32 event; +}; + +int bnxt_re_hwrm_qcaps(struct bnxt_re_dev *rdev); +int bnxt_re_enable_dbr_pacing(struct bnxt_re_dev *rdev); +int bnxt_re_disable_dbr_pacing(struct bnxt_re_dev *rdev); +int bnxt_re_set_dbq_throttling_reg(struct bnxt_re_dev *rdev, + u16 nq_id, u32 throttle); +void bnxt_re_pacing_alert(struct bnxt_re_dev *rdev); +int bnxt_re_hwrm_pri2cos_qcfg(struct bnxt_re_dev *rdev, struct bnxt_re_tc_rec *tc_rec, + u16 target_id); +void writel_fbsd(struct bnxt_softc *bp, u32, u8, u32); +u32 readl_fbsd(struct bnxt_softc *bp, u32, u8); + +static inline unsigned int bnxt_re_get_total_mr_mw_count(struct bnxt_re_dev *rdev) +{ + return (atomic_read(&rdev->stats.rsors.mr_count) + + atomic_read(&rdev->stats.rsors.mw_count)); +} + +static inline void bnxt_re_set_def_pacing_threshold(struct bnxt_re_dev *rdev) +{ + rdev->qplib_res.pacing_data->pacing_th = rdev->pacing_algo_th; + rdev->qplib_res.pacing_data->alarm_th = + rdev->pacing_algo_th * BNXT_RE_PACING_ALARM_TH_MULTIPLE(rdev->chip_ctx); +} + +static inline void bnxt_re_set_def_do_pacing(struct bnxt_re_dev *rdev) +{ + rdev->qplib_res.pacing_data->do_pacing = rdev->dbr_def_do_pacing; +} + +static inline void bnxt_re_set_pacing_dev_state(struct bnxt_re_dev *rdev) +{ + rdev->qplib_res.pacing_data->dev_err_state = + test_bit(BNXT_RE_FLAG_ERR_DEVICE_DETACHED, &rdev->flags); +} +#endif diff --git a/sys/dev/bnxt/bnxt_re/ib_verbs.c b/sys/dev/bnxt/bnxt_re/ib_verbs.c new file mode 100644 index 000000000000..8d43fa96c048 --- /dev/null +++ b/sys/dev/bnxt/bnxt_re/ib_verbs.c @@ -0,0 +1,5498 @@ +/* + * Copyright (c) 2015-2024, Broadcom. All rights reserved. The term + * Broadcom refers to Broadcom Limited and/or its subsidiaries. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR + * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE + * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN + * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * Description: IB Verbs interpreter + */ + +#include +#include +#include + +#include "bnxt_re.h" +#include "ib_verbs.h" + +static inline +struct scatterlist *get_ib_umem_sgl(struct ib_umem *umem, u32 *nmap) +{ + + *nmap = umem->nmap; + return umem->sg_head.sgl; +} + +static inline void bnxt_re_peer_mem_release(struct ib_umem *umem) +{ + dev_dbg(NULL, "ib_umem_release getting invoked \n"); + ib_umem_release(umem); +} + +void bnxt_re_resolve_dmac_task(struct work_struct *work) +{ + int rc = -1; + struct bnxt_re_dev *rdev; + struct ib_ah_attr *ah_attr; + struct bnxt_re_resolve_dmac_work *dmac_work = + container_of(work, struct bnxt_re_resolve_dmac_work, work); + + rdev = dmac_work->rdev; + ah_attr = dmac_work->ah_attr; + rc = ib_resolve_eth_dmac(&rdev->ibdev, ah_attr); + if (rc) + dev_err(rdev_to_dev(dmac_work->rdev), + "Failed to resolve dest mac rc = %d\n", rc); + atomic_set(&dmac_work->status_wait, rc << 8); +} + +static int __from_ib_access_flags(int iflags) +{ + int qflags = 0; + + if (iflags & IB_ACCESS_LOCAL_WRITE) + qflags |= BNXT_QPLIB_ACCESS_LOCAL_WRITE; + if (iflags & IB_ACCESS_REMOTE_READ) + qflags |= BNXT_QPLIB_ACCESS_REMOTE_READ; + if (iflags & IB_ACCESS_REMOTE_WRITE) + qflags |= BNXT_QPLIB_ACCESS_REMOTE_WRITE; + if (iflags & IB_ACCESS_REMOTE_ATOMIC) + qflags |= BNXT_QPLIB_ACCESS_REMOTE_ATOMIC; + if (iflags & IB_ACCESS_MW_BIND) + qflags |= BNXT_QPLIB_ACCESS_MW_BIND; + if (iflags & IB_ZERO_BASED) + qflags |= BNXT_QPLIB_ACCESS_ZERO_BASED; + if (iflags & IB_ACCESS_ON_DEMAND) + qflags |= BNXT_QPLIB_ACCESS_ON_DEMAND; + return qflags; +}; + +static enum ib_access_flags __to_ib_access_flags(int qflags) +{ + enum ib_access_flags iflags = 0; + + if (qflags & BNXT_QPLIB_ACCESS_LOCAL_WRITE) + iflags |= IB_ACCESS_LOCAL_WRITE; + if (qflags & BNXT_QPLIB_ACCESS_REMOTE_WRITE) + iflags |= IB_ACCESS_REMOTE_WRITE; + if (qflags & BNXT_QPLIB_ACCESS_REMOTE_READ) + iflags |= IB_ACCESS_REMOTE_READ; + if (qflags & BNXT_QPLIB_ACCESS_REMOTE_ATOMIC) + iflags |= IB_ACCESS_REMOTE_ATOMIC; + if (qflags & BNXT_QPLIB_ACCESS_MW_BIND) + iflags |= IB_ACCESS_MW_BIND; + if (qflags & BNXT_QPLIB_ACCESS_ZERO_BASED) + iflags |= IB_ZERO_BASED; + if (qflags & BNXT_QPLIB_ACCESS_ON_DEMAND) + iflags |= IB_ACCESS_ON_DEMAND; + return iflags; +}; + +static int bnxt_re_copy_to_udata(struct bnxt_re_dev *rdev, void *data, + int len, struct ib_udata *udata) +{ + int rc; + + rc = ib_copy_to_udata(udata, data, len); + if (rc) + dev_err(rdev_to_dev(rdev), + "ucontext copy failed from %ps rc %d\n", + __builtin_return_address(0), rc); + + return rc; +} + +struct ifnet *bnxt_re_get_netdev(struct ib_device *ibdev, + u8 port_num) +{ + struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev); + struct ifnet *netdev = NULL; + + rcu_read_lock(); + + if (!rdev || !rdev->netdev) + goto end; + + netdev = rdev->netdev; + + /* In case of active-backup bond mode, return active slave */ + if (netdev) + dev_hold(netdev); + +end: + rcu_read_unlock(); + return netdev; +} + +int bnxt_re_query_device(struct ib_device *ibdev, + struct ib_device_attr *ib_attr, + struct ib_udata *udata) +{ + struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev); + struct bnxt_qplib_dev_attr *dev_attr = rdev->dev_attr; + + memset(ib_attr, 0, sizeof(*ib_attr)); + + memcpy(&ib_attr->fw_ver, dev_attr->fw_ver, 4); + bnxt_qplib_get_guid(rdev->dev_addr, (u8 *)&ib_attr->sys_image_guid); + ib_attr->max_mr_size = BNXT_RE_MAX_MR_SIZE; + ib_attr->page_size_cap = dev_attr->page_size_cap; + ib_attr->vendor_id = rdev->en_dev->pdev->vendor; + ib_attr->vendor_part_id = rdev->en_dev->pdev->device; + ib_attr->hw_ver = rdev->en_dev->pdev->subsystem_device; + ib_attr->max_qp = dev_attr->max_qp; + ib_attr->max_qp_wr = dev_attr->max_qp_wqes; + /* + * Read and set from the module param 'min_tx_depth' + * only once after the driver load + */ + if (rdev->min_tx_depth == 1 && + min_tx_depth < dev_attr->max_qp_wqes) + rdev->min_tx_depth = min_tx_depth; + ib_attr->device_cap_flags = + IB_DEVICE_CURR_QP_STATE_MOD + | IB_DEVICE_RC_RNR_NAK_GEN + | IB_DEVICE_SHUTDOWN_PORT + | IB_DEVICE_SYS_IMAGE_GUID + | IB_DEVICE_LOCAL_DMA_LKEY + | IB_DEVICE_RESIZE_MAX_WR + | IB_DEVICE_PORT_ACTIVE_EVENT + | IB_DEVICE_N_NOTIFY_CQ + | IB_DEVICE_MEM_WINDOW + | IB_DEVICE_MEM_WINDOW_TYPE_2B + | IB_DEVICE_MEM_MGT_EXTENSIONS; + ib_attr->max_send_sge = dev_attr->max_qp_sges; + ib_attr->max_recv_sge = dev_attr->max_qp_sges; + ib_attr->max_sge_rd = dev_attr->max_qp_sges; + ib_attr->max_cq = dev_attr->max_cq; + ib_attr->max_cqe = dev_attr->max_cq_wqes; + ib_attr->max_mr = dev_attr->max_mr; + ib_attr->max_pd = dev_attr->max_pd; + ib_attr->max_qp_rd_atom = dev_attr->max_qp_rd_atom; + ib_attr->max_qp_init_rd_atom = dev_attr->max_qp_init_rd_atom; + if (dev_attr->is_atomic) { + ib_attr->atomic_cap = IB_ATOMIC_GLOB; + ib_attr->masked_atomic_cap = IB_ATOMIC_GLOB; + } + ib_attr->max_ee_rd_atom = 0; + ib_attr->max_res_rd_atom = 0; + ib_attr->max_ee_init_rd_atom = 0; + ib_attr->max_ee = 0; + ib_attr->max_rdd = 0; + ib_attr->max_mw = dev_attr->max_mw; + ib_attr->max_raw_ipv6_qp = 0; + ib_attr->max_raw_ethy_qp = dev_attr->max_raw_ethy_qp; + ib_attr->max_mcast_grp = 0; + ib_attr->max_mcast_qp_attach = 0; + ib_attr->max_total_mcast_qp_attach = 0; + ib_attr->max_ah = dev_attr->max_ah; + ib_attr->max_srq = dev_attr->max_srq; + ib_attr->max_srq_wr = dev_attr->max_srq_wqes; + ib_attr->max_srq_sge = dev_attr->max_srq_sges; + + ib_attr->max_fast_reg_page_list_len = MAX_PBL_LVL_1_PGS; + ib_attr->max_pkeys = 1; + ib_attr->local_ca_ack_delay = BNXT_RE_DEFAULT_ACK_DELAY; + ib_attr->sig_prot_cap = 0; + ib_attr->sig_guard_cap = 0; + ib_attr->odp_caps.general_caps = 0; + + return 0; +} + +int bnxt_re_modify_device(struct ib_device *ibdev, + int device_modify_mask, + struct ib_device_modify *device_modify) +{ + dev_dbg(rdev_to_dev(rdev), "Modify device with mask 0x%x\n", + device_modify_mask); + + switch (device_modify_mask) { + case IB_DEVICE_MODIFY_SYS_IMAGE_GUID: + /* Modify the GUID requires the modification of the GID table */ + /* GUID should be made as READ-ONLY */ + break; + case IB_DEVICE_MODIFY_NODE_DESC: + /* Node Desc should be made as READ-ONLY */ + break; + default: + break; + } + return 0; +} + +static void __to_ib_speed_width(u32 espeed, u8 *speed, u8 *width) +{ + switch (espeed) { + case SPEED_1000: + *speed = IB_SPEED_SDR; + *width = IB_WIDTH_1X; + break; + case SPEED_10000: + *speed = IB_SPEED_QDR; + *width = IB_WIDTH_1X; + break; + case SPEED_20000: + *speed = IB_SPEED_DDR; + *width = IB_WIDTH_4X; + break; + case SPEED_25000: + *speed = IB_SPEED_EDR; + *width = IB_WIDTH_1X; + break; + case SPEED_40000: + *speed = IB_SPEED_QDR; + *width = IB_WIDTH_4X; + break; + case SPEED_50000: + *speed = IB_SPEED_EDR; + *width = IB_WIDTH_2X; + break; + case SPEED_100000: + *speed = IB_SPEED_EDR; + *width = IB_WIDTH_4X; + break; + case SPEED_200000: + *speed = IB_SPEED_HDR; + *width = IB_WIDTH_4X; + break; + default: + *speed = IB_SPEED_SDR; + *width = IB_WIDTH_1X; + break; + } +} + +/* Port */ +int bnxt_re_query_port(struct ib_device *ibdev, u8 port_num, + struct ib_port_attr *port_attr) +{ + struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev); + struct bnxt_qplib_dev_attr *dev_attr = rdev->dev_attr; + u8 active_speed = 0, active_width = 0; + + dev_dbg(rdev_to_dev(rdev), "QUERY PORT with port_num 0x%x\n", port_num); + memset(port_attr, 0, sizeof(*port_attr)); + + port_attr->phys_state = IB_PORT_PHYS_STATE_DISABLED; + port_attr->state = bnxt_re_get_link_state(rdev); + if (port_attr->state == IB_PORT_ACTIVE) + port_attr->phys_state = IB_PORT_PHYS_STATE_LINK_UP; + port_attr->max_mtu = IB_MTU_4096; + port_attr->active_mtu = iboe_get_mtu(rdev->netdev->if_mtu); + port_attr->gid_tbl_len = dev_attr->max_sgid; + port_attr->port_cap_flags = IB_PORT_CM_SUP | IB_PORT_REINIT_SUP | + IB_PORT_DEVICE_MGMT_SUP | + IB_PORT_VENDOR_CLASS_SUP | + IB_PORT_IP_BASED_GIDS; + + port_attr->max_msg_sz = (u32)BNXT_RE_MAX_MR_SIZE_LOW; + port_attr->bad_pkey_cntr = 0; + port_attr->qkey_viol_cntr = 0; + port_attr->pkey_tbl_len = dev_attr->max_pkey; + port_attr->lid = 0; + port_attr->sm_lid = 0; + port_attr->lmc = 0; + port_attr->max_vl_num = 4; + port_attr->sm_sl = 0; + port_attr->subnet_timeout = 0; + port_attr->init_type_reply = 0; + rdev->espeed = rdev->en_dev->espeed; + + if (test_bit(BNXT_RE_FLAG_IBDEV_REGISTERED, &rdev->flags)) + __to_ib_speed_width(rdev->espeed, &active_speed, + &active_width); + + port_attr->active_speed = active_speed; + port_attr->active_width = active_width; + + return 0; +} + +int bnxt_re_modify_port(struct ib_device *ibdev, u8 port_num, + int port_modify_mask, + struct ib_port_modify *port_modify) +{ + dev_dbg(rdev_to_dev(rdev), "Modify port with mask 0x%x\n", + port_modify_mask); + + switch (port_modify_mask) { + case IB_PORT_SHUTDOWN: + break; + case IB_PORT_INIT_TYPE: + break; + case IB_PORT_RESET_QKEY_CNTR: + break; + default: + break; + } + return 0; +} + +int bnxt_re_get_port_immutable(struct ib_device *ibdev, u8 port_num, + struct ib_port_immutable *immutable) +{ + struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev); + struct ib_port_attr port_attr; + + if (bnxt_re_query_port(ibdev, port_num, &port_attr)) + return -EINVAL; + + immutable->pkey_tbl_len = port_attr.pkey_tbl_len; + immutable->gid_tbl_len = port_attr.gid_tbl_len; + if (rdev->roce_mode == BNXT_RE_FLAG_ROCEV1_CAP) + immutable->core_cap_flags = RDMA_CORE_PORT_IBA_ROCE; + else if (rdev->roce_mode == BNXT_RE_FLAG_ROCEV2_CAP) + immutable->core_cap_flags = RDMA_CORE_PORT_IBA_ROCE_UDP_ENCAP; + else + immutable->core_cap_flags = RDMA_CORE_PORT_IBA_ROCE | + RDMA_CORE_PORT_IBA_ROCE_UDP_ENCAP; + immutable->max_mad_size = IB_MGMT_MAD_SIZE; + return 0; +} + +void bnxt_re_compat_qfwstr(void) +{ + struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev); + + sprintf(str, "%d.%d.%d.%d", rdev->dev_attr->fw_ver[0], + rdev->dev_attr->fw_ver[1], rdev->dev_attr->fw_ver[2], + rdev->dev_attr->fw_ver[3]); +} + +int bnxt_re_query_pkey(struct ib_device *ibdev, u8 port_num, + u16 index, u16 *pkey) +{ + if (index > 0) + return -EINVAL; + + *pkey = IB_DEFAULT_PKEY_FULL; + + return 0; +} + +int bnxt_re_query_gid(struct ib_device *ibdev, u8 port_num, + int index, union ib_gid *gid) +{ + struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev); + int rc = 0; + + /* Ignore port_num */ + memset(gid, 0, sizeof(*gid)); + rc = bnxt_qplib_get_sgid(&rdev->qplib_res, + &rdev->qplib_res.sgid_tbl, index, + (struct bnxt_qplib_gid *)gid); + return rc; +} + +int bnxt_re_del_gid(struct ib_device *ibdev, u8 port_num, + unsigned int index, void **context) +{ + int rc = 0; + struct bnxt_re_gid_ctx *ctx, **ctx_tbl; + struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev); + struct bnxt_qplib_sgid_tbl *sgid_tbl = &rdev->qplib_res.sgid_tbl; + struct bnxt_qplib_gid *gid_to_del; + u16 vlan_id = 0xFFFF; + + /* Delete the entry from the hardware */ + ctx = *context; + if (!ctx) { + dev_err(rdev_to_dev(rdev), "GID entry has no ctx?!\n"); + return -EINVAL; + } + if (sgid_tbl && sgid_tbl->active) { + if (ctx->idx >= sgid_tbl->max) { + dev_dbg(rdev_to_dev(rdev), "GID index out of range?!\n"); + return -EINVAL; + } + gid_to_del = &sgid_tbl->tbl[ctx->idx].gid; + vlan_id = sgid_tbl->tbl[ctx->idx].vlan_id; + ctx->refcnt--; + /* DEL_GID is called via WQ context(netdevice_event_work_handler) + * or via the ib_unregister_device path. In the former case QP1 + * may not be destroyed yet, in which case just return as FW + * needs that entry to be present and will fail it's deletion. + * We could get invoked again after QP1 is destroyed OR get an + * ADD_GID call with a different GID value for the same index + * where we issue MODIFY_GID cmd to update the GID entry -- TBD + */ + if (ctx->idx == 0 && + rdma_link_local_addr((struct in6_addr *)gid_to_del) && + (rdev->gsi_ctx.gsi_sqp || + rdev->gsi_ctx.gsi_qp_mode == BNXT_RE_GSI_MODE_UD)) { + dev_dbg(rdev_to_dev(rdev), + "Trying to delete GID0 while QP1 is alive\n"); + if (!ctx->refcnt) { + rdev->gid_map[index] = -1; + ctx_tbl = sgid_tbl->ctx; + ctx_tbl[ctx->idx] = NULL; + kfree(ctx); + } + return 0; + } + rdev->gid_map[index] = -1; + if (!ctx->refcnt) { + rc = bnxt_qplib_del_sgid(sgid_tbl, gid_to_del, + vlan_id, true); + if (!rc) { + dev_dbg(rdev_to_dev(rdev), "GID remove success\n"); + ctx_tbl = sgid_tbl->ctx; + ctx_tbl[ctx->idx] = NULL; + kfree(ctx); + } else { + dev_err(rdev_to_dev(rdev), + "Remove GID failed rc = 0x%x\n", rc); + } + } + } else { + dev_dbg(rdev_to_dev(rdev), "GID sgid_tbl does not exist!\n"); + return -EINVAL; + } + return rc; +} + +int bnxt_re_add_gid(struct ib_device *ibdev, u8 port_num, + unsigned int index, const union ib_gid *gid, + const struct ib_gid_attr *attr, void **context) +{ + int rc; + u32 tbl_idx = 0; + u16 vlan_id = 0xFFFF; + struct bnxt_re_gid_ctx *ctx, **ctx_tbl; + struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev); + struct bnxt_qplib_sgid_tbl *sgid_tbl = &rdev->qplib_res.sgid_tbl; + if ((attr->ndev) && is_vlan_dev(attr->ndev)) + vlan_id = vlan_dev_vlan_id(attr->ndev); + + rc = bnxt_qplib_add_sgid(sgid_tbl, gid, + rdev->dev_addr, + vlan_id, true, &tbl_idx); + if (rc == -EALREADY) { + dev_dbg(rdev_to_dev(rdev), "GID %pI6 is already present\n", gid); + ctx_tbl = sgid_tbl->ctx; + if (!ctx_tbl[tbl_idx]) { + ctx = kmalloc(sizeof(*ctx), GFP_KERNEL); + if (!ctx) + return -ENOMEM; + ctx->idx = tbl_idx; + ctx->refcnt = 1; + ctx_tbl[tbl_idx] = ctx; + } else { + ctx_tbl[tbl_idx]->refcnt++; + } + *context = ctx_tbl[tbl_idx]; + /* tbl_idx is the HW table index and index is the stack index */ + rdev->gid_map[index] = tbl_idx; + return 0; + } else if (rc < 0) { + dev_err(rdev_to_dev(rdev), "Add GID failed rc = 0x%x\n", rc); + return rc; + } else { + ctx = kmalloc(sizeof(*ctx), GFP_KERNEL); + if (!ctx) { + dev_err(rdev_to_dev(rdev), "Add GID ctx failed\n"); + return -ENOMEM; + } + ctx_tbl = sgid_tbl->ctx; + ctx->idx = tbl_idx; + ctx->refcnt = 1; + ctx_tbl[tbl_idx] = ctx; + /* tbl_idx is the HW table index and index is the stack index */ + rdev->gid_map[index] = tbl_idx; + *context = ctx; + } + return rc; +} + +enum rdma_link_layer bnxt_re_get_link_layer(struct ib_device *ibdev, + u8 port_num) +{ + return IB_LINK_LAYER_ETHERNET; +} + +static void bnxt_re_legacy_create_fence_wqe(struct bnxt_re_pd *pd) +{ + struct bnxt_re_legacy_fence_data *fence = &pd->fence; + struct ib_mr *ib_mr = &fence->mr->ib_mr; + struct bnxt_qplib_swqe *wqe = &fence->bind_wqe; + struct bnxt_re_dev *rdev = pd->rdev; + + if (!_is_chip_gen_p5_p7(rdev->chip_ctx)) + return; + + memset(wqe, 0, sizeof(*wqe)); + wqe->type = BNXT_QPLIB_SWQE_TYPE_BIND_MW; + wqe->wr_id = BNXT_QPLIB_FENCE_WRID; + wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP; + wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE; + wqe->bind.zero_based = false; + wqe->bind.parent_l_key = ib_mr->lkey; + wqe->bind.va = (u64)fence->va; + wqe->bind.length = fence->size; + wqe->bind.access_cntl = __from_ib_access_flags(IB_ACCESS_REMOTE_READ); + wqe->bind.mw_type = SQ_BIND_MW_TYPE_TYPE1; + + /* Save the initial rkey in fence structure for now; + * wqe->bind.r_key will be set at (re)bind time. + */ + fence->bind_rkey = ib_inc_rkey(fence->mw->rkey); +} + +static int bnxt_re_legacy_bind_fence_mw(struct bnxt_qplib_qp *qplib_qp) +{ + struct bnxt_re_qp *qp = container_of(qplib_qp, struct bnxt_re_qp, + qplib_qp); + struct ib_pd *ib_pd = qp->ib_qp.pd; + struct bnxt_re_pd *pd = to_bnxt_re(ib_pd, struct bnxt_re_pd, ibpd); + struct bnxt_re_legacy_fence_data *fence = &pd->fence; + struct bnxt_qplib_swqe *fence_wqe = &fence->bind_wqe; + struct bnxt_qplib_swqe wqe; + int rc; + + /* TODO: Need SQ locking here when Fence WQE + * posting moves up into bnxt_re from bnxt_qplib. + */ + memcpy(&wqe, fence_wqe, sizeof(wqe)); + wqe.bind.r_key = fence->bind_rkey; + fence->bind_rkey = ib_inc_rkey(fence->bind_rkey); + + dev_dbg(rdev_to_dev(qp->rdev), + "Posting bind fence-WQE: rkey: %#x QP: %d PD: %p\n", + wqe.bind.r_key, qp->qplib_qp.id, pd); + rc = bnxt_qplib_post_send(&qp->qplib_qp, &wqe); + if (rc) { + dev_err(rdev_to_dev(qp->rdev), "Failed to bind fence-WQE\n"); + return rc; + } + bnxt_qplib_post_send_db(&qp->qplib_qp); + + return rc; +} + +static int bnxt_re_legacy_create_fence_mr(struct bnxt_re_pd *pd) +{ + int mr_access_flags = IB_ACCESS_LOCAL_WRITE | IB_ACCESS_MW_BIND; + struct bnxt_re_legacy_fence_data *fence = &pd->fence; + struct bnxt_re_dev *rdev = pd->rdev; + struct bnxt_qplib_mrinfo mrinfo; + struct bnxt_re_mr *mr = NULL; + struct ib_mw *ib_mw = NULL; + dma_addr_t dma_addr = 0; + u32 max_mr_count; + u64 pbl_tbl; + int rc; + + if (!_is_chip_gen_p5_p7(rdev->chip_ctx)) + return 0; + + memset(&mrinfo, 0, sizeof(mrinfo)); + /* Allocate a small chunk of memory and dma-map it */ + fence->va = kzalloc(BNXT_RE_LEGACY_FENCE_BYTES, GFP_KERNEL); + if (!fence->va) + return -ENOMEM; + dma_addr = ib_dma_map_single(&rdev->ibdev, fence->va, + BNXT_RE_LEGACY_FENCE_BYTES, + DMA_BIDIRECTIONAL); + rc = ib_dma_mapping_error(&rdev->ibdev, dma_addr); + if (rc) { + dev_err(rdev_to_dev(rdev), "Failed to dma-map fence-MR-mem\n"); + rc = -EIO; + fence->dma_addr = 0; + goto free_va; + } + fence->dma_addr = dma_addr; + + /* Allocate a MR */ + mr = kzalloc(sizeof(*mr), GFP_KERNEL); + if (!mr) + goto free_dma_addr; + fence->mr = mr; + mr->rdev = rdev; + mr->qplib_mr.pd = &pd->qplib_pd; + mr->qplib_mr.type = CMDQ_ALLOCATE_MRW_MRW_FLAGS_PMR; + mr->qplib_mr.flags = __from_ib_access_flags(mr_access_flags); + if (!_is_alloc_mr_unified(rdev->qplib_res.dattr)) { + rc = bnxt_qplib_alloc_mrw(&rdev->qplib_res, &mr->qplib_mr); + if (rc) { + dev_err(rdev_to_dev(rdev), "Failed to alloc fence-HW-MR\n"); + goto free_mr; + } + /* Register MR */ + mr->ib_mr.lkey = mr->qplib_mr.lkey; + } + mr->qplib_mr.va = (u64)fence->va; + mr->qplib_mr.total_size = BNXT_RE_LEGACY_FENCE_BYTES; + pbl_tbl = dma_addr; + + mrinfo.mrw = &mr->qplib_mr; + mrinfo.ptes = &pbl_tbl; + mrinfo.sg.npages = BNXT_RE_LEGACY_FENCE_PBL_SIZE; + + mrinfo.sg.nmap = 0; + mrinfo.sg.sghead = 0; + mrinfo.sg.pgshft = PAGE_SHIFT; + mrinfo.sg.pgsize = PAGE_SIZE; + rc = bnxt_qplib_reg_mr(&rdev->qplib_res, &mrinfo, false); + if (rc) { + dev_err(rdev_to_dev(rdev), "Failed to register fence-MR\n"); + goto free_mr; + } + mr->ib_mr.lkey = mr->qplib_mr.lkey; + mr->ib_mr.rkey = mr->qplib_mr.rkey; + atomic_inc(&rdev->stats.rsors.mr_count); + max_mr_count = atomic_read(&rdev->stats.rsors.mr_count); + if (max_mr_count > (atomic_read(&rdev->stats.rsors.max_mr_count))) + atomic_set(&rdev->stats.rsors.max_mr_count, max_mr_count); + + ib_mw = bnxt_re_alloc_mw(&pd->ibpd, IB_MW_TYPE_1, NULL); + /* Create a fence MW only for kernel consumers */ + if (!ib_mw) { + dev_err(rdev_to_dev(rdev), + "Failed to create fence-MW for PD: %p\n", pd); + rc = -EINVAL; + goto free_mr; + } + fence->mw = ib_mw; + + bnxt_re_legacy_create_fence_wqe(pd); + return 0; + +free_mr: + if (mr->ib_mr.lkey) { + bnxt_qplib_free_mrw(&rdev->qplib_res, &mr->qplib_mr); + atomic_dec(&rdev->stats.rsors.mr_count); + } + kfree(mr); + fence->mr = NULL; + +free_dma_addr: + ib_dma_unmap_single(&rdev->ibdev, fence->dma_addr, + BNXT_RE_LEGACY_FENCE_BYTES, DMA_BIDIRECTIONAL); + fence->dma_addr = 0; + +free_va: + kfree(fence->va); + fence->va = NULL; + return rc; +} + +static void bnxt_re_legacy_destroy_fence_mr(struct bnxt_re_pd *pd) +{ + struct bnxt_re_legacy_fence_data *fence = &pd->fence; + struct bnxt_re_dev *rdev = pd->rdev; + struct bnxt_re_mr *mr = fence->mr; + + if (!_is_chip_gen_p5_p7(rdev->chip_ctx)) + return; + + if (fence->mw) { + bnxt_re_dealloc_mw(fence->mw); + fence->mw = NULL; + } + if (mr) { + if (mr->ib_mr.rkey) + bnxt_qplib_dereg_mrw(&rdev->qplib_res, &mr->qplib_mr, + false); + if (mr->ib_mr.lkey) + bnxt_qplib_free_mrw(&rdev->qplib_res, &mr->qplib_mr); + kfree(mr); + fence->mr = NULL; + atomic_dec(&rdev->stats.rsors.mr_count); + } + if (fence->dma_addr) { + ib_dma_unmap_single(&rdev->ibdev, fence->dma_addr, + BNXT_RE_LEGACY_FENCE_BYTES, + DMA_BIDIRECTIONAL); + fence->dma_addr = 0; + } + kfree(fence->va); + fence->va = NULL; +} + + +static int bnxt_re_get_user_dpi(struct bnxt_re_dev *rdev, + struct bnxt_re_ucontext *cntx) +{ + struct bnxt_qplib_chip_ctx *cctx = rdev->chip_ctx; + int ret = 0; + u8 type; + /* Allocate DPI in alloc_pd or in create_cq to avoid failing of + * ibv_devinfo and family of application when DPIs are depleted. + */ + type = BNXT_QPLIB_DPI_TYPE_UC; + ret = bnxt_qplib_alloc_dpi(&rdev->qplib_res, &cntx->dpi, cntx, type); + if (ret) { + dev_err(rdev_to_dev(rdev), "Alloc doorbell page failed!\n"); + goto out; + } + + if (cctx->modes.db_push) { + type = BNXT_QPLIB_DPI_TYPE_WC; + ret = bnxt_qplib_alloc_dpi(&rdev->qplib_res, &cntx->wcdpi, + cntx, type); + if (ret) + dev_err(rdev_to_dev(rdev), "push dp alloc failed\n"); + } +out: + return ret; +} + +/* Protection Domains */ +void bnxt_re_dealloc_pd(struct ib_pd *ib_pd, struct ib_udata *udata) +{ + struct bnxt_re_pd *pd = to_bnxt_re(ib_pd, struct bnxt_re_pd, ibpd); + struct bnxt_re_dev *rdev = pd->rdev; + int rc; + + bnxt_re_legacy_destroy_fence_mr(pd); + + rc = bnxt_qplib_dealloc_pd(&rdev->qplib_res, + &rdev->qplib_res.pd_tbl, + &pd->qplib_pd); + if (rc) + dev_err_ratelimited(rdev_to_dev(rdev), + "%s failed rc = %d\n", __func__, rc); + atomic_dec(&rdev->stats.rsors.pd_count); + + return; +} + +int bnxt_re_alloc_pd(struct ib_pd *pd_in, + struct ib_udata *udata) +{ + struct ib_pd *ibpd = pd_in; + struct ib_device *ibdev = ibpd->device; + struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev); + struct bnxt_re_ucontext *ucntx = + rdma_udata_to_drv_context(udata, struct bnxt_re_ucontext, + ibucontext); + u32 max_pd_count; + int rc; + struct bnxt_re_pd *pd = container_of(ibpd, struct bnxt_re_pd, ibpd); + + pd->rdev = rdev; + if (bnxt_qplib_alloc_pd(&rdev->qplib_res, &pd->qplib_pd)) { + dev_err(rdev_to_dev(rdev), + "Allocate HW Protection Domain failed!\n"); + rc = -ENOMEM; + goto fail; + } + + if (udata) { + struct bnxt_re_pd_resp resp = {}; + + if (!ucntx->dpi.dbr) { + rc = bnxt_re_get_user_dpi(rdev, ucntx); + if (rc) + goto dbfail; + } + + resp.pdid = pd->qplib_pd.id; + /* Still allow mapping this DBR to the new user PD. */ + resp.dpi = ucntx->dpi.dpi; + resp.dbr = (u64)ucntx->dpi.umdbr; + /* Copy only on a valid wcpdi */ + if (ucntx->wcdpi.dpi) { + resp.wcdpi = ucntx->wcdpi.dpi; + resp.comp_mask = BNXT_RE_COMP_MASK_PD_HAS_WC_DPI; + } + if (rdev->dbr_pacing) { + WARN_ON(!rdev->dbr_bar_addr); + resp.dbr_bar_addr = (u64)rdev->dbr_bar_addr; + resp.comp_mask |= BNXT_RE_COMP_MASK_PD_HAS_DBR_BAR_ADDR; + } + + rc = bnxt_re_copy_to_udata(rdev, &resp, + min(udata->outlen, sizeof(resp)), + udata); + if (rc) + goto dbfail; + } + + if (!udata) + if (bnxt_re_legacy_create_fence_mr(pd)) + dev_warn(rdev_to_dev(rdev), + "Failed to create Fence-MR\n"); + + atomic_inc(&rdev->stats.rsors.pd_count); + max_pd_count = atomic_read(&rdev->stats.rsors.pd_count); + if (max_pd_count > atomic_read(&rdev->stats.rsors.max_pd_count)) + atomic_set(&rdev->stats.rsors.max_pd_count, max_pd_count); + + return 0; +dbfail: + (void)bnxt_qplib_dealloc_pd(&rdev->qplib_res, &rdev->qplib_res.pd_tbl, + &pd->qplib_pd); +fail: + return rc; +} + +/* Address Handles */ +void bnxt_re_destroy_ah(struct ib_ah *ib_ah, u32 flags) +{ + struct bnxt_re_ah *ah = to_bnxt_re(ib_ah, struct bnxt_re_ah, ibah); + struct bnxt_re_dev *rdev = ah->rdev; + int rc = 0; + bool block = true; + + block = !(flags & RDMA_DESTROY_AH_SLEEPABLE); + + rc = bnxt_qplib_destroy_ah(&rdev->qplib_res, &ah->qplib_ah, block); + if (rc) + dev_err_ratelimited(rdev_to_dev(rdev), + "%s id = %d blocking %d failed rc = %d\n", + __func__, ah->qplib_ah.id, block, rc); + atomic_dec(&rdev->stats.rsors.ah_count); + + return; +} + +static u8 _to_bnxt_re_nw_type(enum rdma_network_type ntype) +{ + u8 nw_type; + switch (ntype) { + case RDMA_NETWORK_IPV4: + nw_type = CMDQ_CREATE_AH_TYPE_V2IPV4; + break; + case RDMA_NETWORK_IPV6: + nw_type = CMDQ_CREATE_AH_TYPE_V2IPV6; + break; + default: + nw_type = CMDQ_CREATE_AH_TYPE_V1; + break; + } + return nw_type; +} + +static inline int +bnxt_re_get_cached_gid(struct ib_device *dev, u8 port_num, int index, + union ib_gid *sgid, struct ib_gid_attr **sgid_attr, + struct ib_global_route *grh, struct ib_ah *ah) +{ + int ret = 0; + + ret = ib_get_cached_gid(dev, port_num, index, sgid, *sgid_attr); + return ret; +} + +static inline enum rdma_network_type +bnxt_re_gid_to_network_type(struct ib_gid_attr *sgid_attr, + union ib_gid *sgid) +{ + return ib_gid_to_network_type(sgid_attr->gid_type, sgid); +} + +static int bnxt_re_get_ah_info(struct bnxt_re_dev *rdev, + struct ib_ah_attr *ah_attr, + struct bnxt_re_ah_info *ah_info) +{ + struct ib_gid_attr *gattr; + enum rdma_network_type ib_ntype; + u8 ntype; + union ib_gid *gid; + int rc = 0; + + gid = &ah_info->sgid; + gattr = &ah_info->sgid_attr; + + rc = bnxt_re_get_cached_gid(&rdev->ibdev, 1, ah_attr->grh.sgid_index, + gid, &gattr, &ah_attr->grh, NULL); + if (rc) + return rc; + + /* Get vlan tag */ + if (gattr->ndev) { + if (is_vlan_dev(gattr->ndev)) + ah_info->vlan_tag = vlan_dev_vlan_id(gattr->ndev); + if_rele(gattr->ndev); + } + + /* Get network header type for this GID */ + + ib_ntype = bnxt_re_gid_to_network_type(gattr, gid); + ntype = _to_bnxt_re_nw_type(ib_ntype); + ah_info->nw_type = ntype; + + return rc; +} + +static u8 _get_sgid_index(struct bnxt_re_dev *rdev, u8 gindx) +{ + gindx = rdev->gid_map[gindx]; + return gindx; +} + +static int bnxt_re_init_dmac(struct bnxt_re_dev *rdev, struct ib_ah_attr *ah_attr, + struct bnxt_re_ah_info *ah_info, bool is_user, + struct bnxt_re_ah *ah) +{ + int rc = 0; + u8 *dmac; + + if (is_user && !rdma_is_multicast_addr((struct in6_addr *) + ah_attr->grh.dgid.raw) && + !rdma_link_local_addr((struct in6_addr *)ah_attr->grh.dgid.raw)) { + + u32 retry_count = BNXT_RE_RESOLVE_RETRY_COUNT_US; + struct bnxt_re_resolve_dmac_work *resolve_dmac_work; + + + resolve_dmac_work = kzalloc(sizeof(*resolve_dmac_work), GFP_ATOMIC); + + resolve_dmac_work->rdev = rdev; + resolve_dmac_work->ah_attr = ah_attr; + resolve_dmac_work->ah_info = ah_info; + + atomic_set(&resolve_dmac_work->status_wait, 1); + INIT_WORK(&resolve_dmac_work->work, bnxt_re_resolve_dmac_task); + queue_work(rdev->resolve_wq, &resolve_dmac_work->work); + + do { + rc = atomic_read(&resolve_dmac_work->status_wait) & 0xFF; + if (!rc) + break; + udelay(1); + } while (--retry_count); + if (atomic_read(&resolve_dmac_work->status_wait)) { + INIT_LIST_HEAD(&resolve_dmac_work->list); + list_add_tail(&resolve_dmac_work->list, + &rdev->mac_wq_list); + return -EFAULT; + } + kfree(resolve_dmac_work); + } + dmac = ROCE_DMAC(ah_attr); + if (dmac) + memcpy(ah->qplib_ah.dmac, dmac, ETH_ALEN); + return rc; +} + +int bnxt_re_create_ah(struct ib_ah *ah_in, struct ib_ah_attr *attr, + u32 flags, struct ib_udata *udata) +{ + + struct ib_ah *ib_ah = ah_in; + struct ib_pd *ib_pd = ib_ah->pd; + struct bnxt_re_ah *ah = container_of(ib_ah, struct bnxt_re_ah, ibah); + struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ibpd); + struct bnxt_re_dev *rdev = pd->rdev; + struct bnxt_re_ah_info ah_info; + u32 max_ah_count; + bool is_user; + int rc; + bool block = true; + struct ib_ah_attr *ah_attr = attr; + block = !(flags & RDMA_CREATE_AH_SLEEPABLE); + + if (!(ah_attr->ah_flags & IB_AH_GRH)) + dev_err(rdev_to_dev(rdev), "ah_attr->ah_flags GRH is not set\n"); + + ah->rdev = rdev; + ah->qplib_ah.pd = &pd->qplib_pd; + is_user = ib_pd->uobject ? true : false; + + /* Supply the configuration for the HW */ + memcpy(ah->qplib_ah.dgid.data, ah_attr->grh.dgid.raw, + sizeof(union ib_gid)); + ah->qplib_ah.sgid_index = _get_sgid_index(rdev, ah_attr->grh.sgid_index); + if (ah->qplib_ah.sgid_index == 0xFF) { + dev_err(rdev_to_dev(rdev), "invalid sgid_index!\n"); + rc = -EINVAL; + goto fail; + } + ah->qplib_ah.host_sgid_index = ah_attr->grh.sgid_index; + ah->qplib_ah.traffic_class = ah_attr->grh.traffic_class; + ah->qplib_ah.flow_label = ah_attr->grh.flow_label; + ah->qplib_ah.hop_limit = ah_attr->grh.hop_limit; + ah->qplib_ah.sl = ah_attr->sl; + rc = bnxt_re_get_ah_info(rdev, ah_attr, &ah_info); + if (rc) + goto fail; + ah->qplib_ah.nw_type = ah_info.nw_type; + + rc = bnxt_re_init_dmac(rdev, ah_attr, &ah_info, is_user, ah); + if (rc) + goto fail; + + rc = bnxt_qplib_create_ah(&rdev->qplib_res, &ah->qplib_ah, block); + if (rc) { + dev_err(rdev_to_dev(rdev), + "Allocate HW Address Handle failed!\n"); + goto fail; + } + + /* Write AVID to shared page. */ + if (ib_pd->uobject) { + struct ib_ucontext *ib_uctx = ib_pd->uobject->context; + struct bnxt_re_ucontext *uctx; + unsigned long flag; + u32 *wrptr; + + uctx = to_bnxt_re(ib_uctx, struct bnxt_re_ucontext, ibucontext); + spin_lock_irqsave(&uctx->sh_lock, flag); + wrptr = (u32 *)((u8 *)uctx->shpg + BNXT_RE_AVID_OFFT); + *wrptr = ah->qplib_ah.id; + wmb(); /* make sure cache is updated. */ + spin_unlock_irqrestore(&uctx->sh_lock, flag); + } + atomic_inc(&rdev->stats.rsors.ah_count); + max_ah_count = atomic_read(&rdev->stats.rsors.ah_count); + if (max_ah_count > atomic_read(&rdev->stats.rsors.max_ah_count)) + atomic_set(&rdev->stats.rsors.max_ah_count, max_ah_count); + + return 0; +fail: + return rc; +} + +int bnxt_re_modify_ah(struct ib_ah *ib_ah, struct ib_ah_attr *ah_attr) +{ + return 0; +} + +int bnxt_re_query_ah(struct ib_ah *ib_ah, struct ib_ah_attr *ah_attr) +{ + struct bnxt_re_ah *ah = to_bnxt_re(ib_ah, struct bnxt_re_ah, ibah); + + memcpy(ah_attr->grh.dgid.raw, ah->qplib_ah.dgid.data, + sizeof(union ib_gid)); + ah_attr->grh.sgid_index = ah->qplib_ah.host_sgid_index; + ah_attr->grh.traffic_class = ah->qplib_ah.traffic_class; + ah_attr->sl = ah->qplib_ah.sl; + memcpy(ROCE_DMAC(ah_attr), ah->qplib_ah.dmac, ETH_ALEN); + ah_attr->ah_flags = IB_AH_GRH; + ah_attr->port_num = 1; + ah_attr->static_rate = 0; + + return 0; +} + +/* Shared Receive Queues */ +void bnxt_re_destroy_srq(struct ib_srq *ib_srq, + struct ib_udata *udata) +{ + struct bnxt_re_srq *srq = to_bnxt_re(ib_srq, struct bnxt_re_srq, ibsrq); + struct bnxt_re_dev *rdev = srq->rdev; + struct bnxt_qplib_srq *qplib_srq = &srq->qplib_srq; + int rc = 0; + + + rc = bnxt_qplib_destroy_srq(&rdev->qplib_res, qplib_srq); + if (rc) + dev_err_ratelimited(rdev_to_dev(rdev), + "%s id = %d failed rc = %d\n", + __func__, qplib_srq->id, rc); + + if (srq->umem && !IS_ERR(srq->umem)) + ib_umem_release(srq->umem); + + atomic_dec(&rdev->stats.rsors.srq_count); + + return; +} + +static u16 _max_rwqe_sz(int nsge) +{ + return sizeof(struct rq_wqe_hdr) + (nsge * sizeof(struct sq_sge)); +} + +static u16 bnxt_re_get_rwqe_size(struct bnxt_qplib_qp *qplqp, + int rsge, int max) +{ + if (qplqp->wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC) + rsge = max; + + return _max_rwqe_sz(rsge); +} + +static inline +struct ib_umem *ib_umem_get_compat(struct bnxt_re_dev *rdev, + struct ib_ucontext *ucontext, + struct ib_udata *udata, + unsigned long addr, + size_t size, int access, int dmasync) +{ + return ib_umem_get(ucontext, addr, size, access, dmasync); +} + +static inline +struct ib_umem *ib_umem_get_flags_compat(struct bnxt_re_dev *rdev, + struct ib_ucontext *ucontext, + struct ib_udata *udata, + unsigned long addr, + size_t size, int access, int dmasync) +{ + return ib_umem_get_compat(rdev, ucontext, udata, addr, size, + access, 0); +} + +static inline size_t ib_umem_num_pages_compat(struct ib_umem *umem) +{ + return ib_umem_num_pages(umem); +} + +static int bnxt_re_init_user_srq(struct bnxt_re_dev *rdev, + struct bnxt_re_pd *pd, + struct bnxt_re_srq *srq, + struct ib_udata *udata) +{ + struct bnxt_qplib_sg_info *sginfo; + struct bnxt_qplib_srq *qplib_srq; + struct bnxt_re_ucontext *cntx; + struct ib_ucontext *context; + struct bnxt_re_srq_req ureq; + struct ib_umem *umem; + int rc, bytes = 0; + + context = pd->ibpd.uobject->context; + cntx = to_bnxt_re(context, struct bnxt_re_ucontext, ibucontext); + qplib_srq = &srq->qplib_srq; + sginfo = &qplib_srq->sginfo; + + if (udata->inlen < sizeof(ureq)) + dev_warn(rdev_to_dev(rdev), + "Update the library ulen %d klen %d\n", + (unsigned int)udata->inlen, + (unsigned int)sizeof(ureq)); + + rc = ib_copy_from_udata(&ureq, udata, + min(udata->inlen, sizeof(ureq))); + if (rc) + return rc; + + bytes = (qplib_srq->max_wqe * qplib_srq->wqe_size); + bytes = PAGE_ALIGN(bytes); + umem = ib_umem_get_compat(rdev, context, udata, ureq.srqva, bytes, + IB_ACCESS_LOCAL_WRITE, 1); + if (IS_ERR(umem)) { + dev_err(rdev_to_dev(rdev), "%s: ib_umem_get failed with %ld\n", + __func__, PTR_ERR(umem)); + return PTR_ERR(umem); + } + + srq->umem = umem; + sginfo->sghead = get_ib_umem_sgl(umem, &sginfo->nmap); + sginfo->npages = ib_umem_num_pages_compat(umem); + qplib_srq->srq_handle = ureq.srq_handle; + qplib_srq->dpi = &cntx->dpi; + qplib_srq->is_user = true; + + return 0; +} + +int bnxt_re_create_srq(struct ib_srq *srq_in, struct ib_srq_init_attr *srq_init_attr, + struct ib_udata *udata) +{ + struct bnxt_qplib_dev_attr *dev_attr; + struct bnxt_re_ucontext *cntx = NULL; + struct ib_ucontext *context; + struct bnxt_re_dev *rdev; + struct bnxt_re_pd *pd; + int rc, entries; + struct ib_srq *ib_srq = srq_in; + struct ib_pd *ib_pd = ib_srq->pd; + struct bnxt_re_srq *srq = + container_of(ib_srq, struct bnxt_re_srq, ibsrq); + u32 max_srq_count; + + pd = to_bnxt_re(ib_pd, struct bnxt_re_pd, ibpd); + rdev = pd->rdev; + dev_attr = rdev->dev_attr; + + if (rdev->mod_exit) { + dev_dbg(rdev_to_dev(rdev), "%s(): in mod_exit, just return!\n", __func__); + rc = -EIO; + goto exit; + } + + if (srq_init_attr->srq_type != IB_SRQT_BASIC) { + dev_err(rdev_to_dev(rdev), "SRQ type not supported\n"); + rc = -ENOTSUPP; + goto exit; + } + + if (udata) { + context = pd->ibpd.uobject->context; + cntx = to_bnxt_re(context, struct bnxt_re_ucontext, ibucontext); + } + + if (atomic_read(&rdev->stats.rsors.srq_count) >= dev_attr->max_srq) { + dev_err(rdev_to_dev(rdev), "Create SRQ failed - max exceeded(SRQs)\n"); + rc = -EINVAL; + goto exit; + } + + if (srq_init_attr->attr.max_wr >= dev_attr->max_srq_wqes) { + dev_err(rdev_to_dev(rdev), "Create SRQ failed - max exceeded(SRQ_WQs)\n"); + rc = -EINVAL; + goto exit; + } + + srq->rdev = rdev; + srq->qplib_srq.pd = &pd->qplib_pd; + srq->qplib_srq.dpi = &rdev->dpi_privileged; + + /* Allocate 1 more than what's provided so posting max doesn't + mean empty */ + entries = srq_init_attr->attr.max_wr + 1; + entries = bnxt_re_init_depth(entries, cntx); + if (entries > dev_attr->max_srq_wqes + 1) + entries = dev_attr->max_srq_wqes + 1; + + srq->qplib_srq.wqe_size = _max_rwqe_sz(6); /* 128 byte wqe size */ + srq->qplib_srq.max_wqe = entries; + srq->qplib_srq.max_sge = srq_init_attr->attr.max_sge; + srq->qplib_srq.threshold = srq_init_attr->attr.srq_limit; + srq->srq_limit = srq_init_attr->attr.srq_limit; + srq->qplib_srq.eventq_hw_ring_id = rdev->nqr.nq[0].ring_id; + srq->qplib_srq.sginfo.pgsize = PAGE_SIZE; + srq->qplib_srq.sginfo.pgshft = PAGE_SHIFT; + + if (udata) { + rc = bnxt_re_init_user_srq(rdev, pd, srq, udata); + if (rc) + goto fail; + } + + rc = bnxt_qplib_create_srq(&rdev->qplib_res, &srq->qplib_srq); + if (rc) { + dev_err(rdev_to_dev(rdev), "Create HW SRQ failed!\n"); + goto fail; + } + + if (udata) { + struct bnxt_re_srq_resp resp; + + resp.srqid = srq->qplib_srq.id; + rc = bnxt_re_copy_to_udata(rdev, &resp, + min(udata->outlen, sizeof(resp)), + udata); + if (rc) { + bnxt_qplib_destroy_srq(&rdev->qplib_res, &srq->qplib_srq); + goto fail; + } + } + atomic_inc(&rdev->stats.rsors.srq_count); + max_srq_count = atomic_read(&rdev->stats.rsors.srq_count); + if (max_srq_count > atomic_read(&rdev->stats.rsors.max_srq_count)) + atomic_set(&rdev->stats.rsors.max_srq_count, max_srq_count); + spin_lock_init(&srq->lock); + + return 0; +fail: + if (udata && srq->umem && !IS_ERR(srq->umem)) { + ib_umem_release(srq->umem); + srq->umem = NULL; + } +exit: + return rc; +} + +int bnxt_re_modify_srq(struct ib_srq *ib_srq, struct ib_srq_attr *srq_attr, + enum ib_srq_attr_mask srq_attr_mask, + struct ib_udata *udata) +{ + struct bnxt_re_srq *srq = to_bnxt_re(ib_srq, struct bnxt_re_srq, + ibsrq); + struct bnxt_re_dev *rdev = srq->rdev; + int rc; + + switch (srq_attr_mask) { + case IB_SRQ_MAX_WR: + /* SRQ resize is not supported */ + break; + case IB_SRQ_LIMIT: + /* Change the SRQ threshold */ + if (srq_attr->srq_limit > srq->qplib_srq.max_wqe) + return -EINVAL; + + srq->qplib_srq.threshold = srq_attr->srq_limit; + rc = bnxt_qplib_modify_srq(&rdev->qplib_res, &srq->qplib_srq); + if (rc) { + dev_err(rdev_to_dev(rdev), "Modify HW SRQ failed!\n"); + return rc; + } + /* On success, update the shadow */ + srq->srq_limit = srq_attr->srq_limit; + + if (udata) { + /* Build and send response back to udata */ + rc = bnxt_re_copy_to_udata(rdev, srq, 0, udata); + if (rc) + return rc; + } + break; + default: + dev_err(rdev_to_dev(rdev), + "Unsupported srq_attr_mask 0x%x\n", srq_attr_mask); + return -EINVAL; + } + return 0; +} + +int bnxt_re_query_srq(struct ib_srq *ib_srq, struct ib_srq_attr *srq_attr) +{ + struct bnxt_re_srq *srq = to_bnxt_re(ib_srq, struct bnxt_re_srq, + ibsrq); + struct bnxt_re_dev *rdev = srq->rdev; + int rc; + + rc = bnxt_qplib_query_srq(&rdev->qplib_res, &srq->qplib_srq); + if (rc) { + dev_err(rdev_to_dev(rdev), "Query HW SRQ (0x%x) failed! rc = %d\n", + srq->qplib_srq.id, rc); + return rc; + } + srq_attr->max_wr = srq->qplib_srq.max_wqe; + srq_attr->max_sge = srq->qplib_srq.max_sge; + srq_attr->srq_limit = srq->qplib_srq.threshold; + + return 0; +} + +int bnxt_re_post_srq_recv(struct ib_srq *ib_srq, const struct ib_recv_wr *wr, + const struct ib_recv_wr **bad_wr) +{ + struct bnxt_re_srq *srq = to_bnxt_re(ib_srq, struct bnxt_re_srq, + ibsrq); + struct bnxt_qplib_swqe wqe = {}; + unsigned long flags; + int rc = 0; + + spin_lock_irqsave(&srq->lock, flags); + while (wr) { + /* Transcribe each ib_recv_wr to qplib_swqe */ + wqe.num_sge = wr->num_sge; + wqe.sg_list = (struct bnxt_qplib_sge *)wr->sg_list; + wqe.wr_id = wr->wr_id; + wqe.type = BNXT_QPLIB_SWQE_TYPE_RECV; + rc = bnxt_qplib_post_srq_recv(&srq->qplib_srq, &wqe); + if (rc) { + *bad_wr = wr; + break; + } + wr = wr->next; + } + spin_unlock_irqrestore(&srq->lock, flags); + + return rc; +} + +unsigned long bnxt_re_lock_cqs(struct bnxt_re_qp *qp) +{ + unsigned long flags; + + spin_lock_irqsave(&qp->scq->cq_lock, flags); + if (qp->rcq && qp->rcq != qp->scq) + spin_lock(&qp->rcq->cq_lock); + + return flags; +} + +void bnxt_re_unlock_cqs(struct bnxt_re_qp *qp, + unsigned long flags) +{ + if (qp->rcq && qp->rcq != qp->scq) + spin_unlock(&qp->rcq->cq_lock); + spin_unlock_irqrestore(&qp->scq->cq_lock, flags); +} + +/* Queue Pairs */ +static int bnxt_re_destroy_gsi_sqp(struct bnxt_re_qp *qp) +{ + struct bnxt_re_qp *gsi_sqp; + struct bnxt_re_ah *gsi_sah; + struct bnxt_re_dev *rdev; + unsigned long flags; + int rc = 0; + + rdev = qp->rdev; + gsi_sqp = rdev->gsi_ctx.gsi_sqp; + gsi_sah = rdev->gsi_ctx.gsi_sah; + + /* remove from active qp list */ + mutex_lock(&rdev->qp_lock); + list_del(&gsi_sqp->list); + mutex_unlock(&rdev->qp_lock); + + if (gsi_sah) { + dev_dbg(rdev_to_dev(rdev), "Destroy the shadow AH\n"); + rc = bnxt_qplib_destroy_ah(&rdev->qplib_res, &gsi_sah->qplib_ah, + true); + if (rc) + dev_err(rdev_to_dev(rdev), + "Destroy HW AH for shadow QP failed!\n"); + atomic_dec(&rdev->stats.rsors.ah_count); + } + + dev_dbg(rdev_to_dev(rdev), "Destroy the shadow QP\n"); + rc = bnxt_qplib_destroy_qp(&rdev->qplib_res, &gsi_sqp->qplib_qp); + if (rc) + dev_err(rdev_to_dev(rdev), "Destroy Shadow QP failed\n"); + + /* Clean the CQ for shadow QP completions */ + flags = bnxt_re_lock_cqs(gsi_sqp); + bnxt_qplib_clean_qp(&gsi_sqp->qplib_qp); + bnxt_re_unlock_cqs(gsi_sqp, flags); + + bnxt_qplib_free_qp_res(&rdev->qplib_res, &gsi_sqp->qplib_qp); + bnxt_qplib_free_hdr_buf(&rdev->qplib_res, &gsi_sqp->qplib_qp); + kfree(rdev->gsi_ctx.sqp_tbl); + kfree(gsi_sah); + kfree(gsi_sqp); + rdev->gsi_ctx.gsi_sqp = NULL; + rdev->gsi_ctx.gsi_sah = NULL; + rdev->gsi_ctx.sqp_tbl = NULL; + atomic_dec(&rdev->stats.rsors.qp_count); + + return 0; +} + +static void bnxt_re_dump_debug_stats(struct bnxt_re_dev *rdev, u32 active_qps) +{ + u32 total_qp = 0; + u64 avg_time = 0; + int i; + + if (!rdev->rcfw.sp_perf_stats_enabled) + return; + + switch (active_qps) { + case 1: + /* Potential hint for Test Stop */ + for (i = 0; i < RCFW_MAX_STAT_INDEX; i++) { + if (rdev->rcfw.qp_destroy_stats[i]) { + total_qp++; + avg_time += rdev->rcfw.qp_destroy_stats[i]; + } + } + if (total_qp >= 0 || avg_time >= 0) + dev_dbg(rdev_to_dev(rdev), + "Perf Debug: %ps Total (%d) QP destroyed in (%ld) msec\n", + __builtin_return_address(0), total_qp, + (long)jiffies_to_msecs(avg_time)); + break; + case 2: + /* Potential hint for Test Start */ + dev_dbg(rdev_to_dev(rdev), + "Perf Debug: %ps active_qps = %d\n", + __builtin_return_address(0), active_qps); + break; + default: + /* Potential hint to know latency of QP destroy. + * Average time taken for 1K QP Destroy. + */ + if (active_qps > 1024 && !(active_qps % 1024)) + dev_dbg(rdev_to_dev(rdev), + "Perf Debug: %ps Active QP (%d) Watermark (%d)\n", + __builtin_return_address(0), active_qps, + atomic_read(&rdev->stats.rsors.max_qp_count)); + break; + } +} + +int bnxt_re_destroy_qp(struct ib_qp *ib_qp, struct ib_udata *udata) +{ + struct bnxt_re_qp *qp = to_bnxt_re(ib_qp, struct bnxt_re_qp, ib_qp); + struct bnxt_re_dev *rdev = qp->rdev; + unsigned long flags; + u32 active_qps; + int rc; + + mutex_lock(&rdev->qp_lock); + list_del(&qp->list); + active_qps = atomic_dec_return(&rdev->stats.rsors.qp_count); + if (qp->qplib_qp.type == CMDQ_CREATE_QP_TYPE_RC) + atomic_dec(&rdev->stats.rsors.rc_qp_count); + else if (qp->qplib_qp.type == CMDQ_CREATE_QP_TYPE_UD) + atomic_dec(&rdev->stats.rsors.ud_qp_count); + mutex_unlock(&rdev->qp_lock); + + rc = bnxt_qplib_destroy_qp(&rdev->qplib_res, &qp->qplib_qp); + if (rc) + dev_err_ratelimited(rdev_to_dev(rdev), + "%s id = %d failed rc = %d\n", + __func__, qp->qplib_qp.id, rc); + + if (!ib_qp->uobject) { + flags = bnxt_re_lock_cqs(qp); + bnxt_qplib_clean_qp(&qp->qplib_qp); + bnxt_re_unlock_cqs(qp, flags); + } + + bnxt_qplib_free_qp_res(&rdev->qplib_res, &qp->qplib_qp); + if (ib_qp->qp_type == IB_QPT_GSI && + rdev->gsi_ctx.gsi_qp_mode != BNXT_RE_GSI_MODE_UD) { + if (rdev->gsi_ctx.gsi_qp_mode == BNXT_RE_GSI_MODE_ALL && + rdev->gsi_ctx.gsi_sqp) { + bnxt_re_destroy_gsi_sqp(qp); + } + bnxt_qplib_free_hdr_buf(&rdev->qplib_res, &qp->qplib_qp); + } + + if (qp->rumem && !IS_ERR(qp->rumem)) + ib_umem_release(qp->rumem); + if (qp->sumem && !IS_ERR(qp->sumem)) + ib_umem_release(qp->sumem); + kfree(qp); + + bnxt_re_dump_debug_stats(rdev, active_qps); + + return 0; +} + +static u8 __from_ib_qp_type(enum ib_qp_type type) +{ + switch (type) { + case IB_QPT_GSI: + return CMDQ_CREATE_QP1_TYPE_GSI; + case IB_QPT_RC: + return CMDQ_CREATE_QP_TYPE_RC; + case IB_QPT_UD: + return CMDQ_CREATE_QP_TYPE_UD; + case IB_QPT_RAW_ETHERTYPE: + return CMDQ_CREATE_QP_TYPE_RAW_ETHERTYPE; + default: + return IB_QPT_MAX; + } +} + +static u16 _get_swqe_sz(int nsge) +{ + return sizeof(struct sq_send_hdr) + nsge * sizeof(struct sq_sge); +} + +static int bnxt_re_get_swqe_size(int ilsize, int nsge) +{ + u16 wqe_size, calc_ils; + + wqe_size = _get_swqe_sz(nsge); + if (ilsize) { + calc_ils = (sizeof(struct sq_send_hdr) + ilsize); + wqe_size = max_t(int, calc_ils, wqe_size); + wqe_size = ALIGN(wqe_size, 32); + } + return wqe_size; +} + +static int bnxt_re_setup_swqe_size(struct bnxt_re_qp *qp, + struct ib_qp_init_attr *init_attr) +{ + struct bnxt_qplib_dev_attr *dev_attr; + struct bnxt_qplib_qp *qplqp; + struct bnxt_re_dev *rdev; + struct bnxt_qplib_q *sq; + int align, ilsize; + + rdev = qp->rdev; + qplqp = &qp->qplib_qp; + sq = &qplqp->sq; + dev_attr = rdev->dev_attr; + + align = sizeof(struct sq_send_hdr); + ilsize = ALIGN(init_attr->cap.max_inline_data, align); + + sq->wqe_size = bnxt_re_get_swqe_size(ilsize, sq->max_sge); + if (sq->wqe_size > _get_swqe_sz(dev_attr->max_qp_sges)) + return -EINVAL; + /* For Cu/Wh and gen p5 backward compatibility mode + * wqe size is fixed to 128 bytes + */ + if (sq->wqe_size < _get_swqe_sz(dev_attr->max_qp_sges) && + qplqp->wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC) + sq->wqe_size = _get_swqe_sz(dev_attr->max_qp_sges); + + if (init_attr->cap.max_inline_data) { + qplqp->max_inline_data = sq->wqe_size - + sizeof(struct sq_send_hdr); + init_attr->cap.max_inline_data = qplqp->max_inline_data; + if (qplqp->wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC) + sq->max_sge = qplqp->max_inline_data / + sizeof(struct sq_sge); + } + + return 0; +} + +static int bnxt_re_init_user_qp(struct bnxt_re_dev *rdev, + struct bnxt_re_pd *pd, struct bnxt_re_qp *qp, + struct ib_udata *udata) +{ + struct bnxt_qplib_sg_info *sginfo; + struct bnxt_qplib_qp *qplib_qp; + struct bnxt_re_ucontext *cntx; + struct ib_ucontext *context; + struct bnxt_re_qp_req ureq; + struct ib_umem *umem; + int rc, bytes = 0; + int psn_nume; + int psn_sz; + + qplib_qp = &qp->qplib_qp; + context = pd->ibpd.uobject->context; + cntx = to_bnxt_re(context, struct bnxt_re_ucontext, ibucontext); + sginfo = &qplib_qp->sq.sginfo; + + if (udata->inlen < sizeof(ureq)) + dev_warn(rdev_to_dev(rdev), + "Update the library ulen %d klen %d\n", + (unsigned int)udata->inlen, + (unsigned int)sizeof(ureq)); + + rc = ib_copy_from_udata(&ureq, udata, + min(udata->inlen, sizeof(ureq))); + if (rc) + return rc; + + bytes = (qplib_qp->sq.max_wqe * qplib_qp->sq.wqe_size); + /* Consider mapping PSN search memory only for RC QPs. */ + if (qplib_qp->type == CMDQ_CREATE_QP_TYPE_RC) { + psn_sz = _is_chip_gen_p5_p7(rdev->chip_ctx) ? + sizeof(struct sq_psn_search_ext) : + sizeof(struct sq_psn_search); + if (rdev->dev_attr && BNXT_RE_HW_RETX(rdev->dev_attr->dev_cap_flags)) + psn_sz = sizeof(struct sq_msn_search); + psn_nume = (qplib_qp->wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC) ? + qplib_qp->sq.max_wqe : + ((qplib_qp->sq.max_wqe * qplib_qp->sq.wqe_size) / + sizeof(struct bnxt_qplib_sge)); + if (BNXT_RE_HW_RETX(rdev->dev_attr->dev_cap_flags)) + psn_nume = roundup_pow_of_two(psn_nume); + + bytes += (psn_nume * psn_sz); + } + bytes = PAGE_ALIGN(bytes); + umem = ib_umem_get_compat(rdev, context, udata, ureq.qpsva, bytes, + IB_ACCESS_LOCAL_WRITE, 1); + if (IS_ERR(umem)) { + dev_err(rdev_to_dev(rdev), "%s: ib_umem_get failed with %ld\n", + __func__, PTR_ERR(umem)); + return PTR_ERR(umem); + } + + qp->sumem = umem; + /* pgsize and pgshft were initialize already. */ + sginfo->sghead = get_ib_umem_sgl(umem, &sginfo->nmap); + sginfo->npages = ib_umem_num_pages_compat(umem); + qplib_qp->qp_handle = ureq.qp_handle; + + if (!qp->qplib_qp.srq) { + sginfo = &qplib_qp->rq.sginfo; + bytes = (qplib_qp->rq.max_wqe * qplib_qp->rq.wqe_size); + bytes = PAGE_ALIGN(bytes); + umem = ib_umem_get_compat(rdev, + context, udata, ureq.qprva, bytes, + IB_ACCESS_LOCAL_WRITE, 1); + if (IS_ERR(umem)) { + dev_err(rdev_to_dev(rdev), + "%s: ib_umem_get failed ret =%ld\n", + __func__, PTR_ERR(umem)); + goto rqfail; + } + qp->rumem = umem; + /* pgsize and pgshft were initialize already. */ + sginfo->sghead = get_ib_umem_sgl(umem, &sginfo->nmap); + sginfo->npages = ib_umem_num_pages_compat(umem); + } + + qplib_qp->dpi = &cntx->dpi; + qplib_qp->is_user = true; + + return 0; +rqfail: + ib_umem_release(qp->sumem); + qp->sumem = NULL; + qplib_qp->sq.sginfo.sghead = NULL; + qplib_qp->sq.sginfo.nmap = 0; + + return PTR_ERR(umem); +} + +static struct bnxt_re_ah *bnxt_re_create_shadow_qp_ah(struct bnxt_re_pd *pd, + struct bnxt_qplib_res *qp1_res, + struct bnxt_qplib_qp *qp1_qp) +{ + struct bnxt_re_dev *rdev = pd->rdev; + struct bnxt_re_ah *ah; + union ib_gid sgid; + int rc; + + ah = kzalloc(sizeof(*ah), GFP_KERNEL); + if (!ah) { + dev_err(rdev_to_dev(rdev), "Allocate Address Handle failed!\n"); + return NULL; + } + memset(ah, 0, sizeof(*ah)); + ah->rdev = rdev; + ah->qplib_ah.pd = &pd->qplib_pd; + + rc = bnxt_re_query_gid(&rdev->ibdev, 1, 0, &sgid); + if (rc) + goto fail; + + /* supply the dgid data same as sgid */ + memcpy(ah->qplib_ah.dgid.data, &sgid.raw, + sizeof(union ib_gid)); + ah->qplib_ah.sgid_index = 0; + + ah->qplib_ah.traffic_class = 0; + ah->qplib_ah.flow_label = 0; + ah->qplib_ah.hop_limit = 1; + ah->qplib_ah.sl = 0; + /* Have DMAC same as SMAC */ + ether_addr_copy(ah->qplib_ah.dmac, rdev->dev_addr); + dev_dbg(rdev_to_dev(rdev), "ah->qplib_ah.dmac = %x:%x:%x:%x:%x:%x\n", + ah->qplib_ah.dmac[0], ah->qplib_ah.dmac[1], ah->qplib_ah.dmac[2], + ah->qplib_ah.dmac[3], ah->qplib_ah.dmac[4], ah->qplib_ah.dmac[5]); + + rc = bnxt_qplib_create_ah(&rdev->qplib_res, &ah->qplib_ah, true); + if (rc) { + dev_err(rdev_to_dev(rdev), + "Allocate HW AH for Shadow QP failed!\n"); + goto fail; + } + dev_dbg(rdev_to_dev(rdev), "AH ID = %d\n", ah->qplib_ah.id); + atomic_inc(&rdev->stats.rsors.ah_count); + + return ah; +fail: + kfree(ah); + return NULL; +} + +void bnxt_re_update_shadow_ah(struct bnxt_re_dev *rdev) +{ + struct bnxt_re_qp *gsi_qp; + struct bnxt_re_ah *sah; + struct bnxt_re_pd *pd; + struct ib_pd *ib_pd; + int rc; + + if (!rdev) + return; + + sah = rdev->gsi_ctx.gsi_sah; + + dev_dbg(rdev_to_dev(rdev), "Updating the AH\n"); + if (sah) { + /* Check if the AH created with current mac address */ + if (!compare_ether_header(sah->qplib_ah.dmac, rdev->dev_addr)) { + dev_dbg(rdev_to_dev(rdev), + "Not modifying shadow AH during AH update\n"); + return; + } + + gsi_qp = rdev->gsi_ctx.gsi_qp; + ib_pd = gsi_qp->ib_qp.pd; + pd = to_bnxt_re(ib_pd, struct bnxt_re_pd, ibpd); + rc = bnxt_qplib_destroy_ah(&rdev->qplib_res, + &sah->qplib_ah, false); + if (rc) { + dev_err(rdev_to_dev(rdev), + "Failed to destroy shadow AH during AH update\n"); + return; + } + atomic_dec(&rdev->stats.rsors.ah_count); + kfree(sah); + rdev->gsi_ctx.gsi_sah = NULL; + + sah = bnxt_re_create_shadow_qp_ah(pd, &rdev->qplib_res, + &gsi_qp->qplib_qp); + if (!sah) { + dev_err(rdev_to_dev(rdev), + "Failed to update AH for ShadowQP\n"); + return; + } + rdev->gsi_ctx.gsi_sah = sah; + atomic_inc(&rdev->stats.rsors.ah_count); + } +} + +static struct bnxt_re_qp *bnxt_re_create_shadow_qp(struct bnxt_re_pd *pd, + struct bnxt_qplib_res *qp1_res, + struct bnxt_qplib_qp *qp1_qp) +{ + struct bnxt_re_dev *rdev = pd->rdev; + struct bnxt_re_qp *qp; + int rc; + + qp = kzalloc(sizeof(*qp), GFP_KERNEL); + if (!qp) { + dev_err(rdev_to_dev(rdev), "Allocate internal UD QP failed!\n"); + return NULL; + } + memset(qp, 0, sizeof(*qp)); + qp->rdev = rdev; + + /* Initialize the shadow QP structure from the QP1 values */ + ether_addr_copy(qp->qplib_qp.smac, rdev->dev_addr); + qp->qplib_qp.pd = &pd->qplib_pd; + qp->qplib_qp.qp_handle = (u64)&qp->qplib_qp; + qp->qplib_qp.type = IB_QPT_UD; + + qp->qplib_qp.max_inline_data = 0; + qp->qplib_qp.sig_type = true; + + /* Shadow QP SQ depth should be same as QP1 RQ depth */ + qp->qplib_qp.sq.wqe_size = bnxt_re_get_swqe_size(0, 6); + qp->qplib_qp.sq.max_wqe = qp1_qp->rq.max_wqe; + qp->qplib_qp.sq.max_sge = 2; + /* Q full delta can be 1 since it is internal QP */ + qp->qplib_qp.sq.q_full_delta = 1; + qp->qplib_qp.sq.sginfo.pgsize = PAGE_SIZE; + qp->qplib_qp.sq.sginfo.pgshft = PAGE_SHIFT; + + qp->qplib_qp.scq = qp1_qp->scq; + qp->qplib_qp.rcq = qp1_qp->rcq; + + qp->qplib_qp.rq.wqe_size = _max_rwqe_sz(6); /* 128 Byte wqe size */ + qp->qplib_qp.rq.max_wqe = qp1_qp->rq.max_wqe; + qp->qplib_qp.rq.max_sge = qp1_qp->rq.max_sge; + qp->qplib_qp.rq.sginfo.pgsize = PAGE_SIZE; + qp->qplib_qp.rq.sginfo.pgshft = PAGE_SHIFT; + /* Q full delta can be 1 since it is internal QP */ + qp->qplib_qp.rq.q_full_delta = 1; + qp->qplib_qp.mtu = qp1_qp->mtu; + qp->qplib_qp.dpi = &rdev->dpi_privileged; + + rc = bnxt_qplib_alloc_hdr_buf(qp1_res, &qp->qplib_qp, 0, + BNXT_QPLIB_MAX_GRH_HDR_SIZE_IPV6); + if (rc) + goto fail; + + rc = bnxt_qplib_create_qp(qp1_res, &qp->qplib_qp); + if (rc) { + dev_err(rdev_to_dev(rdev), "create HW QP failed!\n"); + goto qp_fail; + } + + dev_dbg(rdev_to_dev(rdev), "Created shadow QP with ID = %d\n", + qp->qplib_qp.id); + spin_lock_init(&qp->sq_lock); + INIT_LIST_HEAD(&qp->list); + mutex_lock(&rdev->qp_lock); + list_add_tail(&qp->list, &rdev->qp_list); + atomic_inc(&rdev->stats.rsors.qp_count); + mutex_unlock(&rdev->qp_lock); + return qp; +qp_fail: + bnxt_qplib_free_hdr_buf(qp1_res, &qp->qplib_qp); +fail: + kfree(qp); + return NULL; +} + +static int bnxt_re_init_rq_attr(struct bnxt_re_qp *qp, + struct ib_qp_init_attr *init_attr, void *cntx) +{ + struct bnxt_qplib_dev_attr *dev_attr; + struct bnxt_qplib_qp *qplqp; + struct bnxt_re_dev *rdev; + struct bnxt_qplib_q *rq; + int entries; + + rdev = qp->rdev; + qplqp = &qp->qplib_qp; + rq = &qplqp->rq; + dev_attr = rdev->dev_attr; + + if (init_attr->srq) { + struct bnxt_re_srq *srq; + + srq = to_bnxt_re(init_attr->srq, struct bnxt_re_srq, ibsrq); + if (!srq) { + dev_err(rdev_to_dev(rdev), "SRQ not found\n"); + return -EINVAL; + } + qplqp->srq = &srq->qplib_srq; + rq->max_wqe = 0; + } else { + rq->max_sge = init_attr->cap.max_recv_sge; + if (rq->max_sge > dev_attr->max_qp_sges) + rq->max_sge = dev_attr->max_qp_sges; + init_attr->cap.max_recv_sge = rq->max_sge; + rq->wqe_size = bnxt_re_get_rwqe_size(qplqp, rq->max_sge, + dev_attr->max_qp_sges); + + /* Allocate 1 more than what's provided so posting max doesn't + mean empty */ + entries = init_attr->cap.max_recv_wr + 1; + entries = bnxt_re_init_depth(entries, cntx); + rq->max_wqe = min_t(u32, entries, dev_attr->max_qp_wqes + 1); + rq->q_full_delta = 0; + rq->sginfo.pgsize = PAGE_SIZE; + rq->sginfo.pgshft = PAGE_SHIFT; + } + + return 0; +} + +static void bnxt_re_adjust_gsi_rq_attr(struct bnxt_re_qp *qp) +{ + struct bnxt_qplib_dev_attr *dev_attr; + struct bnxt_qplib_qp *qplqp; + struct bnxt_re_dev *rdev; + + rdev = qp->rdev; + qplqp = &qp->qplib_qp; + dev_attr = rdev->dev_attr; + + if (rdev->gsi_ctx.gsi_qp_mode != BNXT_RE_GSI_MODE_UD) + qplqp->rq.max_sge = dev_attr->max_qp_sges; +} + +static int bnxt_re_init_sq_attr(struct bnxt_re_qp *qp, + struct ib_qp_init_attr *init_attr, + void *cntx) +{ + struct bnxt_qplib_dev_attr *dev_attr; + struct bnxt_qplib_qp *qplqp; + struct bnxt_re_dev *rdev; + struct bnxt_qplib_q *sq; + int diff = 0; + int entries; + int rc; + + rdev = qp->rdev; + qplqp = &qp->qplib_qp; + sq = &qplqp->sq; + dev_attr = rdev->dev_attr; + + sq->max_sge = init_attr->cap.max_send_sge; + if (sq->max_sge > dev_attr->max_qp_sges) { + sq->max_sge = dev_attr->max_qp_sges; + init_attr->cap.max_send_sge = sq->max_sge; + } + rc = bnxt_re_setup_swqe_size(qp, init_attr); + if (rc) + return rc; + /* + * Change the SQ depth if user has requested minimum using + * configfs. Only supported for kernel consumers. Setting + * min_tx_depth to 4096 to handle iser SQ full condition + * in most of the newer OS distros + */ + entries = init_attr->cap.max_send_wr; + if (!cntx && rdev->min_tx_depth && init_attr->qp_type != IB_QPT_GSI) { + /* + * If users specify any value greater than 1 use min_tx_depth + * provided by user for comparison. Else, compare it with the + * BNXT_RE_MIN_KERNEL_QP_TX_DEPTH and adjust it accordingly. + */ + if (rdev->min_tx_depth > 1 && entries < rdev->min_tx_depth) + entries = rdev->min_tx_depth; + else if (entries < BNXT_RE_MIN_KERNEL_QP_TX_DEPTH) + entries = BNXT_RE_MIN_KERNEL_QP_TX_DEPTH; + } + diff = bnxt_re_get_diff(cntx, rdev->chip_ctx); + entries = bnxt_re_init_depth(entries + diff + 1, cntx); + sq->max_wqe = min_t(u32, entries, dev_attr->max_qp_wqes + diff + 1); + sq->q_full_delta = diff + 1; + /* + * Reserving one slot for Phantom WQE. Application can + * post one extra entry in this case. But allowing this to avoid + * unexpected Queue full condition + */ + sq->q_full_delta -= 1; /* becomes 0 for gen-p5 */ + sq->sginfo.pgsize = PAGE_SIZE; + sq->sginfo.pgshft = PAGE_SHIFT; + return 0; +} + +static void bnxt_re_adjust_gsi_sq_attr(struct bnxt_re_qp *qp, + struct ib_qp_init_attr *init_attr, + void *cntx) +{ + struct bnxt_qplib_dev_attr *dev_attr; + struct bnxt_qplib_qp *qplqp; + struct bnxt_re_dev *rdev; + int entries; + + rdev = qp->rdev; + qplqp = &qp->qplib_qp; + dev_attr = rdev->dev_attr; + + if (rdev->gsi_ctx.gsi_qp_mode != BNXT_RE_GSI_MODE_UD) { + entries = init_attr->cap.max_send_wr + 1; + entries = bnxt_re_init_depth(entries, cntx); + qplqp->sq.max_wqe = min_t(u32, entries, + dev_attr->max_qp_wqes + 1); + qplqp->sq.q_full_delta = qplqp->sq.max_wqe - + init_attr->cap.max_send_wr; + qplqp->sq.max_sge++; /* Need one extra sge to put UD header */ + if (qplqp->sq.max_sge > dev_attr->max_qp_sges) + qplqp->sq.max_sge = dev_attr->max_qp_sges; + } +} + +static int bnxt_re_init_qp_type(struct bnxt_re_dev *rdev, + struct ib_qp_init_attr *init_attr) +{ + struct bnxt_qplib_chip_ctx *chip_ctx; + struct bnxt_re_gsi_context *gsi_ctx; + int qptype; + + chip_ctx = rdev->chip_ctx; + gsi_ctx = &rdev->gsi_ctx; + + qptype = __from_ib_qp_type(init_attr->qp_type); + if (qptype == IB_QPT_MAX) { + dev_err(rdev_to_dev(rdev), "QP type 0x%x not supported\n", + qptype); + qptype = -EINVAL; + goto out; + } + + if (_is_chip_gen_p5_p7(chip_ctx) && init_attr->qp_type == IB_QPT_GSI) { + /* For Thor always force UD mode. */ + qptype = CMDQ_CREATE_QP_TYPE_GSI; + gsi_ctx->gsi_qp_mode = BNXT_RE_GSI_MODE_UD; + } +out: + return qptype; +} + +static int bnxt_re_init_qp_wqe_mode(struct bnxt_re_dev *rdev) +{ + return rdev->chip_ctx->modes.wqe_mode; +} + +static int bnxt_re_init_qp_attr(struct bnxt_re_qp *qp, struct bnxt_re_pd *pd, + struct ib_qp_init_attr *init_attr, + struct ib_udata *udata) +{ + struct bnxt_qplib_dev_attr *dev_attr; + struct bnxt_re_ucontext *cntx = NULL; + struct ib_ucontext *context; + struct bnxt_qplib_qp *qplqp; + struct bnxt_re_dev *rdev; + struct bnxt_re_cq *cq; + int rc = 0, qptype; + + rdev = qp->rdev; + qplqp = &qp->qplib_qp; + dev_attr = rdev->dev_attr; + + if (udata) { + context = pd->ibpd.uobject->context; + cntx = to_bnxt_re(context, struct bnxt_re_ucontext, ibucontext); + } + + /* Setup misc params */ + qplqp->is_user = false; + qplqp->pd = &pd->qplib_pd; + qplqp->qp_handle = (u64)qplqp; + qplqp->sig_type = ((init_attr->sq_sig_type == IB_SIGNAL_ALL_WR) ? + true : false); + qptype = bnxt_re_init_qp_type(rdev, init_attr); + if (qptype < 0) { + rc = qptype; + goto out; + } + qplqp->type = (u8)qptype; + qplqp->wqe_mode = bnxt_re_init_qp_wqe_mode(rdev); + ether_addr_copy(qplqp->smac, rdev->dev_addr); + + if (init_attr->qp_type == IB_QPT_RC) { + qplqp->max_rd_atomic = dev_attr->max_qp_rd_atom; + qplqp->max_dest_rd_atomic = dev_attr->max_qp_init_rd_atom; + } + qplqp->mtu = ib_mtu_enum_to_int(iboe_get_mtu(rdev->netdev->if_mtu)); + qplqp->dpi = &rdev->dpi_privileged; /* Doorbell page */ + if (init_attr->create_flags) { + dev_dbg(rdev_to_dev(rdev), + "QP create flags 0x%x not supported\n", + init_attr->create_flags); + return -EOPNOTSUPP; + } + + /* Setup CQs */ + if (init_attr->send_cq) { + cq = to_bnxt_re(init_attr->send_cq, struct bnxt_re_cq, ibcq); + if (!cq) { + dev_err(rdev_to_dev(rdev), "Send CQ not found\n"); + rc = -EINVAL; + goto out; + } + qplqp->scq = &cq->qplib_cq; + qp->scq = cq; + } + + if (init_attr->recv_cq) { + cq = to_bnxt_re(init_attr->recv_cq, struct bnxt_re_cq, ibcq); + if (!cq) { + dev_err(rdev_to_dev(rdev), "Receive CQ not found\n"); + rc = -EINVAL; + goto out; + } + qplqp->rcq = &cq->qplib_cq; + qp->rcq = cq; + } + + /* Setup RQ/SRQ */ + rc = bnxt_re_init_rq_attr(qp, init_attr, cntx); + if (rc) + goto out; + if (init_attr->qp_type == IB_QPT_GSI) + bnxt_re_adjust_gsi_rq_attr(qp); + + /* Setup SQ */ + rc = bnxt_re_init_sq_attr(qp, init_attr, cntx); + if (rc) + goto out; + if (init_attr->qp_type == IB_QPT_GSI) + bnxt_re_adjust_gsi_sq_attr(qp, init_attr, cntx); + + if (udata) /* This will update DPI and qp_handle */ + rc = bnxt_re_init_user_qp(rdev, pd, qp, udata); +out: + return rc; +} + +static int bnxt_re_create_shadow_gsi(struct bnxt_re_qp *qp, + struct bnxt_re_pd *pd) +{ + struct bnxt_re_sqp_entries *sqp_tbl = NULL; + struct bnxt_re_dev *rdev; + struct bnxt_re_qp *sqp; + struct bnxt_re_ah *sah; + int rc = 0; + + rdev = qp->rdev; + /* Create a shadow QP to handle the QP1 traffic */ + sqp_tbl = kzalloc(sizeof(*sqp_tbl) * BNXT_RE_MAX_GSI_SQP_ENTRIES, + GFP_KERNEL); + if (!sqp_tbl) + return -ENOMEM; + rdev->gsi_ctx.sqp_tbl = sqp_tbl; + + sqp = bnxt_re_create_shadow_qp(pd, &rdev->qplib_res, &qp->qplib_qp); + if (!sqp) { + rc = -ENODEV; + dev_err(rdev_to_dev(rdev), + "Failed to create Shadow QP for QP1\n"); + goto out; + } + rdev->gsi_ctx.gsi_sqp = sqp; + + sqp->rcq = qp->rcq; + sqp->scq = qp->scq; + sah = bnxt_re_create_shadow_qp_ah(pd, &rdev->qplib_res, + &qp->qplib_qp); + if (!sah) { + bnxt_qplib_destroy_qp(&rdev->qplib_res, + &sqp->qplib_qp); + rc = -ENODEV; + dev_err(rdev_to_dev(rdev), + "Failed to create AH entry for ShadowQP\n"); + goto out; + } + rdev->gsi_ctx.gsi_sah = sah; + + return 0; +out: + kfree(sqp_tbl); + return rc; +} + +static int __get_rq_hdr_buf_size(u8 gsi_mode) +{ + return (gsi_mode == BNXT_RE_GSI_MODE_ALL) ? + BNXT_QPLIB_MAX_QP1_RQ_HDR_SIZE_V2 : + BNXT_QPLIB_MAX_QP1_RQ_HDR_SIZE; +} + +static int __get_sq_hdr_buf_size(u8 gsi_mode) +{ + return (gsi_mode != BNXT_RE_GSI_MODE_ROCE_V1) ? + BNXT_QPLIB_MAX_QP1_SQ_HDR_SIZE_V2 : + BNXT_QPLIB_MAX_QP1_SQ_HDR_SIZE; +} + +static int bnxt_re_create_gsi_qp(struct bnxt_re_qp *qp, struct bnxt_re_pd *pd) +{ + struct bnxt_qplib_qp *qplqp; + struct bnxt_qplib_res *res; + struct bnxt_re_dev *rdev; + u32 sstep, rstep; + u8 gsi_mode; + int rc = 0; + + rdev = qp->rdev; + qplqp = &qp->qplib_qp; + res = &rdev->qplib_res; + gsi_mode = rdev->gsi_ctx.gsi_qp_mode; + + rstep = __get_rq_hdr_buf_size(gsi_mode); + sstep = __get_sq_hdr_buf_size(gsi_mode); + rc = bnxt_qplib_alloc_hdr_buf(res, qplqp, sstep, rstep); + if (rc) + goto out; + + rc = bnxt_qplib_create_qp1(res, qplqp); + if (rc) { + dev_err(rdev_to_dev(rdev), "create HW QP1 failed!\n"); + goto out; + } + + if (gsi_mode == BNXT_RE_GSI_MODE_ALL) + rc = bnxt_re_create_shadow_gsi(qp, pd); +out: + return rc; +} + +static bool bnxt_re_test_qp_limits(struct bnxt_re_dev *rdev, + struct ib_qp_init_attr *init_attr, + struct bnxt_qplib_dev_attr *dev_attr) +{ + bool rc = true; + int ilsize; + + ilsize = ALIGN(init_attr->cap.max_inline_data, sizeof(struct sq_sge)); + if ((init_attr->cap.max_send_wr > dev_attr->max_qp_wqes) || + (init_attr->cap.max_recv_wr > dev_attr->max_qp_wqes) || + (init_attr->cap.max_send_sge > dev_attr->max_qp_sges) || + (init_attr->cap.max_recv_sge > dev_attr->max_qp_sges) || + (ilsize > dev_attr->max_inline_data)) { + dev_err(rdev_to_dev(rdev), "Create QP failed - max exceeded! " + "0x%x/0x%x 0x%x/0x%x 0x%x/0x%x " + "0x%x/0x%x 0x%x/0x%x\n", + init_attr->cap.max_send_wr, dev_attr->max_qp_wqes, + init_attr->cap.max_recv_wr, dev_attr->max_qp_wqes, + init_attr->cap.max_send_sge, dev_attr->max_qp_sges, + init_attr->cap.max_recv_sge, dev_attr->max_qp_sges, + init_attr->cap.max_inline_data, + dev_attr->max_inline_data); + rc = false; + } + return rc; +} + +static inline struct +bnxt_re_qp *__get_qp_from_qp_in(struct ib_pd *qp_in, + struct bnxt_re_dev *rdev) +{ + struct bnxt_re_qp *qp; + + qp = kzalloc(sizeof(*qp), GFP_KERNEL); + if (!qp) + dev_err(rdev_to_dev(rdev), "Allocate QP failed!\n"); + return qp; +} + +struct ib_qp *bnxt_re_create_qp(struct ib_pd *qp_in, + struct ib_qp_init_attr *qp_init_attr, + struct ib_udata *udata) +{ + struct bnxt_re_pd *pd; + struct ib_pd *ib_pd = qp_in; + struct bnxt_qplib_dev_attr *dev_attr; + struct bnxt_re_dev *rdev; + u32 active_qps, tmp_qps; + struct bnxt_re_qp *qp; + int rc; + + pd = to_bnxt_re(ib_pd, struct bnxt_re_pd, ibpd); + rdev = pd->rdev; + dev_attr = rdev->dev_attr; + if (rdev->mod_exit) { + rc = -EIO; + dev_dbg(rdev_to_dev(rdev), "%s(): in mod_exit, just return!\n", __func__); + goto exit; + } + + if (atomic_read(&rdev->stats.rsors.qp_count) >= dev_attr->max_qp) { + dev_err(rdev_to_dev(rdev), "Create QP failed - max exceeded(QPs Alloc'd %u of max %u)\n", + atomic_read(&rdev->stats.rsors.qp_count), dev_attr->max_qp); + rc = -EINVAL; + goto exit; + } + + rc = bnxt_re_test_qp_limits(rdev, qp_init_attr, dev_attr); + if (!rc) { + rc = -EINVAL; + goto exit; + } + qp = __get_qp_from_qp_in(qp_in, rdev); + if (!qp) { + rc = -ENOMEM; + goto exit; + } + qp->rdev = rdev; + + rc = bnxt_re_init_qp_attr(qp, pd, qp_init_attr, udata); + if (rc) + goto fail; + + if (qp_init_attr->qp_type == IB_QPT_GSI && + !_is_chip_gen_p5_p7(rdev->chip_ctx)) { + rc = bnxt_re_create_gsi_qp(qp, pd); + if (rc == -ENODEV) + goto qp_destroy; + if (rc) + goto fail; + } else { + rc = bnxt_qplib_create_qp(&rdev->qplib_res, &qp->qplib_qp); + if (rc) { + dev_err(rdev_to_dev(rdev), "create HW QP failed!\n"); + goto free_umem; + } + + if (udata) { + struct bnxt_re_qp_resp resp; + + resp.qpid = qp->qplib_qp.id; + rc = bnxt_re_copy_to_udata(rdev, &resp, + min(udata->outlen, sizeof(resp)), + udata); + if (rc) + goto qp_destroy; + } + } + + qp->ib_qp.qp_num = qp->qplib_qp.id; + if (qp_init_attr->qp_type == IB_QPT_GSI) + rdev->gsi_ctx.gsi_qp = qp; + spin_lock_init(&qp->sq_lock); + spin_lock_init(&qp->rq_lock); + INIT_LIST_HEAD(&qp->list); + mutex_lock(&rdev->qp_lock); + list_add_tail(&qp->list, &rdev->qp_list); + mutex_unlock(&rdev->qp_lock); + atomic_inc(&rdev->stats.rsors.qp_count); + active_qps = atomic_read(&rdev->stats.rsors.qp_count); + if (active_qps > atomic_read(&rdev->stats.rsors.max_qp_count)) + atomic_set(&rdev->stats.rsors.max_qp_count, active_qps); + + bnxt_re_dump_debug_stats(rdev, active_qps); + + /* Get the counters for RC QPs and UD QPs */ + if (qp_init_attr->qp_type == IB_QPT_RC) { + tmp_qps = atomic_inc_return(&rdev->stats.rsors.rc_qp_count); + if (tmp_qps > atomic_read(&rdev->stats.rsors.max_rc_qp_count)) + atomic_set(&rdev->stats.rsors.max_rc_qp_count, tmp_qps); + } else if (qp_init_attr->qp_type == IB_QPT_UD) { + tmp_qps = atomic_inc_return(&rdev->stats.rsors.ud_qp_count); + if (tmp_qps > atomic_read(&rdev->stats.rsors.max_ud_qp_count)) + atomic_set(&rdev->stats.rsors.max_ud_qp_count, tmp_qps); + } + + return &qp->ib_qp; + +qp_destroy: + bnxt_qplib_destroy_qp(&rdev->qplib_res, &qp->qplib_qp); +free_umem: + if (udata) { + if (qp->rumem && !IS_ERR(qp->rumem)) + ib_umem_release(qp->rumem); + if (qp->sumem && !IS_ERR(qp->sumem)) + ib_umem_release(qp->sumem); + } +fail: + kfree(qp); +exit: + return ERR_PTR(rc); +} + +static int bnxt_re_modify_shadow_qp(struct bnxt_re_dev *rdev, + struct bnxt_re_qp *qp1_qp, + int qp_attr_mask) +{ + struct bnxt_re_qp *qp = rdev->gsi_ctx.gsi_sqp; + int rc = 0; + + if (qp_attr_mask & IB_QP_STATE) { + qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_STATE; + qp->qplib_qp.state = qp1_qp->qplib_qp.state; + } + if (qp_attr_mask & IB_QP_PKEY_INDEX) { + qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_PKEY; + qp->qplib_qp.pkey_index = qp1_qp->qplib_qp.pkey_index; + } + + if (qp_attr_mask & IB_QP_QKEY) { + qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_QKEY; + /* Using a Random QKEY */ + qp->qplib_qp.qkey = BNXT_RE_QP_RANDOM_QKEY; + } + if (qp_attr_mask & IB_QP_SQ_PSN) { + qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_SQ_PSN; + qp->qplib_qp.sq.psn = qp1_qp->qplib_qp.sq.psn; + } + + rc = bnxt_qplib_modify_qp(&rdev->qplib_res, &qp->qplib_qp); + if (rc) + dev_err(rdev_to_dev(rdev), "Modify Shadow QP for QP1 failed\n"); + return rc; +} + +static u32 ipv4_from_gid(u8 *gid) +{ + return (gid[15] << 24 | gid[14] << 16 | gid[13] << 8 | gid[12]); +} + +static u16 get_source_port(struct bnxt_re_dev *rdev, + struct bnxt_re_qp *qp) +{ + u8 ip_off, data[48], smac[ETH_ALEN]; + u16 crc = 0, buf_len = 0, i; + u8 addr_len; + u32 qpn; + + if (qp->qplib_qp.nw_type == CMDQ_MODIFY_QP_NETWORK_TYPE_ROCEV2_IPV6) { + addr_len = 6; + ip_off = 10; + } else { + addr_len = 4; + ip_off = 12; + } + + memcpy(smac, qp->qplib_qp.smac, ETH_ALEN); + + memset(data, 0, 48); + memcpy(data, qp->qplib_qp.ah.dmac, ETH_ALEN); + buf_len += ETH_ALEN; + + memcpy(data + buf_len, smac, ETH_ALEN); + buf_len += ETH_ALEN; + + memcpy(data + buf_len, qp->qplib_qp.ah.dgid.data + ip_off, addr_len); + buf_len += addr_len; + + memcpy(data + buf_len, qp->qp_info_entry.sgid.raw + ip_off, addr_len); + buf_len += addr_len; + + qpn = htonl(qp->qplib_qp.dest_qpn); + memcpy(data + buf_len, (u8 *)&qpn + 1, 3); + buf_len += 3; + + for (i = 0; i < buf_len; i++) + crc = crc16(crc, (data + i), 1); + + return crc; +} + +static void bnxt_re_update_qp_info(struct bnxt_re_dev *rdev, struct bnxt_re_qp *qp) +{ + u16 type; + + type = __from_hw_to_ib_qp_type(qp->qplib_qp.type); + + /* User-space can extract ip address with sgid_index. */ + if (ipv6_addr_v4mapped((struct in6_addr *)&qp->qplib_qp.ah.dgid)) { + qp->qp_info_entry.s_ip.ipv4_addr = ipv4_from_gid(qp->qp_info_entry.sgid.raw); + qp->qp_info_entry.d_ip.ipv4_addr = ipv4_from_gid(qp->qplib_qp.ah.dgid.data); + } else { + memcpy(&qp->qp_info_entry.s_ip.ipv6_addr, qp->qp_info_entry.sgid.raw, + sizeof(qp->qp_info_entry.s_ip.ipv6_addr)); + memcpy(&qp->qp_info_entry.d_ip.ipv6_addr, qp->qplib_qp.ah.dgid.data, + sizeof(qp->qp_info_entry.d_ip.ipv6_addr)); + } + + if (type == IB_QPT_RC && + (qp->qplib_qp.nw_type == CMDQ_MODIFY_QP_NETWORK_TYPE_ROCEV2_IPV4 || + qp->qplib_qp.nw_type == CMDQ_MODIFY_QP_NETWORK_TYPE_ROCEV2_IPV6)) { + qp->qp_info_entry.s_port = get_source_port(rdev, qp); + } + qp->qp_info_entry.d_port = BNXT_RE_QP_DEST_PORT; +} + +static void bnxt_qplib_manage_flush_qp(struct bnxt_re_qp *qp) +{ + struct bnxt_qplib_q *rq, *sq; + unsigned long flags; + + if (qp->sumem) + return; + + if (qp->qplib_qp.state == CMDQ_MODIFY_QP_NEW_STATE_ERR) { + rq = &qp->qplib_qp.rq; + sq = &qp->qplib_qp.sq; + + dev_dbg(rdev_to_dev(qp->rdev), + "Move QP = %p to flush list\n", qp); + flags = bnxt_re_lock_cqs(qp); + bnxt_qplib_add_flush_qp(&qp->qplib_qp); + bnxt_re_unlock_cqs(qp, flags); + + if (sq->hwq.prod != sq->hwq.cons) + bnxt_re_handle_cqn(&qp->scq->qplib_cq); + + if (qp->rcq && (qp->rcq != qp->scq) && + (rq->hwq.prod != rq->hwq.cons)) + bnxt_re_handle_cqn(&qp->rcq->qplib_cq); + } + + if (qp->qplib_qp.state == CMDQ_MODIFY_QP_NEW_STATE_RESET) { + dev_dbg(rdev_to_dev(qp->rdev), + "Move QP = %p out of flush list\n", qp); + flags = bnxt_re_lock_cqs(qp); + bnxt_qplib_clean_qp(&qp->qplib_qp); + bnxt_re_unlock_cqs(qp, flags); + } +} + +bool ib_modify_qp_is_ok_compat(enum ib_qp_state cur_state, + enum ib_qp_state next_state, + enum ib_qp_type type, + enum ib_qp_attr_mask mask) +{ + return (ib_modify_qp_is_ok(cur_state, next_state, + type, mask)); +} + +int bnxt_re_modify_qp(struct ib_qp *ib_qp, struct ib_qp_attr *qp_attr, + int qp_attr_mask, struct ib_udata *udata) +{ + enum ib_qp_state curr_qp_state, new_qp_state; + struct bnxt_re_modify_qp_ex_resp resp = {}; + struct bnxt_re_modify_qp_ex_req ureq = {}; + struct bnxt_qplib_dev_attr *dev_attr; + struct bnxt_qplib_ppp *ppp = NULL; + struct bnxt_re_dev *rdev; + struct bnxt_re_qp *qp; + struct ib_gid_attr *sgid_attr; + struct ib_gid_attr gid_attr; + union ib_gid sgid, *gid_ptr = NULL; + u8 nw_type; + int rc, entries, status; + bool is_copy_to_udata = false; + bool is_qpmtu_high = false; + + qp = to_bnxt_re(ib_qp, struct bnxt_re_qp, ib_qp); + rdev = qp->rdev; + dev_attr = rdev->dev_attr; + + qp->qplib_qp.modify_flags = 0; + ppp = &qp->qplib_qp.ppp; + if (qp_attr_mask & IB_QP_STATE) { + curr_qp_state = __to_ib_qp_state(qp->qplib_qp.cur_qp_state); + new_qp_state = qp_attr->qp_state; + if (!ib_modify_qp_is_ok_compat(curr_qp_state, new_qp_state, + ib_qp->qp_type, qp_attr_mask)) { + dev_err(rdev_to_dev(rdev),"invalid attribute mask=0x%x" + " specified for qpn=0x%x of type=0x%x" + " current_qp_state=0x%x, new_qp_state=0x%x\n", + qp_attr_mask, ib_qp->qp_num, ib_qp->qp_type, + curr_qp_state, new_qp_state); + return -EINVAL; + } + dev_dbg(rdev_to_dev(rdev), "%s:%d INFO attribute mask=0x%x qpn=0x%x " + "of type=0x%x current_qp_state=0x%x, new_qp_state=0x%x\n", + __func__, __LINE__, qp_attr_mask, ib_qp->qp_num, + ib_qp->qp_type, curr_qp_state, new_qp_state); + qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_STATE; + qp->qplib_qp.state = __from_ib_qp_state(qp_attr->qp_state); + + if (udata && curr_qp_state == IB_QPS_RESET && + new_qp_state == IB_QPS_INIT) { + if (!ib_copy_from_udata(&ureq, udata, sizeof(ureq))) { + if (ureq.comp_mask & + BNXT_RE_COMP_MASK_MQP_EX_PPP_REQ_EN_MASK) { + ppp->req = BNXT_QPLIB_PPP_REQ; + ppp->dpi = ureq.dpi; + } + } + } + } + if (qp_attr_mask & IB_QP_EN_SQD_ASYNC_NOTIFY) { + qp->qplib_qp.modify_flags |= + CMDQ_MODIFY_QP_MODIFY_MASK_EN_SQD_ASYNC_NOTIFY; + qp->qplib_qp.en_sqd_async_notify = true; + } + if (qp_attr_mask & IB_QP_ACCESS_FLAGS) { + qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_ACCESS; + qp->qplib_qp.access = + __from_ib_access_flags(qp_attr->qp_access_flags); + /* LOCAL_WRITE access must be set to allow RC receive */ + qp->qplib_qp.access |= BNXT_QPLIB_ACCESS_LOCAL_WRITE; + qp->qplib_qp.access |= CMDQ_MODIFY_QP_ACCESS_REMOTE_WRITE; + qp->qplib_qp.access |= CMDQ_MODIFY_QP_ACCESS_REMOTE_READ; + } + if (qp_attr_mask & IB_QP_PKEY_INDEX) { + qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_PKEY; + qp->qplib_qp.pkey_index = qp_attr->pkey_index; + } + if (qp_attr_mask & IB_QP_QKEY) { + qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_QKEY; + qp->qplib_qp.qkey = qp_attr->qkey; + } + if (qp_attr_mask & IB_QP_AV) { + qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_DGID | + CMDQ_MODIFY_QP_MODIFY_MASK_FLOW_LABEL | + CMDQ_MODIFY_QP_MODIFY_MASK_SGID_INDEX | + CMDQ_MODIFY_QP_MODIFY_MASK_HOP_LIMIT | + CMDQ_MODIFY_QP_MODIFY_MASK_TRAFFIC_CLASS | + CMDQ_MODIFY_QP_MODIFY_MASK_DEST_MAC | + CMDQ_MODIFY_QP_MODIFY_MASK_VLAN_ID; + memcpy(qp->qplib_qp.ah.dgid.data, qp_attr->ah_attr.grh.dgid.raw, + sizeof(qp->qplib_qp.ah.dgid.data)); + qp->qplib_qp.ah.flow_label = qp_attr->ah_attr.grh.flow_label; + qp->qplib_qp.ah.sgid_index = _get_sgid_index(rdev, + qp_attr->ah_attr.grh.sgid_index); + qp->qplib_qp.ah.host_sgid_index = qp_attr->ah_attr.grh.sgid_index; + qp->qplib_qp.ah.hop_limit = qp_attr->ah_attr.grh.hop_limit; + qp->qplib_qp.ah.traffic_class = + qp_attr->ah_attr.grh.traffic_class; + qp->qplib_qp.ah.sl = qp_attr->ah_attr.sl; + ether_addr_copy(qp->qplib_qp.ah.dmac, ROCE_DMAC(&qp_attr->ah_attr)); + sgid_attr = &gid_attr; + status = bnxt_re_get_cached_gid(&rdev->ibdev, 1, + qp_attr->ah_attr.grh.sgid_index, + &sgid, &sgid_attr, + &qp_attr->ah_attr.grh, NULL); + if (!status) + if_rele(sgid_attr->ndev); + gid_ptr = &sgid; + if (sgid_attr->ndev) { + memcpy(qp->qplib_qp.smac, rdev->dev_addr, + ETH_ALEN); + nw_type = bnxt_re_gid_to_network_type(sgid_attr, &sgid); + dev_dbg(rdev_to_dev(rdev), + "Connection using the nw_type %d\n", nw_type); + switch (nw_type) { + case RDMA_NETWORK_IPV4: + qp->qplib_qp.nw_type = + CMDQ_MODIFY_QP_NETWORK_TYPE_ROCEV2_IPV4; + break; + case RDMA_NETWORK_IPV6: + qp->qplib_qp.nw_type = + CMDQ_MODIFY_QP_NETWORK_TYPE_ROCEV2_IPV6; + break; + default: + qp->qplib_qp.nw_type = + CMDQ_MODIFY_QP_NETWORK_TYPE_ROCEV1; + break; + } + } + memcpy(&qp->qp_info_entry.sgid, gid_ptr, sizeof(qp->qp_info_entry.sgid)); + } + + /* MTU settings allowed only during INIT -> RTR */ + if (qp_attr->qp_state == IB_QPS_RTR) { + bnxt_re_init_qpmtu(qp, rdev->netdev->if_mtu, qp_attr_mask, qp_attr, + &is_qpmtu_high); + if (udata && !ib_copy_from_udata(&ureq, udata, sizeof(ureq))) { + if (ureq.comp_mask & BNXT_RE_COMP_MASK_MQP_EX_PATH_MTU_MASK) { + resp.comp_mask |= BNXT_RE_COMP_MASK_MQP_EX_PATH_MTU_MASK; + resp.path_mtu = qp->qplib_qp.mtu; + is_copy_to_udata = true; + } else if (is_qpmtu_high) { + dev_err(rdev_to_dev(rdev), "qp %#x invalid mtu\n", + qp->qplib_qp.id); + return -EINVAL; + } + } + } + + if (qp_attr_mask & IB_QP_TIMEOUT) { + qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_TIMEOUT; + qp->qplib_qp.timeout = qp_attr->timeout; + } + if (qp_attr_mask & IB_QP_RETRY_CNT) { + qp->qplib_qp.modify_flags |= + CMDQ_MODIFY_QP_MODIFY_MASK_RETRY_CNT; + qp->qplib_qp.retry_cnt = qp_attr->retry_cnt; + } + if (qp_attr_mask & IB_QP_RNR_RETRY) { + qp->qplib_qp.modify_flags |= + CMDQ_MODIFY_QP_MODIFY_MASK_RNR_RETRY; + qp->qplib_qp.rnr_retry = qp_attr->rnr_retry; + } + if (qp_attr_mask & IB_QP_MIN_RNR_TIMER) { + qp->qplib_qp.modify_flags |= + CMDQ_MODIFY_QP_MODIFY_MASK_MIN_RNR_TIMER; + qp->qplib_qp.min_rnr_timer = qp_attr->min_rnr_timer; + } + if (qp_attr_mask & IB_QP_RQ_PSN) { + qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_RQ_PSN; + qp->qplib_qp.rq.psn = qp_attr->rq_psn; + } + if (qp_attr_mask & IB_QP_MAX_QP_RD_ATOMIC) { + qp->qplib_qp.modify_flags |= + CMDQ_MODIFY_QP_MODIFY_MASK_MAX_RD_ATOMIC; + /* Cap the max_rd_atomic to device max */ + if (qp_attr->max_rd_atomic > dev_attr->max_qp_rd_atom) + dev_dbg(rdev_to_dev(rdev), + "max_rd_atomic requested %d is > device max %d\n", + qp_attr->max_rd_atomic, + dev_attr->max_qp_rd_atom); + qp->qplib_qp.max_rd_atomic = min_t(u32, qp_attr->max_rd_atomic, + dev_attr->max_qp_rd_atom); + } + if (qp_attr_mask & IB_QP_SQ_PSN) { + qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_SQ_PSN; + qp->qplib_qp.sq.psn = qp_attr->sq_psn; + } + if (qp_attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) { + if (qp_attr->max_dest_rd_atomic > + dev_attr->max_qp_init_rd_atom) { + dev_err(rdev_to_dev(rdev), + "max_dest_rd_atomic requested %d is > device max %d\n", + qp_attr->max_dest_rd_atomic, + dev_attr->max_qp_init_rd_atom); + return -EINVAL; + } + qp->qplib_qp.modify_flags |= + CMDQ_MODIFY_QP_MODIFY_MASK_MAX_DEST_RD_ATOMIC; + qp->qplib_qp.max_dest_rd_atomic = qp_attr->max_dest_rd_atomic; + } + if (qp_attr_mask & IB_QP_CAP) { + qp->qplib_qp.modify_flags |= + CMDQ_MODIFY_QP_MODIFY_MASK_SQ_SIZE | + CMDQ_MODIFY_QP_MODIFY_MASK_RQ_SIZE | + CMDQ_MODIFY_QP_MODIFY_MASK_SQ_SGE | + CMDQ_MODIFY_QP_MODIFY_MASK_RQ_SGE | + CMDQ_MODIFY_QP_MODIFY_MASK_MAX_INLINE_DATA; + if ((qp_attr->cap.max_send_wr >= dev_attr->max_qp_wqes) || + (qp_attr->cap.max_recv_wr >= dev_attr->max_qp_wqes) || + (qp_attr->cap.max_send_sge >= dev_attr->max_qp_sges) || + (qp_attr->cap.max_recv_sge >= dev_attr->max_qp_sges) || + (qp_attr->cap.max_inline_data >= + dev_attr->max_inline_data)) { + dev_err(rdev_to_dev(rdev), + "Create QP failed - max exceeded\n"); + return -EINVAL; + } + entries = roundup_pow_of_two(qp_attr->cap.max_send_wr); + if (entries > dev_attr->max_qp_wqes) + entries = dev_attr->max_qp_wqes; + entries = min_t(u32, entries, dev_attr->max_qp_wqes); + qp->qplib_qp.sq.max_wqe = entries; + qp->qplib_qp.sq.q_full_delta = qp->qplib_qp.sq.max_wqe - + qp_attr->cap.max_send_wr; + /* + * Reserving one slot for Phantom WQE. Some application can + * post one extra entry in this case. Allowing this to avoid + * unexpected Queue full condition + */ + qp->qplib_qp.sq.q_full_delta -= 1; + qp->qplib_qp.sq.max_sge = qp_attr->cap.max_send_sge; + if (qp->qplib_qp.rq.max_wqe) { + entries = roundup_pow_of_two(qp_attr->cap.max_recv_wr); + if (entries > dev_attr->max_qp_wqes) + entries = dev_attr->max_qp_wqes; + qp->qplib_qp.rq.max_wqe = entries; + qp->qplib_qp.rq.q_full_delta = qp->qplib_qp.rq.max_wqe - + qp_attr->cap.max_recv_wr; + qp->qplib_qp.rq.max_sge = qp_attr->cap.max_recv_sge; + } else { + /* SRQ was used prior, just ignore the RQ caps */ + } + } + if (qp_attr_mask & IB_QP_DEST_QPN) { + qp->qplib_qp.modify_flags |= + CMDQ_MODIFY_QP_MODIFY_MASK_DEST_QP_ID; + qp->qplib_qp.dest_qpn = qp_attr->dest_qp_num; + } + + rc = bnxt_qplib_modify_qp(&rdev->qplib_res, &qp->qplib_qp); + if (rc) { + dev_err(rdev_to_dev(rdev), "Modify HW QP failed!\n"); + return rc; + } + if (qp_attr_mask & IB_QP_STATE) + bnxt_qplib_manage_flush_qp(qp); + if (ureq.comp_mask & BNXT_RE_COMP_MASK_MQP_EX_PPP_REQ_EN_MASK && + ppp->st_idx_en & CREQ_MODIFY_QP_RESP_PINGPONG_PUSH_ENABLED) { + resp.comp_mask |= BNXT_RE_COMP_MASK_MQP_EX_PPP_REQ_EN; + resp.ppp_st_idx = ppp->st_idx_en >> + BNXT_QPLIB_PPP_ST_IDX_SHIFT; + is_copy_to_udata = true; + } + + if (is_copy_to_udata) { + rc = bnxt_re_copy_to_udata(rdev, &resp, + min(udata->outlen, sizeof(resp)), + udata); + if (rc) + return rc; + } + + if (ib_qp->qp_type == IB_QPT_GSI && + rdev->gsi_ctx.gsi_qp_mode == BNXT_RE_GSI_MODE_ALL && + rdev->gsi_ctx.gsi_sqp) + rc = bnxt_re_modify_shadow_qp(rdev, qp, qp_attr_mask); + /* + * Update info when qp_info_info + */ + bnxt_re_update_qp_info(rdev, qp); + return rc; +} + +int bnxt_re_query_qp(struct ib_qp *ib_qp, struct ib_qp_attr *qp_attr, + int qp_attr_mask, struct ib_qp_init_attr *qp_init_attr) +{ + struct bnxt_re_qp *qp = to_bnxt_re(ib_qp, struct bnxt_re_qp, ib_qp); + struct bnxt_re_dev *rdev = qp->rdev; + struct bnxt_qplib_qp *qplib_qp; + int rc; + + qplib_qp = kcalloc(1, sizeof(*qplib_qp), GFP_KERNEL); + if (!qplib_qp) + return -ENOMEM; + + qplib_qp->id = qp->qplib_qp.id; + qplib_qp->ah.host_sgid_index = qp->qplib_qp.ah.host_sgid_index; + + rc = bnxt_qplib_query_qp(&rdev->qplib_res, qplib_qp); + if (rc) { + dev_err(rdev_to_dev(rdev), "Query HW QP (0x%x) failed! rc = %d\n", + qplib_qp->id, rc); + goto free_mem; + } + qp_attr->qp_state = __to_ib_qp_state(qplib_qp->state); + qp_attr->cur_qp_state = __to_ib_qp_state(qplib_qp->cur_qp_state); + qp_attr->en_sqd_async_notify = qplib_qp->en_sqd_async_notify ? 1 : 0; + qp_attr->qp_access_flags = __to_ib_access_flags(qplib_qp->access); + qp_attr->pkey_index = qplib_qp->pkey_index; + qp_attr->qkey = qplib_qp->qkey; + memcpy(qp_attr->ah_attr.grh.dgid.raw, qplib_qp->ah.dgid.data, + sizeof(qplib_qp->ah.dgid.data)); + qp_attr->ah_attr.grh.flow_label = qplib_qp->ah.flow_label; + qp_attr->ah_attr.grh.sgid_index = qplib_qp->ah.host_sgid_index; + qp_attr->ah_attr.grh.hop_limit = qplib_qp->ah.hop_limit; + qp_attr->ah_attr.grh.traffic_class = qplib_qp->ah.traffic_class; + qp_attr->ah_attr.sl = qplib_qp->ah.sl; + ether_addr_copy(ROCE_DMAC(&qp_attr->ah_attr), qplib_qp->ah.dmac); + qp_attr->path_mtu = __to_ib_mtu(qplib_qp->path_mtu); + qp_attr->timeout = qplib_qp->timeout; + qp_attr->retry_cnt = qplib_qp->retry_cnt; + qp_attr->rnr_retry = qplib_qp->rnr_retry; + qp_attr->min_rnr_timer = qplib_qp->min_rnr_timer; + qp_attr->rq_psn = qplib_qp->rq.psn; + qp_attr->max_rd_atomic = qplib_qp->max_rd_atomic; + qp_attr->sq_psn = qplib_qp->sq.psn; + qp_attr->max_dest_rd_atomic = qplib_qp->max_dest_rd_atomic; + qp_init_attr->sq_sig_type = qplib_qp->sig_type ? IB_SIGNAL_ALL_WR : + IB_SIGNAL_REQ_WR; + qp_attr->dest_qp_num = qplib_qp->dest_qpn; + + qp_attr->cap.max_send_wr = qp->qplib_qp.sq.max_wqe; + qp_attr->cap.max_send_sge = qp->qplib_qp.sq.max_sge; + qp_attr->cap.max_recv_wr = qp->qplib_qp.rq.max_wqe; + qp_attr->cap.max_recv_sge = qp->qplib_qp.rq.max_sge; + qp_attr->cap.max_inline_data = qp->qplib_qp.max_inline_data; + qp_init_attr->cap = qp_attr->cap; + +free_mem: + kfree(qplib_qp); + return rc; +} + +/* Builders */ + +/* For Raw, the application is responsible to build the entire packet */ +static void bnxt_re_build_raw_send(const struct ib_send_wr *wr, + struct bnxt_qplib_swqe *wqe) +{ + switch (wr->send_flags) { + case IB_SEND_IP_CSUM: + wqe->rawqp1.lflags |= SQ_SEND_RAWETH_QP1_LFLAGS_IP_CHKSUM; + break; + default: + /* Pad HW RoCE iCRC */ + wqe->rawqp1.lflags |= SQ_SEND_RAWETH_QP1_LFLAGS_ROCE_CRC; + break; + } +} + +/* For QP1, the driver must build the entire RoCE (v1/v2) packet hdr + * as according to the sgid and AV + */ +static int bnxt_re_build_qp1_send(struct bnxt_re_qp *qp, const struct ib_send_wr *wr, + struct bnxt_qplib_swqe *wqe, int payload_size) +{ + struct bnxt_re_ah *ah = to_bnxt_re(ud_wr(wr)->ah, struct bnxt_re_ah, + ibah); + struct bnxt_qplib_ah *qplib_ah = &ah->qplib_ah; + struct bnxt_qplib_sge sge; + int i, rc = 0; + union ib_gid sgid; + u16 vlan_id; + u8 *ptmac; + void *buf; + + memset(&qp->qp1_hdr, 0, sizeof(qp->qp1_hdr)); + + /* Get sgid */ + rc = bnxt_re_query_gid(&qp->rdev->ibdev, 1, qplib_ah->sgid_index, &sgid); + if (rc) + return rc; + + /* ETH */ + qp->qp1_hdr.eth_present = 1; + ptmac = ah->qplib_ah.dmac; + memcpy(qp->qp1_hdr.eth.dmac_h, ptmac, 4); + ptmac += 4; + memcpy(qp->qp1_hdr.eth.dmac_l, ptmac, 2); + + ptmac = qp->qplib_qp.smac; + memcpy(qp->qp1_hdr.eth.smac_h, ptmac, 2); + ptmac += 2; + memcpy(qp->qp1_hdr.eth.smac_l, ptmac, 4); + + qp->qp1_hdr.eth.type = cpu_to_be16(BNXT_QPLIB_ETHTYPE_ROCEV1); + + /* For vlan, check the sgid for vlan existence */ + vlan_id = rdma_get_vlan_id(&sgid); + if (vlan_id && vlan_id < 0x1000) { + qp->qp1_hdr.vlan_present = 1; + qp->qp1_hdr.eth.type = cpu_to_be16(ETH_P_8021Q); + } + /* GRH */ + qp->qp1_hdr.grh_present = 1; + qp->qp1_hdr.grh.ip_version = 6; + qp->qp1_hdr.grh.payload_length = + cpu_to_be16((IB_BTH_BYTES + IB_DETH_BYTES + payload_size + 7) + & ~3); + qp->qp1_hdr.grh.next_header = 0x1b; + memcpy(qp->qp1_hdr.grh.source_gid.raw, sgid.raw, sizeof(sgid)); + memcpy(qp->qp1_hdr.grh.destination_gid.raw, qplib_ah->dgid.data, + sizeof(sgid)); + + /* BTH */ + if (wr->opcode == IB_WR_SEND_WITH_IMM) { + qp->qp1_hdr.bth.opcode = IB_OPCODE_UD_SEND_ONLY_WITH_IMMEDIATE; + qp->qp1_hdr.immediate_present = 1; + } else { + qp->qp1_hdr.bth.opcode = IB_OPCODE_UD_SEND_ONLY; + } + if (wr->send_flags & IB_SEND_SOLICITED) + qp->qp1_hdr.bth.solicited_event = 1; + qp->qp1_hdr.bth.pad_count = (4 - payload_size) & 3; + /* P_key for QP1 is for all members */ + qp->qp1_hdr.bth.pkey = cpu_to_be16(0xFFFF); + qp->qp1_hdr.bth.destination_qpn = IB_QP1; + qp->qp1_hdr.bth.ack_req = 0; + qp->send_psn++; + qp->send_psn &= BTH_PSN_MASK; + qp->qp1_hdr.bth.psn = cpu_to_be32(qp->send_psn); + /* DETH */ + /* Use the priviledged Q_Key for QP1 */ + qp->qp1_hdr.deth.qkey = cpu_to_be32(IB_QP1_QKEY); + qp->qp1_hdr.deth.source_qpn = IB_QP1; + + /* Pack the QP1 to the transmit buffer */ + buf = bnxt_qplib_get_qp1_sq_buf(&qp->qplib_qp, &sge); + if (!buf) { + dev_err(rdev_to_dev(qp->rdev), "QP1 buffer is empty!\n"); + rc = -ENOMEM; + } + for (i = wqe->num_sge; i; i--) { + wqe->sg_list[i].addr = wqe->sg_list[i - 1].addr; + wqe->sg_list[i].lkey = wqe->sg_list[i - 1].lkey; + wqe->sg_list[i].size = wqe->sg_list[i - 1].size; + } + wqe->sg_list[0].addr = sge.addr; + wqe->sg_list[0].lkey = sge.lkey; + wqe->sg_list[0].size = sge.size; + wqe->num_sge++; + + return rc; +} + +static int bnxt_re_build_gsi_send(struct bnxt_re_qp *qp, + const struct ib_send_wr *wr, + struct bnxt_qplib_swqe *wqe) +{ + struct bnxt_re_dev *rdev; + int rc, indx, len = 0; + + rdev = qp->rdev; + + /* Mode UD is applicable to Gen P5 only */ + if (rdev->gsi_ctx.gsi_qp_mode == BNXT_RE_GSI_MODE_UD) + return 0; + + for (indx = 0; indx < wr->num_sge; indx++) { + wqe->sg_list[indx].addr = wr->sg_list[indx].addr; + wqe->sg_list[indx].lkey = wr->sg_list[indx].lkey; + wqe->sg_list[indx].size = wr->sg_list[indx].length; + len += wr->sg_list[indx].length; + } + rc = bnxt_re_build_qp1_send(qp, wr, wqe, len); + wqe->rawqp1.lflags |= SQ_SEND_RAWETH_QP1_LFLAGS_ROCE_CRC; + + return rc; +} + +/* For the MAD layer, it only provides the recv SGE the size of + ib_grh + MAD datagram. No Ethernet headers, Ethertype, BTH, DETH, + nor RoCE iCRC. The Cu+ solution must provide buffer for the entire + receive packet (334 bytes) with no VLAN and then copy the GRH + and the MAD datagram out to the provided SGE. +*/ + +static int bnxt_re_build_qp1_recv(struct bnxt_re_qp *qp, + const struct ib_recv_wr *wr, + struct bnxt_qplib_swqe *wqe) +{ + struct bnxt_re_dev *rdev = qp->rdev; + struct bnxt_qplib_sge ref, sge; + u8 udp_hdr_size = 0; + u8 ip_hdr_size = 0; + int rc = 0; + int size; + + if (bnxt_qplib_get_qp1_rq_buf(&qp->qplib_qp, &sge)) { + /* Create 5 SGEs as according to the following: + * Ethernet header (14) + * ib_grh (40) - as provided from the wr + * ib_bth + ib_deth + UDP(RoCE v2 only) (28) + * MAD (256) - as provided from the wr + * iCRC (4) + */ + + /* Set RoCE v2 header size and offsets */ + if (rdev->gsi_ctx.gsi_qp_mode == BNXT_RE_GSI_MODE_ROCE_V2_IPV4) + ip_hdr_size = 20; + if (rdev->gsi_ctx.gsi_qp_mode != BNXT_RE_GSI_MODE_ROCE_V1) + udp_hdr_size = 8; + + /* Save the reference from ULP */ + ref.addr = wr->sg_list[0].addr; + ref.lkey = wr->sg_list[0].lkey; + ref.size = wr->sg_list[0].length; + + /* SGE 1 */ + size = sge.size; + wqe->sg_list[0].addr = sge.addr; + wqe->sg_list[0].lkey = sge.lkey; + wqe->sg_list[0].size = BNXT_QPLIB_MAX_QP1_RQ_ETH_HDR_SIZE; + size -= wqe->sg_list[0].size; + if (size <= 0) { + dev_err(rdev_to_dev(qp->rdev),"QP1 rq buffer is empty!\n"); + rc = -ENOMEM; + goto done; + } + sge.size = (u32)size; + sge.addr += wqe->sg_list[0].size; + + /* SGE 2 */ + /* In case of RoCE v2 ipv4 lower 20 bytes should have IP hdr */ + wqe->sg_list[1].addr = ref.addr + ip_hdr_size; + wqe->sg_list[1].lkey = ref.lkey; + wqe->sg_list[1].size = sizeof(struct ib_grh) - ip_hdr_size; + ref.size -= wqe->sg_list[1].size; + if (ref.size <= 0) { + dev_err(rdev_to_dev(qp->rdev), + "QP1 ref buffer is empty!\n"); + rc = -ENOMEM; + goto done; + } + ref.addr += wqe->sg_list[1].size + ip_hdr_size; + + /* SGE 3 */ + wqe->sg_list[2].addr = sge.addr; + wqe->sg_list[2].lkey = sge.lkey; + wqe->sg_list[2].size = BNXT_QPLIB_MAX_QP1_RQ_BDETH_HDR_SIZE + + udp_hdr_size; + size -= wqe->sg_list[2].size; + if (size <= 0) { + dev_err(rdev_to_dev(qp->rdev), + "QP1 rq buffer is empty!\n"); + rc = -ENOMEM; + goto done; + } + sge.size = (u32)size; + sge.addr += wqe->sg_list[2].size; + + /* SGE 4 */ + wqe->sg_list[3].addr = ref.addr; + wqe->sg_list[3].lkey = ref.lkey; + wqe->sg_list[3].size = ref.size; + ref.size -= wqe->sg_list[3].size; + if (ref.size) { + dev_err(rdev_to_dev(qp->rdev), + "QP1 ref buffer is incorrect!\n"); + rc = -ENOMEM; + goto done; + } + /* SGE 5 */ + wqe->sg_list[4].addr = sge.addr; + wqe->sg_list[4].lkey = sge.lkey; + wqe->sg_list[4].size = sge.size; + size -= wqe->sg_list[4].size; + if (size) { + dev_err(rdev_to_dev(qp->rdev), + "QP1 rq buffer is incorrect!\n"); + rc = -ENOMEM; + goto done; + } + sge.size = (u32)size; + wqe->num_sge = 5; + } else { + dev_err(rdev_to_dev(qp->rdev), "QP1 buffer is empty!\n"); + rc = -ENOMEM; + } +done: + return rc; +} + +static int bnxt_re_build_qp1_shadow_qp_recv(struct bnxt_re_qp *qp, + const struct ib_recv_wr *wr, + struct bnxt_qplib_swqe *wqe) +{ + struct bnxt_re_sqp_entries *sqp_entry; + struct bnxt_qplib_sge sge; + struct bnxt_re_dev *rdev; + u32 rq_prod_index; + int rc = 0; + + rdev = qp->rdev; + + rq_prod_index = bnxt_qplib_get_rq_prod_index(&qp->qplib_qp); + + if (bnxt_qplib_get_qp1_rq_buf(&qp->qplib_qp, &sge)) { + /* Create 1 SGE to receive the entire + * ethernet packet + */ + /* SGE 1 */ + wqe->sg_list[0].addr = sge.addr; + /* TODO check the lkey to be used */ + wqe->sg_list[0].lkey = sge.lkey; + wqe->sg_list[0].size = BNXT_QPLIB_MAX_QP1_RQ_HDR_SIZE_V2; + if (sge.size < wqe->sg_list[0].size) { + dev_err(rdev_to_dev(qp->rdev), + "QP1 rq buffer is empty!\n"); + rc = -ENOMEM; + goto done; + } + + sqp_entry = &rdev->gsi_ctx.sqp_tbl[rq_prod_index]; + sqp_entry->sge.addr = wr->sg_list[0].addr; + sqp_entry->sge.lkey = wr->sg_list[0].lkey; + sqp_entry->sge.size = wr->sg_list[0].length; + /* Store the wrid for reporting completion */ + sqp_entry->wrid = wqe->wr_id; + /* change the wqe->wrid to table index */ + wqe->wr_id = rq_prod_index; + } +done: + return rc; +} + +static bool is_ud_qp(struct bnxt_re_qp *qp) +{ + return (qp->qplib_qp.type == CMDQ_CREATE_QP_TYPE_UD || + qp->qplib_qp.type == CMDQ_CREATE_QP_TYPE_GSI); +} + +static int bnxt_re_build_send_wqe(struct bnxt_re_qp *qp, + const struct ib_send_wr *wr, + struct bnxt_qplib_swqe *wqe) +{ + struct bnxt_re_ah *ah = NULL; + + if(is_ud_qp(qp)) { + ah = to_bnxt_re(ud_wr(wr)->ah, struct bnxt_re_ah, ibah); + wqe->send.q_key = ud_wr(wr)->remote_qkey; + wqe->send.dst_qp = ud_wr(wr)->remote_qpn; + wqe->send.avid = ah->qplib_ah.id; + } + switch (wr->opcode) { + case IB_WR_SEND: + wqe->type = BNXT_QPLIB_SWQE_TYPE_SEND; + break; + case IB_WR_SEND_WITH_IMM: + wqe->type = BNXT_QPLIB_SWQE_TYPE_SEND_WITH_IMM; + wqe->send.imm_data = wr->ex.imm_data; + break; + case IB_WR_SEND_WITH_INV: + wqe->type = BNXT_QPLIB_SWQE_TYPE_SEND_WITH_INV; + wqe->send.inv_key = wr->ex.invalidate_rkey; + break; + default: + dev_err(rdev_to_dev(qp->rdev), "%s Invalid opcode %d!\n", + __func__, wr->opcode); + return -EINVAL; + } + if (wr->send_flags & IB_SEND_SIGNALED) + wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP; + if (wr->send_flags & IB_SEND_FENCE) + wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE; + if (wr->send_flags & IB_SEND_SOLICITED) + wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SOLICIT_EVENT; + if (wr->send_flags & IB_SEND_INLINE) + wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_INLINE; + + return 0; +} + +static int bnxt_re_build_rdma_wqe(const struct ib_send_wr *wr, + struct bnxt_qplib_swqe *wqe) +{ + switch (wr->opcode) { + case IB_WR_RDMA_WRITE: + wqe->type = BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE; + break; + case IB_WR_RDMA_WRITE_WITH_IMM: + wqe->type = BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE_WITH_IMM; + wqe->rdma.imm_data = wr->ex.imm_data; + break; + case IB_WR_RDMA_READ: + wqe->type = BNXT_QPLIB_SWQE_TYPE_RDMA_READ; + wqe->rdma.inv_key = wr->ex.invalidate_rkey; + break; + default: + return -EINVAL; + } + wqe->rdma.remote_va = rdma_wr(wr)->remote_addr; + wqe->rdma.r_key = rdma_wr(wr)->rkey; + if (wr->send_flags & IB_SEND_SIGNALED) + wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP; + if (wr->send_flags & IB_SEND_FENCE) + wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE; + if (wr->send_flags & IB_SEND_SOLICITED) + wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SOLICIT_EVENT; + if (wr->send_flags & IB_SEND_INLINE) + wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_INLINE; + + return 0; +} + +static int bnxt_re_build_atomic_wqe(const struct ib_send_wr *wr, + struct bnxt_qplib_swqe *wqe) +{ + switch (wr->opcode) { + case IB_WR_ATOMIC_CMP_AND_SWP: + wqe->type = BNXT_QPLIB_SWQE_TYPE_ATOMIC_CMP_AND_SWP; + wqe->atomic.cmp_data = atomic_wr(wr)->compare_add; + wqe->atomic.swap_data = atomic_wr(wr)->swap; + break; + case IB_WR_ATOMIC_FETCH_AND_ADD: + wqe->type = BNXT_QPLIB_SWQE_TYPE_ATOMIC_FETCH_AND_ADD; + wqe->atomic.cmp_data = atomic_wr(wr)->compare_add; + break; + default: + return -EINVAL; + } + wqe->atomic.remote_va = atomic_wr(wr)->remote_addr; + wqe->atomic.r_key = atomic_wr(wr)->rkey; + if (wr->send_flags & IB_SEND_SIGNALED) + wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP; + if (wr->send_flags & IB_SEND_FENCE) + wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE; + if (wr->send_flags & IB_SEND_SOLICITED) + wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SOLICIT_EVENT; + return 0; +} + +static int bnxt_re_build_inv_wqe(const struct ib_send_wr *wr, + struct bnxt_qplib_swqe *wqe) +{ + wqe->type = BNXT_QPLIB_SWQE_TYPE_LOCAL_INV; + wqe->local_inv.inv_l_key = wr->ex.invalidate_rkey; + if (wr->send_flags & IB_SEND_SIGNALED) + wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP; + if (wr->send_flags & IB_SEND_FENCE) + wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE; + if (wr->send_flags & IB_SEND_SOLICITED) + wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SOLICIT_EVENT; + + return 0; +} + +static int bnxt_re_build_reg_wqe(const struct ib_reg_wr *wr, + struct bnxt_qplib_swqe *wqe) +{ + struct bnxt_re_mr *mr = to_bnxt_re(wr->mr, struct bnxt_re_mr, ib_mr); + struct bnxt_qplib_frpl *qplib_frpl = &mr->qplib_frpl; + int reg_len, i, access = wr->access; + + if (mr->npages > qplib_frpl->max_pg_ptrs) { + dev_err_ratelimited(rdev_to_dev(mr->rdev), + " %s: failed npages %d > %d\n", __func__, + mr->npages, qplib_frpl->max_pg_ptrs); + return -EINVAL; + } + + wqe->frmr.pbl_ptr = (__le64 *)qplib_frpl->hwq.pbl_ptr[0]; + wqe->frmr.pbl_dma_ptr = qplib_frpl->hwq.pbl_dma_ptr[0]; + wqe->frmr.levels = qplib_frpl->hwq.level; + wqe->frmr.page_list = mr->pages; + wqe->frmr.page_list_len = mr->npages; + wqe->type = BNXT_QPLIB_SWQE_TYPE_REG_MR; + + if (wr->wr.send_flags & IB_SEND_SIGNALED) + wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP; + if (access & IB_ACCESS_LOCAL_WRITE) + wqe->frmr.access_cntl |= SQ_FR_PMR_ACCESS_CNTL_LOCAL_WRITE; + if (access & IB_ACCESS_REMOTE_READ) + wqe->frmr.access_cntl |= SQ_FR_PMR_ACCESS_CNTL_REMOTE_READ; + if (access & IB_ACCESS_REMOTE_WRITE) + wqe->frmr.access_cntl |= SQ_FR_PMR_ACCESS_CNTL_REMOTE_WRITE; + if (access & IB_ACCESS_REMOTE_ATOMIC) + wqe->frmr.access_cntl |= SQ_FR_PMR_ACCESS_CNTL_REMOTE_ATOMIC; + if (access & IB_ACCESS_MW_BIND) + wqe->frmr.access_cntl |= SQ_FR_PMR_ACCESS_CNTL_WINDOW_BIND; + + /* TODO: OFED provides the rkey of the MR instead of the lkey */ + wqe->frmr.l_key = wr->key; + wqe->frmr.length = wr->mr->length; + wqe->frmr.pbl_pg_sz_log = ilog2(PAGE_SIZE >> PAGE_SHIFT_4K); + wqe->frmr.pg_sz_log = ilog2(wr->mr->page_size >> PAGE_SHIFT_4K); + wqe->frmr.va = wr->mr->iova; + reg_len = wqe->frmr.page_list_len * wr->mr->page_size; + + if (wqe->frmr.length > reg_len) { + dev_err_ratelimited(rdev_to_dev(mr->rdev), + "%s: bnxt_re_mr 0x%px len (%d > %d)\n", + __func__, (void *)mr, wqe->frmr.length, + reg_len); + + for (i = 0; i < mr->npages; i++) + dev_dbg(rdev_to_dev(mr->rdev), + "%s: build_reg_wqe page[%d] = 0x%llx\n", + __func__, i, mr->pages[i]); + + return -EINVAL; + } + + return 0; +} + +static void bnxt_re_set_sg_list(const struct ib_send_wr *wr, + struct bnxt_qplib_swqe *wqe) +{ + wqe->sg_list = (struct bnxt_qplib_sge *)wr->sg_list; + wqe->num_sge = wr->num_sge; +} + +static void bnxt_ud_qp_hw_stall_workaround(struct bnxt_re_qp *qp) +{ + if ((qp->ib_qp.qp_type == IB_QPT_UD || qp->ib_qp.qp_type == IB_QPT_GSI || + qp->ib_qp.qp_type == IB_QPT_RAW_ETHERTYPE) && + qp->qplib_qp.wqe_cnt == BNXT_RE_UD_QP_HW_STALL) { + int qp_attr_mask; + struct ib_qp_attr qp_attr; + + qp_attr_mask = IB_QP_STATE; + qp_attr.qp_state = IB_QPS_RTS; + bnxt_re_modify_qp(&qp->ib_qp, &qp_attr, qp_attr_mask, NULL); + qp->qplib_qp.wqe_cnt = 0; + } +} + +static int bnxt_re_post_send_shadow_qp(struct bnxt_re_dev *rdev, + struct bnxt_re_qp *qp, + const struct ib_send_wr *wr) +{ + struct bnxt_qplib_swqe wqe; + unsigned long flags; + int rc = 0; + + spin_lock_irqsave(&qp->sq_lock, flags); + while (wr) { + /* House keeping */ + memset(&wqe, 0, sizeof(wqe)); + /* Common */ + if (wr->num_sge > qp->qplib_qp.sq.max_sge) { + dev_err(rdev_to_dev(rdev), + "Limit exceeded for Send SGEs\n"); + rc = -EINVAL; + break; + } + + bnxt_re_set_sg_list(wr, &wqe); + wqe.wr_id = wr->wr_id; + wqe.type = BNXT_QPLIB_SWQE_TYPE_SEND; + rc = bnxt_re_build_send_wqe(qp, wr, &wqe); + if (rc) + break; + + rc = bnxt_qplib_post_send(&qp->qplib_qp, &wqe); + if (rc) { + dev_err(rdev_to_dev(rdev), + "bad_wr seen with opcode = 0x%x rc = %d\n", + wr->opcode, rc); + break; + } + wr = wr->next; + } + bnxt_qplib_post_send_db(&qp->qplib_qp); + bnxt_ud_qp_hw_stall_workaround(qp); + spin_unlock_irqrestore(&qp->sq_lock, flags); + return rc; +} + +static void bnxt_re_legacy_set_uc_fence(struct bnxt_qplib_swqe *wqe) +{ + /* Need unconditional fence for non-wire memory opcode + * to work as expected. + */ + if (wqe->type == BNXT_QPLIB_SWQE_TYPE_LOCAL_INV || + wqe->type == BNXT_QPLIB_SWQE_TYPE_FAST_REG_MR || + wqe->type == BNXT_QPLIB_SWQE_TYPE_REG_MR || + wqe->type == BNXT_QPLIB_SWQE_TYPE_BIND_MW) + wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE; +} + +int bnxt_re_post_send(struct ib_qp *ib_qp, const struct ib_send_wr *wr, + const struct ib_send_wr **bad_wr) +{ + struct bnxt_re_qp *qp = to_bnxt_re(ib_qp, struct bnxt_re_qp, ib_qp); + struct bnxt_qplib_sge sge[6]; + struct bnxt_qplib_swqe wqe; + struct bnxt_re_dev *rdev; + unsigned long flags; + int rc = 0; + + rdev = qp->rdev; + spin_lock_irqsave(&qp->sq_lock, flags); + while (wr) { + /* House keeping */ + memset(&wqe, 0, sizeof(wqe)); + /* Common */ + if (wr->num_sge > qp->qplib_qp.sq.max_sge) { + dev_err(rdev_to_dev(rdev), + "Limit exceeded for Send SGEs\n"); + rc = -EINVAL; + goto bad; + } + + bnxt_re_set_sg_list(wr, &wqe); + wqe.wr_id = wr->wr_id; + + switch (wr->opcode) { + case IB_WR_SEND: + case IB_WR_SEND_WITH_IMM: + if (ib_qp->qp_type == IB_QPT_GSI && + rdev->gsi_ctx.gsi_qp_mode != BNXT_RE_GSI_MODE_UD) { + memset(sge, 0, sizeof(sge)); + wqe.sg_list = sge; + rc = bnxt_re_build_gsi_send(qp, wr, &wqe); + if (rc) + goto bad; + } else if (ib_qp->qp_type == IB_QPT_RAW_ETHERTYPE) { + bnxt_re_build_raw_send(wr, &wqe); + } + switch (wr->send_flags) { + case IB_SEND_IP_CSUM: + wqe.rawqp1.lflags |= + SQ_SEND_RAWETH_QP1_LFLAGS_IP_CHKSUM; + break; + default: + break; + } + fallthrough; + case IB_WR_SEND_WITH_INV: + rc = bnxt_re_build_send_wqe(qp, wr, &wqe); + break; + case IB_WR_RDMA_WRITE: + case IB_WR_RDMA_WRITE_WITH_IMM: + case IB_WR_RDMA_READ: + rc = bnxt_re_build_rdma_wqe(wr, &wqe); + break; + case IB_WR_ATOMIC_CMP_AND_SWP: + case IB_WR_ATOMIC_FETCH_AND_ADD: + rc = bnxt_re_build_atomic_wqe(wr, &wqe); + break; + case IB_WR_RDMA_READ_WITH_INV: + dev_err(rdev_to_dev(rdev), + "RDMA Read with Invalidate is not supported\n"); + rc = -EINVAL; + goto bad; + case IB_WR_LOCAL_INV: + rc = bnxt_re_build_inv_wqe(wr, &wqe); + break; + case IB_WR_REG_MR: + rc = bnxt_re_build_reg_wqe(reg_wr(wr), &wqe); + break; + default: + /* Unsupported WRs */ + dev_err(rdev_to_dev(rdev), + "WR (0x%x) is not supported\n", wr->opcode); + rc = -EINVAL; + goto bad; + } + + if (likely(!rc)) { + if (!_is_chip_gen_p5_p7(rdev->chip_ctx)) + bnxt_re_legacy_set_uc_fence(&wqe); + rc = bnxt_qplib_post_send(&qp->qplib_qp, &wqe); + } +bad: + if (unlikely(rc)) { + dev_err(rdev_to_dev(rdev), + "bad_wr seen with opcode = 0x%x\n", wr->opcode); + *bad_wr = wr; + break; + } + wr = wr->next; + } + bnxt_qplib_post_send_db(&qp->qplib_qp); + if (!_is_chip_gen_p5_p7(rdev->chip_ctx)) + bnxt_ud_qp_hw_stall_workaround(qp); + spin_unlock_irqrestore(&qp->sq_lock, flags); + + return rc; +} + +static int bnxt_re_post_recv_shadow_qp(struct bnxt_re_dev *rdev, + struct bnxt_re_qp *qp, + struct ib_recv_wr *wr) +{ + struct bnxt_qplib_swqe wqe; + int rc = 0; + + /* rq lock can be pardoned here. */ + while (wr) { + /* House keeping */ + memset(&wqe, 0, sizeof(wqe)); + /* Common */ + if (wr->num_sge > qp->qplib_qp.rq.max_sge) { + dev_err(rdev_to_dev(rdev), + "Limit exceeded for Receive SGEs\n"); + rc = -EINVAL; + goto bad; + } + + wqe.sg_list = (struct bnxt_qplib_sge *)wr->sg_list; + wqe.num_sge = wr->num_sge; + wqe.wr_id = wr->wr_id; + wqe.type = BNXT_QPLIB_SWQE_TYPE_RECV; + rc = bnxt_qplib_post_recv(&qp->qplib_qp, &wqe); +bad: + if (rc) { + dev_err(rdev_to_dev(rdev), + "bad_wr seen with RQ post\n"); + break; + } + wr = wr->next; + } + bnxt_qplib_post_recv_db(&qp->qplib_qp); + return rc; +} + +static int bnxt_re_build_gsi_recv(struct bnxt_re_qp *qp, + const struct ib_recv_wr *wr, + struct bnxt_qplib_swqe *wqe) +{ + struct bnxt_re_dev *rdev = qp->rdev; + int rc = 0; + + if (rdev->gsi_ctx.gsi_qp_mode == BNXT_RE_GSI_MODE_ALL) + rc = bnxt_re_build_qp1_shadow_qp_recv(qp, wr, wqe); + else + rc = bnxt_re_build_qp1_recv(qp, wr, wqe); + + return rc; +} + +int bnxt_re_post_recv(struct ib_qp *ib_qp, const struct ib_recv_wr *wr, + const struct ib_recv_wr **bad_wr) +{ + struct bnxt_re_qp *qp = to_bnxt_re(ib_qp, struct bnxt_re_qp, ib_qp); + struct bnxt_qplib_sge sge[6]; + struct bnxt_qplib_swqe wqe; + unsigned long flags; + u32 count = 0; + int rc = 0; + + spin_lock_irqsave(&qp->rq_lock, flags); + while (wr) { + memset(&wqe, 0, sizeof(wqe)); + if (wr->num_sge > qp->qplib_qp.rq.max_sge) { + dev_err(rdev_to_dev(qp->rdev), + "Limit exceeded for Receive SGEs\n"); + rc = -EINVAL; + goto bad; + } + wqe.num_sge = wr->num_sge; + wqe.sg_list = (struct bnxt_qplib_sge *)wr->sg_list; + wqe.wr_id = wr->wr_id; + wqe.type = BNXT_QPLIB_SWQE_TYPE_RECV; + + if (ib_qp->qp_type == IB_QPT_GSI && + qp->rdev->gsi_ctx.gsi_qp_mode != BNXT_RE_GSI_MODE_UD) { + memset(sge, 0, sizeof(sge)); + wqe.sg_list = sge; + rc = bnxt_re_build_gsi_recv(qp, wr, &wqe); + if (rc) + goto bad; + } + rc = bnxt_qplib_post_recv(&qp->qplib_qp, &wqe); +bad: + if (rc) { + dev_err(rdev_to_dev(qp->rdev), + "bad_wr seen with RQ post\n"); + *bad_wr = wr; + break; + } + /* Ring DB if the RQEs posted reaches a threshold value */ + if (++count >= BNXT_RE_RQ_WQE_THRESHOLD) { + bnxt_qplib_post_recv_db(&qp->qplib_qp); + count = 0; + } + wr = wr->next; + } + + if (count) + bnxt_qplib_post_recv_db(&qp->qplib_qp); + spin_unlock_irqrestore(&qp->rq_lock, flags); + + return rc; +} + +/* Completion Queues */ +void bnxt_re_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata) +{ + struct bnxt_re_cq *cq = to_bnxt_re(ib_cq, struct bnxt_re_cq, ibcq); + struct bnxt_re_dev *rdev = cq->rdev; + int rc = 0; + + if (cq->uctx_cq_page) { + BNXT_RE_CQ_PAGE_LIST_DEL(cq->uctx, cq); + free_page((u64)cq->uctx_cq_page); + cq->uctx_cq_page = NULL; + } + if (cq->is_dbr_soft_cq && cq->uctx) { + void *dbr_page; + + if (cq->uctx->dbr_recov_cq) { + dbr_page = cq->uctx->dbr_recov_cq_page; + cq->uctx->dbr_recov_cq_page = NULL; + cq->uctx->dbr_recov_cq = NULL; + free_page((unsigned long)dbr_page); + } + goto end; + } + /* CQ getting destroyed. Set this state for cqn handler */ + spin_lock_bh(&cq->qplib_cq.compl_lock); + cq->qplib_cq.destroyed = true; + spin_unlock_bh(&cq->qplib_cq.compl_lock); + if (ib_cq->poll_ctx == IB_POLL_WORKQUEUE || + ib_cq->poll_ctx == IB_POLL_UNBOUND_WORKQUEUE) + cancel_work_sync(&ib_cq->work); + + rc = bnxt_qplib_destroy_cq(&rdev->qplib_res, &cq->qplib_cq); + if (rc) + dev_err_ratelimited(rdev_to_dev(rdev), + "%s id = %d failed rc = %d\n", + __func__, cq->qplib_cq.id, rc); + + bnxt_re_put_nq(rdev, cq->qplib_cq.nq); + if (cq->umem && !IS_ERR(cq->umem)) + ib_umem_release(cq->umem); + + kfree(cq->cql); + atomic_dec(&rdev->stats.rsors.cq_count); +end: + return; +} + +static inline struct +bnxt_re_cq *__get_cq_from_cq_in(struct ib_cq *cq_in, + struct bnxt_re_dev *rdev) +{ + struct bnxt_re_cq *cq; + cq = container_of(cq_in, struct bnxt_re_cq, ibcq); + return cq; +} + +int bnxt_re_create_cq(struct ib_cq *cq_in, + const struct ib_cq_init_attr *attr, + struct ib_udata *udata) +{ + struct bnxt_qplib_dev_attr *dev_attr; + struct bnxt_re_ucontext *uctx = NULL; + struct ib_ucontext *context = NULL; + struct bnxt_qplib_cq *qplcq; + struct bnxt_re_cq_req ureq; + struct bnxt_re_dev *rdev; + int rc, entries; + struct bnxt_re_cq *cq; + u32 max_active_cqs; + int cqe = attr->cqe; + + if (attr->flags) + return -EOPNOTSUPP; + + rdev = rdev_from_cq_in(cq_in); + if (rdev->mod_exit) { + rc = -EIO; + dev_dbg(rdev_to_dev(rdev), "%s(): in mod_exit, just return!\n", __func__); + goto exit; + } + if (udata) { + uctx = rdma_udata_to_drv_context(udata, + struct bnxt_re_ucontext, + ibucontext); + context = &uctx->ibucontext; + } + dev_attr = rdev->dev_attr; + + if (atomic_read(&rdev->stats.rsors.cq_count) >= dev_attr->max_cq) { + dev_err(rdev_to_dev(rdev), "Create CQ failed - max exceeded(CQs)\n"); + rc = -EINVAL; + goto exit; + } + /* Validate CQ fields */ + if (cqe < 1 || cqe > dev_attr->max_cq_wqes) { + dev_err(rdev_to_dev(rdev), "Create CQ failed - max exceeded(CQ_WQs)\n"); + rc = -EINVAL; + goto exit; + } + + cq = __get_cq_from_cq_in(cq_in, rdev); + if (!cq) { + rc = -ENOMEM; + goto exit; + } + cq->rdev = rdev; + cq->uctx = uctx; + qplcq = &cq->qplib_cq; + qplcq->cq_handle = (u64)qplcq; + /* + * Since CQ is for QP1 is shared with Shadow CQ, the size + * should be double the size. There is no way to identify + * whether this CQ is for GSI QP. So assuming that the first + * CQ created is for QP1 + */ + if (!udata && !rdev->gsi_ctx.first_cq_created && + rdev->gsi_ctx.gsi_qp_mode == BNXT_RE_GSI_MODE_ALL) { + rdev->gsi_ctx.first_cq_created = true; + /* + * Total CQE required for the CQ = CQE for QP1 RQ + + * CQE for Shadow QP SQEs + CQE for Shadow QP RQEs. + * Max entries of shadow QP SQ and RQ = QP1 RQEs = cqe + */ + cqe *= 3; + } + + entries = bnxt_re_init_depth(cqe + 1, uctx); + if (entries > dev_attr->max_cq_wqes + 1) + entries = dev_attr->max_cq_wqes + 1; + + qplcq->sginfo.pgshft = PAGE_SHIFT; + qplcq->sginfo.pgsize = PAGE_SIZE; + if (udata) { + if (udata->inlen < sizeof(ureq)) + dev_warn(rdev_to_dev(rdev), + "Update the library ulen %d klen %d\n", + (unsigned int)udata->inlen, + (unsigned int)sizeof(ureq)); + + rc = ib_copy_from_udata(&ureq, udata, + min(udata->inlen, sizeof(ureq))); + if (rc) + goto fail; + + if (BNXT_RE_IS_DBR_PACING_NOTIFY_CQ(ureq)) { + cq->is_dbr_soft_cq = true; + goto success; + } + + if (BNXT_RE_IS_DBR_RECOV_CQ(ureq)) { + void *dbr_page; + u32 *epoch; + + dbr_page = (void *)__get_free_page(GFP_KERNEL); + if (!dbr_page) { + dev_err(rdev_to_dev(rdev), + "DBR recov CQ page allocation failed!"); + rc = -ENOMEM; + goto fail; + } + + /* memset the epoch and epoch_ack to 0 */ + epoch = dbr_page; + epoch[0] = 0x0; + epoch[1] = 0x0; + + uctx->dbr_recov_cq = cq; + uctx->dbr_recov_cq_page = dbr_page; + + cq->is_dbr_soft_cq = true; + goto success; + } + + cq->umem = ib_umem_get_compat + (rdev, context, udata, ureq.cq_va, + entries * sizeof(struct cq_base), + IB_ACCESS_LOCAL_WRITE, 1); + if (IS_ERR(cq->umem)) { + rc = PTR_ERR(cq->umem); + dev_err(rdev_to_dev(rdev), + "%s: ib_umem_get failed! rc = %d\n", + __func__, rc); + goto fail; + } + qplcq->sginfo.sghead = get_ib_umem_sgl(cq->umem, + &qplcq->sginfo.nmap); + qplcq->sginfo.npages = ib_umem_num_pages_compat(cq->umem); + if (!uctx->dpi.dbr) { + rc = bnxt_re_get_user_dpi(rdev, uctx); + if (rc) + goto c2fail; + } + qplcq->dpi = &uctx->dpi; + } else { + cq->max_cql = entries > MAX_CQL_PER_POLL ? MAX_CQL_PER_POLL : entries; + cq->cql = kcalloc(cq->max_cql, sizeof(struct bnxt_qplib_cqe), + GFP_KERNEL); + if (!cq->cql) { + dev_err(rdev_to_dev(rdev), + "Allocate CQL for %d failed!\n", cq->max_cql); + rc = -ENOMEM; + goto fail; + } + qplcq->dpi = &rdev->dpi_privileged; + } + /* + * Allocating the NQ in a round robin fashion. nq_alloc_cnt is a + * used for getting the NQ index. + */ + qplcq->max_wqe = entries; + qplcq->nq = bnxt_re_get_nq(rdev); + qplcq->cnq_hw_ring_id = qplcq->nq->ring_id; + + rc = bnxt_qplib_create_cq(&rdev->qplib_res, qplcq); + if (rc) { + dev_err(rdev_to_dev(rdev), "Create HW CQ failed!\n"); + goto fail; + } + + INIT_LIST_HEAD(&cq->cq_list); + cq->ibcq.cqe = entries; + cq->cq_period = qplcq->period; + + atomic_inc(&rdev->stats.rsors.cq_count); + max_active_cqs = atomic_read(&rdev->stats.rsors.cq_count); + if (max_active_cqs > atomic_read(&rdev->stats.rsors.max_cq_count)) + atomic_set(&rdev->stats.rsors.max_cq_count, max_active_cqs); + spin_lock_init(&cq->cq_lock); + + if (udata) { + struct bnxt_re_cq_resp resp; + + resp.cqid = qplcq->id; + resp.tail = qplcq->hwq.cons; + resp.phase = qplcq->period; + resp.comp_mask = 0; + resp.dbr = (u64)uctx->dpi.umdbr; + resp.dpi = uctx->dpi.dpi; + resp.comp_mask |= BNXT_RE_COMP_MASK_CQ_HAS_DB_INFO; + /* Copy only on a valid wcpdi */ + if (uctx->wcdpi.dpi) { + resp.wcdpi = uctx->wcdpi.dpi; + resp.comp_mask |= BNXT_RE_COMP_MASK_CQ_HAS_WC_DPI; + } + + if (_is_chip_p7(rdev->chip_ctx)) { + cq->uctx_cq_page = (void *)__get_free_page(GFP_KERNEL); + + if (!cq->uctx_cq_page) { + dev_err(rdev_to_dev(rdev), + "CQ page allocation failed!\n"); + bnxt_qplib_destroy_cq(&rdev->qplib_res, qplcq); + rc = -ENOMEM; + goto c2fail; + } + + resp.uctx_cq_page = (u64)cq->uctx_cq_page; + resp.comp_mask |= BNXT_RE_COMP_MASK_CQ_HAS_CQ_PAGE; + } + + rc = bnxt_re_copy_to_udata(rdev, &resp, + min(udata->outlen, sizeof(resp)), + udata); + if (rc) { + free_page((u64)cq->uctx_cq_page); + cq->uctx_cq_page = NULL; + bnxt_qplib_destroy_cq(&rdev->qplib_res, qplcq); + goto c2fail; + } + + if (cq->uctx_cq_page) + BNXT_RE_CQ_PAGE_LIST_ADD(uctx, cq); + } + +success: + return 0; +c2fail: + if (udata && cq->umem && !IS_ERR(cq->umem)) + ib_umem_release(cq->umem); +fail: + if (cq) { + if (cq->cql) + kfree(cq->cql); + } +exit: + return rc; +} + +int bnxt_re_modify_cq(struct ib_cq *ib_cq, u16 cq_count, u16 cq_period) +{ + struct bnxt_re_cq *cq = to_bnxt_re(ib_cq, struct bnxt_re_cq, ibcq); + struct bnxt_re_dev *rdev = cq->rdev; + int rc; + + if ((cq->cq_count != cq_count) || (cq->cq_period != cq_period)) { + cq->qplib_cq.count = cq_count; + cq->qplib_cq.period = cq_period; + rc = bnxt_qplib_modify_cq(&rdev->qplib_res, &cq->qplib_cq); + if (rc) { + dev_err(rdev_to_dev(rdev), "Modify HW CQ %#x failed!\n", + cq->qplib_cq.id); + return rc; + } + /* On success, update the shadow */ + cq->cq_count = cq_count; + cq->cq_period = cq_period; + } + return 0; +} + +static void bnxt_re_resize_cq_complete(struct bnxt_re_cq *cq) +{ + struct bnxt_re_dev *rdev = cq->rdev; + + bnxt_qplib_resize_cq_complete(&rdev->qplib_res, &cq->qplib_cq); + + cq->qplib_cq.max_wqe = cq->resize_cqe; + if (cq->resize_umem) { + ib_umem_release(cq->umem); + cq->umem = cq->resize_umem; + cq->resize_umem = NULL; + cq->resize_cqe = 0; + } +} + +int bnxt_re_resize_cq(struct ib_cq *ib_cq, int cqe, struct ib_udata *udata) +{ + struct bnxt_qplib_sg_info sginfo = {}; + struct bnxt_qplib_dpi *orig_dpi = NULL; + struct bnxt_qplib_dev_attr *dev_attr; + struct bnxt_re_ucontext *uctx = NULL; + struct bnxt_re_resize_cq_req ureq; + struct ib_ucontext *context = NULL; + struct bnxt_re_dev *rdev; + struct bnxt_re_cq *cq; + int rc, entries; + + /* Don't allow more than one resize request at the same time. + * TODO: need a mutex here when we support kernel consumers of resize. + */ + cq = to_bnxt_re(ib_cq, struct bnxt_re_cq, ibcq); + rdev = cq->rdev; + dev_attr = rdev->dev_attr; + if (ib_cq->uobject) { + uctx = rdma_udata_to_drv_context(udata, + struct bnxt_re_ucontext, + ibucontext); + context = &uctx->ibucontext; + } + + if (cq->resize_umem) { + dev_err(rdev_to_dev(rdev), "Resize CQ %#x failed - Busy\n", + cq->qplib_cq.id); + return -EBUSY; + } + + /* Check the requested cq depth out of supported depth */ + if (cqe < 1 || cqe > dev_attr->max_cq_wqes) { + dev_err(rdev_to_dev(rdev), "Resize CQ %#x failed - max exceeded\n", + cq->qplib_cq.id); + return -EINVAL; + } + + entries = bnxt_re_init_depth(cqe + 1, uctx); + entries = min_t(u32, (u32)entries, dev_attr->max_cq_wqes + 1); + + /* Check to see if the new requested size can be handled by already + * existing CQ + */ + if (entries == cq->ibcq.cqe) { + dev_info(rdev_to_dev(rdev), "CQ is already at size %d\n", cqe); + return 0; + } + + if (ib_cq->uobject && udata) { + if (udata->inlen < sizeof(ureq)) + dev_warn(rdev_to_dev(rdev), + "Update the library ulen %d klen %d\n", + (unsigned int)udata->inlen, + (unsigned int)sizeof(ureq)); + + rc = ib_copy_from_udata(&ureq, udata, + min(udata->inlen, sizeof(ureq))); + if (rc) + goto fail; + + dev_dbg(rdev_to_dev(rdev), "%s: va %p\n", __func__, + (void *)ureq.cq_va); + cq->resize_umem = ib_umem_get_compat + (rdev, + context, udata, ureq.cq_va, + entries * sizeof(struct cq_base), + IB_ACCESS_LOCAL_WRITE, 1); + if (IS_ERR(cq->resize_umem)) { + rc = PTR_ERR(cq->resize_umem); + cq->resize_umem = NULL; + dev_err(rdev_to_dev(rdev), "%s: ib_umem_get failed! rc = %d\n", + __func__, rc); + goto fail; + } + cq->resize_cqe = entries; + dev_dbg(rdev_to_dev(rdev), "%s: ib_umem_get() success\n", + __func__); + memcpy(&sginfo, &cq->qplib_cq.sginfo, sizeof(sginfo)); + orig_dpi = cq->qplib_cq.dpi; + + cq->qplib_cq.sginfo.sghead = get_ib_umem_sgl(cq->resize_umem, + &cq->qplib_cq.sginfo.nmap); + cq->qplib_cq.sginfo.npages = + ib_umem_num_pages_compat(cq->resize_umem); + cq->qplib_cq.sginfo.pgsize = PAGE_SIZE; + cq->qplib_cq.sginfo.pgshft = PAGE_SHIFT; + cq->qplib_cq.dpi = &uctx->dpi; + } else { + /* TODO: kernel consumer */ + } + + rc = bnxt_qplib_resize_cq(&rdev->qplib_res, &cq->qplib_cq, entries); + if (rc) { + dev_err(rdev_to_dev(rdev), "Resize HW CQ %#x failed!\n", + cq->qplib_cq.id); + goto fail; + } + + cq->ibcq.cqe = cq->resize_cqe; + /* For kernel consumers complete resize here. For uverbs consumers, + * we complete it in the context of ibv_poll_cq(). + */ + if (!cq->resize_umem) + bnxt_qplib_resize_cq_complete(&rdev->qplib_res, &cq->qplib_cq); + + atomic_inc(&rdev->stats.rsors.resize_count); + return 0; + +fail: + if (cq->resize_umem) { + ib_umem_release(cq->resize_umem); + cq->resize_umem = NULL; + cq->resize_cqe = 0; + memcpy(&cq->qplib_cq.sginfo, &sginfo, sizeof(sginfo)); + cq->qplib_cq.dpi = orig_dpi; + } + return rc; +} + +static enum ib_wc_status __req_to_ib_wc_status(u8 qstatus) +{ + switch(qstatus) { + case CQ_REQ_STATUS_OK: + return IB_WC_SUCCESS; + case CQ_REQ_STATUS_BAD_RESPONSE_ERR: + return IB_WC_BAD_RESP_ERR; + case CQ_REQ_STATUS_LOCAL_LENGTH_ERR: + return IB_WC_LOC_LEN_ERR; + case CQ_REQ_STATUS_LOCAL_QP_OPERATION_ERR: + return IB_WC_LOC_QP_OP_ERR; + case CQ_REQ_STATUS_LOCAL_PROTECTION_ERR: + return IB_WC_LOC_PROT_ERR; + case CQ_REQ_STATUS_MEMORY_MGT_OPERATION_ERR: + return IB_WC_GENERAL_ERR; + case CQ_REQ_STATUS_REMOTE_INVALID_REQUEST_ERR: + return IB_WC_REM_INV_REQ_ERR; + case CQ_REQ_STATUS_REMOTE_ACCESS_ERR: + return IB_WC_REM_ACCESS_ERR; + case CQ_REQ_STATUS_REMOTE_OPERATION_ERR: + return IB_WC_REM_OP_ERR; + case CQ_REQ_STATUS_RNR_NAK_RETRY_CNT_ERR: + return IB_WC_RNR_RETRY_EXC_ERR; + case CQ_REQ_STATUS_TRANSPORT_RETRY_CNT_ERR: + return IB_WC_RETRY_EXC_ERR; + case CQ_REQ_STATUS_WORK_REQUEST_FLUSHED_ERR: + return IB_WC_WR_FLUSH_ERR; + default: + return IB_WC_GENERAL_ERR; + } + return 0; +} + +static enum ib_wc_status __rawqp1_to_ib_wc_status(u8 qstatus) +{ + switch(qstatus) { + case CQ_RES_RAWETH_QP1_STATUS_OK: + return IB_WC_SUCCESS; + case CQ_RES_RAWETH_QP1_STATUS_LOCAL_ACCESS_ERROR: + return IB_WC_LOC_ACCESS_ERR; + case CQ_RES_RAWETH_QP1_STATUS_HW_LOCAL_LENGTH_ERR: + return IB_WC_LOC_LEN_ERR; + case CQ_RES_RAWETH_QP1_STATUS_LOCAL_PROTECTION_ERR: + return IB_WC_LOC_PROT_ERR; + case CQ_RES_RAWETH_QP1_STATUS_LOCAL_QP_OPERATION_ERR: + return IB_WC_LOC_QP_OP_ERR; + case CQ_RES_RAWETH_QP1_STATUS_MEMORY_MGT_OPERATION_ERR: + return IB_WC_GENERAL_ERR; + case CQ_RES_RAWETH_QP1_STATUS_WORK_REQUEST_FLUSHED_ERR: + return IB_WC_WR_FLUSH_ERR; + case CQ_RES_RAWETH_QP1_STATUS_HW_FLUSH_ERR: + return IB_WC_WR_FLUSH_ERR; + default: + return IB_WC_GENERAL_ERR; + } +} + +static enum ib_wc_status __rc_to_ib_wc_status(u8 qstatus) +{ + switch(qstatus) { + case CQ_RES_RC_STATUS_OK: + return IB_WC_SUCCESS; + case CQ_RES_RC_STATUS_LOCAL_ACCESS_ERROR: + return IB_WC_LOC_ACCESS_ERR; + case CQ_RES_RC_STATUS_LOCAL_LENGTH_ERR: + return IB_WC_LOC_LEN_ERR; + case CQ_RES_RC_STATUS_LOCAL_PROTECTION_ERR: + return IB_WC_LOC_PROT_ERR; + case CQ_RES_RC_STATUS_LOCAL_QP_OPERATION_ERR: + return IB_WC_LOC_QP_OP_ERR; + case CQ_RES_RC_STATUS_MEMORY_MGT_OPERATION_ERR: + return IB_WC_GENERAL_ERR; + case CQ_RES_RC_STATUS_REMOTE_INVALID_REQUEST_ERR: + return IB_WC_REM_INV_REQ_ERR; + case CQ_RES_RC_STATUS_WORK_REQUEST_FLUSHED_ERR: + return IB_WC_WR_FLUSH_ERR; + case CQ_RES_RC_STATUS_HW_FLUSH_ERR: + return IB_WC_WR_FLUSH_ERR; + default: + return IB_WC_GENERAL_ERR; + } +} + +static void bnxt_re_process_req_wc(struct ib_wc *wc, struct bnxt_qplib_cqe *cqe) +{ + switch (cqe->type) { + case BNXT_QPLIB_SWQE_TYPE_SEND: + wc->opcode = IB_WC_SEND; + break; + case BNXT_QPLIB_SWQE_TYPE_SEND_WITH_IMM: + wc->opcode = IB_WC_SEND; + wc->wc_flags |= IB_WC_WITH_IMM; + break; + case BNXT_QPLIB_SWQE_TYPE_SEND_WITH_INV: + wc->opcode = IB_WC_SEND; + wc->wc_flags |= IB_WC_WITH_INVALIDATE; + break; + case BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE: + wc->opcode = IB_WC_RDMA_WRITE; + break; + case BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE_WITH_IMM: + wc->opcode = IB_WC_RDMA_WRITE; + wc->wc_flags |= IB_WC_WITH_IMM; + break; + case BNXT_QPLIB_SWQE_TYPE_RDMA_READ: + wc->opcode = IB_WC_RDMA_READ; + break; + case BNXT_QPLIB_SWQE_TYPE_ATOMIC_CMP_AND_SWP: + wc->opcode = IB_WC_COMP_SWAP; + break; + case BNXT_QPLIB_SWQE_TYPE_ATOMIC_FETCH_AND_ADD: + wc->opcode = IB_WC_FETCH_ADD; + break; + case BNXT_QPLIB_SWQE_TYPE_LOCAL_INV: + wc->opcode = IB_WC_LOCAL_INV; + break; + case BNXT_QPLIB_SWQE_TYPE_REG_MR: + wc->opcode = IB_WC_REG_MR; + break; + default: + wc->opcode = IB_WC_SEND; + break; + } + + wc->status = __req_to_ib_wc_status(cqe->status); +} + +static int bnxt_re_check_packet_type(u16 raweth_qp1_flags, u16 raweth_qp1_flags2) +{ + bool is_ipv6 = false, is_ipv4 = false; + + /* raweth_qp1_flags Bit 9-6 indicates itype */ + + if ((raweth_qp1_flags & CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_ROCE) + != CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_ROCE) + return -1; + + if (raweth_qp1_flags2 & + CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_IP_CS_CALC && + raweth_qp1_flags2 & + CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_L4_CS_CALC) { + /* raweth_qp1_flags2 Bit 8 indicates ip_type. 0-v4 1 - v6 */ + (raweth_qp1_flags2 & + CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_IP_TYPE) ? + (is_ipv6 = true) : (is_ipv4 = true); + return ((is_ipv6) ? + BNXT_RE_ROCEV2_IPV6_PACKET : + BNXT_RE_ROCEV2_IPV4_PACKET); + } else { + return BNXT_RE_ROCE_V1_PACKET; + } +} + +static bool bnxt_re_is_loopback_packet(struct bnxt_re_dev *rdev, + void *rq_hdr_buf) +{ + u8 *tmp_buf = NULL; + struct ethhdr *eth_hdr; + u16 eth_type; + bool rc = false; + + tmp_buf = (u8 *)rq_hdr_buf; + /* + * If dest mac is not same as I/F mac, this could be a + * loopback address or multicast address, check whether + * it is a loopback packet + */ + if (!ether_addr_equal(tmp_buf, rdev->dev_addr)) { + tmp_buf += 4; + /* Check the ether type */ + eth_hdr = (struct ethhdr *)tmp_buf; + eth_type = ntohs(eth_hdr->h_proto); + switch (eth_type) { + case BNXT_QPLIB_ETHTYPE_ROCEV1: + rc = true; + break; + default: + break; + } + } + + return rc; +} + +static bool bnxt_re_is_vlan_in_packet(struct bnxt_re_dev *rdev, + void *rq_hdr_buf, + struct bnxt_qplib_cqe *cqe) +{ + struct vlan_hdr *vlan_hdr; + struct ethhdr *eth_hdr; + u8 *tmp_buf = NULL; + u16 eth_type; + + tmp_buf = (u8 *)rq_hdr_buf; + /* Check the ether type */ + eth_hdr = (struct ethhdr *)tmp_buf; + eth_type = ntohs(eth_hdr->h_proto); + if (eth_type == ETH_P_8021Q) { + tmp_buf += sizeof(struct ethhdr); + vlan_hdr = (struct vlan_hdr *)tmp_buf; + cqe->raweth_qp1_metadata = + ntohs(vlan_hdr->h_vlan_TCI) | + (eth_type << + CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_TPID_SFT); + cqe->raweth_qp1_flags2 |= + CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_META_FORMAT_VLAN; + return true; + } + + return false; +} + +static int bnxt_re_process_raw_qp_packet_receive(struct bnxt_re_qp *gsi_qp, + struct bnxt_qplib_cqe *cqe) +{ + struct bnxt_re_sqp_entries *sqp_entry = NULL; + struct bnxt_qplib_hdrbuf *hdr_buf; + dma_addr_t shrq_hdr_buf_map; + struct ib_sge s_sge[2] = {}; + struct ib_sge r_sge[2] = {}; + struct ib_recv_wr rwr = {}; + struct bnxt_re_ah *gsi_sah; + struct bnxt_re_qp *gsi_sqp; + dma_addr_t rq_hdr_buf_map; + struct bnxt_re_dev *rdev; + struct ib_send_wr *swr; + u32 skip_bytes = 0; + void *rq_hdr_buf; + int pkt_type = 0; + u32 offset = 0; + u32 tbl_idx; + int rc; + struct ib_ud_wr udwr = {}; + + swr = &udwr.wr; + rdev = gsi_qp->rdev; + gsi_sqp = rdev->gsi_ctx.gsi_sqp; + tbl_idx = cqe->wr_id; + + hdr_buf = gsi_qp->qplib_qp.rq_hdr_buf; + rq_hdr_buf = (u8 *) hdr_buf->va + tbl_idx * hdr_buf->step; + rq_hdr_buf_map = bnxt_qplib_get_qp_buf_from_index(&gsi_qp->qplib_qp, + tbl_idx); + /* Shadow QP header buffer */ + shrq_hdr_buf_map = bnxt_qplib_get_qp_buf_from_index(&gsi_sqp->qplib_qp, + tbl_idx); + sqp_entry = &rdev->gsi_ctx.sqp_tbl[tbl_idx]; + + /* Find packet type from the cqe */ + pkt_type = bnxt_re_check_packet_type(cqe->raweth_qp1_flags, + cqe->raweth_qp1_flags2); + if (pkt_type < 0) { + dev_err(rdev_to_dev(rdev), "Not handling this packet\n"); + return -EINVAL; + } + + /* Adjust the offset for the user buffer and post in the rq */ + + if (pkt_type == BNXT_RE_ROCEV2_IPV4_PACKET) + offset = 20; + + /* + * QP1 loopback packet has 4 bytes of internal header before + * ether header. Skip these four bytes. + */ + if (bnxt_re_is_loopback_packet(rdev, rq_hdr_buf)) + skip_bytes = 4; + + if (bnxt_re_is_vlan_in_packet(rdev, rq_hdr_buf, cqe)) + skip_bytes += VLAN_HLEN; + + /* Store this cqe */ + memcpy(&sqp_entry->cqe, cqe, sizeof(struct bnxt_qplib_cqe)); + sqp_entry->qp1_qp = gsi_qp; + + /* First send SGE . Skip the ether header*/ + s_sge[0].addr = rq_hdr_buf_map + BNXT_QPLIB_MAX_QP1_RQ_ETH_HDR_SIZE + + skip_bytes; + s_sge[0].lkey = 0xFFFFFFFF; + s_sge[0].length = offset ? BNXT_QPLIB_MAX_GRH_HDR_SIZE_IPV4 : + BNXT_QPLIB_MAX_GRH_HDR_SIZE_IPV6; + + /* Second Send SGE */ + s_sge[1].addr = s_sge[0].addr + s_sge[0].length + + BNXT_QPLIB_MAX_QP1_RQ_BDETH_HDR_SIZE; + if (pkt_type != BNXT_RE_ROCE_V1_PACKET) + s_sge[1].addr += 8; + s_sge[1].lkey = 0xFFFFFFFF; + s_sge[1].length = 256; + + /* First recv SGE */ + r_sge[0].addr = shrq_hdr_buf_map; + r_sge[0].lkey = 0xFFFFFFFF; + r_sge[0].length = 40; + + r_sge[1].addr = sqp_entry->sge.addr + offset; + r_sge[1].lkey = sqp_entry->sge.lkey; + r_sge[1].length = BNXT_QPLIB_MAX_GRH_HDR_SIZE_IPV6 + 256 - offset; + + /* Create receive work request */ + rwr.num_sge = 2; + rwr.sg_list = r_sge; + rwr.wr_id = tbl_idx; + rwr.next = NULL; + + rc = bnxt_re_post_recv_shadow_qp(rdev, gsi_sqp, &rwr); + if (rc) { + dev_err(rdev_to_dev(rdev), + "Failed to post Rx buffers to shadow QP\n"); + return -ENOMEM; + } + + swr->num_sge = 2; + swr->sg_list = s_sge; + swr->wr_id = tbl_idx; + swr->opcode = IB_WR_SEND; + swr->next = NULL; + + gsi_sah = rdev->gsi_ctx.gsi_sah; + udwr.ah = &gsi_sah->ibah; + udwr.remote_qpn = gsi_sqp->qplib_qp.id; + udwr.remote_qkey = gsi_sqp->qplib_qp.qkey; + /* post data received in the send queue */ + rc = bnxt_re_post_send_shadow_qp(rdev, gsi_sqp, swr); + + return rc; +} + +static void bnxt_re_process_res_rawqp1_wc(struct ib_wc *wc, + struct bnxt_qplib_cqe *cqe) +{ + wc->opcode = IB_WC_RECV; + wc->status = __rawqp1_to_ib_wc_status(cqe->status); + wc->wc_flags |= IB_WC_GRH; +} + +static void bnxt_re_process_res_rc_wc(struct ib_wc *wc, + struct bnxt_qplib_cqe *cqe) +{ + wc->opcode = IB_WC_RECV; + wc->status = __rc_to_ib_wc_status(cqe->status); + + if (cqe->flags & CQ_RES_RC_FLAGS_IMM) + wc->wc_flags |= IB_WC_WITH_IMM; + if (cqe->flags & CQ_RES_RC_FLAGS_INV) + wc->wc_flags |= IB_WC_WITH_INVALIDATE; + if ((cqe->flags & (CQ_RES_RC_FLAGS_RDMA | CQ_RES_RC_FLAGS_IMM)) == + (CQ_RES_RC_FLAGS_RDMA | CQ_RES_RC_FLAGS_IMM)) + wc->opcode = IB_WC_RECV_RDMA_WITH_IMM; +} + +/* Returns TRUE if pkt has valid VLAN and if VLAN id is non-zero */ +static bool bnxt_re_is_nonzero_vlanid_pkt(struct bnxt_qplib_cqe *orig_cqe, + u16 *vid, u8 *sl) +{ + u32 metadata; + u16 tpid; + bool ret = false; + metadata = orig_cqe->raweth_qp1_metadata; + if (orig_cqe->raweth_qp1_flags2 & + CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_META_FORMAT_VLAN) { + tpid = ((metadata & + CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_TPID_MASK) >> + CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_TPID_SFT); + if (tpid == ETH_P_8021Q) { + *vid = metadata & + CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_VID_MASK; + *sl = (metadata & + CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_PRI_MASK) >> + CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_PRI_SFT; + ret = !!(*vid); + } + } + + return ret; +} + +static void bnxt_re_process_res_shadow_qp_wc(struct bnxt_re_qp *gsi_sqp, + struct ib_wc *wc, + struct bnxt_qplib_cqe *cqe) +{ + u32 tbl_idx; + struct bnxt_re_dev *rdev = gsi_sqp->rdev; + struct bnxt_re_qp *gsi_qp = NULL; + struct bnxt_qplib_cqe *orig_cqe = NULL; + struct bnxt_re_sqp_entries *sqp_entry = NULL; + int nw_type; + u16 vlan_id; + u8 sl; + + tbl_idx = cqe->wr_id; + + sqp_entry = &rdev->gsi_ctx.sqp_tbl[tbl_idx]; + gsi_qp = sqp_entry->qp1_qp; + orig_cqe = &sqp_entry->cqe; + + wc->wr_id = sqp_entry->wrid; + wc->byte_len = orig_cqe->length; + wc->qp = &gsi_qp->ib_qp; + + wc->ex.imm_data = orig_cqe->immdata; + wc->src_qp = orig_cqe->src_qp; + memcpy(wc->smac, orig_cqe->smac, ETH_ALEN); + if (bnxt_re_is_nonzero_vlanid_pkt(orig_cqe, &vlan_id, &sl)) { + if (bnxt_re_check_if_vlan_valid(rdev, vlan_id)) { + wc->sl = sl; + wc->vlan_id = vlan_id; + wc->wc_flags |= IB_WC_WITH_VLAN; + } + } + wc->port_num = 1; + wc->vendor_err = orig_cqe->status; + + wc->opcode = IB_WC_RECV; + wc->status = __rawqp1_to_ib_wc_status(orig_cqe->status); + wc->wc_flags |= IB_WC_GRH; + + nw_type = bnxt_re_check_packet_type(orig_cqe->raweth_qp1_flags, + orig_cqe->raweth_qp1_flags2); + if(nw_type >= 0) + dev_dbg(rdev_to_dev(rdev), "%s nw_type = %d\n", __func__, nw_type); +} + +static void bnxt_re_process_res_ud_wc(struct bnxt_re_dev *rdev, + struct bnxt_re_qp *qp, struct ib_wc *wc, + struct bnxt_qplib_cqe *cqe) +{ + u16 vlan_id = 0; + + wc->opcode = IB_WC_RECV; + wc->status = __rc_to_ib_wc_status(cqe->status); + if (cqe->flags & CQ_RES_UD_FLAGS_IMM) + wc->wc_flags |= IB_WC_WITH_IMM; + if (cqe->flags & CQ_RES_RC_FLAGS_INV) + wc->wc_flags |= IB_WC_WITH_INVALIDATE; + /* report only on GSI QP for Thor */ + if (rdev->gsi_ctx.gsi_qp->qplib_qp.id == qp->qplib_qp.id && + rdev->gsi_ctx.gsi_qp_mode == BNXT_RE_GSI_MODE_UD) { + wc->wc_flags |= IB_WC_GRH; + memcpy(wc->smac, cqe->smac, ETH_ALEN); + wc->wc_flags |= IB_WC_WITH_SMAC; + if (_is_cqe_v2_supported(rdev->dev_attr->dev_cap_flags)) { + if (cqe->flags & CQ_RES_UD_V2_FLAGS_META_FORMAT_MASK) { + if (cqe->cfa_meta & + BNXT_QPLIB_CQE_CFA_META1_VALID) + vlan_id = (cqe->cfa_meta & 0xFFF); + } + } else if (cqe->flags & CQ_RES_UD_FLAGS_META_FORMAT_VLAN) { + vlan_id = (cqe->cfa_meta & 0xFFF); + } + /* Mark only if vlan_id is non zero */ + if (vlan_id && bnxt_re_check_if_vlan_valid(rdev, vlan_id)) { + wc->vlan_id = vlan_id; + wc->wc_flags |= IB_WC_WITH_VLAN; + } + } +} + +static int bnxt_re_legacy_send_phantom_wqe(struct bnxt_re_qp *qp) +{ + struct bnxt_qplib_qp *lib_qp = &qp->qplib_qp; + unsigned long flags; + int rc = 0; + + spin_lock_irqsave(&qp->sq_lock, flags); + + rc = bnxt_re_legacy_bind_fence_mw(lib_qp); + if (!rc) { + lib_qp->sq.phantom_wqe_cnt++; + dev_dbg(&lib_qp->sq.hwq.pdev->dev, + "qp %#x sq->prod %#x sw_prod %#x phantom_wqe_cnt %d\n", + lib_qp->id, lib_qp->sq.hwq.prod, + HWQ_CMP(lib_qp->sq.hwq.prod, &lib_qp->sq.hwq), + lib_qp->sq.phantom_wqe_cnt); + } + + spin_unlock_irqrestore(&qp->sq_lock, flags); + return rc; +} + +int bnxt_re_poll_cq(struct ib_cq *ib_cq, int num_entries, struct ib_wc *wc) +{ + struct bnxt_re_cq *cq = to_bnxt_re(ib_cq, struct bnxt_re_cq, ibcq); + struct bnxt_re_dev *rdev = cq->rdev; + struct bnxt_re_qp *qp; + struct bnxt_qplib_cqe *cqe; + int i, ncqe, budget, init_budget; + struct bnxt_qplib_q *sq; + struct bnxt_qplib_qp *lib_qp; + u32 tbl_idx; + struct bnxt_re_sqp_entries *sqp_entry = NULL; + unsigned long flags; + u8 gsi_mode; + + /* + * DB recovery CQ; only process the door bell pacing alert from + * the user lib + */ + if (cq->is_dbr_soft_cq) { + bnxt_re_pacing_alert(rdev); + return 0; + } + + /* User CQ; the only processing we do is to + * complete any pending CQ resize operation. + */ + if (cq->umem) { + if (cq->resize_umem) + bnxt_re_resize_cq_complete(cq); + return 0; + } + + spin_lock_irqsave(&cq->cq_lock, flags); + + budget = min_t(u32, num_entries, cq->max_cql); + init_budget = budget; + if (!cq->cql) { + dev_err(rdev_to_dev(rdev), "POLL CQ no CQL to use\n"); + goto exit; + } + cqe = &cq->cql[0]; + gsi_mode = rdev->gsi_ctx.gsi_qp_mode; + while (budget) { + lib_qp = NULL; + ncqe = bnxt_qplib_poll_cq(&cq->qplib_cq, cqe, budget, &lib_qp); + if (lib_qp) { + sq = &lib_qp->sq; + if (sq->legacy_send_phantom == true) { + qp = container_of(lib_qp, struct bnxt_re_qp, qplib_qp); + if (bnxt_re_legacy_send_phantom_wqe(qp) == -ENOMEM) + dev_err(rdev_to_dev(rdev), + "Phantom failed! Scheduled to send again\n"); + else + sq->legacy_send_phantom = false; + } + } + if (ncqe < budget) + ncqe += bnxt_qplib_process_flush_list(&cq->qplib_cq, + cqe + ncqe, + budget - ncqe); + + if (!ncqe) + break; + + for (i = 0; i < ncqe; i++, cqe++) { + /* Transcribe each qplib_wqe back to ib_wc */ + memset(wc, 0, sizeof(*wc)); + + wc->wr_id = cqe->wr_id; + wc->byte_len = cqe->length; + qp = to_bnxt_re((struct bnxt_qplib_qp *)cqe->qp_handle, + struct bnxt_re_qp, qplib_qp); + if (!qp) { + dev_err(rdev_to_dev(rdev), + "POLL CQ bad QP handle\n"); + continue; + } + wc->qp = &qp->ib_qp; + wc->ex.imm_data = cqe->immdata; + wc->src_qp = cqe->src_qp; + memcpy(wc->smac, cqe->smac, ETH_ALEN); + wc->port_num = 1; + wc->vendor_err = cqe->status; + + switch(cqe->opcode) { + case CQ_BASE_CQE_TYPE_REQ: + if (gsi_mode == BNXT_RE_GSI_MODE_ALL && + qp->qplib_qp.id == + rdev->gsi_ctx.gsi_sqp->qplib_qp.id) { + /* Handle this completion with + * the stored completion */ + dev_dbg(rdev_to_dev(rdev), + "Skipping this UD Send CQ\n"); + memset(wc, 0, sizeof(*wc)); + continue; + } + bnxt_re_process_req_wc(wc, cqe); + break; + case CQ_BASE_CQE_TYPE_RES_RAWETH_QP1: + if (gsi_mode == BNXT_RE_GSI_MODE_ALL) { + if (!cqe->status) { + int rc = 0; + rc = bnxt_re_process_raw_qp_packet_receive(qp, cqe); + if (!rc) { + memset(wc, 0, + sizeof(*wc)); + continue; + } + cqe->status = -1; + } + /* Errors need not be looped back. + * But change the wr_id to the one + * stored in the table + */ + tbl_idx = cqe->wr_id; + sqp_entry = &rdev->gsi_ctx.sqp_tbl[tbl_idx]; + wc->wr_id = sqp_entry->wrid; + } + + bnxt_re_process_res_rawqp1_wc(wc, cqe); + break; + case CQ_BASE_CQE_TYPE_RES_RC: + bnxt_re_process_res_rc_wc(wc, cqe); + break; + case CQ_BASE_CQE_TYPE_RES_UD: + if (gsi_mode == BNXT_RE_GSI_MODE_ALL && + qp->qplib_qp.id == + rdev->gsi_ctx.gsi_sqp->qplib_qp.id) { + /* Handle this completion with + * the stored completion + */ + dev_dbg(rdev_to_dev(rdev), + "Handling the UD receive CQ\n"); + if (cqe->status) { + /* TODO handle this completion as a failure in + * loopback porocedure + */ + continue; + } else { + bnxt_re_process_res_shadow_qp_wc(qp, wc, cqe); + break; + } + } + bnxt_re_process_res_ud_wc(rdev, qp, wc, cqe); + break; + default: + dev_err(rdev_to_dev(cq->rdev), + "POLL CQ type 0x%x not handled, skip!\n", + cqe->opcode); + continue; + } + wc++; + budget--; + } + } +exit: + spin_unlock_irqrestore(&cq->cq_lock, flags); + return init_budget - budget; +} + +int bnxt_re_req_notify_cq(struct ib_cq *ib_cq, + enum ib_cq_notify_flags ib_cqn_flags) +{ + struct bnxt_re_cq *cq = to_bnxt_re(ib_cq, struct bnxt_re_cq, ibcq); + int type = 0, rc = 0; + unsigned long flags; + + spin_lock_irqsave(&cq->cq_lock, flags); + /* Trigger on the very next completion */ + if (ib_cqn_flags & IB_CQ_NEXT_COMP) + type = DBC_DBC_TYPE_CQ_ARMALL; + /* Trigger on the next solicited completion */ + else if (ib_cqn_flags & IB_CQ_SOLICITED) + type = DBC_DBC_TYPE_CQ_ARMSE; + + bnxt_qplib_req_notify_cq(&cq->qplib_cq, type); + + /* Poll to see if there are missed events */ + if ((ib_cqn_flags & IB_CQ_REPORT_MISSED_EVENTS) && + !(bnxt_qplib_is_cq_empty(&cq->qplib_cq))) + rc = 1; + + spin_unlock_irqrestore(&cq->cq_lock, flags); + + return rc; +} + +/* Memory Regions */ +struct ib_mr *bnxt_re_get_dma_mr(struct ib_pd *ib_pd, int mr_access_flags) +{ + struct bnxt_qplib_mrinfo mrinfo; + struct bnxt_re_dev *rdev; + struct bnxt_re_mr *mr; + struct bnxt_re_pd *pd; + u32 max_mr_count; + u64 pbl = 0; + int rc; + + memset(&mrinfo, 0, sizeof(mrinfo)); + pd = to_bnxt_re(ib_pd, struct bnxt_re_pd, ibpd); + rdev = pd->rdev; + + mr = kzalloc(sizeof(*mr), GFP_KERNEL); + if (!mr) { + dev_err(rdev_to_dev(rdev), + "Allocate memory for DMA MR failed!\n"); + return ERR_PTR(-ENOMEM); + } + mr->rdev = rdev; + mr->qplib_mr.pd = &pd->qplib_pd; + mr->qplib_mr.flags = __from_ib_access_flags(mr_access_flags); + mr->qplib_mr.type = CMDQ_ALLOCATE_MRW_MRW_FLAGS_PMR; + + /* Allocate and register 0 as the address */ + rc = bnxt_qplib_alloc_mrw(&rdev->qplib_res, &mr->qplib_mr); + if (rc) { + dev_err(rdev_to_dev(rdev), "Allocate DMA MR failed!\n"); + goto fail; + } + mr->qplib_mr.total_size = -1; /* Infinite length */ + mrinfo.ptes = &pbl; + mrinfo.sg.npages = 0; + mrinfo.sg.pgsize = PAGE_SIZE; + mrinfo.sg.pgshft = PAGE_SHIFT; + mrinfo.sg.pgsize = PAGE_SIZE; + mrinfo.mrw = &mr->qplib_mr; + mrinfo.is_dma = true; + rc = bnxt_qplib_reg_mr(&rdev->qplib_res, &mrinfo, false); + if (rc) { + dev_err(rdev_to_dev(rdev), "Register DMA MR failed!\n"); + goto fail_mr; + } + mr->ib_mr.lkey = mr->qplib_mr.lkey; + if (mr_access_flags & (IB_ACCESS_REMOTE_WRITE | IB_ACCESS_REMOTE_READ | + IB_ACCESS_REMOTE_ATOMIC)) + mr->ib_mr.rkey = mr->ib_mr.lkey; + atomic_inc(&rdev->stats.rsors.mr_count); + max_mr_count = atomic_read(&rdev->stats.rsors.mr_count); + if (max_mr_count > atomic_read(&rdev->stats.rsors.max_mr_count)) + atomic_set(&rdev->stats.rsors.max_mr_count, max_mr_count); + + return &mr->ib_mr; + +fail_mr: + bnxt_qplib_free_mrw(&rdev->qplib_res, &mr->qplib_mr); +fail: + kfree(mr); + return ERR_PTR(rc); +} + +int bnxt_re_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata) +{ + struct bnxt_re_mr *mr = to_bnxt_re(ib_mr, struct bnxt_re_mr, ib_mr); + struct bnxt_re_dev *rdev = mr->rdev; + int rc = 0; + + rc = bnxt_qplib_free_mrw(&rdev->qplib_res, &mr->qplib_mr); + if (rc) + dev_err(rdev_to_dev(rdev), "Dereg MR failed (%d): rc - %#x\n", + mr->qplib_mr.lkey, rc); + + if (mr->pages) { + bnxt_qplib_free_fast_reg_page_list(&rdev->qplib_res, + &mr->qplib_frpl); + kfree(mr->pages); + mr->npages = 0; + mr->pages = NULL; + } + if (!IS_ERR(mr->ib_umem) && mr->ib_umem) { + mr->is_invalcb_active = false; + bnxt_re_peer_mem_release(mr->ib_umem); + } + kfree(mr); + atomic_dec(&rdev->stats.rsors.mr_count); + return 0; +} + +static int bnxt_re_set_page(struct ib_mr *ib_mr, u64 addr) +{ + struct bnxt_re_mr *mr = to_bnxt_re(ib_mr, struct bnxt_re_mr, ib_mr); + + if (unlikely(mr->npages == mr->qplib_frpl.max_pg_ptrs)) + return -ENOMEM; + + mr->pages[mr->npages++] = addr; + dev_dbg(NULL, "%s: ibdev %p Set MR pages[%d] = 0x%lx\n", + ROCE_DRV_MODULE_NAME, ib_mr->device, mr->npages - 1, + mr->pages[mr->npages - 1]); + return 0; +} + +int bnxt_re_map_mr_sg(struct ib_mr *ib_mr, struct scatterlist *sg, + int sg_nents, unsigned int *sg_offset) +{ + struct bnxt_re_mr *mr = to_bnxt_re(ib_mr, struct bnxt_re_mr, ib_mr); + + mr->npages = 0; + return ib_sg_to_pages(ib_mr, sg, sg_nents, + sg_offset, bnxt_re_set_page); +} + +struct ib_mr *bnxt_re_alloc_mr(struct ib_pd *ib_pd, enum ib_mr_type type, + u32 max_num_sg, struct ib_udata *udata) +{ + struct bnxt_re_pd *pd = to_bnxt_re(ib_pd, struct bnxt_re_pd, ibpd); + struct bnxt_re_dev *rdev = pd->rdev; + struct bnxt_re_mr *mr; + u32 max_mr_count; + int rc; + + dev_dbg(rdev_to_dev(rdev), "Alloc MR\n"); + if (type != IB_MR_TYPE_MEM_REG) { + dev_dbg(rdev_to_dev(rdev), "MR type 0x%x not supported\n", type); + return ERR_PTR(-EINVAL); + } + if (max_num_sg > MAX_PBL_LVL_1_PGS) { + dev_dbg(rdev_to_dev(rdev), "Max SG exceeded\n"); + return ERR_PTR(-EINVAL); + } + mr = kzalloc(sizeof(*mr), GFP_KERNEL); + if (!mr) { + dev_err(rdev_to_dev(rdev), "Allocate MR mem failed!\n"); + return ERR_PTR(-ENOMEM); + } + mr->rdev = rdev; + mr->qplib_mr.pd = &pd->qplib_pd; + mr->qplib_mr.flags = BNXT_QPLIB_FR_PMR; + mr->qplib_mr.type = CMDQ_ALLOCATE_MRW_MRW_FLAGS_PMR; + + rc = bnxt_qplib_alloc_mrw(&rdev->qplib_res, &mr->qplib_mr); + if (rc) { + dev_err(rdev_to_dev(rdev), "Allocate MR failed!\n"); + goto fail; + } + mr->ib_mr.lkey = mr->qplib_mr.lkey; + mr->ib_mr.rkey = mr->ib_mr.lkey; + mr->pages = kzalloc(sizeof(u64) * max_num_sg, GFP_KERNEL); + if (!mr->pages) { + dev_err(rdev_to_dev(rdev), + "Allocate MR page list mem failed!\n"); + rc = -ENOMEM; + goto fail_mr; + } + rc = bnxt_qplib_alloc_fast_reg_page_list(&rdev->qplib_res, + &mr->qplib_frpl, max_num_sg); + if (rc) { + dev_err(rdev_to_dev(rdev), + "Allocate HW Fast reg page list failed!\n"); + goto free_page; + } + dev_dbg(rdev_to_dev(rdev), "Alloc MR pages = 0x%p\n", mr->pages); + + atomic_inc(&rdev->stats.rsors.mr_count); + max_mr_count = atomic_read(&rdev->stats.rsors.mr_count); + if (max_mr_count > atomic_read(&rdev->stats.rsors.max_mr_count)) + atomic_set(&rdev->stats.rsors.max_mr_count, max_mr_count); + return &mr->ib_mr; + +free_page: + kfree(mr->pages); +fail_mr: + bnxt_qplib_free_mrw(&rdev->qplib_res, &mr->qplib_mr); +fail: + kfree(mr); + return ERR_PTR(rc); +} + +/* Memory Windows */ +struct ib_mw *bnxt_re_alloc_mw(struct ib_pd *ib_pd, enum ib_mw_type type, + struct ib_udata *udata) +{ + struct bnxt_re_pd *pd = to_bnxt_re(ib_pd, struct bnxt_re_pd, ibpd); + struct bnxt_re_dev *rdev = pd->rdev; + struct bnxt_re_mw *mw; + u32 max_mw_count; + int rc; + + mw = kzalloc(sizeof(*mw), GFP_KERNEL); + if (!mw) { + dev_err(rdev_to_dev(rdev), "Allocate MW failed!\n"); + rc = -ENOMEM; + goto exit; + } + mw->rdev = rdev; + mw->qplib_mw.pd = &pd->qplib_pd; + + mw->qplib_mw.type = (type == IB_MW_TYPE_1 ? + CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE1 : + CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE2B); + rc = bnxt_qplib_alloc_mrw(&rdev->qplib_res, &mw->qplib_mw); + if (rc) { + dev_err(rdev_to_dev(rdev), "Allocate MW failed!\n"); + goto fail; + } + mw->ib_mw.rkey = mw->qplib_mw.rkey; + atomic_inc(&rdev->stats.rsors.mw_count); + max_mw_count = atomic_read(&rdev->stats.rsors.mw_count); + if (max_mw_count > atomic_read(&rdev->stats.rsors.max_mw_count)) + atomic_set(&rdev->stats.rsors.max_mw_count, max_mw_count); + + return &mw->ib_mw; +fail: + kfree(mw); +exit: + return ERR_PTR(rc); +} + +int bnxt_re_dealloc_mw(struct ib_mw *ib_mw) +{ + struct bnxt_re_mw *mw = to_bnxt_re(ib_mw, struct bnxt_re_mw, ib_mw); + struct bnxt_re_dev *rdev = mw->rdev; + int rc; + + rc = bnxt_qplib_free_mrw(&rdev->qplib_res, &mw->qplib_mw); + if (rc) { + dev_err(rdev_to_dev(rdev), "Free MW failed: %#x\n", rc); + return rc; + } + + kfree(mw); + atomic_dec(&rdev->stats.rsors.mw_count); + return rc; +} + +static int bnxt_re_page_size_ok(int page_shift) +{ + switch (page_shift) { + case CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_4K: + case CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_8K: + case CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_64K: + case CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_2M: + case CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_256K: + case CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_1M: + case CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_4M: + case CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_256MB: + case CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_1G: + return 1; + default: + return 0; + } +} + +static int bnxt_re_get_page_shift(struct ib_umem *umem, + u64 va, u64 st, u64 cmask) +{ + int pgshft; + + pgshft = ilog2(umem->page_size); + + return pgshft; +} + +static int bnxt_re_get_num_pages(struct ib_umem *umem, u64 start, u64 length, int page_shift) +{ + int npages = 0; + + if (page_shift == PAGE_SHIFT) { + npages = ib_umem_num_pages_compat(umem); + } else { + npages = ALIGN(length, BIT(page_shift)) / BIT(page_shift); + if (start % BIT(page_shift)) + npages++; + } + return npages; +} + +/* uverbs */ +struct ib_mr *bnxt_re_reg_user_mr(struct ib_pd *ib_pd, u64 start, u64 length, + u64 virt_addr, int mr_access_flags, + struct ib_udata *udata) +{ + struct bnxt_re_pd *pd = to_bnxt_re(ib_pd, struct bnxt_re_pd, ibpd); + struct bnxt_re_dev *rdev = pd->rdev; + struct bnxt_qplib_mrinfo mrinfo; + int umem_pgs, page_shift, rc; + struct bnxt_re_mr *mr; + struct ib_umem *umem; + u32 max_mr_count; + int npages; + + dev_dbg(rdev_to_dev(rdev), "Reg user MR\n"); + + if (bnxt_re_get_total_mr_mw_count(rdev) >= rdev->dev_attr->max_mr) + return ERR_PTR(-ENOMEM); + + if (rdev->mod_exit) { + dev_dbg(rdev_to_dev(rdev), "%s(): in mod_exit, just return!\n", __func__); + return ERR_PTR(-EIO); + } + memset(&mrinfo, 0, sizeof(mrinfo)); + if (length > BNXT_RE_MAX_MR_SIZE) { + dev_err(rdev_to_dev(rdev), "Requested MR Size: %lu " + "> Max supported: %ld\n", length, BNXT_RE_MAX_MR_SIZE); + return ERR_PTR(-ENOMEM); + } + mr = kzalloc(sizeof(*mr), GFP_KERNEL); + if (!mr) { + dev_err(rdev_to_dev(rdev), "Allocate MR failed!\n"); + return ERR_PTR (-ENOMEM); + } + mr->rdev = rdev; + mr->qplib_mr.pd = &pd->qplib_pd; + mr->qplib_mr.flags = __from_ib_access_flags(mr_access_flags); + mr->qplib_mr.type = CMDQ_ALLOCATE_MRW_MRW_FLAGS_MR; + + if (!_is_alloc_mr_unified(rdev->qplib_res.dattr)) { + rc = bnxt_qplib_alloc_mrw(&rdev->qplib_res, &mr->qplib_mr); + if (rc) { + dev_err(rdev_to_dev(rdev), "Alloc MR failed!\n"); + goto fail; + } + /* The fixed portion of the rkey is the same as the lkey */ + mr->ib_mr.rkey = mr->qplib_mr.rkey; + } + + umem = ib_umem_get_flags_compat(rdev, ib_pd->uobject->context, + udata, start, length, + mr_access_flags, 0); + if (IS_ERR(umem)) { + rc = PTR_ERR(umem); + dev_err(rdev_to_dev(rdev), "%s: ib_umem_get failed! rc = %d\n", + __func__, rc); + goto free_mr; + } + mr->ib_umem = umem; + + mr->qplib_mr.va = virt_addr; + umem_pgs = ib_umem_num_pages_compat(umem); + if (!umem_pgs) { + dev_err(rdev_to_dev(rdev), "umem is invalid!\n"); + rc = -EINVAL; + goto free_umem; + } + mr->qplib_mr.total_size = length; + page_shift = bnxt_re_get_page_shift(umem, virt_addr, start, + rdev->dev_attr->page_size_cap); + if (!bnxt_re_page_size_ok(page_shift)) { + dev_err(rdev_to_dev(rdev), "umem page size unsupported!\n"); + rc = -EFAULT; + goto free_umem; + } + npages = bnxt_re_get_num_pages(umem, start, length, page_shift); + + /* Map umem buf ptrs to the PBL */ + mrinfo.sg.npages = npages; + mrinfo.sg.sghead = get_ib_umem_sgl(umem, &mrinfo.sg.nmap); + mrinfo.sg.pgshft = page_shift; + mrinfo.sg.pgsize = BIT(page_shift); + + mrinfo.mrw = &mr->qplib_mr; + + rc = bnxt_qplib_reg_mr(&rdev->qplib_res, &mrinfo, false); + if (rc) { + dev_err(rdev_to_dev(rdev), "Reg user MR failed!\n"); + goto free_umem; + } + + mr->ib_mr.lkey = mr->ib_mr.rkey = mr->qplib_mr.lkey; + atomic_inc(&rdev->stats.rsors.mr_count); + max_mr_count = atomic_read(&rdev->stats.rsors.mr_count); + if (max_mr_count > atomic_read(&rdev->stats.rsors.max_mr_count)) + atomic_set(&rdev->stats.rsors.max_mr_count, max_mr_count); + + return &mr->ib_mr; + +free_umem: + bnxt_re_peer_mem_release(mr->ib_umem); +free_mr: + if (!_is_alloc_mr_unified(rdev->qplib_res.dattr)) + bnxt_qplib_free_mrw(&rdev->qplib_res, &mr->qplib_mr); +fail: + kfree(mr); + return ERR_PTR(rc); +} + +int +bnxt_re_rereg_user_mr(struct ib_mr *ib_mr, int flags, u64 start, u64 length, + u64 virt_addr, int mr_access_flags, + struct ib_pd *ib_pd, struct ib_udata *udata) +{ + struct bnxt_re_mr *mr = to_bnxt_re(ib_mr, struct bnxt_re_mr, ib_mr); + struct bnxt_re_pd *pd = to_bnxt_re(ib_pd, struct bnxt_re_pd, ibpd); + int umem_pgs = 0, page_shift = PAGE_SHIFT, rc; + struct bnxt_re_dev *rdev = mr->rdev; + struct bnxt_qplib_mrinfo mrinfo; + struct ib_umem *umem; + u32 npages; + + /* TODO: Must decipher what to modify based on the flags */ + memset(&mrinfo, 0, sizeof(mrinfo)); + if (flags & IB_MR_REREG_TRANS) { + umem = ib_umem_get_flags_compat(rdev, ib_pd->uobject->context, + udata, start, length, + mr_access_flags, 0); + if (IS_ERR(umem)) { + rc = PTR_ERR(umem); + dev_err(rdev_to_dev(rdev), + "%s: ib_umem_get failed! ret = %d\n", + __func__, rc); + goto fail; + } + mr->ib_umem = umem; + + mr->qplib_mr.va = virt_addr; + umem_pgs = ib_umem_num_pages_compat(umem); + if (!umem_pgs) { + dev_err(rdev_to_dev(rdev), "umem is invalid!\n"); + rc = -EINVAL; + goto fail_free_umem; + } + mr->qplib_mr.total_size = length; + page_shift = bnxt_re_get_page_shift(umem, virt_addr, start, + rdev->dev_attr->page_size_cap); + if (!bnxt_re_page_size_ok(page_shift)) { + dev_err(rdev_to_dev(rdev), + "umem page size unsupported!\n"); + rc = -EFAULT; + goto fail_free_umem; + } + npages = bnxt_re_get_num_pages(umem, start, length, page_shift); + /* Map umem buf ptrs to the PBL */ + mrinfo.sg.npages = npages; + mrinfo.sg.sghead = get_ib_umem_sgl(umem, &mrinfo.sg.nmap); + mrinfo.sg.pgshft = page_shift; + mrinfo.sg.pgsize = BIT(page_shift); + } + + mrinfo.mrw = &mr->qplib_mr; + if (flags & IB_MR_REREG_PD) + mr->qplib_mr.pd = &pd->qplib_pd; + + if (flags & IB_MR_REREG_ACCESS) + mr->qplib_mr.flags = __from_ib_access_flags(mr_access_flags); + + rc = bnxt_qplib_reg_mr(&rdev->qplib_res, &mrinfo, false); + if (rc) { + dev_err(rdev_to_dev(rdev), "Rereg user MR failed!\n"); + goto fail_free_umem; + } + mr->ib_mr.rkey = mr->qplib_mr.rkey; + + return 0; + +fail_free_umem: + bnxt_re_peer_mem_release(mr->ib_umem); +fail: + return rc; +} + +static int bnxt_re_check_abi_version(struct bnxt_re_dev *rdev) +{ + struct ib_device *ibdev = &rdev->ibdev; + u32 uverbs_abi_ver; + + uverbs_abi_ver = GET_UVERBS_ABI_VERSION(ibdev); + dev_dbg(rdev_to_dev(rdev), "ABI version requested %d\n", + uverbs_abi_ver); + if (uverbs_abi_ver != BNXT_RE_ABI_VERSION) { + dev_dbg(rdev_to_dev(rdev), " is different from the device %d \n", + BNXT_RE_ABI_VERSION); + return -EPERM; + } + return 0; +} + +int bnxt_re_alloc_ucontext(struct ib_ucontext *uctx_in, + struct ib_udata *udata) +{ + struct ib_ucontext *ctx = uctx_in; + struct ib_device *ibdev = ctx->device; + struct bnxt_re_ucontext *uctx = + container_of(ctx, struct bnxt_re_ucontext, ibucontext); + + struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev); + struct bnxt_qplib_dev_attr *dev_attr = rdev->dev_attr; + struct bnxt_re_uctx_resp resp = {}; + struct bnxt_re_uctx_req ureq = {}; + struct bnxt_qplib_chip_ctx *cctx; + u32 chip_met_rev_num; + bool genp5 = false; + int rc; + + cctx = rdev->chip_ctx; + rc = bnxt_re_check_abi_version(rdev); + if (rc) + goto fail; + + uctx->rdev = rdev; + uctx->shpg = (void *)__get_free_page(GFP_KERNEL); + if (!uctx->shpg) { + dev_err(rdev_to_dev(rdev), "shared memory allocation failed!\n"); + rc = -ENOMEM; + goto fail; + } + spin_lock_init(&uctx->sh_lock); + if (BNXT_RE_ABI_VERSION >= 4) { + chip_met_rev_num = cctx->chip_num; + chip_met_rev_num |= ((u32)cctx->chip_rev & 0xFF) << + BNXT_RE_CHIP_ID0_CHIP_REV_SFT; + chip_met_rev_num |= ((u32)cctx->chip_metal & 0xFF) << + BNXT_RE_CHIP_ID0_CHIP_MET_SFT; + resp.chip_id0 = chip_met_rev_num; + resp.chip_id1 = 0; /* future extension of chip info */ + } + + if (BNXT_RE_ABI_VERSION != 4) { + /*Temp, Use idr_alloc instead*/ + resp.dev_id = rdev->en_dev->pdev->devfn; + resp.max_qp = rdev->qplib_res.hctx->qp_ctx.max; + } + + genp5 = _is_chip_gen_p5_p7(cctx); + if (BNXT_RE_ABI_VERSION > 5) { + resp.modes = genp5 ? cctx->modes.wqe_mode : 0; + if (rdev->dev_attr && BNXT_RE_HW_RETX(rdev->dev_attr->dev_cap_flags)) + resp.comp_mask = BNXT_RE_COMP_MASK_UCNTX_HW_RETX_ENABLED; + } + + resp.pg_size = PAGE_SIZE; + resp.cqe_sz = sizeof(struct cq_base); + resp.max_cqd = dev_attr->max_cq_wqes; + if (genp5 && cctx->modes.db_push) { + resp.comp_mask |= BNXT_RE_COMP_MASK_UCNTX_WC_DPI_ENABLED; + if (_is_chip_p7(cctx) && + !(dev_attr->dev_cap_flags & + CREQ_QUERY_FUNC_RESP_SB_PINGPONG_PUSH_MODE)) + resp.comp_mask &= + ~BNXT_RE_COMP_MASK_UCNTX_WC_DPI_ENABLED; + } + + resp.comp_mask |= BNXT_RE_COMP_MASK_UCNTX_MQP_EX_SUPPORTED; + + if (rdev->dbr_pacing) + resp.comp_mask |= BNXT_RE_COMP_MASK_UCNTX_DBR_PACING_ENABLED; + + if (rdev->dbr_drop_recov && rdev->user_dbr_drop_recov) + resp.comp_mask |= BNXT_RE_COMP_MASK_UCNTX_DBR_RECOVERY_ENABLED; + + if (udata->inlen >= sizeof(ureq)) { + rc = ib_copy_from_udata(&ureq, udata, + min(udata->inlen, sizeof(ureq))); + if (rc) + goto cfail; + if (bnxt_re_init_pow2_flag(&ureq, &resp)) + dev_warn(rdev_to_dev(rdev), + "Enabled roundup logic. Library bug?\n"); + if (bnxt_re_init_rsvd_wqe_flag(&ureq, &resp, genp5)) + dev_warn(rdev_to_dev(rdev), + "Rsvd wqe in use! Try the updated library.\n"); + } else { + dev_warn(rdev_to_dev(rdev), + "Enabled roundup logic. Update the library!\n"); + resp.comp_mask &= ~BNXT_RE_COMP_MASK_UCNTX_POW2_DISABLED; + + dev_warn(rdev_to_dev(rdev), + "Rsvd wqe in use. Update the library!\n"); + resp.comp_mask &= ~BNXT_RE_COMP_MASK_UCNTX_RSVD_WQE_DISABLED; + } + + uctx->cmask = (uint64_t)resp.comp_mask; + rc = bnxt_re_copy_to_udata(rdev, &resp, + min(udata->outlen, sizeof(resp)), + udata); + if (rc) + goto cfail; + + INIT_LIST_HEAD(&uctx->cq_list); + mutex_init(&uctx->cq_lock); + + return 0; +cfail: + free_page((u64)uctx->shpg); + uctx->shpg = NULL; +fail: + return rc; +} + +void bnxt_re_dealloc_ucontext(struct ib_ucontext *ib_uctx) +{ + struct bnxt_re_ucontext *uctx = to_bnxt_re(ib_uctx, + struct bnxt_re_ucontext, + ibucontext); + struct bnxt_re_dev *rdev = uctx->rdev; + int rc = 0; + + if (uctx->shpg) + free_page((u64)uctx->shpg); + + if (uctx->dpi.dbr) { + /* Free DPI only if this is the first PD allocated by the + * application and mark the context dpi as NULL + */ + if (_is_chip_gen_p5_p7(rdev->chip_ctx) && uctx->wcdpi.dbr) { + rc = bnxt_qplib_dealloc_dpi(&rdev->qplib_res, + &uctx->wcdpi); + if (rc) + dev_err(rdev_to_dev(rdev), + "dealloc push dp failed\n"); + uctx->wcdpi.dbr = NULL; + } + + rc = bnxt_qplib_dealloc_dpi(&rdev->qplib_res, + &uctx->dpi); + if (rc) + dev_err(rdev_to_dev(rdev), "Deallocte HW DPI failed!\n"); + /* Don't fail, continue*/ + uctx->dpi.dbr = NULL; + } + return; +} + +static struct bnxt_re_cq *is_bnxt_re_cq_page(struct bnxt_re_ucontext *uctx, + u64 pg_off) +{ + struct bnxt_re_cq *cq = NULL, *tmp_cq; + + if (!_is_chip_p7(uctx->rdev->chip_ctx)) + return NULL; + + mutex_lock(&uctx->cq_lock); + list_for_each_entry(tmp_cq, &uctx->cq_list, cq_list) { + if (((u64)tmp_cq->uctx_cq_page >> PAGE_SHIFT) == pg_off) { + cq = tmp_cq; + break; + } + } + mutex_unlock(&uctx->cq_lock); + return cq; +} + +/* Helper function to mmap the virtual memory from user app */ +int bnxt_re_mmap(struct ib_ucontext *ib_uctx, struct vm_area_struct *vma) +{ + struct bnxt_re_ucontext *uctx = to_bnxt_re(ib_uctx, + struct bnxt_re_ucontext, + ibucontext); + struct bnxt_re_dev *rdev = uctx->rdev; + struct bnxt_re_cq *cq = NULL; + int rc = 0; + u64 pfn; + + switch (vma->vm_pgoff) { + case BNXT_RE_MAP_SH_PAGE: + pfn = vtophys(uctx->shpg) >> PAGE_SHIFT; + return rdma_user_mmap_io(&uctx->ibucontext, vma, pfn, PAGE_SIZE, vma->vm_page_prot, NULL); + dev_dbg(rdev_to_dev(rdev), "%s:%d uctx->shpg 0x%lx, vtophys(uctx->shpg) 0x%lx, pfn = 0x%lx \n", + __func__, __LINE__, (u64) uctx->shpg, vtophys(uctx->shpg), pfn); + if (rc) { + dev_err(rdev_to_dev(rdev), "Shared page mapping failed!\n"); + rc = -EAGAIN; + } + return rc; + case BNXT_RE_MAP_WC: + vma->vm_page_prot = + pgprot_writecombine(vma->vm_page_prot); + pfn = (uctx->wcdpi.umdbr >> PAGE_SHIFT); + if (!pfn) + return -EFAULT; + break; + case BNXT_RE_DBR_PAGE: + /* Driver doesn't expect write access request */ + if (vma->vm_flags & VM_WRITE) + return -EFAULT; + + pfn = vtophys(rdev->dbr_page) >> PAGE_SHIFT; + if (!pfn) + return -EFAULT; + break; + case BNXT_RE_MAP_DB_RECOVERY_PAGE: + pfn = vtophys(uctx->dbr_recov_cq_page) >> PAGE_SHIFT; + if (!pfn) + return -EFAULT; + break; + default: + cq = is_bnxt_re_cq_page(uctx, vma->vm_pgoff); + if (cq) { + pfn = vtophys((void *)cq->uctx_cq_page) >> PAGE_SHIFT; + rc = rdma_user_mmap_io(&uctx->ibucontext, vma, pfn, PAGE_SIZE, vma->vm_page_prot, NULL); + if (rc) { + dev_err(rdev_to_dev(rdev), + "CQ page mapping failed!\n"); + rc = -EAGAIN; + } + goto out; + } else { + vma->vm_page_prot = + pgprot_noncached(vma->vm_page_prot); + pfn = vma->vm_pgoff; + } + break; + } + + rc = rdma_user_mmap_io(&uctx->ibucontext, vma, pfn, PAGE_SIZE, vma->vm_page_prot, NULL); + if (rc) { + dev_err(rdev_to_dev(rdev), "DPI mapping failed!\n"); + return -EAGAIN; + } + rc = __bnxt_re_set_vma_data(uctx, vma); +out: + return rc; +} + +int bnxt_re_process_mad(struct ib_device *ibdev, int mad_flags, u8 port_num, + const struct ib_wc *wc, const struct ib_grh *grh, + const struct ib_mad_hdr *in_mad, size_t in_mad_size, + struct ib_mad_hdr *out_mad, size_t *out_mad_size, + u16 *out_mad_pkey_index) +{ + return IB_MAD_RESULT_SUCCESS; +} + +void bnxt_re_disassociate_ucntx(struct ib_ucontext *ib_uctx) +{ +} diff --git a/sys/dev/bnxt/bnxt_re/ib_verbs.h b/sys/dev/bnxt/bnxt_re/ib_verbs.h new file mode 100644 index 000000000000..cb9f7974e92d --- /dev/null +++ b/sys/dev/bnxt/bnxt_re/ib_verbs.h @@ -0,0 +1,632 @@ +/* + * Copyright (c) 2015-2024, Broadcom. All rights reserved. The term + * Broadcom refers to Broadcom Limited and/or its subsidiaries. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR + * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE + * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN + * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * Description: IB Verbs interpreter (header) + */ + +#ifndef __BNXT_RE_IB_VERBS_H__ +#define __BNXT_RE_IB_VERBS_H__ + +#include +#include "bnxt_re-abi.h" +#include "qplib_res.h" +#include "qplib_fp.h" + +struct bnxt_re_dev; + +#define BNXT_RE_ROCE_V2_UDP_SPORT 0x8CD1 +#define BNXT_RE_QP_RANDOM_QKEY 0x81818181 + +#ifndef IB_MTU_8192 +#define IB_MTU_8192 8192 +#endif + +#ifndef SPEED_1000 +#define SPEED_1000 1000 +#endif + +#ifndef SPEED_10000 +#define SPEED_10000 10000 +#endif + +#ifndef SPEED_20000 +#define SPEED_20000 20000 +#endif + +#ifndef SPEED_25000 +#define SPEED_25000 25000 +#endif + +#ifndef SPEED_40000 +#define SPEED_40000 40000 +#endif + +#ifndef SPEED_50000 +#define SPEED_50000 50000 +#endif + +#ifndef SPEED_100000 +#define SPEED_100000 100000 +#endif + +#ifndef SPEED_200000 +#define SPEED_200000 200000 +#endif + +#ifndef IB_SPEED_HDR +#define IB_SPEED_HDR 64 +#endif + +#define RDMA_NETWORK_IPV4 1 +#define RDMA_NETWORK_IPV6 2 + +#define ROCE_DMAC(x) (x)->dmac + +#define dma_rmb() rmb() + +#define compat_ib_alloc_device(size) ib_alloc_device(size); + +#define rdev_from_cq_in(cq_in) to_bnxt_re_dev(cq_in->device, ibdev) + +#define GET_UVERBS_ABI_VERSION(ibdev) (ibdev->uverbs_abi_ver) + +#define CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_256MB 0x1cUL + +#define IB_POLL_UNBOUND_WORKQUEUE IB_POLL_WORKQUEUE + +#define BNXT_RE_LEGACY_FENCE_BYTES 64 +#define BNXT_RE_LEGACY_FENCE_PBL_SIZE DIV_ROUND_UP(BNXT_RE_LEGACY_FENCE_BYTES, PAGE_SIZE) + + +static inline struct +bnxt_re_cq *__get_cq_from_cq_in(struct ib_cq *cq_in, + struct bnxt_re_dev *rdev); +static inline struct +bnxt_re_qp *__get_qp_from_qp_in(struct ib_pd *qp_in, + struct bnxt_re_dev *rdev); + +static inline bool +bnxt_re_check_if_vlan_valid(struct bnxt_re_dev *rdev, u16 vlan_id); + +#define bnxt_re_compat_qfwstr(void) \ + bnxt_re_query_fw_str(struct ib_device *ibdev, \ + char *str, size_t str_len) + +static inline +struct scatterlist *get_ib_umem_sgl(struct ib_umem *umem, u32 *nmap); + +struct bnxt_re_gid_ctx { + u32 idx; + u32 refcnt; +}; + +struct bnxt_re_legacy_fence_data { + u32 size; + void *va; + dma_addr_t dma_addr; + struct bnxt_re_mr *mr; + struct ib_mw *mw; + struct bnxt_qplib_swqe bind_wqe; + u32 bind_rkey; +}; + +struct bnxt_re_pd { + struct ib_pd ibpd; + struct bnxt_re_dev *rdev; + struct bnxt_qplib_pd qplib_pd; + struct bnxt_re_legacy_fence_data fence; +}; + +struct bnxt_re_ah { + struct ib_ah ibah; + struct bnxt_re_dev *rdev; + struct bnxt_qplib_ah qplib_ah; +}; + +struct bnxt_re_srq { + struct ib_srq ibsrq; + struct bnxt_re_dev *rdev; + u32 srq_limit; + struct bnxt_qplib_srq qplib_srq; + struct ib_umem *umem; + spinlock_t lock; +}; + +union ip_addr { + u32 ipv4_addr; + u8 ipv6_addr[16]; +}; + +struct bnxt_re_qp_info_entry { + union ib_gid sgid; + union ib_gid dgid; + union ip_addr s_ip; + union ip_addr d_ip; + u16 s_port; +#define BNXT_RE_QP_DEST_PORT 4791 + u16 d_port; +}; + +struct bnxt_re_qp { + struct ib_qp ib_qp; + struct list_head list; + struct bnxt_re_dev *rdev; + spinlock_t sq_lock; + spinlock_t rq_lock; + struct bnxt_qplib_qp qplib_qp; + struct ib_umem *sumem; + struct ib_umem *rumem; + /* QP1 */ + u32 send_psn; + struct ib_ud_header qp1_hdr; + struct bnxt_re_cq *scq; + struct bnxt_re_cq *rcq; + struct dentry *qp_info_pdev_dentry; + struct bnxt_re_qp_info_entry qp_info_entry; + void *qp_data; +}; + +struct bnxt_re_cq { + struct ib_cq ibcq; + struct list_head cq_list; + struct bnxt_re_dev *rdev; + struct bnxt_re_ucontext *uctx; + spinlock_t cq_lock; + u16 cq_count; + u16 cq_period; + struct bnxt_qplib_cq qplib_cq; + struct bnxt_qplib_cqe *cql; +#define MAX_CQL_PER_POLL 1024 + u32 max_cql; + struct ib_umem *umem; + struct ib_umem *resize_umem; + struct ib_ucontext *context; + int resize_cqe; + /* list of cq per uctx. Used only for Thor-2 */ + void *uctx_cq_page; + void *dbr_recov_cq_page; + bool is_dbr_soft_cq; +}; + +struct bnxt_re_mr { + struct bnxt_re_dev *rdev; + struct ib_mr ib_mr; + struct ib_umem *ib_umem; + struct bnxt_qplib_mrw qplib_mr; + u32 npages; + u64 *pages; + struct bnxt_qplib_frpl qplib_frpl; + bool is_invalcb_active; +}; + +struct bnxt_re_frpl { + struct bnxt_re_dev *rdev; + struct bnxt_qplib_frpl qplib_frpl; + u64 *page_list; +}; + +struct bnxt_re_mw { + struct bnxt_re_dev *rdev; + struct ib_mw ib_mw; + struct bnxt_qplib_mrw qplib_mw; +}; + +struct bnxt_re_ucontext { + struct ib_ucontext ibucontext; + struct bnxt_re_dev *rdev; + struct list_head cq_list; + struct bnxt_qplib_dpi dpi; + struct bnxt_qplib_dpi wcdpi; + void *shpg; + spinlock_t sh_lock; + uint64_t cmask; + struct mutex cq_lock; /* Protect cq list */ + void *dbr_recov_cq_page; + struct bnxt_re_cq *dbr_recov_cq; +}; + +struct bnxt_re_ah_info { + union ib_gid sgid; + struct ib_gid_attr sgid_attr; + u16 vlan_tag; + u8 nw_type; +}; + +struct ifnet *bnxt_re_get_netdev(struct ib_device *ibdev, + u8 port_num); + +int bnxt_re_query_device(struct ib_device *ibdev, + struct ib_device_attr *ib_attr, + struct ib_udata *udata); +int bnxt_re_modify_device(struct ib_device *ibdev, + int device_modify_mask, + struct ib_device_modify *device_modify); +int bnxt_re_query_port(struct ib_device *ibdev, u8 port_num, + struct ib_port_attr *port_attr); +int bnxt_re_modify_port(struct ib_device *ibdev, u8 port_num, + int port_modify_mask, + struct ib_port_modify *port_modify); +int bnxt_re_get_port_immutable(struct ib_device *ibdev, u8 port_num, + struct ib_port_immutable *immutable); +void bnxt_re_compat_qfwstr(void); +int bnxt_re_query_pkey(struct ib_device *ibdev, u8 port_num, + u16 index, u16 *pkey); +int bnxt_re_del_gid(struct ib_device *ibdev, u8 port_num, + unsigned int index, void **context); +int bnxt_re_add_gid(struct ib_device *ibdev, u8 port_num, + unsigned int index, const union ib_gid *gid, + const struct ib_gid_attr *attr, void **context); +int bnxt_re_query_gid(struct ib_device *ibdev, u8 port_num, + int index, union ib_gid *gid); +enum rdma_link_layer bnxt_re_get_link_layer(struct ib_device *ibdev, + u8 port_num); +int bnxt_re_alloc_pd(struct ib_pd *pd_in, struct ib_udata *udata); +void bnxt_re_dealloc_pd(struct ib_pd *ib_pd, struct ib_udata *udata); + +int bnxt_re_create_ah(struct ib_ah *ah_in, struct ib_ah_attr *attr, + u32 flags, struct ib_udata *udata); + +int bnxt_re_modify_ah(struct ib_ah *ah, struct ib_ah_attr *ah_attr); +int bnxt_re_query_ah(struct ib_ah *ah, struct ib_ah_attr *ah_attr); + +void bnxt_re_destroy_ah(struct ib_ah *ib_ah, u32 flags); +int bnxt_re_create_srq(struct ib_srq *srq_in, + struct ib_srq_init_attr *srq_init_attr, + struct ib_udata *udata); +int bnxt_re_modify_srq(struct ib_srq *srq, struct ib_srq_attr *srq_attr, + enum ib_srq_attr_mask srq_attr_mask, + struct ib_udata *udata); +int bnxt_re_query_srq(struct ib_srq *srq, struct ib_srq_attr *srq_attr); +void bnxt_re_destroy_srq(struct ib_srq *ib_srq, + struct ib_udata *udata); +int bnxt_re_post_srq_recv(struct ib_srq *ib_srq, const struct ib_recv_wr *wr, + const struct ib_recv_wr **bad_wr); +struct ib_qp *bnxt_re_create_qp(struct ib_pd *qp_in, + struct ib_qp_init_attr *qp_init_attr, + struct ib_udata *udata); +int bnxt_re_modify_qp(struct ib_qp *qp, struct ib_qp_attr *qp_attr, + int qp_attr_mask, struct ib_udata *udata); +int bnxt_re_query_qp(struct ib_qp *qp, struct ib_qp_attr *qp_attr, + int qp_attr_mask, struct ib_qp_init_attr *qp_init_attr); +int bnxt_re_destroy_qp(struct ib_qp *ib_qp, struct ib_udata *udata); +int bnxt_re_post_send(struct ib_qp *ib_qp, const struct ib_send_wr *wr, + const struct ib_send_wr **bad_wr); +int bnxt_re_post_recv(struct ib_qp *ib_qp, const struct ib_recv_wr *wr, + const struct ib_recv_wr **bad_wr); +int bnxt_re_create_cq(struct ib_cq *cq_in, + const struct ib_cq_init_attr *attr, + struct ib_udata *udata); +void bnxt_re_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata); +int bnxt_re_modify_cq(struct ib_cq *cq, u16 cq_count, u16 cq_period); +int bnxt_re_resize_cq(struct ib_cq *cq, int cqe, struct ib_udata *udata); +int bnxt_re_poll_cq(struct ib_cq *cq, int num_entries, struct ib_wc *wc); +int bnxt_re_req_notify_cq(struct ib_cq *cq, enum ib_cq_notify_flags flags); +struct ib_mr *bnxt_re_get_dma_mr(struct ib_pd *pd, int mr_access_flags); +int bnxt_re_map_mr_sg(struct ib_mr *ib_mr, struct scatterlist *sg, + int sg_nents, unsigned int *sg_offset); +struct ib_mr *bnxt_re_alloc_mr(struct ib_pd *ib_pd, enum ib_mr_type type, + u32 max_num_sg, struct ib_udata *udata); +int bnxt_re_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata); +struct ib_mw *bnxt_re_alloc_mw(struct ib_pd *ib_pd, enum ib_mw_type type, + struct ib_udata *udata); +int bnxt_re_dealloc_mw(struct ib_mw *mw); +struct ib_mr *bnxt_re_reg_user_mr(struct ib_pd *pd, u64 start, u64 length, + u64 virt_addr, int mr_access_flags, + struct ib_udata *udata); +int +bnxt_re_rereg_user_mr(struct ib_mr *mr, int flags, u64 start, u64 length, + u64 virt_addr, int mr_access_flags, struct ib_pd *pd, + struct ib_udata *udata); +int bnxt_re_alloc_ucontext(struct ib_ucontext *uctx_in, + struct ib_udata *udata); +void bnxt_re_dealloc_ucontext(struct ib_ucontext *ib_uctx); +int bnxt_re_mmap(struct ib_ucontext *context, struct vm_area_struct *vma); +int bnxt_re_process_mad(struct ib_device *ibdev, int mad_flags, u8 port_num, + const struct ib_wc *wc, const struct ib_grh *grh, + const struct ib_mad_hdr *in_mad, size_t in_mad_size, + struct ib_mad_hdr *out_mad, size_t *out_mad_size, + u16 *out_mad_pkey_index); +unsigned long bnxt_re_lock_cqs(struct bnxt_re_qp *qp); +void bnxt_re_unlock_cqs(struct bnxt_re_qp *qp, unsigned long flags); +void bnxt_re_disassociate_ucntx(struct ib_ucontext *ibcontext); +static inline int __bnxt_re_set_vma_data(void *bnxt_re_uctx, + struct vm_area_struct *vma); +void bnxt_re_update_shadow_ah(struct bnxt_re_dev *rdev); +void bnxt_re_handle_cqn(struct bnxt_qplib_cq *cq); +static inline int +bnxt_re_get_cached_gid(struct ib_device *dev, u8 port_num, int index, + union ib_gid *sgid, struct ib_gid_attr **sgid_attr, + struct ib_global_route *grh, struct ib_ah *ah); +static inline enum rdma_network_type +bnxt_re_gid_to_network_type(struct ib_gid_attr *sgid_attr, + union ib_gid *sgid); +static inline +struct ib_umem *ib_umem_get_compat(struct bnxt_re_dev *rdev, + struct ib_ucontext *ucontext, + struct ib_udata *udata, + unsigned long addr, + size_t size, int access, int dmasync); +static inline +struct ib_umem *ib_umem_get_flags_compat(struct bnxt_re_dev *rdev, + struct ib_ucontext *ucontext, + struct ib_udata *udata, + unsigned long addr, + size_t size, int access, int dmasync); +static inline size_t ib_umem_num_pages_compat(struct ib_umem *umem); +static inline void bnxt_re_peer_mem_release(struct ib_umem *umem); +void bnxt_re_resolve_dmac_task(struct work_struct *work); + +static inline enum ib_qp_type __from_hw_to_ib_qp_type(u8 type) +{ + switch (type) { + case CMDQ_CREATE_QP1_TYPE_GSI: + case CMDQ_CREATE_QP_TYPE_GSI: + return IB_QPT_GSI; + case CMDQ_CREATE_QP_TYPE_RC: + return IB_QPT_RC; + case CMDQ_CREATE_QP_TYPE_UD: + return IB_QPT_UD; + case CMDQ_CREATE_QP_TYPE_RAW_ETHERTYPE: + return IB_QPT_RAW_ETHERTYPE; + default: + return IB_QPT_MAX; + } +} + +static inline u8 __from_ib_qp_state(enum ib_qp_state state) +{ + switch (state) { + case IB_QPS_RESET: + return CMDQ_MODIFY_QP_NEW_STATE_RESET; + case IB_QPS_INIT: + return CMDQ_MODIFY_QP_NEW_STATE_INIT; + case IB_QPS_RTR: + return CMDQ_MODIFY_QP_NEW_STATE_RTR; + case IB_QPS_RTS: + return CMDQ_MODIFY_QP_NEW_STATE_RTS; + case IB_QPS_SQD: + return CMDQ_MODIFY_QP_NEW_STATE_SQD; + case IB_QPS_SQE: + return CMDQ_MODIFY_QP_NEW_STATE_SQE; + case IB_QPS_ERR: + default: + return CMDQ_MODIFY_QP_NEW_STATE_ERR; + } +} + +static inline u32 __from_ib_mtu(enum ib_mtu mtu) +{ + switch (mtu) { + case IB_MTU_256: + return CMDQ_MODIFY_QP_PATH_MTU_MTU_256; + case IB_MTU_512: + return CMDQ_MODIFY_QP_PATH_MTU_MTU_512; + case IB_MTU_1024: + return CMDQ_MODIFY_QP_PATH_MTU_MTU_1024; + case IB_MTU_2048: + return CMDQ_MODIFY_QP_PATH_MTU_MTU_2048; + case IB_MTU_4096: + return CMDQ_MODIFY_QP_PATH_MTU_MTU_4096; + default: + return CMDQ_MODIFY_QP_PATH_MTU_MTU_2048; + } +} + +static inline enum ib_mtu __to_ib_mtu(u32 mtu) +{ + switch (mtu & CREQ_QUERY_QP_RESP_SB_PATH_MTU_MASK) { + case CMDQ_MODIFY_QP_PATH_MTU_MTU_256: + return IB_MTU_256; + case CMDQ_MODIFY_QP_PATH_MTU_MTU_512: + return IB_MTU_512; + case CMDQ_MODIFY_QP_PATH_MTU_MTU_1024: + return IB_MTU_1024; + case CMDQ_MODIFY_QP_PATH_MTU_MTU_2048: + return IB_MTU_2048; + case CMDQ_MODIFY_QP_PATH_MTU_MTU_4096: + return IB_MTU_4096; + case CMDQ_MODIFY_QP_PATH_MTU_MTU_8192: + return IB_MTU_8192; + default: + return IB_MTU_2048; + } +} + +static inline enum ib_qp_state __to_ib_qp_state(u8 state) +{ + switch (state) { + case CMDQ_MODIFY_QP_NEW_STATE_RESET: + return IB_QPS_RESET; + case CMDQ_MODIFY_QP_NEW_STATE_INIT: + return IB_QPS_INIT; + case CMDQ_MODIFY_QP_NEW_STATE_RTR: + return IB_QPS_RTR; + case CMDQ_MODIFY_QP_NEW_STATE_RTS: + return IB_QPS_RTS; + case CMDQ_MODIFY_QP_NEW_STATE_SQD: + return IB_QPS_SQD; + case CMDQ_MODIFY_QP_NEW_STATE_SQE: + return IB_QPS_SQE; + case CMDQ_MODIFY_QP_NEW_STATE_ERR: + default: + return IB_QPS_ERR; + } +} + +static inline int bnxt_re_init_pow2_flag(struct bnxt_re_uctx_req *req, + struct bnxt_re_uctx_resp *resp) +{ + resp->comp_mask |= BNXT_RE_COMP_MASK_UCNTX_POW2_DISABLED; + if (!(req->comp_mask & BNXT_RE_COMP_MASK_REQ_UCNTX_POW2_SUPPORT)) { + resp->comp_mask &= ~BNXT_RE_COMP_MASK_UCNTX_POW2_DISABLED; + return -EINVAL; + } + return 0; +} + +static inline u32 bnxt_re_init_depth(u32 ent, struct bnxt_re_ucontext *uctx) +{ + return uctx ? (uctx->cmask & BNXT_RE_COMP_MASK_UCNTX_POW2_DISABLED) ? + ent : roundup_pow_of_two(ent) : ent; +} + +static inline int bnxt_re_init_rsvd_wqe_flag(struct bnxt_re_uctx_req *req, + struct bnxt_re_uctx_resp *resp, + bool genp5) +{ + resp->comp_mask |= BNXT_RE_COMP_MASK_UCNTX_RSVD_WQE_DISABLED; + if (!(req->comp_mask & BNXT_RE_COMP_MASK_REQ_UCNTX_RSVD_WQE)) { + resp->comp_mask &= ~BNXT_RE_COMP_MASK_UCNTX_RSVD_WQE_DISABLED; + return -EINVAL; + } else if (!genp5) { + resp->comp_mask &= ~BNXT_RE_COMP_MASK_UCNTX_RSVD_WQE_DISABLED; + } + return 0; +} + +static inline u32 bnxt_re_get_diff(struct bnxt_re_ucontext *uctx, + struct bnxt_qplib_chip_ctx *cctx) +{ + if (!uctx) { + /* return res-wqe only for gen p4 for user resource */ + return _is_chip_gen_p5_p7(cctx) ? 0 : BNXT_QPLIB_RESERVED_QP_WRS; + } else if (uctx->cmask & BNXT_RE_COMP_MASK_UCNTX_RSVD_WQE_DISABLED) { + return 0; + } + /* old lib */ + return BNXT_QPLIB_RESERVED_QP_WRS; +} + +static inline void bnxt_re_init_qpmtu(struct bnxt_re_qp *qp, int mtu, + int mask, struct ib_qp_attr *qp_attr, + bool *is_qpmtu_high) +{ + int qpmtu, qpmtu_int; + int ifmtu, ifmtu_int; + + ifmtu = iboe_get_mtu(mtu); + ifmtu_int = ib_mtu_enum_to_int(ifmtu); + qpmtu = ifmtu; + qpmtu_int = ifmtu_int; + if (mask & IB_QP_PATH_MTU) { + qpmtu = qp_attr->path_mtu; + qpmtu_int = ib_mtu_enum_to_int(qpmtu); + if (qpmtu_int > ifmtu_int) { + /* Trim the QP path mtu to interface mtu and update + * the new mtu to user qp for retransmission psn + * calculations. + */ + qpmtu = ifmtu; + qpmtu_int = ifmtu_int; + *is_qpmtu_high = true; + } + } + qp->qplib_qp.path_mtu = __from_ib_mtu(qpmtu); + qp->qplib_qp.mtu = qpmtu_int; + qp->qplib_qp.modify_flags |= + CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU; +} + +inline unsigned long compare_ether_header(void *a, void *b) +{ + u32 *a32 = (u32 *)((u8 *)a + 2); + u32 *b32 = (u32 *)((u8 *)b + 2); + + return (*(u16 *)a ^ *(u16 *)b) | (a32[0] ^ b32[0]) | + (a32[1] ^ b32[1]) | (a32[2] ^ b32[2]); +} + +struct vlan_hdr { + __be16 h_vlan_TCI; + __be16 h_vlan_encapsulated_proto; +}; + +inline uint16_t +crc16(uint16_t crc, const void *buffer, unsigned int len) +{ + const unsigned char *cp = buffer; + /* CRC table for the CRC-16. The poly is 0x8005 (x16 + x15 + x2 + 1). */ + static uint16_t const crc16_table[256] = { + 0x0000, 0xC0C1, 0xC181, 0x0140, 0xC301, 0x03C0, 0x0280, 0xC241, + 0xC601, 0x06C0, 0x0780, 0xC741, 0x0500, 0xC5C1, 0xC481, 0x0440, + 0xCC01, 0x0CC0, 0x0D80, 0xCD41, 0x0F00, 0xCFC1, 0xCE81, 0x0E40, + 0x0A00, 0xCAC1, 0xCB81, 0x0B40, 0xC901, 0x09C0, 0x0880, 0xC841, + 0xD801, 0x18C0, 0x1980, 0xD941, 0x1B00, 0xDBC1, 0xDA81, 0x1A40, + 0x1E00, 0xDEC1, 0xDF81, 0x1F40, 0xDD01, 0x1DC0, 0x1C80, 0xDC41, + 0x1400, 0xD4C1, 0xD581, 0x1540, 0xD701, 0x17C0, 0x1680, 0xD641, + 0xD201, 0x12C0, 0x1380, 0xD341, 0x1100, 0xD1C1, 0xD081, 0x1040, + 0xF001, 0x30C0, 0x3180, 0xF141, 0x3300, 0xF3C1, 0xF281, 0x3240, + 0x3600, 0xF6C1, 0xF781, 0x3740, 0xF501, 0x35C0, 0x3480, 0xF441, + 0x3C00, 0xFCC1, 0xFD81, 0x3D40, 0xFF01, 0x3FC0, 0x3E80, 0xFE41, + 0xFA01, 0x3AC0, 0x3B80, 0xFB41, 0x3900, 0xF9C1, 0xF881, 0x3840, + 0x2800, 0xE8C1, 0xE981, 0x2940, 0xEB01, 0x2BC0, 0x2A80, 0xEA41, + 0xEE01, 0x2EC0, 0x2F80, 0xEF41, 0x2D00, 0xEDC1, 0xEC81, 0x2C40, + 0xE401, 0x24C0, 0x2580, 0xE541, 0x2700, 0xE7C1, 0xE681, 0x2640, + 0x2200, 0xE2C1, 0xE381, 0x2340, 0xE101, 0x21C0, 0x2080, 0xE041, + 0xA001, 0x60C0, 0x6180, 0xA141, 0x6300, 0xA3C1, 0xA281, 0x6240, + 0x6600, 0xA6C1, 0xA781, 0x6740, 0xA501, 0x65C0, 0x6480, 0xA441, + 0x6C00, 0xACC1, 0xAD81, 0x6D40, 0xAF01, 0x6FC0, 0x6E80, 0xAE41, + 0xAA01, 0x6AC0, 0x6B80, 0xAB41, 0x6900, 0xA9C1, 0xA881, 0x6840, + 0x7800, 0xB8C1, 0xB981, 0x7940, 0xBB01, 0x7BC0, 0x7A80, 0xBA41, + 0xBE01, 0x7EC0, 0x7F80, 0xBF41, 0x7D00, 0xBDC1, 0xBC81, 0x7C40, + 0xB401, 0x74C0, 0x7580, 0xB541, 0x7700, 0xB7C1, 0xB681, 0x7640, + 0x7200, 0xB2C1, 0xB381, 0x7340, 0xB101, 0x71C0, 0x7080, 0xB041, + 0x5000, 0x90C1, 0x9181, 0x5140, 0x9301, 0x53C0, 0x5280, 0x9241, + 0x9601, 0x56C0, 0x5780, 0x9741, 0x5500, 0x95C1, 0x9481, 0x5440, + 0x9C01, 0x5CC0, 0x5D80, 0x9D41, 0x5F00, 0x9FC1, 0x9E81, 0x5E40, + 0x5A00, 0x9AC1, 0x9B81, 0x5B40, 0x9901, 0x59C0, 0x5880, 0x9841, + 0x8801, 0x48C0, 0x4980, 0x8941, 0x4B00, 0x8BC1, 0x8A81, 0x4A40, + 0x4E00, 0x8EC1, 0x8F81, 0x4F40, 0x8D01, 0x4DC0, 0x4C80, 0x8C41, + 0x4400, 0x84C1, 0x8581, 0x4540, 0x8701, 0x47C0, 0x4680, 0x8641, + 0x8201, 0x42C0, 0x4380, 0x8341, 0x4100, 0x81C1, 0x8081, 0x4040 + }; + + while (len--) + crc = (((crc >> 8) & 0xffU) ^ + crc16_table[(crc ^ *cp++) & 0xffU]) & 0x0000ffffU; + return crc; +} + +static inline int __bnxt_re_set_vma_data(void *bnxt_re_uctx, + struct vm_area_struct *vma) +{ + return 0; +} + +static inline bool bnxt_re_check_if_vlan_valid(struct bnxt_re_dev *rdev, + u16 vlan_id) +{ + bool ret = true; + /* + * Check if the vlan is configured in the host. + * If not configured, it can be a transparent + * VLAN. So dont report the vlan id. + */ + return ret; +} + +#endif diff --git a/sys/dev/bnxt/bnxt_re/main.c b/sys/dev/bnxt/bnxt_re/main.c new file mode 100644 index 000000000000..e6c6f754ea47 --- /dev/null +++ b/sys/dev/bnxt/bnxt_re/main.c @@ -0,0 +1,4467 @@ +/* + * Copyright (c) 2015-2024, Broadcom. All rights reserved. The term + * Broadcom refers to Broadcom Limited and/or its subsidiaries. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR + * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE + * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN + * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * Description: Main component of the bnxt_re driver + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "bnxt_re.h" +#include "ib_verbs.h" +#include "bnxt_re-abi.h" +#include "bnxt.h" + +static char drv_version[] = + "Broadcom NetXtreme-C/E RoCE Driver " ROCE_DRV_MODULE_NAME \ + " v" ROCE_DRV_MODULE_VERSION " (" ROCE_DRV_MODULE_RELDATE ")\n"; + +#define BNXT_RE_DESC "Broadcom NetXtreme RoCE" +#define BNXT_ADEV_NAME "if_bnxt" + +MODULE_DESCRIPTION("Broadcom NetXtreme-C/E RoCE Driver"); +MODULE_LICENSE("Dual BSD/GPL"); +MODULE_DEPEND(bnxt_re, linuxkpi, 1, 1, 1); +MODULE_DEPEND(bnxt_re, ibcore, 1, 1, 1); +MODULE_DEPEND(bnxt_re, if_bnxt, 1, 1, 1); +MODULE_VERSION(bnxt_re, 1); + + +DEFINE_MUTEX(bnxt_re_mutex); /* mutex lock for driver */ + +static unsigned int restrict_mrs = 0; +module_param(restrict_mrs, uint, 0); +MODULE_PARM_DESC(restrict_mrs, " Restrict the no. of MRs 0 = 256K , 1 = 64K"); + +unsigned int restrict_stats = 0; +module_param(restrict_stats, uint, 0); +MODULE_PARM_DESC(restrict_stats, "Restrict stats query frequency to ethtool coalesce value. Disabled by default"); + +unsigned int enable_fc = 1; +module_param(enable_fc, uint, 0); +MODULE_PARM_DESC(enable_fc, "Enable default PFC, CC,ETS during driver load. 1 - fc enable, 0 - fc disable - Default is 1"); + +unsigned int min_tx_depth = 1; +module_param(min_tx_depth, uint, 0); +MODULE_PARM_DESC(min_tx_depth, "Minimum TX depth - Default is 1"); + +static uint8_t max_msix_vec[BNXT_RE_MAX_DEVICES] = {0}; +static unsigned int max_msix_vec_argc; +module_param_array(max_msix_vec, byte, &max_msix_vec_argc, 0444); +MODULE_PARM_DESC(max_msix_vec, "Max MSI-x vectors per PF (2 - 64) - Default is 64"); + +unsigned int cmdq_shadow_qd = RCFW_CMD_NON_BLOCKING_SHADOW_QD; +module_param_named(cmdq_shadow_qd, cmdq_shadow_qd, uint, 0644); +MODULE_PARM_DESC(cmdq_shadow_qd, "Perf Stat Debug: Shadow QD Range (1-64) - Default is 64"); + + +/* globals */ +struct list_head bnxt_re_dev_list = LINUX_LIST_HEAD_INIT(bnxt_re_dev_list); +static int bnxt_re_probe_count; + +DEFINE_MUTEX(bnxt_re_dev_lock); +static u32 gmod_exit; +static u32 gadd_dev_inprogress; + +static void bnxt_re_task(struct work_struct *work_task); +static struct workqueue_struct *bnxt_re_wq; +static int bnxt_re_query_hwrm_intf_version(struct bnxt_re_dev *rdev); +static int bnxt_re_hwrm_qcfg(struct bnxt_re_dev *rdev, u32 *db_len, + u32 *offset); +static int bnxt_re_ib_init(struct bnxt_re_dev *rdev); +static void bnxt_re_ib_init_2(struct bnxt_re_dev *rdev); +void _bnxt_re_remove(struct auxiliary_device *adev); +void writel_fbsd(struct bnxt_softc *bp, u32, u8, u32); +u32 readl_fbsd(struct bnxt_softc *bp, u32, u8); +static int bnxt_re_hwrm_dbr_pacing_qcfg(struct bnxt_re_dev *rdev); + +int bnxt_re_register_netdevice_notifier(struct notifier_block *nb) +{ + int rc; + rc = register_netdevice_notifier(nb); + return rc; +} + +int bnxt_re_unregister_netdevice_notifier(struct notifier_block *nb) +{ + int rc; + rc = unregister_netdevice_notifier(nb); + return rc; +} + +void bnxt_re_set_dma_device(struct ib_device *ibdev, struct bnxt_re_dev *rdev) +{ + ibdev->dma_device = &rdev->en_dev->pdev->dev; +} + +void bnxt_re_init_resolve_wq(struct bnxt_re_dev *rdev) +{ + rdev->resolve_wq = create_singlethread_workqueue("bnxt_re_resolve_wq"); + INIT_LIST_HEAD(&rdev->mac_wq_list); +} + +void bnxt_re_uninit_resolve_wq(struct bnxt_re_dev *rdev) +{ + struct bnxt_re_resolve_dmac_work *tmp_work = NULL, *tmp_st; + if (!rdev->resolve_wq) + return; + flush_workqueue(rdev->resolve_wq); + list_for_each_entry_safe(tmp_work, tmp_st, &rdev->mac_wq_list, list) { + list_del(&tmp_work->list); + kfree(tmp_work); + } + destroy_workqueue(rdev->resolve_wq); + rdev->resolve_wq = NULL; +} + +u32 readl_fbsd(struct bnxt_softc *bp, u32 reg_off, u8 bar_idx) +{ + + if (bar_idx) + return bus_space_read_8(bp->doorbell_bar.tag, bp->doorbell_bar.handle, reg_off); + else + return bus_space_read_8(bp->hwrm_bar.tag, bp->hwrm_bar.handle, reg_off); +} + +void writel_fbsd(struct bnxt_softc *bp, u32 reg_off, u8 bar_idx, u32 val) +{ + if (bar_idx) + bus_space_write_8(bp->doorbell_bar.tag, bp->doorbell_bar.handle, reg_off, htole32(val)); + else + bus_space_write_8(bp->hwrm_bar.tag, bp->hwrm_bar.handle, reg_off, htole32(val)); +} + +static void bnxt_re_update_fifo_occup_slabs(struct bnxt_re_dev *rdev, + u32 fifo_occup) +{ + if (fifo_occup > rdev->dbg_stats->dbq.fifo_occup_water_mark) + rdev->dbg_stats->dbq.fifo_occup_water_mark = fifo_occup; + + if (fifo_occup > 8 * rdev->pacing_algo_th) + rdev->dbg_stats->dbq.fifo_occup_slab_4++; + else if (fifo_occup > 4 * rdev->pacing_algo_th) + rdev->dbg_stats->dbq.fifo_occup_slab_3++; + else if (fifo_occup > 2 * rdev->pacing_algo_th) + rdev->dbg_stats->dbq.fifo_occup_slab_2++; + else if (fifo_occup > rdev->pacing_algo_th) + rdev->dbg_stats->dbq.fifo_occup_slab_1++; +} + +static void bnxt_re_update_do_pacing_slabs(struct bnxt_re_dev *rdev) +{ + struct bnxt_qplib_db_pacing_data *pacing_data = rdev->qplib_res.pacing_data; + + if (pacing_data->do_pacing > rdev->dbg_stats->dbq.do_pacing_water_mark) + rdev->dbg_stats->dbq.do_pacing_water_mark = pacing_data->do_pacing; + + if (pacing_data->do_pacing > 16 * rdev->dbr_def_do_pacing) + rdev->dbg_stats->dbq.do_pacing_slab_5++; + else if (pacing_data->do_pacing > 8 * rdev->dbr_def_do_pacing) + rdev->dbg_stats->dbq.do_pacing_slab_4++; + else if (pacing_data->do_pacing > 4 * rdev->dbr_def_do_pacing) + rdev->dbg_stats->dbq.do_pacing_slab_3++; + else if (pacing_data->do_pacing > 2 * rdev->dbr_def_do_pacing) + rdev->dbg_stats->dbq.do_pacing_slab_2++; + else if (pacing_data->do_pacing > rdev->dbr_def_do_pacing) + rdev->dbg_stats->dbq.do_pacing_slab_1++; +} + +static bool bnxt_re_is_qp1_qp(struct bnxt_re_qp *qp) +{ + return qp->ib_qp.qp_type == IB_QPT_GSI; +} + +static struct bnxt_re_qp *bnxt_re_get_qp1_qp(struct bnxt_re_dev *rdev) +{ + struct bnxt_re_qp *qp; + + mutex_lock(&rdev->qp_lock); + list_for_each_entry(qp, &rdev->qp_list, list) { + if (bnxt_re_is_qp1_qp(qp)) { + mutex_unlock(&rdev->qp_lock); + return qp; + } + } + mutex_unlock(&rdev->qp_lock); + return NULL; +} + +/* Set the maximum number of each resource that the driver actually wants + * to allocate. This may be up to the maximum number the firmware has + * reserved for the function. The driver may choose to allocate fewer + * resources than the firmware maximum. + */ +static void bnxt_re_limit_pf_res(struct bnxt_re_dev *rdev) +{ + struct bnxt_qplib_max_res dev_res = {}; + struct bnxt_qplib_chip_ctx *cctx; + struct bnxt_qplib_dev_attr *attr; + struct bnxt_qplib_ctx *hctx; + int i; + + attr = rdev->dev_attr; + hctx = rdev->qplib_res.hctx; + cctx = rdev->chip_ctx; + + bnxt_qplib_max_res_supported(cctx, &rdev->qplib_res, &dev_res, false); + if (!_is_chip_gen_p5_p7(cctx)) { + hctx->qp_ctx.max = min_t(u32, dev_res.max_qp, attr->max_qp); + hctx->mrw_ctx.max = min_t(u32, dev_res.max_mr, attr->max_mr); + /* To accommodate 16k MRs and 16k AHs, + * driver has to allocate 32k backing store memory + */ + hctx->mrw_ctx.max *= 2; + hctx->srq_ctx.max = min_t(u32, dev_res.max_srq, attr->max_srq); + hctx->cq_ctx.max = min_t(u32, dev_res.max_cq, attr->max_cq); + for (i = 0; i < MAX_TQM_ALLOC_REQ; i++) + hctx->tqm_ctx.qcount[i] = attr->tqm_alloc_reqs[i]; + } else { + hctx->qp_ctx.max = attr->max_qp ? attr->max_qp : dev_res.max_qp; + hctx->mrw_ctx.max = attr->max_mr ? attr->max_mr : dev_res.max_mr; + hctx->srq_ctx.max = attr->max_srq ? attr->max_srq : dev_res.max_srq; + hctx->cq_ctx.max = attr->max_cq ? attr->max_cq : dev_res.max_cq; + } +} + +static void bnxt_re_limit_vf_res(struct bnxt_re_dev *rdev, + struct bnxt_qplib_vf_res *vf_res, + u32 num_vf) +{ + struct bnxt_qplib_chip_ctx *cctx = rdev->chip_ctx; + struct bnxt_qplib_max_res dev_res = {}; + + bnxt_qplib_max_res_supported(cctx, &rdev->qplib_res, &dev_res, true); + vf_res->max_qp = dev_res.max_qp / num_vf; + vf_res->max_srq = dev_res.max_srq / num_vf; + vf_res->max_cq = dev_res.max_cq / num_vf; + /* + * MR and AH shares the same backing store, the value specified + * for max_mrw is split into half by the FW for MR and AH + */ + vf_res->max_mrw = dev_res.max_mr * 2 / num_vf; + vf_res->max_gid = BNXT_RE_MAX_GID_PER_VF; +} + +static void bnxt_re_set_resource_limits(struct bnxt_re_dev *rdev) +{ + struct bnxt_qplib_ctx *hctx; + + hctx = rdev->qplib_res.hctx; + memset(&hctx->vf_res, 0, sizeof(struct bnxt_qplib_vf_res)); + bnxt_re_limit_pf_res(rdev); + + if (rdev->num_vfs) + bnxt_re_limit_vf_res(rdev, &hctx->vf_res, rdev->num_vfs); +} + +static void bnxt_re_dettach_irq(struct bnxt_re_dev *rdev) +{ + struct bnxt_qplib_rcfw *rcfw = NULL; + struct bnxt_qplib_nq *nq; + int indx; + + rcfw = &rdev->rcfw; + for (indx = 0; indx < rdev->nqr.max_init; indx++) { + nq = &rdev->nqr.nq[indx]; + mutex_lock(&nq->lock); + bnxt_qplib_nq_stop_irq(nq, false); + mutex_unlock(&nq->lock); + } + + bnxt_qplib_rcfw_stop_irq(rcfw, false); +} + +static void bnxt_re_detach_err_device(struct bnxt_re_dev *rdev) +{ + /* Free the MSIx vectors only so that L2 can proceed with MSIx disable */ + bnxt_re_dettach_irq(rdev); + + /* Set the state as detached to prevent sending any more commands */ + set_bit(ERR_DEVICE_DETACHED, &rdev->rcfw.cmdq.flags); + set_bit(BNXT_RE_FLAG_ERR_DEVICE_DETACHED, &rdev->flags); + wake_up_all(&rdev->rcfw.cmdq.waitq); +} + +#define MAX_DSCP_PRI_TUPLE 64 + +struct bnxt_re_dcb_work { + struct work_struct work; + struct bnxt_re_dev *rdev; + struct hwrm_async_event_cmpl cmpl; +}; + +static void bnxt_re_init_dcb_wq(struct bnxt_re_dev *rdev) +{ + rdev->dcb_wq = create_singlethread_workqueue("bnxt_re_dcb_wq"); +} + +static void bnxt_re_uninit_dcb_wq(struct bnxt_re_dev *rdev) +{ + if (!rdev->dcb_wq) + return; + flush_workqueue(rdev->dcb_wq); + destroy_workqueue(rdev->dcb_wq); + rdev->dcb_wq = NULL; +} + +static void bnxt_re_init_aer_wq(struct bnxt_re_dev *rdev) +{ + rdev->aer_wq = create_singlethread_workqueue("bnxt_re_aer_wq"); +} + +static void bnxt_re_uninit_aer_wq(struct bnxt_re_dev *rdev) +{ + if (!rdev->aer_wq) + return; + flush_workqueue(rdev->aer_wq); + destroy_workqueue(rdev->aer_wq); + rdev->aer_wq = NULL; +} + +static int bnxt_re_update_qp1_tos_dscp(struct bnxt_re_dev *rdev) +{ + struct bnxt_re_qp *qp; + + if (!_is_chip_gen_p5_p7(rdev->chip_ctx)) + return 0; + + qp = bnxt_re_get_qp1_qp(rdev); + if (!qp) + return 0; + + qp->qplib_qp.modify_flags = CMDQ_MODIFY_QP_MODIFY_MASK_TOS_DSCP; + qp->qplib_qp.tos_dscp = rdev->cc_param.qp1_tos_dscp; + + return bnxt_qplib_modify_qp(&rdev->qplib_res, &qp->qplib_qp); +} + +static void bnxt_re_reconfigure_dscp(struct bnxt_re_dev *rdev) +{ + struct bnxt_qplib_cc_param *cc_param; + struct bnxt_re_tc_rec *tc_rec; + bool update_cc = false; + u8 dscp_user; + int rc; + + cc_param = &rdev->cc_param; + tc_rec = &rdev->tc_rec[0]; + + if (!(cc_param->roce_dscp_user || cc_param->cnp_dscp_user)) + return; + + if (cc_param->cnp_dscp_user) { + dscp_user = (cc_param->cnp_dscp_user & 0x3f); + if ((tc_rec->cnp_dscp_bv & (1ul << dscp_user)) && + (cc_param->alt_tos_dscp != dscp_user)) { + cc_param->alt_tos_dscp = dscp_user; + cc_param->mask |= CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_ALT_TOS_DSCP; + update_cc = true; + } + } + + if (cc_param->roce_dscp_user) { + dscp_user = (cc_param->roce_dscp_user & 0x3f); + if ((tc_rec->roce_dscp_bv & (1ul << dscp_user)) && + (cc_param->tos_dscp != dscp_user)) { + cc_param->tos_dscp = dscp_user; + cc_param->mask |= CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_TOS_DSCP; + update_cc = true; + } + } + + if (update_cc) { + rc = bnxt_qplib_modify_cc(&rdev->qplib_res, cc_param); + if (rc) + dev_err(rdev_to_dev(rdev), "Failed to apply cc settings\n"); + } +} + +static void bnxt_re_dcb_wq_task(struct work_struct *work) +{ + struct bnxt_qplib_cc_param *cc_param; + struct bnxt_re_tc_rec *tc_rec; + struct bnxt_re_dev *rdev; + struct bnxt_re_dcb_work *dcb_work = + container_of(work, struct bnxt_re_dcb_work, work); + int rc; + + rdev = dcb_work->rdev; + if (!rdev) + goto exit; + + mutex_lock(&rdev->cc_lock); + + cc_param = &rdev->cc_param; + rc = bnxt_qplib_query_cc_param(&rdev->qplib_res, cc_param); + if (rc) { + dev_err(rdev_to_dev(rdev), "Failed to query ccparam rc:%d", rc); + goto fail; + } + tc_rec = &rdev->tc_rec[0]; + /* + * Upon the receival of DCB Async event: + * If roce_dscp or cnp_dscp or both (which user configured using configfs) + * is in the list, re-program the value using modify_roce_cc command + */ + bnxt_re_reconfigure_dscp(rdev); + + cc_param->roce_pri = tc_rec->roce_prio; + if (cc_param->qp1_tos_dscp != cc_param->tos_dscp) { + cc_param->qp1_tos_dscp = cc_param->tos_dscp; + rc = bnxt_re_update_qp1_tos_dscp(rdev); + if (rc) { + dev_err(rdev_to_dev(rdev), "%s:Failed to modify QP1 rc:%d", + __func__, rc); + goto fail; + } + } + +fail: + mutex_unlock(&rdev->cc_lock); +exit: + kfree(dcb_work); +} + +static int bnxt_re_hwrm_dbr_pacing_broadcast_event(struct bnxt_re_dev *rdev) +{ + struct hwrm_func_dbr_pacing_broadcast_event_output resp = {0}; + struct hwrm_func_dbr_pacing_broadcast_event_input req = {0}; + struct bnxt_en_dev *en_dev = rdev->en_dev; + struct bnxt_fw_msg fw_msg; + int rc; + + memset(&fw_msg, 0, sizeof(fw_msg)); + bnxt_re_init_hwrm_hdr(rdev, (void *)&req, + HWRM_FUNC_DBR_PACING_BROADCAST_EVENT, -1, -1); + bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp, + sizeof(resp), BNXT_RE_HWRM_CMD_TIMEOUT(rdev)); + rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg); + if (rc) { + dev_dbg(rdev_to_dev(rdev), + "Failed to send dbr pacing broadcast event rc:%d", rc); + return rc; + } + return 0; +} + +static int bnxt_re_hwrm_dbr_pacing_nqlist_query(struct bnxt_re_dev *rdev) +{ + struct hwrm_func_dbr_pacing_nqlist_query_output resp = {0}; + struct hwrm_func_dbr_pacing_nqlist_query_input req = {0}; + struct bnxt_dbq_nq_list *nq_list = &rdev->nq_list; + struct bnxt_en_dev *en_dev = rdev->en_dev; + bool primary_found = false; + struct bnxt_fw_msg fw_msg; + struct bnxt_qplib_nq *nq; + int rc, i, j = 1; + u16 *nql_ptr; + + nq = &rdev->nqr.nq[0]; + + memset(&fw_msg, 0, sizeof(fw_msg)); + bnxt_re_init_hwrm_hdr(rdev, (void *)&req, + HWRM_FUNC_DBR_PACING_NQLIST_QUERY, -1, -1); + bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp, + sizeof(resp), BNXT_RE_HWRM_CMD_TIMEOUT(rdev)); + rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg); + if (rc) { + dev_err(rdev_to_dev(rdev), "Failed to send dbr pacing nq list query rc:%d", rc); + return rc; + } + nq_list->num_nql_entries = le32_to_cpu(resp.num_nqs); + nql_ptr = &resp.nq_ring_id0; + /* populate the nq_list of the primary function with list received + * from FW. Fill the NQ IDs of secondary functions from index 1 to + * num_nql_entries - 1. Fill the nq_list->nq_id[0] with the + * nq_id of the primary pf + */ + for (i = 0; i < nq_list->num_nql_entries; i++) { + u16 nq_id = *nql_ptr; + + dev_dbg(rdev_to_dev(rdev), + "nq_list->nq_id[%d] = %d\n", i, nq_id); + if (nq_id != nq->ring_id) { + nq_list->nq_id[j] = nq_id; + j++; + } else { + primary_found = true; + nq_list->nq_id[0] = nq->ring_id; + } + nql_ptr++; + } + if (primary_found) + bnxt_qplib_dbr_pacing_set_primary_pf(rdev->chip_ctx, 1); + + return 0; +} + +static void __wait_for_fifo_occupancy_below_th(struct bnxt_re_dev *rdev) +{ + struct bnxt_qplib_db_pacing_data *pacing_data = rdev->qplib_res.pacing_data; + u32 read_val, fifo_occup; + bool first_read = true; + + /* loop shouldn't run infintely as the occupancy usually goes + * below pacing algo threshold as soon as pacing kicks in. + */ + while (1) { + read_val = readl_fbsd(rdev->en_dev->softc, rdev->dbr_db_fifo_reg_off, 0); + fifo_occup = pacing_data->fifo_max_depth - + ((read_val & pacing_data->fifo_room_mask) >> + pacing_data->fifo_room_shift); + /* Fifo occupancy cannot be greater the MAX FIFO depth */ + if (fifo_occup > pacing_data->fifo_max_depth) + break; + + if (first_read) { + bnxt_re_update_fifo_occup_slabs(rdev, fifo_occup); + first_read = false; + } + if (fifo_occup < pacing_data->pacing_th) + break; + } +} + +static void bnxt_re_set_default_pacing_data(struct bnxt_re_dev *rdev) +{ + struct bnxt_qplib_db_pacing_data *pacing_data = rdev->qplib_res.pacing_data; + + pacing_data->do_pacing = rdev->dbr_def_do_pacing; + pacing_data->pacing_th = rdev->pacing_algo_th; + pacing_data->alarm_th = + pacing_data->pacing_th * BNXT_RE_PACING_ALARM_TH_MULTIPLE(rdev->chip_ctx); +} + +#define CAG_RING_MASK 0x7FF +#define CAG_RING_SHIFT 17 +#define WATERMARK_MASK 0xFFF +#define WATERMARK_SHIFT 0 + +static bool bnxt_re_check_if_dbq_intr_triggered(struct bnxt_re_dev *rdev) +{ + u32 read_val; + int j; + + for (j = 0; j < 10; j++) { + read_val = readl_fbsd(rdev->en_dev->softc, rdev->dbr_aeq_arm_reg_off, 0); + dev_dbg(rdev_to_dev(rdev), "AEQ ARM status = 0x%x\n", + read_val); + if (!read_val) + return true; + } + return false; +} + +int bnxt_re_set_dbq_throttling_reg(struct bnxt_re_dev *rdev, u16 nq_id, u32 throttle) +{ + u32 cag_ring_water_mark = 0, read_val; + u32 throttle_val; + + /* Convert throttle percentage to value */ + throttle_val = (rdev->qplib_res.pacing_data->fifo_max_depth * throttle) / 100; + + if (bnxt_qplib_dbr_pacing_ext_en(rdev->chip_ctx)) { + cag_ring_water_mark = (nq_id & CAG_RING_MASK) << CAG_RING_SHIFT | + (throttle_val & WATERMARK_MASK); + writel_fbsd(rdev->en_dev->softc, rdev->dbr_throttling_reg_off, 0, cag_ring_water_mark); + read_val = readl_fbsd(rdev->en_dev->softc , rdev->dbr_throttling_reg_off, 0); + dev_dbg(rdev_to_dev(rdev), + "%s: dbr_throttling_reg_off read_val = 0x%x\n", + __func__, read_val); + if (read_val != cag_ring_water_mark) { + dev_dbg(rdev_to_dev(rdev), + "nq_id = %d write_val=0x%x read_val=0x%x\n", + nq_id, cag_ring_water_mark, read_val); + return 1; + } + } + writel_fbsd(rdev->en_dev->softc, rdev->dbr_aeq_arm_reg_off, 0, 1); + return 0; +} + +static void bnxt_re_set_dbq_throttling_for_non_primary(struct bnxt_re_dev *rdev) +{ + struct bnxt_dbq_nq_list *nq_list; + struct bnxt_qplib_nq *nq; + int i; + + nq_list = &rdev->nq_list; + /* Run a loop for other Active functions if this is primary function */ + if (bnxt_qplib_dbr_pacing_is_primary_pf(rdev->chip_ctx)) { + dev_dbg(rdev_to_dev(rdev), "%s: nq_list->num_nql_entries= %d\n", + __func__, nq_list->num_nql_entries); + nq = &rdev->nqr.nq[0]; + for (i = nq_list->num_nql_entries - 1; i > 0; i--) { + u16 nq_id = nq_list->nq_id[i]; + if (nq) + dev_dbg(rdev_to_dev(rdev), + "%s: nq_id = %d cur_fn_ring_id = %d\n", + __func__, nq_id, nq->ring_id); + if (bnxt_re_set_dbq_throttling_reg + (rdev, nq_id, 0)) + break; + bnxt_re_check_if_dbq_intr_triggered(rdev); + } + } +} + +static void bnxt_re_handle_dbr_nq_pacing_notification(struct bnxt_re_dev *rdev) +{ + struct bnxt_qplib_nq *nq; + int rc = 0; + + nq = &rdev->nqr.nq[0]; + + /* Query the NQ list*/ + rc = bnxt_re_hwrm_dbr_pacing_nqlist_query(rdev); + if (rc) { + dev_err(rdev_to_dev(rdev), + "Failed to Query NQ list rc= %d", rc); + return; + } + /*Configure GRC access for Throttling and aeq_arm register */ + writel_fbsd(rdev->en_dev->softc, BNXT_GRCPF_REG_WINDOW_BASE_OUT + 28, 0, + rdev->chip_ctx->dbr_aeq_arm_reg & BNXT_GRC_BASE_MASK); + + rdev->dbr_throttling_reg_off = + (rdev->chip_ctx->dbr_throttling_reg & + BNXT_GRC_OFFSET_MASK) + 0x8000; + rdev->dbr_aeq_arm_reg_off = + (rdev->chip_ctx->dbr_aeq_arm_reg & + BNXT_GRC_OFFSET_MASK) + 0x8000; + + bnxt_re_set_dbq_throttling_reg(rdev, nq->ring_id, rdev->dbq_watermark); +} + +static void bnxt_re_dbq_wq_task(struct work_struct *work) +{ + struct bnxt_re_dbq_work *dbq_work = + container_of(work, struct bnxt_re_dbq_work, work); + struct bnxt_re_dev *rdev; + + rdev = dbq_work->rdev; + + if (!rdev) + goto exit; + switch (dbq_work->event) { + case BNXT_RE_DBQ_EVENT_SCHED: + dev_dbg(rdev_to_dev(rdev), "%s: Handle DBQ Pacing event\n", + __func__); + if (!bnxt_qplib_dbr_pacing_ext_en(rdev->chip_ctx)) + bnxt_re_hwrm_dbr_pacing_broadcast_event(rdev); + else + bnxt_re_pacing_alert(rdev); + break; + case BNXT_RE_DBR_PACING_EVENT: + dev_dbg(rdev_to_dev(rdev), "%s: Sched interrupt/pacing worker\n", + __func__); + if (_is_chip_p7(rdev->chip_ctx)) + bnxt_re_pacing_alert(rdev); + else if (!rdev->chip_ctx->modes.dbr_pacing_v0) + bnxt_re_hwrm_dbr_pacing_qcfg(rdev); + break; + case BNXT_RE_DBR_NQ_PACING_NOTIFICATION: + bnxt_re_handle_dbr_nq_pacing_notification(rdev); + /* Issue a broadcast event to notify other functions + * that primary changed + */ + bnxt_re_hwrm_dbr_pacing_broadcast_event(rdev); + break; + } +exit: + kfree(dbq_work); +} + +static void bnxt_re_async_notifier(void *handle, struct hwrm_async_event_cmpl *cmpl) +{ + struct bnxt_re_en_dev_info *en_info = auxiliary_get_drvdata(handle); + struct bnxt_re_dcb_work *dcb_work; + struct bnxt_re_dbq_work *dbq_work; + struct bnxt_re_dev *rdev; + u16 event_id; + u32 data1; + u32 data2 = 0; + + if (!cmpl) { + pr_err("Async event, bad completion\n"); + return; + } + + if (!en_info || !en_info->en_dev) { + pr_err("Async event, bad en_info or en_dev\n"); + return; + } + rdev = en_info->rdev; + + event_id = le16_to_cpu(cmpl->event_id); + data1 = le32_to_cpu(cmpl->event_data1); + data2 = le32_to_cpu(cmpl->event_data2); + + if (!rdev || !rdev_to_dev(rdev)) { + dev_dbg(NULL, "Async event, bad rdev or netdev\n"); + return; + } + + if (test_bit(BNXT_RE_FLAG_ERR_DEVICE_DETACHED, &rdev->flags) || + !test_bit(BNXT_RE_FLAG_NETDEV_REGISTERED, &rdev->flags)) { + dev_dbg(NULL, "Async event, device already detached\n"); + return; + } + if (data2 >= 0) + dev_dbg(rdev_to_dev(rdev), "Async event_id = %d data1 = %d data2 = %d", + event_id, data1, data2); + + switch (event_id) { + case HWRM_ASYNC_EVENT_CMPL_EVENT_ID_DCB_CONFIG_CHANGE: + /* Not handling the event in older FWs */ + if (!is_qport_service_type_supported(rdev)) + break; + if (!rdev->dcb_wq) + break; + dcb_work = kzalloc(sizeof(*dcb_work), GFP_ATOMIC); + if (!dcb_work) + break; + + dcb_work->rdev = rdev; + memcpy(&dcb_work->cmpl, cmpl, sizeof(*cmpl)); + INIT_WORK(&dcb_work->work, bnxt_re_dcb_wq_task); + queue_work(rdev->dcb_wq, &dcb_work->work); + break; + case HWRM_ASYNC_EVENT_CMPL_EVENT_ID_RESET_NOTIFY: + if (EVENT_DATA1_RESET_NOTIFY_FATAL(data1)) { + /* Set rcfw flag to control commands send to Bono */ + set_bit(ERR_DEVICE_DETACHED, &rdev->rcfw.cmdq.flags); + /* Set bnxt_re flag to control commands send via L2 driver */ + set_bit(BNXT_RE_FLAG_ERR_DEVICE_DETACHED, &rdev->flags); + wake_up_all(&rdev->rcfw.cmdq.waitq); + } + break; + case HWRM_ASYNC_EVENT_CMPL_EVENT_ID_DOORBELL_PACING_THRESHOLD: + if (!rdev->dbr_pacing) + break; + dbq_work = kzalloc(sizeof(*dbq_work), GFP_ATOMIC); + if (!dbq_work) + goto unlock; + dbq_work->rdev = rdev; + dbq_work->event = BNXT_RE_DBR_PACING_EVENT; + INIT_WORK(&dbq_work->work, bnxt_re_dbq_wq_task); + queue_work(rdev->dbq_wq, &dbq_work->work); + rdev->dbr_sw_stats->dbq_int_recv++; + break; + case HWRM_ASYNC_EVENT_CMPL_EVENT_ID_DOORBELL_PACING_NQ_UPDATE: + if (!rdev->dbr_pacing) + break; + + dbq_work = kzalloc(sizeof(*dbq_work), GFP_ATOMIC); + if (!dbq_work) + goto unlock; + dbq_work->rdev = rdev; + dbq_work->event = BNXT_RE_DBR_NQ_PACING_NOTIFICATION; + INIT_WORK(&dbq_work->work, bnxt_re_dbq_wq_task); + queue_work(rdev->dbq_wq, &dbq_work->work); + break; + + default: + break; + } +unlock: + return; +} + +static void bnxt_re_db_fifo_check(struct work_struct *work) +{ + struct bnxt_re_dev *rdev = container_of(work, struct bnxt_re_dev, + dbq_fifo_check_work); + struct bnxt_qplib_db_pacing_data *pacing_data; + u32 pacing_save; + + if (!mutex_trylock(&rdev->dbq_lock)) + return; + pacing_data = rdev->qplib_res.pacing_data; + pacing_save = rdev->do_pacing_save; + __wait_for_fifo_occupancy_below_th(rdev); + cancel_delayed_work_sync(&rdev->dbq_pacing_work); + if (rdev->dbr_recovery_on) + goto recovery_on; + if (pacing_save > rdev->dbr_def_do_pacing) { + /* Double the do_pacing value during the congestion */ + pacing_save = pacing_save << 1; + } else { + /* + * when a new congestion is detected increase the do_pacing + * by 8 times. And also increase the pacing_th by 4 times. The + * reason to increase pacing_th is to give more space for the + * queue to oscillate down without getting empty, but also more + * room for the queue to increase without causing another alarm. + */ + pacing_save = pacing_save << 3; + pacing_data->pacing_th = rdev->pacing_algo_th * 4; + } + + if (pacing_save > BNXT_RE_MAX_DBR_DO_PACING) + pacing_save = BNXT_RE_MAX_DBR_DO_PACING; + + pacing_data->do_pacing = pacing_save; + rdev->do_pacing_save = pacing_data->do_pacing; + pacing_data->alarm_th = + pacing_data->pacing_th * BNXT_RE_PACING_ALARM_TH_MULTIPLE(rdev->chip_ctx); +recovery_on: + schedule_delayed_work(&rdev->dbq_pacing_work, + msecs_to_jiffies(rdev->dbq_pacing_time)); + rdev->dbr_sw_stats->dbq_pacing_alerts++; + mutex_unlock(&rdev->dbq_lock); +} + +static void bnxt_re_pacing_timer_exp(struct work_struct *work) +{ + struct bnxt_re_dev *rdev = container_of(work, struct bnxt_re_dev, + dbq_pacing_work.work); + struct bnxt_qplib_db_pacing_data *pacing_data; + u32 read_val, fifo_occup; + struct bnxt_qplib_nq *nq; + + if (!mutex_trylock(&rdev->dbq_lock)) + return; + + pacing_data = rdev->qplib_res.pacing_data; + read_val = readl_fbsd(rdev->en_dev->softc , rdev->dbr_db_fifo_reg_off, 0); + fifo_occup = pacing_data->fifo_max_depth - + ((read_val & pacing_data->fifo_room_mask) >> + pacing_data->fifo_room_shift); + + if (fifo_occup > pacing_data->pacing_th) + goto restart_timer; + + /* + * Instead of immediately going back to the default do_pacing + * reduce it by 1/8 times and restart the timer. + */ + pacing_data->do_pacing = pacing_data->do_pacing - (pacing_data->do_pacing >> 3); + pacing_data->do_pacing = max_t(u32, rdev->dbr_def_do_pacing, pacing_data->do_pacing); + /* + * If the fifo_occup is less than the interrupt enable threshold + * enable the interrupt on the primary PF. + */ + if (rdev->dbq_int_disable && fifo_occup < rdev->pacing_en_int_th) { + if (bnxt_qplib_dbr_pacing_is_primary_pf(rdev->chip_ctx)) { + if (!rdev->chip_ctx->modes.dbr_pacing_v0) { + nq = &rdev->nqr.nq[0]; + bnxt_re_set_dbq_throttling_reg(rdev, nq->ring_id, + rdev->dbq_watermark); + rdev->dbr_sw_stats->dbq_int_en++; + rdev->dbq_int_disable = false; + } + } + } + if (pacing_data->do_pacing <= rdev->dbr_def_do_pacing) { + bnxt_re_set_default_pacing_data(rdev); + rdev->dbr_sw_stats->dbq_pacing_complete++; + goto dbq_unlock; + } +restart_timer: + schedule_delayed_work(&rdev->dbq_pacing_work, + msecs_to_jiffies(rdev->dbq_pacing_time)); + bnxt_re_update_do_pacing_slabs(rdev); + rdev->dbr_sw_stats->dbq_pacing_resched++; +dbq_unlock: + rdev->do_pacing_save = pacing_data->do_pacing; + mutex_unlock(&rdev->dbq_lock); +} + +void bnxt_re_pacing_alert(struct bnxt_re_dev *rdev) +{ + struct bnxt_qplib_db_pacing_data *pacing_data; + + if (!rdev->dbr_pacing) + return; + mutex_lock(&rdev->dbq_lock); + pacing_data = rdev->qplib_res.pacing_data; + + /* + * Increase the alarm_th to max so that other user lib instances do not + * keep alerting the driver. + */ + pacing_data->alarm_th = pacing_data->fifo_max_depth; + pacing_data->do_pacing = BNXT_RE_MAX_DBR_DO_PACING; + cancel_work_sync(&rdev->dbq_fifo_check_work); + schedule_work(&rdev->dbq_fifo_check_work); + mutex_unlock(&rdev->dbq_lock); +} + +void bnxt_re_schedule_dbq_event(struct bnxt_qplib_res *res) +{ + struct bnxt_re_dbq_work *dbq_work; + struct bnxt_re_dev *rdev; + + rdev = container_of(res, struct bnxt_re_dev, qplib_res); + + atomic_set(&rdev->dbq_intr_running, 1); + + if (test_bit(BNXT_RE_FLAG_ERR_DEVICE_DETACHED, &rdev->flags)) + goto exit; + /* Run the loop to send dbq event to other functions + * for newer FW + */ + if (bnxt_qplib_dbr_pacing_ext_en(rdev->chip_ctx) && + !rdev->chip_ctx->modes.dbr_pacing_v0) + bnxt_re_set_dbq_throttling_for_non_primary(rdev); + + dbq_work = kzalloc(sizeof(*dbq_work), GFP_ATOMIC); + if (!dbq_work) + goto exit; + dbq_work->rdev = rdev; + dbq_work->event = BNXT_RE_DBQ_EVENT_SCHED; + INIT_WORK(&dbq_work->work, bnxt_re_dbq_wq_task); + queue_work(rdev->dbq_wq, &dbq_work->work); + rdev->dbr_sw_stats->dbq_int_recv++; + rdev->dbq_int_disable = true; +exit: + atomic_set(&rdev->dbq_intr_running, 0); +} + +static void bnxt_re_free_msix(struct bnxt_re_dev *rdev) +{ + struct bnxt_en_dev *en_dev = rdev->en_dev; + int rc; + + rc = en_dev->en_ops->bnxt_free_msix(rdev->en_dev, BNXT_ROCE_ULP); + if (rc) + dev_err(rdev_to_dev(rdev), "netdev %p free_msix failed! rc = 0x%x", + rdev->netdev, rc); +} + +static int bnxt_re_request_msix(struct bnxt_re_dev *rdev) +{ + struct bnxt_en_dev *en_dev = rdev->en_dev; + int rc = 0, num_msix_want, num_msix_got; + struct bnxt_msix_entry *entry; + + /* + * Request MSIx based on the function type. This is + * a temporory solution to enable max VFs when NPAR is + * enabled. + * TODO - change the scheme with an adapter specific check + * as the latest adapters can support more NQs. For now + * this change satisfy all adapter versions. + */ + + if (rdev->is_virtfn) + num_msix_want = BNXT_RE_MAX_MSIX_VF; + else if (BNXT_EN_NPAR(en_dev)) + num_msix_want = BNXT_RE_MAX_MSIX_NPAR_PF; + else if (_is_chip_gen_p5_p7(rdev->chip_ctx)) + num_msix_want = rdev->num_msix_requested ?: BNXT_RE_MAX_MSIX_GEN_P5_PF; + else + num_msix_want = BNXT_RE_MAX_MSIX_PF; + + /* + * Since MSIX vectors are used for both NQs and CREQ, we should try to + * allocate num_online_cpus + 1 by taking into account the CREQ. This + * leaves the number of MSIX vectors for NQs match the number of CPUs + * and allows the system to be fully utilized + */ + num_msix_want = min_t(u32, num_msix_want, num_online_cpus() + 1); + num_msix_want = min_t(u32, num_msix_want, BNXT_RE_MAX_MSIX); + num_msix_want = max_t(u32, num_msix_want, BNXT_RE_MIN_MSIX); + + entry = rdev->nqr.msix_entries; + + num_msix_got = en_dev->en_ops->bnxt_request_msix(en_dev, BNXT_ROCE_ULP, + entry, num_msix_want); + if (num_msix_got < BNXT_RE_MIN_MSIX) { + rc = -EINVAL; + goto done; + } + if (num_msix_got != num_msix_want) + dev_warn(rdev_to_dev(rdev), + "bnxt_request_msix: wanted %d vectors, got %d\n", + num_msix_want, num_msix_got); + + rdev->nqr.num_msix = num_msix_got; + return 0; +done: + if (num_msix_got) + bnxt_re_free_msix(rdev); + return rc; +} + +static int __wait_for_ib_unregister(struct bnxt_re_dev *rdev, + struct bnxt_re_en_dev_info *en_info) +{ + u64 timeout = 0; + u32 cur_prod = 0, cur_cons = 0; + int retry = 0, rc = 0, ret = 0; + + cur_prod = rdev->rcfw.cmdq.hwq.prod; + cur_cons = rdev->rcfw.cmdq.hwq.cons; + timeout = msecs_to_jiffies(BNXT_RE_RECOVERY_IB_UNINIT_WAIT_TIME_MS); + retry = BNXT_RE_RECOVERY_IB_UNINIT_WAIT_RETRY; + /* During module exit, increase timeout ten-fold to 100 mins to wait + * as long as possible for ib_unregister() to complete + */ + if (rdev->mod_exit) + retry *= 10; + do { + /* + * Since the caller of this function invokes with bnxt_re_mutex held, + * release it to avoid holding a lock while in wait / sleep mode. + */ + mutex_unlock(&bnxt_re_mutex); + rc = wait_event_timeout(en_info->waitq, + en_info->ib_uninit_done, + timeout); + mutex_lock(&bnxt_re_mutex); + + if (!bnxt_re_is_rdev_valid(rdev)) + break; + + if (rc) + break; + + if (!RCFW_NO_FW_ACCESS(&rdev->rcfw)) { + /* No need to check for cmdq stall during module exit, + * wait for ib unregister to complete + */ + if (!rdev->mod_exit) + ret = __check_cmdq_stall(&rdev->rcfw, &cur_prod, &cur_cons); + if (ret || en_info->ib_uninit_done) + break; + } + } while (retry--); + + return rc; +} + +static int bnxt_re_handle_start(struct auxiliary_device *adev) +{ + struct bnxt_re_en_dev_info *en_info = auxiliary_get_drvdata(adev); + struct bnxt_re_dev *rdev = NULL; + struct ifnet *real_dev; + struct bnxt_en_dev *en_dev; + struct ifnet *netdev; + int rc = 0; + + if (!en_info || !en_info->en_dev) { + pr_err("Start, bad en_info or en_dev\n"); + return -EINVAL; + } + netdev = en_info->en_dev->net; + if (en_info->rdev) { + dev_info(rdev_to_dev(en_info->rdev), + "%s: Device is already added adev %p rdev: %p\n", + __func__, adev, en_info->rdev); + return 0; + } + + en_dev = en_info->en_dev; + real_dev = rdma_vlan_dev_real_dev(netdev); + if (!real_dev) + real_dev = netdev; + rc = bnxt_re_add_device(&rdev, real_dev, + en_info->gsi_mode, + BNXT_RE_POST_RECOVERY_INIT, + en_info->wqe_mode, + en_info->num_msix_requested, adev); + if (rc) { + /* Add device failed. Unregister the device. + * This has to be done explicitly as + * bnxt_re_stop would not have unregistered + */ + rtnl_lock(); + en_dev->en_ops->bnxt_unregister_device(en_dev, BNXT_ROCE_ULP); + rtnl_unlock(); + mutex_lock(&bnxt_re_dev_lock); + gadd_dev_inprogress--; + mutex_unlock(&bnxt_re_dev_lock); + return rc; + } + rdev->adev = adev; + rtnl_lock(); + bnxt_re_get_link_speed(rdev); + rtnl_unlock(); + rc = bnxt_re_ib_init(rdev); + if (rc) { + dev_err(rdev_to_dev(rdev), "Failed ib_init\n"); + return rc; + } + bnxt_re_ib_init_2(rdev); + + return rc; +} + +static void bnxt_re_stop(void *handle) +{ + struct bnxt_re_en_dev_info *en_info = auxiliary_get_drvdata(handle); + struct ifnet *netdev; + struct bnxt_re_dev *rdev; + struct bnxt_en_dev *en_dev; + int rc = 0; + + rtnl_unlock(); + mutex_lock(&bnxt_re_mutex); + if (!en_info || !en_info->en_dev) { + pr_err("Stop, bad en_info or en_dev\n"); + goto exit; + } + netdev = en_info->en_dev->net; + rdev = en_info->rdev; + if (!rdev) + goto exit; + + if (!bnxt_re_is_rdev_valid(rdev)) + goto exit; + + /* + * Check if fw has undergone reset or is in a fatal condition. + * If so, set flags so that no further commands are sent down to FW + */ + en_dev = rdev->en_dev; + if (en_dev->en_state & BNXT_STATE_FW_FATAL_COND || + en_dev->en_state & BNXT_STATE_FW_RESET_DET) { + /* Set rcfw flag to control commands send to Bono */ + set_bit(ERR_DEVICE_DETACHED, &rdev->rcfw.cmdq.flags); + /* Set bnxt_re flag to control commands send via L2 driver */ + set_bit(BNXT_RE_FLAG_ERR_DEVICE_DETACHED, &rdev->flags); + wake_up_all(&rdev->rcfw.cmdq.waitq); + } + + if (test_bit(BNXT_RE_FLAG_STOP_IN_PROGRESS, &rdev->flags)) + goto exit; + set_bit(BNXT_RE_FLAG_STOP_IN_PROGRESS, &rdev->flags); + + en_info->wqe_mode = rdev->chip_ctx->modes.wqe_mode; + en_info->gsi_mode = rdev->gsi_ctx.gsi_qp_mode; + en_info->num_msix_requested = rdev->num_msix_requested; + en_info->ib_uninit_done = false; + + if (rdev->dbr_pacing) + bnxt_re_set_pacing_dev_state(rdev); + + dev_info(rdev_to_dev(rdev), "%s: L2 driver notified to stop." + "Attempting to stop and Dispatching event " + "to inform the stack\n", __func__); + init_waitqueue_head(&en_info->waitq); + /* Schedule a work item to handle IB UNINIT for recovery */ + bnxt_re_schedule_work(rdev, NETDEV_UNREGISTER, + NULL, netdev, rdev->adev); + rc = __wait_for_ib_unregister(rdev, en_info); + if (!bnxt_re_is_rdev_valid(rdev)) + goto exit; + if (!rc) { + dev_info(rdev_to_dev(rdev), "%s: Attempt to stop failed\n", + __func__); + bnxt_re_detach_err_device(rdev); + goto exit; + } + bnxt_re_remove_device(rdev, BNXT_RE_PRE_RECOVERY_REMOVE, rdev->adev); +exit: + mutex_unlock(&bnxt_re_mutex); + /* Take rtnl_lock before return, bnxt_re_stop is called with rtnl_lock */ + rtnl_lock(); + + return; +} + +static void bnxt_re_start(void *handle) +{ + rtnl_unlock(); + mutex_lock(&bnxt_re_mutex); + if (bnxt_re_handle_start((struct auxiliary_device *)handle)) + pr_err("Failed to start RoCE device"); + mutex_unlock(&bnxt_re_mutex); + /* Take rtnl_lock before return, bnxt_re_start is called with rtnl_lock */ + rtnl_lock(); + return; +} + +static void bnxt_re_shutdown(void *p) +{ + struct bnxt_re_en_dev_info *en_info = auxiliary_get_drvdata(p); + struct bnxt_re_dev *rdev; + + if (!en_info) { + pr_err("Shutdown, bad en_info\n"); + return; + } + rtnl_unlock(); + mutex_lock(&bnxt_re_mutex); + rdev = en_info->rdev; + if (!rdev || !bnxt_re_is_rdev_valid(rdev)) + goto exit; + + /* rtnl_lock held by L2 before coming here */ + bnxt_re_stopqps_and_ib_uninit(rdev); + bnxt_re_remove_device(rdev, BNXT_RE_COMPLETE_REMOVE, rdev->adev); +exit: + mutex_unlock(&bnxt_re_mutex); + rtnl_lock(); + return; +} + +static void bnxt_re_stop_irq(void *handle) +{ + struct bnxt_re_en_dev_info *en_info = auxiliary_get_drvdata(handle); + struct bnxt_qplib_rcfw *rcfw = NULL; + struct bnxt_re_dev *rdev; + struct bnxt_qplib_nq *nq; + int indx; + + if (!en_info) { + pr_err("Stop irq, bad en_info\n"); + return; + } + rdev = en_info->rdev; + + if (!rdev) + return; + + rcfw = &rdev->rcfw; + for (indx = 0; indx < rdev->nqr.max_init; indx++) { + nq = &rdev->nqr.nq[indx]; + mutex_lock(&nq->lock); + bnxt_qplib_nq_stop_irq(nq, false); + mutex_unlock(&nq->lock); + } + + if (test_bit(BNXT_RE_FLAG_ALLOC_RCFW, &rdev->flags)) + bnxt_qplib_rcfw_stop_irq(rcfw, false); +} + +static void bnxt_re_start_irq(void *handle, struct bnxt_msix_entry *ent) +{ + struct bnxt_re_en_dev_info *en_info = auxiliary_get_drvdata(handle); + struct bnxt_msix_entry *msix_ent = NULL; + struct bnxt_qplib_rcfw *rcfw = NULL; + struct bnxt_re_dev *rdev; + struct bnxt_qplib_nq *nq; + int indx, rc, vec; + + if (!en_info) { + pr_err("Start irq, bad en_info\n"); + return; + } + rdev = en_info->rdev; + if (!rdev) + return; + if (test_bit(BNXT_RE_FLAG_ERR_DEVICE_DETACHED, &rdev->flags)) + return; + msix_ent = rdev->nqr.msix_entries; + rcfw = &rdev->rcfw; + + if (!ent) { + /* Not setting the f/w timeout bit in rcfw. + * During the driver unload the first command + * to f/w will timeout and that will set the + * timeout bit. + */ + dev_err(rdev_to_dev(rdev), "Failed to re-start IRQs\n"); + return; + } + + /* Vectors may change after restart, so update with new vectors + * in device structure. + */ + for (indx = 0; indx < rdev->nqr.num_msix; indx++) + rdev->nqr.msix_entries[indx].vector = ent[indx].vector; + + if (test_bit(BNXT_RE_FLAG_ALLOC_RCFW, &rdev->flags)) { + rc = bnxt_qplib_rcfw_start_irq(rcfw, msix_ent[BNXT_RE_AEQ_IDX].vector, + false); + if (rc) { + dev_warn(rdev_to_dev(rdev), + "Failed to reinit CREQ\n"); + return; + } + } + for (indx = 0 ; indx < rdev->nqr.max_init; indx++) { + nq = &rdev->nqr.nq[indx]; + vec = indx + 1; + rc = bnxt_qplib_nq_start_irq(nq, indx, msix_ent[vec].vector, + false); + if (rc) { + dev_warn(rdev_to_dev(rdev), + "Failed to reinit NQ index %d\n", indx); + return; + } + } +} + +/* + * Except for ulp_async_notifier, the remaining ulp_ops + * below are called with rtnl_lock held + */ +static struct bnxt_ulp_ops bnxt_re_ulp_ops = { + .ulp_async_notifier = bnxt_re_async_notifier, + .ulp_stop = bnxt_re_stop, + .ulp_start = bnxt_re_start, + .ulp_shutdown = bnxt_re_shutdown, + .ulp_irq_stop = bnxt_re_stop_irq, + .ulp_irq_restart = bnxt_re_start_irq, +}; + +static inline const char *bnxt_re_netevent(unsigned long event) +{ + BNXT_RE_NETDEV_EVENT(event, NETDEV_UP); + BNXT_RE_NETDEV_EVENT(event, NETDEV_DOWN); + BNXT_RE_NETDEV_EVENT(event, NETDEV_CHANGE); + BNXT_RE_NETDEV_EVENT(event, NETDEV_REGISTER); + BNXT_RE_NETDEV_EVENT(event, NETDEV_UNREGISTER); + BNXT_RE_NETDEV_EVENT(event, NETDEV_CHANGEADDR); + return "Unknown"; +} + +/* RoCE -> Net driver */ + +/* Driver registration routines used to let the networking driver (bnxt_en) + * to know that the RoCE driver is now installed */ +static void bnxt_re_unregister_netdev(struct bnxt_re_dev *rdev) +{ + struct bnxt_en_dev *en_dev = rdev->en_dev; + int rc; + + rtnl_lock(); + rc = en_dev->en_ops->bnxt_unregister_device(rdev->en_dev, + BNXT_ROCE_ULP); + rtnl_unlock(); + if (rc) + dev_err(rdev_to_dev(rdev), "netdev %p unregister failed! rc = 0x%x", + rdev->en_dev->net, rc); + + clear_bit(BNXT_RE_FLAG_NETDEV_REGISTERED, &rdev->flags); +} + +static int bnxt_re_register_netdev(struct bnxt_re_dev *rdev) +{ + struct bnxt_en_dev *en_dev = rdev->en_dev; + int rc = 0; + + rtnl_lock(); + rc = en_dev->en_ops->bnxt_register_device(en_dev, + BNXT_ROCE_ULP, + &bnxt_re_ulp_ops, + rdev->adev); + rtnl_unlock(); + if (rc) { + dev_err(rdev_to_dev(rdev), "netdev %p register failed! rc = 0x%x", + rdev->netdev, rc); + return rc; + } + + return rc; +} + +static void bnxt_re_set_db_offset(struct bnxt_re_dev *rdev) +{ + struct bnxt_qplib_chip_ctx *cctx; + struct bnxt_en_dev *en_dev; + struct bnxt_qplib_res *res; + u32 l2db_len = 0; + u32 offset = 0; + u32 barlen; + int rc; + + res = &rdev->qplib_res; + en_dev = rdev->en_dev; + cctx = rdev->chip_ctx; + + /* Issue qcfg */ + rc = bnxt_re_hwrm_qcfg(rdev, &l2db_len, &offset); + if (rc) + dev_info(rdev_to_dev(rdev), + "Couldn't get DB bar size, Low latency framework is disabled\n"); + /* set register offsets for both UC and WC */ + if (_is_chip_p7(cctx)) + res->dpi_tbl.ucreg.offset = offset; + else + res->dpi_tbl.ucreg.offset = res->is_vf ? BNXT_QPLIB_DBR_VF_DB_OFFSET : + BNXT_QPLIB_DBR_PF_DB_OFFSET; + res->dpi_tbl.wcreg.offset = res->dpi_tbl.ucreg.offset; + + /* If WC mapping is disabled by L2 driver then en_dev->l2_db_size + * is equal to the DB-Bar actual size. This indicates that L2 + * is mapping entire bar as UC-. RoCE driver can't enable WC mapping + * in such cases and DB-push will be disabled. + */ + barlen = pci_resource_len(res->pdev, RCFW_DBR_PCI_BAR_REGION); + if (cctx->modes.db_push && l2db_len && en_dev->l2_db_size != barlen) { + res->dpi_tbl.wcreg.offset = en_dev->l2_db_size; + dev_info(rdev_to_dev(rdev), + "Low latency framework is enabled\n"); + } + + return; +} + +static void bnxt_re_set_drv_mode(struct bnxt_re_dev *rdev, u8 mode) +{ + struct bnxt_qplib_chip_ctx *cctx; + struct bnxt_en_dev *en_dev; + + en_dev = rdev->en_dev; + cctx = rdev->chip_ctx; + cctx->modes.wqe_mode = _is_chip_gen_p5_p7(rdev->chip_ctx) ? + mode : BNXT_QPLIB_WQE_MODE_STATIC; + cctx->modes.te_bypass = false; + if (bnxt_re_hwrm_qcaps(rdev)) + dev_err(rdev_to_dev(rdev), + "Failed to query hwrm qcaps\n"); + /* + * TODO: Need a better mechanism for spreading of the + * 512 extended PPP pages in the presence of VF and + * NPAR, until then not enabling push + */ + if (_is_chip_p7(rdev->chip_ctx) && cctx->modes.db_push) { + if (rdev->is_virtfn || BNXT_EN_NPAR(en_dev)) + cctx->modes.db_push = false; + } + + rdev->roce_mode = en_dev->flags & BNXT_EN_FLAG_ROCE_CAP; + dev_dbg(rdev_to_dev(rdev), + "RoCE is supported on the device - caps:0x%x", + rdev->roce_mode); + if (!_is_chip_gen_p5_p7(rdev->chip_ctx)) + rdev->roce_mode = BNXT_RE_FLAG_ROCEV2_CAP; + cctx->hw_stats_size = en_dev->hw_ring_stats_size; +} + +static void bnxt_re_destroy_chip_ctx(struct bnxt_re_dev *rdev) +{ + struct bnxt_qplib_chip_ctx *chip_ctx; + struct bnxt_qplib_res *res; + + if (!rdev->chip_ctx) + return; + + res = &rdev->qplib_res; + bnxt_qplib_unmap_db_bar(res); + + kfree(res->hctx); + res->rcfw = NULL; + kfree(rdev->dev_attr); + rdev->dev_attr = NULL; + + chip_ctx = rdev->chip_ctx; + rdev->chip_ctx = NULL; + res->cctx = NULL; + res->hctx = NULL; + res->pdev = NULL; + res->netdev = NULL; + kfree(chip_ctx); +} + +static int bnxt_re_setup_chip_ctx(struct bnxt_re_dev *rdev, u8 wqe_mode) +{ + struct bnxt_qplib_chip_ctx *chip_ctx; + struct bnxt_en_dev *en_dev; + int rc; + + en_dev = rdev->en_dev; + /* Supply pci device to qplib */ + rdev->qplib_res.pdev = en_dev->pdev; + rdev->qplib_res.netdev = rdev->netdev; + rdev->qplib_res.en_dev = en_dev; + + chip_ctx = kzalloc(sizeof(*chip_ctx), GFP_KERNEL); + if (!chip_ctx) + return -ENOMEM; + rdev->chip_ctx = chip_ctx; + rdev->qplib_res.cctx = chip_ctx; + rc = bnxt_re_query_hwrm_intf_version(rdev); + if (rc) + goto fail; + rdev->dev_attr = kzalloc(sizeof(*rdev->dev_attr), GFP_KERNEL); + if (!rdev->dev_attr) { + rc = -ENOMEM; + goto fail; + } + rdev->qplib_res.dattr = rdev->dev_attr; + rdev->qplib_res.rcfw = &rdev->rcfw; + rdev->qplib_res.is_vf = rdev->is_virtfn; + + rdev->qplib_res.hctx = kzalloc(sizeof(*rdev->qplib_res.hctx), + GFP_KERNEL); + if (!rdev->qplib_res.hctx) { + rc = -ENOMEM; + goto fail; + } + bnxt_re_set_drv_mode(rdev, wqe_mode); + + bnxt_re_set_db_offset(rdev); + rc = bnxt_qplib_map_db_bar(&rdev->qplib_res); + if (rc) + goto fail; + + rc = bnxt_qplib_enable_atomic_ops_to_root(en_dev->pdev); + if (rc) + dev_dbg(rdev_to_dev(rdev), + "platform doesn't support global atomics"); + + return 0; +fail: + kfree(rdev->chip_ctx); + rdev->chip_ctx = NULL; + + kfree(rdev->dev_attr); + rdev->dev_attr = NULL; + + kfree(rdev->qplib_res.hctx); + rdev->qplib_res.hctx = NULL; + return rc; +} + +static u16 bnxt_re_get_rtype(struct bnxt_re_dev *rdev) { + return _is_chip_gen_p5_p7(rdev->chip_ctx) ? + HWRM_RING_ALLOC_INPUT_RING_TYPE_NQ : + HWRM_RING_ALLOC_INPUT_RING_TYPE_ROCE_CMPL; +} + +static int bnxt_re_net_ring_free(struct bnxt_re_dev *rdev, u16 fw_ring_id) +{ + int rc = -EINVAL; + struct hwrm_ring_free_input req = {0}; + struct hwrm_ring_free_output resp; + struct bnxt_en_dev *en_dev = rdev->en_dev; + struct bnxt_fw_msg fw_msg; + + if (!en_dev) + return rc; + + /* To avoid unnecessary error messages during recovery. + * HW is anyway in error state. So dont send down the command */ + if (test_bit(BNXT_RE_FLAG_ERR_DEVICE_DETACHED, &rdev->flags)) + return 0; + + /* allocation had failed, no need to issue hwrm */ + if (fw_ring_id == 0xffff) + return 0; + + memset(&fw_msg, 0, sizeof(fw_msg)); + + bnxt_re_init_hwrm_hdr(rdev, (void *)&req, HWRM_RING_FREE, -1, -1); + req.ring_type = bnxt_re_get_rtype(rdev); + req.ring_id = cpu_to_le16(fw_ring_id); + bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp, + sizeof(resp), DFLT_HWRM_CMD_TIMEOUT); + rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg); + if (rc) { + dev_err(rdev_to_dev(rdev), + "Failed to free HW ring with rc = 0x%x", rc); + return rc; + } + dev_dbg(rdev_to_dev(rdev), "HW ring freed with id = 0x%x\n", + fw_ring_id); + + return rc; +} + +static int bnxt_re_net_ring_alloc(struct bnxt_re_dev *rdev, + struct bnxt_re_ring_attr *ring_attr, + u16 *fw_ring_id) +{ + int rc = -EINVAL; + struct hwrm_ring_alloc_input req = {0}; + struct hwrm_ring_alloc_output resp; + struct bnxt_en_dev *en_dev = rdev->en_dev; + struct bnxt_fw_msg fw_msg; + + if (!en_dev) + return rc; + + memset(&fw_msg, 0, sizeof(fw_msg)); + bnxt_re_init_hwrm_hdr(rdev, (void *)&req, HWRM_RING_ALLOC, -1, -1); + req.flags = cpu_to_le16(ring_attr->flags); + req.enables = 0; + req.page_tbl_addr = cpu_to_le64(ring_attr->dma_arr[0]); + if (ring_attr->pages > 1) { + /* Page size is in log2 units */ + req.page_size = BNXT_PAGE_SHIFT; + req.page_tbl_depth = 1; + } else { + req.page_size = 4; + req.page_tbl_depth = 0; + } + + req.fbo = 0; + /* Association of ring index with doorbell index and MSIX number */ + req.logical_id = cpu_to_le16(ring_attr->lrid); + req.length = cpu_to_le32(ring_attr->depth + 1); + req.ring_type = ring_attr->type; + req.int_mode = ring_attr->mode; + bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp, + sizeof(resp), DFLT_HWRM_CMD_TIMEOUT); + rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg); + if (rc) { + dev_err(rdev_to_dev(rdev), + "Failed to allocate HW ring with rc = 0x%x", rc); + return rc; + } + *fw_ring_id = le16_to_cpu(resp.ring_id); + dev_dbg(rdev_to_dev(rdev), + "HW ring allocated with id = 0x%x at slot 0x%x", + resp.ring_id, ring_attr->lrid); + + return rc; +} + +static int bnxt_re_net_stats_ctx_free(struct bnxt_re_dev *rdev, + u32 fw_stats_ctx_id, u16 tid) +{ + struct bnxt_en_dev *en_dev = rdev->en_dev; + struct hwrm_stat_ctx_free_input req = {0}; + struct hwrm_stat_ctx_free_output resp; + struct bnxt_fw_msg fw_msg; + int rc = -EINVAL; + + if (!en_dev) + return rc; + + /* To avoid unnecessary error messages during recovery. + * HW is anyway in error state. So dont send down the command */ + if (test_bit(BNXT_RE_FLAG_ERR_DEVICE_DETACHED, &rdev->flags)) + return 0; + memset(&fw_msg, 0, sizeof(fw_msg)); + bnxt_re_init_hwrm_hdr(rdev, (void *)&req, HWRM_STAT_CTX_FREE, -1, tid); + req.stat_ctx_id = cpu_to_le32(fw_stats_ctx_id); + bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp, + sizeof(resp), DFLT_HWRM_CMD_TIMEOUT); + rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg); + if (rc) { + dev_err(rdev_to_dev(rdev), + "Failed to free HW stats ctx with rc = 0x%x", rc); + return rc; + } + dev_dbg(rdev_to_dev(rdev), + "HW stats ctx freed with id = 0x%x", fw_stats_ctx_id); + + return rc; +} + +static int bnxt_re_net_stats_ctx_alloc(struct bnxt_re_dev *rdev, u16 tid) +{ + struct hwrm_stat_ctx_alloc_output resp = {}; + struct hwrm_stat_ctx_alloc_input req = {}; + struct bnxt_en_dev *en_dev = rdev->en_dev; + struct bnxt_qplib_stats *stat; + struct bnxt_qplib_ctx *hctx; + struct bnxt_fw_msg fw_msg; + int rc = 0; + + hctx = rdev->qplib_res.hctx; + stat = (tid == 0xffff) ? &hctx->stats : &hctx->stats2; + stat->fw_id = INVALID_STATS_CTX_ID; + + if (!en_dev) + return -EINVAL; + + memset(&fw_msg, 0, sizeof(fw_msg)); + bnxt_re_init_hwrm_hdr(rdev, (void *)&req, + HWRM_STAT_CTX_ALLOC, -1, tid); + req.update_period_ms = cpu_to_le32(1000); + req.stats_dma_length = rdev->chip_ctx->hw_stats_size; + req.stats_dma_addr = cpu_to_le64(stat->dma_map); + req.stat_ctx_flags = HWRM_STAT_CTX_ALLOC_INPUT_STAT_CTX_FLAGS_ROCE; + bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp, + sizeof(resp), DFLT_HWRM_CMD_TIMEOUT); + rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg); + if (rc) { + dev_err(rdev_to_dev(rdev), + "Failed to allocate HW stats ctx, rc = 0x%x", rc); + return rc; + } + stat->fw_id = le32_to_cpu(resp.stat_ctx_id); + dev_dbg(rdev_to_dev(rdev), "HW stats ctx allocated with id = 0x%x", + stat->fw_id); + + return rc; +} + +static void bnxt_re_net_unregister_async_event(struct bnxt_re_dev *rdev) +{ + const struct bnxt_en_ops *en_ops; + + if (rdev->is_virtfn || + test_bit(BNXT_RE_FLAG_ERR_DEVICE_DETACHED, &rdev->flags)) + return; + + memset(rdev->event_bitmap, 0, sizeof(rdev->event_bitmap)); + en_ops = rdev->en_dev->en_ops; + if (en_ops->bnxt_register_fw_async_events + (rdev->en_dev, BNXT_ROCE_ULP, + (unsigned long *)rdev->event_bitmap, + HWRM_ASYNC_EVENT_CMPL_EVENT_ID_DOORBELL_PACING_NQ_UPDATE)) + dev_err(rdev_to_dev(rdev), + "Failed to unregister async event"); +} + +static void bnxt_re_net_register_async_event(struct bnxt_re_dev *rdev) +{ + const struct bnxt_en_ops *en_ops; + + if (rdev->is_virtfn) + return; + + rdev->event_bitmap[0] |= + BIT(HWRM_ASYNC_EVENT_CMPL_EVENT_ID_DCB_CONFIG_CHANGE) | + BIT(HWRM_ASYNC_EVENT_CMPL_EVENT_ID_RESET_NOTIFY); + + rdev->event_bitmap[2] |= + BIT(HWRM_ASYNC_EVENT_CMPL_EVENT_ID_ERROR_REPORT - 64); + rdev->event_bitmap[2] |= + BIT(HWRM_ASYNC_EVENT_CMPL_EVENT_ID_DOORBELL_PACING_THRESHOLD - 64) | + BIT(HWRM_ASYNC_EVENT_CMPL_EVENT_ID_DOORBELL_PACING_NQ_UPDATE - 64); + en_ops = rdev->en_dev->en_ops; + if (en_ops->bnxt_register_fw_async_events + (rdev->en_dev, BNXT_ROCE_ULP, + (unsigned long *)rdev->event_bitmap, + HWRM_ASYNC_EVENT_CMPL_EVENT_ID_DOORBELL_PACING_NQ_UPDATE)) + dev_err(rdev_to_dev(rdev), + "Failed to reg Async event"); +} + +static int bnxt_re_query_hwrm_intf_version(struct bnxt_re_dev *rdev) +{ + struct bnxt_en_dev *en_dev = rdev->en_dev; + struct hwrm_ver_get_output resp = {0}; + struct hwrm_ver_get_input req = {0}; + struct bnxt_qplib_chip_ctx *cctx; + struct bnxt_fw_msg fw_msg; + int rc = 0; + + memset(&fw_msg, 0, sizeof(fw_msg)); + bnxt_re_init_hwrm_hdr(rdev, (void *)&req, + HWRM_VER_GET, -1, -1); + req.hwrm_intf_maj = HWRM_VERSION_MAJOR; + req.hwrm_intf_min = HWRM_VERSION_MINOR; + req.hwrm_intf_upd = HWRM_VERSION_UPDATE; + bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp, + sizeof(resp), DFLT_HWRM_CMD_TIMEOUT); + rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg); + if (rc) { + dev_err(rdev_to_dev(rdev), + "Failed to query HW version, rc = 0x%x", rc); + return rc; + } + cctx = rdev->chip_ctx; + cctx->hwrm_intf_ver = (u64) le16_to_cpu(resp.hwrm_intf_major) << 48 | + (u64) le16_to_cpu(resp.hwrm_intf_minor) << 32 | + (u64) le16_to_cpu(resp.hwrm_intf_build) << 16 | + le16_to_cpu(resp.hwrm_intf_patch); + + cctx->hwrm_cmd_max_timeout = le16_to_cpu(resp.max_req_timeout); + + if (!cctx->hwrm_cmd_max_timeout) + cctx->hwrm_cmd_max_timeout = RCFW_FW_STALL_MAX_TIMEOUT; + + cctx->chip_num = le16_to_cpu(resp.chip_num); + cctx->chip_rev = resp.chip_rev; + cctx->chip_metal = resp.chip_metal; + return 0; +} + +/* Query device config using common hwrm */ +static int bnxt_re_hwrm_qcfg(struct bnxt_re_dev *rdev, u32 *db_len, + u32 *offset) +{ + struct bnxt_en_dev *en_dev = rdev->en_dev; + struct hwrm_func_qcfg_output resp = {0}; + struct hwrm_func_qcfg_input req = {0}; + struct bnxt_fw_msg fw_msg; + int rc; + + memset(&fw_msg, 0, sizeof(fw_msg)); + bnxt_re_init_hwrm_hdr(rdev, (void *)&req, + HWRM_FUNC_QCFG, -1, -1); + req.fid = cpu_to_le16(0xffff); + bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp, + sizeof(resp), DFLT_HWRM_CMD_TIMEOUT); + rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg); + if (rc) { + dev_err(rdev_to_dev(rdev), + "Failed to query config, rc = %#x", rc); + return rc; + } + + *db_len = PAGE_ALIGN(le16_to_cpu(resp.l2_doorbell_bar_size_kb) * 1024); + *offset = PAGE_ALIGN(le16_to_cpu(resp.legacy_l2_db_size_kb) * 1024); + return 0; +} + +/* Query function capabilities using common hwrm */ +int bnxt_re_hwrm_qcaps(struct bnxt_re_dev *rdev) +{ + struct bnxt_en_dev *en_dev = rdev->en_dev; + struct hwrm_func_qcaps_output resp = {0}; + struct hwrm_func_qcaps_input req = {0}; + struct bnxt_qplib_chip_ctx *cctx; + struct bnxt_fw_msg fw_msg; + u8 push_enable = false; + int rc; + + cctx = rdev->chip_ctx; + memset(&fw_msg, 0, sizeof(fw_msg)); + bnxt_re_init_hwrm_hdr(rdev, (void *)&req, + HWRM_FUNC_QCAPS, -1, -1); + req.fid = cpu_to_le16(0xffff); + bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp, + sizeof(resp), DFLT_HWRM_CMD_TIMEOUT); + rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg); + if (rc) { + dev_err(rdev_to_dev(rdev), + "Failed to query capabilities, rc = %#x", rc); + return rc; + } + if (_is_chip_p7(rdev->chip_ctx)) + push_enable = + (resp.flags_ext & + HWRM_FUNC_QCAPS_OUTPUT_FLAGS_EXT_PPP_PUSH_MODE_SUPPORTED) ? + true : false; + else + push_enable = + (resp.flags & HWRM_FUNC_QCAPS_OUTPUT_FLAGS_WCB_PUSH_MODE) ? + true : false; + cctx->modes.db_push = push_enable; + + cctx->modes.dbr_pacing = + resp.flags_ext & HWRM_FUNC_QCAPS_OUTPUT_FLAGS_EXT_DBR_PACING_SUPPORTED ? + true : false; + cctx->modes.dbr_pacing_ext = + resp.flags_ext2 & + HWRM_FUNC_QCAPS_OUTPUT_FLAGS_EXT2_DBR_PACING_EXT_SUPPORTED ? + true : false; + cctx->modes.dbr_drop_recov = + (resp.flags_ext2 & + HWRM_FUNC_QCAPS_OUTPUT_FLAGS_EXT2_SW_DBR_DROP_RECOVERY_SUPPORTED) ? + true : false; + cctx->modes.dbr_pacing_v0 = + (resp.flags_ext2 & + HWRM_FUNC_QCAPS_OUTPUT_FLAGS_EXT2_DBR_PACING_V0_SUPPORTED) ? + true : false; + dev_dbg(rdev_to_dev(rdev), + "%s: cctx->modes.dbr_pacing = %d cctx->modes.dbr_pacing_ext = %d, dbr_drop_recov %d\n", + __func__, cctx->modes.dbr_pacing, cctx->modes.dbr_pacing_ext, cctx->modes.dbr_drop_recov); + + return 0; +} + +static int bnxt_re_hwrm_dbr_pacing_qcfg(struct bnxt_re_dev *rdev) +{ + struct bnxt_qplib_db_pacing_data *pacing_data = rdev->qplib_res.pacing_data; + struct hwrm_func_dbr_pacing_qcfg_output resp = {0}; + struct hwrm_func_dbr_pacing_qcfg_input req = {0}; + struct bnxt_en_dev *en_dev = rdev->en_dev; + struct bnxt_qplib_chip_ctx *cctx; + struct bnxt_fw_msg fw_msg; + u32 primary_nq_id; + int rc; + + cctx = rdev->chip_ctx; + memset(&fw_msg, 0, sizeof(fw_msg)); + bnxt_re_init_hwrm_hdr(rdev, (void *)&req, + HWRM_FUNC_DBR_PACING_QCFG, -1, -1); + bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp, + sizeof(resp), BNXT_RE_HWRM_CMD_TIMEOUT(rdev)); + rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg); + if (rc) { + dev_dbg(rdev_to_dev(rdev), + "Failed to query dbr pacing config, rc = %#x", rc); + return rc; + } + + primary_nq_id = le32_to_cpu(resp.primary_nq_id); + if (primary_nq_id == 0xffffffff && + !bnxt_qplib_dbr_pacing_ext_en(rdev->chip_ctx)) { + dev_err(rdev_to_dev(rdev), "%s:%d Invoke bnxt_qplib_dbr_pacing_set_primary_pf with 1\n", + __func__, __LINE__); + bnxt_qplib_dbr_pacing_set_primary_pf(rdev->chip_ctx, 1); + } + + if (bnxt_qplib_dbr_pacing_ext_en(rdev->chip_ctx)) { + struct bnxt_qplib_nq *nq; + + nq = &rdev->nqr.nq[0]; + /* Reset the primary capability */ + if (nq->ring_id != primary_nq_id) + bnxt_qplib_dbr_pacing_set_primary_pf(rdev->chip_ctx, 0); + } + + if ((resp.dbr_stat_db_fifo_reg & + HWRM_FUNC_DBR_PACING_QCFG_OUTPUT_DBR_STAT_DB_FIFO_REG_ADDR_SPACE_MASK) == + HWRM_FUNC_DBR_PACING_QCFG_OUTPUT_DBR_STAT_DB_FIFO_REG_ADDR_SPACE_GRC) + cctx->dbr_stat_db_fifo = + resp.dbr_stat_db_fifo_reg & + ~HWRM_FUNC_DBR_PACING_QCFG_OUTPUT_DBR_STAT_DB_FIFO_REG_ADDR_SPACE_MASK; + + if ((resp.dbr_throttling_aeq_arm_reg & + HWRM_FUNC_DBR_PACING_QCFG_OUTPUT_DBR_THROTTLING_AEQ_ARM_REG_ADDR_SPACE_MASK) + == HWRM_FUNC_DBR_PACING_QCFG_OUTPUT_DBR_THROTTLING_AEQ_ARM_REG_ADDR_SPACE_GRC) { + cctx->dbr_aeq_arm_reg = resp.dbr_throttling_aeq_arm_reg & + ~HWRM_FUNC_DBR_PACING_QCFG_OUTPUT_DBR_STAT_DB_FIFO_REG_ADDR_SPACE_MASK; + cctx->dbr_throttling_reg = cctx->dbr_aeq_arm_reg - 4; + } + pacing_data->fifo_max_depth = le32_to_cpu(resp.dbr_stat_db_max_fifo_depth); + if (!pacing_data->fifo_max_depth) + pacing_data->fifo_max_depth = BNXT_RE_MAX_FIFO_DEPTH(cctx); + pacing_data->fifo_room_mask = le32_to_cpu(resp.dbr_stat_db_fifo_reg_fifo_room_mask); + pacing_data->fifo_room_shift = resp.dbr_stat_db_fifo_reg_fifo_room_shift; + dev_dbg(rdev_to_dev(rdev), + "%s: nq:0x%x primary_pf:%d db_fifo:0x%x aeq_arm:0x%x i" + "fifo_max_depth 0x%x , resp.dbr_stat_db_max_fifo_depth 0x%x);\n", + __func__, resp.primary_nq_id, cctx->modes.dbr_primary_pf, + cctx->dbr_stat_db_fifo, cctx->dbr_aeq_arm_reg, + pacing_data->fifo_max_depth, + le32_to_cpu(resp.dbr_stat_db_max_fifo_depth)); + return 0; +} + +static int bnxt_re_hwrm_dbr_pacing_cfg(struct bnxt_re_dev *rdev, bool enable) +{ + struct hwrm_func_dbr_pacing_cfg_output resp = {0}; + struct hwrm_func_dbr_pacing_cfg_input req = {0}; + struct bnxt_en_dev *en_dev = rdev->en_dev; + struct bnxt_fw_msg fw_msg; + int rc; + + if (test_bit(BNXT_RE_FLAG_ERR_DEVICE_DETACHED, &rdev->flags)) + return 0; + + memset(&fw_msg, 0, sizeof(fw_msg)); + bnxt_re_init_hwrm_hdr(rdev, (void *)&req, + HWRM_FUNC_DBR_PACING_CFG, -1, -1); + if (enable) { + req.flags = HWRM_FUNC_DBR_PACING_CFG_INPUT_FLAGS_DBR_NQ_EVENT_ENABLE; + req.enables = + cpu_to_le32(HWRM_FUNC_DBR_PACING_CFG_INPUT_ENABLES_PRIMARY_NQ_ID_VALID | + HWRM_FUNC_DBR_PACING_CFG_INPUT_ENABLES_PACING_THRESHOLD_VALID); + } else { + req.flags = HWRM_FUNC_DBR_PACING_CFG_INPUT_FLAGS_DBR_NQ_EVENT_DISABLE; + } + req.primary_nq_id = cpu_to_le32(rdev->dbq_nq_id); + req.pacing_threshold = cpu_to_le32(rdev->dbq_watermark); + dev_dbg(rdev_to_dev(rdev), "%s: nq_id = 0x%x pacing_threshold = 0x%x", + __func__, req.primary_nq_id, req.pacing_threshold); + bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp, + sizeof(resp), BNXT_RE_HWRM_CMD_TIMEOUT(rdev)); + rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg); + if (rc) { + dev_dbg(rdev_to_dev(rdev), + "Failed to set dbr pacing config, rc = %#x", rc); + return rc; + } + return 0; +} + +/* Net -> RoCE driver */ + +/* Device */ +struct bnxt_re_dev *bnxt_re_from_netdev(struct ifnet *netdev) +{ + struct bnxt_re_dev *rdev; + + rcu_read_lock(); + list_for_each_entry_rcu(rdev, &bnxt_re_dev_list, list) { + if (rdev->netdev == netdev) { + rcu_read_unlock(); + dev_dbg(rdev_to_dev(rdev), + "netdev (%p) found, ref_count = 0x%x", + netdev, atomic_read(&rdev->ref_count)); + return rdev; + } + } + rcu_read_unlock(); + return NULL; +} + +static ssize_t show_rev(struct device *device, struct device_attribute *attr, + char *buf) +{ + struct bnxt_re_dev *rdev = to_bnxt_re_dev(device, ibdev.dev); + + return scnprintf(buf, PAGE_SIZE, "0x%x\n", rdev->en_dev->pdev->vendor); +} + + +static ssize_t show_hca(struct device *device, struct device_attribute *attr, + char *buf) +{ + struct bnxt_re_dev *rdev = to_bnxt_re_dev(device, ibdev.dev); + + return scnprintf(buf, PAGE_SIZE, "%s\n", rdev->ibdev.node_desc); +} + +static DEVICE_ATTR(hw_rev, 0444, show_rev, NULL); +static DEVICE_ATTR(hca_type, 0444, show_hca, NULL); +static struct device_attribute *bnxt_re_attributes[] = { + &dev_attr_hw_rev, + &dev_attr_hca_type +}; + +int ib_register_device_compat(struct bnxt_re_dev *rdev) +{ + struct ib_device *ibdev = &rdev->ibdev; + char name[IB_DEVICE_NAME_MAX]; + + memset(name, 0, IB_DEVICE_NAME_MAX); + strlcpy(name, "bnxt_re%d", IB_DEVICE_NAME_MAX); + + strlcpy(ibdev->name, name, IB_DEVICE_NAME_MAX); + + return ib_register_device(ibdev, NULL); +} + +static int bnxt_re_register_ib(struct bnxt_re_dev *rdev) +{ + struct ib_device *ibdev = &rdev->ibdev; + int ret = 0; + + /* ib device init */ + ibdev->owner = THIS_MODULE; + ibdev->uverbs_abi_ver = BNXT_RE_ABI_VERSION; + ibdev->node_type = RDMA_NODE_IB_CA; + strlcpy(ibdev->node_desc, BNXT_RE_DESC " HCA", + strlen(BNXT_RE_DESC) + 5); + ibdev->phys_port_cnt = 1; + + bnxt_qplib_get_guid(rdev->dev_addr, (u8 *)&ibdev->node_guid); + + /* Data path irqs is one less than the max msix vectors */ + ibdev->num_comp_vectors = rdev->nqr.num_msix - 1; + bnxt_re_set_dma_device(ibdev, rdev); + ibdev->local_dma_lkey = BNXT_QPLIB_RSVD_LKEY; + + /* User space */ + ibdev->uverbs_cmd_mask = + (1ull << IB_USER_VERBS_CMD_GET_CONTEXT) | + (1ull << IB_USER_VERBS_CMD_QUERY_DEVICE) | + (1ull << IB_USER_VERBS_CMD_QUERY_PORT) | + (1ull << IB_USER_VERBS_CMD_ALLOC_PD) | + (1ull << IB_USER_VERBS_CMD_DEALLOC_PD) | + (1ull << IB_USER_VERBS_CMD_REG_MR) | + (1ull << IB_USER_VERBS_CMD_DEREG_MR) | + (1ull << IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL) | + (1ull << IB_USER_VERBS_CMD_CREATE_CQ) | + (1ull << IB_USER_VERBS_CMD_DESTROY_CQ) | + (1ull << IB_USER_VERBS_CMD_CREATE_QP) | + (1ull << IB_USER_VERBS_CMD_MODIFY_QP) | + (1ull << IB_USER_VERBS_CMD_QUERY_QP) | + (1ull << IB_USER_VERBS_CMD_DESTROY_QP) | + (1ull << IB_USER_VERBS_CMD_REREG_MR) | + (1ull << IB_USER_VERBS_CMD_RESIZE_CQ) | + (1ull << IB_USER_VERBS_CMD_CREATE_SRQ) | + (1ull << IB_USER_VERBS_CMD_MODIFY_SRQ) | + (1ull << IB_USER_VERBS_CMD_QUERY_SRQ) | + (1ull << IB_USER_VERBS_CMD_DESTROY_SRQ) | + (1ull << IB_USER_VERBS_CMD_ALLOC_MW) | + (1ull << IB_USER_VERBS_CMD_DEALLOC_MW) | + (1ull << IB_USER_VERBS_CMD_CREATE_AH) | + (1ull << IB_USER_VERBS_CMD_MODIFY_AH) | + (1ull << IB_USER_VERBS_CMD_QUERY_AH) | + (1ull << IB_USER_VERBS_CMD_DESTROY_AH); + + ibdev->uverbs_ex_cmd_mask = (1ull << IB_USER_VERBS_EX_CMD_MODIFY_QP); + ibdev->uverbs_cmd_mask |= (1ull << IB_USER_VERBS_CMD_POLL_CQ); + +#define bnxt_re_ib_ah bnxt_re_ah +#define bnxt_re_ib_cq bnxt_re_cq +#define bnxt_re_ib_pd bnxt_re_pd +#define bnxt_re_ib_srq bnxt_re_srq +#define bnxt_re_ib_ucontext bnxt_re_ucontext + INIT_IB_DEVICE_OPS(&ibdev->ops, bnxt_re, BNXT_RE); + + ibdev->query_device = bnxt_re_query_device; + ibdev->modify_device = bnxt_re_modify_device; + ibdev->query_port = bnxt_re_query_port; + ibdev->modify_port = bnxt_re_modify_port; + ibdev->get_port_immutable = bnxt_re_get_port_immutable; + ibdev->query_pkey = bnxt_re_query_pkey; + ibdev->query_gid = bnxt_re_query_gid; + ibdev->get_netdev = bnxt_re_get_netdev; + ibdev->add_gid = bnxt_re_add_gid; + ibdev->del_gid = bnxt_re_del_gid; + ibdev->get_link_layer = bnxt_re_get_link_layer; + ibdev->alloc_pd = bnxt_re_alloc_pd; + ibdev->dealloc_pd = bnxt_re_dealloc_pd; + ibdev->create_ah = bnxt_re_create_ah; + ibdev->modify_ah = bnxt_re_modify_ah; + ibdev->query_ah = bnxt_re_query_ah; + ibdev->destroy_ah = bnxt_re_destroy_ah; + ibdev->create_srq = bnxt_re_create_srq; + ibdev->modify_srq = bnxt_re_modify_srq; + ibdev->query_srq = bnxt_re_query_srq; + ibdev->destroy_srq = bnxt_re_destroy_srq; + ibdev->post_srq_recv = bnxt_re_post_srq_recv; + ibdev->create_qp = bnxt_re_create_qp; + ibdev->modify_qp = bnxt_re_modify_qp; + ibdev->query_qp = bnxt_re_query_qp; + ibdev->destroy_qp = bnxt_re_destroy_qp; + ibdev->post_send = bnxt_re_post_send; + ibdev->post_recv = bnxt_re_post_recv; + ibdev->create_cq = bnxt_re_create_cq; + ibdev->modify_cq = bnxt_re_modify_cq; + ibdev->destroy_cq = bnxt_re_destroy_cq; + ibdev->resize_cq = bnxt_re_resize_cq; + ibdev->poll_cq = bnxt_re_poll_cq; + ibdev->req_notify_cq = bnxt_re_req_notify_cq; + ibdev->get_dma_mr = bnxt_re_get_dma_mr; + ibdev->get_hw_stats = bnxt_re_get_hw_stats; + ibdev->alloc_hw_stats = bnxt_re_alloc_hw_port_stats; + ibdev->dereg_mr = bnxt_re_dereg_mr; + ibdev->alloc_mr = bnxt_re_alloc_mr; + ibdev->map_mr_sg = bnxt_re_map_mr_sg; + ibdev->alloc_mw = bnxt_re_alloc_mw; + ibdev->dealloc_mw = bnxt_re_dealloc_mw; + ibdev->reg_user_mr = bnxt_re_reg_user_mr; + ibdev->rereg_user_mr = bnxt_re_rereg_user_mr; + ibdev->disassociate_ucontext = bnxt_re_disassociate_ucntx; + ibdev->alloc_ucontext = bnxt_re_alloc_ucontext; + ibdev->dealloc_ucontext = bnxt_re_dealloc_ucontext; + ibdev->mmap = bnxt_re_mmap; + ibdev->process_mad = bnxt_re_process_mad; + + ret = ib_register_device_compat(rdev); + return ret; +} + +static void bnxt_re_dev_dealloc(struct bnxt_re_dev *rdev) +{ + int i = BNXT_RE_REF_WAIT_COUNT; + + dev_dbg(rdev_to_dev(rdev), "%s:Remove the device %p\n", __func__, rdev); + /* Wait for rdev refcount to come down */ + while ((atomic_read(&rdev->ref_count) > 1) && i--) + msleep(100); + + if (atomic_read(&rdev->ref_count) > 1) + dev_err(rdev_to_dev(rdev), + "Failed waiting for ref count to deplete %d", + atomic_read(&rdev->ref_count)); + + atomic_set(&rdev->ref_count, 0); + if_rele(rdev->netdev); + rdev->netdev = NULL; + synchronize_rcu(); + + kfree(rdev->gid_map); + kfree(rdev->dbg_stats); + ib_dealloc_device(&rdev->ibdev); +} + +static struct bnxt_re_dev *bnxt_re_dev_alloc(struct ifnet *netdev, + struct bnxt_en_dev *en_dev) +{ + struct bnxt_re_dev *rdev; + u32 count; + + /* Allocate bnxt_re_dev instance here */ + rdev = (struct bnxt_re_dev *)compat_ib_alloc_device(sizeof(*rdev)); + if (!rdev) { + pr_err("%s: bnxt_re_dev allocation failure!", + ROCE_DRV_MODULE_NAME); + return NULL; + } + /* Default values */ + atomic_set(&rdev->ref_count, 0); + rdev->netdev = netdev; + dev_hold(rdev->netdev); + rdev->en_dev = en_dev; + rdev->id = rdev->en_dev->pdev->devfn; + INIT_LIST_HEAD(&rdev->qp_list); + mutex_init(&rdev->qp_lock); + mutex_init(&rdev->cc_lock); + mutex_init(&rdev->dbq_lock); + bnxt_re_clear_rsors_stat(&rdev->stats.rsors); + rdev->cosq[0] = rdev->cosq[1] = 0xFFFF; + rdev->min_tx_depth = 1; + rdev->stats.stats_query_sec = 1; + /* Disable priority vlan as the default mode is DSCP based PFC */ + rdev->cc_param.disable_prio_vlan_tx = 1; + + /* Initialize worker for DBR Pacing */ + INIT_WORK(&rdev->dbq_fifo_check_work, bnxt_re_db_fifo_check); + INIT_DELAYED_WORK(&rdev->dbq_pacing_work, bnxt_re_pacing_timer_exp); + rdev->gid_map = kzalloc(sizeof(*(rdev->gid_map)) * + BNXT_RE_MAX_SGID_ENTRIES, + GFP_KERNEL); + if (!rdev->gid_map) { + ib_dealloc_device(&rdev->ibdev); + return NULL; + } + for(count = 0; count < BNXT_RE_MAX_SGID_ENTRIES; count++) + rdev->gid_map[count] = -1; + + rdev->dbg_stats = kzalloc(sizeof(*rdev->dbg_stats), GFP_KERNEL); + if (!rdev->dbg_stats) { + ib_dealloc_device(&rdev->ibdev); + return NULL; + } + + return rdev; +} + +static int bnxt_re_handle_unaffi_async_event( + struct creq_func_event *unaffi_async) +{ + switch (unaffi_async->event) { + case CREQ_FUNC_EVENT_EVENT_TX_WQE_ERROR: + case CREQ_FUNC_EVENT_EVENT_TX_DATA_ERROR: + case CREQ_FUNC_EVENT_EVENT_RX_WQE_ERROR: + case CREQ_FUNC_EVENT_EVENT_RX_DATA_ERROR: + case CREQ_FUNC_EVENT_EVENT_CQ_ERROR: + case CREQ_FUNC_EVENT_EVENT_TQM_ERROR: + case CREQ_FUNC_EVENT_EVENT_CFCQ_ERROR: + case CREQ_FUNC_EVENT_EVENT_CFCS_ERROR: + case CREQ_FUNC_EVENT_EVENT_CFCC_ERROR: + case CREQ_FUNC_EVENT_EVENT_CFCM_ERROR: + case CREQ_FUNC_EVENT_EVENT_TIM_ERROR: + break; + default: + return -EINVAL; + } + return 0; +} + +static int bnxt_re_handle_qp_async_event(void *qp_event, struct bnxt_re_qp *qp) +{ + struct creq_qp_error_notification *err_event; + struct ib_event event; + unsigned int flags; + + if (qp->qplib_qp.state == CMDQ_MODIFY_QP_NEW_STATE_ERR && + !qp->qplib_qp.is_user) { + flags = bnxt_re_lock_cqs(qp); + bnxt_qplib_add_flush_qp(&qp->qplib_qp); + bnxt_re_unlock_cqs(qp, flags); + } + memset(&event, 0, sizeof(event)); + event.device = &qp->rdev->ibdev; + event.element.qp = &qp->ib_qp; + event.event = IB_EVENT_QP_FATAL; + + err_event = qp_event; + switch(err_event->res_err_state_reason) { + case CFCQ_RES_ERR_STATE_REASON_RES_EXCEED_MAX: + case CFCQ_RES_ERR_STATE_REASON_RES_PAYLOAD_LENGTH_MISMATCH: + case CFCQ_RES_ERR_STATE_REASON_RES_OPCODE_ERROR: + case CFCQ_RES_ERR_STATE_REASON_RES_PSN_SEQ_ERROR_RETRY_LIMIT: + case CFCQ_RES_ERR_STATE_REASON_RES_RX_INVALID_R_KEY: + case CFCQ_RES_ERR_STATE_REASON_RES_RX_DOMAIN_ERROR: + case CFCQ_RES_ERR_STATE_REASON_RES_RX_NO_PERMISSION: + case CFCQ_RES_ERR_STATE_REASON_RES_RX_RANGE_ERROR: + case CFCQ_RES_ERR_STATE_REASON_RES_TX_INVALID_R_KEY: + case CFCQ_RES_ERR_STATE_REASON_RES_TX_DOMAIN_ERROR: + case CFCQ_RES_ERR_STATE_REASON_RES_TX_NO_PERMISSION: + case CFCQ_RES_ERR_STATE_REASON_RES_TX_RANGE_ERROR: + case CFCQ_RES_ERR_STATE_REASON_RES_IVALID_DUP_RKEY: + case CFCQ_RES_ERR_STATE_REASON_RES_UNALIGN_ATOMIC: + event.event = IB_EVENT_QP_ACCESS_ERR; + break; + case CFCQ_RES_ERR_STATE_REASON_RES_EXCEEDS_WQE: + case CFCQ_RES_ERR_STATE_REASON_RES_WQE_FORMAT_ERROR: + case CFCQ_RES_ERR_STATE_REASON_RES_SRQ_LOAD_ERROR: + case CFCQ_RES_ERR_STATE_REASON_RES_UNSUPPORTED_OPCODE: + case CFCQ_RES_ERR_STATE_REASON_RES_REM_INVALIDATE: + event.event = IB_EVENT_QP_REQ_ERR; + break; + case CFCQ_RES_ERR_STATE_REASON_RES_IRRQ_OFLOW: + case CFCQ_RES_ERR_STATE_REASON_RES_CMP_ERROR: + case CFCQ_RES_ERR_STATE_REASON_RES_CQ_LOAD_ERROR: + case CFCQ_RES_ERR_STATE_REASON_RES_TX_PCI_ERROR: + case CFCQ_RES_ERR_STATE_REASON_RES_RX_PCI_ERROR: + case CFCQ_RES_ERR_STATE_REASON_RES_MEMORY_ERROR: + case CFCQ_RES_ERR_STATE_REASON_RES_SRQ_ERROR: + event.event = IB_EVENT_QP_FATAL; + break; + default: + if (qp->qplib_qp.srq) + event.event = IB_EVENT_QP_LAST_WQE_REACHED; + break; + } + + if (err_event->res_err_state_reason) + dev_err(rdev_to_dev(qp->rdev), + "%s %s qp_id: %d cons (%d %d) req (%d %d) res (%d %d)\n", + __func__, qp->qplib_qp.is_user ? "user" : "kernel", + qp->qplib_qp.id, + err_event->sq_cons_idx, + err_event->rq_cons_idx, + err_event->req_slow_path_state, + err_event->req_err_state_reason, + err_event->res_slow_path_state, + err_event->res_err_state_reason); + + if (event.device && qp->ib_qp.event_handler) + qp->ib_qp.event_handler(&event, qp->ib_qp.qp_context); + + return 0; +} + +static int bnxt_re_handle_cq_async_error(void *event, struct bnxt_re_cq *cq) +{ + struct creq_cq_error_notification *cqerr; + bool send = false; + + cqerr = event; + switch (cqerr->cq_err_reason) { + case CREQ_CQ_ERROR_NOTIFICATION_CQ_ERR_REASON_REQ_CQ_INVALID_ERROR: + case CREQ_CQ_ERROR_NOTIFICATION_CQ_ERR_REASON_REQ_CQ_OVERFLOW_ERROR: + case CREQ_CQ_ERROR_NOTIFICATION_CQ_ERR_REASON_REQ_CQ_LOAD_ERROR: + case CREQ_CQ_ERROR_NOTIFICATION_CQ_ERR_REASON_RES_CQ_INVALID_ERROR: + case CREQ_CQ_ERROR_NOTIFICATION_CQ_ERR_REASON_RES_CQ_OVERFLOW_ERROR: + case CREQ_CQ_ERROR_NOTIFICATION_CQ_ERR_REASON_RES_CQ_LOAD_ERROR: + send = true; + default: + break; + } + + if (send && cq->ibcq.event_handler) { + struct ib_event ibevent = {}; + + ibevent.event = IB_EVENT_CQ_ERR; + ibevent.element.cq = &cq->ibcq; + ibevent.device = &cq->rdev->ibdev; + + dev_err(rdev_to_dev(cq->rdev), + "%s err reason %d\n", __func__, cqerr->cq_err_reason); + cq->ibcq.event_handler(&ibevent, cq->ibcq.cq_context); + } + + cq->qplib_cq.is_cq_err_event = true; + + return 0; +} + +static int bnxt_re_handle_affi_async_event(struct creq_qp_event *affi_async, + void *obj) +{ + struct bnxt_qplib_qp *qplqp; + struct bnxt_qplib_cq *qplcq; + struct bnxt_re_qp *qp; + struct bnxt_re_cq *cq; + int rc = 0; + u8 event; + + if (!obj) + return rc; /* QP was already dead, still return success */ + + event = affi_async->event; + switch (event) { + case CREQ_QP_EVENT_EVENT_QP_ERROR_NOTIFICATION: + qplqp = obj; + qp = container_of(qplqp, struct bnxt_re_qp, qplib_qp); + rc = bnxt_re_handle_qp_async_event(affi_async, qp); + break; + case CREQ_QP_EVENT_EVENT_CQ_ERROR_NOTIFICATION: + qplcq = obj; + cq = container_of(qplcq, struct bnxt_re_cq, qplib_cq); + rc = bnxt_re_handle_cq_async_error(affi_async, cq); + break; + default: + rc = -EINVAL; + } + + return rc; +} + +static int bnxt_re_aeq_handler(struct bnxt_qplib_rcfw *rcfw, + void *aeqe, void *obj) +{ + struct creq_func_event *unaffi_async; + struct creq_qp_event *affi_async; + u8 type; + int rc; + + type = ((struct creq_base *)aeqe)->type; + if (type == CREQ_BASE_TYPE_FUNC_EVENT) { + unaffi_async = aeqe; + rc = bnxt_re_handle_unaffi_async_event(unaffi_async); + } else { + affi_async = aeqe; + rc = bnxt_re_handle_affi_async_event(affi_async, obj); + } + + return rc; +} + +static int bnxt_re_srqn_handler(struct bnxt_qplib_nq *nq, + struct bnxt_qplib_srq *handle, u8 event) +{ + struct bnxt_re_srq *srq = to_bnxt_re(handle, struct bnxt_re_srq, + qplib_srq); + struct ib_event ib_event; + + if (srq == NULL) { + pr_err("%s: SRQ is NULL, SRQN not handled", + ROCE_DRV_MODULE_NAME); + return -EINVAL; + } + ib_event.device = &srq->rdev->ibdev; + ib_event.element.srq = &srq->ibsrq; + if (event == NQ_SRQ_EVENT_EVENT_SRQ_THRESHOLD_EVENT) + ib_event.event = IB_EVENT_SRQ_LIMIT_REACHED; + else + ib_event.event = IB_EVENT_SRQ_ERR; + + if (srq->ibsrq.event_handler) { + /* Lock event_handler? */ + (*srq->ibsrq.event_handler)(&ib_event, + srq->ibsrq.srq_context); + } + return 0; +} + +static int bnxt_re_cqn_handler(struct bnxt_qplib_nq *nq, + struct bnxt_qplib_cq *handle) +{ + struct bnxt_re_cq *cq = to_bnxt_re(handle, struct bnxt_re_cq, + qplib_cq); + u32 *cq_ptr; + + if (cq == NULL) { + pr_err("%s: CQ is NULL, CQN not handled", + ROCE_DRV_MODULE_NAME); + return -EINVAL; + } + /* CQ already in destroy path. Do not handle any more events */ + if (handle->destroyed || !atomic_read(&cq->ibcq.usecnt)) { + if (!handle->destroyed) + dev_dbg(NULL, "%s: CQ being destroyed, CQN not handled", + ROCE_DRV_MODULE_NAME); + return 0; + } + + if (cq->ibcq.comp_handler) { + if (cq->uctx_cq_page) { + cq_ptr = (u32 *)cq->uctx_cq_page; + *cq_ptr = cq->qplib_cq.toggle; + } + /* Lock comp_handler? */ + (*cq->ibcq.comp_handler)(&cq->ibcq, cq->ibcq.cq_context); + } + + return 0; +} + +struct bnxt_qplib_nq *bnxt_re_get_nq(struct bnxt_re_dev *rdev) +{ + int min, indx; + + mutex_lock(&rdev->nqr.load_lock); + for (indx = 0, min = 0; indx < (rdev->nqr.num_msix - 1); indx++) { + if (rdev->nqr.nq[min].load > rdev->nqr.nq[indx].load) + min = indx; + } + rdev->nqr.nq[min].load++; + mutex_unlock(&rdev->nqr.load_lock); + + return &rdev->nqr.nq[min]; +} + +void bnxt_re_put_nq(struct bnxt_re_dev *rdev, struct bnxt_qplib_nq *nq) +{ + mutex_lock(&rdev->nqr.load_lock); + nq->load--; + mutex_unlock(&rdev->nqr.load_lock); +} + +static bool bnxt_re_check_min_attr(struct bnxt_re_dev *rdev) +{ + struct bnxt_qplib_dev_attr *attr; + bool rc = true; + + attr = rdev->dev_attr; + + if (!attr->max_cq || !attr->max_qp || + !attr->max_sgid || !attr->max_mr) { + dev_err(rdev_to_dev(rdev),"Insufficient RoCE resources"); + dev_dbg(rdev_to_dev(rdev), + "max_cq = %d, max_qp = %d, max_dpi = %d, max_sgid = %d, max_mr = %d", + attr->max_cq, attr->max_qp, attr->max_dpi, + attr->max_sgid, attr->max_mr); + rc = false; + } + return rc; +} + +static void bnxt_re_dispatch_event(struct ib_device *ibdev, struct ib_qp *qp, + u8 port_num, enum ib_event_type event) +{ + struct ib_event ib_event; + + ib_event.device = ibdev; + if (qp) { + ib_event.element.qp = qp; + ib_event.event = event; + if (qp->event_handler) + qp->event_handler(&ib_event, qp->qp_context); + } else { + ib_event.element.port_num = port_num; + ib_event.event = event; + ib_dispatch_event(&ib_event); + } + + dev_dbg(rdev_to_dev(to_bnxt_re_dev(ibdev, ibdev)), + "ibdev %p Event 0x%x port_num 0x%x", ibdev, event, port_num); +} + +static bool bnxt_re_is_qp1_or_shadow_qp(struct bnxt_re_dev *rdev, + struct bnxt_re_qp *qp) +{ + if (rdev->gsi_ctx.gsi_qp_mode == BNXT_RE_GSI_MODE_ALL) + return (qp->ib_qp.qp_type == IB_QPT_GSI) || + (qp == rdev->gsi_ctx.gsi_sqp); + else + return (qp->ib_qp.qp_type == IB_QPT_GSI); +} + +static void bnxt_re_stop_all_nonqp1_nonshadow_qps(struct bnxt_re_dev *rdev) +{ + struct bnxt_qplib_qp *qpl_qp; + bool dev_detached = false; + struct ib_qp_attr qp_attr; + int num_qps_stopped = 0; + int mask = IB_QP_STATE; + struct bnxt_re_qp *qp; + unsigned long flags; + + if (!rdev) + return; + +restart: + if (test_bit(BNXT_RE_FLAG_ERR_DEVICE_DETACHED, &rdev->flags)) + dev_detached = true; + + qp_attr.qp_state = IB_QPS_ERR; + mutex_lock(&rdev->qp_lock); + list_for_each_entry(qp, &rdev->qp_list, list) { + qpl_qp = &qp->qplib_qp; + if (dev_detached || !bnxt_re_is_qp1_or_shadow_qp(rdev, qp)) { + if (qpl_qp->state != + CMDQ_MODIFY_QP_NEW_STATE_RESET && + qpl_qp->state != + CMDQ_MODIFY_QP_NEW_STATE_ERR) { + if (dev_detached) { + /* + * Cant actually send the command down, + * marking the state for bookkeeping + */ + qpl_qp->state = + CMDQ_MODIFY_QP_NEW_STATE_ERR; + qpl_qp->cur_qp_state = qpl_qp->state; + if (!qpl_qp->is_user) { + /* Add to flush list */ + flags = bnxt_re_lock_cqs(qp); + bnxt_qplib_add_flush_qp(qpl_qp); + bnxt_re_unlock_cqs(qp, flags); + } + } else { + num_qps_stopped++; + bnxt_re_modify_qp(&qp->ib_qp, + &qp_attr, mask, + NULL); + } + + bnxt_re_dispatch_event(&rdev->ibdev, &qp->ib_qp, + 1, IB_EVENT_QP_FATAL); + /* + * 1. Release qp_lock after a budget to unblock other verb + * requests (like qp_destroy) from stack. + * 2. Traverse through the qp_list freshly as addition / deletion + * might have happened since qp_lock is getting released here. + */ + if (num_qps_stopped % BNXT_RE_STOP_QPS_BUDGET == 0) { + mutex_unlock(&rdev->qp_lock); + goto restart; + } + } + } + } + + mutex_unlock(&rdev->qp_lock); +} + +static int bnxt_re_update_gid(struct bnxt_re_dev *rdev) +{ + struct bnxt_qplib_sgid_tbl *sgid_tbl = &rdev->qplib_res.sgid_tbl; + struct bnxt_qplib_gid gid; + u16 gid_idx, index; + int rc = 0; + + if (!test_bit(BNXT_RE_FLAG_IBDEV_REGISTERED, &rdev->flags)) + return 0; + + if (sgid_tbl == NULL) { + dev_err(rdev_to_dev(rdev), "QPLIB: SGID table not allocated"); + return -EINVAL; + } + + for (index = 0; index < sgid_tbl->active; index++) { + gid_idx = sgid_tbl->hw_id[index]; + + if (!memcmp(&sgid_tbl->tbl[index], &bnxt_qplib_gid_zero, + sizeof(bnxt_qplib_gid_zero))) + continue; + /* Need to modify the VLAN enable setting of non VLAN GID only + * as setting is done for VLAN GID while adding GID + * + * If disable_prio_vlan_tx is enable, then we'll need to remove the + * vlan entry from the sgid_tbl. + */ + if (sgid_tbl->vlan[index] == true) + continue; + + memcpy(&gid, &sgid_tbl->tbl[index], sizeof(gid)); + + rc = bnxt_qplib_update_sgid(sgid_tbl, &gid, gid_idx, + rdev->dev_addr); + } + + return rc; +} + +static void bnxt_re_clear_cc(struct bnxt_re_dev *rdev) +{ + struct bnxt_qplib_cc_param *cc_param = &rdev->cc_param; + + if (_is_chip_p7(rdev->chip_ctx)) { + cc_param->mask = CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_TOS_DSCP; + } else { + cc_param->mask = (CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_CC_MODE | + CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_ENABLE_CC | + CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_TOS_ECN); + + if (!is_qport_service_type_supported(rdev)) + cc_param->mask |= + (CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_ALT_VLAN_PCP | + CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_ALT_TOS_DSCP | + CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_TOS_DSCP); + } + + cc_param->cur_mask = cc_param->mask; + + if (bnxt_qplib_modify_cc(&rdev->qplib_res, cc_param)) + dev_err(rdev_to_dev(rdev), "Failed to modify cc\n"); +} + +static int bnxt_re_setup_cc(struct bnxt_re_dev *rdev) +{ + struct bnxt_qplib_cc_param *cc_param = &rdev->cc_param; + int rc; + + if (_is_chip_p7(rdev->chip_ctx)) { + cc_param->enable = 0x0; + cc_param->mask = CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_TOS_DSCP; + } else { + cc_param->enable = 0x1; + cc_param->mask = (CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_CC_MODE | + CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_ENABLE_CC | + CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_TOS_ECN); + + if (!is_qport_service_type_supported(rdev)) + cc_param->mask |= + (CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_ALT_VLAN_PCP | + CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_ALT_TOS_DSCP | + CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_TOS_DSCP); + } + + cc_param->cur_mask = cc_param->mask; + + rc = bnxt_qplib_modify_cc(&rdev->qplib_res, cc_param); + if (rc) { + dev_err(rdev_to_dev(rdev), "Failed to modify cc\n"); + return rc; + } + /* Reset the programming mask */ + cc_param->mask = 0; + if (cc_param->qp1_tos_dscp != cc_param->tos_dscp) { + cc_param->qp1_tos_dscp = cc_param->tos_dscp; + rc = bnxt_re_update_qp1_tos_dscp(rdev); + if (rc) { + dev_err(rdev_to_dev(rdev), "%s:Failed to modify QP1:%d", + __func__, rc); + goto clear; + } + } + return 0; + +clear: + bnxt_re_clear_cc(rdev); + return rc; +} + +int bnxt_re_query_hwrm_dscp2pri(struct bnxt_re_dev *rdev, + struct bnxt_re_dscp2pri *d2p, u16 *count, + u16 target_id) +{ + struct bnxt_en_dev *en_dev = rdev->en_dev; + struct hwrm_queue_dscp2pri_qcfg_input req; + struct hwrm_queue_dscp2pri_qcfg_output resp; + struct bnxt_re_dscp2pri *dscp2pri; + struct bnxt_fw_msg fw_msg; + u16 in_count = *count; + dma_addr_t dma_handle; + int rc = 0, i; + u16 data_len; + u8 *kmem; + + data_len = *count * sizeof(*dscp2pri); + memset(&fw_msg, 0, sizeof(fw_msg)); + memset(&req, 0, sizeof(req)); + bnxt_re_init_hwrm_hdr(rdev, (void *)&req, + HWRM_QUEUE_DSCP2PRI_QCFG, -1, target_id); + req.port_id = (target_id == 0xFFFF) ? en_dev->pf_port_id : 1; + + kmem = dma_zalloc_coherent(&en_dev->pdev->dev, data_len, &dma_handle, + GFP_KERNEL); + if (!kmem) { + dev_err(rdev_to_dev(rdev), + "dma_zalloc_coherent failure, length = %u\n", + (unsigned)data_len); + return -ENOMEM; + } + req.dest_data_addr = cpu_to_le64(dma_handle); + req.dest_data_buffer_size = cpu_to_le16(data_len); + bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp, + sizeof(resp), DFLT_HWRM_CMD_TIMEOUT); + rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg); + if (rc) + goto out; + + /* Upload the DSCP-MASK-PRI tuple(s) */ + dscp2pri = (struct bnxt_re_dscp2pri *)kmem; + for (i = 0; i < le16_to_cpu(resp.entry_cnt) && i < in_count; i++) { + d2p[i].dscp = dscp2pri->dscp; + d2p[i].mask = dscp2pri->mask; + d2p[i].pri = dscp2pri->pri; + dscp2pri++; + } + *count = le16_to_cpu(resp.entry_cnt); +out: + dma_free_coherent(&en_dev->pdev->dev, data_len, kmem, dma_handle); + return rc; +} + +int bnxt_re_prio_vlan_tx_update(struct bnxt_re_dev *rdev) +{ + /* Remove the VLAN from the GID entry */ + if (rdev->cc_param.disable_prio_vlan_tx) + rdev->qplib_res.prio = false; + else + rdev->qplib_res.prio = true; + + return bnxt_re_update_gid(rdev); +} + +int bnxt_re_set_hwrm_dscp2pri(struct bnxt_re_dev *rdev, + struct bnxt_re_dscp2pri *d2p, u16 count, + u16 target_id) +{ + struct bnxt_en_dev *en_dev = rdev->en_dev; + struct hwrm_queue_dscp2pri_cfg_input req; + struct hwrm_queue_dscp2pri_cfg_output resp; + struct bnxt_fw_msg fw_msg; + struct bnxt_re_dscp2pri *dscp2pri; + int i, rc, data_len = 3 * 256; + dma_addr_t dma_handle; + u8 *kmem; + + memset(&req, 0, sizeof(req)); + memset(&fw_msg, 0, sizeof(fw_msg)); + bnxt_re_init_hwrm_hdr(rdev, (void *)&req, + HWRM_QUEUE_DSCP2PRI_CFG, -1, target_id); + req.port_id = (target_id == 0xFFFF) ? en_dev->pf_port_id : 1; + + kmem = dma_alloc_coherent(&en_dev->pdev->dev, data_len, &dma_handle, + GFP_KERNEL); + if (!kmem) { + dev_err(rdev_to_dev(rdev), + "dma_alloc_coherent failure, length = %u\n", + (unsigned)data_len); + return -ENOMEM; + } + req.src_data_addr = cpu_to_le64(dma_handle); + + /* Download the DSCP-MASK-PRI tuple(s) */ + dscp2pri = (struct bnxt_re_dscp2pri *)kmem; + for (i = 0; i < count; i++) { + dscp2pri->dscp = d2p[i].dscp; + dscp2pri->mask = d2p[i].mask; + dscp2pri->pri = d2p[i].pri; + dscp2pri++; + } + + req.entry_cnt = cpu_to_le16(count); + bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp, + sizeof(resp), DFLT_HWRM_CMD_TIMEOUT); + rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg); + dma_free_coherent(&en_dev->pdev->dev, data_len, kmem, dma_handle); + return rc; +} + +int bnxt_re_query_hwrm_qportcfg(struct bnxt_re_dev *rdev, + struct bnxt_re_tc_rec *tc_rec, u16 tid) +{ + u8 max_tc, tc, *qptr, *type_ptr0, *type_ptr1; + struct hwrm_queue_qportcfg_output resp = {0}; + struct hwrm_queue_qportcfg_input req = {0}; + struct bnxt_en_dev *en_dev = rdev->en_dev; + struct bnxt_fw_msg fw_msg; + bool def_init = false; + u8 *tmp_type; + u8 cos_id; + int rc; + + memset(&fw_msg, 0, sizeof(fw_msg)); + bnxt_re_init_hwrm_hdr(rdev, (void *)&req, HWRM_QUEUE_QPORTCFG, + -1, tid); + req.port_id = (tid == 0xFFFF) ? en_dev->pf_port_id : 1; + if (BNXT_EN_ASYM_Q(en_dev)) + req.flags = htole32(HWRM_QUEUE_QPORTCFG_INPUT_FLAGS_PATH_RX); + + bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp, + sizeof(resp), DFLT_HWRM_CMD_TIMEOUT); + rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg); + if (rc) + return rc; + + if (!resp.max_configurable_queues) + return -EINVAL; + + max_tc = resp.max_configurable_queues; + tc_rec->max_tc = max_tc; + + if (resp.queue_cfg_info & HWRM_QUEUE_QPORTCFG_OUTPUT_QUEUE_CFG_INFO_USE_PROFILE_TYPE) + tc_rec->serv_type_enabled = true; + + qptr = &resp.queue_id0; + type_ptr0 = &resp.queue_id0_service_profile_type; + type_ptr1 = &resp.queue_id1_service_profile_type; + for (tc = 0; tc < max_tc; tc++) { + tmp_type = tc ? type_ptr1 + (tc - 1) : type_ptr0; + + cos_id = *qptr++; + /* RoCE CoS queue is the first cos queue. + * For MP12 and MP17 order is 405 and 141015. + */ + if (is_bnxt_roce_queue(rdev, *qptr, *tmp_type)) { + tc_rec->cos_id_roce = cos_id; + tc_rec->tc_roce = tc; + } else if (is_bnxt_cnp_queue(rdev, *qptr, *tmp_type)) { + tc_rec->cos_id_cnp = cos_id; + tc_rec->tc_cnp = tc; + } else if (!def_init) { + def_init = true; + tc_rec->tc_def = tc; + tc_rec->cos_id_def = cos_id; + } + qptr++; + } + + return rc; +} + +int bnxt_re_hwrm_cos2bw_qcfg(struct bnxt_re_dev *rdev, u16 target_id, + struct bnxt_re_cos2bw_cfg *cfg) +{ + struct bnxt_en_dev *en_dev = rdev->en_dev; + struct hwrm_queue_cos2bw_qcfg_output resp; + struct hwrm_queue_cos2bw_qcfg_input req = {0}; + struct bnxt_fw_msg fw_msg; + int rc, indx; + void *data; + + memset(&fw_msg, 0, sizeof(fw_msg)); + bnxt_re_init_hwrm_hdr(rdev, (void *)&req, + HWRM_QUEUE_COS2BW_QCFG, -1, target_id); + req.port_id = (target_id == 0xFFFF) ? en_dev->pf_port_id : 1; + + bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp, + sizeof(resp), DFLT_HWRM_CMD_TIMEOUT); + rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg); + if (rc) + return rc; + data = &resp.queue_id0 + offsetof(struct bnxt_re_cos2bw_cfg, + queue_id); + for (indx = 0; indx < 8; indx++, data += (sizeof(cfg->cfg))) { + memcpy(&cfg->cfg, data, sizeof(cfg->cfg)); + if (indx == 0) + cfg->queue_id = resp.queue_id0; + cfg++; + } + + return rc; +} + +int bnxt_re_hwrm_cos2bw_cfg(struct bnxt_re_dev *rdev, u16 target_id, + struct bnxt_re_cos2bw_cfg *cfg) +{ + struct bnxt_en_dev *en_dev = rdev->en_dev; + struct hwrm_queue_cos2bw_cfg_input req = {0}; + struct hwrm_queue_cos2bw_cfg_output resp = {0}; + struct bnxt_fw_msg fw_msg; + void *data; + int indx; + int rc; + + memset(&fw_msg, 0, sizeof(fw_msg)); + bnxt_re_init_hwrm_hdr(rdev, (void *)&req, + HWRM_QUEUE_COS2BW_CFG, -1, target_id); + req.port_id = (target_id == 0xFFFF) ? en_dev->pf_port_id : 1; + + /* Chimp wants enable bit to retain previous + * config done by L2 driver + */ + for (indx = 0; indx < 8; indx++) { + if (cfg[indx].queue_id < 40) { + req.enables |= cpu_to_le32( + HWRM_QUEUE_COS2BW_CFG_INPUT_ENABLES_COS_QUEUE_ID0_VALID << + indx); + } + + data = (char *)&req.unused_0 + indx * (sizeof(*cfg) - 4); + memcpy(data, &cfg[indx].queue_id, sizeof(*cfg) - 4); + if (indx == 0) { + req.queue_id0 = cfg[0].queue_id; + req.unused_0 = 0; + } + } + + memset(&resp, 0, sizeof(resp)); + bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp, + sizeof(resp), DFLT_HWRM_CMD_TIMEOUT); + rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg); + return rc; +} + +int bnxt_re_host_pf_id_query(struct bnxt_re_dev *rdev, + struct bnxt_qplib_query_fn_info *fn_info, + u32 *pf_mask, u32 *first_pf) +{ + struct hwrm_func_host_pf_ids_query_output resp = {0}; + struct hwrm_func_host_pf_ids_query_input req; + struct bnxt_en_dev *en_dev = rdev->en_dev; + struct bnxt_fw_msg fw_msg; + int rc; + + memset(&fw_msg, 0, sizeof(fw_msg)); + memset(&req, 0, sizeof(req)); + bnxt_re_init_hwrm_hdr(rdev, (void *)&req, + HWRM_FUNC_HOST_PF_IDS_QUERY, -1, -1); + /* To query the info from the host EPs */ + switch (fn_info->host) { + case HWRM_FUNC_HOST_PF_IDS_QUERY_INPUT_HOST_SOC: + case HWRM_FUNC_HOST_PF_IDS_QUERY_INPUT_HOST_EP_0: + case HWRM_FUNC_HOST_PF_IDS_QUERY_INPUT_HOST_EP_1: + case HWRM_FUNC_HOST_PF_IDS_QUERY_INPUT_HOST_EP_2: + case HWRM_FUNC_HOST_PF_IDS_QUERY_INPUT_HOST_EP_3: + req.host = fn_info->host; + break; + default: + req.host = HWRM_FUNC_HOST_PF_IDS_QUERY_INPUT_HOST_EP_0; + break; + } + + req.filter = fn_info->filter; + if (req.filter > HWRM_FUNC_HOST_PF_IDS_QUERY_INPUT_FILTER_ROCE) + req.filter = HWRM_FUNC_HOST_PF_IDS_QUERY_INPUT_FILTER_ALL; + + bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp, + sizeof(resp), DFLT_HWRM_CMD_TIMEOUT); + rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg); + + + *first_pf = le16_to_cpu(resp.first_pf_id); + *pf_mask = le16_to_cpu(resp.pf_ordinal_mask); + + return rc; +} + +static void bnxt_re_put_stats_ctx(struct bnxt_re_dev *rdev) +{ + struct bnxt_qplib_ctx *hctx; + struct bnxt_qplib_res *res; + u16 tid = 0xffff; + + res = &rdev->qplib_res; + hctx = res->hctx; + + if (test_and_clear_bit(BNXT_RE_FLAG_STATS_CTX_ALLOC, &rdev->flags)) { + bnxt_re_net_stats_ctx_free(rdev, hctx->stats.fw_id, tid); + bnxt_qplib_free_stat_mem(res, &hctx->stats); + } +} + +static void bnxt_re_put_stats2_ctx(struct bnxt_re_dev *rdev) +{ + test_and_clear_bit(BNXT_RE_FLAG_STATS_CTX2_ALLOC, &rdev->flags); +} + +static int bnxt_re_get_stats_ctx(struct bnxt_re_dev *rdev) +{ + struct bnxt_qplib_ctx *hctx; + struct bnxt_qplib_res *res; + u16 tid = 0xffff; + int rc; + + res = &rdev->qplib_res; + hctx = res->hctx; + + rc = bnxt_qplib_alloc_stat_mem(res->pdev, rdev->chip_ctx, &hctx->stats); + if (rc) + return -ENOMEM; + rc = bnxt_re_net_stats_ctx_alloc(rdev, tid); + if (rc) + goto free_stat_mem; + set_bit(BNXT_RE_FLAG_STATS_CTX_ALLOC, &rdev->flags); + + return 0; + +free_stat_mem: + bnxt_qplib_free_stat_mem(res, &hctx->stats); + + return rc; +} + +static int bnxt_re_update_dev_attr(struct bnxt_re_dev *rdev) +{ + int rc; + + rc = bnxt_qplib_get_dev_attr(&rdev->rcfw); + if (rc) + return rc; + if (!bnxt_re_check_min_attr(rdev)) + return -EINVAL; + return 0; +} + +static void bnxt_re_free_tbls(struct bnxt_re_dev *rdev) +{ + bnxt_qplib_clear_tbls(&rdev->qplib_res); + bnxt_qplib_free_tbls(&rdev->qplib_res); +} + +static int bnxt_re_alloc_init_tbls(struct bnxt_re_dev *rdev) +{ + struct bnxt_qplib_chip_ctx *chip_ctx = rdev->chip_ctx; + u8 pppp_factor = 0; + int rc; + + /* + * TODO: Need a better mechanism for spreading of the + * 512 extended PPP pages. For now, spreading it + * based on port_count + */ + if (_is_chip_p7(chip_ctx) && chip_ctx->modes.db_push) + pppp_factor = rdev->en_dev->port_count; + rc = bnxt_qplib_alloc_tbls(&rdev->qplib_res, pppp_factor); + if (rc) + return rc; + bnxt_qplib_init_tbls(&rdev->qplib_res); + set_bit(BNXT_RE_FLAG_TBLS_ALLOCINIT, &rdev->flags); + + return 0; +} + +static void bnxt_re_clean_nqs(struct bnxt_re_dev *rdev) +{ + struct bnxt_qplib_nq *nq; + int i; + + if (!rdev->nqr.max_init) + return; + + for (i = (rdev->nqr.max_init - 1) ; i >= 0; i--) { + nq = &rdev->nqr.nq[i]; + bnxt_qplib_disable_nq(nq); + bnxt_re_net_ring_free(rdev, nq->ring_id); + bnxt_qplib_free_nq_mem(nq); + } + rdev->nqr.max_init = 0; +} + +static int bnxt_re_setup_nqs(struct bnxt_re_dev *rdev) +{ + struct bnxt_re_ring_attr rattr = {}; + struct bnxt_qplib_nq *nq; + int rc, i; + int depth; + u32 offt; + u16 vec; + + mutex_init(&rdev->nqr.load_lock); + /* + * TODO: Optimize the depth based on the + * number of NQs. + */ + depth = BNXT_QPLIB_NQE_MAX_CNT; + for (i = 0; i < rdev->nqr.num_msix - 1; i++) { + nq = &rdev->nqr.nq[i]; + vec = rdev->nqr.msix_entries[i + 1].vector; + offt = rdev->nqr.msix_entries[i + 1].db_offset; + nq->hwq.max_elements = depth; + rc = bnxt_qplib_alloc_nq_mem(&rdev->qplib_res, nq); + if (rc) { + dev_err(rdev_to_dev(rdev), + "Failed to get mem for NQ %d, rc = 0x%x", + i, rc); + goto fail_mem; + } + + rattr.dma_arr = nq->hwq.pbl[PBL_LVL_0].pg_map_arr; + rattr.pages = nq->hwq.pbl[rdev->nqr.nq[i].hwq.level].pg_count; + rattr.type = bnxt_re_get_rtype(rdev); + rattr.mode = HWRM_RING_ALLOC_INPUT_INT_MODE_MSIX; + rattr.depth = nq->hwq.max_elements - 1; + rattr.lrid = rdev->nqr.msix_entries[i + 1].ring_idx; + + /* Set DBR pacing capability on the first NQ ring only */ + if (!i && bnxt_qplib_dbr_pacing_ext_en(rdev->chip_ctx)) + rattr.flags = HWRM_RING_ALLOC_INPUT_FLAGS_NQ_DBR_PACING; + else + rattr.flags = 0; + + rc = bnxt_re_net_ring_alloc(rdev, &rattr, &nq->ring_id); + if (rc) { + nq->ring_id = 0xffff; /* Invalid ring-id */ + dev_err(rdev_to_dev(rdev), + "Failed to get fw id for NQ %d, rc = 0x%x", + i, rc); + goto fail_ring; + } + + rc = bnxt_qplib_enable_nq(nq, i, vec, offt, + &bnxt_re_cqn_handler, + &bnxt_re_srqn_handler); + if (rc) { + dev_err(rdev_to_dev(rdev), + "Failed to enable NQ %d, rc = 0x%x", i, rc); + goto fail_en; + } + } + + rdev->nqr.max_init = i; + return 0; +fail_en: + /* *nq was i'th nq */ + bnxt_re_net_ring_free(rdev, nq->ring_id); +fail_ring: + bnxt_qplib_free_nq_mem(nq); +fail_mem: + rdev->nqr.max_init = i; + return rc; +} + +static void bnxt_re_sysfs_destroy_file(struct bnxt_re_dev *rdev) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(bnxt_re_attributes); i++) + device_remove_file(&rdev->ibdev.dev, bnxt_re_attributes[i]); +} + +static int bnxt_re_sysfs_create_file(struct bnxt_re_dev *rdev) +{ + int i, j, rc = 0; + + for (i = 0; i < ARRAY_SIZE(bnxt_re_attributes); i++) { + rc = device_create_file(&rdev->ibdev.dev, + bnxt_re_attributes[i]); + if (rc) { + dev_err(rdev_to_dev(rdev), + "Failed to create IB sysfs with rc = 0x%x", rc); + /* Must clean up all created device files */ + for (j = 0; j < i; j++) + device_remove_file(&rdev->ibdev.dev, + bnxt_re_attributes[j]); + clear_bit(BNXT_RE_FLAG_IBDEV_REGISTERED, &rdev->flags); + ib_unregister_device(&rdev->ibdev); + return 1; + } + } + return 0; +} + +/* worker thread for polling periodic events. Now used for QoS programming*/ +static void bnxt_re_worker(struct work_struct *work) +{ + struct bnxt_re_dev *rdev = container_of(work, struct bnxt_re_dev, + worker.work); + int rc; + + /* QoS is in 30s cadence for PFs*/ + if (!rdev->is_virtfn && !rdev->worker_30s--) + rdev->worker_30s = 30; + /* Use trylock for bnxt_re_dev_lock as this can be + * held for long time by debugfs show path while issuing + * HWRMS. If the debugfs name update is not done in this + * iteration, the driver will check for the same in the + * next schedule of the worker i.e after 1 sec. + */ + if (mutex_trylock(&bnxt_re_dev_lock)) + mutex_unlock(&bnxt_re_dev_lock); + + if (!rdev->stats.stats_query_sec) + goto resched; + + if (test_bit(BNXT_RE_FLAG_ISSUE_CFA_FLOW_STATS, &rdev->flags) && + (rdev->is_virtfn || + !_is_ext_stats_supported(rdev->dev_attr->dev_cap_flags))) { + if (!(rdev->stats.stats_query_counter++ % + rdev->stats.stats_query_sec)) { + rc = bnxt_re_get_qos_stats(rdev); + if (rc && rc != -ENOMEM) + clear_bit(BNXT_RE_FLAG_ISSUE_CFA_FLOW_STATS, + &rdev->flags); + } + } + +resched: + schedule_delayed_work(&rdev->worker, msecs_to_jiffies(1000)); +} + +static int bnxt_re_alloc_dbr_sw_stats_mem(struct bnxt_re_dev *rdev) +{ + if (!(rdev->dbr_drop_recov || rdev->dbr_pacing)) + return 0; + + rdev->dbr_sw_stats = kzalloc(sizeof(*rdev->dbr_sw_stats), GFP_KERNEL); + if (!rdev->dbr_sw_stats) + return -ENOMEM; + + return 0; +} + +static void bnxt_re_free_dbr_sw_stats_mem(struct bnxt_re_dev *rdev) +{ + kfree(rdev->dbr_sw_stats); + rdev->dbr_sw_stats = NULL; +} + +static int bnxt_re_initialize_dbr_drop_recov(struct bnxt_re_dev *rdev) +{ + rdev->dbr_drop_recov_wq = + create_singlethread_workqueue("bnxt_re_dbr_drop_recov"); + if (!rdev->dbr_drop_recov_wq) { + dev_err(rdev_to_dev(rdev), "DBR Drop Revov wq alloc failed!"); + return -EINVAL; + } + rdev->dbr_drop_recov = true; + + /* Enable configfs setting dbr_drop_recov by default*/ + rdev->user_dbr_drop_recov = true; + + rdev->user_dbr_drop_recov_timeout = BNXT_RE_DBR_RECOV_USERLAND_TIMEOUT; + return 0; +} + +static void bnxt_re_deinitialize_dbr_drop_recov(struct bnxt_re_dev *rdev) +{ + if (rdev->dbr_drop_recov_wq) { + flush_workqueue(rdev->dbr_drop_recov_wq); + destroy_workqueue(rdev->dbr_drop_recov_wq); + rdev->dbr_drop_recov_wq = NULL; + } + rdev->dbr_drop_recov = false; +} + +static int bnxt_re_initialize_dbr_pacing(struct bnxt_re_dev *rdev) +{ + int rc; + + /* Allocate a page for app use */ + rdev->dbr_page = (void *)__get_free_page(GFP_KERNEL); + if (!rdev->dbr_page) { + dev_err(rdev_to_dev(rdev), "DBR page allocation failed!"); + return -ENOMEM; + } + memset((u8 *)rdev->dbr_page, 0, PAGE_SIZE); + rdev->qplib_res.pacing_data = (struct bnxt_qplib_db_pacing_data *)rdev->dbr_page; + rc = bnxt_re_hwrm_dbr_pacing_qcfg(rdev); + if (rc) { + dev_err(rdev_to_dev(rdev), + "Failed to query dbr pacing config %d\n", rc); + goto fail; + } + /* Create a work queue for scheduling dbq event */ + rdev->dbq_wq = create_singlethread_workqueue("bnxt_re_dbq"); + if (!rdev->dbq_wq) { + dev_err(rdev_to_dev(rdev), "DBQ wq alloc failed!"); + rc = -ENOMEM; + goto fail; + } + /* MAP grc window 2 for reading db fifo depth */ + writel_fbsd(rdev->en_dev->softc, BNXT_GRCPF_REG_WINDOW_BASE_OUT + 4, 0, + rdev->chip_ctx->dbr_stat_db_fifo & BNXT_GRC_BASE_MASK); + rdev->dbr_db_fifo_reg_off = + (rdev->chip_ctx->dbr_stat_db_fifo & BNXT_GRC_OFFSET_MASK) + + 0x2000; + rdev->qplib_res.pacing_data->grc_reg_offset = rdev->dbr_db_fifo_reg_off; + + rdev->dbr_bar_addr = + pci_resource_start(rdev->qplib_res.pdev, 0) + + rdev->dbr_db_fifo_reg_off; + + /* Percentage of DB FIFO */ + rdev->dbq_watermark = BNXT_RE_PACING_DBQ_THRESHOLD; + rdev->pacing_en_int_th = BNXT_RE_PACING_EN_INT_THRESHOLD; + rdev->pacing_algo_th = BNXT_RE_PACING_ALGO_THRESHOLD; + rdev->dbq_pacing_time = BNXT_RE_DBR_INT_TIME; + rdev->dbr_def_do_pacing = BNXT_RE_DBR_DO_PACING_NO_CONGESTION; + rdev->do_pacing_save = rdev->dbr_def_do_pacing; + bnxt_re_set_default_pacing_data(rdev); + dev_dbg(rdev_to_dev(rdev), "Initialized db pacing\n"); + + return 0; +fail: + free_page((u64)rdev->dbr_page); + rdev->dbr_page = NULL; + return rc; +} + +static void bnxt_re_deinitialize_dbr_pacing(struct bnxt_re_dev *rdev) +{ + if (rdev->dbq_wq) + flush_workqueue(rdev->dbq_wq); + + cancel_work_sync(&rdev->dbq_fifo_check_work); + cancel_delayed_work_sync(&rdev->dbq_pacing_work); + + if (rdev->dbq_wq) { + destroy_workqueue(rdev->dbq_wq); + rdev->dbq_wq = NULL; + } + + if (rdev->dbr_page) + free_page((u64)rdev->dbr_page); + rdev->dbr_page = NULL; + rdev->dbr_pacing = false; +} + +/* enable_dbr_pacing needs to be done only for older FWs + * where host selects primary function. ie. pacing_ext + * flags is not set. + */ +int bnxt_re_enable_dbr_pacing(struct bnxt_re_dev *rdev) +{ + struct bnxt_qplib_nq *nq; + + nq = &rdev->nqr.nq[0]; + rdev->dbq_nq_id = nq->ring_id; + + if (!bnxt_qplib_dbr_pacing_ext_en(rdev->chip_ctx) && + bnxt_qplib_dbr_pacing_is_primary_pf(rdev->chip_ctx)) { + if (bnxt_re_hwrm_dbr_pacing_cfg(rdev, true)) { + dev_err(rdev_to_dev(rdev), + "Failed to set dbr pacing config\n"); + return -EIO; + } + /* MAP grc window 8 for ARMing the NQ DBQ */ + writel_fbsd(rdev->en_dev->softc, BNXT_GRCPF_REG_WINDOW_BASE_OUT + 28 , 0, + rdev->chip_ctx->dbr_aeq_arm_reg & BNXT_GRC_BASE_MASK); + rdev->dbr_aeq_arm_reg_off = + (rdev->chip_ctx->dbr_aeq_arm_reg & + BNXT_GRC_OFFSET_MASK) + 0x8000; + writel_fbsd(rdev->en_dev->softc, rdev->dbr_aeq_arm_reg_off , 0, 1); + } + + return 0; +} + +/* disable_dbr_pacing needs to be done only for older FWs + * where host selects primary function. ie. pacing_ext + * flags is not set. + */ + +int bnxt_re_disable_dbr_pacing(struct bnxt_re_dev *rdev) +{ + int rc = 0; + + if (!bnxt_qplib_dbr_pacing_ext_en(rdev->chip_ctx) && + bnxt_qplib_dbr_pacing_is_primary_pf(rdev->chip_ctx)) + rc = bnxt_re_hwrm_dbr_pacing_cfg(rdev, false); + + return rc; +} + +static void bnxt_re_ib_uninit(struct bnxt_re_dev *rdev) +{ + if (test_bit(BNXT_RE_FLAG_IBDEV_REGISTERED, &rdev->flags)) { + bnxt_re_sysfs_destroy_file(rdev); + /* Cleanup ib dev */ + ib_unregister_device(&rdev->ibdev); + clear_bit(BNXT_RE_FLAG_IBDEV_REGISTERED, &rdev->flags); + return; + } +} + +static void bnxt_re_dev_uninit(struct bnxt_re_dev *rdev, u8 op_type) +{ + struct bnxt_qplib_dpi *kdpi; + int rc, wait_count = BNXT_RE_RES_FREE_WAIT_COUNT; + + bnxt_re_net_unregister_async_event(rdev); + + bnxt_re_put_stats2_ctx(rdev); + if (test_and_clear_bit(BNXT_RE_FLAG_DEV_LIST_INITIALIZED, + &rdev->flags)) { + /* did the caller hold the lock? */ + mutex_lock(&bnxt_re_dev_lock); + list_del_rcu(&rdev->list); + mutex_unlock(&bnxt_re_dev_lock); + } + + bnxt_re_uninit_resolve_wq(rdev); + bnxt_re_uninit_dcb_wq(rdev); + bnxt_re_uninit_aer_wq(rdev); + + bnxt_re_deinitialize_dbr_drop_recov(rdev); + + if (bnxt_qplib_dbr_pacing_en(rdev->chip_ctx)) + (void)bnxt_re_disable_dbr_pacing(rdev); + + if (test_and_clear_bit(BNXT_RE_FLAG_WORKER_REG, &rdev->flags)) { + cancel_delayed_work_sync(&rdev->worker); + } + + /* Wait for ULPs to release references */ + while (atomic_read(&rdev->stats.rsors.cq_count) && --wait_count) + usleep_range(500, 1000); + if (!wait_count) + dev_err(rdev_to_dev(rdev), + "CQ resources not freed by stack, count = 0x%x", + atomic_read(&rdev->stats.rsors.cq_count)); + + kdpi = &rdev->dpi_privileged; + if (kdpi->umdbr) { /* kernel DPI was allocated with success */ + (void)bnxt_qplib_dealloc_dpi(&rdev->qplib_res, kdpi); + /* + * Driver just need to know no command had failed + * during driver load sequence and below command is + * required indeed. Piggybacking dpi allocation status. + */ + } + + /* Protect the device uninitialization and start_irq/stop_irq L2 + * callbacks with rtnl lock to avoid race condition between these calls + */ + rtnl_lock(); + if (test_and_clear_bit(BNXT_RE_FLAG_SETUP_NQ, &rdev->flags)) + bnxt_re_clean_nqs(rdev); + rtnl_unlock(); + + if (test_and_clear_bit(BNXT_RE_FLAG_TBLS_ALLOCINIT, &rdev->flags)) + bnxt_re_free_tbls(rdev); + if (test_and_clear_bit(BNXT_RE_FLAG_RCFW_CHANNEL_INIT, &rdev->flags)) { + rc = bnxt_qplib_deinit_rcfw(&rdev->rcfw); + if (rc) + dev_warn(rdev_to_dev(rdev), + "Failed to deinitialize fw, rc = 0x%x", rc); + } + + bnxt_re_put_stats_ctx(rdev); + + if (test_and_clear_bit(BNXT_RE_FLAG_ALLOC_CTX, &rdev->flags)) + bnxt_qplib_free_hwctx(&rdev->qplib_res); + + rtnl_lock(); + if (test_and_clear_bit(BNXT_RE_FLAG_RCFW_CHANNEL_EN, &rdev->flags)) + bnxt_qplib_disable_rcfw_channel(&rdev->rcfw); + + if (rdev->dbr_pacing) + bnxt_re_deinitialize_dbr_pacing(rdev); + + bnxt_re_free_dbr_sw_stats_mem(rdev); + + if (test_and_clear_bit(BNXT_RE_FLAG_NET_RING_ALLOC, &rdev->flags)) + bnxt_re_net_ring_free(rdev, rdev->rcfw.creq.ring_id); + + if (test_and_clear_bit(BNXT_RE_FLAG_ALLOC_RCFW, &rdev->flags)) + bnxt_qplib_free_rcfw_channel(&rdev->qplib_res); + + if (test_and_clear_bit(BNXT_RE_FLAG_GOT_MSIX, &rdev->flags)) + bnxt_re_free_msix(rdev); + rtnl_unlock(); + + bnxt_re_destroy_chip_ctx(rdev); + + if (op_type != BNXT_RE_PRE_RECOVERY_REMOVE) { + if (test_and_clear_bit(BNXT_RE_FLAG_NETDEV_REGISTERED, + &rdev->flags)) + bnxt_re_unregister_netdev(rdev); + } +} + +static int bnxt_re_dev_init(struct bnxt_re_dev *rdev, u8 op_type, u8 wqe_mode) +{ + struct bnxt_re_ring_attr rattr = {}; + struct bnxt_qplib_creq_ctx *creq; + int vec, offset; + int rc = 0; + + if (op_type != BNXT_RE_POST_RECOVERY_INIT) { + /* Registered a new RoCE device instance to netdev */ + rc = bnxt_re_register_netdev(rdev); + if (rc) + return -EINVAL; + } + set_bit(BNXT_RE_FLAG_NETDEV_REGISTERED, &rdev->flags); + + rc = bnxt_re_setup_chip_ctx(rdev, wqe_mode); + if (rc) { + dev_err(rdev_to_dev(rdev), "Failed to get chip context rc 0x%x", rc); + bnxt_re_unregister_netdev(rdev); + clear_bit(BNXT_RE_FLAG_NETDEV_REGISTERED, &rdev->flags); + rc = -EINVAL; + return rc; + } + + /* Protect the device initialization and start_irq/stop_irq L2 callbacks + * with rtnl lock to avoid race condition between these calls + */ + rtnl_lock(); + rc = bnxt_re_request_msix(rdev); + if (rc) { + dev_err(rdev_to_dev(rdev), + "Requesting MSI-X vectors failed with rc = 0x%x", rc); + rc = -EINVAL; + goto release_rtnl; + } + set_bit(BNXT_RE_FLAG_GOT_MSIX, &rdev->flags); + + /* Establish RCFW Communication Channel to initialize the context + memory for the function and all child VFs */ + rc = bnxt_qplib_alloc_rcfw_channel(&rdev->qplib_res); + if (rc) { + dev_err(rdev_to_dev(rdev), + "Failed to alloc mem for rcfw, rc = %#x\n", rc); + goto release_rtnl; + } + set_bit(BNXT_RE_FLAG_ALLOC_RCFW, &rdev->flags); + + creq = &rdev->rcfw.creq; + rattr.dma_arr = creq->hwq.pbl[PBL_LVL_0].pg_map_arr; + rattr.pages = creq->hwq.pbl[creq->hwq.level].pg_count; + rattr.type = bnxt_re_get_rtype(rdev); + rattr.mode = HWRM_RING_ALLOC_INPUT_INT_MODE_MSIX; + rattr.depth = BNXT_QPLIB_CREQE_MAX_CNT - 1; + rattr.lrid = rdev->nqr.msix_entries[BNXT_RE_AEQ_IDX].ring_idx; + rc = bnxt_re_net_ring_alloc(rdev, &rattr, &creq->ring_id); + if (rc) { + creq->ring_id = 0xffff; + dev_err(rdev_to_dev(rdev), + "Failed to allocate CREQ fw id with rc = 0x%x", rc); + goto release_rtnl; + } + + if (!rdev->chip_ctx) + goto release_rtnl; + /* Program the NQ ID for DBQ notification */ + if (rdev->chip_ctx->modes.dbr_pacing_v0 || + bnxt_qplib_dbr_pacing_en(rdev->chip_ctx) || + bnxt_qplib_dbr_pacing_ext_en(rdev->chip_ctx)) { + rc = bnxt_re_initialize_dbr_pacing(rdev); + if (!rc) + rdev->dbr_pacing = true; + else + rdev->dbr_pacing = false; + dev_dbg(rdev_to_dev(rdev), "%s: initialize db pacing ret %d\n", + __func__, rc); + } + + vec = rdev->nqr.msix_entries[BNXT_RE_AEQ_IDX].vector; + offset = rdev->nqr.msix_entries[BNXT_RE_AEQ_IDX].db_offset; + rc = bnxt_qplib_enable_rcfw_channel(&rdev->rcfw, vec, offset, + &bnxt_re_aeq_handler); + if (rc) { + dev_err(rdev_to_dev(rdev), + "Failed to enable RCFW channel with rc = 0x%x", rc); + goto release_rtnl; + } + set_bit(BNXT_RE_FLAG_RCFW_CHANNEL_EN, &rdev->flags); + + rc = bnxt_re_update_dev_attr(rdev); + if (rc) + goto release_rtnl; + bnxt_re_set_resource_limits(rdev); + if (!rdev->is_virtfn && !_is_chip_gen_p5_p7(rdev->chip_ctx)) { + rc = bnxt_qplib_alloc_hwctx(&rdev->qplib_res); + if (rc) { + dev_err(rdev_to_dev(rdev), + "Failed to alloc hw contexts, rc = 0x%x", rc); + goto release_rtnl; + } + set_bit(BNXT_RE_FLAG_ALLOC_CTX, &rdev->flags); + } + + rc = bnxt_re_get_stats_ctx(rdev); + if (rc) + goto release_rtnl; + + rc = bnxt_qplib_init_rcfw(&rdev->rcfw, rdev->is_virtfn); + if (rc) { + dev_err(rdev_to_dev(rdev), + "Failed to initialize fw with rc = 0x%x", rc); + goto release_rtnl; + } + set_bit(BNXT_RE_FLAG_RCFW_CHANNEL_INIT, &rdev->flags); + + /* Based resource count on the 'new' device caps */ + rc = bnxt_re_update_dev_attr(rdev); + if (rc) + goto release_rtnl; + rc = bnxt_re_alloc_init_tbls(rdev); + if (rc) { + dev_err(rdev_to_dev(rdev), "tbls alloc-init failed rc = %#x", + rc); + goto release_rtnl; + } + rc = bnxt_re_setup_nqs(rdev); + if (rc) { + dev_err(rdev_to_dev(rdev), "NQs alloc-init failed rc = %#x\n", + rc); + if (rdev->nqr.max_init == 0) + goto release_rtnl; + + dev_warn(rdev_to_dev(rdev), + "expected nqs %d available nqs %d\n", + rdev->nqr.num_msix, rdev->nqr.max_init); + } + set_bit(BNXT_RE_FLAG_SETUP_NQ, &rdev->flags); + rtnl_unlock(); + + rc = bnxt_qplib_alloc_dpi(&rdev->qplib_res, &rdev->dpi_privileged, + rdev, BNXT_QPLIB_DPI_TYPE_KERNEL); + if (rc) + goto fail; + + if (rdev->dbr_pacing) + bnxt_re_enable_dbr_pacing(rdev); + + if (rdev->chip_ctx->modes.dbr_drop_recov) + bnxt_re_initialize_dbr_drop_recov(rdev); + + rc = bnxt_re_alloc_dbr_sw_stats_mem(rdev); + if (rc) + goto fail; + + /* This block of code is needed for error recovery support */ + if (!rdev->is_virtfn) { + struct bnxt_re_tc_rec *tc_rec; + + tc_rec = &rdev->tc_rec[0]; + rc = bnxt_re_query_hwrm_qportcfg(rdev, tc_rec, 0xFFFF); + if (rc) { + dev_err(rdev_to_dev(rdev), + "Failed to query port config rc:%d", rc); + return rc; + } + + /* Query f/w defaults of CC params */ + rc = bnxt_qplib_query_cc_param(&rdev->qplib_res, &rdev->cc_param); + if (rc) + dev_warn(rdev_to_dev(rdev), + "Failed to query CC defaults\n"); + if (1) { + rdev->num_vfs = pci_num_vf(rdev->en_dev->pdev); + if (rdev->num_vfs) { + bnxt_re_set_resource_limits(rdev); + bnxt_qplib_set_func_resources(&rdev->qplib_res); + } + } + } + INIT_DELAYED_WORK(&rdev->worker, bnxt_re_worker); + set_bit(BNXT_RE_FLAG_WORKER_REG, &rdev->flags); + schedule_delayed_work(&rdev->worker, msecs_to_jiffies(1000)); + + bnxt_re_init_dcb_wq(rdev); + bnxt_re_init_aer_wq(rdev); + bnxt_re_init_resolve_wq(rdev); + mutex_lock(&bnxt_re_dev_lock); + list_add_tail_rcu(&rdev->list, &bnxt_re_dev_list); + /* Added to the list, not in progress anymore */ + gadd_dev_inprogress--; + set_bit(BNXT_RE_FLAG_DEV_LIST_INITIALIZED, &rdev->flags); + mutex_unlock(&bnxt_re_dev_lock); + + + return rc; +release_rtnl: + rtnl_unlock(); +fail: + bnxt_re_dev_uninit(rdev, BNXT_RE_COMPLETE_REMOVE); + + return rc; +} + +static int bnxt_re_ib_init(struct bnxt_re_dev *rdev) +{ + int rc = 0; + + rc = bnxt_re_register_ib(rdev); + if (rc) { + dev_err(rdev_to_dev(rdev), + "Register IB failed with rc = 0x%x", rc); + goto fail; + } + if (bnxt_re_sysfs_create_file(rdev)) { + bnxt_re_stopqps_and_ib_uninit(rdev); + goto fail; + } + + set_bit(BNXT_RE_FLAG_IBDEV_REGISTERED, &rdev->flags); + set_bit(BNXT_RE_FLAG_ISSUE_ROCE_STATS, &rdev->flags); + set_bit(BNXT_RE_FLAG_ISSUE_CFA_FLOW_STATS, &rdev->flags); + bnxt_re_dispatch_event(&rdev->ibdev, NULL, 1, IB_EVENT_PORT_ACTIVE); + bnxt_re_dispatch_event(&rdev->ibdev, NULL, 1, IB_EVENT_GID_CHANGE); + + return rc; +fail: + bnxt_re_dev_uninit(rdev, BNXT_RE_COMPLETE_REMOVE); + return rc; +} + +/* wrapper for ib_init funcs */ +int _bnxt_re_ib_init(struct bnxt_re_dev *rdev) +{ + return bnxt_re_ib_init(rdev); +} + +/* wrapper for aux init funcs */ +int _bnxt_re_ib_init2(struct bnxt_re_dev *rdev) +{ + bnxt_re_ib_init_2(rdev); + return 0; /* add return for future proof */ +} + +static void bnxt_re_dev_unreg(struct bnxt_re_dev *rdev) +{ + bnxt_re_dev_dealloc(rdev); +} + + +static int bnxt_re_dev_reg(struct bnxt_re_dev **rdev, struct ifnet *netdev, + struct bnxt_en_dev *en_dev) +{ + struct ifnet *realdev = NULL; + + realdev = netdev; + if (realdev) + dev_dbg(NULL, "%s: realdev = %p netdev = %p\n", __func__, + realdev, netdev); + /* + * Note: + * The first argument to bnxt_re_dev_alloc() is 'netdev' and + * not 'realdev', since in the case of bonding we want to + * register the bonded virtual netdev (master) to the ib stack. + * And 'en_dev' (for L2/PCI communication) is the first slave + * device (PF0 on the card). + * In the case of a regular netdev, both netdev and the en_dev + * correspond to the same device. + */ + *rdev = bnxt_re_dev_alloc(netdev, en_dev); + if (!*rdev) { + pr_err("%s: netdev %p not handled", + ROCE_DRV_MODULE_NAME, netdev); + return -ENOMEM; + } + bnxt_re_hold(*rdev); + + return 0; +} + +void bnxt_re_get_link_speed(struct bnxt_re_dev *rdev) +{ + rdev->espeed = rdev->en_dev->espeed; + return; +} + +void bnxt_re_stopqps_and_ib_uninit(struct bnxt_re_dev *rdev) +{ + dev_dbg(rdev_to_dev(rdev), "%s: Stopping QPs, IB uninit on rdev: %p\n", + __func__, rdev); + bnxt_re_stop_all_nonqp1_nonshadow_qps(rdev); + bnxt_re_ib_uninit(rdev); +} + +void bnxt_re_remove_device(struct bnxt_re_dev *rdev, u8 op_type, + struct auxiliary_device *aux_dev) +{ + struct bnxt_re_en_dev_info *en_info; + struct bnxt_qplib_cmdq_ctx *cmdq; + struct bnxt_qplib_rcfw *rcfw; + + rcfw = &rdev->rcfw; + cmdq = &rcfw->cmdq; + if (test_bit(FIRMWARE_STALL_DETECTED, &cmdq->flags)) + set_bit(BNXT_RE_FLAG_ERR_DEVICE_DETACHED, &rdev->flags); + + dev_dbg(rdev_to_dev(rdev), "%s: Removing rdev: %p\n", __func__, rdev); + bnxt_re_dev_uninit(rdev, op_type); + en_info = auxiliary_get_drvdata(aux_dev); + if (en_info) { + rtnl_lock(); + en_info->rdev = NULL; + rtnl_unlock(); + if (op_type != BNXT_RE_PRE_RECOVERY_REMOVE) { + clear_bit(BNXT_RE_FLAG_EN_DEV_PRIMARY_DEV, &en_info->flags); + clear_bit(BNXT_RE_FLAG_EN_DEV_SECONDARY_DEV, &en_info->flags); + clear_bit(BNXT_RE_FLAG_EN_DEV_NETDEV_REG, &en_info->flags); + } + } + bnxt_re_dev_unreg(rdev); +} + +int bnxt_re_add_device(struct bnxt_re_dev **rdev, + struct ifnet *netdev, + u8 qp_mode, u8 op_type, u8 wqe_mode, + u32 num_msix_requested, + struct auxiliary_device *aux_dev) +{ + struct bnxt_re_en_dev_info *en_info; + struct bnxt_en_dev *en_dev; + int rc = 0; + + en_info = auxiliary_get_drvdata(aux_dev); + en_dev = en_info->en_dev; + + mutex_lock(&bnxt_re_dev_lock); + /* Check if driver already in mod exit and aux_dev is valid */ + if (gmod_exit || !aux_dev) { + mutex_unlock(&bnxt_re_dev_lock); + return -ENODEV; + } + /* Add device in progress */ + gadd_dev_inprogress++; + mutex_unlock(&bnxt_re_dev_lock); + + rc = bnxt_re_dev_reg(rdev, netdev, en_dev); + if (rc) { + dev_dbg(NULL, "Failed to create add device for netdev %p\n", + netdev); + /* + * For BNXT_RE_POST_RECOVERY_INIT special case + * called from bnxt_re_start, the work is + * complete only after, bnxt_re_start completes + * bnxt_unregister_device in case of failure. + * So bnxt_re_start will decrement gadd_dev_inprogress + * in case of failure. + */ + if (op_type != BNXT_RE_POST_RECOVERY_INIT) { + mutex_lock(&bnxt_re_dev_lock); + gadd_dev_inprogress--; + mutex_unlock(&bnxt_re_dev_lock); + } + return rc; + } + + if (rc != 0) + goto ref_error; + + /* + * num_msix_requested = BNXT_RE_MSIX_FROM_MOD_PARAM indicates fresh driver load. + * Otherwaise, this invocation can be the result of lag create / destroy, + * err revovery, hot fw upgrade, etc.. + */ + if (num_msix_requested == BNXT_RE_MSIX_FROM_MOD_PARAM) { + if (bnxt_re_probe_count < BNXT_RE_MAX_DEVICES) + num_msix_requested = max_msix_vec[bnxt_re_probe_count++]; + else + /* Consider as default when probe_count exceeds its limit */ + num_msix_requested = 0; + + /* if user specifies only one value, use the same for all PFs */ + if (max_msix_vec_argc == 1) + num_msix_requested = max_msix_vec[0]; + } + + (*rdev)->num_msix_requested = num_msix_requested; + (*rdev)->gsi_ctx.gsi_qp_mode = qp_mode; + (*rdev)->adev = aux_dev; + (*rdev)->dev_addr = en_dev->softc->func.mac_addr; + /* Before updating the rdev pointer in bnxt_re_en_dev_info structure, + * take the rtnl lock to avoid accessing invalid rdev pointer from + * L2 ULP callbacks. This is applicable in all the places where rdev + * pointer is updated in bnxt_re_en_dev_info. + */ + rtnl_lock(); + en_info->rdev = *rdev; + rtnl_unlock(); + rc = bnxt_re_dev_init(*rdev, op_type, wqe_mode); + if (rc) { +ref_error: + bnxt_re_dev_unreg(*rdev); + *rdev = NULL; + /* + * For BNXT_RE_POST_RECOVERY_INIT special case + * called from bnxt_re_start, the work is + * complete only after, bnxt_re_start completes + * bnxt_unregister_device in case of failure. + * So bnxt_re_start will decrement gadd_dev_inprogress + * in case of failure. + */ + if (op_type != BNXT_RE_POST_RECOVERY_INIT) { + mutex_lock(&bnxt_re_dev_lock); + gadd_dev_inprogress--; + mutex_unlock(&bnxt_re_dev_lock); + } + } + dev_dbg(rdev_to_dev(*rdev), "%s: Adding rdev: %p\n", __func__, *rdev); + if (!rc) { + set_bit(BNXT_RE_FLAG_EN_DEV_NETDEV_REG, &en_info->flags); + } + return rc; +} + +struct bnxt_re_dev *bnxt_re_get_peer_pf(struct bnxt_re_dev *rdev) +{ + struct pci_dev *pdev_in = rdev->en_dev->pdev; + int tmp_bus_num, bus_num = pdev_in->bus->number; + int tmp_dev_num, dev_num = PCI_SLOT(pdev_in->devfn); + int tmp_func_num, func_num = PCI_FUNC(pdev_in->devfn); + struct bnxt_re_dev *tmp_rdev; + + rcu_read_lock(); + list_for_each_entry_rcu(tmp_rdev, &bnxt_re_dev_list, list) { + tmp_bus_num = tmp_rdev->en_dev->pdev->bus->number; + tmp_dev_num = PCI_SLOT(tmp_rdev->en_dev->pdev->devfn); + tmp_func_num = PCI_FUNC(tmp_rdev->en_dev->pdev->devfn); + + if (bus_num == tmp_bus_num && dev_num == tmp_dev_num && + func_num != tmp_func_num) { + rcu_read_unlock(); + return tmp_rdev; + } + } + rcu_read_unlock(); + return NULL; +} + + +int bnxt_re_schedule_work(struct bnxt_re_dev *rdev, unsigned long event, + struct ifnet *vlan_dev, + struct ifnet *netdev, + struct auxiliary_device *adev) +{ + struct bnxt_re_work *re_work; + + /* Allocate for the deferred task */ + re_work = kzalloc(sizeof(*re_work), GFP_KERNEL); + if (!re_work) + return -ENOMEM; + + re_work->rdev = rdev; + re_work->event = event; + re_work->vlan_dev = vlan_dev; + re_work->adev = adev; + INIT_WORK(&re_work->work, bnxt_re_task); + if (rdev) + atomic_inc(&rdev->sched_count); + re_work->netdev = netdev; + queue_work(bnxt_re_wq, &re_work->work); + + return 0; +} + + +int bnxt_re_get_slot_pf_count(struct bnxt_re_dev *rdev) +{ + struct pci_dev *pdev_in = rdev->en_dev->pdev; + int tmp_bus_num, bus_num = pdev_in->bus->number; + int tmp_dev_num, dev_num = PCI_SLOT(pdev_in->devfn); + struct bnxt_re_dev *tmp_rdev; + int pf_cnt = 0; + + rcu_read_lock(); + list_for_each_entry_rcu(tmp_rdev, &bnxt_re_dev_list, list) { + tmp_bus_num = tmp_rdev->en_dev->pdev->bus->number; + tmp_dev_num = PCI_SLOT(tmp_rdev->en_dev->pdev->devfn); + + if (bus_num == tmp_bus_num && dev_num == tmp_dev_num) + pf_cnt++; + } + rcu_read_unlock(); + return pf_cnt; +} + +/* Handle all deferred netevents tasks */ +static void bnxt_re_task(struct work_struct *work) +{ + struct bnxt_re_en_dev_info *en_info; + struct auxiliary_device *aux_dev; + struct bnxt_re_work *re_work; + struct bnxt_re_dev *rdev; + + re_work = container_of(work, struct bnxt_re_work, work); + + mutex_lock(&bnxt_re_mutex); + rdev = re_work->rdev; + + /* + * If the previous rdev is deleted due to bond creation + * do not handle the event + */ + if (!bnxt_re_is_rdev_valid(rdev)) + goto exit; + + /* Ignore the event, if the device is not registred with IB stack. This + * is to avoid handling any event while the device is added/removed. + */ + if (rdev && !test_bit(BNXT_RE_FLAG_IBDEV_REGISTERED, &rdev->flags)) { + dev_dbg(rdev_to_dev(rdev), "%s: Ignoring netdev event 0x%lx", + __func__, re_work->event); + goto done; + } + + /* Extra check to silence coverity. We shouldn't handle any event + * when rdev is NULL. + */ + if (!rdev) + goto exit; + + dev_dbg(rdev_to_dev(rdev), "Scheduled work for event 0x%lx", + re_work->event); + + switch (re_work->event) { + case NETDEV_UP: + bnxt_re_dispatch_event(&rdev->ibdev, NULL, 1, + IB_EVENT_PORT_ACTIVE); + bnxt_re_net_register_async_event(rdev); + break; + + case NETDEV_DOWN: + bnxt_qplib_dbr_pacing_set_primary_pf(rdev->chip_ctx, 0); + bnxt_re_stop_all_nonqp1_nonshadow_qps(rdev); + bnxt_re_dispatch_event(&rdev->ibdev, NULL, 1, + IB_EVENT_PORT_ERR); + break; + + case NETDEV_CHANGE: + if (bnxt_re_get_link_state(rdev) == IB_PORT_DOWN) { + bnxt_re_stop_all_nonqp1_nonshadow_qps(rdev); + bnxt_re_dispatch_event(&rdev->ibdev, NULL, 1, + IB_EVENT_PORT_ERR); + break; + } else if (bnxt_re_get_link_state(rdev) == IB_PORT_ACTIVE) { + bnxt_re_dispatch_event(&rdev->ibdev, NULL, 1, + IB_EVENT_PORT_ACTIVE); + } + + /* temporarily disable the check for SR2 */ + if (!bnxt_qplib_query_cc_param(&rdev->qplib_res, + &rdev->cc_param) && + !_is_chip_p7(rdev->chip_ctx)) { + /* + * Disable CC for 10G speed + * for non p5 devices + */ + if (rdev->sl_espeed == SPEED_10000 && + !_is_chip_gen_p5_p7(rdev->chip_ctx)) { + if (rdev->cc_param.enable) + bnxt_re_clear_cc(rdev); + } else { + if (!rdev->cc_param.enable && + rdev->cc_param.admin_enable) + bnxt_re_setup_cc(rdev); + } + } + break; + + case NETDEV_UNREGISTER: + bnxt_re_stopqps_and_ib_uninit(rdev); + aux_dev = rdev->adev; + if (re_work->adev) + goto done; + + bnxt_re_remove_device(rdev, BNXT_RE_COMPLETE_REMOVE, aux_dev); + + break; + + default: + break; + } +done: + if (rdev) { + /* memory barrier to guarantee task completion + * before decrementing sched count + */ + mmiowb(); + atomic_dec(&rdev->sched_count); + } +exit: + if (re_work->adev && re_work->event == NETDEV_UNREGISTER) { + en_info = auxiliary_get_drvdata(re_work->adev); + en_info->ib_uninit_done = true; + wake_up(&en_info->waitq); + } + kfree(re_work); + mutex_unlock(&bnxt_re_mutex); +} + +/* + "Notifier chain callback can be invoked for the same chain from + different CPUs at the same time". + + For cases when the netdev is already present, our call to the + register_netdevice_notifier() will actually get the rtnl_lock() + before sending NETDEV_REGISTER and (if up) NETDEV_UP + events. + + But for cases when the netdev is not already present, the notifier + chain is subjected to be invoked from different CPUs simultaneously. + + This is protected by the netdev_mutex. +*/ +static int bnxt_re_netdev_event(struct notifier_block *notifier, + unsigned long event, void *ptr) +{ + struct ifnet *real_dev, *netdev; + struct bnxt_re_dev *rdev = NULL; + + netdev = netdev_notifier_info_to_ifp(ptr); + real_dev = rdma_vlan_dev_real_dev(netdev); + if (!real_dev) + real_dev = netdev; + /* In case of bonding,this will be bond's rdev */ + rdev = bnxt_re_from_netdev(real_dev); + + if (!rdev) + goto exit; + + dev_info(rdev_to_dev(rdev), "%s: Event = %s (0x%lx), rdev %s (real_dev %s)\n", + __func__, bnxt_re_netevent(event), event, + rdev ? rdev->netdev ? rdev->netdev->if_dname : "->netdev = NULL" : "= NULL", + (real_dev == netdev) ? "= netdev" : real_dev->if_dname); + + if (!test_bit(BNXT_RE_FLAG_IBDEV_REGISTERED, &rdev->flags)) + goto exit; + + bnxt_re_hold(rdev); + + if (real_dev != netdev) { + switch (event) { + case NETDEV_UP: + bnxt_re_schedule_work(rdev, event, netdev, + NULL, NULL); + break; + case NETDEV_DOWN: + break; + default: + break; + } + goto done; + } + + switch (event) { + case NETDEV_CHANGEADDR: + if (!_is_chip_gen_p5_p7(rdev->chip_ctx)) + bnxt_re_update_shadow_ah(rdev); + bnxt_qplib_get_guid(rdev->dev_addr, + (u8 *)&rdev->ibdev.node_guid); + break; + + case NETDEV_CHANGE: + bnxt_re_get_link_speed(rdev); + bnxt_re_schedule_work(rdev, event, NULL, NULL, NULL); + break; + case NETDEV_UNREGISTER: + /* netdev notifier will call NETDEV_UNREGISTER again later since + * we are still holding the reference to the netdev + */ + + /* + * Workaround to avoid ib_unregister hang. Check for module + * reference and dont free up the device if the reference + * is non zero. Checking only for PF functions. + */ + + if (rdev) { + dev_info(rdev_to_dev(rdev), + "bnxt_re:Unreg recvd when module refcnt > 0"); + dev_info(rdev_to_dev(rdev), + "bnxt_re:Close all apps using bnxt_re devs"); + dev_info(rdev_to_dev(rdev), + "bnxt_re:Remove the configfs entry created for the device"); + dev_info(rdev_to_dev(rdev), + "bnxt_re:Refer documentation for details"); + goto done; + } + + if (atomic_read(&rdev->sched_count) > 0) + goto done; + if (!rdev->unreg_sched) { + bnxt_re_schedule_work(rdev, NETDEV_UNREGISTER, + NULL, NULL, NULL); + rdev->unreg_sched = true; + goto done; + } + + break; + default: + break; + } +done: + if (rdev) + bnxt_re_put(rdev); +exit: + return NOTIFY_DONE; +} + +static struct notifier_block bnxt_re_netdev_notifier = { + .notifier_call = bnxt_re_netdev_event +}; + +static void bnxt_re_remove_base_interface(struct bnxt_re_dev *rdev, + struct auxiliary_device *adev) +{ + bnxt_re_stopqps_and_ib_uninit(rdev); + bnxt_re_remove_device(rdev, BNXT_RE_COMPLETE_REMOVE, adev); + auxiliary_set_drvdata(adev, NULL); +} + +/* + * bnxt_re_remove - Removes the roce aux device + * @adev - aux device pointer + * + * This function removes the roce device. This gets + * called in the mod exit path and pci unbind path. + * If the rdev is bond interace, destroys the lag + * in module exit path, and in pci unbind case + * destroys the lag and recreates other base interface. + * If the device is already removed in error recovery + * path, it just unregister with the L2. + */ +static void bnxt_re_remove(struct auxiliary_device *adev) +{ + struct bnxt_re_en_dev_info *en_info = auxiliary_get_drvdata(adev); + struct bnxt_en_dev *en_dev; + struct bnxt_re_dev *rdev; + bool primary_dev = false; + bool secondary_dev = false; + + if (!en_info) + return; + + mutex_lock(&bnxt_re_mutex); + en_dev = en_info->en_dev; + + rdev = en_info->rdev; + + if (rdev && bnxt_re_is_rdev_valid(rdev)) { + if (pci_channel_offline(rdev->rcfw.pdev)) + set_bit(ERR_DEVICE_DETACHED, &rdev->rcfw.cmdq.flags); + + if (test_bit(BNXT_RE_FLAG_EN_DEV_PRIMARY_DEV, &en_info->flags)) + primary_dev = true; + if (test_bit(BNXT_RE_FLAG_EN_DEV_SECONDARY_DEV, &en_info->flags)) + secondary_dev = true; + + /* + * en_dev_info of primary device and secondary device have the + * same rdev pointer when LAG is configured. This rdev pointer + * is rdev of bond interface. + */ + if (!primary_dev && !secondary_dev) { + /* removal of non bond interface */ + bnxt_re_remove_base_interface(rdev, adev); + } else { + /* + * removal of bond primary/secondary interface. In this + * case bond device is already removed, so rdev->binfo + * is NULL. + */ + auxiliary_set_drvdata(adev, NULL); + } + } else { + /* device is removed from ulp stop, unregister the net dev */ + if (test_bit(BNXT_RE_FLAG_EN_DEV_NETDEV_REG, &en_info->flags)) { + rtnl_lock(); + en_dev->en_ops->bnxt_unregister_device(en_dev, + BNXT_ROCE_ULP); + rtnl_unlock(); + } + } + mutex_unlock(&bnxt_re_mutex); + return; +} + +/* wrapper for all external user context callers */ +void _bnxt_re_remove(struct auxiliary_device *adev) +{ + bnxt_re_remove(adev); +} + +static void bnxt_re_ib_init_2(struct bnxt_re_dev *rdev) +{ + int rc; + + rc = bnxt_re_get_device_stats(rdev); + if (rc) + dev_err(rdev_to_dev(rdev), + "Failed initial device stat query"); + + bnxt_re_net_register_async_event(rdev); +} + +static int bnxt_re_probe(struct auxiliary_device *adev, + const struct auxiliary_device_id *id) +{ + struct bnxt_aux_dev *aux_dev = + container_of(adev, struct bnxt_aux_dev, aux_dev); + struct bnxt_re_en_dev_info *en_info; + struct bnxt_en_dev *en_dev = NULL; + struct bnxt_re_dev *rdev; + int rc = -ENODEV; + + if (aux_dev) + en_dev = aux_dev->edev; + + if (!en_dev) + return rc; + + if (en_dev->ulp_version != BNXT_ULP_VERSION) { + pr_err("%s: probe error: bnxt_en ulp version magic %x is not compatible!\n", + ROCE_DRV_MODULE_NAME, en_dev->ulp_version); + return -EINVAL; + } + + en_info = kzalloc(sizeof(*en_info), GFP_KERNEL); + if (!en_info) + return -ENOMEM; + memset(en_info, 0, sizeof(struct bnxt_re_en_dev_info)); + en_info->en_dev = en_dev; + auxiliary_set_drvdata(adev, en_info); + + mutex_lock(&bnxt_re_mutex); + rc = bnxt_re_add_device(&rdev, en_dev->net, + BNXT_RE_GSI_MODE_ALL, + BNXT_RE_COMPLETE_INIT, + BNXT_QPLIB_WQE_MODE_STATIC, + BNXT_RE_MSIX_FROM_MOD_PARAM, adev); + if (rc) { + mutex_unlock(&bnxt_re_mutex); + return rc; + } + + rc = bnxt_re_ib_init(rdev); + if (rc) + goto err; + + bnxt_re_ib_init_2(rdev); + + dev_dbg(rdev_to_dev(rdev), "%s: adev: %p\n", __func__, adev); + rdev->adev = adev; + + mutex_unlock(&bnxt_re_mutex); + + return 0; + +err: + mutex_unlock(&bnxt_re_mutex); + bnxt_re_remove(adev); + + return rc; +} + +static const struct auxiliary_device_id bnxt_re_id_table[] = { + { .name = BNXT_ADEV_NAME ".rdma", }, + {}, +}; + +MODULE_DEVICE_TABLE(auxiliary, bnxt_re_id_table); + +static struct auxiliary_driver bnxt_re_driver = { + .name = "rdma", + .probe = bnxt_re_probe, + .remove = bnxt_re_remove, + .id_table = bnxt_re_id_table, +}; + +static int __init bnxt_re_mod_init(void) +{ + int rc = 0; + + pr_info("%s: %s", ROCE_DRV_MODULE_NAME, drv_version); + + bnxt_re_wq = create_singlethread_workqueue("bnxt_re"); + if (!bnxt_re_wq) + return -ENOMEM; + + rc = bnxt_re_register_netdevice_notifier(&bnxt_re_netdev_notifier); + if (rc) { + pr_err("%s: Cannot register to netdevice_notifier", + ROCE_DRV_MODULE_NAME); + goto err_netdev; + } + + INIT_LIST_HEAD(&bnxt_re_dev_list); + + rc = auxiliary_driver_register(&bnxt_re_driver); + if (rc) { + pr_err("%s: Failed to register auxiliary driver\n", + ROCE_DRV_MODULE_NAME); + goto err_auxdrv; + } + + return 0; + +err_auxdrv: + bnxt_re_unregister_netdevice_notifier(&bnxt_re_netdev_notifier); + +err_netdev: + destroy_workqueue(bnxt_re_wq); + + return rc; +} + +static void __exit bnxt_re_mod_exit(void) +{ + gmod_exit = 1; + auxiliary_driver_unregister(&bnxt_re_driver); + + bnxt_re_unregister_netdevice_notifier(&bnxt_re_netdev_notifier); + + if (bnxt_re_wq) + destroy_workqueue(bnxt_re_wq); +} + +module_init(bnxt_re_mod_init); +module_exit(bnxt_re_mod_exit); diff --git a/sys/dev/bnxt/bnxt_re/qplib_fp.c b/sys/dev/bnxt/bnxt_re/qplib_fp.c new file mode 100644 index 000000000000..438f7ddd468e --- /dev/null +++ b/sys/dev/bnxt/bnxt_re/qplib_fp.c @@ -0,0 +1,3544 @@ +/* + * Copyright (c) 2015-2024, Broadcom. All rights reserved. The term + * Broadcom refers to Broadcom Limited and/or its subsidiaries. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR + * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE + * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN + * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * Description: Fast Path Operators + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "hsi_struct_def.h" +#include "qplib_tlv.h" +#include "qplib_res.h" +#include "qplib_rcfw.h" +#include "qplib_sp.h" +#include "qplib_fp.h" +#include "ib_verbs.h" + +static void __clean_cq(struct bnxt_qplib_cq *cq, u64 qp); + +static void bnxt_re_legacy_cancel_phantom_processing(struct bnxt_qplib_qp *qp) +{ + qp->sq.condition = false; + qp->sq.legacy_send_phantom = false; + qp->sq.single = false; +} + +static void __bnxt_qplib_add_flush_qp(struct bnxt_qplib_qp *qp) +{ + struct bnxt_qplib_cq *scq, *rcq; + + scq = qp->scq; + rcq = qp->rcq; + + if (!qp->sq.flushed) { + dev_dbg(&scq->hwq.pdev->dev, + "QPLIB: FP: Adding to SQ Flush list = %p\n", + qp); + bnxt_re_legacy_cancel_phantom_processing(qp); + list_add_tail(&qp->sq_flush, &scq->sqf_head); + qp->sq.flushed = true; + } + if (!qp->srq) { + if (!qp->rq.flushed) { + dev_dbg(&rcq->hwq.pdev->dev, + "QPLIB: FP: Adding to RQ Flush list = %p\n", + qp); + list_add_tail(&qp->rq_flush, &rcq->rqf_head); + qp->rq.flushed = true; + } + } +} + +static void bnxt_qplib_acquire_cq_flush_locks(struct bnxt_qplib_qp *qp) + __acquires(&qp->scq->flush_lock) __acquires(&qp->rcq->flush_lock) +{ + /* Interrupts are already disabled in calling functions */ + spin_lock(&qp->scq->flush_lock); + if (qp->scq == qp->rcq) + __acquire(&qp->rcq->flush_lock); + else + spin_lock(&qp->rcq->flush_lock); +} + +static void bnxt_qplib_release_cq_flush_locks(struct bnxt_qplib_qp *qp) + __releases(&qp->scq->flush_lock) __releases(&qp->rcq->flush_lock) +{ + if (qp->scq == qp->rcq) + __release(&qp->rcq->flush_lock); + else + spin_unlock(&qp->rcq->flush_lock); + spin_unlock(&qp->scq->flush_lock); +} + +void bnxt_qplib_add_flush_qp(struct bnxt_qplib_qp *qp) +{ + + bnxt_qplib_acquire_cq_flush_locks(qp); + __bnxt_qplib_add_flush_qp(qp); + bnxt_qplib_release_cq_flush_locks(qp); +} + +static void __bnxt_qplib_del_flush_qp(struct bnxt_qplib_qp *qp) +{ + if (qp->sq.flushed) { + qp->sq.flushed = false; + list_del(&qp->sq_flush); + } + if (!qp->srq) { + if (qp->rq.flushed) { + qp->rq.flushed = false; + list_del(&qp->rq_flush); + } + } +} + +void bnxt_qplib_clean_qp(struct bnxt_qplib_qp *qp) +{ + + bnxt_qplib_acquire_cq_flush_locks(qp); + __clean_cq(qp->scq, (u64)(unsigned long)qp); + qp->sq.hwq.prod = 0; + qp->sq.hwq.cons = 0; + qp->sq.swq_start = 0; + qp->sq.swq_last = 0; + __clean_cq(qp->rcq, (u64)(unsigned long)qp); + qp->rq.hwq.prod = 0; + qp->rq.hwq.cons = 0; + qp->rq.swq_start = 0; + qp->rq.swq_last = 0; + + __bnxt_qplib_del_flush_qp(qp); + bnxt_qplib_release_cq_flush_locks(qp); +} + +static void bnxt_qpn_cqn_sched_task(struct work_struct *work) +{ + struct bnxt_qplib_nq_work *nq_work = + container_of(work, struct bnxt_qplib_nq_work, work); + + struct bnxt_qplib_cq *cq = nq_work->cq; + struct bnxt_qplib_nq *nq = nq_work->nq; + + if (cq && nq) { + spin_lock_bh(&cq->compl_lock); + if (nq->cqn_handler) { + dev_dbg(&nq->res->pdev->dev, + "%s:Trigger cq = %p event nq = %p\n", + __func__, cq, nq); + nq->cqn_handler(nq, cq); + } + spin_unlock_bh(&cq->compl_lock); + } + kfree(nq_work); +} + +static void bnxt_qplib_put_hdr_buf(struct pci_dev *pdev, + struct bnxt_qplib_hdrbuf *buf) +{ + dma_free_coherent(&pdev->dev, buf->len, buf->va, buf->dma_map); + kfree(buf); +} + +static void *bnxt_qplib_get_hdr_buf(struct pci_dev *pdev, u32 step, u32 cnt) +{ + struct bnxt_qplib_hdrbuf *hdrbuf; + u32 len; + + hdrbuf = kmalloc(sizeof(*hdrbuf), GFP_KERNEL); + if (!hdrbuf) + return NULL; + + len = ALIGN((step * cnt), PAGE_SIZE); + hdrbuf->va = dma_alloc_coherent(&pdev->dev, len, + &hdrbuf->dma_map, GFP_KERNEL); + if (!hdrbuf->va) + goto out; + + hdrbuf->len = len; + hdrbuf->step = step; + return hdrbuf; +out: + kfree(hdrbuf); + return NULL; +} + +void bnxt_qplib_free_hdr_buf(struct bnxt_qplib_res *res, + struct bnxt_qplib_qp *qp) +{ + if (qp->rq_hdr_buf) { + bnxt_qplib_put_hdr_buf(res->pdev, qp->rq_hdr_buf); + qp->rq_hdr_buf = NULL; + } + + if (qp->sq_hdr_buf) { + bnxt_qplib_put_hdr_buf(res->pdev, qp->sq_hdr_buf); + qp->sq_hdr_buf = NULL; + } +} + +int bnxt_qplib_alloc_hdr_buf(struct bnxt_qplib_res *res, + struct bnxt_qplib_qp *qp, u32 sstep, u32 rstep) +{ + struct pci_dev *pdev; + int rc = 0; + + pdev = res->pdev; + if (sstep) { + qp->sq_hdr_buf = bnxt_qplib_get_hdr_buf(pdev, sstep, + qp->sq.max_wqe); + if (!qp->sq_hdr_buf) { + dev_err(&pdev->dev, "QPLIB: Failed to get sq_hdr_buf\n"); + return -ENOMEM; + } + } + + if (rstep) { + qp->rq_hdr_buf = bnxt_qplib_get_hdr_buf(pdev, rstep, + qp->rq.max_wqe); + if (!qp->rq_hdr_buf) { + rc = -ENOMEM; + dev_err(&pdev->dev, "QPLIB: Failed to get rq_hdr_buf\n"); + goto fail; + } + } + + return 0; +fail: + bnxt_qplib_free_hdr_buf(res, qp); + return rc; +} + +/* + * clean_nq - Invalidate cqe from given nq. + * @cq - Completion queue + * + * Traverse whole notification queue and invalidate any completion + * associated cq handler provided by caller. + * Note - This function traverse the hardware queue but do not update + * consumer index. Invalidated cqe(marked from this function) will be + * ignored from actual completion of notification queue. + */ +static void clean_nq(struct bnxt_qplib_cq *cq) +{ + struct bnxt_qplib_hwq *nq_hwq = NULL; + struct bnxt_qplib_nq *nq = NULL; + struct nq_base *hw_nqe = NULL; + struct nq_cn *nqcne = NULL; + u32 peek_flags, peek_cons; + u64 q_handle; + u32 type; + int i; + + nq = cq->nq; + nq_hwq = &nq->hwq; + + spin_lock_bh(&nq_hwq->lock); + peek_flags = nq->nq_db.dbinfo.flags; + peek_cons = nq_hwq->cons; + for (i = 0; i < nq_hwq->max_elements; i++) { + hw_nqe = bnxt_qplib_get_qe(nq_hwq, peek_cons, NULL); + if (!NQE_CMP_VALID(hw_nqe, peek_flags)) + break; + + /* The valid test of the entry must be done first + * before reading any further. + */ + dma_rmb(); + type = le16_to_cpu(hw_nqe->info10_type) & + NQ_BASE_TYPE_MASK; + + /* Processing only NQ_BASE_TYPE_CQ_NOTIFICATION */ + if (type == NQ_BASE_TYPE_CQ_NOTIFICATION) { + nqcne = (struct nq_cn *)hw_nqe; + + q_handle = le32_to_cpu(nqcne->cq_handle_low); + q_handle |= (u64)le32_to_cpu(nqcne->cq_handle_high) << 32; + if (q_handle == (u64)cq) { + nqcne->cq_handle_low = 0; + nqcne->cq_handle_high = 0; + cq->cnq_events++; + } + } + bnxt_qplib_hwq_incr_cons(nq_hwq->max_elements, &peek_cons, + 1, &peek_flags); + } + spin_unlock_bh(&nq_hwq->lock); +} + +/* + * Wait for receiving all NQEs for this CQ. + * clean_nq is tried 100 times, each time clean_cq + * loops upto budget times. budget is based on the + * number of CQs shared by that NQ. So any NQE from + * CQ would be already in the NQ. + */ +static void __wait_for_all_nqes(struct bnxt_qplib_cq *cq, u16 cnq_events) +{ + u32 retry_cnt = 100; + u16 total_events; + + if (!cnq_events) { + clean_nq(cq); + return; + } + while (retry_cnt--) { + total_events = cq->cnq_events; + + /* Increment total_events by 1 if any CREQ event received with CQ notification */ + if (cq->is_cq_err_event) + total_events++; + + if (cnq_events == total_events) { + dev_dbg(&cq->nq->res->pdev->dev, + "QPLIB: NQ cleanup - Received all NQ events\n"); + return; + } + msleep(1); + clean_nq(cq); + } +} + +static void bnxt_qplib_service_nq(unsigned long data) +{ + struct bnxt_qplib_nq *nq = (struct bnxt_qplib_nq *)data; + struct bnxt_qplib_hwq *nq_hwq = &nq->hwq; + int budget = nq->budget; + struct bnxt_qplib_res *res; + struct bnxt_qplib_cq *cq; + struct pci_dev *pdev; + struct nq_base *nqe; + u32 hw_polled = 0; + u64 q_handle; + u32 type; + + res = nq->res; + pdev = res->pdev; + + spin_lock_bh(&nq_hwq->lock); + /* Service the NQ until empty or budget expired */ + while (budget--) { + nqe = bnxt_qplib_get_qe(nq_hwq, nq_hwq->cons, NULL); + if (!NQE_CMP_VALID(nqe, nq->nq_db.dbinfo.flags)) + break; + /* The valid test of the entry must be done first before + * reading any further. + */ + dma_rmb(); + type = le16_to_cpu(nqe->info10_type) & NQ_BASE_TYPE_MASK; + switch (type) { + case NQ_BASE_TYPE_CQ_NOTIFICATION: + { + struct nq_cn *nqcne = (struct nq_cn *)nqe; + + q_handle = le32_to_cpu(nqcne->cq_handle_low); + q_handle |= (u64)le32_to_cpu(nqcne->cq_handle_high) << 32; + cq = (struct bnxt_qplib_cq *)q_handle; + if (!cq) + break; + cq->toggle = (le16_to_cpu(nqe->info10_type) & NQ_CN_TOGGLE_MASK) >> NQ_CN_TOGGLE_SFT; + cq->dbinfo.toggle = cq->toggle; + bnxt_qplib_armen_db(&cq->dbinfo, + DBC_DBC_TYPE_CQ_ARMENA); + spin_lock_bh(&cq->compl_lock); + atomic_set(&cq->arm_state, 0) ; + if (!nq->cqn_handler(nq, (cq))) + nq->stats.num_cqne_processed++; + else + dev_warn(&pdev->dev, + "QPLIB: cqn - type 0x%x not handled\n", + type); + cq->cnq_events++; + spin_unlock_bh(&cq->compl_lock); + break; + } + case NQ_BASE_TYPE_SRQ_EVENT: + { + struct bnxt_qplib_srq *srq; + struct nq_srq_event *nqsrqe = + (struct nq_srq_event *)nqe; + + q_handle = le32_to_cpu(nqsrqe->srq_handle_low); + q_handle |= (u64)le32_to_cpu(nqsrqe->srq_handle_high) << 32; + srq = (struct bnxt_qplib_srq *)q_handle; + bnxt_qplib_armen_db(&srq->dbinfo, + DBC_DBC_TYPE_SRQ_ARMENA); + if (!nq->srqn_handler(nq, + (struct bnxt_qplib_srq *)q_handle, + nqsrqe->event)) + nq->stats.num_srqne_processed++; + else + dev_warn(&pdev->dev, + "QPLIB: SRQ event 0x%x not handled\n", + nqsrqe->event); + break; + } + default: + dev_warn(&pdev->dev, + "QPLIB: nqe with opcode = 0x%x not handled\n", + type); + break; + } + hw_polled++; + bnxt_qplib_hwq_incr_cons(nq_hwq->max_elements, &nq_hwq->cons, + 1, &nq->nq_db.dbinfo.flags); + } + nqe = bnxt_qplib_get_qe(nq_hwq, nq_hwq->cons, NULL); + if (!NQE_CMP_VALID(nqe, nq->nq_db.dbinfo.flags)) { + nq->stats.num_nq_rearm++; + bnxt_qplib_ring_nq_db(&nq->nq_db.dbinfo, res->cctx, true); + } else if (nq->requested) { + bnxt_qplib_ring_nq_db(&nq->nq_db.dbinfo, res->cctx, true); + nq->stats.num_tasklet_resched++; + } + dev_dbg(&pdev->dev, "QPLIB: cqn/srqn/dbqn \n"); + if (hw_polled >= 0) + dev_dbg(&pdev->dev, + "QPLIB: serviced %llu/%llu/%llu budget 0x%x reaped 0x%x\n", + nq->stats.num_cqne_processed, nq->stats.num_srqne_processed, + nq->stats.num_dbqne_processed, budget, hw_polled); + dev_dbg(&pdev->dev, + "QPLIB: resched_cnt = %llu arm_count = %llu\n", + nq->stats.num_tasklet_resched, nq->stats.num_nq_rearm); + spin_unlock_bh(&nq_hwq->lock); +} + +static irqreturn_t bnxt_qplib_nq_irq(int irq, void *dev_instance) +{ + struct bnxt_qplib_nq *nq = dev_instance; + struct bnxt_qplib_hwq *nq_hwq = &nq->hwq; + u32 sw_cons; + + /* Prefetch the NQ element */ + sw_cons = HWQ_CMP(nq_hwq->cons, nq_hwq); + if (sw_cons >= 0) + prefetch(bnxt_qplib_get_qe(nq_hwq, sw_cons, NULL)); + + bnxt_qplib_service_nq((unsigned long)nq); + + return IRQ_HANDLED; +} + +void bnxt_qplib_nq_stop_irq(struct bnxt_qplib_nq *nq, bool kill) +{ + struct bnxt_qplib_res *res; + + if (!nq->requested) + return; + + nq->requested = false; + res = nq->res; + /* Mask h/w interrupt */ + bnxt_qplib_ring_nq_db(&nq->nq_db.dbinfo, res->cctx, false); + /* Sync with last running IRQ handler */ + synchronize_irq(nq->msix_vec); + free_irq(nq->msix_vec, nq); + kfree(nq->name); + nq->name = NULL; +} + +void bnxt_qplib_disable_nq(struct bnxt_qplib_nq *nq) +{ + if (nq->cqn_wq) { + destroy_workqueue(nq->cqn_wq); + nq->cqn_wq = NULL; + } + /* Make sure the HW is stopped! */ + bnxt_qplib_nq_stop_irq(nq, true); + + nq->nq_db.reg.bar_reg = NULL; + nq->nq_db.db = NULL; + + nq->cqn_handler = NULL; + nq->srqn_handler = NULL; + nq->msix_vec = 0; +} + +int bnxt_qplib_nq_start_irq(struct bnxt_qplib_nq *nq, int nq_indx, + int msix_vector, bool need_init) +{ + struct bnxt_qplib_res *res; + int rc; + + res = nq->res; + if (nq->requested) + return -EFAULT; + + nq->msix_vec = msix_vector; + nq->name = kasprintf(GFP_KERNEL, "bnxt_re-nq-%d@pci:%s\n", + nq_indx, pci_name(res->pdev)); + if (!nq->name) + return -ENOMEM; + rc = request_irq(nq->msix_vec, bnxt_qplib_nq_irq, 0, nq->name, nq); + if (rc) { + kfree(nq->name); + nq->name = NULL; + return rc; + } + nq->requested = true; + bnxt_qplib_ring_nq_db(&nq->nq_db.dbinfo, res->cctx, true); + + return rc; +} + +static void bnxt_qplib_map_nq_db(struct bnxt_qplib_nq *nq, u32 reg_offt) +{ + struct bnxt_qplib_reg_desc *dbreg; + struct bnxt_qplib_nq_db *nq_db; + struct bnxt_qplib_res *res; + + nq_db = &nq->nq_db; + res = nq->res; + dbreg = &res->dpi_tbl.ucreg; + + nq_db->reg.bar_id = dbreg->bar_id; + nq_db->reg.bar_base = dbreg->bar_base; + nq_db->reg.bar_reg = dbreg->bar_reg + reg_offt; + nq_db->reg.len = _is_chip_gen_p5_p7(res->cctx) ? sizeof(u64) : + sizeof(u32); + + nq_db->dbinfo.db = nq_db->reg.bar_reg; + nq_db->dbinfo.hwq = &nq->hwq; + nq_db->dbinfo.xid = nq->ring_id; + nq_db->dbinfo.seed = nq->ring_id; + nq_db->dbinfo.flags = 0; + spin_lock_init(&nq_db->dbinfo.lock); + nq_db->dbinfo.shadow_key = BNXT_QPLIB_DBR_KEY_INVALID; + nq_db->dbinfo.res = nq->res; + + return; +} + +int bnxt_qplib_enable_nq(struct bnxt_qplib_nq *nq, int nq_idx, + int msix_vector, int bar_reg_offset, + cqn_handler_t cqn_handler, + srqn_handler_t srqn_handler) +{ + struct pci_dev *pdev; + int rc; + + pdev = nq->res->pdev; + nq->cqn_handler = cqn_handler; + nq->srqn_handler = srqn_handler; + nq->load = 0; + mutex_init(&nq->lock); + + /* Have a task to schedule CQ notifiers in post send case */ + nq->cqn_wq = create_singlethread_workqueue("bnxt_qplib_nq\n"); + if (!nq->cqn_wq) + return -ENOMEM; + + bnxt_qplib_map_nq_db(nq, bar_reg_offset); + rc = bnxt_qplib_nq_start_irq(nq, nq_idx, msix_vector, true); + if (rc) { + dev_err(&pdev->dev, + "QPLIB: Failed to request irq for nq-idx %d\n", nq_idx); + goto fail; + } + dev_dbg(&pdev->dev, "QPLIB: NQ max = 0x%x\n", nq->hwq.max_elements); + + return 0; +fail: + bnxt_qplib_disable_nq(nq); + return rc; +} + +void bnxt_qplib_free_nq_mem(struct bnxt_qplib_nq *nq) +{ + if (nq->hwq.max_elements) { + bnxt_qplib_free_hwq(nq->res, &nq->hwq); + nq->hwq.max_elements = 0; + } +} + +int bnxt_qplib_alloc_nq_mem(struct bnxt_qplib_res *res, + struct bnxt_qplib_nq *nq) +{ + struct bnxt_qplib_hwq_attr hwq_attr = {}; + struct bnxt_qplib_sg_info sginfo = {}; + + nq->res = res; + if (!nq->hwq.max_elements || + nq->hwq.max_elements > BNXT_QPLIB_NQE_MAX_CNT) + nq->hwq.max_elements = BNXT_QPLIB_NQE_MAX_CNT; + + sginfo.pgsize = PAGE_SIZE; + sginfo.pgshft = PAGE_SHIFT; + hwq_attr.res = res; + hwq_attr.sginfo = &sginfo; + hwq_attr.depth = nq->hwq.max_elements; + hwq_attr.stride = sizeof(struct nq_base); + hwq_attr.type = _get_hwq_type(res); + if (bnxt_qplib_alloc_init_hwq(&nq->hwq, &hwq_attr)) { + dev_err(&res->pdev->dev, "QPLIB: FP NQ allocation failed\n"); + return -ENOMEM; + } + nq->budget = 8; + return 0; +} + +/* SRQ */ +static int __qplib_destroy_srq(struct bnxt_qplib_rcfw *rcfw, + struct bnxt_qplib_srq *srq) +{ + struct creq_destroy_srq_resp resp = {}; + struct bnxt_qplib_cmdqmsg msg = {}; + struct cmdq_destroy_srq req = {}; + /* Configure the request */ + req.srq_cid = cpu_to_le32(srq->id); + bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_DESTROY_SRQ, + sizeof(req)); + bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL, sizeof(req), + sizeof(resp), 0); + return bnxt_qplib_rcfw_send_message(rcfw, &msg); +} + +int bnxt_qplib_destroy_srq(struct bnxt_qplib_res *res, + struct bnxt_qplib_srq *srq) +{ + struct bnxt_qplib_rcfw *rcfw = res->rcfw; + int rc; + + rc = __qplib_destroy_srq(rcfw, srq); + if (rc) + return rc; + bnxt_qplib_free_hwq(res, &srq->hwq); + kfree(srq->swq); + return 0; +} + +int bnxt_qplib_create_srq(struct bnxt_qplib_res *res, + struct bnxt_qplib_srq *srq) +{ + struct bnxt_qplib_hwq_attr hwq_attr = {}; + struct bnxt_qplib_rcfw *rcfw = res->rcfw; + struct creq_create_srq_resp resp = {}; + struct bnxt_qplib_cmdqmsg msg = {}; + struct cmdq_create_srq req = {}; + u16 pg_sz_lvl = 0; + u16 srq_size; + int rc, idx; + + hwq_attr.res = res; + hwq_attr.sginfo = &srq->sginfo; + hwq_attr.depth = srq->max_wqe; + hwq_attr.stride = srq->wqe_size; + hwq_attr.type = HWQ_TYPE_QUEUE; + rc = bnxt_qplib_alloc_init_hwq(&srq->hwq, &hwq_attr); + if (rc) + goto exit; + /* Configure the request */ + req.dpi = cpu_to_le32(srq->dpi->dpi); + req.srq_handle = cpu_to_le64(srq); + srq_size = min_t(u32, srq->hwq.depth, U16_MAX); + req.srq_size = cpu_to_le16(srq_size); + pg_sz_lvl |= (_get_base_pg_size(&srq->hwq) << + CMDQ_CREATE_SRQ_PG_SIZE_SFT); + pg_sz_lvl |= (srq->hwq.level & CMDQ_CREATE_SRQ_LVL_MASK); + req.pg_size_lvl = cpu_to_le16(pg_sz_lvl); + req.pbl = cpu_to_le64(_get_base_addr(&srq->hwq)); + req.pd_id = cpu_to_le32(srq->pd->id); + req.eventq_id = cpu_to_le16(srq->eventq_hw_ring_id); + bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_CREATE_SRQ, + sizeof(req)); + bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL, sizeof(req), + sizeof(resp), 0); + rc = bnxt_qplib_rcfw_send_message(rcfw, &msg); + if (rc) + goto fail; + if (!srq->is_user) { + srq->swq = kcalloc(srq->hwq.depth, sizeof(*srq->swq), + GFP_KERNEL); + if (!srq->swq) + goto srq_fail; + srq->start_idx = 0; + srq->last_idx = srq->hwq.depth - 1; + for (idx = 0; idx < srq->hwq.depth; idx++) + srq->swq[idx].next_idx = idx + 1; + srq->swq[srq->last_idx].next_idx = -1; + } + + spin_lock_init(&srq->lock); + srq->id = le32_to_cpu(resp.xid); + srq->cctx = res->cctx; + srq->dbinfo.hwq = &srq->hwq; + srq->dbinfo.xid = srq->id; + srq->dbinfo.db = srq->dpi->dbr; + srq->dbinfo.max_slot = 1; + srq->dbinfo.priv_db = res->dpi_tbl.priv_db; + srq->dbinfo.flags = 0; + spin_lock_init(&srq->dbinfo.lock); + srq->dbinfo.shadow_key = BNXT_QPLIB_DBR_KEY_INVALID; + srq->dbinfo.shadow_key_arm_ena = BNXT_QPLIB_DBR_KEY_INVALID; + srq->dbinfo.res = res; + srq->dbinfo.seed = srq->id; + if (srq->threshold) + bnxt_qplib_armen_db(&srq->dbinfo, DBC_DBC_TYPE_SRQ_ARMENA); + srq->arm_req = false; + return 0; +srq_fail: + __qplib_destroy_srq(rcfw, srq); +fail: + bnxt_qplib_free_hwq(res, &srq->hwq); +exit: + return rc; +} + +int bnxt_qplib_modify_srq(struct bnxt_qplib_res *res, + struct bnxt_qplib_srq *srq) +{ + struct bnxt_qplib_hwq *srq_hwq = &srq->hwq; + u32 avail = 0; + + avail = __bnxt_qplib_get_avail(srq_hwq); + if (avail <= srq->threshold) { + srq->arm_req = false; + bnxt_qplib_srq_arm_db(&srq->dbinfo); + } else { + /* Deferred arming */ + srq->arm_req = true; + } + return 0; +} + +int bnxt_qplib_query_srq(struct bnxt_qplib_res *res, + struct bnxt_qplib_srq *srq) +{ + struct bnxt_qplib_rcfw *rcfw = res->rcfw; + struct creq_query_srq_resp resp = {}; + struct bnxt_qplib_cmdqmsg msg = {}; + struct creq_query_srq_resp_sb *sb; + struct bnxt_qplib_rcfw_sbuf sbuf; + struct cmdq_query_srq req = {}; + int rc = 0; + + bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_QUERY_SRQ, + sizeof(req)); + sbuf.size = ALIGN(sizeof(*sb), BNXT_QPLIB_CMDQE_UNITS); + sbuf.sb = dma_zalloc_coherent(&rcfw->pdev->dev, sbuf.size, + &sbuf.dma_addr, GFP_KERNEL); + if (!sbuf.sb) + return -ENOMEM; + req.resp_size = sbuf.size / BNXT_QPLIB_CMDQE_UNITS; + req.srq_cid = cpu_to_le32(srq->id); + bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, &sbuf, sizeof(req), + sizeof(resp), 0); + rc = bnxt_qplib_rcfw_send_message(rcfw, &msg); + /* TODO: What to do with the query? */ + dma_free_coherent(&rcfw->pdev->dev, sbuf.size, + sbuf.sb, sbuf.dma_addr); + + return rc; +} + +int bnxt_qplib_post_srq_recv(struct bnxt_qplib_srq *srq, + struct bnxt_qplib_swqe *wqe) +{ + struct bnxt_qplib_hwq *srq_hwq = &srq->hwq; + struct sq_sge *hw_sge; + struct rq_wqe *srqe; + int i, rc = 0, next; + u32 avail; + + spin_lock(&srq_hwq->lock); + if (srq->start_idx == srq->last_idx) { + dev_err(&srq_hwq->pdev->dev, "QPLIB: FP: SRQ (0x%x) is full!\n", + srq->id); + rc = -EINVAL; + spin_unlock(&srq_hwq->lock); + goto done; + } + next = srq->start_idx; + srq->start_idx = srq->swq[next].next_idx; + spin_unlock(&srq_hwq->lock); + + srqe = bnxt_qplib_get_qe(srq_hwq, srq_hwq->prod, NULL); + memset(srqe, 0, srq->wqe_size); + /* Calculate wqe_size and data_len */ + for (i = 0, hw_sge = (struct sq_sge *)srqe->data; + i < wqe->num_sge; i++, hw_sge++) { + hw_sge->va_or_pa = cpu_to_le64(wqe->sg_list[i].addr); + hw_sge->l_key = cpu_to_le32(wqe->sg_list[i].lkey); + hw_sge->size = cpu_to_le32(wqe->sg_list[i].size); + } + srqe->wqe_type = wqe->type; + srqe->flags = wqe->flags; + srqe->wqe_size = wqe->num_sge + + ((offsetof(typeof(*srqe), data) + 15) >> 4); + if (!wqe->num_sge) + srqe->wqe_size++; + srqe->wr_id |= cpu_to_le32((u32)next); + srq->swq[next].wr_id = wqe->wr_id; + bnxt_qplib_hwq_incr_prod(&srq->dbinfo, srq_hwq, srq->dbinfo.max_slot); + /* retaining srq_hwq->cons for this logic actually the lock is only + * required to read srq_hwq->cons. + */ + spin_lock(&srq_hwq->lock); + avail = __bnxt_qplib_get_avail(srq_hwq); + spin_unlock(&srq_hwq->lock); + /* Ring DB */ + bnxt_qplib_ring_prod_db(&srq->dbinfo, DBC_DBC_TYPE_SRQ); + if (srq->arm_req && avail <= srq->threshold) { + srq->arm_req = false; + bnxt_qplib_srq_arm_db(&srq->dbinfo); + } +done: + return rc; +} + +/* QP */ +static int __qplib_destroy_qp(struct bnxt_qplib_rcfw *rcfw, + struct bnxt_qplib_qp *qp) +{ + struct creq_destroy_qp_resp resp = {}; + struct bnxt_qplib_cmdqmsg msg = {}; + struct cmdq_destroy_qp req = {}; + + req.qp_cid = cpu_to_le32(qp->id); + bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_DESTROY_QP, + sizeof(req)); + bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL, sizeof(req), + sizeof(resp), 0); + return bnxt_qplib_rcfw_send_message(rcfw, &msg); +} + +static int bnxt_qplib_alloc_init_swq(struct bnxt_qplib_q *que) +{ + int rc = 0; + int indx; + + que->swq = kcalloc(que->max_wqe, sizeof(*que->swq), GFP_KERNEL); + if (!que->swq) { + rc = -ENOMEM; + goto out; + } + + que->swq_start = 0; + que->swq_last = que->max_wqe - 1; + for (indx = 0; indx < que->max_wqe; indx++) + que->swq[indx].next_idx = indx + 1; + que->swq[que->swq_last].next_idx = 0; /* Make it circular */ + que->swq_last = 0; +out: + return rc; +} + +static struct bnxt_qplib_swq *bnxt_qplib_get_swqe(struct bnxt_qplib_q *que, + u32 *swq_idx) +{ + u32 idx; + + idx = que->swq_start; + if (swq_idx) + *swq_idx = idx; + return &que->swq[idx]; +} + +static void bnxt_qplib_swq_mod_start(struct bnxt_qplib_q *que, u32 idx) +{ + que->swq_start = que->swq[idx].next_idx; +} + +static u32 bnxt_qplib_get_stride(void) +{ + return sizeof(struct sq_sge); +} + +static u32 bnxt_qplib_get_depth(struct bnxt_qplib_q *que) +{ + u8 stride; + + stride = bnxt_qplib_get_stride(); + return (que->wqe_size * que->max_wqe) / stride; +} + +static u32 _set_sq_size(struct bnxt_qplib_q *que, u8 wqe_mode) +{ + /* For Variable mode supply number of 16B slots */ + return (wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC) ? + que->max_wqe : bnxt_qplib_get_depth(que); +} + +static u32 _set_sq_max_slot(u8 wqe_mode) +{ + /* for static mode index divisor is 8 */ + return (wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC) ? + sizeof(struct sq_send) / sizeof(struct sq_sge) : 1; +} + +static u32 _set_rq_max_slot(struct bnxt_qplib_q *que) +{ + return (que->wqe_size / sizeof(struct sq_sge)); +} + +int bnxt_qplib_create_qp1(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp) +{ + struct bnxt_qplib_hwq_attr hwq_attr = {}; + struct bnxt_qplib_rcfw *rcfw = res->rcfw; + struct creq_create_qp1_resp resp = {}; + struct bnxt_qplib_cmdqmsg msg = {}; + struct bnxt_qplib_q *sq = &qp->sq; + struct bnxt_qplib_q *rq = &qp->rq; + struct cmdq_create_qp1 req = {}; + struct bnxt_qplib_reftbl *tbl; + unsigned long flag; + u8 pg_sz_lvl = 0; + u32 qp_flags = 0; + int rc; + + /* General */ + req.type = qp->type; + req.dpi = cpu_to_le32(qp->dpi->dpi); + req.qp_handle = cpu_to_le64(qp->qp_handle); + /* SQ */ + hwq_attr.res = res; + hwq_attr.sginfo = &sq->sginfo; + hwq_attr.stride = bnxt_qplib_get_stride(); + hwq_attr.depth = bnxt_qplib_get_depth(sq); + hwq_attr.type = HWQ_TYPE_QUEUE; + rc = bnxt_qplib_alloc_init_hwq(&sq->hwq, &hwq_attr); + if (rc) + goto exit; + + req.sq_size = cpu_to_le32(_set_sq_size(sq, qp->wqe_mode)); + req.sq_pbl = cpu_to_le64(_get_base_addr(&sq->hwq)); + pg_sz_lvl = _get_base_pg_size(&sq->hwq) << + CMDQ_CREATE_QP1_SQ_PG_SIZE_SFT; + pg_sz_lvl |= ((sq->hwq.level & CMDQ_CREATE_QP1_SQ_LVL_MASK) << + CMDQ_CREATE_QP1_SQ_LVL_SFT); + req.sq_pg_size_sq_lvl = pg_sz_lvl; + req.sq_fwo_sq_sge = cpu_to_le16(((0 << CMDQ_CREATE_QP1_SQ_FWO_SFT) & + CMDQ_CREATE_QP1_SQ_FWO_MASK) | + (sq->max_sge & + CMDQ_CREATE_QP1_SQ_SGE_MASK)); + req.scq_cid = cpu_to_le32(qp->scq->id); + + /* RQ */ + if (!qp->srq) { + hwq_attr.res = res; + hwq_attr.sginfo = &rq->sginfo; + hwq_attr.stride = bnxt_qplib_get_stride(); + hwq_attr.depth = bnxt_qplib_get_depth(rq); + hwq_attr.type = HWQ_TYPE_QUEUE; + rc = bnxt_qplib_alloc_init_hwq(&rq->hwq, &hwq_attr); + if (rc) + goto fail_sq; + req.rq_size = cpu_to_le32(rq->max_wqe); + req.rq_pbl = cpu_to_le64(_get_base_addr(&rq->hwq)); + pg_sz_lvl = _get_base_pg_size(&rq->hwq) << + CMDQ_CREATE_QP1_RQ_PG_SIZE_SFT; + pg_sz_lvl |= ((rq->hwq.level & CMDQ_CREATE_QP1_RQ_LVL_MASK) << + CMDQ_CREATE_QP1_RQ_LVL_SFT); + req.rq_pg_size_rq_lvl = pg_sz_lvl; + req.rq_fwo_rq_sge = + cpu_to_le16(((0 << CMDQ_CREATE_QP1_RQ_FWO_SFT) & + CMDQ_CREATE_QP1_RQ_FWO_MASK) | + (rq->max_sge & + CMDQ_CREATE_QP1_RQ_SGE_MASK)); + } else { + /* SRQ */ + qp_flags |= CMDQ_CREATE_QP1_QP_FLAGS_SRQ_USED; + req.srq_cid = cpu_to_le32(qp->srq->id); + } + req.rcq_cid = cpu_to_le32(qp->rcq->id); + + qp_flags |= CMDQ_CREATE_QP1_QP_FLAGS_RESERVED_LKEY_ENABLE; + req.qp_flags = cpu_to_le32(qp_flags); + req.pd_id = cpu_to_le32(qp->pd->id); + + bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_CREATE_QP1, + sizeof(req)); + bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL, sizeof(req), + sizeof(resp), 0); + rc = bnxt_qplib_rcfw_send_message(rcfw, &msg); + if (rc) + goto fail_rq; + + rc = bnxt_qplib_alloc_init_swq(sq); + if (rc) + goto sq_swq; + + if (!qp->srq) { + rc = bnxt_qplib_alloc_init_swq(rq); + if (rc) + goto rq_swq; + } + + qp->id = le32_to_cpu(resp.xid); + qp->cur_qp_state = CMDQ_MODIFY_QP_NEW_STATE_RESET; + qp->cctx = res->cctx; + sq->dbinfo.hwq = &sq->hwq; + sq->dbinfo.xid = qp->id; + sq->dbinfo.db = qp->dpi->dbr; + sq->dbinfo.max_slot = _set_sq_max_slot(qp->wqe_mode); + sq->dbinfo.flags = 0; + spin_lock_init(&sq->dbinfo.lock); + sq->dbinfo.shadow_key = BNXT_QPLIB_DBR_KEY_INVALID; + sq->dbinfo.res = res; + if (rq->max_wqe) { + rq->dbinfo.hwq = &rq->hwq; + rq->dbinfo.xid = qp->id; + rq->dbinfo.db = qp->dpi->dbr; + rq->dbinfo.max_slot = _set_rq_max_slot(rq); + rq->dbinfo.flags = 0; + spin_lock_init(&rq->dbinfo.lock); + rq->dbinfo.shadow_key = BNXT_QPLIB_DBR_KEY_INVALID; + rq->dbinfo.res = res; + } + + tbl = &res->reftbl.qpref; + spin_lock_irqsave(&tbl->lock, flag); + tbl->rec[tbl->max].xid = qp->id; + tbl->rec[tbl->max].handle = qp; + spin_unlock_irqrestore(&tbl->lock, flag); + + return 0; +rq_swq: + kfree(sq->swq); +sq_swq: + __qplib_destroy_qp(rcfw, qp); +fail_rq: + bnxt_qplib_free_hwq(res, &rq->hwq); +fail_sq: + bnxt_qplib_free_hwq(res, &sq->hwq); +exit: + return rc; +} + +static void bnxt_qplib_init_psn_ptr(struct bnxt_qplib_qp *qp, int size) +{ + struct bnxt_qplib_hwq *sq_hwq; + struct bnxt_qplib_q *sq; + u64 fpsne, psn_pg; + u16 indx_pad = 0; + + sq = &qp->sq; + sq_hwq = &sq->hwq; + /* First psn entry */ + fpsne = (u64)bnxt_qplib_get_qe(sq_hwq, sq_hwq->depth, &psn_pg); + if (!IS_ALIGNED(fpsne, PAGE_SIZE)) + indx_pad = (fpsne & ~PAGE_MASK) / size; + sq_hwq->pad_pgofft = indx_pad; + sq_hwq->pad_pg = (u64 *)psn_pg; + sq_hwq->pad_stride = size; +} + +int bnxt_qplib_create_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp) +{ + struct bnxt_qplib_hwq_attr hwq_attr = {}; + struct bnxt_qplib_rcfw *rcfw = res->rcfw; + struct bnxt_qplib_sg_info sginfo = {}; + struct creq_create_qp_resp resp = {}; + struct bnxt_qplib_cmdqmsg msg = {}; + struct bnxt_qplib_q *sq = &qp->sq; + struct bnxt_qplib_q *rq = &qp->rq; + struct cmdq_create_qp req = {}; + struct bnxt_qplib_reftbl *tbl; + struct bnxt_qplib_hwq *xrrq; + int rc, req_size, psn_sz; + unsigned long flag; + u8 pg_sz_lvl = 0; + u32 qp_flags = 0; + u32 qp_idx; + u16 nsge; + u32 sqsz; + + qp->cctx = res->cctx; + if (res->dattr) + qp->dev_cap_flags = res->dattr->dev_cap_flags; + /* General */ + req.type = qp->type; + req.dpi = cpu_to_le32(qp->dpi->dpi); + req.qp_handle = cpu_to_le64(qp->qp_handle); + + /* SQ */ + if (qp->type == CMDQ_CREATE_QP_TYPE_RC) { + psn_sz = _is_chip_gen_p5_p7(qp->cctx) ? + sizeof(struct sq_psn_search_ext) : + sizeof(struct sq_psn_search); + if (BNXT_RE_HW_RETX(qp->dev_cap_flags)) { + psn_sz = sizeof(struct sq_msn_search); + qp->msn = 0; + } + } else { + psn_sz = 0; + } + + hwq_attr.res = res; + hwq_attr.sginfo = &sq->sginfo; + hwq_attr.stride = bnxt_qplib_get_stride(); + hwq_attr.depth = bnxt_qplib_get_depth(sq); + hwq_attr.aux_stride = psn_sz; + hwq_attr.aux_depth = (psn_sz) ? + _set_sq_size(sq, qp->wqe_mode) : 0; + /* Update msn tbl size */ + if (BNXT_RE_HW_RETX(qp->dev_cap_flags) && psn_sz) { + if (qp->wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC) + hwq_attr.aux_depth = roundup_pow_of_two(_set_sq_size(sq, qp->wqe_mode)); + else + hwq_attr.aux_depth = roundup_pow_of_two(_set_sq_size(sq, qp->wqe_mode)) / 2; + qp->msn_tbl_sz = hwq_attr.aux_depth; + qp->msn = 0; + } + hwq_attr.type = HWQ_TYPE_QUEUE; + rc = bnxt_qplib_alloc_init_hwq(&sq->hwq, &hwq_attr); + if (rc) + goto exit; + + sqsz = _set_sq_size(sq, qp->wqe_mode); + /* 0xffff is the max sq size hw limits to */ + if (sqsz > BNXT_QPLIB_MAX_SQSZ) { + pr_err("QPLIB: FP: QP (0x%x) exceeds sq size %d\n", qp->id, sqsz); + goto fail_sq; + } + req.sq_size = cpu_to_le32(sqsz); + req.sq_pbl = cpu_to_le64(_get_base_addr(&sq->hwq)); + pg_sz_lvl = _get_base_pg_size(&sq->hwq) << + CMDQ_CREATE_QP_SQ_PG_SIZE_SFT; + pg_sz_lvl |= ((sq->hwq.level & CMDQ_CREATE_QP_SQ_LVL_MASK) << + CMDQ_CREATE_QP_SQ_LVL_SFT); + req.sq_pg_size_sq_lvl = pg_sz_lvl; + req.sq_fwo_sq_sge = cpu_to_le16(((0 << CMDQ_CREATE_QP_SQ_FWO_SFT) & + CMDQ_CREATE_QP_SQ_FWO_MASK) | + ((BNXT_RE_HW_RETX(qp->dev_cap_flags)) ? + BNXT_MSN_TBLE_SGE : sq->max_sge & + CMDQ_CREATE_QP_SQ_SGE_MASK)); + req.scq_cid = cpu_to_le32(qp->scq->id); + + /* RQ/SRQ */ + if (!qp->srq) { + hwq_attr.res = res; + hwq_attr.sginfo = &rq->sginfo; + hwq_attr.stride = bnxt_qplib_get_stride(); + hwq_attr.depth = bnxt_qplib_get_depth(rq); + hwq_attr.aux_stride = 0; + hwq_attr.aux_depth = 0; + hwq_attr.type = HWQ_TYPE_QUEUE; + rc = bnxt_qplib_alloc_init_hwq(&rq->hwq, &hwq_attr); + if (rc) + goto fail_sq; + req.rq_size = cpu_to_le32(rq->max_wqe); + req.rq_pbl = cpu_to_le64(_get_base_addr(&rq->hwq)); + pg_sz_lvl = _get_base_pg_size(&rq->hwq) << + CMDQ_CREATE_QP_RQ_PG_SIZE_SFT; + pg_sz_lvl |= ((rq->hwq.level & CMDQ_CREATE_QP_RQ_LVL_MASK) << + CMDQ_CREATE_QP_RQ_LVL_SFT); + req.rq_pg_size_rq_lvl = pg_sz_lvl; + nsge = (qp->wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC) ? + res->dattr->max_qp_sges : rq->max_sge; + req.rq_fwo_rq_sge = + cpu_to_le16(((0 << CMDQ_CREATE_QP_RQ_FWO_SFT) & + CMDQ_CREATE_QP_RQ_FWO_MASK) | + (nsge & CMDQ_CREATE_QP_RQ_SGE_MASK)); + } else { + qp_flags |= CMDQ_CREATE_QP_QP_FLAGS_SRQ_USED; + req.srq_cid = cpu_to_le32(qp->srq->id); + } + req.rcq_cid = cpu_to_le32(qp->rcq->id); + + qp_flags |= CMDQ_CREATE_QP_QP_FLAGS_RESERVED_LKEY_ENABLE; + qp_flags |= CMDQ_CREATE_QP_QP_FLAGS_FR_PMR_ENABLED; + if (qp->sig_type) + qp_flags |= CMDQ_CREATE_QP_QP_FLAGS_FORCE_COMPLETION; + if (qp->wqe_mode == BNXT_QPLIB_WQE_MODE_VARIABLE) + qp_flags |= CMDQ_CREATE_QP_QP_FLAGS_VARIABLE_SIZED_WQE_ENABLED; + if (res->cctx->modes.te_bypass) + qp_flags |= CMDQ_CREATE_QP_QP_FLAGS_OPTIMIZED_TRANSMIT_ENABLED; + if (res->dattr && + bnxt_ext_stats_supported(qp->cctx, res->dattr->dev_cap_flags, res->is_vf)) + qp_flags |= CMDQ_CREATE_QP_QP_FLAGS_EXT_STATS_ENABLED; + req.qp_flags = cpu_to_le32(qp_flags); + + /* ORRQ and IRRQ */ + if (psn_sz) { + xrrq = &qp->orrq; + xrrq->max_elements = + ORD_LIMIT_TO_ORRQ_SLOTS(qp->max_rd_atomic); + req_size = xrrq->max_elements * + BNXT_QPLIB_MAX_ORRQE_ENTRY_SIZE + PAGE_SIZE - 1; + req_size &= ~(PAGE_SIZE - 1); + sginfo.pgsize = req_size; + sginfo.pgshft = PAGE_SHIFT; + + hwq_attr.res = res; + hwq_attr.sginfo = &sginfo; + hwq_attr.depth = xrrq->max_elements; + hwq_attr.stride = BNXT_QPLIB_MAX_ORRQE_ENTRY_SIZE; + hwq_attr.aux_stride = 0; + hwq_attr.aux_depth = 0; + hwq_attr.type = HWQ_TYPE_CTX; + rc = bnxt_qplib_alloc_init_hwq(xrrq, &hwq_attr); + if (rc) + goto fail_rq; + req.orrq_addr = cpu_to_le64(_get_base_addr(xrrq)); + + xrrq = &qp->irrq; + xrrq->max_elements = IRD_LIMIT_TO_IRRQ_SLOTS( + qp->max_dest_rd_atomic); + req_size = xrrq->max_elements * + BNXT_QPLIB_MAX_IRRQE_ENTRY_SIZE + PAGE_SIZE - 1; + req_size &= ~(PAGE_SIZE - 1); + sginfo.pgsize = req_size; + hwq_attr.depth = xrrq->max_elements; + hwq_attr.stride = BNXT_QPLIB_MAX_IRRQE_ENTRY_SIZE; + rc = bnxt_qplib_alloc_init_hwq(xrrq, &hwq_attr); + if (rc) + goto fail_orrq; + req.irrq_addr = cpu_to_le64(_get_base_addr(xrrq)); + } + req.pd_id = cpu_to_le32(qp->pd->id); + + bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_CREATE_QP, + sizeof(req)); + bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL, sizeof(req), + sizeof(resp), 0); + rc = bnxt_qplib_rcfw_send_message(rcfw, &msg); + if (rc) + goto fail; + + if (!qp->is_user) { + rc = bnxt_qplib_alloc_init_swq(sq); + if (rc) + goto swq_sq; + if (!qp->srq) { + rc = bnxt_qplib_alloc_init_swq(rq); + if (rc) + goto swq_rq; + } + if (psn_sz) + bnxt_qplib_init_psn_ptr(qp, psn_sz); + } + qp->id = le32_to_cpu(resp.xid); + qp->cur_qp_state = CMDQ_MODIFY_QP_NEW_STATE_RESET; + INIT_LIST_HEAD(&qp->sq_flush); + INIT_LIST_HEAD(&qp->rq_flush); + + sq->dbinfo.hwq = &sq->hwq; + sq->dbinfo.xid = qp->id; + sq->dbinfo.db = qp->dpi->dbr; + sq->dbinfo.max_slot = _set_sq_max_slot(qp->wqe_mode); + sq->dbinfo.flags = 0; + spin_lock_init(&sq->dbinfo.lock); + sq->dbinfo.shadow_key = BNXT_QPLIB_DBR_KEY_INVALID; + sq->dbinfo.res = res; + sq->dbinfo.seed = qp->id; + if (rq->max_wqe) { + rq->dbinfo.hwq = &rq->hwq; + rq->dbinfo.xid = qp->id; + rq->dbinfo.db = qp->dpi->dbr; + rq->dbinfo.max_slot = _set_rq_max_slot(rq); + rq->dbinfo.flags = 0; + spin_lock_init(&rq->dbinfo.lock); + rq->dbinfo.shadow_key = BNXT_QPLIB_DBR_KEY_INVALID; + rq->dbinfo.res = res; + rq->dbinfo.seed = qp->id; + } + + tbl = &res->reftbl.qpref; + qp_idx = map_qp_id_to_tbl_indx(qp->id, tbl); + spin_lock_irqsave(&tbl->lock, flag); + tbl->rec[qp_idx].xid = qp->id; + tbl->rec[qp_idx].handle = qp; + spin_unlock_irqrestore(&tbl->lock, flag); + + return 0; +swq_rq: + kfree(sq->swq); +swq_sq: + __qplib_destroy_qp(rcfw, qp); +fail: + bnxt_qplib_free_hwq(res, &qp->irrq); +fail_orrq: + bnxt_qplib_free_hwq(res, &qp->orrq); +fail_rq: + bnxt_qplib_free_hwq(res, &rq->hwq); +fail_sq: + bnxt_qplib_free_hwq(res, &sq->hwq); +exit: + return rc; +} + +static void __filter_modify_flags(struct bnxt_qplib_qp *qp) +{ + switch (qp->cur_qp_state) { + case CMDQ_MODIFY_QP_NEW_STATE_RESET: + switch (qp->state) { + case CMDQ_MODIFY_QP_NEW_STATE_INIT: + break; + default: + break; + } + break; + case CMDQ_MODIFY_QP_NEW_STATE_INIT: + switch (qp->state) { + case CMDQ_MODIFY_QP_NEW_STATE_RTR: + if (!(qp->modify_flags & + CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU)) { + qp->modify_flags |= + CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU; + qp->path_mtu = CMDQ_MODIFY_QP_PATH_MTU_MTU_2048; + } + qp->modify_flags &= + ~CMDQ_MODIFY_QP_MODIFY_MASK_VLAN_ID; + /* Bono FW requires the max_dest_rd_atomic to be >= 1 */ + if (qp->max_dest_rd_atomic < 1) + qp->max_dest_rd_atomic = 1; + qp->modify_flags &= ~CMDQ_MODIFY_QP_MODIFY_MASK_SRC_MAC; + /* Bono FW 20.6.5 requires SGID_INDEX to be configured */ + if (!(qp->modify_flags & + CMDQ_MODIFY_QP_MODIFY_MASK_SGID_INDEX)) { + qp->modify_flags |= + CMDQ_MODIFY_QP_MODIFY_MASK_SGID_INDEX; + qp->ah.sgid_index = 0; + } + break; + default: + break; + } + break; + case CMDQ_MODIFY_QP_NEW_STATE_RTR: + switch (qp->state) { + case CMDQ_MODIFY_QP_NEW_STATE_RTS: + /* Bono FW requires the max_rd_atomic to be >= 1 */ + if (qp->max_rd_atomic < 1) + qp->max_rd_atomic = 1; + qp->modify_flags &= + ~(CMDQ_MODIFY_QP_MODIFY_MASK_PKEY | + CMDQ_MODIFY_QP_MODIFY_MASK_DGID | + CMDQ_MODIFY_QP_MODIFY_MASK_FLOW_LABEL | + CMDQ_MODIFY_QP_MODIFY_MASK_SGID_INDEX | + CMDQ_MODIFY_QP_MODIFY_MASK_HOP_LIMIT | + CMDQ_MODIFY_QP_MODIFY_MASK_TRAFFIC_CLASS | + CMDQ_MODIFY_QP_MODIFY_MASK_DEST_MAC | + CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU | + CMDQ_MODIFY_QP_MODIFY_MASK_RQ_PSN | + CMDQ_MODIFY_QP_MODIFY_MASK_MIN_RNR_TIMER | + CMDQ_MODIFY_QP_MODIFY_MASK_MAX_DEST_RD_ATOMIC | + CMDQ_MODIFY_QP_MODIFY_MASK_DEST_QP_ID); + break; + default: + break; + } + break; + case CMDQ_MODIFY_QP_NEW_STATE_RTS: + break; + case CMDQ_MODIFY_QP_NEW_STATE_SQD: + break; + case CMDQ_MODIFY_QP_NEW_STATE_SQE: + break; + case CMDQ_MODIFY_QP_NEW_STATE_ERR: + break; + default: + break; + } +} + +int bnxt_qplib_modify_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp) +{ + struct bnxt_qplib_rcfw *rcfw = res->rcfw; + struct creq_modify_qp_resp resp = {}; + struct bnxt_qplib_cmdqmsg msg = {}; + struct cmdq_modify_qp req = {}; + bool ppp_requested = false; + u32 temp32[4]; + u32 bmask; + int rc; + + bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_MODIFY_QP, + sizeof(req)); + + /* Filter out the qp_attr_mask based on the state->new transition */ + __filter_modify_flags(qp); + bmask = qp->modify_flags; + req.modify_mask = cpu_to_le32(qp->modify_flags); + req.qp_cid = cpu_to_le32(qp->id); + if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_STATE) { + req.network_type_en_sqd_async_notify_new_state = + (qp->state & CMDQ_MODIFY_QP_NEW_STATE_MASK) | + (qp->en_sqd_async_notify == true ? + CMDQ_MODIFY_QP_EN_SQD_ASYNC_NOTIFY : 0); + if (__can_request_ppp(qp)) { + req.path_mtu_pingpong_push_enable = + CMDQ_MODIFY_QP_PINGPONG_PUSH_ENABLE; + req.pingpong_push_dpi = qp->ppp.dpi; + ppp_requested = true; + } + } + req.network_type_en_sqd_async_notify_new_state |= qp->nw_type; + + if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_ACCESS) { + req.access = qp->access; + } + if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_PKEY) + req.pkey = IB_DEFAULT_PKEY_FULL; + + if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_QKEY) { + req.qkey = cpu_to_le32(qp->qkey); + } + if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_DGID) { + memcpy(temp32, qp->ah.dgid.data, sizeof(struct bnxt_qplib_gid)); + req.dgid[0] = cpu_to_le32(temp32[0]); + req.dgid[1] = cpu_to_le32(temp32[1]); + req.dgid[2] = cpu_to_le32(temp32[2]); + req.dgid[3] = cpu_to_le32(temp32[3]); + } + if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_FLOW_LABEL) { + req.flow_label = cpu_to_le32(qp->ah.flow_label); + } + if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_SGID_INDEX) { + req.sgid_index = cpu_to_le16(res->sgid_tbl.hw_id[qp->ah.sgid_index]); + } + if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_HOP_LIMIT) { + req.hop_limit = qp->ah.hop_limit; + } + if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_TRAFFIC_CLASS) { + req.traffic_class = qp->ah.traffic_class; + } + if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_DEST_MAC) { + memcpy(req.dest_mac, qp->ah.dmac, 6); + } + if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU) { + req.path_mtu_pingpong_push_enable = qp->path_mtu; + } + if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_TIMEOUT) { + req.timeout = qp->timeout; + } + if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_RETRY_CNT) { + req.retry_cnt = qp->retry_cnt; + } + if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_RNR_RETRY) { + req.rnr_retry = qp->rnr_retry; + } + if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_MIN_RNR_TIMER) { + req.min_rnr_timer = qp->min_rnr_timer; + } + if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_RQ_PSN) { + req.rq_psn = cpu_to_le32(qp->rq.psn); + } + if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_SQ_PSN) { + req.sq_psn = cpu_to_le32(qp->sq.psn); + } + if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_MAX_RD_ATOMIC) { + req.max_rd_atomic = + ORD_LIMIT_TO_ORRQ_SLOTS(qp->max_rd_atomic); + } + if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_MAX_DEST_RD_ATOMIC) { + req.max_dest_rd_atomic = + IRD_LIMIT_TO_IRRQ_SLOTS(qp->max_dest_rd_atomic); + } + req.sq_size = cpu_to_le32(qp->sq.hwq.max_elements); + req.rq_size = cpu_to_le32(qp->rq.hwq.max_elements); + req.sq_sge = cpu_to_le16(qp->sq.max_sge); + req.rq_sge = cpu_to_le16(qp->rq.max_sge); + req.max_inline_data = cpu_to_le32(qp->max_inline_data); + if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_DEST_QP_ID) + req.dest_qp_id = cpu_to_le32(qp->dest_qpn); + if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_ENABLE_CC) + req.enable_cc = cpu_to_le16(CMDQ_MODIFY_QP_ENABLE_CC); + if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_TOS_ECN) + req.tos_dscp_tos_ecn = + ((qp->tos_ecn << CMDQ_MODIFY_QP_TOS_ECN_SFT) & + CMDQ_MODIFY_QP_TOS_ECN_MASK); + if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_TOS_DSCP) + req.tos_dscp_tos_ecn |= + ((qp->tos_dscp << CMDQ_MODIFY_QP_TOS_DSCP_SFT) & + CMDQ_MODIFY_QP_TOS_DSCP_MASK); + req.vlan_pcp_vlan_dei_vlan_id = cpu_to_le16(qp->vlan_id); + bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL, sizeof(req), + sizeof(resp), 0); + msg.qp_state = qp->state; + + rc = bnxt_qplib_rcfw_send_message(rcfw, &msg); + if (rc == -ETIMEDOUT && (qp->state == CMDQ_MODIFY_QP_NEW_STATE_ERR)) { + qp->cur_qp_state = qp->state; + return 0; + } else if (rc) { + return rc; + } + if (qp->state == CMDQ_MODIFY_QP_NEW_STATE_RTR) + qp->lag_src_mac = be32_to_cpu(resp.lag_src_mac); + + if (ppp_requested) + qp->ppp.st_idx_en = resp.pingpong_push_state_index_enabled; + + qp->cur_qp_state = qp->state; + return 0; +} + +int bnxt_qplib_query_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp) +{ + struct bnxt_qplib_rcfw *rcfw = res->rcfw; + struct creq_query_qp_resp resp = {}; + struct bnxt_qplib_cmdqmsg msg = {}; + struct bnxt_qplib_rcfw_sbuf sbuf; + struct creq_query_qp_resp_sb *sb; + struct cmdq_query_qp req = {}; + u32 temp32[4]; + int i, rc; + + sbuf.size = ALIGN(sizeof(*sb), BNXT_QPLIB_CMDQE_UNITS); + sbuf.sb = dma_zalloc_coherent(&rcfw->pdev->dev, sbuf.size, + &sbuf.dma_addr, GFP_KERNEL); + if (!sbuf.sb) + return -ENOMEM; + sb = sbuf.sb; + + bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_QUERY_QP, + sizeof(req)); + req.qp_cid = cpu_to_le32(qp->id); + req.resp_size = sbuf.size / BNXT_QPLIB_CMDQE_UNITS; + bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, &sbuf, sizeof(req), + sizeof(resp), 0); + rc = bnxt_qplib_rcfw_send_message(rcfw, &msg); + if (rc) + goto bail; + + /* Extract the context from the side buffer */ + qp->state = sb->en_sqd_async_notify_state & + CREQ_QUERY_QP_RESP_SB_STATE_MASK; + qp->cur_qp_state = qp->state; + qp->en_sqd_async_notify = sb->en_sqd_async_notify_state & + CREQ_QUERY_QP_RESP_SB_EN_SQD_ASYNC_NOTIFY ? + true : false; + qp->access = sb->access; + qp->pkey_index = le16_to_cpu(sb->pkey); + qp->qkey = le32_to_cpu(sb->qkey); + + temp32[0] = le32_to_cpu(sb->dgid[0]); + temp32[1] = le32_to_cpu(sb->dgid[1]); + temp32[2] = le32_to_cpu(sb->dgid[2]); + temp32[3] = le32_to_cpu(sb->dgid[3]); + memcpy(qp->ah.dgid.data, temp32, sizeof(qp->ah.dgid.data)); + + qp->ah.flow_label = le32_to_cpu(sb->flow_label); + + qp->ah.sgid_index = 0; + for (i = 0; i < res->sgid_tbl.max; i++) { + if (res->sgid_tbl.hw_id[i] == le16_to_cpu(sb->sgid_index)) { + qp->ah.sgid_index = i; + break; + } + } + if (i == res->sgid_tbl.max) + dev_dbg(&res->pdev->dev, + "QPLIB: SGID not found qp->id = 0x%x sgid_index = 0x%x\n", + qp->id, le16_to_cpu(sb->sgid_index)); + + qp->ah.hop_limit = sb->hop_limit; + qp->ah.traffic_class = sb->traffic_class; + memcpy(qp->ah.dmac, sb->dest_mac, ETH_ALEN); + qp->ah.vlan_id = le16_to_cpu(sb->path_mtu_dest_vlan_id) & + CREQ_QUERY_QP_RESP_SB_VLAN_ID_MASK >> + CREQ_QUERY_QP_RESP_SB_VLAN_ID_SFT; + qp->path_mtu = le16_to_cpu(sb->path_mtu_dest_vlan_id) & + CREQ_QUERY_QP_RESP_SB_PATH_MTU_MASK; + qp->timeout = sb->timeout; + qp->retry_cnt = sb->retry_cnt; + qp->rnr_retry = sb->rnr_retry; + qp->min_rnr_timer = sb->min_rnr_timer; + qp->rq.psn = le32_to_cpu(sb->rq_psn); + qp->max_rd_atomic = ORRQ_SLOTS_TO_ORD_LIMIT(sb->max_rd_atomic); + qp->sq.psn = le32_to_cpu(sb->sq_psn); + qp->max_dest_rd_atomic = + IRRQ_SLOTS_TO_IRD_LIMIT(sb->max_dest_rd_atomic); + qp->sq.max_wqe = qp->sq.hwq.max_elements; + qp->rq.max_wqe = qp->rq.hwq.max_elements; + qp->sq.max_sge = le16_to_cpu(sb->sq_sge); + qp->rq.max_sge = le16_to_cpu(sb->rq_sge); + qp->max_inline_data = le32_to_cpu(sb->max_inline_data); + qp->dest_qpn = le32_to_cpu(sb->dest_qp_id); + memcpy(qp->smac, sb->src_mac, ETH_ALEN); + qp->vlan_id = le16_to_cpu(sb->vlan_pcp_vlan_dei_vlan_id); + qp->port_id = le16_to_cpu(sb->port_id); +bail: + dma_free_coherent(&rcfw->pdev->dev, sbuf.size, + sbuf.sb, sbuf.dma_addr); + return rc; +} + + +static void __clean_cq(struct bnxt_qplib_cq *cq, u64 qp) +{ + struct bnxt_qplib_hwq *cq_hwq = &cq->hwq; + u32 peek_flags, peek_cons; + struct cq_base *hw_cqe; + int i; + + peek_flags = cq->dbinfo.flags; + peek_cons = cq_hwq->cons; + for (i = 0; i < cq_hwq->depth; i++) { + hw_cqe = bnxt_qplib_get_qe(cq_hwq, peek_cons, NULL); + if (CQE_CMP_VALID(hw_cqe, peek_flags)) { + dma_rmb(); + switch (hw_cqe->cqe_type_toggle & CQ_BASE_CQE_TYPE_MASK) { + case CQ_BASE_CQE_TYPE_REQ: + case CQ_BASE_CQE_TYPE_TERMINAL: + { + struct cq_req *cqe = (struct cq_req *)hw_cqe; + + if (qp == le64_to_cpu(cqe->qp_handle)) + cqe->qp_handle = 0; + break; + } + case CQ_BASE_CQE_TYPE_RES_RC: + case CQ_BASE_CQE_TYPE_RES_UD: + case CQ_BASE_CQE_TYPE_RES_RAWETH_QP1: + { + struct cq_res_rc *cqe = (struct cq_res_rc *)hw_cqe; + + if (qp == le64_to_cpu(cqe->qp_handle)) + cqe->qp_handle = 0; + break; + } + default: + break; + } + } + bnxt_qplib_hwq_incr_cons(cq_hwq->depth, &peek_cons, + 1, &peek_flags); + } +} + +int bnxt_qplib_destroy_qp(struct bnxt_qplib_res *res, + struct bnxt_qplib_qp *qp) +{ + struct bnxt_qplib_rcfw *rcfw = res->rcfw; + struct bnxt_qplib_reftbl *tbl; + unsigned long flags; + u32 qp_idx; + int rc; + + tbl = &res->reftbl.qpref; + qp_idx = map_qp_id_to_tbl_indx(qp->id, tbl); + spin_lock_irqsave(&tbl->lock, flags); + tbl->rec[qp_idx].xid = BNXT_QPLIB_QP_ID_INVALID; + tbl->rec[qp_idx].handle = NULL; + spin_unlock_irqrestore(&tbl->lock, flags); + + rc = __qplib_destroy_qp(rcfw, qp); + if (rc) { + spin_lock_irqsave(&tbl->lock, flags); + tbl->rec[qp_idx].xid = qp->id; + tbl->rec[qp_idx].handle = qp; + spin_unlock_irqrestore(&tbl->lock, flags); + return rc; + } + + return 0; +} + +void bnxt_qplib_free_qp_res(struct bnxt_qplib_res *res, + struct bnxt_qplib_qp *qp) +{ + if (qp->irrq.max_elements) + bnxt_qplib_free_hwq(res, &qp->irrq); + if (qp->orrq.max_elements) + bnxt_qplib_free_hwq(res, &qp->orrq); + + if (!qp->is_user) + kfree(qp->rq.swq); + bnxt_qplib_free_hwq(res, &qp->rq.hwq); + + if (!qp->is_user) + kfree(qp->sq.swq); + bnxt_qplib_free_hwq(res, &qp->sq.hwq); +} + +void *bnxt_qplib_get_qp1_sq_buf(struct bnxt_qplib_qp *qp, + struct bnxt_qplib_sge *sge) +{ + struct bnxt_qplib_q *sq = &qp->sq; + struct bnxt_qplib_hdrbuf *buf; + u32 sw_prod; + + memset(sge, 0, sizeof(*sge)); + + buf = qp->sq_hdr_buf; + if (buf) { + sw_prod = sq->swq_start; + sge->addr = (dma_addr_t)(buf->dma_map + sw_prod * buf->step); + sge->lkey = 0xFFFFFFFF; + sge->size = buf->step; + return buf->va + sw_prod * sge->size; + } + return NULL; +} + +u32 bnxt_qplib_get_rq_prod_index(struct bnxt_qplib_qp *qp) +{ + struct bnxt_qplib_q *rq = &qp->rq; + + return rq->swq_start; +} + +void *bnxt_qplib_get_qp1_rq_buf(struct bnxt_qplib_qp *qp, + struct bnxt_qplib_sge *sge) +{ + struct bnxt_qplib_q *rq = &qp->rq; + struct bnxt_qplib_hdrbuf *buf; + u32 sw_prod; + + memset(sge, 0, sizeof(*sge)); + + buf = qp->rq_hdr_buf; + if (buf) { + sw_prod = rq->swq_start; + sge->addr = (dma_addr_t)(buf->dma_map + sw_prod * buf->step); + sge->lkey = 0xFFFFFFFF; + sge->size = buf->step; + return buf->va + sw_prod * sge->size; + } + return NULL; +} + +/* Fil the MSN table into the next psn row */ +static void bnxt_qplib_fill_msn_search(struct bnxt_qplib_qp *qp, + struct bnxt_qplib_swqe *wqe, + struct bnxt_qplib_swq *swq) +{ + struct sq_msn_search *msns; + u32 start_psn, next_psn; + u16 start_idx; + + msns = (struct sq_msn_search *)swq->psn_search; + msns->start_idx_next_psn_start_psn = 0; + + start_psn = swq->start_psn; + next_psn = swq->next_psn; + start_idx = swq->slot_idx; + msns->start_idx_next_psn_start_psn |= + bnxt_re_update_msn_tbl(start_idx, next_psn, start_psn); + pr_debug("QP_LIB MSN %d START_IDX %u NEXT_PSN %u START_PSN %u\n", + qp->msn, + (u16) + cpu_to_le16(BNXT_RE_MSN_IDX(msns->start_idx_next_psn_start_psn)), + (u32) + cpu_to_le32(BNXT_RE_MSN_NPSN(msns->start_idx_next_psn_start_psn)), + (u32) + cpu_to_le32(BNXT_RE_MSN_SPSN(msns->start_idx_next_psn_start_psn))); + qp->msn++; + qp->msn %= qp->msn_tbl_sz; +} + +static void bnxt_qplib_fill_psn_search(struct bnxt_qplib_qp *qp, + struct bnxt_qplib_swqe *wqe, + struct bnxt_qplib_swq *swq) +{ + struct sq_psn_search_ext *psns_ext; + struct sq_psn_search *psns; + u32 flg_npsn; + u32 op_spsn; + + if (!swq->psn_search) + return; + + /* Handle MSN differently on cap flags */ + if (BNXT_RE_HW_RETX(qp->dev_cap_flags)) { + bnxt_qplib_fill_msn_search(qp, wqe, swq); + return; + } + psns = (struct sq_psn_search *)swq->psn_search; + psns_ext = (struct sq_psn_search_ext *)swq->psn_search; + + op_spsn = ((swq->start_psn << SQ_PSN_SEARCH_START_PSN_SFT) & + SQ_PSN_SEARCH_START_PSN_MASK); + op_spsn |= ((wqe->type << SQ_PSN_SEARCH_OPCODE_SFT) & + SQ_PSN_SEARCH_OPCODE_MASK); + flg_npsn = ((swq->next_psn << SQ_PSN_SEARCH_NEXT_PSN_SFT) & + SQ_PSN_SEARCH_NEXT_PSN_MASK); + + if (_is_chip_gen_p5_p7(qp->cctx)) { + psns_ext->opcode_start_psn = cpu_to_le32(op_spsn); + psns_ext->flags_next_psn = cpu_to_le32(flg_npsn); + psns_ext->start_slot_idx = cpu_to_le16(swq->slot_idx); + } else { + psns->opcode_start_psn = cpu_to_le32(op_spsn); + psns->flags_next_psn = cpu_to_le32(flg_npsn); + } +} + +static u16 _calc_ilsize(struct bnxt_qplib_swqe *wqe) +{ + u16 size = 0; + int indx; + + for (indx = 0; indx < wqe->num_sge; indx++) + size += wqe->sg_list[indx].size; + return size; +} + +static int bnxt_qplib_put_inline(struct bnxt_qplib_qp *qp, + struct bnxt_qplib_swqe *wqe, + u32 *sw_prod) +{ + struct bnxt_qplib_hwq *sq_hwq; + int len, t_len, offt = 0; + int t_cplen = 0, cplen; + bool pull_dst = true; + void *il_dst = NULL; + void *il_src = NULL; + int indx; + + sq_hwq = &qp->sq.hwq; + t_len = 0; + for (indx = 0; indx < wqe->num_sge; indx++) { + len = wqe->sg_list[indx].size; + il_src = (void *)wqe->sg_list[indx].addr; + t_len += len; + if (t_len > qp->max_inline_data) + goto bad; + while (len) { + if (pull_dst) { + pull_dst = false; + il_dst = bnxt_qplib_get_qe(sq_hwq, ((*sw_prod) % + sq_hwq->depth), NULL); + (*sw_prod)++; + t_cplen = 0; + offt = 0; + } + cplen = min_t(int, len, sizeof(struct sq_sge)); + cplen = min_t(int, cplen, + (sizeof(struct sq_sge) - offt)); + memcpy(il_dst, il_src, cplen); + t_cplen += cplen; + il_src += cplen; + il_dst += cplen; + offt += cplen; + len -= cplen; + if (t_cplen == sizeof(struct sq_sge)) + pull_dst = true; + } + } + + return t_len; +bad: + return -ENOMEM; +} + +static int bnxt_qplib_put_sges(struct bnxt_qplib_hwq *sq_hwq, + struct bnxt_qplib_sge *ssge, + u32 nsge, u32 *sw_prod) +{ + struct sq_sge *dsge; + int indx, len = 0; + + for (indx = 0; indx < nsge; indx++, (*sw_prod)++) { + dsge = bnxt_qplib_get_qe(sq_hwq, ((*sw_prod) % sq_hwq->depth), NULL); + dsge->va_or_pa = cpu_to_le64(ssge[indx].addr); + dsge->l_key = cpu_to_le32(ssge[indx].lkey); + dsge->size = cpu_to_le32(ssge[indx].size); + len += ssge[indx].size; + } + return len; +} + +static u16 _calculate_wqe_byte(struct bnxt_qplib_qp *qp, + struct bnxt_qplib_swqe *wqe, u16 *wqe_byte) +{ + u16 wqe_size; + u32 ilsize; + u16 nsge; + + nsge = wqe->num_sge; + if (wqe->flags & BNXT_QPLIB_SWQE_FLAGS_INLINE) { + ilsize = _calc_ilsize(wqe); + wqe_size = (ilsize > qp->max_inline_data) ? + qp->max_inline_data : ilsize; + wqe_size = ALIGN(wqe_size, sizeof(struct sq_sge)); + } else { + wqe_size = nsge * sizeof(struct sq_sge); + } + /* Adding sq_send_hdr is a misnomer, for rq also hdr size is same. */ + wqe_size += sizeof(struct sq_send_hdr); + if (wqe_byte) + *wqe_byte = wqe_size; + return wqe_size / sizeof(struct sq_sge); +} + +static u16 _translate_q_full_delta(struct bnxt_qplib_q *que, u16 wqe_bytes) +{ + /* For Cu/Wh delta = 128, stride = 16, wqe_bytes = 128 + * For Gen-p5 B/C mode delta = 0, stride = 16, wqe_bytes = 128. + * For Gen-p5 delta = 0, stride = 16, 32 <= wqe_bytes <= 512. + * when 8916 is disabled. + */ + return (que->q_full_delta * wqe_bytes) / que->hwq.element_size; +} + +static void bnxt_qplib_pull_psn_buff(struct bnxt_qplib_qp *qp, struct bnxt_qplib_q *sq, + struct bnxt_qplib_swq *swq, bool hw_retx) +{ + struct bnxt_qplib_hwq *sq_hwq; + u32 pg_num, pg_indx; + void *buff; + u32 tail; + + sq_hwq = &sq->hwq; + if (!sq_hwq->pad_pg) + return; + + tail = swq->slot_idx / sq->dbinfo.max_slot; + if (hw_retx) + tail %= qp->msn_tbl_sz; + pg_num = (tail + sq_hwq->pad_pgofft) / (PAGE_SIZE / sq_hwq->pad_stride); + pg_indx = (tail + sq_hwq->pad_pgofft) % (PAGE_SIZE / sq_hwq->pad_stride); + buff = (void *)(sq_hwq->pad_pg[pg_num] + pg_indx * sq_hwq->pad_stride); + /* the start ptr for buff is same ie after the SQ */ + swq->psn_search = buff; +} + +void bnxt_qplib_post_send_db(struct bnxt_qplib_qp *qp) +{ + struct bnxt_qplib_q *sq = &qp->sq; + + bnxt_qplib_ring_prod_db(&sq->dbinfo, DBC_DBC_TYPE_SQ); +} + +int bnxt_qplib_post_send(struct bnxt_qplib_qp *qp, + struct bnxt_qplib_swqe *wqe) +{ + struct bnxt_qplib_nq_work *nq_work = NULL; + int i, rc = 0, data_len = 0, pkt_num = 0; + struct bnxt_qplib_q *sq = &qp->sq; + struct bnxt_qplib_hwq *sq_hwq; + struct bnxt_qplib_swq *swq; + bool sch_handler = false; + u16 slots_needed; + void *base_hdr; + void *ext_hdr; + __le32 temp32; + u16 qfd_slots; + u8 wqe_slots; + u16 wqe_size; + u32 sw_prod; + u32 wqe_idx; + + sq_hwq = &sq->hwq; + if (qp->state != CMDQ_MODIFY_QP_NEW_STATE_RTS && + qp->state != CMDQ_MODIFY_QP_NEW_STATE_ERR) { + dev_err(&sq_hwq->pdev->dev, + "QPLIB: FP: QP (0x%x) is in the 0x%x state\n", + qp->id, qp->state); + rc = -EINVAL; + goto done; + } + + wqe_slots = _calculate_wqe_byte(qp, wqe, &wqe_size); + slots_needed = (qp->wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC) ? + sq->dbinfo.max_slot : wqe_slots; + qfd_slots = _translate_q_full_delta(sq, wqe_size); + if (bnxt_qplib_queue_full(sq_hwq, (slots_needed + qfd_slots))) { + dev_err(&sq_hwq->pdev->dev, + "QPLIB: FP: QP (0x%x) SQ is full!\n", qp->id); + dev_err(&sq_hwq->pdev->dev, + "QPLIB: prod = %#x cons = %#x qdepth = %#x delta = %#x slots = %#x\n", + HWQ_CMP(sq_hwq->prod, sq_hwq), + HWQ_CMP(sq_hwq->cons, sq_hwq), + sq_hwq->max_elements, qfd_slots, slots_needed); + dev_err(&sq_hwq->pdev->dev, + "QPLIB: phantom_wqe_cnt: %d phantom_cqe_cnt: %d\n", + sq->phantom_wqe_cnt, sq->phantom_cqe_cnt); + rc = -ENOMEM; + goto done; + } + + sw_prod = sq_hwq->prod; + swq = bnxt_qplib_get_swqe(sq, &wqe_idx); + swq->slot_idx = sw_prod; + bnxt_qplib_pull_psn_buff(qp, sq, swq, BNXT_RE_HW_RETX(qp->dev_cap_flags)); + + swq->wr_id = wqe->wr_id; + swq->type = wqe->type; + swq->flags = wqe->flags; + swq->slots = slots_needed; + swq->start_psn = sq->psn & BTH_PSN_MASK; + if (qp->sig_type || wqe->flags & BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP) + swq->flags |= SQ_SEND_FLAGS_SIGNAL_COMP; + + dev_dbg(&sq_hwq->pdev->dev, + "QPLIB: FP: QP(0x%x) post SQ wr_id[%d] = 0x%llx\n", + qp->id, wqe_idx, swq->wr_id); + if (qp->cur_qp_state == CMDQ_MODIFY_QP_NEW_STATE_ERR) { + sch_handler = true; + dev_dbg(&sq_hwq->pdev->dev, + "%s Error QP. Scheduling for poll_cq\n", __func__); + goto queue_err; + } + + base_hdr = bnxt_qplib_get_qe(sq_hwq, sw_prod, NULL); + sw_prod++; + ext_hdr = bnxt_qplib_get_qe(sq_hwq, (sw_prod % sq_hwq->depth), NULL); + sw_prod++; + memset(base_hdr, 0, sizeof(struct sq_sge)); + memset(ext_hdr, 0, sizeof(struct sq_sge)); + + if (wqe->flags & BNXT_QPLIB_SWQE_FLAGS_INLINE) + data_len = bnxt_qplib_put_inline(qp, wqe, &sw_prod); + else + data_len = bnxt_qplib_put_sges(sq_hwq, wqe->sg_list, + wqe->num_sge, &sw_prod); + if (data_len < 0) + goto queue_err; + /* Specifics */ + switch (wqe->type) { + case BNXT_QPLIB_SWQE_TYPE_SEND: + if (qp->type == CMDQ_CREATE_QP_TYPE_RAW_ETHERTYPE || + qp->type == CMDQ_CREATE_QP1_TYPE_GSI) { + /* Assemble info for Raw Ethertype QPs */ + struct sq_send_raweth_qp1_hdr *sqe = base_hdr; + struct sq_raw_ext_hdr *ext_sqe = ext_hdr; + + sqe->wqe_type = wqe->type; + sqe->flags = wqe->flags; + sqe->wqe_size = wqe_slots; + sqe->cfa_action = cpu_to_le16(wqe->rawqp1.cfa_action); + sqe->lflags = cpu_to_le16(wqe->rawqp1.lflags); + sqe->length = cpu_to_le32(data_len); + ext_sqe->cfa_meta = cpu_to_le32((wqe->rawqp1.cfa_meta & + SQ_SEND_RAWETH_QP1_CFA_META_VLAN_VID_MASK) << + SQ_SEND_RAWETH_QP1_CFA_META_VLAN_VID_SFT); + + dev_dbg(&sq_hwq->pdev->dev, + "QPLIB: FP: RAW/QP1 Send WQE:\n" + "\twqe_type = 0x%x\n" + "\tflags = 0x%x\n" + "\twqe_size = 0x%x\n" + "\tlflags = 0x%x\n" + "\tcfa_action = 0x%x\n" + "\tlength = 0x%x\n" + "\tcfa_meta = 0x%x\n", + sqe->wqe_type, sqe->flags, sqe->wqe_size, + sqe->lflags, sqe->cfa_action, + sqe->length, ext_sqe->cfa_meta); + break; + } + fallthrough; + case BNXT_QPLIB_SWQE_TYPE_SEND_WITH_IMM: + fallthrough; + case BNXT_QPLIB_SWQE_TYPE_SEND_WITH_INV: + { + struct sq_send_hdr *sqe = base_hdr; + struct sq_ud_ext_hdr *ext_sqe = ext_hdr; + + sqe->wqe_type = wqe->type; + sqe->flags = wqe->flags; + sqe->wqe_size = wqe_slots; + sqe->inv_key_or_imm_data = cpu_to_le32(wqe->send.inv_key); + if (qp->type == CMDQ_CREATE_QP_TYPE_UD || + qp->type == CMDQ_CREATE_QP_TYPE_GSI) { + sqe->q_key = cpu_to_le32(wqe->send.q_key); + sqe->length = cpu_to_le32(data_len); + ext_sqe->dst_qp = cpu_to_le32( + wqe->send.dst_qp & SQ_SEND_DST_QP_MASK); + ext_sqe->avid = cpu_to_le32(wqe->send.avid & + SQ_SEND_AVID_MASK); + sq->psn = (sq->psn + 1) & BTH_PSN_MASK; + } else { + sqe->length = cpu_to_le32(data_len); + if (qp->mtu) + pkt_num = (data_len + qp->mtu - 1) / qp->mtu; + if (!pkt_num) + pkt_num = 1; + sq->psn = (sq->psn + pkt_num) & BTH_PSN_MASK; + } + dev_dbg(&sq_hwq->pdev->dev, + "QPLIB: FP: Send WQE:\n" + "\twqe_type = 0x%x\n" + "\tflags = 0x%x\n" + "\twqe_size = 0x%x\n" + "\tinv_key/immdata = 0x%x\n" + "\tq_key = 0x%x\n" + "\tdst_qp = 0x%x\n" + "\tlength = 0x%x\n" + "\tavid = 0x%x\n", + sqe->wqe_type, sqe->flags, sqe->wqe_size, + sqe->inv_key_or_imm_data, sqe->q_key, ext_sqe->dst_qp, + sqe->length, ext_sqe->avid); + break; + } + case BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE: + /* fall-thru */ + case BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE_WITH_IMM: + /* fall-thru */ + case BNXT_QPLIB_SWQE_TYPE_RDMA_READ: + { + struct sq_rdma_hdr *sqe = base_hdr; + struct sq_rdma_ext_hdr *ext_sqe = ext_hdr; + + sqe->wqe_type = wqe->type; + sqe->flags = wqe->flags; + sqe->wqe_size = wqe_slots; + sqe->imm_data = cpu_to_le32(wqe->rdma.inv_key); + sqe->length = cpu_to_le32((u32)data_len); + ext_sqe->remote_va = cpu_to_le64(wqe->rdma.remote_va); + ext_sqe->remote_key = cpu_to_le32(wqe->rdma.r_key); + if (qp->mtu) + pkt_num = (data_len + qp->mtu - 1) / qp->mtu; + if (!pkt_num) + pkt_num = 1; + sq->psn = (sq->psn + pkt_num) & BTH_PSN_MASK; + + dev_dbg(&sq_hwq->pdev->dev, + "QPLIB: FP: RDMA WQE:\n" + "\twqe_type = 0x%x\n" + "\tflags = 0x%x\n" + "\twqe_size = 0x%x\n" + "\timmdata = 0x%x\n" + "\tlength = 0x%x\n" + "\tremote_va = 0x%llx\n" + "\tremote_key = 0x%x\n", + sqe->wqe_type, sqe->flags, sqe->wqe_size, + sqe->imm_data, sqe->length, ext_sqe->remote_va, + ext_sqe->remote_key); + break; + } + case BNXT_QPLIB_SWQE_TYPE_ATOMIC_CMP_AND_SWP: + /* fall-thru */ + case BNXT_QPLIB_SWQE_TYPE_ATOMIC_FETCH_AND_ADD: + { + struct sq_atomic_hdr *sqe = base_hdr; + struct sq_atomic_ext_hdr *ext_sqe = ext_hdr; + + sqe->wqe_type = wqe->type; + sqe->flags = wqe->flags; + sqe->remote_key = cpu_to_le32(wqe->atomic.r_key); + sqe->remote_va = cpu_to_le64(wqe->atomic.remote_va); + ext_sqe->swap_data = cpu_to_le64(wqe->atomic.swap_data); + ext_sqe->cmp_data = cpu_to_le64(wqe->atomic.cmp_data); + if (qp->mtu) + pkt_num = (data_len + qp->mtu - 1) / qp->mtu; + if (!pkt_num) + pkt_num = 1; + sq->psn = (sq->psn + pkt_num) & BTH_PSN_MASK; + break; + } + case BNXT_QPLIB_SWQE_TYPE_LOCAL_INV: + { + struct sq_localinvalidate_hdr *sqe = base_hdr; + + sqe->wqe_type = wqe->type; + sqe->flags = wqe->flags; + sqe->inv_l_key = cpu_to_le32(wqe->local_inv.inv_l_key); + + dev_dbg(&sq_hwq->pdev->dev, + "QPLIB: FP: LOCAL INV WQE:\n" + "\twqe_type = 0x%x\n" + "\tflags = 0x%x\n" + "\tinv_l_key = 0x%x\n", + sqe->wqe_type, sqe->flags, sqe->inv_l_key); + break; + } + case BNXT_QPLIB_SWQE_TYPE_FAST_REG_MR: + { + struct sq_fr_pmr_hdr *sqe = base_hdr; + struct sq_fr_pmr_ext_hdr *ext_sqe = ext_hdr; + + sqe->wqe_type = wqe->type; + sqe->flags = wqe->flags; + sqe->access_cntl = wqe->frmr.access_cntl | + SQ_FR_PMR_ACCESS_CNTL_LOCAL_WRITE; + sqe->zero_based_page_size_log = + (wqe->frmr.pg_sz_log & SQ_FR_PMR_PAGE_SIZE_LOG_MASK) << + SQ_FR_PMR_PAGE_SIZE_LOG_SFT | + (wqe->frmr.zero_based == true ? SQ_FR_PMR_ZERO_BASED : 0); + sqe->l_key = cpu_to_le32(wqe->frmr.l_key); + /* TODO: OFED only provides length of MR up to 32-bits for FRMR */ + temp32 = cpu_to_le32(wqe->frmr.length); + memcpy(sqe->length, &temp32, sizeof(wqe->frmr.length)); + sqe->numlevels_pbl_page_size_log = + ((wqe->frmr.pbl_pg_sz_log << + SQ_FR_PMR_PBL_PAGE_SIZE_LOG_SFT) & + SQ_FR_PMR_PBL_PAGE_SIZE_LOG_MASK) | + ((wqe->frmr.levels << SQ_FR_PMR_NUMLEVELS_SFT) & + SQ_FR_PMR_NUMLEVELS_MASK); + if (!wqe->frmr.levels && !wqe->frmr.pbl_ptr) { + ext_sqe->pblptr = cpu_to_le64(wqe->frmr.page_list[0]); + } else { + for (i = 0; i < wqe->frmr.page_list_len; i++) + wqe->frmr.pbl_ptr[i] = cpu_to_le64( + wqe->frmr.page_list[i] | + PTU_PTE_VALID); + ext_sqe->pblptr = cpu_to_le64(wqe->frmr.pbl_dma_ptr); + } + ext_sqe->va = cpu_to_le64(wqe->frmr.va); + dev_dbg(&sq_hwq->pdev->dev, + "QPLIB: FP: FRMR WQE:\n" + "\twqe_type = 0x%x\n" + "\tflags = 0x%x\n" + "\taccess_cntl = 0x%x\n" + "\tzero_based_page_size_log = 0x%x\n" + "\tl_key = 0x%x\n" + "\tlength = 0x%x\n" + "\tnumlevels_pbl_page_size_log = 0x%x\n" + "\tpblptr = 0x%llx\n" + "\tva = 0x%llx\n", + sqe->wqe_type, sqe->flags, sqe->access_cntl, + sqe->zero_based_page_size_log, sqe->l_key, + *(u32 *)sqe->length, sqe->numlevels_pbl_page_size_log, + ext_sqe->pblptr, ext_sqe->va); + break; + } + case BNXT_QPLIB_SWQE_TYPE_BIND_MW: + { + struct sq_bind_hdr *sqe = base_hdr; + struct sq_bind_ext_hdr *ext_sqe = ext_hdr; + + sqe->wqe_type = wqe->type; + sqe->flags = wqe->flags; + sqe->access_cntl = wqe->bind.access_cntl; + sqe->mw_type_zero_based = wqe->bind.mw_type | + (wqe->bind.zero_based == true ? SQ_BIND_ZERO_BASED : 0); + sqe->parent_l_key = cpu_to_le32(wqe->bind.parent_l_key); + sqe->l_key = cpu_to_le32(wqe->bind.r_key); + ext_sqe->va = cpu_to_le64(wqe->bind.va); + ext_sqe->length_lo = cpu_to_le32(wqe->bind.length); + dev_dbg(&sq_hwq->pdev->dev, + "QPLIB: FP: BIND WQE:\n" + "\twqe_type = 0x%x\n" + "\tflags = 0x%x\n" + "\taccess_cntl = 0x%x\n" + "\tmw_type_zero_based = 0x%x\n" + "\tparent_l_key = 0x%x\n" + "\tl_key = 0x%x\n" + "\tva = 0x%llx\n" + "\tlength = 0x%x\n", + sqe->wqe_type, sqe->flags, sqe->access_cntl, + sqe->mw_type_zero_based, sqe->parent_l_key, + sqe->l_key, sqe->va, ext_sqe->length_lo); + break; + } + default: + /* Bad wqe, return error */ + rc = -EINVAL; + goto done; + } + swq->next_psn = sq->psn & BTH_PSN_MASK; + bnxt_qplib_fill_psn_search(qp, wqe, swq); + +queue_err: + bnxt_qplib_swq_mod_start(sq, wqe_idx); + bnxt_qplib_hwq_incr_prod(&sq->dbinfo, sq_hwq, swq->slots); + qp->wqe_cnt++; +done: + if (sch_handler) { + nq_work = kzalloc(sizeof(*nq_work), GFP_ATOMIC); + if (nq_work) { + nq_work->cq = qp->scq; + nq_work->nq = qp->scq->nq; + INIT_WORK(&nq_work->work, bnxt_qpn_cqn_sched_task); + queue_work(qp->scq->nq->cqn_wq, &nq_work->work); + } else { + dev_err(&sq->hwq.pdev->dev, + "QPLIB: FP: Failed to allocate SQ nq_work!\n"); + rc = -ENOMEM; + } + } + return rc; +} + +void bnxt_qplib_post_recv_db(struct bnxt_qplib_qp *qp) +{ + struct bnxt_qplib_q *rq = &qp->rq; + + bnxt_qplib_ring_prod_db(&rq->dbinfo, DBC_DBC_TYPE_RQ); +} + +void bnxt_re_handle_cqn(struct bnxt_qplib_cq *cq) +{ + struct bnxt_qplib_nq *nq; + + if (!(cq && cq->nq)) + return; + + nq = cq->nq; + spin_lock_bh(&cq->compl_lock); + if (nq->cqn_handler) { + dev_dbg(&nq->res->pdev->dev, + "%s:Trigger cq = %p event nq = %p\n", + __func__, cq, nq); + nq->cqn_handler(nq, cq); + } + spin_unlock_bh(&cq->compl_lock); +} + +int bnxt_qplib_post_recv(struct bnxt_qplib_qp *qp, + struct bnxt_qplib_swqe *wqe) +{ + struct bnxt_qplib_nq_work *nq_work = NULL; + struct bnxt_qplib_q *rq = &qp->rq; + struct bnxt_qplib_hwq *rq_hwq; + struct bnxt_qplib_swq *swq; + bool sch_handler = false; + struct rq_wqe_hdr *base_hdr; + struct rq_ext_hdr *ext_hdr; + struct sq_sge *dsge; + u8 wqe_slots; + u32 wqe_idx; + u32 sw_prod; + int rc = 0; + + rq_hwq = &rq->hwq; + if (qp->state == CMDQ_MODIFY_QP_NEW_STATE_RESET) { + dev_err(&rq_hwq->pdev->dev, + "QPLIB: FP: QP (0x%x) is in the 0x%x state\n", + qp->id, qp->state); + rc = -EINVAL; + goto done; + } + + wqe_slots = _calculate_wqe_byte(qp, wqe, NULL); + if (bnxt_qplib_queue_full(rq_hwq, rq->dbinfo.max_slot)) { + dev_err(&rq_hwq->pdev->dev, + "QPLIB: FP: QP (0x%x) RQ is full!\n", qp->id); + rc = -EINVAL; + goto done; + } + + swq = bnxt_qplib_get_swqe(rq, &wqe_idx); + swq->wr_id = wqe->wr_id; + swq->slots = rq->dbinfo.max_slot; + dev_dbg(&rq_hwq->pdev->dev, + "QPLIB: FP: post RQ wr_id[%d] = 0x%llx\n", + wqe_idx, swq->wr_id); + if (qp->cur_qp_state == CMDQ_MODIFY_QP_NEW_STATE_ERR) { + sch_handler = true; + dev_dbg(&rq_hwq->pdev->dev, "%s Error QP. Sched a flushed cmpl\n", + __func__); + goto queue_err; + } + + sw_prod = rq_hwq->prod; + base_hdr = bnxt_qplib_get_qe(rq_hwq, sw_prod, NULL); + sw_prod++; + ext_hdr = bnxt_qplib_get_qe(rq_hwq, (sw_prod % rq_hwq->depth), NULL); + sw_prod++; + memset(base_hdr, 0, sizeof(struct sq_sge)); + memset(ext_hdr, 0, sizeof(struct sq_sge)); + + if (!wqe->num_sge) { + dsge = bnxt_qplib_get_qe(rq_hwq, (sw_prod % rq_hwq->depth), NULL); + dsge->size = 0; + wqe_slots++; + } else { + bnxt_qplib_put_sges(rq_hwq, wqe->sg_list, wqe->num_sge, &sw_prod); + } + base_hdr->wqe_type = wqe->type; + base_hdr->flags = wqe->flags; + base_hdr->wqe_size = wqe_slots; + base_hdr->wr_id |= cpu_to_le32(wqe_idx); +queue_err: + bnxt_qplib_swq_mod_start(rq, wqe_idx); + bnxt_qplib_hwq_incr_prod(&rq->dbinfo, &rq->hwq, swq->slots); +done: + if (sch_handler) { + nq_work = kzalloc(sizeof(*nq_work), GFP_ATOMIC); + if (nq_work) { + nq_work->cq = qp->rcq; + nq_work->nq = qp->rcq->nq; + INIT_WORK(&nq_work->work, bnxt_qpn_cqn_sched_task); + queue_work(qp->rcq->nq->cqn_wq, &nq_work->work); + } else { + dev_err(&rq->hwq.pdev->dev, + "QPLIB: FP: Failed to allocate RQ nq_work!\n"); + rc = -ENOMEM; + } + } + return rc; +} + +/* CQ */ +int bnxt_qplib_create_cq(struct bnxt_qplib_res *res, struct bnxt_qplib_cq *cq) +{ + struct bnxt_qplib_hwq_attr hwq_attr = {}; + struct bnxt_qplib_rcfw *rcfw = res->rcfw; + struct creq_create_cq_resp resp = {}; + struct bnxt_qplib_cmdqmsg msg = {}; + struct cmdq_create_cq req = {}; + struct bnxt_qplib_reftbl *tbl; + unsigned long flag; + u32 pg_sz_lvl = 0; + int rc; + + hwq_attr.res = res; + hwq_attr.depth = cq->max_wqe; + hwq_attr.stride = sizeof(struct cq_base); + hwq_attr.type = HWQ_TYPE_QUEUE; + hwq_attr.sginfo = &cq->sginfo; + rc = bnxt_qplib_alloc_init_hwq(&cq->hwq, &hwq_attr); + if (rc) + goto exit; + + bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_CREATE_CQ, + sizeof(req)); + + if (!cq->dpi) { + dev_err(&rcfw->pdev->dev, + "QPLIB: FP: CREATE_CQ failed due to NULL DPI\n"); + return -EINVAL; + } + req.dpi = cpu_to_le32(cq->dpi->dpi); + req.cq_handle = cpu_to_le64(cq->cq_handle); + + req.cq_size = cpu_to_le32(cq->max_wqe); + req.pbl = cpu_to_le64(_get_base_addr(&cq->hwq)); + pg_sz_lvl = _get_base_pg_size(&cq->hwq) << CMDQ_CREATE_CQ_PG_SIZE_SFT; + pg_sz_lvl |= ((cq->hwq.level & CMDQ_CREATE_CQ_LVL_MASK) << + CMDQ_CREATE_CQ_LVL_SFT); + req.pg_size_lvl = cpu_to_le32(pg_sz_lvl); + + req.cq_fco_cnq_id = cpu_to_le32( + (cq->cnq_hw_ring_id & CMDQ_CREATE_CQ_CNQ_ID_MASK) << + CMDQ_CREATE_CQ_CNQ_ID_SFT); + bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL, sizeof(req), + sizeof(resp), 0); + rc = bnxt_qplib_rcfw_send_message(rcfw, &msg); + if (rc) + goto fail; + cq->id = le32_to_cpu(resp.xid); + cq->period = BNXT_QPLIB_QUEUE_START_PERIOD; + init_waitqueue_head(&cq->waitq); + INIT_LIST_HEAD(&cq->sqf_head); + INIT_LIST_HEAD(&cq->rqf_head); + spin_lock_init(&cq->flush_lock); + spin_lock_init(&cq->compl_lock); + + /* init dbinfo */ + cq->cctx = res->cctx; + cq->dbinfo.hwq = &cq->hwq; + cq->dbinfo.xid = cq->id; + cq->dbinfo.db = cq->dpi->dbr; + cq->dbinfo.priv_db = res->dpi_tbl.priv_db; + cq->dbinfo.flags = 0; + cq->dbinfo.toggle = 0; + cq->dbinfo.res = res; + cq->dbinfo.seed = cq->id; + spin_lock_init(&cq->dbinfo.lock); + cq->dbinfo.shadow_key = BNXT_QPLIB_DBR_KEY_INVALID; + cq->dbinfo.shadow_key_arm_ena = BNXT_QPLIB_DBR_KEY_INVALID; + + tbl = &res->reftbl.cqref; + spin_lock_irqsave(&tbl->lock, flag); + tbl->rec[GET_TBL_INDEX(cq->id, tbl)].xid = cq->id; + tbl->rec[GET_TBL_INDEX(cq->id, tbl)].handle = cq; + spin_unlock_irqrestore(&tbl->lock, flag); + + bnxt_qplib_armen_db(&cq->dbinfo, DBC_DBC_TYPE_CQ_ARMENA); + return 0; + +fail: + bnxt_qplib_free_hwq(res, &cq->hwq); +exit: + return rc; +} + +int bnxt_qplib_modify_cq(struct bnxt_qplib_res *res, struct bnxt_qplib_cq *cq) +{ + /* TODO: Modify CQ threshold are passed to the HW via DBR */ + return 0; +} + +void bnxt_qplib_resize_cq_complete(struct bnxt_qplib_res *res, + struct bnxt_qplib_cq *cq) +{ + bnxt_qplib_free_hwq(res, &cq->hwq); + memcpy(&cq->hwq, &cq->resize_hwq, sizeof(cq->hwq)); + /* Reset only the cons bit in the flags */ + cq->dbinfo.flags &= ~(1UL << BNXT_QPLIB_FLAG_EPOCH_CONS_SHIFT); + + /* Tell HW to switch over to the new CQ */ + if (!cq->resize_hwq.is_user) + bnxt_qplib_cq_coffack_db(&cq->dbinfo); +} + +int bnxt_qplib_resize_cq(struct bnxt_qplib_res *res, struct bnxt_qplib_cq *cq, + int new_cqes) +{ + struct bnxt_qplib_hwq_attr hwq_attr = {}; + struct bnxt_qplib_rcfw *rcfw = res->rcfw; + struct creq_resize_cq_resp resp = {}; + struct bnxt_qplib_cmdqmsg msg = {}; + struct cmdq_resize_cq req = {}; + u32 pgsz = 0, lvl = 0, nsz = 0; + struct bnxt_qplib_pbl *pbl; + u16 count = -1; + int rc; + + bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_RESIZE_CQ, + sizeof(req)); + + hwq_attr.sginfo = &cq->sginfo; + hwq_attr.res = res; + hwq_attr.depth = new_cqes; + hwq_attr.stride = sizeof(struct cq_base); + hwq_attr.type = HWQ_TYPE_QUEUE; + rc = bnxt_qplib_alloc_init_hwq(&cq->resize_hwq, &hwq_attr); + if (rc) + return rc; + + dev_dbg(&rcfw->pdev->dev, "QPLIB: FP: %s: pbl_lvl: %d\n", __func__, + cq->resize_hwq.level); + req.cq_cid = cpu_to_le32(cq->id); + pbl = &cq->resize_hwq.pbl[PBL_LVL_0]; + pgsz = ((pbl->pg_size == ROCE_PG_SIZE_4K ? CMDQ_RESIZE_CQ_PG_SIZE_PG_4K : + pbl->pg_size == ROCE_PG_SIZE_8K ? CMDQ_RESIZE_CQ_PG_SIZE_PG_8K : + pbl->pg_size == ROCE_PG_SIZE_64K ? CMDQ_RESIZE_CQ_PG_SIZE_PG_64K : + pbl->pg_size == ROCE_PG_SIZE_2M ? CMDQ_RESIZE_CQ_PG_SIZE_PG_2M : + pbl->pg_size == ROCE_PG_SIZE_8M ? CMDQ_RESIZE_CQ_PG_SIZE_PG_8M : + pbl->pg_size == ROCE_PG_SIZE_1G ? CMDQ_RESIZE_CQ_PG_SIZE_PG_1G : + CMDQ_RESIZE_CQ_PG_SIZE_PG_4K) & CMDQ_RESIZE_CQ_PG_SIZE_MASK); + lvl = (cq->resize_hwq.level << CMDQ_RESIZE_CQ_LVL_SFT) & + CMDQ_RESIZE_CQ_LVL_MASK; + nsz = (new_cqes << CMDQ_RESIZE_CQ_NEW_CQ_SIZE_SFT) & + CMDQ_RESIZE_CQ_NEW_CQ_SIZE_MASK; + req.new_cq_size_pg_size_lvl = cpu_to_le32(nsz|pgsz|lvl); + req.new_pbl = cpu_to_le64(pbl->pg_map_arr[0]); + + if (!cq->resize_hwq.is_user) + set_bit(CQ_FLAGS_RESIZE_IN_PROG, &cq->flags); + + bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL, sizeof(req), + sizeof(resp), 0); + rc = bnxt_qplib_rcfw_send_message(rcfw, &msg); + if (rc) + goto fail; + + if (!cq->resize_hwq.is_user) { +wait: + /* Wait here for the HW to switch the CQ over */ + if (wait_event_interruptible_timeout(cq->waitq, + !test_bit(CQ_FLAGS_RESIZE_IN_PROG, &cq->flags), + msecs_to_jiffies(CQ_RESIZE_WAIT_TIME_MS)) == + -ERESTARTSYS && count--) + goto wait; + + if (test_bit(CQ_FLAGS_RESIZE_IN_PROG, &cq->flags)) { + dev_err(&rcfw->pdev->dev, + "QPLIB: FP: RESIZE_CQ timed out\n"); + rc = -ETIMEDOUT; + goto fail; + } + + bnxt_qplib_resize_cq_complete(res, cq); + } + + return 0; +fail: + if (!cq->resize_hwq.is_user) { + bnxt_qplib_free_hwq(res, &cq->resize_hwq); + clear_bit(CQ_FLAGS_RESIZE_IN_PROG, &cq->flags); + } + return rc; +} + +void bnxt_qplib_free_cq(struct bnxt_qplib_res *res, struct bnxt_qplib_cq *cq) +{ + bnxt_qplib_free_hwq(res, &cq->hwq); +} + +static void bnxt_qplib_sync_cq(struct bnxt_qplib_cq *cq) +{ + struct bnxt_qplib_nq *nq = cq->nq; + /* Flush any pending work and synchronize irq */ + flush_workqueue(cq->nq->cqn_wq); + mutex_lock(&nq->lock); + if (nq->requested) + synchronize_irq(nq->msix_vec); + mutex_unlock(&nq->lock); +} + +int bnxt_qplib_destroy_cq(struct bnxt_qplib_res *res, struct bnxt_qplib_cq *cq) +{ + struct bnxt_qplib_rcfw *rcfw = res->rcfw; + struct creq_destroy_cq_resp resp = {}; + struct bnxt_qplib_cmdqmsg msg = {}; + struct cmdq_destroy_cq req = {}; + struct bnxt_qplib_reftbl *tbl; + u16 total_cnq_events; + unsigned long flag; + int rc; + + tbl = &res->reftbl.cqref; + spin_lock_irqsave(&tbl->lock, flag); + tbl->rec[GET_TBL_INDEX(cq->id, tbl)].handle = NULL; + tbl->rec[GET_TBL_INDEX(cq->id, tbl)].xid = 0; + spin_unlock_irqrestore(&tbl->lock, flag); + + bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_DESTROY_CQ, + sizeof(req)); + + req.cq_cid = cpu_to_le32(cq->id); + bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL, sizeof(req), + sizeof(resp), 0); + rc = bnxt_qplib_rcfw_send_message(rcfw, &msg); + if (rc) + return rc; + + total_cnq_events = le16_to_cpu(resp.total_cnq_events); + if (total_cnq_events >= 0) + dev_dbg(&rcfw->pdev->dev, + "%s: cq_id = 0x%x cq = 0x%p resp.total_cnq_events = 0x%x\n", + __func__, cq->id, cq, total_cnq_events); + __wait_for_all_nqes(cq, total_cnq_events); + bnxt_qplib_sync_cq(cq); + bnxt_qplib_free_hwq(res, &cq->hwq); + return 0; +} + +static int __flush_sq(struct bnxt_qplib_q *sq, struct bnxt_qplib_qp *qp, + struct bnxt_qplib_cqe **pcqe, int *budget) +{ + struct bnxt_qplib_cqe *cqe; + u32 start, last; + int rc = 0; + + /* Now complete all outstanding SQEs with FLUSHED_ERR */ + start = sq->swq_start; + cqe = *pcqe; + while (*budget) { + last = sq->swq_last; + if (start == last) { + break; + } + /* Skip the FENCE WQE completions */ + if (sq->swq[last].wr_id == BNXT_QPLIB_FENCE_WRID) { + bnxt_re_legacy_cancel_phantom_processing(qp); + goto skip_compl; + } + + memset(cqe, 0, sizeof(*cqe)); + cqe->status = CQ_REQ_STATUS_WORK_REQUEST_FLUSHED_ERR; + cqe->opcode = CQ_BASE_CQE_TYPE_REQ; + cqe->qp_handle = (u64)qp; + cqe->wr_id = sq->swq[last].wr_id; + cqe->src_qp = qp->id; + cqe->type = sq->swq[last].type; + dev_dbg(&sq->hwq.pdev->dev, + "QPLIB: FP: CQ Processed terminal Req \n"); + dev_dbg(&sq->hwq.pdev->dev, + "QPLIB: wr_id[%d] = 0x%llx with status 0x%x\n", + last, cqe->wr_id, cqe->status); + cqe++; + (*budget)--; +skip_compl: + bnxt_qplib_hwq_incr_cons(sq->hwq.depth, + &sq->hwq.cons, + sq->swq[last].slots, + &sq->dbinfo.flags); + sq->swq_last = sq->swq[last].next_idx; + } + *pcqe = cqe; + if (!*budget && sq->swq_last != start) + /* Out of budget */ + rc = -EAGAIN; + dev_dbg(&sq->hwq.pdev->dev, "QPLIB: FP: Flush SQ rc = 0x%x\n", rc); + + return rc; +} + +static int __flush_rq(struct bnxt_qplib_q *rq, struct bnxt_qplib_qp *qp, + struct bnxt_qplib_cqe **pcqe, int *budget) +{ + struct bnxt_qplib_cqe *cqe; + u32 start, last; + int opcode = 0; + int rc = 0; + + switch (qp->type) { + case CMDQ_CREATE_QP1_TYPE_GSI: + opcode = CQ_BASE_CQE_TYPE_RES_RAWETH_QP1; + break; + case CMDQ_CREATE_QP_TYPE_RC: + opcode = CQ_BASE_CQE_TYPE_RES_RC; + break; + case CMDQ_CREATE_QP_TYPE_UD: + opcode = CQ_BASE_CQE_TYPE_RES_UD; + break; + } + + /* Flush the rest of the RQ */ + start = rq->swq_start; + cqe = *pcqe; + while (*budget) { + last = rq->swq_last; + if (last == start) + break; + memset(cqe, 0, sizeof(*cqe)); + cqe->status = + CQ_RES_RC_STATUS_WORK_REQUEST_FLUSHED_ERR; + cqe->opcode = opcode; + cqe->qp_handle = (u64)qp; + cqe->wr_id = rq->swq[last].wr_id; + dev_dbg(&rq->hwq.pdev->dev, "QPLIB: FP: CQ Processed Res RC \n"); + dev_dbg(&rq->hwq.pdev->dev, + "QPLIB: rq[%d] = 0x%llx with status 0x%x\n", + last, cqe->wr_id, cqe->status); + cqe++; + (*budget)--; + bnxt_qplib_hwq_incr_cons(rq->hwq.depth, + &rq->hwq.cons, + rq->swq[last].slots, + &rq->dbinfo.flags); + rq->swq_last = rq->swq[last].next_idx; + } + *pcqe = cqe; + if (!*budget && rq->swq_last != start) + /* Out of budget */ + rc = -EAGAIN; + + dev_dbg(&rq->hwq.pdev->dev, "QPLIB: FP: Flush RQ rc = 0x%x\n", rc); + return rc; +} + +void bnxt_qplib_mark_qp_error(void *qp_handle) +{ + struct bnxt_qplib_qp *qp = qp_handle; + + if (!qp) + return; + + /* Must block new posting of SQ and RQ */ + qp->cur_qp_state = CMDQ_MODIFY_QP_NEW_STATE_ERR; + qp->state = qp->cur_qp_state; + + /* Add qp to flush list of the CQ */ + if (!qp->is_user) + bnxt_qplib_add_flush_qp(qp); +} + +/* Note: SQE is valid from sw_sq_cons up to cqe_sq_cons (exclusive) + * CQE is track from sw_cq_cons to max_element but valid only if VALID=1 + */ +static int bnxt_re_legacy_do_wa9060(struct bnxt_qplib_qp *qp, + struct bnxt_qplib_cq *cq, + u32 cq_cons, u32 swq_last, + u32 cqe_sq_cons) +{ + struct bnxt_qplib_q *sq = &qp->sq; + struct bnxt_qplib_swq *swq; + u32 peek_sw_cq_cons, peek_sq_cons_idx, peek_flags; + struct cq_terminal *peek_term_hwcqe; + struct cq_req *peek_req_hwcqe; + struct bnxt_qplib_qp *peek_qp; + struct bnxt_qplib_q *peek_sq; + struct cq_base *peek_hwcqe; + int i, rc = 0; + + /* Check for the psn_search marking before completing */ + swq = &sq->swq[swq_last]; + if (swq->psn_search && + le32_to_cpu(swq->psn_search->flags_next_psn) & 0x80000000) { + /* Unmark */ + swq->psn_search->flags_next_psn = cpu_to_le32 + (le32_to_cpu(swq->psn_search->flags_next_psn) + & ~0x80000000); + dev_dbg(&cq->hwq.pdev->dev, + "FP: Process Req cq_cons=0x%x qp=0x%x sq cons sw=0x%x cqe=0x%x marked!\n", + cq_cons, qp->id, swq_last, cqe_sq_cons); + sq->condition = true; + sq->legacy_send_phantom = true; + + /* TODO: Only ARM if the previous SQE is ARMALL */ + bnxt_qplib_ring_db(&cq->dbinfo, DBC_DBC_TYPE_CQ_ARMALL); + + rc = -EAGAIN; + goto out; + } + if (sq->condition == true) { + /* Peek at the completions */ + peek_flags = cq->dbinfo.flags; + peek_sw_cq_cons = cq_cons; + i = cq->hwq.depth; + while (i--) { + peek_hwcqe = bnxt_qplib_get_qe(&cq->hwq, + peek_sw_cq_cons, NULL); + /* If the next hwcqe is VALID */ + if (CQE_CMP_VALID(peek_hwcqe, peek_flags)) { + /* If the next hwcqe is a REQ */ + dma_rmb(); + switch (peek_hwcqe->cqe_type_toggle & + CQ_BASE_CQE_TYPE_MASK) { + case CQ_BASE_CQE_TYPE_REQ: + peek_req_hwcqe = (struct cq_req *) + peek_hwcqe; + peek_qp = (struct bnxt_qplib_qp *) + le64_to_cpu( + peek_req_hwcqe->qp_handle); + peek_sq = &peek_qp->sq; + peek_sq_cons_idx = + ((le16_to_cpu( + peek_req_hwcqe->sq_cons_idx) + - 1) % sq->max_wqe); + /* If the hwcqe's sq's wr_id matches */ + if (peek_sq == sq && + sq->swq[peek_sq_cons_idx].wr_id == + BNXT_QPLIB_FENCE_WRID) { + /* Unbreak only if the phantom + comes back */ + dev_dbg(&cq->hwq.pdev->dev, + "FP: Process Req qp=0x%x current sq cons sw=0x%x cqe=0x%x\n", + qp->id, swq_last, + cqe_sq_cons); + sq->condition = false; + sq->single = true; + sq->phantom_cqe_cnt++; + dev_dbg(&cq->hwq.pdev->dev, + "qp %#x condition restored at peek cq_cons=%#x sq_cons_idx %#x, phantom_cqe_cnt: %d unmark\n", + peek_qp->id, + peek_sw_cq_cons, + peek_sq_cons_idx, + sq->phantom_cqe_cnt); + rc = 0; + goto out; + } + break; + + case CQ_BASE_CQE_TYPE_TERMINAL: + /* In case the QP has gone into the + error state */ + peek_term_hwcqe = (struct cq_terminal *) + peek_hwcqe; + peek_qp = (struct bnxt_qplib_qp *) + le64_to_cpu( + peek_term_hwcqe->qp_handle); + if (peek_qp == qp) { + sq->condition = false; + rc = 0; + goto out; + } + break; + default: + break; + } + /* Valid but not the phantom, so keep looping */ + } else { + /* Not valid yet, just exit and wait */ + rc = -EINVAL; + goto out; + } + bnxt_qplib_hwq_incr_cons(cq->hwq.depth, + &peek_sw_cq_cons, + 1, &peek_flags); + } + dev_err(&cq->hwq.pdev->dev, + "Should not have come here! cq_cons=0x%x qp=0x%x sq cons sw=0x%x hw=0x%x\n", + cq_cons, qp->id, swq_last, cqe_sq_cons); + rc = -EINVAL; + } +out: + return rc; +} + +static int bnxt_qplib_cq_process_req(struct bnxt_qplib_cq *cq, + struct cq_req *hwcqe, + struct bnxt_qplib_cqe **pcqe, int *budget, + u32 cq_cons, struct bnxt_qplib_qp **lib_qp) +{ + struct bnxt_qplib_qp *qp; + struct bnxt_qplib_q *sq; + struct bnxt_qplib_cqe *cqe; + u32 cqe_sq_cons; + struct bnxt_qplib_swq *swq; + int rc = 0; + + qp = (struct bnxt_qplib_qp *)le64_to_cpu(hwcqe->qp_handle); + dev_dbg(&cq->hwq.pdev->dev, "FP: Process Req qp=0x%p\n", qp); + if (!qp) { + dev_err(&cq->hwq.pdev->dev, + "QPLIB: FP: Process Req qp is NULL\n"); + return -EINVAL; + } + sq = &qp->sq; + + cqe_sq_cons = le16_to_cpu(hwcqe->sq_cons_idx) % sq->max_wqe; + if (qp->sq.flushed) { + dev_dbg(&cq->hwq.pdev->dev, + "%s: QPLIB: QP in Flush QP = %p\n", __func__, qp); + goto done; + } + + /* Require to walk the sq's swq to fabricate CQEs for all previously + * signaled SWQEs due to CQE aggregation from the current sq cons + * to the cqe_sq_cons + */ + cqe = *pcqe; + while (*budget) { + if (sq->swq_last == cqe_sq_cons) + /* Done */ + break; + + swq = &sq->swq[sq->swq_last]; + memset(cqe, 0, sizeof(*cqe)); + cqe->opcode = CQ_BASE_CQE_TYPE_REQ; + cqe->qp_handle = (u64)qp; + cqe->src_qp = qp->id; + cqe->wr_id = swq->wr_id; + + if (cqe->wr_id == BNXT_QPLIB_FENCE_WRID) + goto skip; + + cqe->type = swq->type; + + /* For the last CQE, check for status. For errors, regardless + * of the request being signaled or not, it must complete with + * the hwcqe error status + */ + if (swq->next_idx == cqe_sq_cons && + hwcqe->status != CQ_REQ_STATUS_OK) { + cqe->status = hwcqe->status; + dev_err(&cq->hwq.pdev->dev, + "QPLIB: FP: CQ Processed Req \n"); + dev_err(&cq->hwq.pdev->dev, + "QPLIB: QP 0x%x wr_id[%d] = 0x%lx vendor type 0x%x with vendor status 0x%x\n", + cqe->src_qp, sq->swq_last, cqe->wr_id, cqe->type, cqe->status); + cqe++; + (*budget)--; + bnxt_qplib_mark_qp_error(qp); + } else { + /* Before we complete, do WA 9060 */ + if (!_is_chip_gen_p5_p7(qp->cctx)) { + if (bnxt_re_legacy_do_wa9060(qp, cq, cq_cons, + sq->swq_last, + cqe_sq_cons)) { + *lib_qp = qp; + goto out; + } + } + if (swq->flags & SQ_SEND_FLAGS_SIGNAL_COMP) { + + dev_dbg(&cq->hwq.pdev->dev, + "QPLIB: FP: CQ Processed Req \n"); + dev_dbg(&cq->hwq.pdev->dev, + "QPLIB: wr_id[%d] = 0x%llx \n", + sq->swq_last, cqe->wr_id); + dev_dbg(&cq->hwq.pdev->dev, + "QPLIB: with status 0x%x\n", cqe->status); + cqe->status = CQ_REQ_STATUS_OK; + cqe++; + (*budget)--; + } + } +skip: + bnxt_qplib_hwq_incr_cons(sq->hwq.depth, &sq->hwq.cons, + swq->slots, &sq->dbinfo.flags); + sq->swq_last = swq->next_idx; + if (sq->single == true) + break; + } +out: + *pcqe = cqe; + if (sq->swq_last != cqe_sq_cons) { + /* Out of budget */ + rc = -EAGAIN; + goto done; + } + /* Back to normal completion mode only after it has completed all of + the WC for this CQE */ + sq->single = false; +done: + return rc; +} + +static void bnxt_qplib_release_srqe(struct bnxt_qplib_srq *srq, u32 tag) +{ + spin_lock(&srq->hwq.lock); + srq->swq[srq->last_idx].next_idx = (int)tag; + srq->last_idx = (int)tag; + srq->swq[srq->last_idx].next_idx = -1; + bnxt_qplib_hwq_incr_cons(srq->hwq.depth, &srq->hwq.cons, + srq->dbinfo.max_slot, &srq->dbinfo.flags); + spin_unlock(&srq->hwq.lock); +} + +static int bnxt_qplib_cq_process_res_rc(struct bnxt_qplib_cq *cq, + struct cq_res_rc *hwcqe, + struct bnxt_qplib_cqe **pcqe, + int *budget) +{ + struct bnxt_qplib_srq *srq; + struct bnxt_qplib_cqe *cqe; + struct bnxt_qplib_qp *qp; + struct bnxt_qplib_q *rq; + u32 wr_id_idx; + int rc = 0; + + qp = (struct bnxt_qplib_qp *)le64_to_cpu(hwcqe->qp_handle); + if (!qp) { + dev_err(&cq->hwq.pdev->dev, "QPLIB: process_cq RC qp is NULL\n"); + return -EINVAL; + } + if (qp->rq.flushed) { + dev_dbg(&cq->hwq.pdev->dev, + "%s: QPLIB: QP in Flush QP = %p\n", __func__, qp); + goto done; + } + + cqe = *pcqe; + cqe->opcode = hwcqe->cqe_type_toggle & CQ_BASE_CQE_TYPE_MASK; + cqe->length = le32_to_cpu(hwcqe->length); + cqe->invrkey = le32_to_cpu(hwcqe->imm_data_or_inv_r_key); + cqe->mr_handle = le64_to_cpu(hwcqe->mr_handle); + cqe->flags = le16_to_cpu(hwcqe->flags); + cqe->status = hwcqe->status; + cqe->qp_handle = (u64)(unsigned long)qp; + + wr_id_idx = le32_to_cpu(hwcqe->srq_or_rq_wr_id) & + CQ_RES_RC_SRQ_OR_RQ_WR_ID_MASK; + if (cqe->flags & CQ_RES_RC_FLAGS_SRQ_SRQ) { + srq = qp->srq; + if (!srq) { + dev_err(&cq->hwq.pdev->dev, + "QPLIB: FP: SRQ used but not defined??\n"); + return -EINVAL; + } + if (wr_id_idx > srq->hwq.depth - 1) { + dev_err(&cq->hwq.pdev->dev, + "QPLIB: FP: CQ Process RC \n"); + dev_err(&cq->hwq.pdev->dev, + "QPLIB: wr_id idx 0x%x exceeded SRQ max 0x%x\n", + wr_id_idx, srq->hwq.depth); + return -EINVAL; + } + cqe->wr_id = srq->swq[wr_id_idx].wr_id; + bnxt_qplib_release_srqe(srq, wr_id_idx); + dev_dbg(&srq->hwq.pdev->dev, + "QPLIB: FP: CQ Processed RC SRQ wr_id[%d] = 0x%llx\n", + wr_id_idx, cqe->wr_id); + cqe++; + (*budget)--; + *pcqe = cqe; + } else { + rq = &qp->rq; + if (wr_id_idx > (rq->max_wqe - 1)) { + dev_err(&cq->hwq.pdev->dev, + "QPLIB: FP: CQ Process RC \n"); + dev_err(&cq->hwq.pdev->dev, + "QPLIB: wr_id idx 0x%x exceeded RQ max 0x%x\n", + wr_id_idx, rq->hwq.depth); + return -EINVAL; + } + if (wr_id_idx != rq->swq_last) + return -EINVAL; + cqe->wr_id = rq->swq[rq->swq_last].wr_id; + dev_dbg(&cq->hwq.pdev->dev, + "QPLIB: FP: CQ Processed RC RQ wr_id[%d] = 0x%llx\n", + rq->swq_last, cqe->wr_id); + cqe++; + (*budget)--; + bnxt_qplib_hwq_incr_cons(rq->hwq.depth, &rq->hwq.cons, + rq->swq[rq->swq_last].slots, + &rq->dbinfo.flags); + rq->swq_last = rq->swq[rq->swq_last].next_idx; + *pcqe = cqe; + + if (hwcqe->status != CQ_RES_RC_STATUS_OK) + bnxt_qplib_mark_qp_error(qp); + } +done: + return rc; +} + +static int bnxt_qplib_cq_process_res_ud(struct bnxt_qplib_cq *cq, + struct cq_res_ud_v2 *hwcqe, + struct bnxt_qplib_cqe **pcqe, + int *budget) +{ + struct bnxt_qplib_srq *srq; + struct bnxt_qplib_cqe *cqe; + struct bnxt_qplib_qp *qp; + struct bnxt_qplib_q *rq; + u32 wr_id_idx; + int rc = 0; + u16 *smac; + + qp = (struct bnxt_qplib_qp *)le64_to_cpu(hwcqe->qp_handle); + if (!qp) { + dev_err(&cq->hwq.pdev->dev, "QPLIB: process_cq UD qp is NULL\n"); + return -EINVAL; + } + if (qp->rq.flushed) { + dev_dbg(&cq->hwq.pdev->dev, + "%s: QPLIB: QP in Flush QP = %p\n", __func__, qp); + goto done; + } + cqe = *pcqe; + cqe->opcode = hwcqe->cqe_type_toggle & CQ_RES_UD_V2_CQE_TYPE_MASK; + cqe->length = le32_to_cpu((hwcqe->length & CQ_RES_UD_V2_LENGTH_MASK)); + cqe->cfa_meta = le16_to_cpu(hwcqe->cfa_metadata0); + /* V2 format has metadata1 */ + cqe->cfa_meta |= (((le32_to_cpu(hwcqe->src_qp_high_srq_or_rq_wr_id) & + CQ_RES_UD_V2_CFA_METADATA1_MASK) >> + CQ_RES_UD_V2_CFA_METADATA1_SFT) << + BNXT_QPLIB_META1_SHIFT); + cqe->invrkey = le32_to_cpu(hwcqe->imm_data); + cqe->flags = le16_to_cpu(hwcqe->flags); + cqe->status = hwcqe->status; + cqe->qp_handle = (u64)(unsigned long)qp; + smac = (u16 *)cqe->smac; + smac[2] = ntohs(le16_to_cpu(hwcqe->src_mac[0])); + smac[1] = ntohs(le16_to_cpu(hwcqe->src_mac[1])); + smac[0] = ntohs(le16_to_cpu(hwcqe->src_mac[2])); + wr_id_idx = le32_to_cpu(hwcqe->src_qp_high_srq_or_rq_wr_id) + & CQ_RES_UD_V2_SRQ_OR_RQ_WR_ID_MASK; + cqe->src_qp = le16_to_cpu(hwcqe->src_qp_low) | + ((le32_to_cpu( + hwcqe->src_qp_high_srq_or_rq_wr_id) & + CQ_RES_UD_V2_SRC_QP_HIGH_MASK) >> 8); + + if (cqe->flags & CQ_RES_UD_V2_FLAGS_SRQ) { + srq = qp->srq; + if (!srq) { + dev_err(&cq->hwq.pdev->dev, + "QPLIB: FP: SRQ used but not defined??\n"); + return -EINVAL; + } + if (wr_id_idx > srq->hwq.depth - 1) { + dev_err(&cq->hwq.pdev->dev, + "QPLIB: FP: CQ Process UD \n"); + dev_err(&cq->hwq.pdev->dev, + "QPLIB: wr_id idx 0x%x exceeded SRQ max 0x%x\n", + wr_id_idx, srq->hwq.depth); + return -EINVAL; + } + cqe->wr_id = srq->swq[wr_id_idx].wr_id; + bnxt_qplib_release_srqe(srq, wr_id_idx); + dev_dbg(&cq->hwq.pdev->dev, + "QPLIB: FP: CQ Processed UD SRQ wr_id[%d] = 0x%llx\n", + wr_id_idx, cqe->wr_id); + cqe++; + (*budget)--; + *pcqe = cqe; + } else { + rq = &qp->rq; + if (wr_id_idx > (rq->max_wqe - 1)) { + dev_err(&cq->hwq.pdev->dev, + "QPLIB: FP: CQ Process UD \n"); + dev_err(&cq->hwq.pdev->dev, + "QPLIB: wr_id idx 0x%x exceeded RQ max 0x%x\n", + wr_id_idx, rq->hwq.depth); + return -EINVAL; + } + if (rq->swq_last != wr_id_idx) + return -EINVAL; + + cqe->wr_id = rq->swq[rq->swq_last].wr_id; + dev_dbg(&cq->hwq.pdev->dev, + "QPLIB: FP: CQ Processed UD RQ wr_id[%d] = 0x%llx\n", + rq->swq_last, cqe->wr_id); + cqe++; + (*budget)--; + bnxt_qplib_hwq_incr_cons(rq->hwq.depth, &rq->hwq.cons, + rq->swq[rq->swq_last].slots, + &rq->dbinfo.flags); + rq->swq_last = rq->swq[rq->swq_last].next_idx; + *pcqe = cqe; + + if (hwcqe->status != CQ_RES_UD_V2_STATUS_OK) + bnxt_qplib_mark_qp_error(qp); + } +done: + return rc; +} + +bool bnxt_qplib_is_cq_empty(struct bnxt_qplib_cq *cq) +{ + + struct cq_base *hw_cqe; + unsigned long flags; + bool rc = true; + + spin_lock_irqsave(&cq->hwq.lock, flags); + hw_cqe = bnxt_qplib_get_qe(&cq->hwq, cq->hwq.cons, NULL); + + /* Check for Valid bit. If the CQE is valid, return false */ + rc = !CQE_CMP_VALID(hw_cqe, cq->dbinfo.flags); + spin_unlock_irqrestore(&cq->hwq.lock, flags); + return rc; +} + +static int bnxt_qplib_cq_process_res_raweth_qp1(struct bnxt_qplib_cq *cq, + struct cq_res_raweth_qp1 *hwcqe, + struct bnxt_qplib_cqe **pcqe, + int *budget) +{ + struct bnxt_qplib_qp *qp; + struct bnxt_qplib_q *rq; + struct bnxt_qplib_srq *srq; + struct bnxt_qplib_cqe *cqe; + u32 wr_id_idx; + int rc = 0; + + qp = (struct bnxt_qplib_qp *)le64_to_cpu(hwcqe->qp_handle); + if (!qp) { + dev_err(&cq->hwq.pdev->dev, + "QPLIB: process_cq Raw/QP1 qp is NULL\n"); + return -EINVAL; + } + if (qp->rq.flushed) { + dev_dbg(&cq->hwq.pdev->dev, + "%s: QPLIB: QP in Flush QP = %p\n", __func__, qp); + goto done; + } + cqe = *pcqe; + cqe->opcode = hwcqe->cqe_type_toggle & CQ_BASE_CQE_TYPE_MASK; + cqe->flags = le16_to_cpu(hwcqe->flags); + cqe->qp_handle = (u64)(unsigned long)qp; + + wr_id_idx = le32_to_cpu(hwcqe->raweth_qp1_payload_offset_srq_or_rq_wr_id) + & CQ_RES_RAWETH_QP1_SRQ_OR_RQ_WR_ID_MASK; + cqe->src_qp = qp->id; + if (qp->id == 1 && !cqe->length) { + /* Add workaround for the length misdetection */ + cqe->length = 296; + } else { + cqe->length = le16_to_cpu(hwcqe->length); + } + cqe->pkey_index = qp->pkey_index; + memcpy(cqe->smac, qp->smac, 6); + + cqe->raweth_qp1_flags = le16_to_cpu(hwcqe->raweth_qp1_flags); + cqe->raweth_qp1_flags2 = le32_to_cpu(hwcqe->raweth_qp1_flags2); + cqe->raweth_qp1_metadata = le32_to_cpu(hwcqe->raweth_qp1_metadata); + + dev_dbg(&cq->hwq.pdev->dev, + "QPLIB: raweth_qp1_flags = 0x%x raweth_qp1_flags2 = 0x%x\n", + cqe->raweth_qp1_flags, cqe->raweth_qp1_flags2); + + if (cqe->flags & CQ_RES_RAWETH_QP1_FLAGS_SRQ_SRQ) { + srq = qp->srq; + if (!srq) { + dev_err(&cq->hwq.pdev->dev, + "QPLIB: FP: SRQ used but not defined??\n"); + return -EINVAL; + } + if (wr_id_idx > srq->hwq.depth - 1) { + dev_err(&cq->hwq.pdev->dev, + "QPLIB: FP: CQ Process Raw/QP1 \n"); + dev_err(&cq->hwq.pdev->dev, + "QPLIB: wr_id idx 0x%x exceeded SRQ max 0x%x\n", + wr_id_idx, srq->hwq.depth); + return -EINVAL; + } + cqe->wr_id = srq->swq[wr_id_idx].wr_id; + dev_dbg(&cq->hwq.pdev->dev, + "QPLIB: FP: CQ Processed Raw/QP1 SRQ \n"); + dev_dbg(&cq->hwq.pdev->dev, + "QPLIB: wr_id[%d] = 0x%llx with status = 0x%x\n", + wr_id_idx, cqe->wr_id, hwcqe->status); + cqe++; + (*budget)--; + srq->hwq.cons++; + *pcqe = cqe; + } else { + rq = &qp->rq; + if (wr_id_idx > (rq->max_wqe - 1)) { + dev_err(&cq->hwq.pdev->dev, + "QPLIB: FP: CQ Process Raw/QP1 RQ wr_id \n"); + dev_err(&cq->hwq.pdev->dev, + "QPLIB: ix 0x%x exceeded RQ max 0x%x\n", + wr_id_idx, rq->max_wqe); + return -EINVAL; + } + if (wr_id_idx != rq->swq_last) + return -EINVAL; + cqe->wr_id = rq->swq[rq->swq_last].wr_id; + dev_dbg(&cq->hwq.pdev->dev, + "QPLIB: FP: CQ Processed Raw/QP1 RQ \n"); + dev_dbg(&cq->hwq.pdev->dev, + "QPLIB: wr_id[%d] = 0x%llx with status = 0x%x\n", + wr_id_idx, cqe->wr_id, hwcqe->status); + cqe++; + (*budget)--; + bnxt_qplib_hwq_incr_cons(rq->hwq.depth, &rq->hwq.cons, + rq->swq[wr_id_idx].slots, + &rq->dbinfo.flags); + rq->swq_last = rq->swq[rq->swq_last].next_idx; + *pcqe = cqe; + + if (hwcqe->status != CQ_RES_RC_STATUS_OK) + bnxt_qplib_mark_qp_error(qp); + } +done: + return rc; +} + +static int bnxt_qplib_cq_process_terminal(struct bnxt_qplib_cq *cq, + struct cq_terminal *hwcqe, + struct bnxt_qplib_cqe **pcqe, + int *budget) +{ + struct bnxt_qplib_q *sq, *rq; + struct bnxt_qplib_cqe *cqe; + struct bnxt_qplib_qp *qp; + u32 swq_last; + u32 cqe_cons; + int rc = 0; + + /* Check the Status */ + if (hwcqe->status != CQ_TERMINAL_STATUS_OK) + dev_warn(&cq->hwq.pdev->dev, + "QPLIB: FP: CQ Process Terminal Error status = 0x%x\n", + hwcqe->status); + + qp = (struct bnxt_qplib_qp *)le64_to_cpu(hwcqe->qp_handle); + if (!qp) + return -EINVAL; + dev_dbg(&cq->hwq.pdev->dev, + "QPLIB: FP: CQ Process terminal for qp (0x%x)\n", qp->id); + + /* Terminal CQE requires all posted RQEs to complete with FLUSHED_ERR + * from the current rq->cons to the rq->prod regardless what the + * rq->cons the terminal CQE indicates. + */ + bnxt_qplib_mark_qp_error(qp); + + sq = &qp->sq; + rq = &qp->rq; + + cqe_cons = le16_to_cpu(hwcqe->sq_cons_idx); + if (cqe_cons == 0xFFFF) + goto do_rq; + + cqe_cons %= sq->max_wqe; + if (qp->sq.flushed) { + dev_dbg(&cq->hwq.pdev->dev, + "%s: QPLIB: QP in Flush QP = %p\n", __func__, qp); + goto sq_done; + } + + /* Terminal CQE can also include aggregated successful CQEs prior. + So we must complete all CQEs from the current sq's cons to the + cq_cons with status OK */ + cqe = *pcqe; + while (*budget) { + /*sw_cons = HWQ_CMP(sq->hwq.cons, &sq->hwq);*/ + swq_last = sq->swq_last; + if (swq_last == cqe_cons) + break; + if (sq->swq[swq_last].flags & SQ_SEND_FLAGS_SIGNAL_COMP) { + memset(cqe, 0, sizeof(*cqe)); + cqe->status = CQ_REQ_STATUS_OK; + cqe->opcode = CQ_BASE_CQE_TYPE_REQ; + cqe->qp_handle = (u64)qp; + cqe->src_qp = qp->id; + cqe->wr_id = sq->swq[swq_last].wr_id; + cqe->type = sq->swq[swq_last].type; + dev_dbg(&cq->hwq.pdev->dev, + "QPLIB: FP: CQ Processed terminal Req \n"); + dev_dbg(&cq->hwq.pdev->dev, + "QPLIB: wr_id[%d] = 0x%llx with status 0x%x\n", + swq_last, cqe->wr_id, cqe->status); + cqe++; + (*budget)--; + } + bnxt_qplib_hwq_incr_cons(sq->hwq.depth, &sq->hwq.cons, + sq->swq[swq_last].slots, + &sq->dbinfo.flags); + sq->swq_last = sq->swq[swq_last].next_idx; + } + *pcqe = cqe; + if (!*budget && swq_last != cqe_cons) { + /* Out of budget */ + rc = -EAGAIN; + goto sq_done; + } +sq_done: + if (rc) + return rc; +do_rq: + cqe_cons = le16_to_cpu(hwcqe->rq_cons_idx); + if (cqe_cons == 0xFFFF) { + goto done; + } else if (cqe_cons > (rq->max_wqe - 1)) { + dev_err(&cq->hwq.pdev->dev, + "QPLIB: FP: CQ Processed terminal \n"); + dev_err(&cq->hwq.pdev->dev, + "QPLIB: reported rq_cons_idx 0x%x exceeds max 0x%x\n", + cqe_cons, rq->hwq.depth); + goto done; + } + if (qp->rq.flushed) { + dev_dbg(&cq->hwq.pdev->dev, + "%s: QPLIB: QP in Flush QP = %p\n", __func__, qp); + rc = 0; + goto rq_done; + } + +rq_done: +done: + return rc; +} + +static int bnxt_qplib_cq_process_cutoff(struct bnxt_qplib_cq *cq, + struct cq_cutoff *hwcqe) +{ + /* Check the Status */ + if (hwcqe->status != CQ_CUTOFF_STATUS_OK) { + dev_err(&cq->hwq.pdev->dev, + "QPLIB: FP: CQ Process Cutoff Error status = 0x%x\n", + hwcqe->status); + return -EINVAL; + } + clear_bit(CQ_FLAGS_RESIZE_IN_PROG, &cq->flags); + wake_up_interruptible(&cq->waitq); + + dev_dbg(&cq->hwq.pdev->dev, "QPLIB: FP: CQ Processed Cutoff\n"); + return 0; +} + +int bnxt_qplib_process_flush_list(struct bnxt_qplib_cq *cq, + struct bnxt_qplib_cqe *cqe, + int num_cqes) +{ + struct bnxt_qplib_qp *qp = NULL; + u32 budget = num_cqes; + unsigned long flags; + + spin_lock_irqsave(&cq->flush_lock, flags); + list_for_each_entry(qp, &cq->sqf_head, sq_flush) { + dev_dbg(&cq->hwq.pdev->dev, + "QPLIB: FP: Flushing SQ QP= %p\n", + qp); + __flush_sq(&qp->sq, qp, &cqe, &budget); + } + + list_for_each_entry(qp, &cq->rqf_head, rq_flush) { + dev_dbg(&cq->hwq.pdev->dev, + "QPLIB: FP: Flushing RQ QP= %p\n", + qp); + __flush_rq(&qp->rq, qp, &cqe, &budget); + } + spin_unlock_irqrestore(&cq->flush_lock, flags); + + return num_cqes - budget; +} + +int bnxt_qplib_poll_cq(struct bnxt_qplib_cq *cq, struct bnxt_qplib_cqe *cqe, + int num_cqes, struct bnxt_qplib_qp **lib_qp) +{ + struct cq_base *hw_cqe; + u32 hw_polled = 0; + int budget, rc = 0; + u8 type; + + budget = num_cqes; + + while (budget) { + hw_cqe = bnxt_qplib_get_qe(&cq->hwq, cq->hwq.cons, NULL); + + /* Check for Valid bit */ + if (!CQE_CMP_VALID(hw_cqe, cq->dbinfo.flags)) + break; + + /* The valid test of the entry must be done first before + * reading any further. + */ + dma_rmb(); + /* From the device's respective CQE format to qplib_wc*/ + type = hw_cqe->cqe_type_toggle & CQ_BASE_CQE_TYPE_MASK; + switch (type) { + case CQ_BASE_CQE_TYPE_REQ: + rc = bnxt_qplib_cq_process_req(cq, + (struct cq_req *)hw_cqe, &cqe, &budget, + cq->hwq.cons, lib_qp); + break; + case CQ_BASE_CQE_TYPE_RES_RC: + rc = bnxt_qplib_cq_process_res_rc(cq, + (struct cq_res_rc *)hw_cqe, &cqe, + &budget); + break; + case CQ_BASE_CQE_TYPE_RES_UD: + rc = bnxt_qplib_cq_process_res_ud(cq, + (struct cq_res_ud_v2 *)hw_cqe, + &cqe, &budget); + break; + case CQ_BASE_CQE_TYPE_RES_RAWETH_QP1: + rc = bnxt_qplib_cq_process_res_raweth_qp1(cq, + (struct cq_res_raweth_qp1 *) + hw_cqe, &cqe, &budget); + break; + case CQ_BASE_CQE_TYPE_TERMINAL: + rc = bnxt_qplib_cq_process_terminal(cq, + (struct cq_terminal *)hw_cqe, + &cqe, &budget); + break; + case CQ_BASE_CQE_TYPE_CUT_OFF: + bnxt_qplib_cq_process_cutoff(cq, + (struct cq_cutoff *)hw_cqe); + /* Done processing this CQ */ + goto exit; + default: + dev_err(&cq->hwq.pdev->dev, + "QPLIB: process_cq unknown type 0x%x\n", + hw_cqe->cqe_type_toggle & + CQ_BASE_CQE_TYPE_MASK); + rc = -EINVAL; + break; + } + if (rc < 0) { + dev_dbg(&cq->hwq.pdev->dev, + "QPLIB: process_cqe rc = 0x%x\n", rc); + if (rc == -EAGAIN) + break; + /* Error while processing the CQE, just skip to the + next one */ + if (type != CQ_BASE_CQE_TYPE_TERMINAL) + dev_err(&cq->hwq.pdev->dev, + "QPLIB: process_cqe error rc = 0x%x\n", + rc); + } + hw_polled++; + bnxt_qplib_hwq_incr_cons(cq->hwq.depth, &cq->hwq.cons, + 1, &cq->dbinfo.flags); + } + if (hw_polled) + bnxt_qplib_ring_db(&cq->dbinfo, DBC_DBC_TYPE_CQ); +exit: + return num_cqes - budget; +} + +void bnxt_qplib_req_notify_cq(struct bnxt_qplib_cq *cq, u32 arm_type) +{ + cq->dbinfo.toggle = cq->toggle; + if (arm_type) + bnxt_qplib_ring_db(&cq->dbinfo, arm_type); + /* Using cq->arm_state variable to track whether to issue cq handler */ + atomic_set(&cq->arm_state, 1); +} + +void bnxt_qplib_flush_cqn_wq(struct bnxt_qplib_qp *qp) +{ + flush_workqueue(qp->scq->nq->cqn_wq); + if (qp->scq != qp->rcq) + flush_workqueue(qp->rcq->nq->cqn_wq); +} diff --git a/sys/dev/bnxt/bnxt_re/qplib_fp.h b/sys/dev/bnxt/bnxt_re/qplib_fp.h new file mode 100644 index 000000000000..527c377f0aa5 --- /dev/null +++ b/sys/dev/bnxt/bnxt_re/qplib_fp.h @@ -0,0 +1,638 @@ +/* + * Copyright (c) 2015-2024, Broadcom. All rights reserved. The term + * Broadcom refers to Broadcom Limited and/or its subsidiaries. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR + * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE + * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN + * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * Description: Fast Path Operators (header) + */ + +#ifndef __BNXT_QPLIB_FP_H__ +#define __BNXT_QPLIB_FP_H__ + +/* Temp header structures for SQ */ +struct sq_ud_ext_hdr { + __le32 dst_qp; + __le32 avid; + __le64 rsvd; +}; + +struct sq_raw_ext_hdr { + __le32 cfa_meta; + __le32 rsvd0; + __le64 rsvd1; +}; + +struct sq_rdma_ext_hdr { + __le64 remote_va; + __le32 remote_key; + __le32 rsvd; +}; + +struct sq_atomic_ext_hdr { + __le64 swap_data; + __le64 cmp_data; +}; + +struct sq_fr_pmr_ext_hdr { + __le64 pblptr; + __le64 va; +}; + +struct sq_bind_ext_hdr { + __le64 va; + __le32 length_lo; + __le32 length_hi; +}; + +struct rq_ext_hdr { + __le64 rsvd1; + __le64 rsvd2; +}; + +#define BNXT_QPLIB_ETHTYPE_ROCEV1 0x8915 + +struct bnxt_qplib_srq { + struct bnxt_qplib_pd *pd; + struct bnxt_qplib_dpi *dpi; + struct bnxt_qplib_chip_ctx *cctx; + struct bnxt_qplib_cq *cq; + struct bnxt_qplib_swq *swq; + struct bnxt_qplib_hwq hwq; + struct bnxt_qplib_db_info dbinfo; + struct bnxt_qplib_sg_info sginfo; + u64 srq_handle; + u32 id; + u16 wqe_size; + u32 max_wqe; + u32 max_sge; + u32 threshold; + bool arm_req; + int start_idx; + int last_idx; + u16 eventq_hw_ring_id; + bool is_user; + spinlock_t lock; +}; + +struct bnxt_qplib_sge { + u64 addr; + u32 size; + u32 lkey; +}; + +/* + * Buffer space for ETH(14), IP or GRH(40), UDP header(8) + * and ib_bth + ib_deth (20). + * Max required is 82 when RoCE V2 is enabled + */ + +/* + * RoCE V1 (38 bytes needed) + * +------------+----------+--------+--------+-------+ + * |Eth-hdr(14B)| GRH (40B)|bth+deth| Mad | iCRC | + * | | supplied | 20B |payload | 4B | + * | | by user |supplied| 256B | | + * | | mad | |by user | | + * | | | | | | + * | sge 1 | sge 2 | sge 3 | sge 4 | sge 5 | + * +------------+----------+--------+--------+-------+ + */ + +/* + * RoCE V2-IPv4 (46 Bytes needed) + * +------------+----------+--------+--------+-------+ + * |Eth-hdr(14B)| IP-hdr |UDP-hdr | Mad | iCRC | + * | | supplied | 8B |payload | 4B | + * | | by user |bth+deth| 256B | | + * | | mad lower| 20B |supplied| | + * | | 20B out | (sge 3)|by user | | + * | | of 40B | | | | + * | | grh space| | | | + * | sge 1 | sge 2 | sge 3 | sge 4 | sge 5 | + * +------------+----------+--------+--------+-------+ + */ + +/* + * RoCE V2-IPv6 (46 Bytes needed) + * +------------+----------+--------+--------+-------+ + * |Eth-hdr(14B)| IPv6 |UDP-hdr | Mad | iCRC | + * | | supplied | 8B |payload | 4B | + * | | by user |bth+deth| 256B | | + * | | mad lower| 20B |supplied| | + * | | 40 bytes | |by user | | + * | | grh space| | | | + * | | | | | | + * | sge 1 | sge 2 | sge 3 | sge 4 | sge 5 | + * +------------+----------+--------+--------+-------+ + */ + +#define BNXT_QPLIB_MAX_QP1_SQ_HDR_SIZE 74 +#define BNXT_QPLIB_MAX_QP1_SQ_HDR_SIZE_V2 86 +#define BNXT_QPLIB_MAX_QP1_RQ_HDR_SIZE 46 +#define BNXT_QPLIB_MAX_QP1_RQ_ETH_HDR_SIZE 14 +#define BNXT_QPLIB_MAX_QP1_RQ_HDR_SIZE_V2 512 +#define BNXT_QPLIB_MAX_GRH_HDR_SIZE_IPV4 20 +#define BNXT_QPLIB_MAX_GRH_HDR_SIZE_IPV6 40 +#define BNXT_QPLIB_MAX_QP1_RQ_BDETH_HDR_SIZE 20 +#define BNXT_QPLIB_MAX_SQSZ 0xFFFF + +struct bnxt_qplib_hdrbuf { + dma_addr_t dma_map; + void *va; + u32 len; + u32 step; +}; + +struct bnxt_qplib_swq { + u64 wr_id; + int next_idx; + u8 type; + u8 flags; + u32 start_psn; + u32 next_psn; + u32 slot_idx; + u8 slots; + /* WIP: make it void * to handle legacy also */ + struct sq_psn_search *psn_search; + void *inline_data; +}; + +struct bnxt_qplib_swqe { + /* General */ +#define BNXT_QPLIB_FENCE_WRID 0x46454E43 /* "FENC" */ +#define BNXT_QPLIB_QP1_DUMMY_WRID 0x44554D59 /* "DUMY" */ + u64 wr_id; + u8 reqs_type; + u8 type; +#define BNXT_QPLIB_SWQE_TYPE_SEND 0 +#define BNXT_QPLIB_SWQE_TYPE_SEND_WITH_IMM 1 +#define BNXT_QPLIB_SWQE_TYPE_SEND_WITH_INV 2 +#define BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE 4 +#define BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE_WITH_IMM 5 +#define BNXT_QPLIB_SWQE_TYPE_RDMA_READ 6 +#define BNXT_QPLIB_SWQE_TYPE_ATOMIC_CMP_AND_SWP 8 +#define BNXT_QPLIB_SWQE_TYPE_ATOMIC_FETCH_AND_ADD 11 +#define BNXT_QPLIB_SWQE_TYPE_LOCAL_INV 12 +#define BNXT_QPLIB_SWQE_TYPE_FAST_REG_MR 13 +#define BNXT_QPLIB_SWQE_TYPE_REG_MR 13 +#define BNXT_QPLIB_SWQE_TYPE_BIND_MW 14 +#define BNXT_QPLIB_SWQE_TYPE_RECV 128 +#define BNXT_QPLIB_SWQE_TYPE_RECV_RDMA_IMM 129 + u8 flags; +#define BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP (1 << 0) +#define BNXT_QPLIB_SWQE_FLAGS_RD_ATOMIC_FENCE (1 << 1) +#define BNXT_QPLIB_SWQE_FLAGS_UC_FENCE (1 << 2) +#define BNXT_QPLIB_SWQE_FLAGS_SOLICIT_EVENT (1 << 3) +#define BNXT_QPLIB_SWQE_FLAGS_INLINE (1 << 4) + struct bnxt_qplib_sge *sg_list; + int num_sge; + + union { + /* Send, with imm, inval key */ + struct { + union { + __be32 imm_data; + u32 inv_key; + }; + u32 q_key; + u32 dst_qp; + u16 avid; + } send; + + /* Send Raw Ethernet and QP1 */ + struct { + u16 lflags; + u16 cfa_action; + u32 cfa_meta; + } rawqp1; + + /* RDMA write, with imm, read */ + struct { + union { + __be32 imm_data; + u32 inv_key; + }; + u64 remote_va; + u32 r_key; + } rdma; + + /* Atomic cmp/swap, fetch/add */ + struct { + u64 remote_va; + u32 r_key; + u64 swap_data; + u64 cmp_data; + } atomic; + + /* Local Invalidate */ + struct { + u32 inv_l_key; + } local_inv; + + /* FR-PMR */ + struct { + u8 access_cntl; + u8 pg_sz_log; + bool zero_based; + u32 l_key; + u32 length; + u8 pbl_pg_sz_log; +#define BNXT_QPLIB_SWQE_PAGE_SIZE_4K 0 +#define BNXT_QPLIB_SWQE_PAGE_SIZE_8K 1 +#define BNXT_QPLIB_SWQE_PAGE_SIZE_64K 4 +#define BNXT_QPLIB_SWQE_PAGE_SIZE_256K 6 +#define BNXT_QPLIB_SWQE_PAGE_SIZE_1M 8 +#define BNXT_QPLIB_SWQE_PAGE_SIZE_2M 9 +#define BNXT_QPLIB_SWQE_PAGE_SIZE_4M 10 +#define BNXT_QPLIB_SWQE_PAGE_SIZE_1G 18 + u8 levels; +#define PAGE_SHIFT_4K 12 + __le64 *pbl_ptr; + dma_addr_t pbl_dma_ptr; + u64 *page_list; + u16 page_list_len; + u64 va; + } frmr; + + /* Bind */ + struct { + u8 access_cntl; +#define BNXT_QPLIB_BIND_SWQE_ACCESS_LOCAL_WRITE (1 << 0) +#define BNXT_QPLIB_BIND_SWQE_ACCESS_REMOTE_READ (1 << 1) +#define BNXT_QPLIB_BIND_SWQE_ACCESS_REMOTE_WRITE (1 << 2) +#define BNXT_QPLIB_BIND_SWQE_ACCESS_REMOTE_ATOMIC (1 << 3) +#define BNXT_QPLIB_BIND_SWQE_ACCESS_WINDOW_BIND (1 << 4) + bool zero_based; + u8 mw_type; + u32 parent_l_key; + u32 r_key; + u64 va; + u32 length; + } bind; + }; +}; + +struct bnxt_qplib_q { + struct bnxt_qplib_swq *swq; + struct bnxt_qplib_db_info dbinfo; + struct bnxt_qplib_sg_info sginfo; + struct bnxt_qplib_hwq hwq; + u32 max_wqe; + u16 max_sge; + u16 wqe_size; + u16 q_full_delta; + u32 psn; + bool condition; + bool single; + bool legacy_send_phantom; + u32 phantom_wqe_cnt; + u32 phantom_cqe_cnt; + u32 next_cq_cons; + bool flushed; + u32 swq_start; + u32 swq_last; +}; + +#define BNXT_QPLIB_PPP_REQ 0x1 +#define BNXT_QPLIB_PPP_ST_IDX_SHIFT 0x1 + +struct bnxt_qplib_ppp { + u32 dpi; + u8 req; + u8 st_idx_en; +}; + +struct bnxt_qplib_qp { + struct bnxt_qplib_pd *pd; + struct bnxt_qplib_dpi *dpi; + struct bnxt_qplib_chip_ctx *cctx; + u64 qp_handle; +#define BNXT_QPLIB_QP_ID_INVALID 0xFFFFFFFF + u32 id; + u8 type; + u8 sig_type; + u8 wqe_mode; + u8 state; + u8 cur_qp_state; + u8 is_user; + u64 modify_flags; + u32 max_inline_data; + u32 mtu; + u32 path_mtu; + bool en_sqd_async_notify; + u16 pkey_index; + u32 qkey; + u32 dest_qp_id; + u8 access; + u8 timeout; + u8 retry_cnt; + u8 rnr_retry; + u64 wqe_cnt; + u32 min_rnr_timer; + u32 max_rd_atomic; + u32 max_dest_rd_atomic; + u32 dest_qpn; + u8 smac[6]; + u16 vlan_id; + u8 nw_type; + u16 port_id; + struct bnxt_qplib_ah ah; + struct bnxt_qplib_ppp ppp; + +#define BTH_PSN_MASK ((1 << 24) - 1) + /* SQ */ + struct bnxt_qplib_q sq; + /* RQ */ + struct bnxt_qplib_q rq; + /* SRQ */ + struct bnxt_qplib_srq *srq; + /* CQ */ + struct bnxt_qplib_cq *scq; + struct bnxt_qplib_cq *rcq; + /* IRRQ and ORRQ */ + struct bnxt_qplib_hwq irrq; + struct bnxt_qplib_hwq orrq; + /* Header buffer for QP1 */ + struct bnxt_qplib_hdrbuf *sq_hdr_buf; + struct bnxt_qplib_hdrbuf *rq_hdr_buf; + + /* ToS */ + u8 tos_ecn; + u8 tos_dscp; + /* To track the SQ and RQ flush list */ + struct list_head sq_flush; + struct list_head rq_flush; + /* 4 bytes of QP's scrabled mac received from FW */ + u32 lag_src_mac; + u32 msn; + u32 msn_tbl_sz; + /* get devflags in PI code */ + u16 dev_cap_flags; +}; + + +#define CQE_CMP_VALID(hdr, pass) \ + (!!((hdr)->cqe_type_toggle & CQ_BASE_TOGGLE) == \ + !(pass & BNXT_QPLIB_FLAG_EPOCH_CONS_MASK)) + +static inline u32 __bnxt_qplib_get_avail(struct bnxt_qplib_hwq *hwq) +{ + int cons, prod, avail; + + /* False full is possible retrying post-send makes sense */ + cons = hwq->cons; + prod = hwq->prod; + avail = cons - prod; + if (cons <= prod) + avail += hwq->depth; + return avail; +} + +static inline bool bnxt_qplib_queue_full(struct bnxt_qplib_hwq *hwq, u8 slots) +{ + return __bnxt_qplib_get_avail(hwq) <= slots; +} + +struct bnxt_qplib_cqe { + u8 status; + u8 type; + u8 opcode; + u32 length; + /* Lower 16 is cfa_metadata0, Upper 16 is cfa_metadata1 */ + u32 cfa_meta; +#define BNXT_QPLIB_META1_SHIFT 16 +#define BNXT_QPLIB_CQE_CFA_META1_VALID 0x80000UL + u64 wr_id; + union { + __be32 immdata; + u32 invrkey; + }; + u64 qp_handle; + u64 mr_handle; + u16 flags; + u8 smac[6]; + u32 src_qp; + u16 raweth_qp1_flags; + u16 raweth_qp1_errors; + u16 raweth_qp1_cfa_code; + u32 raweth_qp1_flags2; + u32 raweth_qp1_metadata; + u8 raweth_qp1_payload_offset; + u16 pkey_index; +}; + +#define BNXT_QPLIB_QUEUE_START_PERIOD 0x01 +struct bnxt_qplib_cq { + struct bnxt_qplib_dpi *dpi; + struct bnxt_qplib_chip_ctx *cctx; + struct bnxt_qplib_nq *nq; + struct bnxt_qplib_db_info dbinfo; + struct bnxt_qplib_sg_info sginfo; + struct bnxt_qplib_hwq hwq; + struct bnxt_qplib_hwq resize_hwq; + struct list_head sqf_head; + struct list_head rqf_head; + u32 max_wqe; + u32 id; + u16 count; + u16 period; + u32 cnq_hw_ring_id; + u64 cq_handle; + atomic_t arm_state; +#define CQ_RESIZE_WAIT_TIME_MS 500 + unsigned long flags; +#define CQ_FLAGS_RESIZE_IN_PROG 1 + wait_queue_head_t waitq; + spinlock_t flush_lock; /* lock flush queue list */ + spinlock_t compl_lock; /* synch CQ handlers */ + u16 cnq_events; + bool is_cq_err_event; + bool destroyed; + u8 toggle; +}; + +#define BNXT_QPLIB_MAX_IRRQE_ENTRY_SIZE sizeof(struct xrrq_irrq) +#define BNXT_QPLIB_MAX_ORRQE_ENTRY_SIZE sizeof(struct xrrq_orrq) +#define IRD_LIMIT_TO_IRRQ_SLOTS(x) (2 * x + 2) +#define IRRQ_SLOTS_TO_IRD_LIMIT(s) ((s >> 1) - 1) +#define ORD_LIMIT_TO_ORRQ_SLOTS(x) (x + 1) +#define ORRQ_SLOTS_TO_ORD_LIMIT(s) (s - 1) + +#define NQE_CMP_VALID(hdr, pass) \ + (!!(le32_to_cpu((hdr)->info63_v & 0xffffffff) & NQ_BASE_V) == \ + !(pass & BNXT_QPLIB_FLAG_EPOCH_CONS_MASK)) + +#define BNXT_QPLIB_NQE_MAX_CNT (128 * 1024) + +/* MSN table print macros for debugging */ +#define BNXT_RE_MSN_IDX(m) (((m) & SQ_MSN_SEARCH_START_IDX_MASK) >> \ + SQ_MSN_SEARCH_START_IDX_SFT) +#define BNXT_RE_MSN_NPSN(m) (((m) & SQ_MSN_SEARCH_NEXT_PSN_MASK) >> \ + SQ_MSN_SEARCH_NEXT_PSN_SFT) +#define BNXT_RE_MSN_SPSN(m) (((m) & SQ_MSN_SEARCH_START_PSN_MASK) >> \ + SQ_MSN_SEARCH_START_PSN_SFT) +#define BNXT_MSN_TBLE_SGE 6 + +struct bnxt_qplib_nq_stats { + u64 num_dbqne_processed; + u64 num_srqne_processed; + u64 num_cqne_processed; + u64 num_tasklet_resched; + u64 num_nq_rearm; +}; + +struct bnxt_qplib_nq_db { + struct bnxt_qplib_reg_desc reg; + void __iomem *db; + struct bnxt_qplib_db_info dbinfo; +}; + +typedef int (*cqn_handler_t)(struct bnxt_qplib_nq *nq, + struct bnxt_qplib_cq *cq); +typedef int (*srqn_handler_t)(struct bnxt_qplib_nq *nq, + struct bnxt_qplib_srq *srq, u8 event); + +struct bnxt_qplib_nq { + struct bnxt_qplib_res *res; + struct bnxt_qplib_hwq hwq; + struct bnxt_qplib_nq_db nq_db; + + char *name; + u16 ring_id; + int msix_vec; + bool requested; + int budget; + u32 load; + struct mutex lock; + + cqn_handler_t cqn_handler; + srqn_handler_t srqn_handler; + struct workqueue_struct *cqn_wq; + struct bnxt_qplib_nq_stats stats; +}; + +struct bnxt_qplib_nq_work { + struct work_struct work; + struct bnxt_qplib_nq *nq; + struct bnxt_qplib_cq *cq; +}; + +static inline dma_addr_t +bnxt_qplib_get_qp_buf_from_index(struct bnxt_qplib_qp *qp, u32 index) +{ + struct bnxt_qplib_hdrbuf *buf; + + buf = qp->rq_hdr_buf; + return (buf->dma_map + index * buf->step); +} + +void bnxt_qplib_nq_stop_irq(struct bnxt_qplib_nq *nq, bool kill); +void bnxt_qplib_disable_nq(struct bnxt_qplib_nq *nq); +int bnxt_qplib_nq_start_irq(struct bnxt_qplib_nq *nq, int nq_indx, + int msix_vector, bool need_init); +int bnxt_qplib_enable_nq(struct bnxt_qplib_nq *nq, int nq_idx, + int msix_vector, int bar_reg_offset, + cqn_handler_t cqn_handler, + srqn_handler_t srq_handler); +int bnxt_qplib_create_srq(struct bnxt_qplib_res *res, + struct bnxt_qplib_srq *srq); +int bnxt_qplib_modify_srq(struct bnxt_qplib_res *res, + struct bnxt_qplib_srq *srq); +int bnxt_qplib_query_srq(struct bnxt_qplib_res *res, + struct bnxt_qplib_srq *srq); +int bnxt_qplib_destroy_srq(struct bnxt_qplib_res *res, + struct bnxt_qplib_srq *srq); +int bnxt_qplib_post_srq_recv(struct bnxt_qplib_srq *srq, + struct bnxt_qplib_swqe *wqe); +int bnxt_qplib_create_qp1(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp); +int bnxt_qplib_create_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp); +int bnxt_qplib_modify_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp); +int bnxt_qplib_query_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp); +int bnxt_qplib_destroy_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp); +void bnxt_qplib_clean_qp(struct bnxt_qplib_qp *qp); +void bnxt_qplib_free_qp_res(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp); +void *bnxt_qplib_get_qp1_sq_buf(struct bnxt_qplib_qp *qp, + struct bnxt_qplib_sge *sge); +void *bnxt_qplib_get_qp1_rq_buf(struct bnxt_qplib_qp *qp, + struct bnxt_qplib_sge *sge); +u32 bnxt_qplib_get_rq_prod_index(struct bnxt_qplib_qp *qp); +void bnxt_qplib_post_send_db(struct bnxt_qplib_qp *qp); +int bnxt_qplib_post_send(struct bnxt_qplib_qp *qp, + struct bnxt_qplib_swqe *wqe); +void bnxt_qplib_post_recv_db(struct bnxt_qplib_qp *qp); +int bnxt_qplib_post_recv(struct bnxt_qplib_qp *qp, + struct bnxt_qplib_swqe *wqe); +int bnxt_qplib_create_cq(struct bnxt_qplib_res *res, struct bnxt_qplib_cq *cq); +int bnxt_qplib_modify_cq(struct bnxt_qplib_res *res, struct bnxt_qplib_cq *cq); +int bnxt_qplib_resize_cq(struct bnxt_qplib_res *res, struct bnxt_qplib_cq *cq, + int new_cqes); +void bnxt_qplib_resize_cq_complete(struct bnxt_qplib_res *res, + struct bnxt_qplib_cq *cq); +int bnxt_qplib_destroy_cq(struct bnxt_qplib_res *res, struct bnxt_qplib_cq *cq); +void bnxt_qplib_free_cq(struct bnxt_qplib_res *res, struct bnxt_qplib_cq *cq); +int bnxt_qplib_poll_cq(struct bnxt_qplib_cq *cq, struct bnxt_qplib_cqe *cqe, + int num, struct bnxt_qplib_qp **qp); +bool bnxt_qplib_is_cq_empty(struct bnxt_qplib_cq *cq); +void bnxt_qplib_req_notify_cq(struct bnxt_qplib_cq *cq, u32 arm_type); +void bnxt_qplib_free_nq_mem(struct bnxt_qplib_nq *nq); +int bnxt_qplib_alloc_nq_mem(struct bnxt_qplib_res *res, + struct bnxt_qplib_nq *nq); +void bnxt_qplib_add_flush_qp(struct bnxt_qplib_qp *qp); +void bnxt_qplib_del_flush_qp(struct bnxt_qplib_qp *qp); +int bnxt_qplib_process_flush_list(struct bnxt_qplib_cq *cq, + struct bnxt_qplib_cqe *cqe, + int num_cqes); +void bnxt_qplib_flush_cqn_wq(struct bnxt_qplib_qp *qp); +void bnxt_qplib_free_hdr_buf(struct bnxt_qplib_res *res, + struct bnxt_qplib_qp *qp); +int bnxt_qplib_alloc_hdr_buf(struct bnxt_qplib_res *res, + struct bnxt_qplib_qp *qp, u32 slen, u32 rlen); + +static inline bool __can_request_ppp(struct bnxt_qplib_qp *qp) +{ + bool can_request = false; + + if (qp->cur_qp_state == CMDQ_MODIFY_QP_NEW_STATE_RESET && + qp->state == CMDQ_MODIFY_QP_NEW_STATE_INIT && + qp->ppp.req && + !(qp->ppp.st_idx_en & + CREQ_MODIFY_QP_RESP_PINGPONG_PUSH_ENABLED)) + can_request = true; + return can_request; +} + +/* MSN table update inlin */ +static inline uint64_t bnxt_re_update_msn_tbl(uint32_t st_idx, uint32_t npsn, uint32_t start_psn) +{ + return cpu_to_le64((((u64)(st_idx) << SQ_MSN_SEARCH_START_IDX_SFT) & + SQ_MSN_SEARCH_START_IDX_MASK) | + (((u64)(npsn) << SQ_MSN_SEARCH_NEXT_PSN_SFT) & + SQ_MSN_SEARCH_NEXT_PSN_MASK) | + (((start_psn) << SQ_MSN_SEARCH_START_PSN_SFT) & + SQ_MSN_SEARCH_START_PSN_MASK)); +} + +void bnxt_re_schedule_dbq_event(struct bnxt_qplib_res *res); +#endif diff --git a/sys/dev/bnxt/bnxt_re/qplib_rcfw.c b/sys/dev/bnxt/bnxt_re/qplib_rcfw.c new file mode 100644 index 000000000000..7e3453a1e044 --- /dev/null +++ b/sys/dev/bnxt/bnxt_re/qplib_rcfw.c @@ -0,0 +1,1338 @@ +/* + * Copyright (c) 2015-2024, Broadcom. All rights reserved. The term + * Broadcom refers to Broadcom Limited and/or its subsidiaries. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR + * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE + * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN + * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * Description: RDMA Controller HW interface + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "hsi_struct_def.h" +#include "qplib_tlv.h" +#include "qplib_res.h" +#include "qplib_sp.h" +#include "qplib_rcfw.h" +#include "bnxt_re.h" + +static void bnxt_qplib_service_creq(unsigned long data); + +int __check_cmdq_stall(struct bnxt_qplib_rcfw *rcfw, + u32 *cur_prod, u32 *cur_cons) +{ + struct bnxt_qplib_cmdq_ctx *cmdq; + + cmdq = &rcfw->cmdq; + + if (*cur_prod == cmdq->hwq.prod && + *cur_cons == cmdq->hwq.cons) + /* No activity on CMDQ or CREQ. FW down */ + return -ETIMEDOUT; + + *cur_prod = cmdq->hwq.prod; + *cur_cons = cmdq->hwq.cons; + return 0; +} + +static int bnxt_qplib_map_rc(u8 opcode) +{ + switch (opcode) { + case CMDQ_BASE_OPCODE_DESTROY_QP: + case CMDQ_BASE_OPCODE_DESTROY_SRQ: + case CMDQ_BASE_OPCODE_DESTROY_CQ: + case CMDQ_BASE_OPCODE_DEALLOCATE_KEY: + case CMDQ_BASE_OPCODE_DEREGISTER_MR: + case CMDQ_BASE_OPCODE_DELETE_GID: + case CMDQ_BASE_OPCODE_DESTROY_QP1: + case CMDQ_BASE_OPCODE_DESTROY_AH: + case CMDQ_BASE_OPCODE_DEINITIALIZE_FW: + case CMDQ_BASE_OPCODE_MODIFY_ROCE_CC: + case CMDQ_BASE_OPCODE_SET_LINK_AGGR_MODE: + return 0; + default: + return -ETIMEDOUT; + } +} + +/** + * bnxt_re_is_fw_stalled - Check firmware health + * @rcfw - rcfw channel instance of rdev + * @cookie - cookie to track the command + * + * If firmware has not responded any rcfw command within + * rcfw->max_timeout, consider firmware as stalled. + * + * Returns: + * 0 if firmware is responding + * -ENODEV if firmware is not responding + */ +static int bnxt_re_is_fw_stalled(struct bnxt_qplib_rcfw *rcfw, u16 cookie) +{ + struct bnxt_qplib_cmdq_ctx *cmdq; + struct bnxt_qplib_crsqe *crsqe; + + crsqe = &rcfw->crsqe_tbl[cookie]; + cmdq = &rcfw->cmdq; + + if (time_after(jiffies, cmdq->last_seen + + (rcfw->max_timeout * HZ))) { + dev_warn_ratelimited(&rcfw->pdev->dev, + "%s: FW STALL Detected. cmdq[%#x]=%#x waited (%ld > %d) msec active %d\n", + __func__, cookie, crsqe->opcode, + (long)jiffies_to_msecs(jiffies - cmdq->last_seen), + rcfw->max_timeout * 1000, + crsqe->is_in_used); + return -ENODEV; + } + + return 0; +} +/** + * __wait_for_resp - Don't hold the cpu context and wait for response + * @rcfw - rcfw channel instance of rdev + * @cookie - cookie to track the command + * + * Wait for command completion in sleepable context. + * + * Returns: + * 0 if command is completed by firmware. + * Non zero error code for rest of the case. + */ +static int __wait_for_resp(struct bnxt_qplib_rcfw *rcfw, u16 cookie) +{ + struct bnxt_qplib_cmdq_ctx *cmdq; + struct bnxt_qplib_crsqe *crsqe; + unsigned long issue_time; + int ret; + + cmdq = &rcfw->cmdq; + issue_time = jiffies; + crsqe = &rcfw->crsqe_tbl[cookie]; + + do { + if (RCFW_NO_FW_ACCESS(rcfw)) + return bnxt_qplib_map_rc(crsqe->opcode); + if (test_bit(FIRMWARE_STALL_DETECTED, &cmdq->flags)) + return -ETIMEDOUT; + + /* Non zero means command completed */ + ret = wait_event_timeout(cmdq->waitq, + !crsqe->is_in_used || + RCFW_NO_FW_ACCESS(rcfw), + msecs_to_jiffies(rcfw->max_timeout * 1000)); + + if (!crsqe->is_in_used) + return 0; + /* + * Take care if interrupt miss or other cases like DBR drop + */ + bnxt_qplib_service_creq((unsigned long)rcfw); + dev_warn_ratelimited(&rcfw->pdev->dev, + "Non-Blocking QPLIB: cmdq[%#x]=%#x waited (%lu) msec bit %d\n", + cookie, crsqe->opcode, + (long)jiffies_to_msecs(jiffies - issue_time), + crsqe->is_in_used); + + if (!crsqe->is_in_used) + return 0; + + ret = bnxt_re_is_fw_stalled(rcfw, cookie); + if (ret) + return ret; + + } while (true); +}; + +/** + * __block_for_resp - hold the cpu context and wait for response + * @rcfw - rcfw channel instance of rdev + * @cookie - cookie to track the command + * + * This function will hold the cpu (non-sleepable context) and + * wait for command completion. Maximum holding interval is 8 second. + * + * Returns: + * -ETIMEOUT if command is not completed in specific time interval. + * 0 if command is completed by firmware. + */ +static int __block_for_resp(struct bnxt_qplib_rcfw *rcfw, u16 cookie) +{ + struct bnxt_qplib_cmdq_ctx *cmdq = &rcfw->cmdq; + struct bnxt_qplib_crsqe *crsqe; + unsigned long issue_time = 0; + + issue_time = jiffies; + crsqe = &rcfw->crsqe_tbl[cookie]; + + do { + if (RCFW_NO_FW_ACCESS(rcfw)) + return bnxt_qplib_map_rc(crsqe->opcode); + if (test_bit(FIRMWARE_STALL_DETECTED, &cmdq->flags)) + return -ETIMEDOUT; + + udelay(1); + + /* Below call is must since there can be a deadlock + * if interrupt is mapped to the same cpu + */ + bnxt_qplib_service_creq((unsigned long)rcfw); + if (!crsqe->is_in_used) + return 0; + + } while (time_before(jiffies, issue_time + (8 * HZ))); + + dev_warn_ratelimited(&rcfw->pdev->dev, + "Blocking QPLIB: cmdq[%#x]=%#x taken (%lu) msec", + cookie, crsqe->opcode, + (long)jiffies_to_msecs(jiffies - issue_time)); + + return -ETIMEDOUT; +}; + +/* __send_message_no_waiter - get cookie and post the message. + * @rcfw - rcfw channel instance of rdev + * @msg - qplib message internal + * + * This function will just post and don't bother about completion. + * Current design of this function is - + * user must hold the completion queue hwq->lock. + * user must have used existing completion and free the resources. + * this function will not check queue full condition. + * this function will explicitly set is_waiter_alive=false. + * current use case is - send destroy_ah if create_ah is return + * after waiter of create_ah is lost. It can be extended for other + * use case as well. + * + * Returns: Nothing + * + */ +static void __send_message_no_waiter(struct bnxt_qplib_rcfw *rcfw, + struct bnxt_qplib_cmdqmsg *msg) +{ + struct bnxt_qplib_cmdq_ctx *cmdq = &rcfw->cmdq; + struct bnxt_qplib_hwq *cmdq_hwq = &cmdq->hwq; + struct bnxt_qplib_crsqe *crsqe; + struct bnxt_qplib_cmdqe *cmdqe; + u32 sw_prod, cmdq_prod, bsize; + u16 cookie; + u8 *preq; + + cookie = cmdq->seq_num & RCFW_MAX_COOKIE_VALUE; + __set_cmdq_base_cookie(msg->req, msg->req_sz, cpu_to_le16(cookie)); + crsqe = &rcfw->crsqe_tbl[cookie]; + + /* Set cmd_size in terms of 16B slots in req. */ + bsize = bnxt_qplib_set_cmd_slots(msg->req); + /* GET_CMD_SIZE would return number of slots in either case of tlv + * and non-tlv commands after call to bnxt_qplib_set_cmd_slots() + */ + crsqe->send_timestamp = jiffies; + crsqe->is_internal_cmd = true; + crsqe->is_waiter_alive = false; + crsqe->is_in_used = true; + crsqe->req_size = __get_cmdq_base_cmd_size(msg->req, msg->req_sz); + + preq = (u8 *)msg->req; + do { + /* Locate the next cmdq slot */ + sw_prod = HWQ_CMP(cmdq_hwq->prod, cmdq_hwq); + cmdqe = bnxt_qplib_get_qe(cmdq_hwq, sw_prod, NULL); + /* Copy a segment of the req cmd to the cmdq */ + memset(cmdqe, 0, sizeof(*cmdqe)); + memcpy(cmdqe, preq, min_t(u32, bsize, sizeof(*cmdqe))); + preq += min_t(u32, bsize, sizeof(*cmdqe)); + bsize -= min_t(u32, bsize, sizeof(*cmdqe)); + cmdq_hwq->prod++; + } while (bsize > 0); + cmdq->seq_num++; + + cmdq_prod = cmdq_hwq->prod & 0xFFFF; + atomic_inc(&rcfw->timeout_send); + /* ring CMDQ DB */ + wmb(); + writel(cmdq_prod, cmdq->cmdq_mbox.prod); + writel(RCFW_CMDQ_TRIG_VAL, cmdq->cmdq_mbox.db); +} + +static int __send_message(struct bnxt_qplib_rcfw *rcfw, + struct bnxt_qplib_cmdqmsg *msg) +{ + u32 bsize, free_slots, required_slots; + struct bnxt_qplib_cmdq_ctx *cmdq; + struct bnxt_qplib_crsqe *crsqe; + struct bnxt_qplib_cmdqe *cmdqe; + struct bnxt_qplib_hwq *cmdq_hwq; + u32 sw_prod, cmdq_prod; + struct pci_dev *pdev; + unsigned long flags; + u16 cookie; + u8 opcode; + u8 *preq; + + cmdq = &rcfw->cmdq; + cmdq_hwq = &cmdq->hwq; + pdev = rcfw->pdev; + opcode = __get_cmdq_base_opcode(msg->req, msg->req_sz); + + /* Cmdq are in 16-byte units, each request can consume 1 or more + cmdqe */ + spin_lock_irqsave(&cmdq_hwq->lock, flags); + required_slots = bnxt_qplib_get_cmd_slots(msg->req); + free_slots = HWQ_FREE_SLOTS(cmdq_hwq); + cookie = cmdq->seq_num & RCFW_MAX_COOKIE_VALUE; + crsqe = &rcfw->crsqe_tbl[cookie]; + + if (required_slots >= free_slots) { + dev_warn_ratelimited(&pdev->dev, + "QPLIB: RCFW: CMDQ is full req/free %d/%d!\n", + required_slots, free_slots); + rcfw->cmdq_full_dbg++; + spin_unlock_irqrestore(&cmdq_hwq->lock, flags); + return -EAGAIN; + } + + if (crsqe->is_in_used) + panic("QPLIB: Cookie was not requested %d\n", + cookie); + + if (msg->block) + cookie |= RCFW_CMD_IS_BLOCKING; + __set_cmdq_base_cookie(msg->req, msg->req_sz, cpu_to_le16(cookie)); + + /* Set cmd_size in terms of 16B slots in req. */ + bsize = bnxt_qplib_set_cmd_slots(msg->req); + /* GET_CMD_SIZE would return number of slots in either case of tlv + * and non-tlv commands after call to bnxt_qplib_set_cmd_slots() + */ + crsqe->send_timestamp = jiffies; + crsqe->free_slots = free_slots; + crsqe->resp = (struct creq_qp_event *)msg->resp; + crsqe->resp->cookie = cpu_to_le16(cookie); + crsqe->is_internal_cmd = false; + crsqe->is_waiter_alive = true; + crsqe->is_in_used = true; + crsqe->opcode = opcode; + crsqe->requested_qp_state = msg->qp_state; + + crsqe->req_size = __get_cmdq_base_cmd_size(msg->req, msg->req_sz); + if (__get_cmdq_base_resp_size(msg->req, msg->req_sz) && msg->sb) { + struct bnxt_qplib_rcfw_sbuf *sbuf = msg->sb; + + __set_cmdq_base_resp_addr(msg->req, msg->req_sz, + cpu_to_le64(sbuf->dma_addr)); + __set_cmdq_base_resp_size(msg->req, msg->req_sz, + ALIGN(sbuf->size, BNXT_QPLIB_CMDQE_UNITS) / + BNXT_QPLIB_CMDQE_UNITS); + } + + preq = (u8 *)msg->req; + do { + /* Locate the next cmdq slot */ + sw_prod = HWQ_CMP(cmdq_hwq->prod, cmdq_hwq); + cmdqe = bnxt_qplib_get_qe(cmdq_hwq, sw_prod, NULL); + /* Copy a segment of the req cmd to the cmdq */ + memset(cmdqe, 0, sizeof(*cmdqe)); + memcpy(cmdqe, preq, min_t(u32, bsize, sizeof(*cmdqe))); + preq += min_t(u32, bsize, sizeof(*cmdqe)); + bsize -= min_t(u32, bsize, sizeof(*cmdqe)); + cmdq_hwq->prod++; + } while (bsize > 0); + cmdq->seq_num++; + + cmdq_prod = cmdq_hwq->prod & 0xFFFF; + if (test_bit(FIRMWARE_FIRST_FLAG, &cmdq->flags)) { + /* The very first doorbell write + * is required to set this flag + * which prompts the FW to reset + * its internal pointers + */ + cmdq_prod |= BIT(FIRMWARE_FIRST_FLAG); + clear_bit(FIRMWARE_FIRST_FLAG, &cmdq->flags); + } + /* ring CMDQ DB */ + wmb(); + writel(cmdq_prod, cmdq->cmdq_mbox.prod); + writel(RCFW_CMDQ_TRIG_VAL, cmdq->cmdq_mbox.db); + + dev_dbg(&pdev->dev, "QPLIB: RCFW sent request with 0x%x 0x%x 0x%x\n", + cmdq_prod, cmdq_hwq->prod, crsqe->req_size); + dev_dbg(&pdev->dev, + "QPLIB: opcode 0x%x with cookie 0x%x at cmdq/crsq 0x%p/0x%p\n", + opcode, + __get_cmdq_base_cookie(msg->req, msg->req_sz), + cmdqe, crsqe); + spin_unlock_irqrestore(&cmdq_hwq->lock, flags); + /* Return the CREQ response pointer */ + return 0; +} + +/** + * __poll_for_resp - self poll completion for rcfw command + * @rcfw - rcfw channel instance of rdev + * @cookie - cookie to track the command + * + * It works same as __wait_for_resp except this function will + * do self polling in sort interval since interrupt is disabled. + * This function can not be called from non-sleepable context. + * + * Returns: + * -ETIMEOUT if command is not completed in specific time interval. + * 0 if command is completed by firmware. + */ +static int __poll_for_resp(struct bnxt_qplib_rcfw *rcfw, u16 cookie) +{ + struct bnxt_qplib_cmdq_ctx *cmdq = &rcfw->cmdq; + struct bnxt_qplib_crsqe *crsqe; + unsigned long issue_time; + int ret; + + issue_time = jiffies; + crsqe = &rcfw->crsqe_tbl[cookie]; + + do { + if (RCFW_NO_FW_ACCESS(rcfw)) + return bnxt_qplib_map_rc(crsqe->opcode); + if (test_bit(FIRMWARE_STALL_DETECTED, &cmdq->flags)) + return -ETIMEDOUT; + + usleep_range(1000, 1001); + + bnxt_qplib_service_creq((unsigned long)rcfw); + if (!crsqe->is_in_used) + return 0; + + if (jiffies_to_msecs(jiffies - issue_time) > + (rcfw->max_timeout * 1000)) { + dev_warn_ratelimited(&rcfw->pdev->dev, + "Self Polling QPLIB: cmdq[%#x]=%#x taken (%lu) msec", + cookie, crsqe->opcode, + (long)jiffies_to_msecs(jiffies - issue_time)); + ret = bnxt_re_is_fw_stalled(rcfw, cookie); + if (ret) + return ret; + } + } while (true); + +}; + +static int __send_message_basic_sanity(struct bnxt_qplib_rcfw *rcfw, + struct bnxt_qplib_cmdqmsg *msg, u8 opcode) +{ + struct bnxt_qplib_cmdq_ctx *cmdq; + + cmdq = &rcfw->cmdq; + + /* Prevent posting if f/w is not in a state to process */ + if (RCFW_NO_FW_ACCESS(rcfw)) + return -ENXIO; + + if (test_bit(FIRMWARE_STALL_DETECTED, &cmdq->flags)) + return -ETIMEDOUT; + + if (test_bit(FIRMWARE_INITIALIZED_FLAG, &cmdq->flags) && + opcode == CMDQ_BASE_OPCODE_INITIALIZE_FW) { + dev_err(&rcfw->pdev->dev, "QPLIB: RCFW already initialized!\n"); + return -EINVAL; + } + + if (!test_bit(FIRMWARE_INITIALIZED_FLAG, &cmdq->flags) && + (opcode != CMDQ_BASE_OPCODE_QUERY_FUNC && + opcode != CMDQ_BASE_OPCODE_INITIALIZE_FW && + opcode != CMDQ_BASE_OPCODE_QUERY_VERSION)) { + dev_err(&rcfw->pdev->dev, + "QPLIB: RCFW not initialized, reject opcode 0x%x\n", + opcode); + return -ENOTSUPP; + } + + return 0; +} + +/* This function will just post and do not bother about completion */ +static void __destroy_timedout_ah(struct bnxt_qplib_rcfw *rcfw, + struct creq_create_ah_resp *create_ah_resp) +{ + struct bnxt_qplib_cmdqmsg msg = {}; + struct cmdq_destroy_ah req = {}; + + bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_DESTROY_AH, + sizeof(req)); + req.ah_cid = create_ah_resp->xid; + msg.req = (struct cmdq_base *)&req; + msg.req_sz = sizeof(req); + __send_message_no_waiter(rcfw, &msg); + dev_warn_ratelimited(&rcfw->pdev->dev, + "From %s: ah_cid = %d timeout_send %d\n", __func__, + req.ah_cid, + atomic_read(&rcfw->timeout_send)); +} + +/** + * __bnxt_qplib_rcfw_send_message - qplib interface to send + * and complete rcfw command. + * @rcfw - rcfw channel instance of rdev + * @msg - qplib message internal + * + * This function does not account shadow queue depth. It will send + * all the command unconditionally as long as send queue is not full. + * + * Returns: + * 0 if command completed by firmware. + * Non zero if the command is not completed by firmware. + */ +static int __bnxt_qplib_rcfw_send_message(struct bnxt_qplib_rcfw *rcfw, + struct bnxt_qplib_cmdqmsg *msg) +{ + struct bnxt_qplib_crsqe *crsqe; + struct creq_qp_event *event; + unsigned long flags; + u16 cookie; + int rc = 0; + u8 opcode; + + opcode = __get_cmdq_base_opcode(msg->req, msg->req_sz); + + rc = __send_message_basic_sanity(rcfw, msg, opcode); + if (rc) + return rc == -ENXIO ? bnxt_qplib_map_rc(opcode) : rc; + + rc = __send_message(rcfw, msg); + if (rc) + return rc; + + cookie = le16_to_cpu(__get_cmdq_base_cookie(msg->req, + msg->req_sz)) & RCFW_MAX_COOKIE_VALUE; + + + if (msg->block) + rc = __block_for_resp(rcfw, cookie); + else if (atomic_read(&rcfw->rcfw_intr_enabled)) + rc = __wait_for_resp(rcfw, cookie); + else + rc = __poll_for_resp(rcfw, cookie); + + if (rc) { + /* First check if it is FW stall. + * Use hwq.lock to avoid race with actual completion. + */ + spin_lock_irqsave(&rcfw->cmdq.hwq.lock, flags); + crsqe = &rcfw->crsqe_tbl[cookie]; + crsqe->is_waiter_alive = false; + if (rc == -ENODEV) + set_bit(FIRMWARE_STALL_DETECTED, &rcfw->cmdq.flags); + spin_unlock_irqrestore(&rcfw->cmdq.hwq.lock, flags); + + return -ETIMEDOUT; + } + + event = (struct creq_qp_event *)msg->resp; + if (event->status) { + /* failed with status */ + dev_err(&rcfw->pdev->dev, "QPLIB: cmdq[%#x]=%#x (%s) status %d\n", + cookie, opcode, GET_OPCODE_TYPE(opcode), event->status); + rc = -EFAULT; + /* + * Workaround to avoid errors in the stack during bond + * creation and deletion. + * Disable error returned for ADD_GID/DEL_GID + */ + if (opcode == CMDQ_BASE_OPCODE_ADD_GID || + opcode == CMDQ_BASE_OPCODE_DELETE_GID) + rc = 0; + } + + dev_dbg(&pdev->dev, "QPLIB: %s:%d - op 0x%x (%s), cookie 0x%x -- Return: e->status 0x%x, rc = 0x%x\n", + __func__, __LINE__, opcode, GET_OPCODE_TYPE(opcode), cookie, event->status, rc); + return rc; +} + +/** + * bnxt_qplib_rcfw_send_message - qplib interface to send + * and complete rcfw command. + * @rcfw - rcfw channel instance of rdev + * @msg - qplib message internal + * + * Driver interact with Firmware through rcfw channel/slow path in two ways. + * a. Blocking rcfw command send. In this path, driver cannot hold + * the context for longer period since it is holding cpu until + * command is not completed. + * b. Non-blocking rcfw command send. In this path, driver can hold the + * context for longer period. There may be many pending command waiting + * for completion because of non-blocking nature. + * + * Driver will use shadow queue depth. Current queue depth of 8K + * (due to size of rcfw message it can be actual ~4K rcfw outstanding) + * is not optimal for rcfw command processing in firmware. + * RCFW_CMD_NON_BLOCKING_SHADOW_QD is defined as 64. + * Restrict at max 64 Non-Blocking rcfw commands. + * Do not allow more than 64 non-blocking command to the Firmware. + * Allow all blocking commands until there is no queue full. + * + * Returns: + * 0 if command completed by firmware. + * Non zero if the command is not completed by firmware. + */ +int bnxt_qplib_rcfw_send_message(struct bnxt_qplib_rcfw *rcfw, + struct bnxt_qplib_cmdqmsg *msg) +{ + int ret; + + if (!msg->block) { + down(&rcfw->rcfw_inflight); + ret = __bnxt_qplib_rcfw_send_message(rcfw, msg); + up(&rcfw->rcfw_inflight); + } else { + ret = __bnxt_qplib_rcfw_send_message(rcfw, msg); + } + + return ret; +} + +static void bnxt_re_add_perf_stats(struct bnxt_qplib_rcfw *rcfw, + struct bnxt_qplib_crsqe *crsqe) +{ + u32 latency_msec, dest_stats_id; + u64 *dest_stats_ptr = NULL; + + latency_msec = jiffies_to_msecs(rcfw->cmdq.last_seen - + crsqe->send_timestamp); + if (latency_msec/1000 < RCFW_MAX_LATENCY_SEC_SLAB_INDEX) + rcfw->rcfw_lat_slab_sec[latency_msec/1000]++; + + if (!rcfw->sp_perf_stats_enabled) + return; + + if (latency_msec < RCFW_MAX_LATENCY_MSEC_SLAB_INDEX) + rcfw->rcfw_lat_slab_msec[latency_msec]++; + + switch (crsqe->opcode) { + case CMDQ_BASE_OPCODE_CREATE_QP: + dest_stats_id = rcfw->qp_create_stats_id++; + dest_stats_id = dest_stats_id % RCFW_MAX_STAT_INDEX; + dest_stats_ptr = &rcfw->qp_create_stats[dest_stats_id]; + break; + case CMDQ_BASE_OPCODE_DESTROY_QP: + dest_stats_id = rcfw->qp_destroy_stats_id++; + dest_stats_id = dest_stats_id % RCFW_MAX_STAT_INDEX; + dest_stats_ptr = &rcfw->qp_destroy_stats[dest_stats_id]; + break; + case CMDQ_BASE_OPCODE_REGISTER_MR: + dest_stats_id = rcfw->mr_create_stats_id++; + dest_stats_id = dest_stats_id % RCFW_MAX_STAT_INDEX; + dest_stats_ptr = &rcfw->mr_create_stats[dest_stats_id]; + break; + case CMDQ_BASE_OPCODE_DEREGISTER_MR: + case CMDQ_BASE_OPCODE_DEALLOCATE_KEY: + dest_stats_id = rcfw->mr_destroy_stats_id++; + dest_stats_id = dest_stats_id % RCFW_MAX_STAT_INDEX; + dest_stats_ptr = &rcfw->mr_destroy_stats[dest_stats_id]; + break; + case CMDQ_BASE_OPCODE_MODIFY_QP: + if (crsqe->requested_qp_state != IB_QPS_ERR) + break; + dest_stats_id = rcfw->qp_modify_stats_id++; + dest_stats_id = dest_stats_id % RCFW_MAX_STAT_INDEX; + dest_stats_ptr = &rcfw->qp_modify_stats[dest_stats_id]; + break; + default: + break; + } + if (dest_stats_ptr) + *dest_stats_ptr = max_t(unsigned long, + (rcfw->cmdq.last_seen - crsqe->send_timestamp), 1); + +} + +/* Completions */ +static int bnxt_qplib_process_qp_event(struct bnxt_qplib_rcfw *rcfw, + struct creq_qp_event *event, + u32 *num_wait) +{ + struct bnxt_qplib_hwq *cmdq_hwq = &rcfw->cmdq.hwq; + struct creq_cq_error_notification *cqerr; + struct creq_qp_error_notification *qperr; + struct bnxt_qplib_crsqe *crsqe; + struct bnxt_qplib_reftbl *tbl; + struct bnxt_qplib_qp *qp; + struct bnxt_qplib_cq *cq; + u16 cookie, blocked = 0; + struct pci_dev *pdev; + bool is_waiter_alive; + unsigned long flags; + u32 wait_cmds = 0; + u32 xid, qp_idx; + u32 req_size; + int rc = 0; + + pdev = rcfw->pdev; + switch (event->event) { + case CREQ_QP_EVENT_EVENT_QP_ERROR_NOTIFICATION: + tbl = &rcfw->res->reftbl.qpref; + qperr = (struct creq_qp_error_notification *)event; + xid = le32_to_cpu(qperr->xid); + qp_idx = map_qp_id_to_tbl_indx(xid, tbl); + spin_lock(&tbl->lock); + qp = tbl->rec[qp_idx].handle; + if (!qp) { + spin_unlock(&tbl->lock); + break; + } + bnxt_qplib_mark_qp_error(qp); + rc = rcfw->creq.aeq_handler(rcfw, event, qp); + spin_unlock(&tbl->lock); + /* + * Keeping these prints as debug to avoid flooding of log + * messages during modify QP to error state by applications + */ + dev_dbg(&pdev->dev, "QPLIB: QP Error encountered!\n"); + dev_dbg(&pdev->dev, + "QPLIB: qpid 0x%x, req_err=0x%x, resp_err=0x%x\n", + xid, qperr->req_err_state_reason, + qperr->res_err_state_reason); + break; + case CREQ_QP_EVENT_EVENT_CQ_ERROR_NOTIFICATION: + tbl = &rcfw->res->reftbl.cqref; + cqerr = (struct creq_cq_error_notification *)event; + xid = le32_to_cpu(cqerr->xid); + spin_lock(&tbl->lock); + cq = tbl->rec[GET_TBL_INDEX(xid, tbl)].handle; + if (!cq) { + spin_unlock(&tbl->lock); + break; + } + rc = rcfw->creq.aeq_handler(rcfw, event, cq); + spin_unlock(&tbl->lock); + dev_dbg(&pdev->dev, "QPLIB: CQ error encountered!\n"); + break; + default: + /* + * Command Response + * cmdq hwq lock needs to be acquired to synchronize + * the command send and completion reaping. This function + * is always called with creq hwq lock held. So there is no + * chance of deadlock here as the locking is in correct sequence. + * Using the nested variant of spin_lock to annotate + */ + spin_lock_irqsave_nested(&cmdq_hwq->lock, flags, + SINGLE_DEPTH_NESTING); + cookie = le16_to_cpu(event->cookie); + blocked = cookie & RCFW_CMD_IS_BLOCKING; + cookie &= RCFW_MAX_COOKIE_VALUE; + + crsqe = &rcfw->crsqe_tbl[cookie]; + + bnxt_re_add_perf_stats(rcfw, crsqe); + + if (WARN_ONCE(test_bit(FIRMWARE_STALL_DETECTED, + &rcfw->cmdq.flags), + "QPLIB: Unreponsive rcfw channel detected.!!")) { + dev_info(&pdev->dev, "rcfw timedout: cookie = %#x," + " latency_msec = %ld free_slots = %d\n", cookie, + (long)jiffies_to_msecs(rcfw->cmdq.last_seen - + crsqe->send_timestamp), + crsqe->free_slots); + spin_unlock_irqrestore(&cmdq_hwq->lock, flags); + return rc; + } + + if (crsqe->is_internal_cmd && !event->status) + atomic_dec(&rcfw->timeout_send); + + if (crsqe->is_waiter_alive) { + if (crsqe->resp) + memcpy(crsqe->resp, event, sizeof(*event)); + if (!blocked) + wait_cmds++; + } + + req_size = crsqe->req_size; + is_waiter_alive = crsqe->is_waiter_alive; + + crsqe->req_size = 0; + if (!crsqe->is_waiter_alive) + crsqe->resp = NULL; + crsqe->is_in_used = false; + /* Consumer is updated so that __send_message_no_waiter + * can never see queue full. + * It is safe since we are still holding cmdq_hwq->lock. + */ + cmdq_hwq->cons += req_size; + + /* This is a case to handle below scenario - + * Create AH is completed successfully by firmware, + * but completion took more time and driver already lost + * the context of create_ah from caller. + * We have already return failure for create_ah verbs, + * so let's destroy the same address vector since it is + * no more used in stack. We don't care about completion + * in __send_message_no_waiter. + * If destroy_ah is failued by firmware, there will be AH + * resource leak and relatively not critical + unlikely + * scenario. Current design is not to handle such case. + */ + if (!is_waiter_alive && !event->status && + event->event == CREQ_QP_EVENT_EVENT_CREATE_AH) + __destroy_timedout_ah(rcfw, + (struct creq_create_ah_resp *) + event); + + spin_unlock_irqrestore(&cmdq_hwq->lock, flags); + } + *num_wait += wait_cmds; + return rc; +} + +/* SP - CREQ Completion handlers */ +static void bnxt_qplib_service_creq(unsigned long data) +{ + struct bnxt_qplib_rcfw *rcfw = (struct bnxt_qplib_rcfw *)data; + struct bnxt_qplib_creq_ctx *creq = &rcfw->creq; + struct bnxt_qplib_res *res; + u32 type, budget = CREQ_ENTRY_POLL_BUDGET; + struct bnxt_qplib_hwq *creq_hwq = &creq->hwq; + struct creq_base *creqe; + struct pci_dev *pdev; + unsigned long flags; + u32 num_wakeup = 0; + int rc; + + pdev = rcfw->pdev; + res = rcfw->res; + /* Service the CREQ until empty */ + spin_lock_irqsave(&creq_hwq->lock, flags); + while (budget > 0) { + if (RCFW_NO_FW_ACCESS(rcfw)) { + spin_unlock_irqrestore(&creq_hwq->lock, flags); + return; + } + creqe = bnxt_qplib_get_qe(creq_hwq, creq_hwq->cons, NULL); + if (!CREQ_CMP_VALID(creqe, creq->creq_db.dbinfo.flags)) + break; + /* The valid test of the entry must be done first before + * reading any further. + */ + dma_rmb(); + type = creqe->type & CREQ_BASE_TYPE_MASK; + rcfw->cmdq.last_seen = jiffies; + + switch (type) { + case CREQ_BASE_TYPE_QP_EVENT: + bnxt_qplib_process_qp_event + (rcfw,(struct creq_qp_event *)creqe, + &num_wakeup); + creq->stats.creq_qp_event_processed++; + break; + case CREQ_BASE_TYPE_FUNC_EVENT: + rc = rcfw->creq.aeq_handler(rcfw, creqe, NULL); + if (rc) + dev_warn(&pdev->dev, + "QPLIB: async event type = 0x%x not handled", + type); + creq->stats.creq_func_event_processed++; + break; + default: + if (type != HWRM_ASYNC_EVENT_CMPL_TYPE_HWRM_ASYNC_EVENT) { + dev_warn(&pdev->dev, + "QPLIB: op_event = 0x%x not handled\n", + type); + } + break; + } + budget--; + bnxt_qplib_hwq_incr_cons(creq_hwq->max_elements, &creq_hwq->cons, + 1, &creq->creq_db.dbinfo.flags); + } + if (budget == CREQ_ENTRY_POLL_BUDGET && + !CREQ_CMP_VALID(creqe, creq->creq_db.dbinfo.flags)) { + /* No completions received during this poll. Enable interrupt now */ + bnxt_qplib_ring_nq_db(&creq->creq_db.dbinfo, res->cctx, true); + creq->stats.creq_arm_count++; + dev_dbg(&pdev->dev, "QPLIB: Num of Func (0x%llx) \n", + creq->stats.creq_func_event_processed); + dev_dbg(&pdev->dev, "QPLIB: QP (0x%llx) events processed\n", + creq->stats.creq_qp_event_processed); + dev_dbg(&pdev->dev, "QPLIB: Armed:%#llx resched:%#llx \n", + creq->stats.creq_arm_count, + creq->stats.creq_tasklet_schedule_count); + } else if (creq->requested) { + /* + * Currently there is no bottom half implementation to process + * completions, all completions are processed in interrupt context + * only. So enable interrupts. + */ + bnxt_qplib_ring_nq_db(&creq->creq_db.dbinfo, res->cctx, true); + creq->stats.creq_tasklet_schedule_count++; + } + spin_unlock_irqrestore(&creq_hwq->lock, flags); + if (num_wakeup) + wake_up_all(&rcfw->cmdq.waitq); +} + +static irqreturn_t bnxt_qplib_creq_irq(int irq, void *dev_instance) +{ + struct bnxt_qplib_rcfw *rcfw = dev_instance; + + bnxt_qplib_service_creq((unsigned long)rcfw); + return IRQ_HANDLED; +} + +/* RCFW */ +int bnxt_qplib_deinit_rcfw(struct bnxt_qplib_rcfw *rcfw) +{ + struct creq_deinitialize_fw_resp resp = {}; + struct cmdq_deinitialize_fw req = {}; + struct bnxt_qplib_cmdqmsg msg = {}; + int rc; + + bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_DEINITIALIZE_FW, + sizeof(req)); + bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL, + sizeof(req), sizeof(resp), 0); + rc = bnxt_qplib_rcfw_send_message(rcfw, &msg); + if (rc) + return rc; + clear_bit(FIRMWARE_INITIALIZED_FLAG, &rcfw->cmdq.flags); + return 0; +} + +int bnxt_qplib_init_rcfw(struct bnxt_qplib_rcfw *rcfw, int is_virtfn) +{ + struct creq_initialize_fw_resp resp = {}; + struct cmdq_initialize_fw req = {}; + struct bnxt_qplib_cmdqmsg msg = {}; + struct bnxt_qplib_chip_ctx *cctx; + struct bnxt_qplib_ctx *hctx; + struct bnxt_qplib_res *res; + struct bnxt_qplib_hwq *hwq; + int rc; + + res = rcfw->res; + cctx = res->cctx; + hctx = res->hctx; + + bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_INITIALIZE_FW, + sizeof(req)); + /* Supply (log-base-2-of-host-page-size - base-page-shift) + * to bono to adjust the doorbell page sizes. + */ + req.log2_dbr_pg_size = cpu_to_le16(PAGE_SHIFT - + RCFW_DBR_BASE_PAGE_SHIFT); + /* + * VFs need not setup the HW context area, PF + * shall setup this area for VF. Skipping the + * HW programming + */ + if (is_virtfn || _is_chip_gen_p5_p7(cctx)) + goto skip_ctx_setup; + + hwq = &hctx->qp_ctx.hwq; + req.qpc_page_dir = cpu_to_le64(_get_base_addr(hwq)); + req.number_of_qp = cpu_to_le32(hwq->max_elements); + req.qpc_pg_size_qpc_lvl = (_get_pte_pg_size(hwq) << + CMDQ_INITIALIZE_FW_QPC_PG_SIZE_SFT) | + (u8)hwq->level; + + hwq = &hctx->mrw_ctx.hwq; + req.mrw_page_dir = cpu_to_le64(_get_base_addr(hwq)); + req.number_of_mrw = cpu_to_le32(hwq->max_elements); + req.mrw_pg_size_mrw_lvl = (_get_pte_pg_size(hwq) << + CMDQ_INITIALIZE_FW_MRW_PG_SIZE_SFT) | + (u8)hwq->level; + + hwq = &hctx->srq_ctx.hwq; + req.srq_page_dir = cpu_to_le64(_get_base_addr(hwq)); + req.number_of_srq = cpu_to_le32(hwq->max_elements); + req.srq_pg_size_srq_lvl = (_get_pte_pg_size(hwq) << + CMDQ_INITIALIZE_FW_SRQ_PG_SIZE_SFT) | + (u8)hwq->level; + + hwq = &hctx->cq_ctx.hwq; + req.cq_page_dir = cpu_to_le64(_get_base_addr(hwq)); + req.number_of_cq = cpu_to_le32(hwq->max_elements); + req.cq_pg_size_cq_lvl = (_get_pte_pg_size(hwq) << + CMDQ_INITIALIZE_FW_CQ_PG_SIZE_SFT) | + (u8)hwq->level; + + hwq = &hctx->tim_ctx.hwq; + req.tim_page_dir = cpu_to_le64(_get_base_addr(hwq)); + req.tim_pg_size_tim_lvl = (_get_pte_pg_size(hwq) << + CMDQ_INITIALIZE_FW_TIM_PG_SIZE_SFT) | + (u8)hwq->level; + hwq = &hctx->tqm_ctx.pde; + req.tqm_page_dir = cpu_to_le64(_get_base_addr(hwq)); + req.tqm_pg_size_tqm_lvl = (_get_pte_pg_size(hwq) << + CMDQ_INITIALIZE_FW_TQM_PG_SIZE_SFT) | + (u8)hwq->level; +skip_ctx_setup: + if (BNXT_RE_HW_RETX(res->dattr->dev_cap_flags)) + req.flags |= CMDQ_INITIALIZE_FW_FLAGS_HW_REQUESTER_RETX_SUPPORTED; + req.stat_ctx_id = cpu_to_le32(hctx->stats.fw_id); + bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL, + sizeof(req), sizeof(resp), 0); + rc = bnxt_qplib_rcfw_send_message(rcfw, &msg); + if (rc) + return rc; + set_bit(FIRMWARE_INITIALIZED_FLAG, &rcfw->cmdq.flags); + + return 0; +} + +void bnxt_qplib_free_rcfw_channel(struct bnxt_qplib_res *res) +{ + struct bnxt_qplib_rcfw *rcfw = res->rcfw; + + vfree(rcfw->rcfw_lat_slab_msec); + rcfw->rcfw_lat_slab_msec = NULL; + vfree(rcfw->qp_create_stats); + rcfw->qp_create_stats = NULL; + vfree(rcfw->qp_destroy_stats); + rcfw->qp_destroy_stats = NULL; + vfree(rcfw->mr_create_stats); + rcfw->mr_create_stats = NULL; + vfree(rcfw->mr_destroy_stats); + rcfw->mr_destroy_stats = NULL; + vfree(rcfw->qp_modify_stats); + rcfw->qp_modify_stats = NULL; + rcfw->sp_perf_stats_enabled = false; + + kfree(rcfw->crsqe_tbl); + rcfw->crsqe_tbl = NULL; + + bnxt_qplib_free_hwq(res, &rcfw->cmdq.hwq); + bnxt_qplib_free_hwq(res, &rcfw->creq.hwq); + rcfw->pdev = NULL; +} + +int bnxt_qplib_alloc_rcfw_channel(struct bnxt_qplib_res *res) +{ + struct bnxt_qplib_hwq_attr hwq_attr = {}; + struct bnxt_qplib_rcfw *rcfw = res->rcfw; + struct bnxt_qplib_sg_info sginfo = {}; + struct bnxt_qplib_cmdq_ctx *cmdq; + struct bnxt_qplib_creq_ctx *creq; + + rcfw->pdev = res->pdev; + rcfw->res = res; + cmdq = &rcfw->cmdq; + creq = &rcfw->creq; + + sginfo.pgsize = PAGE_SIZE; + sginfo.pgshft = PAGE_SHIFT; + + hwq_attr.sginfo = &sginfo; + hwq_attr.res = rcfw->res; + hwq_attr.depth = BNXT_QPLIB_CREQE_MAX_CNT; + hwq_attr.stride = BNXT_QPLIB_CREQE_UNITS; + hwq_attr.type = _get_hwq_type(res); + + if (bnxt_qplib_alloc_init_hwq(&creq->hwq, &hwq_attr)) { + dev_err(&rcfw->pdev->dev, + "QPLIB: HW channel CREQ allocation failed\n"); + return -ENOMEM; + } + + sginfo.pgsize = BNXT_QPLIB_CMDQE_PAGE_SIZE; + hwq_attr.depth = BNXT_QPLIB_CMDQE_MAX_CNT & 0x7FFFFFFF; + hwq_attr.stride = BNXT_QPLIB_CMDQE_UNITS; + hwq_attr.type = HWQ_TYPE_CTX; + if (bnxt_qplib_alloc_init_hwq(&cmdq->hwq, &hwq_attr)) { + dev_err(&rcfw->pdev->dev, + "QPLIB: HW channel CMDQ allocation failed\n"); + goto fail_free_creq_hwq; + } + + rcfw->crsqe_tbl = kcalloc(cmdq->hwq.max_elements, + sizeof(*rcfw->crsqe_tbl), GFP_KERNEL); + if (!rcfw->crsqe_tbl) { + dev_err(&rcfw->pdev->dev, + "QPLIB: HW channel CRSQ allocation failed\n"); + goto fail_free_cmdq_hwq; + } + + rcfw->max_timeout = res->cctx->hwrm_cmd_max_timeout; + + rcfw->sp_perf_stats_enabled = false; + rcfw->rcfw_lat_slab_msec = vzalloc(sizeof(u32) * + RCFW_MAX_LATENCY_MSEC_SLAB_INDEX); + rcfw->qp_create_stats = vzalloc(sizeof(u64) * RCFW_MAX_STAT_INDEX); + rcfw->qp_destroy_stats = vzalloc(sizeof(u64) * RCFW_MAX_STAT_INDEX); + rcfw->mr_create_stats = vzalloc(sizeof(u64) * RCFW_MAX_STAT_INDEX); + rcfw->mr_destroy_stats = vzalloc(sizeof(u64) * RCFW_MAX_STAT_INDEX); + rcfw->qp_modify_stats = vzalloc(sizeof(u64) * RCFW_MAX_STAT_INDEX); + + if (rcfw->rcfw_lat_slab_msec && + rcfw->qp_create_stats && + rcfw->qp_destroy_stats && + rcfw->mr_create_stats && + rcfw->mr_destroy_stats && + rcfw->qp_modify_stats) + rcfw->sp_perf_stats_enabled = true; + + return 0; +fail_free_cmdq_hwq: + bnxt_qplib_free_hwq(res, &rcfw->cmdq.hwq); +fail_free_creq_hwq: + bnxt_qplib_free_hwq(res, &rcfw->creq.hwq); + return -ENOMEM; +} + +void bnxt_qplib_rcfw_stop_irq(struct bnxt_qplib_rcfw *rcfw, bool kill) +{ + struct bnxt_qplib_creq_ctx *creq; + struct bnxt_qplib_res *res; + + creq = &rcfw->creq; + res = rcfw->res; + + if (!creq->requested) + return; + + creq->requested = false; + /* Mask h/w interrupts */ + bnxt_qplib_ring_nq_db(&creq->creq_db.dbinfo, res->cctx, false); + /* Sync with last running IRQ-handler */ + synchronize_irq(creq->msix_vec); + free_irq(creq->msix_vec, rcfw); + kfree(creq->irq_name); + creq->irq_name = NULL; + /* rcfw_intr_enabled should not be greater than 1. Debug + * print to check if that is the case + */ + if (atomic_read(&rcfw->rcfw_intr_enabled) > 1) { + dev_err(&rcfw->pdev->dev, + "%s: rcfw->rcfw_intr_enabled = 0x%x\n", __func__, + atomic_read(&rcfw->rcfw_intr_enabled)); + } + atomic_set(&rcfw->rcfw_intr_enabled, 0); + rcfw->num_irq_stopped++; +} + +void bnxt_qplib_disable_rcfw_channel(struct bnxt_qplib_rcfw *rcfw) +{ + struct bnxt_qplib_creq_ctx *creq; + struct bnxt_qplib_cmdq_ctx *cmdq; + + creq = &rcfw->creq; + cmdq = &rcfw->cmdq; + /* Make sure the HW channel is stopped! */ + bnxt_qplib_rcfw_stop_irq(rcfw, true); + + creq->creq_db.reg.bar_reg = NULL; + creq->creq_db.db = NULL; + + if (cmdq->cmdq_mbox.reg.bar_reg) { + iounmap(cmdq->cmdq_mbox.reg.bar_reg); + cmdq->cmdq_mbox.reg.bar_reg = NULL; + cmdq->cmdq_mbox.prod = NULL; + cmdq->cmdq_mbox.db = NULL; + } + + creq->aeq_handler = NULL; + creq->msix_vec = 0; +} + +int bnxt_qplib_rcfw_start_irq(struct bnxt_qplib_rcfw *rcfw, int msix_vector, + bool need_init) +{ + struct bnxt_qplib_creq_ctx *creq; + struct bnxt_qplib_res *res; + int rc; + + creq = &rcfw->creq; + res = rcfw->res; + + if (creq->requested) + return -EFAULT; + + creq->msix_vec = msix_vector; + + creq->irq_name = kasprintf(GFP_KERNEL, "bnxt_re-creq@pci:%s\n", + pci_name(res->pdev)); + if (!creq->irq_name) + return -ENOMEM; + + rc = request_irq(creq->msix_vec, bnxt_qplib_creq_irq, 0, + creq->irq_name, rcfw); + if (rc) { + kfree(creq->irq_name); + creq->irq_name = NULL; + return rc; + } + creq->requested = true; + + bnxt_qplib_ring_nq_db(&creq->creq_db.dbinfo, res->cctx, true); + + rcfw->num_irq_started++; + /* Debug print to check rcfw interrupt enable/disable is invoked + * out of sequence + */ + if (atomic_read(&rcfw->rcfw_intr_enabled) > 0) { + dev_err(&rcfw->pdev->dev, + "%s: rcfw->rcfw_intr_enabled = 0x%x\n", __func__, + atomic_read(&rcfw->rcfw_intr_enabled)); + } + atomic_inc(&rcfw->rcfw_intr_enabled); + return 0; +} + +static int bnxt_qplib_map_cmdq_mbox(struct bnxt_qplib_rcfw *rcfw) +{ + struct bnxt_qplib_cmdq_mbox *mbox; + resource_size_t bar_reg; + struct pci_dev *pdev; + + pdev = rcfw->pdev; + mbox = &rcfw->cmdq.cmdq_mbox; + + mbox->reg.bar_id = RCFW_COMM_PCI_BAR_REGION; + mbox->reg.len = RCFW_COMM_SIZE; + mbox->reg.bar_base = pci_resource_start(pdev, mbox->reg.bar_id); + if (!mbox->reg.bar_base) { + dev_err(&pdev->dev, + "QPLIB: CMDQ BAR region %d resc start is 0!\n", + mbox->reg.bar_id); + return -ENOMEM; + } + + bar_reg = mbox->reg.bar_base + RCFW_COMM_BASE_OFFSET; + mbox->reg.len = RCFW_COMM_SIZE; + mbox->reg.bar_reg = ioremap(bar_reg, mbox->reg.len); + if (!mbox->reg.bar_reg) { + dev_err(&pdev->dev, + "QPLIB: CMDQ BAR region %d mapping failed\n", + mbox->reg.bar_id); + return -ENOMEM; + } + + mbox->prod = (void __iomem *)((char *)mbox->reg.bar_reg + + RCFW_PF_VF_COMM_PROD_OFFSET); + mbox->db = (void __iomem *)((char *)mbox->reg.bar_reg + + RCFW_COMM_TRIG_OFFSET); + return 0; +} + +static int bnxt_qplib_map_creq_db(struct bnxt_qplib_rcfw *rcfw, u32 reg_offt) +{ + struct bnxt_qplib_creq_db *creq_db; + struct bnxt_qplib_reg_desc *dbreg; + struct bnxt_qplib_res *res; + + res = rcfw->res; + creq_db = &rcfw->creq.creq_db; + dbreg = &res->dpi_tbl.ucreg; + + creq_db->reg.bar_id = dbreg->bar_id; + creq_db->reg.bar_base = dbreg->bar_base; + creq_db->reg.bar_reg = dbreg->bar_reg + reg_offt; + creq_db->reg.len = _is_chip_gen_p5_p7(res->cctx) ? sizeof(u64) : + sizeof(u32); + + creq_db->dbinfo.db = creq_db->reg.bar_reg; + creq_db->dbinfo.hwq = &rcfw->creq.hwq; + creq_db->dbinfo.xid = rcfw->creq.ring_id; + creq_db->dbinfo.seed = rcfw->creq.ring_id; + creq_db->dbinfo.flags = 0; + spin_lock_init(&creq_db->dbinfo.lock); + creq_db->dbinfo.shadow_key = BNXT_QPLIB_DBR_KEY_INVALID; + creq_db->dbinfo.res = rcfw->res; + + return 0; +} + +static void bnxt_qplib_start_rcfw(struct bnxt_qplib_rcfw *rcfw) +{ + struct bnxt_qplib_cmdq_ctx *cmdq; + struct bnxt_qplib_creq_ctx *creq; + struct bnxt_qplib_cmdq_mbox *mbox; + struct cmdq_init init = {0}; + + cmdq = &rcfw->cmdq; + creq = &rcfw->creq; + mbox = &cmdq->cmdq_mbox; + + init.cmdq_pbl = cpu_to_le64(cmdq->hwq.pbl[PBL_LVL_0].pg_map_arr[0]); + init.cmdq_size_cmdq_lvl = cpu_to_le16( + ((BNXT_QPLIB_CMDQE_MAX_CNT << CMDQ_INIT_CMDQ_SIZE_SFT) & + CMDQ_INIT_CMDQ_SIZE_MASK) | + ((cmdq->hwq.level << CMDQ_INIT_CMDQ_LVL_SFT) & + CMDQ_INIT_CMDQ_LVL_MASK)); + init.creq_ring_id = cpu_to_le16(creq->ring_id); + /* Write to the Bono mailbox register */ + __iowrite32_copy(mbox->reg.bar_reg, &init, sizeof(init) / 4); +} + +int bnxt_qplib_enable_rcfw_channel(struct bnxt_qplib_rcfw *rcfw, + int msix_vector, + int cp_bar_reg_off, + aeq_handler_t aeq_handler) +{ + struct bnxt_qplib_cmdq_ctx *cmdq; + struct bnxt_qplib_creq_ctx *creq; + int rc; + + cmdq = &rcfw->cmdq; + creq = &rcfw->creq; + + /* Clear to defaults */ + cmdq->seq_num = 0; + set_bit(FIRMWARE_FIRST_FLAG, &cmdq->flags); + init_waitqueue_head(&cmdq->waitq); + + creq->stats.creq_qp_event_processed = 0; + creq->stats.creq_func_event_processed = 0; + creq->aeq_handler = aeq_handler; + + rc = bnxt_qplib_map_cmdq_mbox(rcfw); + if (rc) + return rc; + + rc = bnxt_qplib_map_creq_db(rcfw, cp_bar_reg_off); + if (rc) + return rc; + + rc = bnxt_qplib_rcfw_start_irq(rcfw, msix_vector, true); + if (rc) { + dev_err(&rcfw->pdev->dev, + "QPLIB: Failed to request IRQ for CREQ rc = 0x%x\n", rc); + bnxt_qplib_disable_rcfw_channel(rcfw); + return rc; + } + + rcfw->curr_shadow_qd = min_not_zero(cmdq_shadow_qd, + (unsigned int)RCFW_CMD_NON_BLOCKING_SHADOW_QD); + sema_init(&rcfw->rcfw_inflight, rcfw->curr_shadow_qd); + dev_dbg(&rcfw->pdev->dev, + "Perf Debug: shadow qd %d\n", rcfw->curr_shadow_qd); + bnxt_qplib_start_rcfw(rcfw); + + return 0; +} diff --git a/sys/dev/bnxt/bnxt_re/qplib_rcfw.h b/sys/dev/bnxt/bnxt_re/qplib_rcfw.h new file mode 100644 index 000000000000..f117525daacb --- /dev/null +++ b/sys/dev/bnxt/bnxt_re/qplib_rcfw.h @@ -0,0 +1,354 @@ +/* + * Copyright (c) 2015-2024, Broadcom. All rights reserved. The term + * Broadcom refers to Broadcom Limited and/or its subsidiaries. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR + * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE + * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN + * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * Description: RDMA Controller HW interface (header) + */ + +#ifndef __BNXT_QPLIB_RCFW_H__ +#define __BNXT_QPLIB_RCFW_H__ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "qplib_tlv.h" + +#define RCFW_CMDQ_TRIG_VAL 1 +#define RCFW_COMM_PCI_BAR_REGION 0 +#define RCFW_COMM_CONS_PCI_BAR_REGION 2 +#define RCFW_COMM_BASE_OFFSET 0x600 +#define RCFW_PF_VF_COMM_PROD_OFFSET 0xc +#define RCFW_COMM_TRIG_OFFSET 0x100 +#define RCFW_COMM_SIZE 0x104 + +#define RCFW_DBR_PCI_BAR_REGION 2 +#define RCFW_DBR_BASE_PAGE_SHIFT 12 +#define RCFW_MAX_LATENCY_SEC_SLAB_INDEX 128 +#define RCFW_MAX_LATENCY_MSEC_SLAB_INDEX 3000 +#define RCFW_MAX_STAT_INDEX 0xFFFF +#define RCFW_FW_STALL_MAX_TIMEOUT 40 + +#define GET_OPCODE_TYPE(x) \ + ((x) == 0x1 ? "CREATE_QP": \ + ((x) == 0x2 ? "DESTROY_QP": \ + ((x) == 0x3 ? "MODIFY_QP": \ + ((x) == 0x4 ? "QUERY_QP": \ + ((x) == 0x5 ? "CREATE_SRQ": \ + ((x) == 0x6 ? "DESTROY_SRQ": \ + ((x) == 0x8 ? "QUERY_SRQ": \ + ((x) == 0x9 ? "CREATE_CQ": \ + ((x) == 0xa ? "DESTROY_CQ": \ + ((x) == 0xc ? "RESIZE_CQ": \ + ((x) == 0xd ? "ALLOCATE_MRW": \ + ((x) == 0xe ? "DEALLOCATE_KEY": \ + ((x) == 0xf ? "REGISTER_MR": \ + ((x) == 0x10 ? "DEREGISTER_MR": \ + ((x) == 0x11 ? "ADD_GID": \ + ((x) == 0x12 ? "DELETE_GID": \ + ((x) == 0x17 ? "MODIFY_GID": \ + ((x) == 0x18 ? "QUERY_GID": \ + ((x) == 0x13 ? "CREATE_QP1": \ + ((x) == 0x14 ? "DESTROY_QP1": \ + ((x) == 0x15 ? "CREATE_AH": \ + ((x) == 0x16 ? "DESTROY_AH": \ + ((x) == 0x80 ? "INITIALIZE_FW": \ + ((x) == 0x81 ? "DEINITIALIZE_FW": \ + ((x) == 0x82 ? "STOP_FUNC": \ + ((x) == 0x83 ? "QUERY_FUNC": \ + ((x) == 0x84 ? "SET_FUNC_RESOURCES": \ + ((x) == 0x85 ? "READ_CONTEXT": \ + ((x) == 0x86 ? "VF_BACKCHANNEL_REQUEST": \ + ((x) == 0x87 ? "READ_VF_MEMORY": \ + ((x) == 0x88 ? "COMPLETE_VF_REQUEST": \ + ((x) == 0x89 ? "EXTEND_CONTEXT_ARRRAY": \ + ((x) == 0x8a ? "MAP_TC_TO_COS": \ + ((x) == 0x8b ? "QUERY_VERSION": \ + ((x) == 0x8c ? "MODIFY_ROCE_CC": \ + ((x) == 0x8d ? "QUERY_ROCE_CC": \ + ((x) == 0x8e ? "QUERY_ROCE_STATS": \ + ((x) == 0x8f ? "SET_LINK_AGGR_MODE": \ + ((x) == 0x90 ? "MODIFY_CQ": \ + ((x) == 0x91 ? "QUERY_QP_EXTEND": \ + ((x) == 0x92 ? "QUERY_ROCE_STATS_EXT": \ + "Unknown OPCODE" \ + ))))))))))))))))))))))))))))))))))))))))) + +extern unsigned int cmdq_shadow_qd; +/* Cmdq contains a fix number of a 16-Byte slots */ +struct bnxt_qplib_cmdqe { + u8 data[16]; +}; +#define BNXT_QPLIB_CMDQE_UNITS sizeof(struct bnxt_qplib_cmdqe) + +static inline void bnxt_qplib_rcfw_cmd_prep(void *r, u8 opcode, u8 cmd_size) +{ + struct cmdq_base *req = r; + + req->opcode = opcode; + req->cmd_size = cmd_size; +} + +/* Shadow queue depth for non blocking command */ +#define RCFW_CMD_NON_BLOCKING_SHADOW_QD 64 +#define RCFW_CMD_DEV_ERR_CHECK_TIME_MS 1000 /* 1 Second time out*/ +#define RCFW_ERR_RETRY_COUNT (RCFW_CMD_WAIT_TIME_MS / RCFW_CMD_DEV_ERR_CHECK_TIME_MS) + +/* CMDQ elements */ +#define BNXT_QPLIB_CMDQE_MAX_CNT 8192 +#define BNXT_QPLIB_CMDQE_BYTES (BNXT_QPLIB_CMDQE_MAX_CNT * \ + BNXT_QPLIB_CMDQE_UNITS) +#define BNXT_QPLIB_CMDQE_NPAGES ((BNXT_QPLIB_CMDQE_BYTES % \ + PAGE_SIZE) ? \ + ((BNXT_QPLIB_CMDQE_BYTES / \ + PAGE_SIZE) + 1) : \ + (BNXT_QPLIB_CMDQE_BYTES / \ + PAGE_SIZE)) +#define BNXT_QPLIB_CMDQE_PAGE_SIZE (BNXT_QPLIB_CMDQE_NPAGES * \ + PAGE_SIZE) + +#define RCFW_MAX_OUTSTANDING_CMD BNXT_QPLIB_CMDQE_MAX_CNT +#define RCFW_MAX_COOKIE_VALUE (BNXT_QPLIB_CMDQE_MAX_CNT - 1) +#define RCFW_CMD_IS_BLOCKING 0x8000 +#define RCFW_NO_FW_ACCESS(rcfw) \ + (test_bit(ERR_DEVICE_DETACHED, &(rcfw)->cmdq.flags) || \ + pci_channel_offline((rcfw)->pdev)) + +/* Crsq buf is 1024-Byte */ +struct bnxt_qplib_crsbe { + u8 data[1024]; +}; + +/* Get the number of command units required for the req. The + * function returns correct value only if called before + * setting using bnxt_qplib_set_cmd_slots + */ +static inline u32 bnxt_qplib_get_cmd_slots(struct cmdq_base *req) +{ + u32 cmd_units = 0; + + if (HAS_TLV_HEADER(req)) { + struct roce_tlv *tlv_req = (struct roce_tlv *)req; + cmd_units = tlv_req->total_size; + } else { + cmd_units = (req->cmd_size + BNXT_QPLIB_CMDQE_UNITS - 1) / + BNXT_QPLIB_CMDQE_UNITS; + } + return cmd_units; +} + +/* Set the cmd_size to a factor of CMDQE unit */ +static inline u32 bnxt_qplib_set_cmd_slots(struct cmdq_base *req) +{ + u32 cmd_byte = 0; + + if (HAS_TLV_HEADER(req)) { + struct roce_tlv *tlv_req = (struct roce_tlv *)req; + cmd_byte = tlv_req->total_size * BNXT_QPLIB_CMDQE_UNITS; + } else { + cmd_byte = req->cmd_size; + req->cmd_size = (req->cmd_size + BNXT_QPLIB_CMDQE_UNITS - 1) / + BNXT_QPLIB_CMDQE_UNITS; + } + + return cmd_byte; +} + +/* CREQ */ +/* Allocate 1 per QP for async error notification for now */ +#define BNXT_QPLIB_CREQE_MAX_CNT (64 * 1024) +#define BNXT_QPLIB_CREQE_UNITS 16 /* 16-Bytes per prod unit */ + +#define CREQ_CMP_VALID(hdr, pass) \ + (!!((hdr)->v & CREQ_BASE_V) == \ + !(pass & BNXT_QPLIB_FLAG_EPOCH_CONS_MASK)) + +#define CREQ_ENTRY_POLL_BUDGET 8 + +typedef int (*aeq_handler_t)(struct bnxt_qplib_rcfw *, void *, void *); + +struct bnxt_qplib_crsqe { + struct creq_qp_event *resp; + u32 req_size; + bool is_waiter_alive; + bool is_internal_cmd; + bool is_in_used; + + /* Free slots at the time of submission */ + u32 free_slots; + unsigned long send_timestamp; + u8 opcode; + u8 requested_qp_state; +}; + +struct bnxt_qplib_rcfw_sbuf { + void *sb; + dma_addr_t dma_addr; + u32 size; +}; + +#define BNXT_QPLIB_OOS_COUNT_MASK 0xFFFFFFFF + +#define FIRMWARE_INITIALIZED_FLAG (0) +#define FIRMWARE_FIRST_FLAG (31) +#define FIRMWARE_STALL_DETECTED (3) +#define ERR_DEVICE_DETACHED (4) +struct bnxt_qplib_cmdq_mbox { + struct bnxt_qplib_reg_desc reg; + void __iomem *prod; + void __iomem *db; +}; + +struct bnxt_qplib_cmdq_ctx { + struct bnxt_qplib_hwq hwq; + struct bnxt_qplib_cmdq_mbox cmdq_mbox; + wait_queue_head_t waitq; + unsigned long flags; + unsigned long last_seen; + u32 seq_num; +}; + +struct bnxt_qplib_creq_db { + struct bnxt_qplib_reg_desc reg; + void __iomem *db; + struct bnxt_qplib_db_info dbinfo; +}; + +struct bnxt_qplib_creq_stat { + u64 creq_arm_count; + u64 creq_tasklet_schedule_count; + u64 creq_qp_event_processed; + u64 creq_func_event_processed; +}; + +struct bnxt_qplib_creq_ctx { + struct bnxt_qplib_hwq hwq; + struct bnxt_qplib_creq_db creq_db; + struct bnxt_qplib_creq_stat stats; + aeq_handler_t aeq_handler; + u16 ring_id; + int msix_vec; + bool requested; + char *irq_name; +}; + +/* RCFW Communication Channels */ +#define BNXT_QPLIB_RCFW_SEND_RETRY_COUNT 4000 +struct bnxt_qplib_rcfw { + struct pci_dev *pdev; + struct bnxt_qplib_res *res; + struct bnxt_qplib_cmdq_ctx cmdq; + struct bnxt_qplib_creq_ctx creq; + struct bnxt_qplib_crsqe *crsqe_tbl; + u32 rcfw_lat_slab_sec[RCFW_MAX_LATENCY_SEC_SLAB_INDEX]; + + /* Slow path Perf Stats */ + bool sp_perf_stats_enabled; + u32 *rcfw_lat_slab_msec; + u64 *qp_create_stats; + u64 *qp_destroy_stats; + u64 *qp_modify_stats; + u64 *mr_create_stats; + u64 *mr_destroy_stats; + u32 qp_create_stats_id; + u32 qp_destroy_stats_id; + u32 qp_modify_stats_id; + u32 mr_create_stats_id; + u32 mr_destroy_stats_id; + bool init_oos_stats; + u64 oos_prev; + u32 num_irq_stopped; + u32 num_irq_started; + u32 poll_in_intr_en; + u32 poll_in_intr_dis; + atomic_t rcfw_intr_enabled; + u32 cmdq_full_dbg; + struct semaphore rcfw_inflight; + unsigned int curr_shadow_qd; + atomic_t timeout_send; + /* cached from chip cctx for quick reference in slow path */ + u16 max_timeout; +}; + +struct bnxt_qplib_cmdqmsg { + struct cmdq_base *req; + struct creq_base *resp; + void *sb; + u32 req_sz; + u32 res_sz; + u8 block; + u8 qp_state; +}; + +static inline void bnxt_qplib_fill_cmdqmsg(struct bnxt_qplib_cmdqmsg *msg, + void *req, void *resp, void *sb, + u32 req_sz, u32 res_sz, u8 block) +{ + msg->req = req; + msg->resp = resp; + msg->sb = sb; + msg->req_sz = req_sz; + msg->res_sz = res_sz; + msg->block = block; +} + +void bnxt_qplib_free_rcfw_channel(struct bnxt_qplib_res *res); +int bnxt_qplib_alloc_rcfw_channel(struct bnxt_qplib_res *res); +void bnxt_qplib_rcfw_stop_irq(struct bnxt_qplib_rcfw *rcfw, bool kill); +void bnxt_qplib_disable_rcfw_channel(struct bnxt_qplib_rcfw *rcfw); +int bnxt_qplib_rcfw_start_irq(struct bnxt_qplib_rcfw *rcfw, int msix_vector, + bool need_init); +int bnxt_qplib_enable_rcfw_channel(struct bnxt_qplib_rcfw *rcfw, + int msix_vector, + int cp_bar_reg_off, + aeq_handler_t aeq_handler); + +struct bnxt_qplib_rcfw_sbuf *bnxt_qplib_rcfw_alloc_sbuf( + struct bnxt_qplib_rcfw *rcfw, + u32 size); +void bnxt_qplib_rcfw_free_sbuf(struct bnxt_qplib_rcfw *rcfw, + struct bnxt_qplib_rcfw_sbuf *sbuf); +int bnxt_qplib_rcfw_send_message(struct bnxt_qplib_rcfw *rcfw, + struct bnxt_qplib_cmdqmsg *msg); + +int bnxt_qplib_deinit_rcfw(struct bnxt_qplib_rcfw *rcfw); +int bnxt_qplib_init_rcfw(struct bnxt_qplib_rcfw *rcfw, int is_virtfn); +void bnxt_qplib_mark_qp_error(void *qp_handle); +int __check_cmdq_stall(struct bnxt_qplib_rcfw *rcfw, + u32 *cur_prod, u32 *cur_cons); +#endif diff --git a/sys/dev/bnxt/bnxt_re/qplib_res.c b/sys/dev/bnxt/bnxt_re/qplib_res.c new file mode 100644 index 000000000000..69661c67708c --- /dev/null +++ b/sys/dev/bnxt/bnxt_re/qplib_res.c @@ -0,0 +1,1226 @@ +/* + * Copyright (c) 2015-2024, Broadcom. All rights reserved. The term + * Broadcom refers to Broadcom Limited and/or its subsidiaries. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR + * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE + * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN + * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * Description: QPLib resource manager + */ + +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include "hsi_struct_def.h" +#include "qplib_res.h" +#include "qplib_sp.h" +#include "qplib_rcfw.h" +#include "bnxt.h" +#include "bnxt_ulp.h" + +uint8_t _get_chip_gen_p5_type(struct bnxt_qplib_chip_ctx *cctx) +{ + /* Extend this for granular type */ + return(BNXT_RE_DEFAULT); +} + +inline bool _is_alloc_mr_unified(struct bnxt_qplib_dev_attr *dattr) +{ + return dattr->dev_cap_flags & + CREQ_QUERY_FUNC_RESP_SB_MR_REGISTER_ALLOC; +} + +/* PBL */ +static void __free_pbl(struct bnxt_qplib_res *res, + struct bnxt_qplib_pbl *pbl, bool is_umem) +{ + struct pci_dev *pdev; + int i; + + pdev = res->pdev; + if (is_umem == false) { + for (i = 0; i < pbl->pg_count; i++) { + if (pbl->pg_arr[i]) { + dma_free_coherent(&pdev->dev, pbl->pg_size, + (void *)((u64)pbl->pg_arr[i] & + PAGE_MASK), + pbl->pg_map_arr[i]); + } + else + dev_warn(&pdev->dev, + "QPLIB: PBL free pg_arr[%d] empty?!\n", + i); + pbl->pg_arr[i] = NULL; + } + } + + if (pbl->pg_arr) { + vfree(pbl->pg_arr); + pbl->pg_arr = NULL; + } + if (pbl->pg_map_arr) { + vfree(pbl->pg_map_arr); + pbl->pg_map_arr = NULL; + } + pbl->pg_count = 0; + pbl->pg_size = 0; +} + +struct qplib_sg { + dma_addr_t pg_map_arr; + u32 size; +}; + +static int __fill_user_dma_pages(struct bnxt_qplib_pbl *pbl, + struct bnxt_qplib_sg_info *sginfo) +{ + int sg_indx, pg_indx, tmp_size, offset; + struct qplib_sg *tmp_sg = NULL; + struct scatterlist *sg; + u64 pmask, addr; + + tmp_sg = vzalloc(sginfo->nmap * sizeof(struct qplib_sg)); + if (!tmp_sg) + return -ENOMEM; + + pmask = BIT_ULL(sginfo->pgshft) - 1; + sg_indx = 0; + for_each_sg(sginfo->sghead, sg, sginfo->nmap, sg_indx) { + tmp_sg[sg_indx].pg_map_arr = sg_dma_address(sg); + tmp_sg[sg_indx].size = sg_dma_len(sg); + } + pg_indx = 0; + for (sg_indx = 0; sg_indx < sginfo->nmap; sg_indx++) { + tmp_size = tmp_sg[sg_indx].size; + offset = 0; + while (tmp_size > 0) { + addr = tmp_sg[sg_indx].pg_map_arr + offset; + if ((!sg_indx && !pg_indx) || !(addr & pmask)) { + pbl->pg_map_arr[pg_indx] = addr &(~pmask); + pbl->pg_count++; + pg_indx++; + } + offset += sginfo->pgsize; + tmp_size -= sginfo->pgsize; + } + } + + vfree(tmp_sg); + return 0; +} + +static int bnxt_qplib_fill_user_dma_pages(struct bnxt_qplib_pbl *pbl, + struct bnxt_qplib_sg_info *sginfo) +{ + int rc = 0; + + rc = __fill_user_dma_pages(pbl, sginfo); + + return rc; +} + +static int __alloc_pbl(struct bnxt_qplib_res *res, struct bnxt_qplib_pbl *pbl, + struct bnxt_qplib_sg_info *sginfo) +{ + struct pci_dev *pdev; + bool is_umem = false; + int i; + + if (sginfo->nopte) + return 0; + + pdev = res->pdev; + /* page ptr arrays */ + pbl->pg_arr = vmalloc(sginfo->npages * sizeof(void *)); + if (!pbl->pg_arr) + return -ENOMEM; + + pbl->pg_map_arr = vmalloc(sginfo->npages * sizeof(dma_addr_t)); + if (!pbl->pg_map_arr) { + vfree(pbl->pg_arr); + return -ENOMEM; + } + pbl->pg_count = 0; + pbl->pg_size = sginfo->pgsize; + if (!sginfo->sghead) { + for (i = 0; i < sginfo->npages; i++) { + pbl->pg_arr[i] = dma_zalloc_coherent(&pdev->dev, + pbl->pg_size, + &pbl->pg_map_arr[i], + GFP_KERNEL); + if (!pbl->pg_arr[i]) + goto fail; + pbl->pg_count++; + } + } else { + is_umem = true; + if (bnxt_qplib_fill_user_dma_pages(pbl, sginfo)) + goto fail; + } + + return 0; +fail: + __free_pbl(res, pbl, is_umem); + return -ENOMEM; +} + +/* HWQ */ +void bnxt_qplib_free_hwq(struct bnxt_qplib_res *res, + struct bnxt_qplib_hwq *hwq) +{ + int i; + + if (!hwq->max_elements) + return; + if (hwq->level >= PBL_LVL_MAX) + return; + + for (i = 0; i < hwq->level + 1; i++) { + if (i == hwq->level) + __free_pbl(res, &hwq->pbl[i], hwq->is_user); + else + __free_pbl(res, &hwq->pbl[i], false); + } + + hwq->level = PBL_LVL_MAX; + hwq->max_elements = 0; + hwq->element_size = 0; + hwq->prod = hwq->cons = 0; + hwq->cp_bit = 0; +} + +/* All HWQs are power of 2 in size */ +int bnxt_qplib_alloc_init_hwq(struct bnxt_qplib_hwq *hwq, + struct bnxt_qplib_hwq_attr *hwq_attr) +{ + u32 npages = 0, depth, stride, aux_pages = 0; + dma_addr_t *src_phys_ptr, **dst_virt_ptr; + struct bnxt_qplib_sg_info sginfo = {}; + u32 aux_size = 0, npbl, npde; + void *umem; + struct bnxt_qplib_res *res; + u32 aux_slots, pg_size; + struct pci_dev *pdev; + int i, rc, lvl; + + res = hwq_attr->res; + pdev = res->pdev; + umem = hwq_attr->sginfo->sghead; + pg_size = hwq_attr->sginfo->pgsize; + hwq->level = PBL_LVL_MAX; + + depth = roundup_pow_of_two(hwq_attr->depth); + stride = roundup_pow_of_two(hwq_attr->stride); + if (hwq_attr->aux_depth) { + aux_slots = hwq_attr->aux_depth; + aux_size = roundup_pow_of_two(hwq_attr->aux_stride); + aux_pages = (aux_slots * aux_size) / pg_size; + if ((aux_slots * aux_size) % pg_size) + aux_pages++; + } + + if (!umem) { + hwq->is_user = false; + npages = (depth * stride) / pg_size + aux_pages; + if ((depth * stride) % pg_size) + npages++; + if (!npages) + return -EINVAL; + hwq_attr->sginfo->npages = npages; + } else { + hwq->is_user = true; + npages = hwq_attr->sginfo->npages; + npages = (npages * (u64)pg_size) / + BIT_ULL(hwq_attr->sginfo->pgshft); + if ((hwq_attr->sginfo->npages * (u64)pg_size) % + BIT_ULL(hwq_attr->sginfo->pgshft)) + npages++; + } + if (npages == MAX_PBL_LVL_0_PGS && !hwq_attr->sginfo->nopte) { + /* This request is Level 0, map PTE */ + rc = __alloc_pbl(res, &hwq->pbl[PBL_LVL_0], hwq_attr->sginfo); + if (rc) + goto fail; + hwq->level = PBL_LVL_0; + goto done; + } + + if (npages >= MAX_PBL_LVL_0_PGS) { + if (npages > MAX_PBL_LVL_1_PGS) { + u32 flag = (hwq_attr->type == HWQ_TYPE_L2_CMPL) ? + 0 : PTU_PTE_VALID; + /* 2 levels of indirection */ + npbl = npages >> MAX_PBL_LVL_1_PGS_SHIFT; + if (npages % BIT(MAX_PBL_LVL_1_PGS_SHIFT)) + npbl++; + npde = npbl >> MAX_PDL_LVL_SHIFT; + if(npbl % BIT(MAX_PDL_LVL_SHIFT)) + npde++; + /* Alloc PDE pages */ + sginfo.pgsize = npde * PAGE_SIZE; + sginfo.npages = 1; + rc = __alloc_pbl(res, &hwq->pbl[PBL_LVL_0], &sginfo); + + /* Alloc PBL pages */ + sginfo.npages = npbl; + sginfo.pgsize = PAGE_SIZE; + rc = __alloc_pbl(res, &hwq->pbl[PBL_LVL_1], &sginfo); + if (rc) + goto fail; + /* Fill PDL with PBL page pointers */ + dst_virt_ptr = + (dma_addr_t **)hwq->pbl[PBL_LVL_0].pg_arr; + src_phys_ptr = hwq->pbl[PBL_LVL_1].pg_map_arr; + if (hwq_attr->type == HWQ_TYPE_MR) { + /* For MR it is expected that we supply only 1 contigous + * page i.e only 1 entry in the PDL that will contain + * all the PBLs for the user supplied memory region + */ + for (i = 0; i < hwq->pbl[PBL_LVL_1].pg_count; i++) + dst_virt_ptr[0][i] = src_phys_ptr[i] | + flag; + } else { + for (i = 0; i < hwq->pbl[PBL_LVL_1].pg_count; i++) + dst_virt_ptr[PTR_PG(i)][PTR_IDX(i)] = + src_phys_ptr[i] | PTU_PDE_VALID; + } + /* Alloc or init PTEs */ + rc = __alloc_pbl(res, &hwq->pbl[PBL_LVL_2], + hwq_attr->sginfo); + if (rc) + goto fail; + hwq->level = PBL_LVL_2; + if (hwq_attr->sginfo->nopte) + goto done; + /* Fill PBLs with PTE pointers */ + dst_virt_ptr = + (dma_addr_t **)hwq->pbl[PBL_LVL_1].pg_arr; + src_phys_ptr = hwq->pbl[PBL_LVL_2].pg_map_arr; + for (i = 0; i < hwq->pbl[PBL_LVL_2].pg_count; i++) { + dst_virt_ptr[PTR_PG(i)][PTR_IDX(i)] = + src_phys_ptr[i] | PTU_PTE_VALID; + } + if (hwq_attr->type == HWQ_TYPE_QUEUE) { + /* Find the last pg of the size */ + i = hwq->pbl[PBL_LVL_2].pg_count; + dst_virt_ptr[PTR_PG(i - 1)][PTR_IDX(i - 1)] |= + PTU_PTE_LAST; + if (i > 1) + dst_virt_ptr[PTR_PG(i - 2)] + [PTR_IDX(i - 2)] |= + PTU_PTE_NEXT_TO_LAST; + } + } else { /* pages < 512 npbl = 1, npde = 0 */ + u32 flag = (hwq_attr->type == HWQ_TYPE_L2_CMPL) ? + 0 : PTU_PTE_VALID; + + /* 1 level of indirection */ + npbl = npages >> MAX_PBL_LVL_1_PGS_SHIFT; + if (npages % BIT(MAX_PBL_LVL_1_PGS_SHIFT)) + npbl++; + sginfo.npages = npbl; + sginfo.pgsize = PAGE_SIZE; + /* Alloc PBL page */ + rc = __alloc_pbl(res, &hwq->pbl[PBL_LVL_0], &sginfo); + if (rc) + goto fail; + /* Alloc or init PTEs */ + rc = __alloc_pbl(res, &hwq->pbl[PBL_LVL_1], + hwq_attr->sginfo); + if (rc) + goto fail; + hwq->level = PBL_LVL_1; + if (hwq_attr->sginfo->nopte) + goto done; + /* Fill PBL with PTE pointers */ + dst_virt_ptr = + (dma_addr_t **)hwq->pbl[PBL_LVL_0].pg_arr; + src_phys_ptr = hwq->pbl[PBL_LVL_1].pg_map_arr; + for (i = 0; i < hwq->pbl[PBL_LVL_1].pg_count; i++) + dst_virt_ptr[PTR_PG(i)][PTR_IDX(i)] = + src_phys_ptr[i] | flag; + if (hwq_attr->type == HWQ_TYPE_QUEUE) { + /* Find the last pg of the size */ + i = hwq->pbl[PBL_LVL_1].pg_count; + dst_virt_ptr[PTR_PG(i - 1)][PTR_IDX(i - 1)] |= + PTU_PTE_LAST; + if (i > 1) + dst_virt_ptr[PTR_PG(i - 2)] + [PTR_IDX(i - 2)] |= + PTU_PTE_NEXT_TO_LAST; + } + } + } +done: + hwq->prod = 0; + hwq->cons = 0; + hwq->pdev = pdev; + hwq->depth = hwq_attr->depth; + hwq->max_elements = depth; + hwq->element_size = stride; + hwq->qe_ppg = (pg_size/stride); + + if (hwq->level >= PBL_LVL_MAX) + goto fail; + /* For direct access to the elements */ + lvl = hwq->level; + if (hwq_attr->sginfo->nopte && hwq->level) + lvl = hwq->level - 1; + hwq->pbl_ptr = hwq->pbl[lvl].pg_arr; + hwq->pbl_dma_ptr = hwq->pbl[lvl].pg_map_arr; + spin_lock_init(&hwq->lock); + + return 0; +fail: + bnxt_qplib_free_hwq(res, hwq); + return -ENOMEM; +} + +/* Context Tables */ +void bnxt_qplib_free_hwctx(struct bnxt_qplib_res *res) +{ + struct bnxt_qplib_ctx *hctx; + int i; + + hctx = res->hctx; + bnxt_qplib_free_hwq(res, &hctx->qp_ctx.hwq); + bnxt_qplib_free_hwq(res, &hctx->mrw_ctx.hwq); + bnxt_qplib_free_hwq(res, &hctx->srq_ctx.hwq); + bnxt_qplib_free_hwq(res, &hctx->cq_ctx.hwq); + bnxt_qplib_free_hwq(res, &hctx->tim_ctx.hwq); + for (i = 0; i < MAX_TQM_ALLOC_REQ; i++) + bnxt_qplib_free_hwq(res, &hctx->tqm_ctx.qtbl[i]); + /* restor original pde level before destroy */ + hctx->tqm_ctx.pde.level = hctx->tqm_ctx.pde_level; + bnxt_qplib_free_hwq(res, &hctx->tqm_ctx.pde); +} + +static int bnxt_qplib_alloc_tqm_rings(struct bnxt_qplib_res *res, + struct bnxt_qplib_ctx *hctx) +{ + struct bnxt_qplib_hwq_attr hwq_attr = {}; + struct bnxt_qplib_sg_info sginfo = {}; + struct bnxt_qplib_tqm_ctx *tqmctx; + int rc = 0; + int i; + + tqmctx = &hctx->tqm_ctx; + + sginfo.pgsize = PAGE_SIZE; + sginfo.pgshft = PAGE_SHIFT; + hwq_attr.sginfo = &sginfo; + hwq_attr.res = res; + hwq_attr.type = HWQ_TYPE_CTX; + hwq_attr.depth = 512; + hwq_attr.stride = sizeof(u64); + /* Alloc pdl buffer */ + rc = bnxt_qplib_alloc_init_hwq(&tqmctx->pde, &hwq_attr); + if (rc) + goto out; + /* Save original pdl level */ + tqmctx->pde_level = tqmctx->pde.level; + + hwq_attr.stride = 1; + for (i = 0; i < MAX_TQM_ALLOC_REQ; i++) { + if (!tqmctx->qcount[i]) + continue; + hwq_attr.depth = hctx->qp_ctx.max * tqmctx->qcount[i]; + rc = bnxt_qplib_alloc_init_hwq(&tqmctx->qtbl[i], &hwq_attr); + if (rc) + goto out; + } +out: + return rc; +} + +static void bnxt_qplib_map_tqm_pgtbl(struct bnxt_qplib_tqm_ctx *ctx) +{ + struct bnxt_qplib_hwq *qtbl_hwq; + dma_addr_t *dma_ptr; + __le64 **pbl_ptr, *ptr; + int i, j, k; + int fnz_idx = -1; + int pg_count; + + pbl_ptr = (__le64 **)ctx->pde.pbl_ptr; + + for (i = 0, j = 0; i < MAX_TQM_ALLOC_REQ; + i++, j += MAX_TQM_ALLOC_BLK_SIZE) { + qtbl_hwq = &ctx->qtbl[i]; + if (!qtbl_hwq->max_elements) + continue; + if (fnz_idx == -1) + fnz_idx = i; /* first non-zero index */ + switch (qtbl_hwq->level) { + case PBL_LVL_2: + pg_count = qtbl_hwq->pbl[PBL_LVL_1].pg_count; + for (k = 0; k < pg_count; k++) { + ptr = &pbl_ptr[PTR_PG(j + k)][PTR_IDX(j + k)]; + dma_ptr = &qtbl_hwq->pbl[PBL_LVL_1].pg_map_arr[k]; + *ptr = cpu_to_le64(*dma_ptr | PTU_PTE_VALID); + } + break; + case PBL_LVL_1: + case PBL_LVL_0: + default: + ptr = &pbl_ptr[PTR_PG(j)][PTR_IDX(j)]; + *ptr = cpu_to_le64(qtbl_hwq->pbl[PBL_LVL_0].pg_map_arr[0] | + PTU_PTE_VALID); + break; + } + } + if (fnz_idx == -1) + fnz_idx = 0; + /* update pde level as per page table programming */ + ctx->pde.level = (ctx->qtbl[fnz_idx].level == PBL_LVL_2) ? PBL_LVL_2 : + ctx->qtbl[fnz_idx].level + 1; +} + +static int bnxt_qplib_setup_tqm_rings(struct bnxt_qplib_res *res, + struct bnxt_qplib_ctx *hctx) +{ + int rc = 0; + + rc = bnxt_qplib_alloc_tqm_rings(res, hctx); + if (rc) + goto fail; + + bnxt_qplib_map_tqm_pgtbl(&hctx->tqm_ctx); +fail: + return rc; +} + +/* + * Routine: bnxt_qplib_alloc_hwctx + * Description: + * Context tables are memories which are used by the chip. + * The 6 tables defined are: + * QPC ctx - holds QP states + * MRW ctx - holds memory region and window + * SRQ ctx - holds shared RQ states + * CQ ctx - holds completion queue states + * TQM ctx - holds Tx Queue Manager context + * TIM ctx - holds timer context + * Depending on the size of the tbl requested, either a 1 Page Buffer List + * or a 1-to-2-stage indirection Page Directory List + 1 PBL is used + * instead. + * Table might be employed as follows: + * For 0 < ctx size <= 1 PAGE, 0 level of ind is used + * For 1 PAGE < ctx size <= 512 entries size, 1 level of ind is used + * For 512 < ctx size <= MAX, 2 levels of ind is used + * Returns: + * 0 if success, else -ERRORS + */ +int bnxt_qplib_alloc_hwctx(struct bnxt_qplib_res *res) +{ + struct bnxt_qplib_hwq_attr hwq_attr = {}; + struct bnxt_qplib_sg_info sginfo = {}; + struct bnxt_qplib_ctx *hctx; + struct bnxt_qplib_hwq *hwq; + int rc = 0; + + hctx = res->hctx; + /* QPC Tables */ + sginfo.pgsize = PAGE_SIZE; + sginfo.pgshft = PAGE_SHIFT; + hwq_attr.sginfo = &sginfo; + + hwq_attr.res = res; + hwq_attr.depth = hctx->qp_ctx.max; + hwq_attr.stride = BNXT_QPLIB_MAX_QP_CTX_ENTRY_SIZE; + hwq_attr.type = HWQ_TYPE_CTX; + hwq = &hctx->qp_ctx.hwq; + rc = bnxt_qplib_alloc_init_hwq(hwq, &hwq_attr); + if (rc) + goto fail; + + /* MRW Tables */ + hwq_attr.depth = hctx->mrw_ctx.max; + hwq_attr.stride = BNXT_QPLIB_MAX_MRW_CTX_ENTRY_SIZE; + hwq = &hctx->mrw_ctx.hwq; + rc = bnxt_qplib_alloc_init_hwq(hwq, &hwq_attr); + if (rc) + goto fail; + + /* SRQ Tables */ + hwq_attr.depth = hctx->srq_ctx.max; + hwq_attr.stride = BNXT_QPLIB_MAX_SRQ_CTX_ENTRY_SIZE; + hwq = &hctx->srq_ctx.hwq; + rc = bnxt_qplib_alloc_init_hwq(hwq, &hwq_attr); + if (rc) + goto fail; + + /* CQ Tables */ + hwq_attr.depth = hctx->cq_ctx.max; + hwq_attr.stride = BNXT_QPLIB_MAX_CQ_CTX_ENTRY_SIZE; + hwq = &hctx->cq_ctx.hwq; + rc = bnxt_qplib_alloc_init_hwq(hwq, &hwq_attr); + if (rc) + goto fail; + + /* TQM Buffer */ + rc = bnxt_qplib_setup_tqm_rings(res, hctx); + if (rc) + goto fail; + /* TIM Buffer */ + hwq_attr.depth = hctx->qp_ctx.max * 16; + hwq_attr.stride = 1; + hwq = &hctx->tim_ctx.hwq; + rc = bnxt_qplib_alloc_init_hwq(hwq, &hwq_attr); + if (rc) + goto fail; + + return 0; +fail: + bnxt_qplib_free_hwctx(res); + return rc; +} + +/* GUID */ +void bnxt_qplib_get_guid(const u8 *dev_addr, u8 *guid) +{ + u8 mac[ETH_ALEN]; + + /* MAC-48 to EUI-64 mapping */ + memcpy(mac, dev_addr, ETH_ALEN); + guid[0] = mac[0] ^ 2; + guid[1] = mac[1]; + guid[2] = mac[2]; + guid[3] = 0xff; + guid[4] = 0xfe; + guid[5] = mac[3]; + guid[6] = mac[4]; + guid[7] = mac[5]; +} + +static void bnxt_qplib_free_sgid_tbl(struct bnxt_qplib_res *res) +{ + struct bnxt_qplib_sgid_tbl *sgid_tbl; + + sgid_tbl = &res->sgid_tbl; + + if (sgid_tbl->tbl) { + kfree(sgid_tbl->tbl); + sgid_tbl->tbl = NULL; + kfree(sgid_tbl->hw_id); + sgid_tbl->hw_id = NULL; + kfree(sgid_tbl->ctx); + sgid_tbl->ctx = NULL; + kfree(sgid_tbl->vlan); + sgid_tbl->vlan = NULL; + } else { + dev_dbg(&res->pdev->dev, "QPLIB: SGID tbl not present"); + } + sgid_tbl->max = 0; + sgid_tbl->active = 0; +} + +static void bnxt_qplib_free_reftbls(struct bnxt_qplib_res *res) +{ + struct bnxt_qplib_reftbl *tbl; + + tbl = &res->reftbl.srqref; + vfree(tbl->rec); + + tbl = &res->reftbl.cqref; + vfree(tbl->rec); + + tbl = &res->reftbl.qpref; + vfree(tbl->rec); +} + +static int bnxt_qplib_alloc_reftbl(struct bnxt_qplib_reftbl *tbl, u32 max) +{ + tbl->max = max; + tbl->rec = vzalloc(sizeof(*tbl->rec) * max); + if (!tbl->rec) + return -ENOMEM; + spin_lock_init(&tbl->lock); + return 0; +} + +static int bnxt_qplib_alloc_reftbls(struct bnxt_qplib_res *res, + struct bnxt_qplib_dev_attr *dattr) +{ + u32 max_cq = BNXT_QPLIB_MAX_CQ_COUNT; + struct bnxt_qplib_reftbl *tbl; + u32 res_cnt; + int rc; + + /* + * Allocating one extra entry to hold QP1 info. + * Store QP1 info at the last entry of the table. + * Decrement the tbl->max by one so that modulo + * operation to get the qp table index from qp id + * returns any value between 0 and max_qp-1 + */ + res_cnt = max_t(u32, BNXT_QPLIB_MAX_QPC_COUNT + 1, dattr->max_qp); + tbl = &res->reftbl.qpref; + rc = bnxt_qplib_alloc_reftbl(tbl, res_cnt); + if (rc) + goto fail; + tbl->max--; + + if (_is_chip_gen_p5_p7(res->cctx)) + max_cq = BNXT_QPLIB_MAX_CQ_COUNT_P5; + res_cnt = max_t(u32, max_cq, dattr->max_cq); + tbl = &res->reftbl.cqref; + rc = bnxt_qplib_alloc_reftbl(tbl, res_cnt); + if (rc) + goto fail; + + res_cnt = max_t(u32, BNXT_QPLIB_MAX_SRQC_COUNT, dattr->max_cq); + tbl = &res->reftbl.srqref; + rc = bnxt_qplib_alloc_reftbl(tbl, BNXT_QPLIB_MAX_SRQC_COUNT); + if (rc) + goto fail; + + return 0; +fail: + return rc; +} + +static int bnxt_qplib_alloc_sgid_tbl(struct bnxt_qplib_res *res, u16 max) +{ + struct bnxt_qplib_sgid_tbl *sgid_tbl; + + sgid_tbl = &res->sgid_tbl; + + sgid_tbl->tbl = kcalloc(max, sizeof(*sgid_tbl->tbl), GFP_KERNEL); + if (!sgid_tbl->tbl) + return -ENOMEM; + + sgid_tbl->hw_id = kcalloc(max, sizeof(u32), GFP_KERNEL); + if (!sgid_tbl->hw_id) + goto free_tbl; + + sgid_tbl->ctx = kcalloc(max, sizeof(void *), GFP_KERNEL); + if (!sgid_tbl->ctx) + goto free_hw_id; + + sgid_tbl->vlan = kcalloc(max, sizeof(u8), GFP_KERNEL); + if (!sgid_tbl->vlan) + goto free_ctx; + + sgid_tbl->max = max; + return 0; +free_ctx: + kfree(sgid_tbl->ctx); +free_hw_id: + kfree(sgid_tbl->hw_id); +free_tbl: + kfree(sgid_tbl->tbl); + return -ENOMEM; +}; + +static void bnxt_qplib_cleanup_sgid_tbl(struct bnxt_qplib_res *res, + struct bnxt_qplib_sgid_tbl *sgid_tbl) +{ + int i; + + for (i = 0; i < sgid_tbl->max; i++) { + if (memcmp(&sgid_tbl->tbl[i], &bnxt_qplib_gid_zero, + sizeof(bnxt_qplib_gid_zero))) + bnxt_qplib_del_sgid(sgid_tbl, &sgid_tbl->tbl[i].gid, + sgid_tbl->tbl[i].vlan_id, true); + } + memset(sgid_tbl->tbl, 0, sizeof(*sgid_tbl->tbl) * sgid_tbl->max); + memset(sgid_tbl->hw_id, -1, sizeof(u16) * sgid_tbl->max); + memset(sgid_tbl->vlan, 0, sizeof(u8) * sgid_tbl->max); + sgid_tbl->active = 0; +} + +static void bnxt_qplib_init_sgid_tbl(struct bnxt_qplib_sgid_tbl *sgid_tbl, + struct ifnet *netdev) +{ + u32 i; + + for (i = 0; i < sgid_tbl->max; i++) + sgid_tbl->tbl[i].vlan_id = 0xffff; + memset(sgid_tbl->hw_id, -1, sizeof(u16) * sgid_tbl->max); +} + +/* PDs */ +int bnxt_qplib_alloc_pd(struct bnxt_qplib_res *res, struct bnxt_qplib_pd *pd) +{ + u32 bit_num; + int rc = 0; + struct bnxt_qplib_pd_tbl *pdt = &res->pd_tbl; + + mutex_lock(&res->pd_tbl_lock); + bit_num = find_first_bit(pdt->tbl, pdt->max); + if (bit_num == pdt->max - 1) {/* Last bit is reserved */ + rc = -ENOMEM; + goto fail; + } + + /* Found unused PD */ + clear_bit(bit_num, pdt->tbl); + pd->id = bit_num; +fail: + mutex_unlock(&res->pd_tbl_lock); + return rc; +} + +int bnxt_qplib_dealloc_pd(struct bnxt_qplib_res *res, + struct bnxt_qplib_pd_tbl *pdt, + struct bnxt_qplib_pd *pd) +{ + mutex_lock(&res->pd_tbl_lock); + if (test_and_set_bit(pd->id, pdt->tbl)) { + dev_warn(&res->pdev->dev, "Freeing an unused PD? pdn = %d\n", + pd->id); + mutex_unlock(&res->pd_tbl_lock); + return -EINVAL; + } + /* Reset to reserved pdid. */ + pd->id = pdt->max - 1; + + mutex_unlock(&res->pd_tbl_lock); + return 0; +} + +static void bnxt_qplib_free_pd_tbl(struct bnxt_qplib_pd_tbl *pdt) +{ + if (pdt->tbl) { + kfree(pdt->tbl); + pdt->tbl = NULL; + } + pdt->max = 0; +} + +static int bnxt_qplib_alloc_pd_tbl(struct bnxt_qplib_res *res, u32 max) +{ + struct bnxt_qplib_pd_tbl *pdt; + u32 bytes; + + pdt = &res->pd_tbl; + + max++; /* One extra for reserved pdid. */ + bytes = DIV_ROUND_UP(max, 8); + + if (!bytes) + bytes = 1; + pdt->tbl = kmalloc(bytes, GFP_KERNEL); + if (!pdt->tbl) { + dev_err(&res->pdev->dev, + "QPLIB: PD tbl allocation failed for size = %d\n", bytes); + return -ENOMEM; + } + pdt->max = max; + memset((u8 *)pdt->tbl, 0xFF, bytes); + mutex_init(&res->pd_tbl_lock); + + return 0; +} + +/* DPIs */ +int bnxt_qplib_alloc_dpi(struct bnxt_qplib_res *res, + struct bnxt_qplib_dpi *dpi, + void *app, u8 type) +{ + struct bnxt_qplib_dpi_tbl *dpit = &res->dpi_tbl; + struct bnxt_qplib_reg_desc *reg; + u32 bit_num; + u64 umaddr; + int rc = 0; + + reg = &dpit->wcreg; + mutex_lock(&res->dpi_tbl_lock); + if (type == BNXT_QPLIB_DPI_TYPE_WC && _is_chip_p7(res->cctx) && + !dpit->avail_ppp) { + rc = -ENOMEM; + goto fail; + } + bit_num = find_first_bit(dpit->tbl, dpit->max); + if (bit_num == dpit->max) { + rc = -ENOMEM; + goto fail; + } + /* Found unused DPI */ + clear_bit(bit_num, dpit->tbl); + dpit->app_tbl[bit_num] = app; + dpi->bit = bit_num; + dpi->dpi = bit_num + (reg->offset - dpit->ucreg.offset) / PAGE_SIZE; + + umaddr = reg->bar_base + reg->offset + bit_num * PAGE_SIZE; + dpi->umdbr = umaddr; + switch (type) { + case BNXT_QPLIB_DPI_TYPE_KERNEL: + /* priviledged dbr was already mapped just initialize it. */ + dpi->umdbr = dpit->ucreg.bar_base + + dpit->ucreg.offset + bit_num * PAGE_SIZE; + dpi->dbr = dpit->priv_db; + dpi->dpi = dpi->bit; + break; + case BNXT_QPLIB_DPI_TYPE_WC: + dpi->dbr = ioremap_wc(umaddr, PAGE_SIZE); + if (_is_chip_p7(res->cctx) && dpi->dbr) + dpit->avail_ppp--; + break; + default: + dpi->dbr = ioremap(umaddr, PAGE_SIZE); + } + if (!dpi->dbr) { + dev_err(&res->pdev->dev, "QPLIB: DB remap failed, type = %d\n", + type); + rc = -ENOMEM; + } + dpi->type = type; +fail: + mutex_unlock(&res->dpi_tbl_lock); + return rc; +} + +int bnxt_qplib_dealloc_dpi(struct bnxt_qplib_res *res, + struct bnxt_qplib_dpi *dpi) +{ + struct bnxt_qplib_dpi_tbl *dpit = &res->dpi_tbl; + int rc = 0; + + mutex_lock(&res->dpi_tbl_lock); + if (dpi->bit >= dpit->max) { + dev_warn(&res->pdev->dev, + "Invalid DPI? dpi = %d, bit = %d\n", + dpi->dpi, dpi->bit); + rc = -EINVAL; + goto fail; + } + + if (dpi->dpi && dpi->type != BNXT_QPLIB_DPI_TYPE_KERNEL) { + if (dpi->type == BNXT_QPLIB_DPI_TYPE_WC && + _is_chip_p7(res->cctx) && dpi->dbr) + dpit->avail_ppp++; + pci_iounmap(res->pdev, dpi->dbr); + } + + if (test_and_set_bit(dpi->bit, dpit->tbl)) { + dev_warn(&res->pdev->dev, + "Freeing an unused DPI? dpi = %d, bit = %d\n", + dpi->dpi, dpi->bit); + rc = -EINVAL; + goto fail; + } + if (dpit->app_tbl) + dpit->app_tbl[dpi->bit] = NULL; + memset(dpi, 0, sizeof(*dpi)); +fail: + mutex_unlock(&res->dpi_tbl_lock); + return rc; +} + +static void bnxt_qplib_free_dpi_tbl(struct bnxt_qplib_dpi_tbl *dpit) +{ + kfree(dpit->tbl); + kfree(dpit->app_tbl); + dpit->tbl = NULL; + dpit->app_tbl = NULL; + dpit->max = 0; +} + +static int bnxt_qplib_alloc_dpi_tbl(struct bnxt_qplib_res *res, + struct bnxt_qplib_dev_attr *dev_attr, + u8 pppp_factor) +{ + struct bnxt_qplib_dpi_tbl *dpit; + struct bnxt_qplib_reg_desc *reg; + unsigned long bar_len; + u32 dbr_offset; + u32 bytes; + + dpit = &res->dpi_tbl; + reg = &dpit->wcreg; + + if (!_is_chip_gen_p5_p7(res->cctx)) { + /* Offest should come from L2 driver */ + dbr_offset = dev_attr->l2_db_size; + dpit->ucreg.offset = dbr_offset; + dpit->wcreg.offset = dbr_offset; + } + + bar_len = pci_resource_len(res->pdev, reg->bar_id); + dpit->max = (bar_len - reg->offset) / PAGE_SIZE; + if (dev_attr->max_dpi) + dpit->max = min_t(u32, dpit->max, dev_attr->max_dpi); + + dpit->app_tbl = kzalloc(dpit->max * sizeof(void*), GFP_KERNEL); + if (!dpit->app_tbl) { + dev_err(&res->pdev->dev, + "QPLIB: DPI app tbl allocation failed"); + return -ENOMEM; + } + + bytes = dpit->max >> 3; + if (!bytes) + bytes = 1; + + dpit->tbl = kmalloc(bytes, GFP_KERNEL); + if (!dpit->tbl) { + kfree(dpit->app_tbl); + dev_err(&res->pdev->dev, + "QPLIB: DPI tbl allocation failed for size = %d\n", + bytes); + return -ENOMEM; + } + + memset((u8 *)dpit->tbl, 0xFF, bytes); + /* + * On SR2, 2nd doorbell page of each function + * is reserved for L2 PPP. Now that the tbl is + * initialized, mark it as unavailable. By default + * RoCE can make use of the 512 extended pages for + * PPP. + */ + if (_is_chip_p7(res->cctx)) { + clear_bit(1, dpit->tbl); + if (pppp_factor) + dpit->avail_ppp = + BNXT_QPLIB_MAX_EXTENDED_PPP_PAGES / pppp_factor; + } + mutex_init(&res->dpi_tbl_lock); + dpit->priv_db = dpit->ucreg.bar_reg + dpit->ucreg.offset; + + return 0; +} + +/* Stats */ +void bnxt_qplib_free_stat_mem(struct bnxt_qplib_res *res, + struct bnxt_qplib_stats *stats) +{ + struct pci_dev *pdev; + + pdev = res->pdev; + if (stats->dma) + dma_free_coherent(&pdev->dev, stats->size, + stats->dma, stats->dma_map); + + memset(stats, 0, sizeof(*stats)); + stats->fw_id = -1; +} + +int bnxt_qplib_alloc_stat_mem(struct pci_dev *pdev, + struct bnxt_qplib_chip_ctx *cctx, + struct bnxt_qplib_stats *stats) +{ + cctx->hw_stats_size = 168; + + memset(stats, 0, sizeof(*stats)); + stats->fw_id = -1; + stats->size = cctx->hw_stats_size; + stats->dma = dma_alloc_coherent(&pdev->dev, stats->size, + &stats->dma_map, GFP_KERNEL); + if (!stats->dma) { + dev_err(&pdev->dev, "QPLIB: Stats DMA allocation failed"); + return -ENOMEM; + } + return 0; +} + +/* Resource */ +int bnxt_qplib_stop_res(struct bnxt_qplib_res *res) +{ + struct bnxt_qplib_rcfw *rcfw = res->rcfw; + struct creq_stop_func_resp resp = {}; + struct bnxt_qplib_cmdqmsg msg = {}; + struct cmdq_stop_func req = {}; + int rc; + + bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_STOP_FUNC, + sizeof(req)); + bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL, sizeof(req), + sizeof(resp), 0); + rc = bnxt_qplib_rcfw_send_message(rcfw, &msg); + return rc; +} + +void bnxt_qplib_clear_tbls(struct bnxt_qplib_res *res) +{ + bnxt_qplib_cleanup_sgid_tbl(res, &res->sgid_tbl); +} + +int bnxt_qplib_init_tbls(struct bnxt_qplib_res *res) +{ + bnxt_qplib_init_sgid_tbl(&res->sgid_tbl, res->netdev); + + return 0; +} + +void bnxt_qplib_free_tbls(struct bnxt_qplib_res *res) +{ + bnxt_qplib_free_sgid_tbl(res); + bnxt_qplib_free_pd_tbl(&res->pd_tbl); + bnxt_qplib_free_dpi_tbl(&res->dpi_tbl); + bnxt_qplib_free_reftbls(res); +} + +int bnxt_qplib_alloc_tbls(struct bnxt_qplib_res *res, u8 pppp_factor) +{ + struct bnxt_qplib_dev_attr *dev_attr; + int rc = 0; + + dev_attr = res->dattr; + + rc = bnxt_qplib_alloc_reftbls(res, dev_attr); + if (rc) + goto fail; + + rc = bnxt_qplib_alloc_sgid_tbl(res, dev_attr->max_sgid); + if (rc) + goto fail; + + rc = bnxt_qplib_alloc_pd_tbl(res, dev_attr->max_pd); + if (rc) + goto fail; + + rc = bnxt_qplib_alloc_dpi_tbl(res, dev_attr, pppp_factor); + if (rc) + goto fail; + + return 0; +fail: + bnxt_qplib_free_tbls(res); + return rc; +} + +void bnxt_qplib_unmap_db_bar(struct bnxt_qplib_res *res) +{ + struct bnxt_qplib_reg_desc *reg; + + reg = &res->dpi_tbl.ucreg; + if (reg->bar_reg) + pci_iounmap(res->pdev, reg->bar_reg); + reg->bar_reg = NULL; + reg->bar_base = 0; + reg->len = 0; + reg->bar_id = 0; /* Zero? or ff */ +} + +int bnxt_qplib_map_db_bar(struct bnxt_qplib_res *res) +{ + struct bnxt_qplib_reg_desc *ucreg; + struct bnxt_qplib_reg_desc *wcreg; + + wcreg = &res->dpi_tbl.wcreg; + wcreg->bar_id = RCFW_DBR_PCI_BAR_REGION; + if (!res || !res->pdev || !wcreg) + return -1; + wcreg->bar_base = pci_resource_start(res->pdev, wcreg->bar_id); + /* No need to set the wcreg->len here */ + + ucreg = &res->dpi_tbl.ucreg; + ucreg->bar_id = RCFW_DBR_PCI_BAR_REGION; + ucreg->bar_base = pci_resource_start(res->pdev, ucreg->bar_id); + + ucreg->offset = 65536; + + ucreg->len = ucreg->offset + PAGE_SIZE; + + if (!ucreg->len || ((ucreg->len & (PAGE_SIZE - 1)) != 0)) { + dev_err(&res->pdev->dev, "QPLIB: invalid dbr length %d\n", + (int)ucreg->len); + return -EINVAL; + } + ucreg->bar_reg = ioremap(ucreg->bar_base, ucreg->len); + if (!ucreg->bar_reg) { + dev_err(&res->pdev->dev, "priviledged dpi map failed!\n"); + return -ENOMEM; + } + + return 0; +} + +/** + * pci_enable_atomic_ops_to_root - enable AtomicOp requests to root port + * @dev: the PCI device + * @cap_mask: mask of desired AtomicOp sizes, including one or more of: + * PCI_EXP_DEVCAP2_ATOMIC_COMP32 + * PCI_EXP_DEVCAP2_ATOMIC_COMP64 + * PCI_EXP_DEVCAP2_ATOMIC_COMP128 + * + * Return 0 if all upstream bridges support AtomicOp routing, egress + * blocking is disabled on all upstream ports, and the root port supports + * the requested completion capabilities (32-bit, 64-bit and/or 128-bit + * AtomicOp completion), or negative otherwise. + */ +int pci_enable_atomic_ops_to_root(struct pci_dev *dev, u32 cap_mask) +{ + struct pci_bus *bus = dev->bus; + struct pci_dev *bridge; + u32 cap; + + if (!pci_is_pcie(dev)) + return -EINVAL; + + /* + * Per PCIe r4.0, sec 6.15, endpoints and root ports may be + * AtomicOp requesters. For now, we only support endpoints as + * requesters and root ports as completers. No endpoints as + * completers, and no peer-to-peer. + */ + + switch (pci_pcie_type(dev)) { + case PCI_EXP_TYPE_ENDPOINT: + case PCI_EXP_TYPE_LEG_END: + break; + default: + return -EINVAL; + } + + bridge = bus->self; + + pcie_capability_read_dword(bridge, PCI_EXP_DEVCAP2, &cap); + + switch (pci_pcie_type(bridge)) { + case PCI_EXP_TYPE_DOWNSTREAM: + if (!(cap & PCI_EXP_DEVCAP2_ATOMIC_ROUTE)) + return -EINVAL; + break; + + /* Ensure root port supports all the sizes we care about */ + case PCI_EXP_TYPE_ROOT_PORT: + if ((cap & cap_mask) != cap_mask) + return -EINVAL; + break; + } + return 0; +} + +int bnxt_qplib_enable_atomic_ops_to_root(struct pci_dev *dev) +{ + u16 ctl2; + + if(pci_enable_atomic_ops_to_root(dev, PCI_EXP_DEVCAP2_ATOMIC_COMP32) && + pci_enable_atomic_ops_to_root(dev, PCI_EXP_DEVCAP2_ATOMIC_COMP64)) + return -EOPNOTSUPP; + + pcie_capability_read_word(dev, PCI_EXP_DEVCTL2, &ctl2); + return !(ctl2 & PCI_EXP_DEVCTL2_ATOMIC_REQ); +} diff --git a/sys/dev/bnxt/bnxt_re/qplib_res.h b/sys/dev/bnxt/bnxt_re/qplib_res.h new file mode 100644 index 000000000000..6468207a49aa --- /dev/null +++ b/sys/dev/bnxt/bnxt_re/qplib_res.h @@ -0,0 +1,840 @@ +/* + * Copyright (c) 2015-2024, Broadcom. All rights reserved. The term + * Broadcom refers to Broadcom Limited and/or its subsidiaries. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR + * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE + * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN + * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * Description: QPLib resource manager (header) + */ + +#ifndef __BNXT_QPLIB_RES_H__ +#define __BNXT_QPLIB_RES_H__ + +#include "hsi_struct_def.h" + +extern const struct bnxt_qplib_gid bnxt_qplib_gid_zero; + +#define CHIP_NUM_57508 0x1750 +#define CHIP_NUM_57504 0x1751 +#define CHIP_NUM_57502 0x1752 +#define CHIP_NUM_58818 0xd818 +#define CHIP_NUM_57608 0x1760 + +#define BNXT_QPLIB_MAX_QPC_COUNT (64 * 1024) +#define BNXT_QPLIB_MAX_SRQC_COUNT (64 * 1024) +#define BNXT_QPLIB_MAX_CQ_COUNT (64 * 1024) +#define BNXT_QPLIB_MAX_CQ_COUNT_P5 (128 * 1024) + +#define BNXT_QPLIB_DBR_VALID (0x1UL << 26) +#define BNXT_QPLIB_DBR_EPOCH_SHIFT 24 +#define BNXT_QPLIB_DBR_TOGGLE_SHIFT 25 + +#define BNXT_QPLIB_DBR_PF_DB_OFFSET 0x10000 +#define BNXT_QPLIB_DBR_VF_DB_OFFSET 0x4000 + +#define BNXT_QPLIB_DBR_KEY_INVALID -1 + +/* chip gen type */ +#define BNXT_RE_DEFAULT 0xf + +enum bnxt_qplib_wqe_mode { + BNXT_QPLIB_WQE_MODE_STATIC = 0x00, + BNXT_QPLIB_WQE_MODE_VARIABLE = 0x01, + BNXT_QPLIB_WQE_MODE_INVALID = 0x02 +}; + +#define BNXT_RE_PUSH_MODE_NONE 0 +#define BNXT_RE_PUSH_MODE_WCB 1 +#define BNXT_RE_PUSH_MODE_PPP 2 +#define BNXT_RE_PUSH_ENABLED(mode) ((mode) == BNXT_RE_PUSH_MODE_WCB ||\ + (mode) == BNXT_RE_PUSH_MODE_PPP) +#define BNXT_RE_PPP_ENABLED(cctx) ((cctx)->modes.db_push_mode ==\ + BNXT_RE_PUSH_MODE_PPP) +#define PCI_EXP_DEVCAP2_ATOMIC_ROUTE 0x00000040 /* Atomic Op routing */ +#define PCI_EXP_DEVCAP2_ATOMIC_COMP32 0x00000080 /* 32b AtomicOp completion */ +#define PCI_EXP_DEVCAP2_ATOMIC_COMP64 0x00000100 /* 64b AtomicOp completion */ +#define PCI_EXP_DEVCTL2_ATOMIC_EGRESS_BLOCK 0x0080 /* Block atomic egress */ +#define PCI_EXP_DEVCTL2_ATOMIC_REQ 0x0040 /* Set Atomic requests */ + +int pci_enable_atomic_ops_to_root(struct pci_dev *dev, u32 cap_mask); + +struct bnxt_qplib_drv_modes { + u8 wqe_mode; + u8 te_bypass; + u8 db_push; + /* To control advanced cc params display in configfs */ + u8 cc_pr_mode; + /* Other modes to follow here e.g. GSI QP mode */ + u8 dbr_pacing; + u8 dbr_pacing_ext; + u8 dbr_drop_recov; + u8 dbr_primary_pf; + u8 dbr_pacing_v0; +}; + +struct bnxt_qplib_chip_ctx { + u16 chip_num; + u8 chip_rev; + u8 chip_metal; + u64 hwrm_intf_ver; + struct bnxt_qplib_drv_modes modes; + u32 dbr_stat_db_fifo; + u32 dbr_aeq_arm_reg; + u32 dbr_throttling_reg; + u16 hw_stats_size; + u16 hwrm_cmd_max_timeout; +}; + +static inline bool _is_chip_num_p7(u16 chip_num) +{ + return (chip_num == CHIP_NUM_58818 || + chip_num == CHIP_NUM_57608); +} + +static inline bool _is_chip_p7(struct bnxt_qplib_chip_ctx *cctx) +{ + return _is_chip_num_p7(cctx->chip_num); +} + +/* SR2 is Gen P5 */ +static inline bool _is_chip_gen_p5(struct bnxt_qplib_chip_ctx *cctx) +{ + return (cctx->chip_num == CHIP_NUM_57508 || + cctx->chip_num == CHIP_NUM_57504 || + cctx->chip_num == CHIP_NUM_57502); +} + +static inline bool _is_chip_gen_p5_p7(struct bnxt_qplib_chip_ctx *cctx) +{ + return (_is_chip_gen_p5(cctx) || _is_chip_p7(cctx)); +} + +static inline bool _is_wqe_mode_variable(struct bnxt_qplib_chip_ctx *cctx) +{ + return cctx->modes.wqe_mode == BNXT_QPLIB_WQE_MODE_VARIABLE; +} + +struct bnxt_qplib_db_pacing_data { + u32 do_pacing; + u32 pacing_th; + u32 dev_err_state; + u32 alarm_th; + u32 grc_reg_offset; + u32 fifo_max_depth; + u32 fifo_room_mask; + u8 fifo_room_shift; +}; + +static inline u8 bnxt_qplib_dbr_pacing_en(struct bnxt_qplib_chip_ctx *cctx) +{ + return cctx->modes.dbr_pacing; +} + +static inline u8 bnxt_qplib_dbr_pacing_ext_en(struct bnxt_qplib_chip_ctx *cctx) +{ + return cctx->modes.dbr_pacing_ext; +} + +static inline u8 bnxt_qplib_dbr_pacing_is_primary_pf(struct bnxt_qplib_chip_ctx *cctx) +{ + return cctx->modes.dbr_primary_pf; +} + +static inline void bnxt_qplib_dbr_pacing_set_primary_pf + (struct bnxt_qplib_chip_ctx *cctx, u8 val) +{ + cctx->modes.dbr_primary_pf = val; +} + +/* Defines for handling the HWRM version check */ +#define HWRM_VERSION_DEV_ATTR_MAX_DPI 0x1000A0000000D +#define HWRM_VERSION_ROCE_STATS_FN_ID 0x1000A00000045 + +#define PTR_CNT_PER_PG (PAGE_SIZE / sizeof(void *)) +#define PTR_MAX_IDX_PER_PG (PTR_CNT_PER_PG - 1) +#define PTR_PG(x) (((x) & ~PTR_MAX_IDX_PER_PG) / PTR_CNT_PER_PG) +#define PTR_IDX(x) ((x) & PTR_MAX_IDX_PER_PG) + +#define HWQ_CMP(idx, hwq) ((idx) & ((hwq)->max_elements - 1)) +#define HWQ_FREE_SLOTS(hwq) (hwq->max_elements - \ + ((HWQ_CMP(hwq->prod, hwq)\ + - HWQ_CMP(hwq->cons, hwq))\ + & (hwq->max_elements - 1))) +enum bnxt_qplib_hwq_type { + HWQ_TYPE_CTX, + HWQ_TYPE_QUEUE, + HWQ_TYPE_L2_CMPL, + HWQ_TYPE_MR +}; + +#define MAX_PBL_LVL_0_PGS 1 +#define MAX_PBL_LVL_1_PGS 512 +#define MAX_PBL_LVL_1_PGS_SHIFT 9 +#define MAX_PDL_LVL_SHIFT 9 + +enum bnxt_qplib_pbl_lvl { + PBL_LVL_0, + PBL_LVL_1, + PBL_LVL_2, + PBL_LVL_MAX +}; + +#define ROCE_PG_SIZE_4K (4 * 1024) +#define ROCE_PG_SIZE_8K (8 * 1024) +#define ROCE_PG_SIZE_64K (64 * 1024) +#define ROCE_PG_SIZE_2M (2 * 1024 * 1024) +#define ROCE_PG_SIZE_8M (8 * 1024 * 1024) +#define ROCE_PG_SIZE_1G (1024 * 1024 * 1024) +enum bnxt_qplib_hwrm_pg_size { + BNXT_QPLIB_HWRM_PG_SIZE_4K = 0, + BNXT_QPLIB_HWRM_PG_SIZE_8K = 1, + BNXT_QPLIB_HWRM_PG_SIZE_64K = 2, + BNXT_QPLIB_HWRM_PG_SIZE_2M = 3, + BNXT_QPLIB_HWRM_PG_SIZE_8M = 4, + BNXT_QPLIB_HWRM_PG_SIZE_1G = 5, +}; + +struct bnxt_qplib_reg_desc { + u8 bar_id; + resource_size_t bar_base; + unsigned long offset; + void __iomem *bar_reg; + size_t len; +}; + +struct bnxt_qplib_pbl { + u32 pg_count; + u32 pg_size; + void **pg_arr; + dma_addr_t *pg_map_arr; +}; + +struct bnxt_qplib_sg_info { + struct scatterlist *sghead; + u32 nmap; + u32 npages; + u32 pgshft; + u32 pgsize; + bool nopte; +}; + +struct bnxt_qplib_hwq_attr { + struct bnxt_qplib_res *res; + struct bnxt_qplib_sg_info *sginfo; + enum bnxt_qplib_hwq_type type; + u32 depth; + u32 stride; + u32 aux_stride; + u32 aux_depth; +}; + +struct bnxt_qplib_hwq { + struct pci_dev *pdev; + spinlock_t lock; + struct bnxt_qplib_pbl pbl[PBL_LVL_MAX]; + enum bnxt_qplib_pbl_lvl level; /* 0, 1, or 2 */ + void **pbl_ptr; /* ptr for easy access + to the PBL entries */ + dma_addr_t *pbl_dma_ptr; /* ptr for easy access + to the dma_addr */ + u32 max_elements; + u32 depth; /* original requested depth */ + u16 element_size; /* Size of each entry */ + u16 qe_ppg; /* queue entry per page */ + + u32 prod; /* raw */ + u32 cons; /* raw */ + u8 cp_bit; + u8 is_user; + u64 *pad_pg; + u32 pad_stride; + u32 pad_pgofft; +}; + +struct bnxt_qplib_db_info { + void __iomem *db; + void __iomem *priv_db; + struct bnxt_qplib_hwq *hwq; + struct bnxt_qplib_res *res; + u32 xid; + u32 max_slot; + u32 flags; + u8 toggle; + spinlock_t lock; + u64 shadow_key; + u64 shadow_key_arm_ena; + u32 seed; /* For DB pacing */ +}; + +enum bnxt_qplib_db_info_flags_mask { + BNXT_QPLIB_FLAG_EPOCH_CONS_SHIFT = 0x0UL, + BNXT_QPLIB_FLAG_EPOCH_PROD_SHIFT = 0x1UL, + BNXT_QPLIB_FLAG_EPOCH_CONS_MASK = 0x1UL, + BNXT_QPLIB_FLAG_EPOCH_PROD_MASK = 0x2UL, +}; + +enum bnxt_qplib_db_epoch_flag_shift { + BNXT_QPLIB_DB_EPOCH_CONS_SHIFT = BNXT_QPLIB_DBR_EPOCH_SHIFT, + BNXT_QPLIB_DB_EPOCH_PROD_SHIFT = (BNXT_QPLIB_DBR_EPOCH_SHIFT - 1) +}; + +/* Tables */ +struct bnxt_qplib_pd_tbl { + unsigned long *tbl; + u32 max; +}; + +struct bnxt_qplib_sgid_tbl { + struct bnxt_qplib_gid_info *tbl; + u16 *hw_id; + u16 max; + u16 active; + void *ctx; + bool *vlan; +}; + +enum { + BNXT_QPLIB_DPI_TYPE_KERNEL = 0, + BNXT_QPLIB_DPI_TYPE_UC = 1, + BNXT_QPLIB_DPI_TYPE_WC = 2 +}; + +struct bnxt_qplib_dpi { + u32 dpi; + u32 bit; + void __iomem *dbr; + u64 umdbr; + u8 type; +}; + +#define BNXT_QPLIB_MAX_EXTENDED_PPP_PAGES 512 +struct bnxt_qplib_dpi_tbl { + void **app_tbl; + unsigned long *tbl; + u16 max; + u16 avail_ppp; + struct bnxt_qplib_reg_desc ucreg; /* Hold entire DB bar. */ + struct bnxt_qplib_reg_desc wcreg; + void __iomem *priv_db; +}; + +struct bnxt_qplib_stats { + dma_addr_t dma_map; + void *dma; + u32 size; + u32 fw_id; +}; + +struct bnxt_qplib_vf_res { + u32 max_qp; + u32 max_mrw; + u32 max_srq; + u32 max_cq; + u32 max_gid; +}; + +#define BNXT_QPLIB_MAX_QP_CTX_ENTRY_SIZE 448 +#define BNXT_QPLIB_MAX_SRQ_CTX_ENTRY_SIZE 64 +#define BNXT_QPLIB_MAX_CQ_CTX_ENTRY_SIZE 64 +#define BNXT_QPLIB_MAX_MRW_CTX_ENTRY_SIZE 128 + +#define MAX_TQM_ALLOC_REQ 48 +#define MAX_TQM_ALLOC_BLK_SIZE 8 +struct bnxt_qplib_tqm_ctx { + struct bnxt_qplib_hwq pde; + enum bnxt_qplib_pbl_lvl pde_level; /* Original level */ + struct bnxt_qplib_hwq qtbl[MAX_TQM_ALLOC_REQ]; + u8 qcount[MAX_TQM_ALLOC_REQ]; +}; + +struct bnxt_qplib_hctx { + struct bnxt_qplib_hwq hwq; + u32 max; +}; + +struct bnxt_qplib_refrec { + void *handle; + u32 xid; +}; + +struct bnxt_qplib_reftbl { + struct bnxt_qplib_refrec *rec; + u32 max; + spinlock_t lock; /* reftbl lock */ +}; + +struct bnxt_qplib_reftbls { + struct bnxt_qplib_reftbl qpref; + struct bnxt_qplib_reftbl cqref; + struct bnxt_qplib_reftbl srqref; +}; + +#define GET_TBL_INDEX(id, tbl) ((id) % (((tbl)->max) - 1)) +static inline u32 map_qp_id_to_tbl_indx(u32 qid, struct bnxt_qplib_reftbl *tbl) +{ + return (qid == 1) ? tbl->max : GET_TBL_INDEX(qid, tbl); +} + +/* + * This structure includes the number of various roce resource table sizes + * actually allocated by the driver. May be less than the maximums the firmware + * allows if the driver imposes lower limits than the firmware. + */ +struct bnxt_qplib_ctx { + struct bnxt_qplib_hctx qp_ctx; + struct bnxt_qplib_hctx mrw_ctx; + struct bnxt_qplib_hctx srq_ctx; + struct bnxt_qplib_hctx cq_ctx; + struct bnxt_qplib_hctx tim_ctx; + struct bnxt_qplib_tqm_ctx tqm_ctx; + + struct bnxt_qplib_stats stats; + struct bnxt_qplib_stats stats2; + struct bnxt_qplib_vf_res vf_res; +}; + +struct bnxt_qplib_res { + struct pci_dev *pdev; + struct bnxt_qplib_chip_ctx *cctx; + struct bnxt_qplib_dev_attr *dattr; + struct bnxt_qplib_ctx *hctx; + struct ifnet *netdev; + struct bnxt_en_dev *en_dev; + + struct bnxt_qplib_rcfw *rcfw; + + struct bnxt_qplib_pd_tbl pd_tbl; + struct mutex pd_tbl_lock; + struct bnxt_qplib_sgid_tbl sgid_tbl; + struct bnxt_qplib_dpi_tbl dpi_tbl; + struct mutex dpi_tbl_lock; + struct bnxt_qplib_reftbls reftbl; + bool prio; + bool is_vf; + struct bnxt_qplib_db_pacing_data *pacing_data; +}; + +struct bnxt_qplib_query_stats_info { + u32 function_id; + u8 collection_id; + bool vf_valid; +}; + +struct bnxt_qplib_query_qp_info { + u32 function_id; + u32 num_qps; + u32 start_index; + bool vf_valid; +}; + +struct bnxt_qplib_query_fn_info { + bool vf_valid; + u32 host; + u32 filter; +}; + + +#define to_bnxt_qplib(ptr, type, member) \ + container_of(ptr, type, member) + +struct bnxt_qplib_pd; +struct bnxt_qplib_dev_attr; + +bool _is_chip_gen_p5(struct bnxt_qplib_chip_ctx *cctx); +bool _is_chip_gen_p5_p7(struct bnxt_qplib_chip_ctx *cctx); +bool _is_chip_a0(struct bnxt_qplib_chip_ctx *cctx); +bool _is_chip_p7(struct bnxt_qplib_chip_ctx *cctx); +bool _is_alloc_mr_unified(struct bnxt_qplib_dev_attr *dattr); +void bnxt_qplib_free_hwq(struct bnxt_qplib_res *res, + struct bnxt_qplib_hwq *hwq); +int bnxt_qplib_alloc_init_hwq(struct bnxt_qplib_hwq *hwq, + struct bnxt_qplib_hwq_attr *hwq_attr); +void bnxt_qplib_get_guid(const u8 *dev_addr, u8 *guid); +int bnxt_qplib_alloc_pd(struct bnxt_qplib_res *res, + struct bnxt_qplib_pd *pd); +int bnxt_qplib_dealloc_pd(struct bnxt_qplib_res *res, + struct bnxt_qplib_pd_tbl *pd_tbl, + struct bnxt_qplib_pd *pd); +int bnxt_qplib_alloc_dpi(struct bnxt_qplib_res *res, + struct bnxt_qplib_dpi *dpi, + void *app, u8 type); +int bnxt_qplib_dealloc_dpi(struct bnxt_qplib_res *res, + struct bnxt_qplib_dpi *dpi); +int bnxt_qplib_stop_res(struct bnxt_qplib_res *res); +void bnxt_qplib_clear_tbls(struct bnxt_qplib_res *res); +int bnxt_qplib_init_tbls(struct bnxt_qplib_res *res); +void bnxt_qplib_free_tbls(struct bnxt_qplib_res *res); +int bnxt_qplib_alloc_tbls(struct bnxt_qplib_res *res, u8 pppp_factor); +void bnxt_qplib_free_hwctx(struct bnxt_qplib_res *res); +int bnxt_qplib_alloc_hwctx(struct bnxt_qplib_res *res); +int bnxt_qplib_alloc_stat_mem(struct pci_dev *pdev, + struct bnxt_qplib_chip_ctx *cctx, + struct bnxt_qplib_stats *stats); +void bnxt_qplib_free_stat_mem(struct bnxt_qplib_res *res, + struct bnxt_qplib_stats *stats); + +int bnxt_qplib_map_db_bar(struct bnxt_qplib_res *res); +void bnxt_qplib_unmap_db_bar(struct bnxt_qplib_res *res); +int bnxt_qplib_enable_atomic_ops_to_root(struct pci_dev *dev); +u8 _get_chip_gen_p5_type(struct bnxt_qplib_chip_ctx *cctx); + +static inline void *bnxt_qplib_get_qe(struct bnxt_qplib_hwq *hwq, + u32 indx, u64 *pg) +{ + u32 pg_num, pg_idx; + + pg_num = (indx / hwq->qe_ppg); + pg_idx = (indx % hwq->qe_ppg); + if (pg) + *pg = (u64)&hwq->pbl_ptr[pg_num]; + return (void *)((u8 *)hwq->pbl_ptr[pg_num] + hwq->element_size * pg_idx); +} + +static inline void bnxt_qplib_hwq_incr_prod(struct bnxt_qplib_db_info *dbinfo, + struct bnxt_qplib_hwq *hwq, u32 cnt) +{ + /* move prod and update toggle/epoch if wrap around */ + hwq->prod += cnt; + if (hwq->prod >= hwq->depth) { + hwq->prod %= hwq->depth; + dbinfo->flags ^= 1UL << BNXT_QPLIB_FLAG_EPOCH_PROD_SHIFT; + } +} + +static inline void bnxt_qplib_hwq_incr_cons(u32 max_elements, u32 *cons, + u32 cnt, u32 *dbinfo_flags) +{ + /* move cons and update toggle/epoch if wrap around */ + *cons += cnt; + if (*cons >= max_elements) { + *cons %= max_elements; + *dbinfo_flags ^= 1UL << BNXT_QPLIB_FLAG_EPOCH_CONS_SHIFT; + } +} + +static inline u8 _get_pte_pg_size(struct bnxt_qplib_hwq *hwq) +{ + u8 pg_size = BNXT_QPLIB_HWRM_PG_SIZE_4K; + struct bnxt_qplib_pbl *pbl; + + pbl = &hwq->pbl[hwq->level]; + switch (pbl->pg_size) { + case ROCE_PG_SIZE_4K: pg_size = BNXT_QPLIB_HWRM_PG_SIZE_4K; + break; + case ROCE_PG_SIZE_8K: pg_size = BNXT_QPLIB_HWRM_PG_SIZE_8K; + break; + case ROCE_PG_SIZE_64K: pg_size = BNXT_QPLIB_HWRM_PG_SIZE_64K; + break; + case ROCE_PG_SIZE_2M: pg_size = BNXT_QPLIB_HWRM_PG_SIZE_2M; + break; + case ROCE_PG_SIZE_8M: pg_size = BNXT_QPLIB_HWRM_PG_SIZE_8M; + break; + case ROCE_PG_SIZE_1G: pg_size = BNXT_QPLIB_HWRM_PG_SIZE_1G; + break; + default: + break; + } + return pg_size; +} + +static inline u64 _get_base_addr(struct bnxt_qplib_hwq *hwq) +{ + return hwq->pbl[PBL_LVL_0].pg_map_arr[0]; +} + +static inline u8 _get_base_pg_size(struct bnxt_qplib_hwq *hwq) +{ + u8 pg_size = BNXT_QPLIB_HWRM_PG_SIZE_4K; + struct bnxt_qplib_pbl *pbl; + + pbl = &hwq->pbl[PBL_LVL_0]; + switch (pbl->pg_size) { + case ROCE_PG_SIZE_4K: pg_size = BNXT_QPLIB_HWRM_PG_SIZE_4K; + break; + case ROCE_PG_SIZE_8K: pg_size = BNXT_QPLIB_HWRM_PG_SIZE_8K; + break; + case ROCE_PG_SIZE_64K: pg_size = BNXT_QPLIB_HWRM_PG_SIZE_64K; + break; + case ROCE_PG_SIZE_2M: pg_size = BNXT_QPLIB_HWRM_PG_SIZE_2M; + break; + case ROCE_PG_SIZE_8M: pg_size = BNXT_QPLIB_HWRM_PG_SIZE_8M; + break; + case ROCE_PG_SIZE_1G: pg_size = BNXT_QPLIB_HWRM_PG_SIZE_1G; + break; + default: + break; + } + return pg_size; +} + +static inline enum bnxt_qplib_hwq_type _get_hwq_type(struct bnxt_qplib_res *res) +{ + return _is_chip_gen_p5_p7(res->cctx) ? HWQ_TYPE_QUEUE : HWQ_TYPE_L2_CMPL; +} + +static inline bool _is_ext_stats_supported(u16 dev_cap_flags) +{ + return dev_cap_flags & + CREQ_QUERY_FUNC_RESP_SB_EXT_STATS; +} + +static inline int bnxt_ext_stats_supported(struct bnxt_qplib_chip_ctx *ctx, + u16 flags, bool virtfn) +{ + return (_is_ext_stats_supported(flags) && + ((virtfn && _is_chip_p7(ctx)) || (!virtfn))); +} + +static inline bool _is_hw_retx_supported(u16 dev_cap_flags) +{ + return dev_cap_flags & + (CREQ_QUERY_FUNC_RESP_SB_HW_REQUESTER_RETX_ENABLED | + CREQ_QUERY_FUNC_RESP_SB_HW_RESPONDER_RETX_ENABLED); +} + +/* Disable HW_RETX */ +#define BNXT_RE_HW_RETX(a) _is_hw_retx_supported((a)) + +static inline bool _is_cqe_v2_supported(u16 dev_cap_flags) +{ + return dev_cap_flags & + CREQ_QUERY_FUNC_RESP_SB_CQE_V2; +} + +#define BNXT_DB_FIFO_ROOM_MASK 0x1fff8000 +#define BNXT_DB_FIFO_ROOM_SHIFT 15 +#define BNXT_MAX_FIFO_DEPTH 0x2c00 + +#define BNXT_DB_PACING_ALGO_THRESHOLD 250 +#define BNXT_DEFAULT_PACING_PROBABILITY 0xFFFF + +#define BNXT_DBR_PACING_WIN_BASE 0x2000 +#define BNXT_DBR_PACING_WIN_MAP_OFF 4 +#define BNXT_DBR_PACING_WIN_OFF(reg) (BNXT_DBR_PACING_WIN_BASE + \ + +static inline void bnxt_qplib_ring_db32(struct bnxt_qplib_db_info *info, + bool arm) +{ + u32 key = 0; + + key = info->hwq->cons | (CMPL_DOORBELL_IDX_VALID | + (CMPL_DOORBELL_KEY_CMPL & CMPL_DOORBELL_KEY_MASK)); + if (!arm) + key |= CMPL_DOORBELL_MASK; + /* memory barrier */ + wmb(); + writel(key, info->db); +} + +#define BNXT_QPLIB_INIT_DBHDR(xid, type, indx, toggle) \ + (((u64)(((xid) & DBC_DBC_XID_MASK) | DBC_DBC_PATH_ROCE | \ + (type) | BNXT_QPLIB_DBR_VALID) << 32) | (indx) | \ + ((toggle) << (BNXT_QPLIB_DBR_TOGGLE_SHIFT))) + +static inline void bnxt_qplib_write_db(struct bnxt_qplib_db_info *info, + u64 key, void __iomem *db, + u64 *shadow_key) +{ + unsigned long flags; + + spin_lock_irqsave(&info->lock, flags); + *shadow_key = key; + writeq(key, db); + spin_unlock_irqrestore(&info->lock, flags); +} + +static inline void __replay_writeq(u64 key, void __iomem *db) +{ + /* No need to replay uninitialised shadow_keys */ + if (key != BNXT_QPLIB_DBR_KEY_INVALID) + writeq(key, db); +} + +static inline void bnxt_qplib_replay_db(struct bnxt_qplib_db_info *info, + bool is_arm_ena) + +{ + if (!spin_trylock_irq(&info->lock)) + return; + + if (is_arm_ena) + __replay_writeq(info->shadow_key_arm_ena, info->priv_db); + else + __replay_writeq(info->shadow_key, info->db); + + spin_unlock_irq(&info->lock); +} + +static inline void bnxt_qplib_ring_db(struct bnxt_qplib_db_info *info, + u32 type) +{ + u64 key = 0; + u32 indx; + u8 toggle = 0; + + if (type == DBC_DBC_TYPE_CQ_ARMALL || + type == DBC_DBC_TYPE_CQ_ARMSE) + toggle = info->toggle; + + indx = ((info->hwq->cons & DBC_DBC_INDEX_MASK) | + ((info->flags & BNXT_QPLIB_FLAG_EPOCH_CONS_MASK) << + BNXT_QPLIB_DB_EPOCH_CONS_SHIFT)); + + key = BNXT_QPLIB_INIT_DBHDR(info->xid, type, indx, toggle); + bnxt_qplib_write_db(info, key, info->db, &info->shadow_key); +} + +static inline void bnxt_qplib_ring_prod_db(struct bnxt_qplib_db_info *info, + u32 type) +{ + u64 key = 0; + u32 indx; + + indx = (((info->hwq->prod / info->max_slot) & DBC_DBC_INDEX_MASK) | + ((info->flags & BNXT_QPLIB_FLAG_EPOCH_PROD_MASK) << + BNXT_QPLIB_DB_EPOCH_PROD_SHIFT)); + key = BNXT_QPLIB_INIT_DBHDR(info->xid, type, indx, 0); + bnxt_qplib_write_db(info, key, info->db, &info->shadow_key); +} + +static inline void bnxt_qplib_armen_db(struct bnxt_qplib_db_info *info, + u32 type) +{ + u64 key = 0; + u8 toggle = 0; + + if (type == DBC_DBC_TYPE_CQ_ARMENA) + toggle = info->toggle; + /* Index always at 0 */ + key = BNXT_QPLIB_INIT_DBHDR(info->xid, type, 0, toggle); + bnxt_qplib_write_db(info, key, info->priv_db, + &info->shadow_key_arm_ena); +} + +static inline void bnxt_qplib_cq_coffack_db(struct bnxt_qplib_db_info *info) +{ + u64 key = 0; + + /* Index always at 0 */ + key = BNXT_QPLIB_INIT_DBHDR(info->xid, DBC_DBC_TYPE_CQ_CUTOFF_ACK, 0, 0); + bnxt_qplib_write_db(info, key, info->priv_db, &info->shadow_key); +} + +static inline void bnxt_qplib_srq_arm_db(struct bnxt_qplib_db_info *info) +{ + u64 key = 0; + + /* Index always at 0 */ + key = BNXT_QPLIB_INIT_DBHDR(info->xid, DBC_DBC_TYPE_SRQ_ARM, 0, 0); + bnxt_qplib_write_db(info, key, info->priv_db, &info->shadow_key); +} + +static inline void bnxt_qplib_ring_nq_db(struct bnxt_qplib_db_info *info, + struct bnxt_qplib_chip_ctx *cctx, + bool arm) +{ + u32 type; + + type = arm ? DBC_DBC_TYPE_NQ_ARM : DBC_DBC_TYPE_NQ; + if (_is_chip_gen_p5_p7(cctx)) + bnxt_qplib_ring_db(info, type); + else + bnxt_qplib_ring_db32(info, arm); +} + +struct bnxt_qplib_max_res { + u32 max_qp; + u32 max_mr; + u32 max_cq; + u32 max_srq; + u32 max_ah; + u32 max_pd; +}; + +/* + * Defines for maximum resources supported for chip revisions + * Maximum PDs supported are restricted to Max QPs + * GENP4 - Wh+ + * DEFAULT - Thor + */ +#define BNXT_QPLIB_GENP4_PF_MAX_QP (16 * 1024) +#define BNXT_QPLIB_GENP4_PF_MAX_MRW (16 * 1024) +#define BNXT_QPLIB_GENP4_PF_MAX_CQ (16 * 1024) +#define BNXT_QPLIB_GENP4_PF_MAX_SRQ (1 * 1024) +#define BNXT_QPLIB_GENP4_PF_MAX_AH (16 * 1024) +#define BNXT_QPLIB_GENP4_PF_MAX_PD BNXT_QPLIB_GENP4_PF_MAX_QP + +#define BNXT_QPLIB_DEFAULT_PF_MAX_QP (64 * 1024) +#define BNXT_QPLIB_DEFAULT_PF_MAX_MRW (256 * 1024) +#define BNXT_QPLIB_DEFAULT_PF_MAX_CQ (64 * 1024) +#define BNXT_QPLIB_DEFAULT_PF_MAX_SRQ (4 * 1024) +#define BNXT_QPLIB_DEFAULT_PF_MAX_AH (64 * 1024) +#define BNXT_QPLIB_DEFAULT_PF_MAX_PD BNXT_QPLIB_DEFAULT_PF_MAX_QP + +#define BNXT_QPLIB_DEFAULT_VF_MAX_QP (6 * 1024) +#define BNXT_QPLIB_DEFAULT_VF_MAX_MRW (6 * 1024) +#define BNXT_QPLIB_DEFAULT_VF_MAX_CQ (6 * 1024) +#define BNXT_QPLIB_DEFAULT_VF_MAX_SRQ (4 * 1024) +#define BNXT_QPLIB_DEFAULT_VF_MAX_AH (6 * 1024) +#define BNXT_QPLIB_DEFAULT_VF_MAX_PD BNXT_QPLIB_DEFAULT_VF_MAX_QP + +static inline void bnxt_qplib_max_res_supported(struct bnxt_qplib_chip_ctx *cctx, + struct bnxt_qplib_res *qpl_res, + struct bnxt_qplib_max_res *max_res, + bool vf_res_limit) +{ + switch (cctx->chip_num) { + case CHIP_NUM_57608: + case CHIP_NUM_58818: + case CHIP_NUM_57504: + case CHIP_NUM_57502: + case CHIP_NUM_57508: + if (!qpl_res->is_vf) { + max_res->max_qp = BNXT_QPLIB_DEFAULT_PF_MAX_QP; + max_res->max_mr = BNXT_QPLIB_DEFAULT_PF_MAX_MRW; + max_res->max_cq = BNXT_QPLIB_DEFAULT_PF_MAX_CQ; + max_res->max_srq = BNXT_QPLIB_DEFAULT_PF_MAX_SRQ; + max_res->max_ah = BNXT_QPLIB_DEFAULT_PF_MAX_AH; + max_res->max_pd = BNXT_QPLIB_DEFAULT_PF_MAX_PD; + } else { + max_res->max_qp = BNXT_QPLIB_DEFAULT_VF_MAX_QP; + max_res->max_mr = BNXT_QPLIB_DEFAULT_VF_MAX_MRW; + max_res->max_cq = BNXT_QPLIB_DEFAULT_VF_MAX_CQ; + max_res->max_srq = BNXT_QPLIB_DEFAULT_VF_MAX_SRQ; + max_res->max_ah = BNXT_QPLIB_DEFAULT_VF_MAX_AH; + max_res->max_pd = BNXT_QPLIB_DEFAULT_VF_MAX_PD; + } + break; + default: + /* Wh+/Stratus max resources */ + max_res->max_qp = BNXT_QPLIB_GENP4_PF_MAX_QP; + max_res->max_mr = BNXT_QPLIB_GENP4_PF_MAX_MRW; + max_res->max_cq = BNXT_QPLIB_GENP4_PF_MAX_CQ; + max_res->max_srq = BNXT_QPLIB_GENP4_PF_MAX_SRQ; + max_res->max_ah = BNXT_QPLIB_GENP4_PF_MAX_AH; + max_res->max_pd = BNXT_QPLIB_GENP4_PF_MAX_PD; + break; + } +} +#endif diff --git a/sys/dev/bnxt/bnxt_re/qplib_sp.c b/sys/dev/bnxt/bnxt_re/qplib_sp.c new file mode 100644 index 000000000000..8faa3cd9390c --- /dev/null +++ b/sys/dev/bnxt/bnxt_re/qplib_sp.c @@ -0,0 +1,1234 @@ +/* + * Copyright (c) 2015-2024, Broadcom. All rights reserved. The term + * Broadcom refers to Broadcom Limited and/or its subsidiaries. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR + * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE + * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN + * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * Description: Slow Path Operators + */ + +#include +#include +#include +#include +#include +#include + +#include "hsi_struct_def.h" +#include "qplib_tlv.h" +#include "qplib_res.h" +#include "qplib_rcfw.h" +#include "qplib_sp.h" + +const struct bnxt_qplib_gid bnxt_qplib_gid_zero = {{ 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0 }}; + +/* Device */ +static u8 bnxt_qplib_is_atomic_cap(struct bnxt_qplib_rcfw *rcfw) +{ + u16 pcie_ctl2 = 0; + + if (!_is_chip_gen_p5_p7(rcfw->res->cctx)) + return false; + pcie_capability_read_word(rcfw->pdev, PCI_EXP_DEVCTL2, &pcie_ctl2); + return (pcie_ctl2 & PCI_EXP_DEVCTL2_ATOMIC_REQ); +} + +static void bnxt_qplib_query_version(struct bnxt_qplib_rcfw *rcfw, char *fw_ver) +{ + struct creq_query_version_resp resp = {}; + struct bnxt_qplib_cmdqmsg msg = {}; + struct cmdq_query_version req = {}; + int rc = 0; + + bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_QUERY_VERSION, + sizeof(req)); + bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL, sizeof(req), + sizeof(resp), 0); + rc = bnxt_qplib_rcfw_send_message(rcfw, &msg); + if (rc) { + dev_err(&rcfw->pdev->dev, "QPLIB: Failed to query version\n"); + return; + } + fw_ver[0] = resp.fw_maj; + fw_ver[1] = resp.fw_minor; + fw_ver[2] = resp.fw_bld; + fw_ver[3] = resp.fw_rsvd; +} + +int bnxt_qplib_get_dev_attr(struct bnxt_qplib_rcfw *rcfw) +{ + struct creq_query_func_resp resp = {}; + struct bnxt_qplib_cmdqmsg msg = {}; + struct creq_query_func_resp_sb *sb; + struct bnxt_qplib_rcfw_sbuf sbuf; + struct bnxt_qplib_dev_attr *attr; + struct bnxt_qplib_chip_ctx *cctx; + struct cmdq_query_func req = {}; + u8 *tqm_alloc; + int i, rc = 0; + u32 temp; + u8 chip_gen = BNXT_RE_DEFAULT; + + cctx = rcfw->res->cctx; + attr = rcfw->res->dattr; + + bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_QUERY_FUNC, + sizeof(req)); + + sbuf.size = sizeof(*sb); + sbuf.sb = dma_zalloc_coherent(&rcfw->pdev->dev, sbuf.size, + &sbuf.dma_addr, GFP_KERNEL); + if (!sbuf.sb) + return -ENOMEM; + + sb = sbuf.sb; + req.resp_size = sbuf.size / BNXT_QPLIB_CMDQE_UNITS; + bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, &sbuf, sizeof(req), + sizeof(resp), 0); + rc = bnxt_qplib_rcfw_send_message(rcfw, &msg); + if (rc) + goto bail; + /* Extract the context from the side buffer */ + chip_gen = _get_chip_gen_p5_type(cctx); + attr->max_qp = le32_to_cpu(sb->max_qp); + attr->max_qp = min_t(u32, attr->max_qp, BNXT_RE_MAX_QP_SUPPORTED(chip_gen)); + /* max_qp value reported by FW does not include the QP1 */ + attr->max_qp += 1; + attr->max_qp_rd_atom = + sb->max_qp_rd_atom > BNXT_QPLIB_MAX_OUT_RD_ATOM ? + BNXT_QPLIB_MAX_OUT_RD_ATOM : sb->max_qp_rd_atom; + attr->max_qp_init_rd_atom = + sb->max_qp_init_rd_atom > BNXT_QPLIB_MAX_OUT_RD_ATOM ? + BNXT_QPLIB_MAX_OUT_RD_ATOM : sb->max_qp_init_rd_atom; + /* Report 1 less than the max_qp_wqes reported by FW as driver adds + * one extra entry while creating the qp + */ + attr->max_qp_wqes = le16_to_cpu(sb->max_qp_wr) - 1; + /* Adjust for max_qp_wqes for variable wqe */ + if (cctx->modes.wqe_mode == BNXT_QPLIB_WQE_MODE_VARIABLE) { + attr->max_qp_wqes = (BNXT_MAX_SQ_SIZE) / + (BNXT_MAX_VAR_WQE_SIZE / BNXT_SGE_SIZE) - 1; + } + if (!_is_chip_gen_p5_p7(cctx)) { + /* + * 128 WQEs needs to be reserved for the HW (8916). Prevent + * reporting the max number for gen-p4 only. + */ + attr->max_qp_wqes -= BNXT_QPLIB_RESERVED_QP_WRS; + } + attr->max_qp_sges = sb->max_sge; + if (_is_chip_gen_p5_p7(cctx) && + cctx->modes.wqe_mode == BNXT_QPLIB_WQE_MODE_VARIABLE) + attr->max_qp_sges = sb->max_sge_var_wqe; + attr->max_cq = le32_to_cpu(sb->max_cq); + attr->max_cq = min_t(u32, attr->max_cq, BNXT_RE_MAX_CQ_SUPPORTED(chip_gen)); + + attr->max_cq_wqes = le32_to_cpu(sb->max_cqe); + attr->max_cq_wqes = min_t(u32, BNXT_QPLIB_MAX_CQ_WQES, attr->max_cq_wqes); + + attr->max_cq_sges = attr->max_qp_sges; + attr->max_mr = le32_to_cpu(sb->max_mr); + attr->max_mr = min_t(u32, attr->max_mr, BNXT_RE_MAX_MRW_SUPPORTED(chip_gen)); + attr->max_mw = le32_to_cpu(sb->max_mw); + attr->max_mw = min_t(u32, attr->max_mw, BNXT_RE_MAX_MRW_SUPPORTED(chip_gen)); + + attr->max_mr_size = le64_to_cpu(sb->max_mr_size); + attr->max_pd = BNXT_QPLIB_MAX_PD; + attr->max_raw_ethy_qp = le32_to_cpu(sb->max_raw_eth_qp); + attr->max_ah = le32_to_cpu(sb->max_ah); + attr->max_ah = min_t(u32, attr->max_ah, BNXT_RE_MAX_AH_SUPPORTED(chip_gen)); + + attr->max_fmr = le32_to_cpu(sb->max_fmr); + attr->max_map_per_fmr = sb->max_map_per_fmr; + + attr->max_srq = le16_to_cpu(sb->max_srq); + attr->max_srq = min_t(u32, attr->max_srq, BNXT_RE_MAX_SRQ_SUPPORTED(chip_gen)); + attr->max_srq_wqes = le32_to_cpu(sb->max_srq_wr) - 1; + attr->max_srq_sges = sb->max_srq_sge; + attr->max_pkey = 1; + + attr->max_inline_data = !cctx->modes.wqe_mode ? + le32_to_cpu(sb->max_inline_data) : + le16_to_cpu(sb->max_inline_data_var_wqe); + if (!_is_chip_p7(cctx)) { + attr->l2_db_size = (sb->l2_db_space_size + 1) * + (0x01 << RCFW_DBR_BASE_PAGE_SHIFT); + } + attr->max_sgid = le32_to_cpu(sb->max_gid); + + /* TODO: remove this hack for statically allocated gid_map */ + bnxt_re_set_max_gid(&attr->max_sgid); + + attr->dev_cap_flags = le16_to_cpu(sb->dev_cap_flags); + attr->page_size_cap = BIT_ULL(28) | BIT_ULL(21) | BIT_ULL(12); + + bnxt_qplib_query_version(rcfw, attr->fw_ver); + + for (i = 0; i < MAX_TQM_ALLOC_REQ / 4; i++) { + temp = le32_to_cpu(sb->tqm_alloc_reqs[i]); + tqm_alloc = (u8 *)&temp; + attr->tqm_alloc_reqs[i * 4] = *tqm_alloc; + attr->tqm_alloc_reqs[i * 4 + 1] = *(++tqm_alloc); + attr->tqm_alloc_reqs[i * 4 + 2] = *(++tqm_alloc); + attr->tqm_alloc_reqs[i * 4 + 3] = *(++tqm_alloc); + } + + if (rcfw->res->cctx->hwrm_intf_ver >= HWRM_VERSION_DEV_ATTR_MAX_DPI) + attr->max_dpi = le32_to_cpu(sb->max_dpi); + + attr->is_atomic = bnxt_qplib_is_atomic_cap(rcfw); +bail: + dma_free_coherent(&rcfw->pdev->dev, sbuf.size, + sbuf.sb, sbuf.dma_addr); + return rc; +} + +int bnxt_qplib_set_func_resources(struct bnxt_qplib_res *res) +{ + struct creq_set_func_resources_resp resp = {}; + struct cmdq_set_func_resources req = {}; + struct bnxt_qplib_cmdqmsg msg = {}; + struct bnxt_qplib_rcfw *rcfw; + struct bnxt_qplib_ctx *hctx; + int rc = 0; + + rcfw = res->rcfw; + hctx = res->hctx; + bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_SET_FUNC_RESOURCES, + sizeof(req)); + + req.number_of_qp = cpu_to_le32(hctx->qp_ctx.max); + req.number_of_mrw = cpu_to_le32(hctx->mrw_ctx.max); + req.number_of_srq = cpu_to_le32(hctx->srq_ctx.max); + req.number_of_cq = cpu_to_le32(hctx->cq_ctx.max); + + req.max_qp_per_vf = cpu_to_le32(hctx->vf_res.max_qp); + req.max_mrw_per_vf = cpu_to_le32(hctx->vf_res.max_mrw); + req.max_srq_per_vf = cpu_to_le32(hctx->vf_res.max_srq); + req.max_cq_per_vf = cpu_to_le32(hctx->vf_res.max_cq); + req.max_gid_per_vf = cpu_to_le32(hctx->vf_res.max_gid); + + /* Keep the old stats context id of PF */ + req.stat_ctx_id = cpu_to_le32(hctx->stats.fw_id); + + bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL, sizeof(req), + sizeof(resp), 0); + rc = bnxt_qplib_rcfw_send_message(rcfw, &msg); + if (rc) + dev_err(&res->pdev->dev, + "QPLIB: Failed to set function resources\n"); + + return rc; +} + +int bnxt_qplib_update_sgid(struct bnxt_qplib_sgid_tbl *sgid_tbl, + struct bnxt_qplib_gid *gid, u16 gid_idx, const u8 *smac) +{ + struct bnxt_qplib_res *res = to_bnxt_qplib(sgid_tbl, + struct bnxt_qplib_res, + sgid_tbl); + struct bnxt_qplib_rcfw *rcfw = res->rcfw; + struct creq_modify_gid_resp resp = {}; + struct bnxt_qplib_cmdqmsg msg = {}; + struct cmdq_modify_gid req = {}; + int rc; + + bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_MODIFY_GID, + sizeof(req)); + + req.gid[0] = cpu_to_be32(((u32 *)gid->data)[3]); + req.gid[1] = cpu_to_be32(((u32 *)gid->data)[2]); + req.gid[2] = cpu_to_be32(((u32 *)gid->data)[1]); + req.gid[3] = cpu_to_be32(((u32 *)gid->data)[0]); + if (res->prio) { + req.vlan |= cpu_to_le16(CMDQ_ADD_GID_VLAN_TPID_TPID_8100 | + CMDQ_ADD_GID_VLAN_VLAN_EN); + } + + /* MAC in network format */ + req.src_mac[0] = cpu_to_be16(((u16 *)smac)[0]); + req.src_mac[1] = cpu_to_be16(((u16 *)smac)[1]); + req.src_mac[2] = cpu_to_be16(((u16 *)smac)[2]); + req.gid_index = cpu_to_le16(gid_idx); + + bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL, sizeof(req), + sizeof(resp), 0); + rc = bnxt_qplib_rcfw_send_message(rcfw, &msg); + if (rc) { + dev_err(&res->pdev->dev, + "QPLIB: update SGID table failed\n"); + return rc; + } + return 0; +} + +/* SGID */ +int bnxt_qplib_get_sgid(struct bnxt_qplib_res *res, + struct bnxt_qplib_sgid_tbl *sgid_tbl, int index, + struct bnxt_qplib_gid *gid) +{ + if (index > sgid_tbl->max) { + dev_err(&res->pdev->dev, + "QPLIB: Index %d exceeded SGID table max (%d)\n", + index, sgid_tbl->max); + return -EINVAL; + } + memcpy(gid, &sgid_tbl->tbl[index].gid, sizeof(*gid)); + return 0; +} + +int bnxt_qplib_del_sgid(struct bnxt_qplib_sgid_tbl *sgid_tbl, + struct bnxt_qplib_gid *gid, + u16 vlan_id, bool update) +{ + struct bnxt_qplib_res *res = to_bnxt_qplib(sgid_tbl, + struct bnxt_qplib_res, + sgid_tbl); + struct bnxt_qplib_rcfw *rcfw = res->rcfw; + int index; + + if (sgid_tbl == NULL) { + dev_err(&res->pdev->dev, "QPLIB: SGID table not allocated\n"); + return -EINVAL; + } + /* Do we need a sgid_lock here? */ + if (!sgid_tbl->active) { + dev_err(&res->pdev->dev, + "QPLIB: SGID table has no active entries\n"); + return -ENOMEM; + } + for (index = 0; index < sgid_tbl->max; index++) { + if (!memcmp(&sgid_tbl->tbl[index].gid, gid, sizeof(*gid)) && + vlan_id == sgid_tbl->tbl[index].vlan_id) + break; + } + if (index == sgid_tbl->max) { + dev_warn(&res->pdev->dev, "GID not found in the SGID table\n"); + return 0; + } + + if (update) { + struct creq_delete_gid_resp resp = {}; + struct bnxt_qplib_cmdqmsg msg = {}; + struct cmdq_delete_gid req = {}; + int rc; + + bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_DELETE_GID, + sizeof(req)); + if (sgid_tbl->hw_id[index] == 0xFFFF) { + dev_err(&res->pdev->dev, + "QPLIB: GID entry contains an invalid HW id"); + return -EINVAL; + } + req.gid_index = cpu_to_le16(sgid_tbl->hw_id[index]); + bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL, sizeof(req), + sizeof(resp), 0); + rc = bnxt_qplib_rcfw_send_message(rcfw, &msg); + if (rc) + return rc; + } + memcpy(&sgid_tbl->tbl[index].gid, &bnxt_qplib_gid_zero, + sizeof(bnxt_qplib_gid_zero)); + sgid_tbl->tbl[index].vlan_id = 0xFFFF; + sgid_tbl->vlan[index] = false; + sgid_tbl->active--; + dev_dbg(&res->pdev->dev, + "QPLIB: SGID deleted hw_id[0x%x] = 0x%x active = 0x%x\n", + index, sgid_tbl->hw_id[index], sgid_tbl->active); + sgid_tbl->hw_id[index] = (u16)-1; + + return 0; +} + +int bnxt_qplib_add_sgid(struct bnxt_qplib_sgid_tbl *sgid_tbl, + const union ib_gid *gid, const u8 *smac, u16 vlan_id, + bool update, u32 *index) +{ + struct bnxt_qplib_res *res = to_bnxt_qplib(sgid_tbl, + struct bnxt_qplib_res, + sgid_tbl); + struct bnxt_qplib_rcfw *rcfw = res->rcfw; + int i, free_idx; + + if (sgid_tbl == NULL) { + dev_err(&res->pdev->dev, "QPLIB: SGID table not allocated\n"); + return -EINVAL; + } + /* Do we need a sgid_lock here? */ + if (sgid_tbl->active == sgid_tbl->max) { + dev_err(&res->pdev->dev, "QPLIB: SGID table is full\n"); + return -ENOMEM; + } + free_idx = sgid_tbl->max; + for (i = 0; i < sgid_tbl->max; i++) { + if (!memcmp(&sgid_tbl->tbl[i], gid, sizeof(*gid)) && + sgid_tbl->tbl[i].vlan_id == vlan_id) { + dev_dbg(&res->pdev->dev, + "QPLIB: SGID entry already exist in entry %d!\n", + i); + *index = i; + return -EALREADY; + } else if (!memcmp(&sgid_tbl->tbl[i], &bnxt_qplib_gid_zero, + sizeof(bnxt_qplib_gid_zero)) && + free_idx == sgid_tbl->max) { + free_idx = i; + } + } + if (free_idx == sgid_tbl->max) { + dev_err(&res->pdev->dev, + "QPLIB: SGID table is FULL but count is not MAX??\n"); + return -ENOMEM; + } + if (update) { + struct creq_add_gid_resp resp = {}; + struct bnxt_qplib_cmdqmsg msg = {}; + struct cmdq_add_gid req = {}; + int rc; + + bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_ADD_GID, + sizeof(req)); + + req.gid[0] = cpu_to_be32(((u32 *)gid->raw)[3]); + req.gid[1] = cpu_to_be32(((u32 *)gid->raw)[2]); + req.gid[2] = cpu_to_be32(((u32 *)gid->raw)[1]); + req.gid[3] = cpu_to_be32(((u32 *)gid->raw)[0]); + /* driver should ensure that all RoCE traffic is always VLAN tagged + * if RoCE traffic is running on non-zero VLAN ID or + * RoCE traffic is running on non-zero Priority. + */ + if ((vlan_id != 0xFFFF) || res->prio) { + if (vlan_id != 0xFFFF) + req.vlan = cpu_to_le16(vlan_id & + CMDQ_ADD_GID_VLAN_VLAN_ID_MASK); + req.vlan |= + cpu_to_le16(CMDQ_ADD_GID_VLAN_TPID_TPID_8100 | + CMDQ_ADD_GID_VLAN_VLAN_EN); + } + + /* MAC in network format */ + req.src_mac[0] = cpu_to_be16(((u16 *)smac)[0]); + req.src_mac[1] = cpu_to_be16(((u16 *)smac)[1]); + req.src_mac[2] = cpu_to_be16(((u16 *)smac)[2]); + + bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL, sizeof(req), + sizeof(resp), 0); + rc = bnxt_qplib_rcfw_send_message(rcfw, &msg); + if (rc) + return rc; + sgid_tbl->hw_id[free_idx] = le32_to_cpu(resp.xid); + } + + if (vlan_id != 0xFFFF) + sgid_tbl->vlan[free_idx] = true; + + memcpy(&sgid_tbl->tbl[free_idx], gid, sizeof(*gid)); + sgid_tbl->tbl[free_idx].vlan_id = vlan_id; + sgid_tbl->active++; + dev_dbg(&res->pdev->dev, + "QPLIB: SGID added hw_id[0x%x] = 0x%x active = 0x%x\n", + free_idx, sgid_tbl->hw_id[free_idx], sgid_tbl->active); + + *index = free_idx; + /* unlock */ + return 0; +} + +/* AH */ +int bnxt_qplib_create_ah(struct bnxt_qplib_res *res, struct bnxt_qplib_ah *ah, + bool block) +{ + struct bnxt_qplib_rcfw *rcfw = res->rcfw; + struct creq_create_ah_resp resp = {}; + struct bnxt_qplib_cmdqmsg msg = {}; + struct cmdq_create_ah req = {}; + u32 temp32[4]; + u16 temp16[3]; + int rc; + + bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_CREATE_AH, + sizeof(req)); + + memcpy(temp32, ah->dgid.data, sizeof(struct bnxt_qplib_gid)); + req.dgid[0] = cpu_to_le32(temp32[0]); + req.dgid[1] = cpu_to_le32(temp32[1]); + req.dgid[2] = cpu_to_le32(temp32[2]); + req.dgid[3] = cpu_to_le32(temp32[3]); + + req.type = ah->nw_type; + req.hop_limit = ah->hop_limit; + req.sgid_index = cpu_to_le16(res->sgid_tbl.hw_id[ah->sgid_index]); + req.dest_vlan_id_flow_label = cpu_to_le32((ah->flow_label & + CMDQ_CREATE_AH_FLOW_LABEL_MASK) | + CMDQ_CREATE_AH_DEST_VLAN_ID_MASK); + req.pd_id = cpu_to_le32(ah->pd->id); + req.traffic_class = ah->traffic_class; + + /* MAC in network format */ + memcpy(temp16, ah->dmac, ETH_ALEN); + req.dest_mac[0] = cpu_to_le16(temp16[0]); + req.dest_mac[1] = cpu_to_le16(temp16[1]); + req.dest_mac[2] = cpu_to_le16(temp16[2]); + + bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL, sizeof(req), + sizeof(resp), block); + rc = bnxt_qplib_rcfw_send_message(rcfw, &msg); + if (rc) + return rc; + + ah->id = le32_to_cpu(resp.xid); + /* for Cu/Wh AHID 0 is not valid */ + if (!_is_chip_gen_p5_p7(res->cctx) && !ah->id) + rc = -EINVAL; + + return rc; +} + +int bnxt_qplib_destroy_ah(struct bnxt_qplib_res *res, struct bnxt_qplib_ah *ah, + bool block) +{ + struct bnxt_qplib_rcfw *rcfw = res->rcfw; + struct creq_destroy_ah_resp resp = {}; + struct bnxt_qplib_cmdqmsg msg = {}; + struct cmdq_destroy_ah req = {}; + int rc; + + bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_DESTROY_AH, + sizeof(req)); + + req.ah_cid = cpu_to_le32(ah->id); + + bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL, sizeof(req), + sizeof(resp), block); + rc = bnxt_qplib_rcfw_send_message(rcfw, &msg); + return rc; +} + +/* MRW */ +int bnxt_qplib_free_mrw(struct bnxt_qplib_res *res, struct bnxt_qplib_mrw *mrw) +{ + struct creq_deallocate_key_resp resp = {}; + struct bnxt_qplib_rcfw *rcfw = res->rcfw; + struct cmdq_deallocate_key req = {}; + struct bnxt_qplib_cmdqmsg msg = {}; + int rc; + + if (mrw->lkey == 0xFFFFFFFF) { + dev_info(&res->pdev->dev, + "QPLIB: SP: Free a reserved lkey MRW\n"); + return 0; + } + + bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_DEALLOCATE_KEY, + sizeof(req)); + + req.mrw_flags = mrw->type; + + if ((mrw->type == CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE1) || + (mrw->type == CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE2A) || + (mrw->type == CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE2B)) + req.key = cpu_to_le32(mrw->rkey); + else + req.key = cpu_to_le32(mrw->lkey); + + bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL, sizeof(req), + sizeof(resp), 0); + rc = bnxt_qplib_rcfw_send_message(rcfw, &msg); + if (rc) + return rc; + + if (mrw->hwq.max_elements) + bnxt_qplib_free_hwq(res, &mrw->hwq); + + return 0; +} + +int bnxt_qplib_alloc_mrw(struct bnxt_qplib_res *res, struct bnxt_qplib_mrw *mrw) +{ + struct bnxt_qplib_rcfw *rcfw = res->rcfw; + struct creq_allocate_mrw_resp resp = {}; + struct bnxt_qplib_cmdqmsg msg = {}; + struct cmdq_allocate_mrw req = {}; + int rc; + + bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_ALLOCATE_MRW, + sizeof(req)); + + req.pd_id = cpu_to_le32(mrw->pd->id); + req.mrw_flags = mrw->type; + if ((mrw->type == CMDQ_ALLOCATE_MRW_MRW_FLAGS_PMR && + mrw->flags & BNXT_QPLIB_FR_PMR) || + mrw->type == CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE2A || + mrw->type == CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE2B) + req.access = CMDQ_ALLOCATE_MRW_ACCESS_CONSUMER_OWNED_KEY; + req.mrw_handle = cpu_to_le64(mrw); + + bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL, sizeof(req), + sizeof(resp), 0); + rc = bnxt_qplib_rcfw_send_message(rcfw, &msg); + if (rc) + return rc; + if ((mrw->type == CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE1) || + (mrw->type == CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE2A) || + (mrw->type == CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE2B)) + mrw->rkey = le32_to_cpu(resp.xid); + else + mrw->lkey = le32_to_cpu(resp.xid); + + return 0; +} + +int bnxt_qplib_dereg_mrw(struct bnxt_qplib_res *res, struct bnxt_qplib_mrw *mrw, + bool block) +{ + struct bnxt_qplib_rcfw *rcfw = res->rcfw; + struct creq_deregister_mr_resp resp = {}; + struct bnxt_qplib_cmdqmsg msg = {}; + struct cmdq_deregister_mr req = {}; + int rc; + + bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_DEREGISTER_MR, + sizeof(req)); + + req.lkey = cpu_to_le32(mrw->lkey); + bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL, sizeof(req), + sizeof(resp), block); + rc = bnxt_qplib_rcfw_send_message(rcfw, &msg); + if (rc) + return rc; + + if (mrw->hwq.max_elements) { + mrw->va = 0; + mrw->total_size = 0; + bnxt_qplib_free_hwq(res, &mrw->hwq); + } + + return 0; +} + +int bnxt_qplib_reg_mr(struct bnxt_qplib_res *res, + struct bnxt_qplib_mrinfo *mrinfo, + bool block) +{ + struct bnxt_qplib_hwq_attr hwq_attr = {}; + struct bnxt_qplib_rcfw *rcfw = res->rcfw; + struct creq_register_mr_resp resp = {}; + struct bnxt_qplib_cmdqmsg msg = {}; + struct cmdq_register_mr req = {}; + struct bnxt_qplib_mrw *mr; + u32 buf_pg_size; + u32 pg_size; + u16 level; + u16 flags; + int rc; + + mr = mrinfo->mrw; + buf_pg_size = 0x01ULL << mrinfo->sg.pgshft; + if (mrinfo->sg.npages) { + /* Free the hwq if it already exist, must be a rereg */ + if (mr->hwq.max_elements) + bnxt_qplib_free_hwq(res, &mr->hwq); + /* Use system PAGE_SIZE */ + hwq_attr.res = res; + hwq_attr.depth = mrinfo->sg.npages; + hwq_attr.stride = PAGE_SIZE; + hwq_attr.type = HWQ_TYPE_MR; + hwq_attr.sginfo = &mrinfo->sg; + rc = bnxt_qplib_alloc_init_hwq(&mr->hwq, &hwq_attr); + if (rc) { + dev_err(&res->pdev->dev, + "SP: Reg MR memory allocation failed\n"); + return -ENOMEM; + } + } + + bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_REGISTER_MR, + sizeof(req)); + /* Configure the request */ + if (mrinfo->is_dma) { + /* No PBL provided, just use system PAGE_SIZE */ + level = 0; + req.pbl = 0; + pg_size = PAGE_SIZE; + } else { + level = mr->hwq.level; + req.pbl = cpu_to_le64(mr->hwq.pbl[PBL_LVL_0].pg_map_arr[0]); + } + + pg_size = buf_pg_size ? buf_pg_size : PAGE_SIZE; + req.log2_pg_size_lvl = (level << CMDQ_REGISTER_MR_LVL_SFT) | + ((ilog2(pg_size) << + CMDQ_REGISTER_MR_LOG2_PG_SIZE_SFT) & + CMDQ_REGISTER_MR_LOG2_PG_SIZE_MASK); + req.log2_pbl_pg_size = cpu_to_le16(((ilog2(PAGE_SIZE) << + CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_SFT) & + CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_MASK)); + req.access = (mr->flags & 0xFFFF); + req.va = cpu_to_le64(mr->va); + req.key = cpu_to_le32(mr->lkey); + if (_is_alloc_mr_unified(res->dattr)) { + flags = 0; + req.key = cpu_to_le32(mr->pd->id); + flags |= CMDQ_REGISTER_MR_FLAGS_ALLOC_MR; + req.flags = cpu_to_le16(flags); + } + req.mr_size = cpu_to_le64(mr->total_size); + + bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL, sizeof(req), + sizeof(resp), block); + rc = bnxt_qplib_rcfw_send_message(rcfw, &msg); + if (rc) + goto fail; + + if (_is_alloc_mr_unified(res->dattr)) { + mr->lkey = le32_to_cpu(resp.xid); + mr->rkey = mr->lkey; + } + + return 0; +fail: + if (mr->hwq.max_elements) + bnxt_qplib_free_hwq(res, &mr->hwq); + return rc; +} + +int bnxt_qplib_alloc_fast_reg_page_list(struct bnxt_qplib_res *res, + struct bnxt_qplib_frpl *frpl, + int max_pg_ptrs) +{ + struct bnxt_qplib_hwq_attr hwq_attr = {}; + struct bnxt_qplib_sg_info sginfo = {}; + int pg_ptrs, rc; + + /* Re-calculate the max to fit the HWQ allocation model */ + pg_ptrs = roundup_pow_of_two(max_pg_ptrs); + + sginfo.pgsize = PAGE_SIZE; + sginfo.nopte = true; + + hwq_attr.res = res; + hwq_attr.depth = pg_ptrs; + hwq_attr.stride = PAGE_SIZE; + hwq_attr.sginfo = &sginfo; + hwq_attr.type = HWQ_TYPE_CTX; + rc = bnxt_qplib_alloc_init_hwq(&frpl->hwq, &hwq_attr); + if (!rc) + frpl->max_pg_ptrs = pg_ptrs; + + return rc; +} + +void bnxt_qplib_free_fast_reg_page_list(struct bnxt_qplib_res *res, + struct bnxt_qplib_frpl *frpl) +{ + bnxt_qplib_free_hwq(res, &frpl->hwq); +} + +int bnxt_qplib_map_tc2cos(struct bnxt_qplib_res *res, u16 *cids) +{ + struct bnxt_qplib_rcfw *rcfw = res->rcfw; + struct creq_map_tc_to_cos_resp resp = {}; + struct bnxt_qplib_cmdqmsg msg = {}; + struct cmdq_map_tc_to_cos req = {}; + int rc; + + bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_MAP_TC_TO_COS, + sizeof(req)); + req.cos0 = cpu_to_le16(cids[0]); + req.cos1 = cpu_to_le16(cids[1]); + + bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL, sizeof(req), + sizeof(resp), 0); + rc = bnxt_qplib_rcfw_send_message(rcfw, &msg); + return rc; +} + +static void bnxt_qplib_fill_cc_gen1(struct cmdq_modify_roce_cc_gen1_tlv *ext_req, + struct bnxt_qplib_cc_param_ext *cc_ext) +{ + ext_req->modify_mask = cpu_to_le64(cc_ext->ext_mask); + cc_ext->ext_mask = 0; + ext_req->inactivity_th_hi = cpu_to_le16(cc_ext->inact_th_hi); + ext_req->min_time_between_cnps = cpu_to_le16(cc_ext->min_delta_cnp); + ext_req->init_cp = cpu_to_le16(cc_ext->init_cp); + ext_req->tr_update_mode = cc_ext->tr_update_mode; + ext_req->tr_update_cycles = cc_ext->tr_update_cyls; + ext_req->fr_num_rtts = cc_ext->fr_rtt; + ext_req->ai_rate_increase = cc_ext->ai_rate_incr; + ext_req->reduction_relax_rtts_th = cpu_to_le16(cc_ext->rr_rtt_th); + ext_req->additional_relax_cr_th = cpu_to_le16(cc_ext->ar_cr_th); + ext_req->cr_min_th = cpu_to_le16(cc_ext->cr_min_th); + ext_req->bw_avg_weight = cc_ext->bw_avg_weight; + ext_req->actual_cr_factor = cc_ext->cr_factor; + ext_req->max_cp_cr_th = cpu_to_le16(cc_ext->cr_th_max_cp); + ext_req->cp_bias_en = cc_ext->cp_bias_en; + ext_req->cp_bias = cc_ext->cp_bias; + ext_req->cnp_ecn = cc_ext->cnp_ecn; + ext_req->rtt_jitter_en = cc_ext->rtt_jitter_en; + ext_req->link_bytes_per_usec = cpu_to_le16(cc_ext->bytes_per_usec); + ext_req->reset_cc_cr_th = cpu_to_le16(cc_ext->cc_cr_reset_th); + ext_req->cr_width = cc_ext->cr_width; + ext_req->quota_period_min = cc_ext->min_quota; + ext_req->quota_period_max = cc_ext->max_quota; + ext_req->quota_period_abs_max = cc_ext->abs_max_quota; + ext_req->tr_lower_bound = cpu_to_le16(cc_ext->tr_lb); + ext_req->cr_prob_factor = cc_ext->cr_prob_fac; + ext_req->tr_prob_factor = cc_ext->tr_prob_fac; + ext_req->fairness_cr_th = cpu_to_le16(cc_ext->fair_cr_th); + ext_req->red_div = cc_ext->red_div; + ext_req->cnp_ratio_th = cc_ext->cnp_ratio_th; + ext_req->exp_ai_rtts = cpu_to_le16(cc_ext->ai_ext_rtt); + ext_req->exp_ai_cr_cp_ratio = cc_ext->exp_crcp_ratio; + ext_req->use_rate_table = cc_ext->low_rate_en; + ext_req->cp_exp_update_th = cpu_to_le16(cc_ext->cpcr_update_th); + ext_req->high_exp_ai_rtts_th1 = cpu_to_le16(cc_ext->ai_rtt_th1); + ext_req->high_exp_ai_rtts_th2 = cpu_to_le16(cc_ext->ai_rtt_th2); + ext_req->actual_cr_cong_free_rtts_th = cpu_to_le16(cc_ext->cf_rtt_th); + ext_req->severe_cong_cr_th1 = cpu_to_le16(cc_ext->sc_cr_th1); + ext_req->severe_cong_cr_th2 = cpu_to_le16(cc_ext->sc_cr_th2); + ext_req->link64B_per_rtt = cpu_to_le32(cc_ext->l64B_per_rtt); + ext_req->cc_ack_bytes = cc_ext->cc_ack_bytes; + ext_req->reduce_init_cong_free_rtts_th = cpu_to_le16(cc_ext->reduce_cf_rtt_th); +} + +int bnxt_qplib_modify_cc(struct bnxt_qplib_res *res, + struct bnxt_qplib_cc_param *cc_param) +{ + struct bnxt_qplib_tlv_modify_cc_req tlv_req = {}; + struct creq_modify_roce_cc_resp resp = {}; + struct bnxt_qplib_cmdqmsg msg = {}; + struct cmdq_modify_roce_cc *req; + int req_size; + void *cmd; + int rc; + + /* Prepare the older base command */ + req = &tlv_req.base_req; + cmd = req; + req_size = sizeof(*req); + bnxt_qplib_rcfw_cmd_prep(req, CMDQ_BASE_OPCODE_MODIFY_ROCE_CC, + sizeof(*req)); + req->modify_mask = cpu_to_le32(cc_param->mask); + req->enable_cc = cc_param->enable; + req->g = cc_param->g; + req->num_phases_per_state = cc_param->nph_per_state; + req->time_per_phase = cc_param->time_pph; + req->pkts_per_phase = cc_param->pkts_pph; + req->init_cr = cpu_to_le16(cc_param->init_cr); + req->init_tr = cpu_to_le16(cc_param->init_tr); + req->tos_dscp_tos_ecn = (cc_param->tos_dscp << + CMDQ_MODIFY_ROCE_CC_TOS_DSCP_SFT) | + (cc_param->tos_ecn & + CMDQ_MODIFY_ROCE_CC_TOS_ECN_MASK); + req->alt_vlan_pcp = cc_param->alt_vlan_pcp; + req->alt_tos_dscp = cpu_to_le16(cc_param->alt_tos_dscp); + req->rtt = cpu_to_le16(cc_param->rtt); + req->tcp_cp = cpu_to_le16(cc_param->tcp_cp); + req->cc_mode = cc_param->cc_mode; + req->inactivity_th = cpu_to_le16(cc_param->inact_th); + + /* For chip gen P5 onwards fill extended cmd and header */ + if (_is_chip_gen_p5_p7(res->cctx)) { + struct roce_tlv *hdr; + u32 payload; + u32 chunks; + + cmd = &tlv_req; + req_size = sizeof(tlv_req); + /* Prepare primary tlv header */ + hdr = &tlv_req.tlv_hdr; + chunks = CHUNKS(sizeof(struct bnxt_qplib_tlv_modify_cc_req)); + payload = sizeof(struct cmdq_modify_roce_cc); + ROCE_1ST_TLV_PREP(hdr, chunks, payload, true); + /* Prepare secondary tlv header */ + hdr = (struct roce_tlv *)&tlv_req.ext_req; + payload = sizeof(struct cmdq_modify_roce_cc_gen1_tlv) - + sizeof(struct roce_tlv); + ROCE_EXT_TLV_PREP(hdr, TLV_TYPE_MODIFY_ROCE_CC_GEN1, payload, + false, true); + bnxt_qplib_fill_cc_gen1(&tlv_req.ext_req, &cc_param->cc_ext); + } + + bnxt_qplib_fill_cmdqmsg(&msg, cmd, &resp, NULL, req_size, + sizeof(resp), 0); + rc = bnxt_qplib_rcfw_send_message(res->rcfw, &msg); + return rc; +} + +static void bnxt_qplib_read_cc_gen1(struct bnxt_qplib_cc_param_ext *cc_ext, + struct creq_query_roce_cc_gen1_resp_sb_tlv *sb) +{ + cc_ext->inact_th_hi = le16_to_cpu(sb->inactivity_th_hi); + cc_ext->min_delta_cnp = le16_to_cpu(sb->min_time_between_cnps); + cc_ext->init_cp = le16_to_cpu(sb->init_cp); + cc_ext->tr_update_mode = sb->tr_update_mode; + cc_ext->tr_update_cyls = sb->tr_update_cycles; + cc_ext->fr_rtt = sb->fr_num_rtts; + cc_ext->ai_rate_incr = sb->ai_rate_increase; + cc_ext->rr_rtt_th = le16_to_cpu(sb->reduction_relax_rtts_th); + cc_ext->ar_cr_th = le16_to_cpu(sb->additional_relax_cr_th); + cc_ext->cr_min_th = le16_to_cpu(sb->cr_min_th); + cc_ext->bw_avg_weight = sb->bw_avg_weight; + cc_ext->cr_factor = sb->actual_cr_factor; + cc_ext->cr_th_max_cp = le16_to_cpu(sb->max_cp_cr_th); + cc_ext->cp_bias_en = sb->cp_bias_en; + cc_ext->cp_bias = sb->cp_bias; + cc_ext->cnp_ecn = sb->cnp_ecn; + cc_ext->rtt_jitter_en = sb->rtt_jitter_en; + cc_ext->bytes_per_usec = le16_to_cpu(sb->link_bytes_per_usec); + cc_ext->cc_cr_reset_th = le16_to_cpu(sb->reset_cc_cr_th); + cc_ext->cr_width = sb->cr_width; + cc_ext->min_quota = sb->quota_period_min; + cc_ext->max_quota = sb->quota_period_max; + cc_ext->abs_max_quota = sb->quota_period_abs_max; + cc_ext->tr_lb = le16_to_cpu(sb->tr_lower_bound); + cc_ext->cr_prob_fac = sb->cr_prob_factor; + cc_ext->tr_prob_fac = sb->tr_prob_factor; + cc_ext->fair_cr_th = le16_to_cpu(sb->fairness_cr_th); + cc_ext->red_div = sb->red_div; + cc_ext->cnp_ratio_th = sb->cnp_ratio_th; + cc_ext->ai_ext_rtt = le16_to_cpu(sb->exp_ai_rtts); + cc_ext->exp_crcp_ratio = sb->exp_ai_cr_cp_ratio; + cc_ext->low_rate_en = sb->use_rate_table; + cc_ext->cpcr_update_th = le16_to_cpu(sb->cp_exp_update_th); + cc_ext->ai_rtt_th1 = le16_to_cpu(sb->high_exp_ai_rtts_th1); + cc_ext->ai_rtt_th2 = le16_to_cpu(sb->high_exp_ai_rtts_th2); + cc_ext->cf_rtt_th = le16_to_cpu(sb->actual_cr_cong_free_rtts_th); + cc_ext->sc_cr_th1 = le16_to_cpu(sb->severe_cong_cr_th1); + cc_ext->sc_cr_th2 = le16_to_cpu(sb->severe_cong_cr_th2); + cc_ext->l64B_per_rtt = le32_to_cpu(sb->link64B_per_rtt); + cc_ext->cc_ack_bytes = sb->cc_ack_bytes; + cc_ext->reduce_cf_rtt_th = le16_to_cpu(sb->reduce_init_cong_free_rtts_th); +} + +int bnxt_qplib_query_cc_param(struct bnxt_qplib_res *res, + struct bnxt_qplib_cc_param *cc_param) +{ + struct creq_query_roce_cc_gen1_resp_sb_tlv *gen1_sb; + struct bnxt_qplib_tlv_query_rcc_sb *ext_sb; + struct bnxt_qplib_rcfw *rcfw = res->rcfw; + struct creq_query_roce_cc_resp resp = {}; + struct creq_query_roce_cc_resp_sb *sb; + struct bnxt_qplib_cmdqmsg msg = {}; + struct cmdq_query_roce_cc req = {}; + struct bnxt_qplib_rcfw_sbuf sbuf; + size_t resp_size; + int rc; + + /* Query the parameters from chip */ + bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_QUERY_ROCE_CC, + sizeof(req)); + if (_is_chip_gen_p5_p7(res->cctx)) + resp_size = sizeof(*ext_sb); + else + resp_size = sizeof(*sb); + sbuf.size = ALIGN(resp_size, BNXT_QPLIB_CMDQE_UNITS); + sbuf.sb = dma_zalloc_coherent(&rcfw->pdev->dev, sbuf.size, + &sbuf.dma_addr, GFP_KERNEL); + if (!sbuf.sb) + return -ENOMEM; + + req.resp_size = sbuf.size / BNXT_QPLIB_CMDQE_UNITS; + bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, &sbuf, sizeof(req), + sizeof(resp), 0); + rc = bnxt_qplib_rcfw_send_message(res->rcfw, &msg); + if (rc) { + dev_dbg(&res->pdev->dev, "%s:Query CC param failed:0x%x\n", + __func__, rc); + goto out; + } + + ext_sb = sbuf.sb; + gen1_sb = &ext_sb->gen1_sb; + sb = _is_chip_gen_p5_p7(res->cctx) ? &ext_sb->base_sb : + (struct creq_query_roce_cc_resp_sb *)ext_sb; + + cc_param->enable = sb->enable_cc & CREQ_QUERY_ROCE_CC_RESP_SB_ENABLE_CC; + cc_param->tos_ecn = (sb->tos_dscp_tos_ecn & + CREQ_QUERY_ROCE_CC_RESP_SB_TOS_ECN_MASK) >> + CREQ_QUERY_ROCE_CC_RESP_SB_TOS_ECN_SFT; + cc_param->tos_dscp = (sb->tos_dscp_tos_ecn & + CREQ_QUERY_ROCE_CC_RESP_SB_TOS_DSCP_MASK) >> + CREQ_QUERY_ROCE_CC_RESP_SB_TOS_DSCP_SFT; + cc_param->alt_tos_dscp = sb->alt_tos_dscp; + cc_param->alt_vlan_pcp = sb->alt_vlan_pcp; + + cc_param->g = sb->g; + cc_param->nph_per_state = sb->num_phases_per_state; + cc_param->init_cr = le16_to_cpu(sb->init_cr); + cc_param->init_tr = le16_to_cpu(sb->init_tr); + cc_param->cc_mode = sb->cc_mode; + cc_param->inact_th = le16_to_cpu(sb->inactivity_th); + cc_param->rtt = le16_to_cpu(sb->rtt); + cc_param->tcp_cp = le16_to_cpu(sb->tcp_cp); + cc_param->time_pph = sb->time_per_phase; + cc_param->pkts_pph = sb->pkts_per_phase; + if (_is_chip_gen_p5_p7(res->cctx)) + bnxt_qplib_read_cc_gen1(&cc_param->cc_ext, gen1_sb); +out: + dma_free_coherent(&rcfw->pdev->dev, sbuf.size, + sbuf.sb, sbuf.dma_addr); + return rc; +} + + +int bnxt_qplib_get_roce_error_stats(struct bnxt_qplib_rcfw *rcfw, + struct bnxt_qplib_roce_stats *stats, + struct bnxt_qplib_query_stats_info *sinfo) +{ + struct creq_query_roce_stats_resp resp = {}; + struct creq_query_roce_stats_resp_sb *sb; + struct cmdq_query_roce_stats req = {}; + struct bnxt_qplib_cmdqmsg msg = {}; + struct bnxt_qplib_rcfw_sbuf sbuf; + u16 cmd_flags = 0; + u32 fn_id = 0; + int rc = 0; + + bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_QUERY_ROCE_STATS, + sizeof(req)); + + sbuf.size = sizeof(*sb); + sbuf.sb = dma_zalloc_coherent(&rcfw->pdev->dev, sbuf.size, + &sbuf.dma_addr, GFP_KERNEL); + if (!sbuf.sb) + return -ENOMEM; + sb = sbuf.sb; + + if (rcfw->res->cctx->hwrm_intf_ver >= HWRM_VERSION_ROCE_STATS_FN_ID) { + if (sinfo->function_id != 0xFFFFFFFF) { + cmd_flags = CMDQ_QUERY_ROCE_STATS_FLAGS_FUNCTION_ID; + if (sinfo->vf_valid) { + fn_id = CMDQ_QUERY_ROCE_STATS_VF_VALID; + fn_id |= (sinfo->function_id << + CMDQ_QUERY_ROCE_STATS_VF_NUM_SFT) & + CMDQ_QUERY_ROCE_STATS_VF_NUM_MASK; + } else { + fn_id = sinfo->function_id & + CMDQ_QUERY_ROCE_STATS_PF_NUM_MASK; + } + } + + req.flags = cpu_to_le16(cmd_flags); + req.function_id = cpu_to_le32(fn_id); + + if (sinfo->collection_id != 0xFF) { + cmd_flags |= CMDQ_QUERY_ROCE_STATS_FLAGS_COLLECTION_ID; + req.collection_id = sinfo->collection_id; + } + } else { + /* For older HWRM version, the command length has to be + * adjusted. 8 bytes are more in the newer command. + * So subtract these 8 bytes for older HWRM version. + * command units are adjusted inside + * bnxt_qplib_rcfw_send_message. + */ + req.cmd_size -= 8; + } + + req.resp_size = sbuf.size / BNXT_QPLIB_CMDQE_UNITS; + bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, &sbuf, sizeof(req), + sizeof(resp), 0); + rc = bnxt_qplib_rcfw_send_message(rcfw, &msg); + if (rc) + goto bail; + /* Extract the context from the side buffer */ + stats->to_retransmits = le64_to_cpu(sb->to_retransmits); + stats->seq_err_naks_rcvd = le64_to_cpu(sb->seq_err_naks_rcvd); + stats->max_retry_exceeded = le64_to_cpu(sb->max_retry_exceeded); + stats->rnr_naks_rcvd = le64_to_cpu(sb->rnr_naks_rcvd); + stats->missing_resp = le64_to_cpu(sb->missing_resp); + stats->unrecoverable_err = le64_to_cpu(sb->unrecoverable_err); + stats->bad_resp_err = le64_to_cpu(sb->bad_resp_err); + stats->local_qp_op_err = le64_to_cpu(sb->local_qp_op_err); + stats->local_protection_err = le64_to_cpu(sb->local_protection_err); + stats->mem_mgmt_op_err = le64_to_cpu(sb->mem_mgmt_op_err); + stats->remote_invalid_req_err = le64_to_cpu(sb->remote_invalid_req_err); + stats->remote_access_err = le64_to_cpu(sb->remote_access_err); + stats->remote_op_err = le64_to_cpu(sb->remote_op_err); + stats->dup_req = le64_to_cpu(sb->dup_req); + stats->res_exceed_max = le64_to_cpu(sb->res_exceed_max); + stats->res_length_mismatch = le64_to_cpu(sb->res_length_mismatch); + stats->res_exceeds_wqe = le64_to_cpu(sb->res_exceeds_wqe); + stats->res_opcode_err = le64_to_cpu(sb->res_opcode_err); + stats->res_rx_invalid_rkey = le64_to_cpu(sb->res_rx_invalid_rkey); + stats->res_rx_domain_err = le64_to_cpu(sb->res_rx_domain_err); + stats->res_rx_no_perm = le64_to_cpu(sb->res_rx_no_perm); + stats->res_rx_range_err = le64_to_cpu(sb->res_rx_range_err); + stats->res_tx_invalid_rkey = le64_to_cpu(sb->res_tx_invalid_rkey); + stats->res_tx_domain_err = le64_to_cpu(sb->res_tx_domain_err); + stats->res_tx_no_perm = le64_to_cpu(sb->res_tx_no_perm); + stats->res_tx_range_err = le64_to_cpu(sb->res_tx_range_err); + stats->res_irrq_oflow = le64_to_cpu(sb->res_irrq_oflow); + stats->res_unsup_opcode = le64_to_cpu(sb->res_unsup_opcode); + stats->res_unaligned_atomic = le64_to_cpu(sb->res_unaligned_atomic); + stats->res_rem_inv_err = le64_to_cpu(sb->res_rem_inv_err); + stats->res_mem_error = le64_to_cpu(sb->res_mem_error); + stats->res_srq_err = le64_to_cpu(sb->res_srq_err); + stats->res_cmp_err = le64_to_cpu(sb->res_cmp_err); + stats->res_invalid_dup_rkey = le64_to_cpu(sb->res_invalid_dup_rkey); + stats->res_wqe_format_err = le64_to_cpu(sb->res_wqe_format_err); + stats->res_cq_load_err = le64_to_cpu(sb->res_cq_load_err); + stats->res_srq_load_err = le64_to_cpu(sb->res_srq_load_err); + stats->res_tx_pci_err = le64_to_cpu(sb->res_tx_pci_err); + stats->res_rx_pci_err = le64_to_cpu(sb->res_rx_pci_err); + + if (!rcfw->init_oos_stats) { + rcfw->oos_prev = le64_to_cpu(sb->res_oos_drop_count); + rcfw->init_oos_stats = true; + } else { + stats->res_oos_drop_count += (le64_to_cpu(sb->res_oos_drop_count) - + rcfw->oos_prev) & + BNXT_QPLIB_OOS_COUNT_MASK; + rcfw->oos_prev = le64_to_cpu(sb->res_oos_drop_count); + } + + stats->active_qp_count_p0 = le64_to_cpu(sb->active_qp_count_p0); + stats->active_qp_count_p1 = le64_to_cpu(sb->active_qp_count_p1); + stats->active_qp_count_p2 = le64_to_cpu(sb->active_qp_count_p2); + stats->active_qp_count_p3 = le64_to_cpu(sb->active_qp_count_p3); +bail: + dma_free_coherent(&rcfw->pdev->dev, sbuf.size, + sbuf.sb, sbuf.dma_addr); + return rc; +} + +int bnxt_qplib_set_link_aggr_mode(struct bnxt_qplib_res *res, + u8 aggr_mode, u8 member_port_map, + u8 active_port_map, bool aggr_en, + u32 stats_fw_id) +{ + struct creq_set_link_aggr_mode_resources_resp resp = {}; + struct cmdq_set_link_aggr_mode_cc req = {}; + struct bnxt_qplib_rcfw *rcfw = res->rcfw; + struct bnxt_qplib_cmdqmsg msg = {}; + int rc = 0; + + bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_SET_LINK_AGGR_MODE, + sizeof(req)); + + req.aggr_enable = aggr_en; + req.active_port_map = active_port_map; + req.member_port_map = member_port_map; + req.link_aggr_mode = aggr_mode; + + /* need to specify only second port stats ctx id for now */ + req.stat_ctx_id[1] = cpu_to_le16((u16)(stats_fw_id)); + + req.modify_mask = + cpu_to_le32(CMDQ_SET_LINK_AGGR_MODE_MODIFY_MASK_AGGR_EN | + CMDQ_SET_LINK_AGGR_MODE_MODIFY_MASK_ACTIVE_PORT_MAP | + CMDQ_SET_LINK_AGGR_MODE_MODIFY_MASK_MEMBER_PORT_MAP | + CMDQ_SET_LINK_AGGR_MODE_MODIFY_MASK_AGGR_MODE | + CMDQ_SET_LINK_AGGR_MODE_MODIFY_MASK_STAT_CTX_ID); + + bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL, sizeof(req), + sizeof(resp), 0); + rc = bnxt_qplib_rcfw_send_message(rcfw, &msg); + if (rc) + dev_err(&res->pdev->dev, + "QPLIB: Failed to set link aggr mode, %#x\n", rc); + + return rc; +} + +int bnxt_qplib_qext_stat(struct bnxt_qplib_rcfw *rcfw, u32 fid, + struct bnxt_qplib_ext_stat *estat, + struct bnxt_qplib_query_stats_info *sinfo) +{ + struct creq_query_roce_stats_ext_resp resp = {}; + struct creq_query_roce_stats_ext_resp_sb *sb; + struct cmdq_query_roce_stats_ext req = {}; + struct bnxt_qplib_cmdqmsg msg = {}; + struct bnxt_qplib_rcfw_sbuf sbuf; + int rc; + + sbuf.size = sizeof(*sb); + sbuf.sb = dma_zalloc_coherent(&rcfw->pdev->dev, sbuf.size, + &sbuf.dma_addr, GFP_KERNEL); + if (!sbuf.sb) { + dev_err(&rcfw->pdev->dev, + "QPLIB: SP: QUERY_ROCE_STATS_EXT alloc sb failed\n"); + return -ENOMEM; + } + sb = sbuf.sb; + + bnxt_qplib_rcfw_cmd_prep(&req, + CMDQ_QUERY_ROCE_STATS_EXT_OPCODE_QUERY_ROCE_STATS, + sizeof(req)); + req.resp_size = sbuf.size; + req.resp_addr = cpu_to_le64(sbuf.dma_addr); + req.flags = cpu_to_le16(CMDQ_QUERY_ROCE_STATS_EXT_FLAGS_FUNCTION_ID); + if (_is_chip_p7(rcfw->res->cctx) && rcfw->res->is_vf) { + if (sinfo->vf_valid) + req.function_id = + cpu_to_le32(CMDQ_QUERY_ROCE_STATS_EXT_VF_VALID | + (fid << CMDQ_QUERY_ROCE_STATS_EXT_VF_NUM_SFT)); + else + req.flags = cpu_to_le16(0); + } else { + req.function_id = cpu_to_le32(fid); + } + + bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, &sbuf, sizeof(req), + sizeof(resp), 0); + rc = bnxt_qplib_rcfw_send_message(rcfw, &msg); + if (rc) + goto bail; + + /* dump when dyndbg is enabled */ + print_hex_dump_bytes("", DUMP_PREFIX_OFFSET, sb, sizeof(*sb)); + estat->tx_atomic_req = le64_to_cpu(sb->tx_atomic_req_pkts); + estat->tx_read_req = le64_to_cpu(sb->tx_read_req_pkts); + estat->tx_read_res = le64_to_cpu(sb->tx_read_res_pkts); + estat->tx_write_req = le64_to_cpu(sb->tx_write_req_pkts); + estat->tx_send_req = le64_to_cpu(sb->tx_send_req_pkts); + estat->tx_roce_pkts = le64_to_cpu(sb->tx_roce_pkts); + estat->tx_roce_bytes = le64_to_cpu(sb->tx_roce_bytes); + estat->rx_atomic_req = le64_to_cpu(sb->rx_atomic_req_pkts); + estat->rx_read_req = le64_to_cpu(sb->rx_read_req_pkts); + estat->rx_read_res = le64_to_cpu(sb->rx_read_res_pkts); + estat->rx_write_req = le64_to_cpu(sb->rx_write_req_pkts); + estat->rx_send_req = le64_to_cpu(sb->rx_send_req_pkts); + estat->rx_roce_pkts = le64_to_cpu(sb->rx_roce_pkts); + estat->rx_roce_bytes = le64_to_cpu(sb->rx_roce_bytes); + estat->rx_roce_good_pkts = le64_to_cpu(sb->rx_roce_good_pkts); + estat->rx_roce_good_bytes = le64_to_cpu(sb->rx_roce_good_bytes); + estat->rx_out_of_buffer = le64_to_cpu(sb->rx_out_of_buffer_pkts); + estat->rx_out_of_sequence = le64_to_cpu(sb->rx_out_of_sequence_pkts); + estat->tx_cnp = le64_to_cpu(sb->tx_cnp_pkts); + estat->rx_cnp = le64_to_cpu(sb->rx_cnp_pkts); + estat->rx_ecn_marked = le64_to_cpu(sb->rx_ecn_marked_pkts); + estat->seq_err_naks_rcvd = le64_to_cpu(sb->seq_err_naks_rcvd); + estat->rnr_naks_rcvd = le64_to_cpu(sb->rnr_naks_rcvd); + estat->missing_resp = le64_to_cpu(sb->missing_resp); + estat->to_retransmits = le64_to_cpu(sb->to_retransmit); + estat->dup_req = le64_to_cpu(sb->dup_req); + estat->rx_dcn_payload_cut = le64_to_cpu(sb->rx_dcn_payload_cut); + estat->te_bypassed = le64_to_cpu(sb->te_bypassed); +bail: + dma_free_coherent(&rcfw->pdev->dev, sbuf.size, + sbuf.sb, sbuf.dma_addr); + return rc; +} diff --git a/sys/dev/bnxt/bnxt_re/qplib_sp.h b/sys/dev/bnxt/bnxt_re/qplib_sp.h new file mode 100644 index 000000000000..e306db3b9d8e --- /dev/null +++ b/sys/dev/bnxt/bnxt_re/qplib_sp.h @@ -0,0 +1,432 @@ +/* + * Copyright (c) 2015-2024, Broadcom. All rights reserved. The term + * Broadcom refers to Broadcom Limited and/or its subsidiaries. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR + * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE + * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN + * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * Description: Slow Path Operators (header) + */ + +#ifndef __BNXT_QPLIB_SP_H__ +#define __BNXT_QPLIB_SP_H__ + +#include + +#define BNXT_QPLIB_RESERVED_QP_WRS 128 + +/* Resource maximums reported by the firmware */ +struct bnxt_qplib_dev_attr { +#define FW_VER_ARR_LEN 4 + u8 fw_ver[FW_VER_ARR_LEN]; + u16 max_sgid; + u16 max_mrw; + u32 max_qp; +#define BNXT_QPLIB_MAX_OUT_RD_ATOM 126 + u32 max_qp_rd_atom; + u32 max_qp_init_rd_atom; + u32 max_qp_wqes; + u32 max_qp_sges; + u32 max_cq; + /* HW supports only 8K entries in PBL. + * So max CQEs that can be supported per CQ is 1M. + */ +#define BNXT_QPLIB_MAX_CQ_WQES 0xfffff + u32 max_cq_wqes; + u32 max_cq_sges; + u32 max_mr; + u64 max_mr_size; +#define BNXT_QPLIB_MAX_PD (64 * 1024) + u32 max_pd; + u32 max_mw; + u32 max_raw_ethy_qp; + u32 max_ah; + u32 max_fmr; + u32 max_map_per_fmr; + u32 max_srq; + u32 max_srq_wqes; + u32 max_srq_sges; + u32 max_pkey; + u32 max_inline_data; + u32 l2_db_size; + u8 tqm_alloc_reqs[MAX_TQM_ALLOC_REQ]; + u8 is_atomic; + u16 dev_cap_flags; + u64 page_size_cap; + u32 max_dpi; +}; + +struct bnxt_qplib_pd { + u32 id; +}; + +struct bnxt_qplib_gid { + u8 data[16]; +}; + +struct bnxt_qplib_gid_info { + struct bnxt_qplib_gid gid; + u16 vlan_id; +}; + +struct bnxt_qplib_ah { + struct bnxt_qplib_gid dgid; + struct bnxt_qplib_pd *pd; + u32 id; + u8 sgid_index; + u8 host_sgid_index; /* For Query AH if the hw table and SW table are differnt */ + u8 traffic_class; + u32 flow_label; + u8 hop_limit; + u8 sl; + u8 dmac[6]; + u16 vlan_id; + u8 nw_type; + u8 enable_cc; +}; + +struct bnxt_qplib_mrw { + struct bnxt_qplib_pd *pd; + int type; + u32 flags; +#define BNXT_QPLIB_FR_PMR 0x80000000 + u32 lkey; + u32 rkey; +#define BNXT_QPLIB_RSVD_LKEY 0xFFFFFFFF + u64 va; + u64 total_size; + u32 npages; + u64 mr_handle; + struct bnxt_qplib_hwq hwq; +}; + +struct bnxt_qplib_mrinfo { + struct bnxt_qplib_mrw *mrw; + struct bnxt_qplib_sg_info sg; + u64 *ptes; + bool is_dma; +}; + +struct bnxt_qplib_frpl { + int max_pg_ptrs; + struct bnxt_qplib_hwq hwq; +}; + +struct bnxt_qplib_cc_param_ext { + u64 ext_mask; + u16 inact_th_hi; + u16 min_delta_cnp; + u16 init_cp; + u8 tr_update_mode; + u8 tr_update_cyls; + u8 fr_rtt; + u8 ai_rate_incr; + u16 rr_rtt_th; + u16 ar_cr_th; + u16 cr_min_th; + u8 bw_avg_weight; + u8 cr_factor; + u16 cr_th_max_cp; + u8 cp_bias_en; + u8 cp_bias; + u8 cnp_ecn; + u8 rtt_jitter_en; + u16 bytes_per_usec; + u16 cc_cr_reset_th; + u8 cr_width; + u8 min_quota; + u8 max_quota; + u8 abs_max_quota; + u16 tr_lb; + u8 cr_prob_fac; + u8 tr_prob_fac; + u16 fair_cr_th; + u8 red_div; + u8 cnp_ratio_th; + u16 ai_ext_rtt; + u8 exp_crcp_ratio; + u8 low_rate_en; + u16 cpcr_update_th; + u16 ai_rtt_th1; + u16 ai_rtt_th2; + u16 cf_rtt_th; + u16 sc_cr_th1; /* severe congestion cr threshold 1 */ + u16 sc_cr_th2; /* severe congestion cr threshold 2 */ + u32 l64B_per_rtt; + u8 cc_ack_bytes; + u16 reduce_cf_rtt_th; +}; + +struct bnxt_qplib_cc_param { + u8 alt_vlan_pcp; + u16 alt_tos_dscp; +#define BNXT_QPLIB_USER_DSCP_VALID 0x80 + u8 cnp_dscp_user; + u8 roce_dscp_user; + u8 cc_mode; + u8 enable; + u16 inact_th; + u16 init_cr; + u16 init_tr; + u16 rtt; + u8 g; + u8 nph_per_state; + u8 time_pph; + u8 pkts_pph; + u8 tos_ecn; + u8 tos_dscp; + u8 qp1_tos_dscp; + u16 tcp_cp; + struct bnxt_qplib_cc_param_ext cc_ext; + u8 disable_prio_vlan_tx; + /* Mask used while programming the configfs values */ + u32 mask; + /* Mask used while displaying the configfs values */ + u32 cur_mask; + u8 roce_pri; +#define BNXT_QPLIB_CC_PARAM_MASK_VLAN_TX_DISABLE 0x40000 +#define BNXT_QPLIB_CC_PARAM_MASK_ROCE_PRI 0x80000 + /* prev value to clear dscp table */ + u8 prev_roce_pri; + u8 prev_alt_vlan_pcp; + u8 prev_tos_dscp; + u16 prev_alt_tos_dscp; + /* To track if admin has enabled ECN explicitly */ + u8 admin_enable; +}; + +struct bnxt_qplib_roce_stats { + u64 to_retransmits; + u64 seq_err_naks_rcvd; + /* seq_err_naks_rcvd is 64 b */ + u64 max_retry_exceeded; + /* max_retry_exceeded is 64 b */ + u64 rnr_naks_rcvd; + /* rnr_naks_rcvd is 64 b */ + u64 missing_resp; + u64 unrecoverable_err; + /* unrecoverable_err is 64 b */ + u64 bad_resp_err; + /* bad_resp_err is 64 b */ + u64 local_qp_op_err; + /* local_qp_op_err is 64 b */ + u64 local_protection_err; + /* local_protection_err is 64 b */ + u64 mem_mgmt_op_err; + /* mem_mgmt_op_err is 64 b */ + u64 remote_invalid_req_err; + /* remote_invalid_req_err is 64 b */ + u64 remote_access_err; + /* remote_access_err is 64 b */ + u64 remote_op_err; + /* remote_op_err is 64 b */ + u64 dup_req; + /* dup_req is 64 b */ + u64 res_exceed_max; + /* res_exceed_max is 64 b */ + u64 res_length_mismatch; + /* res_length_mismatch is 64 b */ + u64 res_exceeds_wqe; + /* res_exceeds_wqe is 64 b */ + u64 res_opcode_err; + /* res_opcode_err is 64 b */ + u64 res_rx_invalid_rkey; + /* res_rx_invalid_rkey is 64 b */ + u64 res_rx_domain_err; + /* res_rx_domain_err is 64 b */ + u64 res_rx_no_perm; + /* res_rx_no_perm is 64 b */ + u64 res_rx_range_err; + /* res_rx_range_err is 64 b */ + u64 res_tx_invalid_rkey; + /* res_tx_invalid_rkey is 64 b */ + u64 res_tx_domain_err; + /* res_tx_domain_err is 64 b */ + u64 res_tx_no_perm; + /* res_tx_no_perm is 64 b */ + u64 res_tx_range_err; + /* res_tx_range_err is 64 b */ + u64 res_irrq_oflow; + /* res_irrq_oflow is 64 b */ + u64 res_unsup_opcode; + /* res_unsup_opcode is 64 b */ + u64 res_unaligned_atomic; + /* res_unaligned_atomic is 64 b */ + u64 res_rem_inv_err; + /* res_rem_inv_err is 64 b */ + u64 res_mem_error; + /* res_mem_error is 64 b */ + u64 res_srq_err; + /* res_srq_err is 64 b */ + u64 res_cmp_err; + /* res_cmp_err is 64 b */ + u64 res_invalid_dup_rkey; + /* res_invalid_dup_rkey is 64 b */ + u64 res_wqe_format_err; + /* res_wqe_format_err is 64 b */ + u64 res_cq_load_err; + /* res_cq_load_err is 64 b */ + u64 res_srq_load_err; + /* res_srq_load_err is 64 b */ + u64 res_tx_pci_err; + /* res_tx_pci_err is 64 b */ + u64 res_rx_pci_err; + /* res_rx_pci_err is 64 b */ + u64 res_oos_drop_count; + /* res_oos_drop_count */ + u64 active_qp_count_p0; + /* port 0 active qps */ + u64 active_qp_count_p1; + /* port 1 active qps */ + u64 active_qp_count_p2; + /* port 2 active qps */ + u64 active_qp_count_p3; + /* port 3 active qps */ +}; + +struct bnxt_qplib_ext_stat { + u64 tx_atomic_req; + u64 tx_read_req; + u64 tx_read_res; + u64 tx_write_req; + u64 tx_send_req; + u64 tx_roce_pkts; + u64 tx_roce_bytes; + u64 rx_atomic_req; + u64 rx_read_req; + u64 rx_read_res; + u64 rx_write_req; + u64 rx_send_req; + u64 rx_roce_pkts; + u64 rx_roce_bytes; + u64 rx_roce_good_pkts; + u64 rx_roce_good_bytes; + u64 rx_out_of_buffer; + u64 rx_out_of_sequence; + u64 tx_cnp; + u64 rx_cnp; + u64 rx_ecn_marked; + u64 seq_err_naks_rcvd; + u64 rnr_naks_rcvd; + u64 missing_resp; + u64 to_retransmits; + u64 dup_req; + u64 rx_dcn_payload_cut; + u64 te_bypassed; +}; + +#define BNXT_QPLIB_ACCESS_LOCAL_WRITE (1 << 0) +#define BNXT_QPLIB_ACCESS_REMOTE_READ (1 << 1) +#define BNXT_QPLIB_ACCESS_REMOTE_WRITE (1 << 2) +#define BNXT_QPLIB_ACCESS_REMOTE_ATOMIC (1 << 3) +#define BNXT_QPLIB_ACCESS_MW_BIND (1 << 4) +#define BNXT_QPLIB_ACCESS_ZERO_BASED (1 << 5) +#define BNXT_QPLIB_ACCESS_ON_DEMAND (1 << 6) + +int bnxt_qplib_get_sgid(struct bnxt_qplib_res *res, + struct bnxt_qplib_sgid_tbl *sgid_tbl, int index, + struct bnxt_qplib_gid *gid); +int bnxt_qplib_del_sgid(struct bnxt_qplib_sgid_tbl *sgid_tbl, + struct bnxt_qplib_gid *gid, u16 vlan_id, bool update); +int bnxt_qplib_add_sgid(struct bnxt_qplib_sgid_tbl *sgid_tbl, + const union ib_gid *gid, const u8 *mac, u16 vlan_id, + bool update, u32 *index); +int bnxt_qplib_update_sgid(struct bnxt_qplib_sgid_tbl *sgid_tbl, + struct bnxt_qplib_gid *gid, u16 gid_idx, const u8 *smac); +int bnxt_qplib_get_dev_attr(struct bnxt_qplib_rcfw *rcfw); +int bnxt_qplib_set_func_resources(struct bnxt_qplib_res *res); +int bnxt_qplib_create_ah(struct bnxt_qplib_res *res, struct bnxt_qplib_ah *ah, + bool block); +int bnxt_qplib_destroy_ah(struct bnxt_qplib_res *res, struct bnxt_qplib_ah *ah, + bool block); +int bnxt_qplib_alloc_mrw(struct bnxt_qplib_res *res, struct bnxt_qplib_mrw *mrw); +int bnxt_qplib_dereg_mrw(struct bnxt_qplib_res *res, struct bnxt_qplib_mrw *mrw, + bool block); +int bnxt_qplib_reg_mr(struct bnxt_qplib_res *res, + struct bnxt_qplib_mrinfo *mrinfo, bool block); +int bnxt_qplib_free_mrw(struct bnxt_qplib_res *res, struct bnxt_qplib_mrw *mr); +int bnxt_qplib_alloc_fast_reg_mr(struct bnxt_qplib_res *res, + struct bnxt_qplib_mrw *mr, int max); +int bnxt_qplib_alloc_fast_reg_page_list(struct bnxt_qplib_res *res, + struct bnxt_qplib_frpl *frpl, int max); +void bnxt_qplib_free_fast_reg_page_list(struct bnxt_qplib_res *res, + struct bnxt_qplib_frpl *frpl); +int bnxt_qplib_map_tc2cos(struct bnxt_qplib_res *res, u16 *cids); +int bnxt_qplib_modify_cc(struct bnxt_qplib_res *res, + struct bnxt_qplib_cc_param *cc_param); +int bnxt_qplib_query_cc_param(struct bnxt_qplib_res *res, + struct bnxt_qplib_cc_param *cc_param); +int bnxt_qplib_set_link_aggr_mode(struct bnxt_qplib_res *res, + u8 aggr_mode, u8 member_port_map, + u8 active_port_map, bool aggr_en, + u32 stats_fw_id); +int bnxt_qplib_get_roce_error_stats(struct bnxt_qplib_rcfw *rcfw, + struct bnxt_qplib_roce_stats *stats, + struct bnxt_qplib_query_stats_info *sinfo); +int bnxt_qplib_qext_stat(struct bnxt_qplib_rcfw *rcfw, u32 fid, + struct bnxt_qplib_ext_stat *estat, + struct bnxt_qplib_query_stats_info *sinfo); +static inline void bnxt_re_set_max_gid(u16 *max_sgid); +bool ib_modify_qp_is_ok_compat(enum ib_qp_state cur_state, enum ib_qp_state next_state, + enum ib_qp_type type, enum ib_qp_attr_mask mask); + +#define BNXT_MAX_SQ_SIZE 0xFFFF +#define BNXT_MAX_VAR_WQE_SIZE 512 +#define BNXT_SGE_SIZE 16 + +/* PF defines */ +#define BNXT_RE_MAX_QP_SUPPORTED(chip_gen) \ + chip_gen == BNXT_RE_DEFAULT ? (64 * 1024) : 0 + +#define BNXT_RE_MAX_MRW_SUPPORTED(chip_gen) \ + chip_gen == BNXT_RE_DEFAULT ? (256 * 1024) : 0 + +#define BNXT_RE_MAX_CQ_SUPPORTED(chip_gen) \ + chip_gen == BNXT_RE_DEFAULT ? (64 * 1024) : 0 + +#define BNXT_RE_MAX_SRQ_SUPPORTED(chip_gen) \ + chip_gen == BNXT_RE_DEFAULT ? (4 * 1024) : 0 + +#define BNXT_RE_MAX_AH_SUPPORTED(chip_gen) \ + chip_gen == BNXT_RE_DEFAULT ? (64 * 1024) : 0 + +/* VF defines */ +#define BNXT_RE_VF_MAX_QP_SUPPORTED(chip_gen) \ + chip_gen == BNXT_RE_DEFAULT ? (6 * 1024) : 0 + +#define BNXT_RE_VF_MAX_MRW_SUPPORTED(chip_gen) \ + chip_gen == BNXT_RE_DEFAULT ? (6 * 1024) : 0 + +#define BNXT_RE_VF_MAX_CQ_SUPPORTED(chip_gen) \ + chip_gen == BNXT_RE_DEFAULT ? (6 * 1024) : 0 + +#define BNXT_RE_VF_MAX_SRQ_SUPPORTED(chip_gen) \ + chip_gen == BNXT_RE_DEFAULT ? (4 * 1024) : 0 + +static inline void bnxt_re_set_max_gid(u16 *max_sgid) +{ + *max_sgid = max_t(u32, 256, *max_sgid); + *max_sgid = min_t(u32, 256, *max_sgid); +} + +#endif diff --git a/sys/dev/bnxt/bnxt_re/qplib_tlv.h b/sys/dev/bnxt/bnxt_re/qplib_tlv.h new file mode 100644 index 000000000000..eeaea41a37e6 --- /dev/null +++ b/sys/dev/bnxt/bnxt_re/qplib_tlv.h @@ -0,0 +1,187 @@ +/* + * Copyright (c) 2017 - 2024, Broadcom. All rights reserved. The term + * Broadcom refers to Broadcom Limited and/or its subsidiaries. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR + * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE + * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN + * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#ifndef __QPLIB_TLV_H__ +#define __QPLIB_TLV_H__ + +struct roce_tlv { + struct tlv tlv; + u8 total_size; + u8 unused[7]; +}; + +#define CHUNK_SIZE 16 +#define CHUNKS(x) (((x) + CHUNK_SIZE - 1) / CHUNK_SIZE) + +#define ROCE_1ST_TLV_PREP(rtlv, tot_chunks, content_bytes, more) \ + do { \ + (rtlv)->tlv.cmd_discr = CMD_DISCR_TLV_ENCAP; \ + (rtlv)->tlv.tlv_type = TLV_TYPE_ROCE_SP_COMMAND; \ + (rtlv)->tlv.length = (content_bytes); \ + (rtlv)->tlv.flags = TLV_FLAGS_REQUIRED; \ + (rtlv)->tlv.flags |= (more) ? TLV_FLAGS_MORE : 0; \ + (rtlv)->total_size = (tot_chunks); \ + } while (0) + +#define ROCE_EXT_TLV_PREP(rtlv, ext_type, content_bytes, more, reqd) \ + do { \ + (rtlv)->tlv.cmd_discr = CMD_DISCR_TLV_ENCAP; \ + (rtlv)->tlv.tlv_type = (ext_type); \ + (rtlv)->tlv.length = (content_bytes); \ + (rtlv)->tlv.flags |= (more) ? TLV_FLAGS_MORE : 0; \ + (rtlv)->tlv.flags |= (reqd) ? TLV_FLAGS_REQUIRED : 0; \ + } while (0) + +/* + * TLV size in units of 16 byte chunks + */ +#define TLV_SIZE ((sizeof(struct roce_tlv) + 15) / 16) +/* + * TLV length in bytes + */ +#define TLV_BYTES (TLV_SIZE * 16) + +#define HAS_TLV_HEADER(msg) (((struct tlv *)(msg))->cmd_discr == CMD_DISCR_TLV_ENCAP) +#define GET_TLV_DATA(tlv) ((void *)&((uint8_t *)(tlv))[TLV_BYTES]) + +static inline u8 __get_cmdq_base_opcode(struct cmdq_base *req, u32 size) +{ + if (HAS_TLV_HEADER(req) && size > TLV_BYTES) + return ((struct cmdq_base *)GET_TLV_DATA(req))->opcode; + else + return req->opcode; +} + +static inline void __set_cmdq_base_opcode(struct cmdq_base *req, + u32 size, u8 val) +{ + if (HAS_TLV_HEADER(req) && size > TLV_BYTES) + ((struct cmdq_base *)GET_TLV_DATA(req))->opcode = val; + else + req->opcode = val; +} + +static inline __le16 __get_cmdq_base_cookie(struct cmdq_base *req, u32 size) +{ + if (HAS_TLV_HEADER(req) && size > TLV_BYTES) + return ((struct cmdq_base *)GET_TLV_DATA(req))->cookie; + else + return req->cookie; +} + +static inline void __set_cmdq_base_cookie(struct cmdq_base *req, + u32 size, __le16 val) +{ + if (HAS_TLV_HEADER(req) && size > TLV_BYTES) + ((struct cmdq_base *)GET_TLV_DATA(req))->cookie = val; + else + req->cookie = val; +} + +static inline __le64 __get_cmdq_base_resp_addr(struct cmdq_base *req, u32 size) +{ + if (HAS_TLV_HEADER(req) && size > TLV_BYTES) + return ((struct cmdq_base *)GET_TLV_DATA(req))->resp_addr; + else + return req->resp_addr; +} + +static inline void __set_cmdq_base_resp_addr(struct cmdq_base *req, + u32 size, __le64 val) +{ + if (HAS_TLV_HEADER(req) && size > TLV_BYTES) + ((struct cmdq_base *)GET_TLV_DATA(req))->resp_addr = val; + else + req->resp_addr = val; +} + +static inline u8 __get_cmdq_base_resp_size(struct cmdq_base *req, u32 size) +{ + if (HAS_TLV_HEADER(req) && size > TLV_BYTES) + return ((struct cmdq_base *)GET_TLV_DATA(req))->resp_size; + else + return req->resp_size; +} + +static inline void __set_cmdq_base_resp_size(struct cmdq_base *req, + u32 size, u8 val) +{ + if (HAS_TLV_HEADER(req) && size > TLV_BYTES) + ((struct cmdq_base *)GET_TLV_DATA(req))->resp_size = val; + else + req->resp_size = val; +} + +static inline u8 __get_cmdq_base_cmd_size(struct cmdq_base *req, u32 size) +{ + if (HAS_TLV_HEADER(req) && size > TLV_BYTES) + return ((struct roce_tlv *)(req))->total_size; + else + return req->cmd_size; +} + +static inline void __set_cmdq_base_cmd_size(struct cmdq_base *req, + u32 size, u8 val) +{ + if (HAS_TLV_HEADER(req) && size > TLV_BYTES) + ((struct cmdq_base *)GET_TLV_DATA(req))->cmd_size = val; + else + req->cmd_size = val; +} + +static inline __le16 __get_cmdq_base_flags(struct cmdq_base *req, u32 size) +{ + if (HAS_TLV_HEADER(req) && size > TLV_BYTES) + return ((struct cmdq_base *)GET_TLV_DATA(req))->flags; + else + return req->flags; +} + +static inline void __set_cmdq_base_flags(struct cmdq_base *req, + u32 size, __le16 val) +{ + if (HAS_TLV_HEADER(req) && size > TLV_BYTES) + ((struct cmdq_base *)GET_TLV_DATA(req))->flags = val; + else + req->flags = val; +} + +struct bnxt_qplib_tlv_modify_cc_req { + struct roce_tlv tlv_hdr; + struct cmdq_modify_roce_cc base_req; + __le64 tlvpad; + struct cmdq_modify_roce_cc_gen1_tlv ext_req; +}; + +struct bnxt_qplib_tlv_query_rcc_sb { + struct roce_tlv tlv_hdr; + struct creq_query_roce_cc_resp_sb base_sb; + struct creq_query_roce_cc_gen1_resp_sb_tlv gen1_sb; +}; +#endif diff --git a/sys/dev/bnxt/bnxt_re/stats.c b/sys/dev/bnxt/bnxt_re/stats.c new file mode 100644 index 000000000000..7b0e6097aae6 --- /dev/null +++ b/sys/dev/bnxt/bnxt_re/stats.c @@ -0,0 +1,773 @@ +/* + * Copyright (c) 2015-2024, Broadcom. All rights reserved. The term + * Broadcom refers to Broadcom Limited and/or its subsidiaries. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR + * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE + * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN + * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * Description: statistics related functions + */ + +#include "bnxt_re.h" +#include "bnxt.h" + +int bnxt_re_get_flow_stats_from_service_pf(struct bnxt_re_dev *rdev, + struct bnxt_re_flow_counters *stats, + struct bnxt_qplib_query_stats_info *sinfo) +{ + struct hwrm_cfa_flow_stats_output resp = {}; + struct hwrm_cfa_flow_stats_input req = {}; + struct bnxt_en_dev *en_dev = rdev->en_dev; + struct bnxt_fw_msg fw_msg = {}; + u16 target_id; + int rc = 0; + + if (sinfo->function_id == 0xFFFFFFFF) + target_id = -1; + else + target_id = sinfo->function_id + 1; + + /* Issue HWRM cmd to read flow counters for CNP tx and rx */ + bnxt_re_init_hwrm_hdr(rdev, (void *)&req, HWRM_CFA_FLOW_STATS, -1, target_id); + req.num_flows = cpu_to_le16(6); + req.flow_handle_0 = cpu_to_le16(HWRM_CFA_FLOW_INFO_INPUT_FLOW_HANDLE_CNP_CNT); + req.flow_handle_1 = cpu_to_le16(HWRM_CFA_FLOW_INFO_INPUT_FLOW_HANDLE_CNP_CNT | + HWRM_CFA_FLOW_INFO_INPUT_FLOW_HANDLE_DIR_RX); + req.flow_handle_2 = cpu_to_le16(HWRM_CFA_FLOW_INFO_INPUT_FLOW_HANDLE_ROCEV1_CNT); + req.flow_handle_3 = cpu_to_le16(HWRM_CFA_FLOW_INFO_INPUT_FLOW_HANDLE_ROCEV1_CNT | + HWRM_CFA_FLOW_INFO_INPUT_FLOW_HANDLE_DIR_RX); + req.flow_handle_4 = cpu_to_le16(HWRM_CFA_FLOW_INFO_INPUT_FLOW_HANDLE_ROCEV2_CNT); + req.flow_handle_5 = cpu_to_le16(HWRM_CFA_FLOW_INFO_INPUT_FLOW_HANDLE_ROCEV2_CNT | + HWRM_CFA_FLOW_INFO_INPUT_FLOW_HANDLE_DIR_RX); + bnxt_re_fill_fw_msg(&fw_msg, &req, sizeof(req), &resp, + sizeof(resp), BNXT_RE_HWRM_CMD_TIMEOUT(rdev)); + rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg); + if (rc) { + dev_err(rdev_to_dev(rdev), + "Failed to get CFA Flow stats : rc = 0x%x\n", rc); + return rc; + } + + stats->cnp_stats.cnp_tx_pkts = le64_to_cpu(resp.packet_0); + stats->cnp_stats.cnp_tx_bytes = le64_to_cpu(resp.byte_0); + stats->cnp_stats.cnp_rx_pkts = le64_to_cpu(resp.packet_1); + stats->cnp_stats.cnp_rx_bytes = le64_to_cpu(resp.byte_1); + + stats->ro_stats.tx_pkts = le64_to_cpu(resp.packet_2) + + le64_to_cpu(resp.packet_4); + stats->ro_stats.tx_bytes = le64_to_cpu(resp.byte_2) + + le64_to_cpu(resp.byte_4); + stats->ro_stats.rx_pkts = le64_to_cpu(resp.packet_3) + + le64_to_cpu(resp.packet_5); + stats->ro_stats.rx_bytes = le64_to_cpu(resp.byte_3) + + le64_to_cpu(resp.byte_5); + + return 0; +} + +int bnxt_re_get_qos_stats(struct bnxt_re_dev *rdev) +{ + struct bnxt_re_ro_counters roce_only_tmp[2] = {{}, {}}; + struct bnxt_re_cnp_counters tmp_counters[2] = {{}, {}}; + struct hwrm_cfa_flow_stats_output resp = {}; + struct hwrm_cfa_flow_stats_input req = {}; + struct bnxt_en_dev *en_dev = rdev->en_dev; + struct bnxt_fw_msg fw_msg = {}; + struct bnxt_re_cc_stat *cnps; + struct bnxt_re_rstat *dstat; + int rc = 0; + u64 bytes; + u64 pkts; + + /* Issue HWRM cmd to read flow counters for CNP tx and rx */ + bnxt_re_init_hwrm_hdr(rdev, (void *)&req, HWRM_CFA_FLOW_STATS, -1, -1); + req.num_flows = cpu_to_le16(6); + req.flow_handle_0 = cpu_to_le16(HWRM_CFA_FLOW_INFO_INPUT_FLOW_HANDLE_CNP_CNT); + req.flow_handle_1 = cpu_to_le16(HWRM_CFA_FLOW_INFO_INPUT_FLOW_HANDLE_CNP_CNT | + HWRM_CFA_FLOW_INFO_INPUT_FLOW_HANDLE_DIR_RX); + req.flow_handle_2 = cpu_to_le16(HWRM_CFA_FLOW_INFO_INPUT_FLOW_HANDLE_ROCEV1_CNT); + req.flow_handle_3 = cpu_to_le16(HWRM_CFA_FLOW_INFO_INPUT_FLOW_HANDLE_ROCEV1_CNT | + HWRM_CFA_FLOW_INFO_INPUT_FLOW_HANDLE_DIR_RX); + req.flow_handle_4 = cpu_to_le16(HWRM_CFA_FLOW_INFO_INPUT_FLOW_HANDLE_ROCEV2_CNT); + req.flow_handle_5 = cpu_to_le16(HWRM_CFA_FLOW_INFO_INPUT_FLOW_HANDLE_ROCEV2_CNT | + HWRM_CFA_FLOW_INFO_INPUT_FLOW_HANDLE_DIR_RX); + bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp, + sizeof(resp), BNXT_RE_HWRM_CMD_TIMEOUT(rdev)); + rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg); + if (rc) { + dev_err(rdev_to_dev(rdev), + "Failed to get CFA Flow stats : rc = 0x%x\n", rc); + goto done; + } + + tmp_counters[0].cnp_tx_pkts = le64_to_cpu(resp.packet_0); + tmp_counters[0].cnp_tx_bytes = le64_to_cpu(resp.byte_0); + tmp_counters[0].cnp_rx_pkts = le64_to_cpu(resp.packet_1); + tmp_counters[0].cnp_rx_bytes = le64_to_cpu(resp.byte_1); + + roce_only_tmp[0].tx_pkts = le64_to_cpu(resp.packet_2) + + le64_to_cpu(resp.packet_4); + roce_only_tmp[0].tx_bytes = le64_to_cpu(resp.byte_2) + + le64_to_cpu(resp.byte_4); + roce_only_tmp[0].rx_pkts = le64_to_cpu(resp.packet_3) + + le64_to_cpu(resp.packet_5); + roce_only_tmp[0].rx_bytes = le64_to_cpu(resp.byte_3) + + le64_to_cpu(resp.byte_5); + + cnps = &rdev->stats.cnps; + dstat = &rdev->stats.dstat; + if (!cnps->is_first) { + /* First query done.. */ + cnps->is_first = true; + cnps->prev[0].cnp_tx_pkts = tmp_counters[0].cnp_tx_pkts; + cnps->prev[0].cnp_tx_bytes = tmp_counters[0].cnp_tx_bytes; + cnps->prev[0].cnp_rx_pkts = tmp_counters[0].cnp_rx_pkts; + cnps->prev[0].cnp_rx_bytes = tmp_counters[0].cnp_rx_bytes; + + cnps->prev[1].cnp_tx_pkts = tmp_counters[1].cnp_tx_pkts; + cnps->prev[1].cnp_tx_bytes = tmp_counters[1].cnp_tx_bytes; + cnps->prev[1].cnp_rx_pkts = tmp_counters[1].cnp_rx_pkts; + cnps->prev[1].cnp_rx_bytes = tmp_counters[1].cnp_rx_bytes; + + dstat->prev[0].tx_pkts = roce_only_tmp[0].tx_pkts; + dstat->prev[0].tx_bytes = roce_only_tmp[0].tx_bytes; + dstat->prev[0].rx_pkts = roce_only_tmp[0].rx_pkts; + dstat->prev[0].rx_bytes = roce_only_tmp[0].rx_bytes; + + dstat->prev[1].tx_pkts = roce_only_tmp[1].tx_pkts; + dstat->prev[1].tx_bytes = roce_only_tmp[1].tx_bytes; + dstat->prev[1].rx_pkts = roce_only_tmp[1].rx_pkts; + dstat->prev[1].rx_bytes = roce_only_tmp[1].rx_bytes; + } else { + u64 byte_mask, pkts_mask; + u64 diff; + + byte_mask = bnxt_re_get_cfa_stat_mask(rdev->chip_ctx, + BYTE_MASK); + pkts_mask = bnxt_re_get_cfa_stat_mask(rdev->chip_ctx, + PKTS_MASK); + /* + * Calculate the number of cnp packets and use + * the value to calculate the CRC bytes. + * Multply pkts with 4 and add it to total bytes + */ + pkts = bnxt_re_stat_diff(tmp_counters[0].cnp_tx_pkts, + &cnps->prev[0].cnp_tx_pkts, + pkts_mask); + cnps->cur[0].cnp_tx_pkts += pkts; + diff = bnxt_re_stat_diff(tmp_counters[0].cnp_tx_bytes, + &cnps->prev[0].cnp_tx_bytes, + byte_mask); + bytes = diff + pkts * 4; + cnps->cur[0].cnp_tx_bytes += bytes; + pkts = bnxt_re_stat_diff(tmp_counters[0].cnp_rx_pkts, + &cnps->prev[0].cnp_rx_pkts, + pkts_mask); + cnps->cur[0].cnp_rx_pkts += pkts; + bytes = bnxt_re_stat_diff(tmp_counters[0].cnp_rx_bytes, + &cnps->prev[0].cnp_rx_bytes, + byte_mask); + cnps->cur[0].cnp_rx_bytes += bytes; + + /* + * Calculate the number of cnp packets and use + * the value to calculate the CRC bytes. + * Multply pkts with 4 and add it to total bytes + */ + pkts = bnxt_re_stat_diff(tmp_counters[1].cnp_tx_pkts, + &cnps->prev[1].cnp_tx_pkts, + pkts_mask); + cnps->cur[1].cnp_tx_pkts += pkts; + diff = bnxt_re_stat_diff(tmp_counters[1].cnp_tx_bytes, + &cnps->prev[1].cnp_tx_bytes, + byte_mask); + cnps->cur[1].cnp_tx_bytes += diff + pkts * 4; + pkts = bnxt_re_stat_diff(tmp_counters[1].cnp_rx_pkts, + &cnps->prev[1].cnp_rx_pkts, + pkts_mask); + cnps->cur[1].cnp_rx_pkts += pkts; + bytes = bnxt_re_stat_diff(tmp_counters[1].cnp_rx_bytes, + &cnps->prev[1].cnp_rx_bytes, + byte_mask); + cnps->cur[1].cnp_rx_bytes += bytes; + + pkts = bnxt_re_stat_diff(roce_only_tmp[0].tx_pkts, + &dstat->prev[0].tx_pkts, + pkts_mask); + dstat->cur[0].tx_pkts += pkts; + diff = bnxt_re_stat_diff(roce_only_tmp[0].tx_bytes, + &dstat->prev[0].tx_bytes, + byte_mask); + dstat->cur[0].tx_bytes += diff + pkts * 4; + pkts = bnxt_re_stat_diff(roce_only_tmp[0].rx_pkts, + &dstat->prev[0].rx_pkts, + pkts_mask); + dstat->cur[0].rx_pkts += pkts; + + bytes = bnxt_re_stat_diff(roce_only_tmp[0].rx_bytes, + &dstat->prev[0].rx_bytes, + byte_mask); + dstat->cur[0].rx_bytes += bytes; + pkts = bnxt_re_stat_diff(roce_only_tmp[1].tx_pkts, + &dstat->prev[1].tx_pkts, + pkts_mask); + dstat->cur[1].tx_pkts += pkts; + diff = bnxt_re_stat_diff(roce_only_tmp[1].tx_bytes, + &dstat->prev[1].tx_bytes, + byte_mask); + dstat->cur[1].tx_bytes += diff + pkts * 4; + pkts = bnxt_re_stat_diff(roce_only_tmp[1].rx_pkts, + &dstat->prev[1].rx_pkts, + pkts_mask); + dstat->cur[1].rx_pkts += pkts; + bytes = bnxt_re_stat_diff(roce_only_tmp[1].rx_bytes, + &dstat->prev[1].rx_bytes, + byte_mask); + dstat->cur[1].rx_bytes += bytes; + } +done: + return rc; +} + +static void bnxt_re_copy_ext_stats(struct bnxt_re_dev *rdev, + u8 indx, struct bnxt_qplib_ext_stat *s) +{ + struct bnxt_re_ext_roce_stats *e_errs; + struct bnxt_re_cnp_counters *cnp; + struct bnxt_re_ext_rstat *ext_d; + struct bnxt_re_ro_counters *ro; + + cnp = &rdev->stats.cnps.cur[indx]; + ro = &rdev->stats.dstat.cur[indx]; + ext_d = &rdev->stats.dstat.ext_rstat[indx]; + e_errs = &rdev->stats.dstat.e_errs; + + cnp->cnp_tx_pkts = s->tx_cnp; + cnp->cnp_rx_pkts = s->rx_cnp; + /* In bonding mode do not duplicate other stats */ + if (indx) + return; + cnp->ecn_marked = s->rx_ecn_marked; + + ro->tx_pkts = s->tx_roce_pkts; + ro->tx_bytes = s->tx_roce_bytes; + ro->rx_pkts = s->rx_roce_pkts; + ro->rx_bytes = s->rx_roce_bytes; + + ext_d->tx.atomic_req = s->tx_atomic_req; + ext_d->tx.read_req = s->tx_read_req; + ext_d->tx.read_resp = s->tx_read_res; + ext_d->tx.write_req = s->tx_write_req; + ext_d->tx.send_req = s->tx_send_req; + ext_d->rx.atomic_req = s->rx_atomic_req; + ext_d->rx.read_req = s->rx_read_req; + ext_d->rx.read_resp = s->rx_read_res; + ext_d->rx.write_req = s->rx_write_req; + ext_d->rx.send_req = s->rx_send_req; + ext_d->grx.rx_pkts = s->rx_roce_good_pkts; + ext_d->grx.rx_bytes = s->rx_roce_good_bytes; + ext_d->rx_dcn_payload_cut = s->rx_dcn_payload_cut; + ext_d->te_bypassed = s->te_bypassed; + e_errs->oob = s->rx_out_of_buffer; + e_errs->oos = s->rx_out_of_sequence; + e_errs->seq_err_naks_rcvd = s->seq_err_naks_rcvd; + e_errs->rnr_naks_rcvd = s->rnr_naks_rcvd; + e_errs->missing_resp = s->missing_resp; + e_errs->to_retransmits = s->to_retransmits; + e_errs->dup_req = s->dup_req; +} + +static int bnxt_re_get_ext_stat(struct bnxt_re_dev *rdev) +{ + struct bnxt_qplib_ext_stat estat[2] = {{}, {}}; + struct bnxt_qplib_query_stats_info sinfo; + u32 fid; + int rc; + + fid = PCI_FUNC(rdev->en_dev->pdev->devfn); + /* Set default values for sinfo */ + sinfo.function_id = 0xFFFFFFFF; + sinfo.collection_id = 0xFF; + sinfo.vf_valid = false; + rc = bnxt_qplib_qext_stat(&rdev->rcfw, fid, &estat[0], &sinfo); + if (rc) + goto done; + bnxt_re_copy_ext_stats(rdev, 0, &estat[0]); + +done: + return rc; +} + +static void bnxt_re_copy_rstat(struct bnxt_re_rdata_counters *d, + struct ctx_hw_stats_ext *s, + bool is_thor) +{ + d->tx_ucast_pkts = le64_to_cpu(s->tx_ucast_pkts); + d->tx_mcast_pkts = le64_to_cpu(s->tx_mcast_pkts); + d->tx_bcast_pkts = le64_to_cpu(s->tx_bcast_pkts); + d->tx_discard_pkts = le64_to_cpu(s->tx_discard_pkts); + d->tx_error_pkts = le64_to_cpu(s->tx_error_pkts); + d->tx_ucast_bytes = le64_to_cpu(s->tx_ucast_bytes); + /* Add four bytes of CRC bytes per packet */ + d->tx_ucast_bytes += d->tx_ucast_pkts * 4; + d->tx_mcast_bytes = le64_to_cpu(s->tx_mcast_bytes); + d->tx_bcast_bytes = le64_to_cpu(s->tx_bcast_bytes); + d->rx_ucast_pkts = le64_to_cpu(s->rx_ucast_pkts); + d->rx_mcast_pkts = le64_to_cpu(s->rx_mcast_pkts); + d->rx_bcast_pkts = le64_to_cpu(s->rx_bcast_pkts); + d->rx_discard_pkts = le64_to_cpu(s->rx_discard_pkts); + d->rx_error_pkts = le64_to_cpu(s->rx_error_pkts); + d->rx_ucast_bytes = le64_to_cpu(s->rx_ucast_bytes); + d->rx_mcast_bytes = le64_to_cpu(s->rx_mcast_bytes); + d->rx_bcast_bytes = le64_to_cpu(s->rx_bcast_bytes); + if (is_thor) { + d->rx_agg_pkts = le64_to_cpu(s->rx_tpa_pkt); + d->rx_agg_bytes = le64_to_cpu(s->rx_tpa_bytes); + d->rx_agg_events = le64_to_cpu(s->rx_tpa_events); + d->rx_agg_aborts = le64_to_cpu(s->rx_tpa_errors); + } +} + +static void bnxt_re_get_roce_data_stats(struct bnxt_re_dev *rdev) +{ + bool is_thor = _is_chip_gen_p5_p7(rdev->chip_ctx); + struct bnxt_re_rdata_counters *rstat; + + rstat = &rdev->stats.dstat.rstat[0]; + bnxt_re_copy_rstat(rstat, rdev->qplib_res.hctx->stats.dma, is_thor); + +} + +int bnxt_re_get_device_stats(struct bnxt_re_dev *rdev) +{ + struct bnxt_qplib_query_stats_info sinfo; + int rc = 0; + + /* Stats are in 1s cadence */ + if (test_bit(BNXT_RE_FLAG_ISSUE_CFA_FLOW_STATS, &rdev->flags)) { + if (bnxt_ext_stats_supported(rdev->chip_ctx, rdev->dev_attr->dev_cap_flags, + rdev->is_virtfn)) + rc = bnxt_re_get_ext_stat(rdev); + else + rc = bnxt_re_get_qos_stats(rdev); + + if (rc && rc != -ENOMEM) + clear_bit(BNXT_RE_FLAG_ISSUE_CFA_FLOW_STATS, + &rdev->flags); + } + + if (test_bit(BNXT_RE_FLAG_ISSUE_ROCE_STATS, &rdev->flags)) { + bnxt_re_get_roce_data_stats(rdev); + + /* Set default values for sinfo */ + sinfo.function_id = 0xFFFFFFFF; + sinfo.collection_id = 0xFF; + sinfo.vf_valid = false; + rc = bnxt_qplib_get_roce_error_stats(&rdev->rcfw, + &rdev->stats.dstat.errs, + &sinfo); + if (rc && rc != -ENOMEM) + clear_bit(BNXT_RE_FLAG_ISSUE_ROCE_STATS, + &rdev->flags); + } + + return rc; +} + +static const char * const bnxt_re_stat_descs[] = { + "link_state", + "max_qp", + "max_srq", + "max_cq", + "max_mr", + "max_mw", + "max_ah", + "max_pd", + "active_qp", + "active_rc_qp", + "active_ud_qp", + "active_srq", + "active_cq", + "active_mr", + "active_mw", + "active_ah", + "active_pd", + "qp_watermark", + "rc_qp_watermark", + "ud_qp_watermark", + "srq_watermark", + "cq_watermark", + "mr_watermark", + "mw_watermark", + "ah_watermark", + "pd_watermark", + "resize_cq_count", + "hw_retransmission", + "recoverable_errors", + "rx_pkts", + "rx_bytes", + "tx_pkts", + "tx_bytes", + "cnp_tx_pkts", + "cnp_tx_bytes", + "cnp_rx_pkts", + "cnp_rx_bytes", + "roce_only_rx_pkts", + "roce_only_rx_bytes", + "roce_only_tx_pkts", + "roce_only_tx_bytes", + "rx_roce_error_pkts", + "rx_roce_discard_pkts", + "tx_roce_error_pkts", + "tx_roce_discards_pkts", + "res_oob_drop_count", + "tx_atomic_req", + "rx_atomic_req", + "tx_read_req", + "tx_read_resp", + "rx_read_req", + "rx_read_resp", + "tx_write_req", + "rx_write_req", + "tx_send_req", + "rx_send_req", + "rx_good_pkts", + "rx_good_bytes", + "rx_dcn_payload_cut", + "te_bypassed", + "rx_ecn_marked_pkts", + "max_retry_exceeded", + "to_retransmits", + "seq_err_naks_rcvd", + "rnr_naks_rcvd", + "missing_resp", + "dup_reqs", + "unrecoverable_err", + "bad_resp_err", + "local_qp_op_err", + "local_protection_err", + "mem_mgmt_op_err", + "remote_invalid_req_err", + "remote_access_err", + "remote_op_err", + "res_exceed_max", + "res_length_mismatch", + "res_exceeds_wqe", + "res_opcode_err", + "res_rx_invalid_rkey", + "res_rx_domain_err", + "res_rx_no_perm", + "res_rx_range_err", + "res_tx_invalid_rkey", + "res_tx_domain_err", + "res_tx_no_perm", + "res_tx_range_err", + "res_irrq_oflow", + "res_unsup_opcode", + "res_unaligned_atomic", + "res_rem_inv_err", + "res_mem_error64", + "res_srq_err", + "res_cmp_err", + "res_invalid_dup_rkey", + "res_wqe_format_err", + "res_cq_load_err", + "res_srq_load_err", + "res_tx_pci_err", + "res_rx_pci_err", + "res_oos_drop_count", + "num_irq_started", + "num_irq_stopped", + "poll_in_intr_en", + "poll_in_intr_dis", + "cmdq_full_dbg_cnt", + "fw_service_prof_type_sup", + "dbq_int_recv", + "dbq_int_en", + "dbq_pacing_resched", + "dbq_pacing_complete", + "dbq_pacing_alerts", + "dbq_dbr_fifo_reg" + +}; + +static void bnxt_re_print_ext_stat(struct bnxt_re_dev *rdev, + struct rdma_hw_stats *stats) +{ + struct bnxt_re_cnp_counters *cnp; + struct bnxt_re_ext_rstat *ext_s; + + ext_s = &rdev->stats.dstat.ext_rstat[0]; + cnp = &rdev->stats.cnps.cur[0]; + + stats->value[BNXT_RE_TX_ATOMIC_REQ] = ext_s->tx.atomic_req; + stats->value[BNXT_RE_RX_ATOMIC_REQ] = ext_s->rx.atomic_req; + stats->value[BNXT_RE_TX_READ_REQ] = ext_s->tx.read_req; + stats->value[BNXT_RE_TX_READ_RESP] = ext_s->tx.read_resp; + stats->value[BNXT_RE_RX_READ_REQ] = ext_s->rx.read_req; + stats->value[BNXT_RE_RX_READ_RESP] = ext_s->rx.read_resp; + stats->value[BNXT_RE_TX_WRITE_REQ] = ext_s->tx.write_req; + stats->value[BNXT_RE_RX_WRITE_REQ] = ext_s->rx.write_req; + stats->value[BNXT_RE_TX_SEND_REQ] = ext_s->tx.send_req; + stats->value[BNXT_RE_RX_SEND_REQ] = ext_s->rx.send_req; + stats->value[BNXT_RE_RX_GOOD_PKTS] = ext_s->grx.rx_pkts; + stats->value[BNXT_RE_RX_GOOD_BYTES] = ext_s->grx.rx_bytes; + if (_is_chip_p7(rdev->chip_ctx)) { + stats->value[BNXT_RE_RX_DCN_PAYLOAD_CUT] = ext_s->rx_dcn_payload_cut; + stats->value[BNXT_RE_TE_BYPASSED] = ext_s->te_bypassed; + } + stats->value[BNXT_RE_RX_ECN_MARKED_PKTS] = cnp->ecn_marked; +} + +static void bnxt_re_print_roce_only_counters(struct bnxt_re_dev *rdev, + struct rdma_hw_stats *stats) +{ + struct bnxt_re_ro_counters *roce_only = &rdev->stats.dstat.cur[0]; + + stats->value[BNXT_RE_ROCE_ONLY_RX_PKTS] = roce_only->rx_pkts; + stats->value[BNXT_RE_ROCE_ONLY_RX_BYTES] = roce_only->rx_bytes; + stats->value[BNXT_RE_ROCE_ONLY_TX_PKTS] = roce_only->tx_pkts; + stats->value[BNXT_RE_ROCE_ONLY_TX_BYTES] = roce_only->tx_bytes; +} + +static void bnxt_re_print_normal_total_counters(struct bnxt_re_dev *rdev, + struct rdma_hw_stats *stats) +{ + struct bnxt_re_ro_counters *roce_only; + struct bnxt_re_cc_stat *cnps; + + cnps = &rdev->stats.cnps; + roce_only = &rdev->stats.dstat.cur[0]; + + stats->value[BNXT_RE_RX_PKTS] = cnps->cur[0].cnp_rx_pkts + roce_only->rx_pkts; + stats->value[BNXT_RE_RX_BYTES] = cnps->cur[0].cnp_rx_bytes + roce_only->rx_bytes; + stats->value[BNXT_RE_TX_PKTS] = cnps->cur[0].cnp_tx_pkts + roce_only->tx_pkts; + stats->value[BNXT_RE_TX_BYTES] = cnps->cur[0].cnp_tx_bytes + roce_only->tx_bytes; +} + +static void bnxt_re_print_normal_counters(struct bnxt_re_dev *rdev, + struct rdma_hw_stats *rstats) +{ + struct bnxt_re_rdata_counters *stats; + struct bnxt_re_cc_stat *cnps; + bool en_disp; + + stats = &rdev->stats.dstat.rstat[0]; + cnps = &rdev->stats.cnps; + en_disp = !_is_chip_gen_p5_p7(rdev->chip_ctx); + + bnxt_re_print_normal_total_counters(rdev, rstats); + if (!rdev->is_virtfn) { + rstats->value[BNXT_RE_CNP_TX_PKTS] = cnps->cur[0].cnp_tx_pkts; + if (en_disp) + rstats->value[BNXT_RE_CNP_TX_BYTES] = cnps->cur[0].cnp_tx_bytes; + rstats->value[BNXT_RE_CNP_RX_PKTS] = cnps->cur[0].cnp_rx_pkts; + if (en_disp) + rstats->value[BNXT_RE_CNP_RX_BYTES] = cnps->cur[0].cnp_rx_bytes; + } + /* Print RoCE only bytes.. CNP counters include RoCE packets also */ + bnxt_re_print_roce_only_counters(rdev, rstats); + + rstats->value[BNXT_RE_RX_ROCE_ERROR_PKTS] = stats ? stats->rx_error_pkts : 0; + rstats->value[BNXT_RE_RX_ROCE_DISCARD_PKTS] = stats ? stats->rx_discard_pkts : 0; + if (!en_disp) { + rstats->value[BNXT_RE_TX_ROCE_ERROR_PKTS] = stats ? stats->tx_error_pkts : 0; + rstats->value[BNXT_RE_TX_ROCE_DISCARDS_PKTS] = stats ? stats->tx_discard_pkts : 0; + } + + if (bnxt_ext_stats_supported(rdev->chip_ctx, rdev->dev_attr->dev_cap_flags, + rdev->is_virtfn)) { + rstats->value[BNXT_RE_RES_OOB_DROP_COUNT] = rdev->stats.dstat.e_errs.oob; + bnxt_re_print_ext_stat(rdev, rstats); + } +} + +static void bnxt_re_copy_db_pacing_stats(struct bnxt_re_dev *rdev, + struct rdma_hw_stats *stats) +{ + struct bnxt_re_dbr_sw_stats *dbr_sw_stats = rdev->dbr_sw_stats; + + stats->value[BNXT_RE_DBQ_PACING_RESCHED] = dbr_sw_stats->dbq_pacing_resched; + stats->value[BNXT_RE_DBQ_PACING_CMPL] = dbr_sw_stats->dbq_pacing_complete; + stats->value[BNXT_RE_DBQ_PACING_ALERT] = dbr_sw_stats->dbq_pacing_alerts; + stats->value[BNXT_RE_DBQ_DBR_FIFO_REG] = readl_fbsd(rdev->en_dev->softc, + rdev->dbr_db_fifo_reg_off, 0); +} + +int bnxt_re_get_hw_stats(struct ib_device *ibdev, + struct rdma_hw_stats *stats, + u8 port, int index) +{ + struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev); + struct bnxt_re_ext_roce_stats *e_errs; + struct bnxt_re_rdata_counters *rstat; + struct bnxt_qplib_roce_stats *errs; + unsigned long tstamp_diff; + struct pci_dev *pdev; + int sched_msec; + int rc = 0; + + if (!port || !stats) + return -EINVAL; + + if (!rdev) + return -ENODEV; + + if (!__bnxt_re_is_rdev_valid(rdev)) { + return -ENODEV; + } + + pdev = rdev->en_dev->pdev; + errs = &rdev->stats.dstat.errs; + rstat = &rdev->stats.dstat.rstat[0]; + e_errs = &rdev->stats.dstat.e_errs; +#define BNXT_RE_STATS_CTX_UPDATE_TIMER 250 + sched_msec = BNXT_RE_STATS_CTX_UPDATE_TIMER; + tstamp_diff = jiffies - rdev->stats.read_tstamp; + if (test_bit(BNXT_RE_FLAG_IBDEV_REGISTERED, &rdev->flags)) { + if (/* restrict_stats && */ tstamp_diff < msecs_to_jiffies(sched_msec)) + goto skip_query; + rc = bnxt_re_get_device_stats(rdev); + if (rc) + dev_err(rdev_to_dev(rdev), + "Failed to query device stats\n"); + rdev->stats.read_tstamp = jiffies; + } + + if (rdev->dbr_pacing) + bnxt_re_copy_db_pacing_stats(rdev, stats); + +skip_query: + + if (rdev->netdev) + stats->value[BNXT_RE_LINK_STATE] = bnxt_re_link_state(rdev); + stats->value[BNXT_RE_MAX_QP] = rdev->dev_attr->max_qp; + stats->value[BNXT_RE_MAX_SRQ] = rdev->dev_attr->max_srq; + stats->value[BNXT_RE_MAX_CQ] = rdev->dev_attr->max_cq; + stats->value[BNXT_RE_MAX_MR] = rdev->dev_attr->max_mr; + stats->value[BNXT_RE_MAX_MW] = rdev->dev_attr->max_mw; + stats->value[BNXT_RE_MAX_AH] = rdev->dev_attr->max_ah; + stats->value[BNXT_RE_MAX_PD] = rdev->dev_attr->max_pd; + stats->value[BNXT_RE_ACTIVE_QP] = atomic_read(&rdev->stats.rsors.qp_count); + stats->value[BNXT_RE_ACTIVE_RC_QP] = atomic_read(&rdev->stats.rsors.rc_qp_count); + stats->value[BNXT_RE_ACTIVE_UD_QP] = atomic_read(&rdev->stats.rsors.ud_qp_count); + stats->value[BNXT_RE_ACTIVE_SRQ] = atomic_read(&rdev->stats.rsors.srq_count); + stats->value[BNXT_RE_ACTIVE_CQ] = atomic_read(&rdev->stats.rsors.cq_count); + stats->value[BNXT_RE_ACTIVE_MR] = atomic_read(&rdev->stats.rsors.mr_count); + stats->value[BNXT_RE_ACTIVE_MW] = atomic_read(&rdev->stats.rsors.mw_count); + stats->value[BNXT_RE_ACTIVE_AH] = atomic_read(&rdev->stats.rsors.ah_count); + stats->value[BNXT_RE_ACTIVE_PD] = atomic_read(&rdev->stats.rsors.pd_count); + stats->value[BNXT_RE_QP_WATERMARK] = atomic_read(&rdev->stats.rsors.max_qp_count); + stats->value[BNXT_RE_RC_QP_WATERMARK] = atomic_read(&rdev->stats.rsors.max_rc_qp_count); + stats->value[BNXT_RE_UD_QP_WATERMARK] = atomic_read(&rdev->stats.rsors.max_ud_qp_count); + stats->value[BNXT_RE_SRQ_WATERMARK] = atomic_read(&rdev->stats.rsors.max_srq_count); + stats->value[BNXT_RE_CQ_WATERMARK] = atomic_read(&rdev->stats.rsors.max_cq_count); + stats->value[BNXT_RE_MR_WATERMARK] = atomic_read(&rdev->stats.rsors.max_mr_count); + stats->value[BNXT_RE_MW_WATERMARK] = atomic_read(&rdev->stats.rsors.max_mw_count); + stats->value[BNXT_RE_AH_WATERMARK] = atomic_read(&rdev->stats.rsors.max_ah_count); + stats->value[BNXT_RE_PD_WATERMARK] = atomic_read(&rdev->stats.rsors.max_pd_count); + stats->value[BNXT_RE_RESIZE_CQ_COUNT] = atomic_read(&rdev->stats.rsors.resize_count); + stats->value[BNXT_RE_HW_RETRANSMISSION] = BNXT_RE_HW_RETX(rdev->dev_attr->dev_cap_flags) ? 1 : 0; + stats->value[BNXT_RE_RECOVERABLE_ERRORS] = rstat ? rstat->tx_bcast_pkts : 0; + + bnxt_re_print_normal_counters(rdev, stats); + + + stats->value[BNXT_RE_MAX_RETRY_EXCEEDED] = errs->max_retry_exceeded; + if (bnxt_ext_stats_supported(rdev->chip_ctx, rdev->dev_attr->dev_cap_flags, + rdev->is_virtfn) && + _is_hw_retx_supported(rdev->dev_attr->dev_cap_flags)) { + stats->value[BNXT_RE_TO_RETRANSMITS] = e_errs->to_retransmits; + stats->value[BNXT_RE_SEQ_ERR_NAKS_RCVD] = e_errs->seq_err_naks_rcvd; + stats->value[BNXT_RE_RNR_NAKS_RCVD] = e_errs->rnr_naks_rcvd; + stats->value[BNXT_RE_MISSING_RESP] = e_errs->missing_resp; + stats->value[BNXT_RE_DUP_REQS] = e_errs->dup_req; + } else { + stats->value[BNXT_RE_TO_RETRANSMITS] = errs->to_retransmits; + stats->value[BNXT_RE_SEQ_ERR_NAKS_RCVD] = errs->seq_err_naks_rcvd; + stats->value[BNXT_RE_RNR_NAKS_RCVD] = errs->rnr_naks_rcvd; + stats->value[BNXT_RE_MISSING_RESP] = errs->missing_resp; + stats->value[BNXT_RE_DUP_REQS] = errs->dup_req; + } + + stats->value[BNXT_RE_UNRECOVERABLE_ERR] = errs->unrecoverable_err; + stats->value[BNXT_RE_BAD_RESP_ERR] = errs->bad_resp_err; + stats->value[BNXT_RE_LOCAL_QP_OP_ERR] = errs->local_qp_op_err; + stats->value[BNXT_RE_LOCAL_PROTECTION_ERR] = errs->local_protection_err; + stats->value[BNXT_RE_MEM_MGMT_OP_ERR] = errs->mem_mgmt_op_err; + stats->value[BNXT_RE_REMOTE_INVALID_REQ_ERR] = errs->remote_invalid_req_err; + stats->value[BNXT_RE_REMOTE_ACCESS_ERR] = errs->remote_access_err; + stats->value[BNXT_RE_REMOTE_OP_ERR] = errs->remote_op_err; + stats->value[BNXT_RE_RES_EXCEED_MAX] = errs->res_exceed_max; + stats->value[BNXT_RE_RES_LENGTH_MISMATCH] = errs->res_length_mismatch; + stats->value[BNXT_RE_RES_EXCEEDS_WQE] = errs->res_exceeds_wqe; + stats->value[BNXT_RE_RES_OPCODE_ERR] = errs->res_opcode_err; + stats->value[BNXT_RE_RES_RX_INVALID_RKEY] = errs->res_rx_invalid_rkey; + stats->value[BNXT_RE_RES_RX_DOMAIN_ERR] = errs->res_rx_domain_err; + stats->value[BNXT_RE_RES_RX_NO_PERM] = errs->res_rx_no_perm; + stats->value[BNXT_RE_RES_RX_RANGE_ERR] = errs->res_rx_range_err; + stats->value[BNXT_RE_RES_TX_INVALID_RKEY] = errs->res_tx_invalid_rkey; + stats->value[BNXT_RE_RES_TX_DOMAIN_ERR] = errs->res_tx_domain_err; + stats->value[BNXT_RE_RES_TX_NO_PERM] = errs->res_tx_no_perm; + stats->value[BNXT_RE_RES_TX_RANGE_ERR] = errs->res_tx_range_err; + stats->value[BNXT_RE_RES_IRRQ_OFLOW] = errs->res_irrq_oflow; + stats->value[BNXT_RE_RES_UNSUP_OPCODE] = errs->res_unsup_opcode; + stats->value[BNXT_RE_RES_UNALIGNED_ATOMIC] = errs->res_unaligned_atomic; + stats->value[BNXT_RE_RES_REM_INV_ERR] = errs->res_rem_inv_err; + stats->value[BNXT_RE_RES_MEM_ERROR64] = errs->res_mem_error; + stats->value[BNXT_RE_RES_SRQ_ERR] = errs->res_srq_err; + stats->value[BNXT_RE_RES_CMP_ERR] = errs->res_cmp_err; + stats->value[BNXT_RE_RES_INVALID_DUP_RKEY] = errs->res_invalid_dup_rkey; + stats->value[BNXT_RE_RES_WQE_FORMAT_ERR] = errs->res_wqe_format_err; + stats->value[BNXT_RE_RES_CQ_LOAD_ERR] = errs->res_cq_load_err; + stats->value[BNXT_RE_RES_SRQ_LOAD_ERR] = errs->res_srq_load_err; + stats->value[BNXT_RE_RES_TX_PCI_ERR] = errs->res_tx_pci_err; + stats->value[BNXT_RE_RES_RX_PCI_ERR] = errs->res_rx_pci_err; + + + if (bnxt_ext_stats_supported(rdev->chip_ctx, rdev->dev_attr->dev_cap_flags, + rdev->is_virtfn)) { + stats->value[BNXT_RE_RES_OOS_DROP_COUNT] = e_errs->oos; + } else { + /* Display on function 0 as OOS counters are chip-wide */ + if (PCI_FUNC(pdev->devfn) == 0) + stats->value[BNXT_RE_RES_OOS_DROP_COUNT] = errs->res_oos_drop_count; + } + stats->value[BNXT_RE_NUM_IRQ_STARTED] = rdev->rcfw.num_irq_started; + stats->value[BNXT_RE_NUM_IRQ_STOPPED] = rdev->rcfw.num_irq_stopped; + stats->value[BNXT_RE_POLL_IN_INTR_EN] = rdev->rcfw.poll_in_intr_en; + stats->value[BNXT_RE_POLL_IN_INTR_DIS] = rdev->rcfw.poll_in_intr_dis; + stats->value[BNXT_RE_CMDQ_FULL_DBG_CNT] = rdev->rcfw.cmdq_full_dbg; + if (!rdev->is_virtfn) + stats->value[BNXT_RE_FW_SERVICE_PROF_TYPE_SUP] = is_qport_service_type_supported(rdev); + + return ARRAY_SIZE(bnxt_re_stat_descs); +} + +struct rdma_hw_stats *bnxt_re_alloc_hw_port_stats(struct ib_device *ibdev, + u8 port_num) +{ + return rdma_alloc_hw_stats_struct(bnxt_re_stat_descs, + ARRAY_SIZE(bnxt_re_stat_descs), + RDMA_HW_STATS_DEFAULT_LIFESPAN); +} diff --git a/sys/dev/bnxt/bnxt_re/stats.h b/sys/dev/bnxt/bnxt_re/stats.h new file mode 100644 index 000000000000..748d8165947b --- /dev/null +++ b/sys/dev/bnxt/bnxt_re/stats.h @@ -0,0 +1,353 @@ +/* + * Copyright (c) 2015-2024, Broadcom. All rights reserved. The term + * Broadcom refers to Broadcom Limited and/or its subsidiaries. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR + * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE + * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN + * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * Description: statistics related data structures + */ + +#ifndef __STATS_H__ +#define __STATS_H__ + +#define BNXT_RE_CFA_STAT_BYTES_MASK 0xFFFFFFFFF +#define BNXT_RE_CFA_STAT_PKTS_MASK 0xFFFFFFF +enum { + BYTE_MASK = 0, + PKTS_MASK = 1 +}; + +struct bnxt_re_cnp_counters { + u64 cnp_tx_pkts; + u64 cnp_tx_bytes; + u64 cnp_rx_pkts; + u64 cnp_rx_bytes; + u64 ecn_marked; +}; + +struct bnxt_re_ro_counters { + u64 tx_pkts; + u64 tx_bytes; + u64 rx_pkts; + u64 rx_bytes; +}; + +struct bnxt_re_flow_counters { + struct bnxt_re_ro_counters ro_stats; + struct bnxt_re_cnp_counters cnp_stats; +}; + +struct bnxt_re_ext_cntr { + u64 atomic_req; + u64 read_req; + u64 read_resp; + u64 write_req; + u64 send_req; +}; + +struct bnxt_re_ext_good { + u64 rx_pkts; + u64 rx_bytes; +}; + +struct bnxt_re_ext_rstat { + struct bnxt_re_ext_cntr tx; + struct bnxt_re_ext_cntr rx; + struct bnxt_re_ext_good grx; + u64 rx_dcn_payload_cut; + u64 te_bypassed; +}; + +struct bnxt_re_rdata_counters { + u64 tx_ucast_pkts; + u64 tx_mcast_pkts; + u64 tx_bcast_pkts; + u64 tx_discard_pkts; + u64 tx_error_pkts; + u64 tx_ucast_bytes; + u64 tx_mcast_bytes; + u64 tx_bcast_bytes; + u64 rx_ucast_pkts; + u64 rx_mcast_pkts; + u64 rx_bcast_pkts; + u64 rx_discard_pkts; + u64 rx_error_pkts; + u64 rx_ucast_bytes; + u64 rx_mcast_bytes; + u64 rx_bcast_bytes; + u64 rx_agg_pkts; + u64 rx_agg_bytes; + u64 rx_agg_events; + u64 rx_agg_aborts; +}; + +struct bnxt_re_cc_stat { + struct bnxt_re_cnp_counters prev[2]; + struct bnxt_re_cnp_counters cur[2]; + bool is_first; +}; + +struct bnxt_re_ext_roce_stats { + u64 oob; + u64 oos; + u64 seq_err_naks_rcvd; + u64 rnr_naks_rcvd; + u64 missing_resp; + u64 to_retransmits; + u64 dup_req; +}; + +struct bnxt_re_rstat { + struct bnxt_re_ro_counters prev[2]; + struct bnxt_re_ro_counters cur[2]; + struct bnxt_re_rdata_counters rstat[2]; + struct bnxt_re_ext_rstat ext_rstat[2]; + struct bnxt_re_ext_roce_stats e_errs; + struct bnxt_qplib_roce_stats errs; + unsigned long long prev_oob; +}; + +struct bnxt_re_res_cntrs { + atomic_t qp_count; + atomic_t rc_qp_count; + atomic_t ud_qp_count; + atomic_t cq_count; + atomic_t srq_count; + atomic_t mr_count; + atomic_t mw_count; + atomic_t ah_count; + atomic_t pd_count; + atomic_t resize_count; + atomic_t max_qp_count; + atomic_t max_rc_qp_count; + atomic_t max_ud_qp_count; + atomic_t max_cq_count; + atomic_t max_srq_count; + atomic_t max_mr_count; + atomic_t max_mw_count; + atomic_t max_ah_count; + atomic_t max_pd_count; +}; + +struct bnxt_re_device_stats { + struct bnxt_re_rstat dstat; + struct bnxt_re_res_cntrs rsors; + struct bnxt_re_cc_stat cnps; + unsigned long read_tstamp; + /* To be used in case to disable stats query from worker or change + * query interval. 0 means stats_query disabled. + */ + u32 stats_query_sec; + /* A free running counter to be used along with stats_query_sec to + * decide whether to issue the command to FW. + */ + u32 stats_query_counter; +}; + +static inline u64 bnxt_re_get_cfa_stat_mask(struct bnxt_qplib_chip_ctx *cctx, + bool type) +{ + u64 mask; + + if (type == BYTE_MASK) { + mask = BNXT_RE_CFA_STAT_BYTES_MASK; /* 36 bits */ + if (_is_chip_gen_p5_p7(cctx)) + mask >>= 0x01; /* 35 bits */ + } else { + mask = BNXT_RE_CFA_STAT_PKTS_MASK; /* 28 bits */ + if (_is_chip_gen_p5_p7(cctx)) + mask |= (0x10000000ULL); /* 29 bits */ + } + + return mask; +} + +static inline u64 bnxt_re_stat_diff(u64 cur, u64 *prev, u64 mask) +{ + u64 diff; + + if (!cur) + return 0; + diff = (cur - *prev) & mask; + if (diff) + *prev = cur; + return diff; +} + +static inline void bnxt_re_clear_rsors_stat(struct bnxt_re_res_cntrs *rsors) +{ + atomic_set(&rsors->qp_count, 0); + atomic_set(&rsors->cq_count, 0); + atomic_set(&rsors->srq_count, 0); + atomic_set(&rsors->mr_count, 0); + atomic_set(&rsors->mw_count, 0); + atomic_set(&rsors->ah_count, 0); + atomic_set(&rsors->pd_count, 0); + atomic_set(&rsors->resize_count, 0); + atomic_set(&rsors->max_qp_count, 0); + atomic_set(&rsors->max_cq_count, 0); + atomic_set(&rsors->max_srq_count, 0); + atomic_set(&rsors->max_mr_count, 0); + atomic_set(&rsors->max_mw_count, 0); + atomic_set(&rsors->max_ah_count, 0); + atomic_set(&rsors->max_pd_count, 0); + atomic_set(&rsors->max_rc_qp_count, 0); + atomic_set(&rsors->max_ud_qp_count, 0); +} + +enum bnxt_re_hw_stats { + BNXT_RE_LINK_STATE, + BNXT_RE_MAX_QP, + BNXT_RE_MAX_SRQ, + BNXT_RE_MAX_CQ, + BNXT_RE_MAX_MR, + BNXT_RE_MAX_MW, + BNXT_RE_MAX_AH, + BNXT_RE_MAX_PD, + BNXT_RE_ACTIVE_QP, + BNXT_RE_ACTIVE_RC_QP, + BNXT_RE_ACTIVE_UD_QP, + BNXT_RE_ACTIVE_SRQ, + BNXT_RE_ACTIVE_CQ, + BNXT_RE_ACTIVE_MR, + BNXT_RE_ACTIVE_MW, + BNXT_RE_ACTIVE_AH, + BNXT_RE_ACTIVE_PD, + BNXT_RE_QP_WATERMARK, + BNXT_RE_RC_QP_WATERMARK, + BNXT_RE_UD_QP_WATERMARK, + BNXT_RE_SRQ_WATERMARK, + BNXT_RE_CQ_WATERMARK, + BNXT_RE_MR_WATERMARK, + BNXT_RE_MW_WATERMARK, + BNXT_RE_AH_WATERMARK, + BNXT_RE_PD_WATERMARK, + BNXT_RE_RESIZE_CQ_COUNT, + BNXT_RE_HW_RETRANSMISSION, + BNXT_RE_RECOVERABLE_ERRORS, + BNXT_RE_RX_PKTS, + BNXT_RE_RX_BYTES, + BNXT_RE_TX_PKTS, + BNXT_RE_TX_BYTES, + BNXT_RE_CNP_TX_PKTS, + BNXT_RE_CNP_TX_BYTES, + BNXT_RE_CNP_RX_PKTS, + BNXT_RE_CNP_RX_BYTES, + BNXT_RE_ROCE_ONLY_RX_PKTS, + BNXT_RE_ROCE_ONLY_RX_BYTES, + BNXT_RE_ROCE_ONLY_TX_PKTS, + BNXT_RE_ROCE_ONLY_TX_BYTES, + BNXT_RE_RX_ROCE_ERROR_PKTS, + BNXT_RE_RX_ROCE_DISCARD_PKTS, + BNXT_RE_TX_ROCE_ERROR_PKTS, + BNXT_RE_TX_ROCE_DISCARDS_PKTS, + BNXT_RE_RES_OOB_DROP_COUNT, + BNXT_RE_TX_ATOMIC_REQ, + BNXT_RE_RX_ATOMIC_REQ, + BNXT_RE_TX_READ_REQ, + BNXT_RE_TX_READ_RESP, + BNXT_RE_RX_READ_REQ, + BNXT_RE_RX_READ_RESP, + BNXT_RE_TX_WRITE_REQ, + BNXT_RE_RX_WRITE_REQ, + BNXT_RE_TX_SEND_REQ, + BNXT_RE_RX_SEND_REQ, + BNXT_RE_RX_GOOD_PKTS, + BNXT_RE_RX_GOOD_BYTES, + BNXT_RE_RX_DCN_PAYLOAD_CUT, + BNXT_RE_TE_BYPASSED, + BNXT_RE_RX_ECN_MARKED_PKTS, + BNXT_RE_MAX_RETRY_EXCEEDED, + BNXT_RE_TO_RETRANSMITS, + BNXT_RE_SEQ_ERR_NAKS_RCVD, + BNXT_RE_RNR_NAKS_RCVD, + BNXT_RE_MISSING_RESP, + BNXT_RE_DUP_REQS, + BNXT_RE_UNRECOVERABLE_ERR, + BNXT_RE_BAD_RESP_ERR, + BNXT_RE_LOCAL_QP_OP_ERR, + BNXT_RE_LOCAL_PROTECTION_ERR, + BNXT_RE_MEM_MGMT_OP_ERR, + BNXT_RE_REMOTE_INVALID_REQ_ERR, + BNXT_RE_REMOTE_ACCESS_ERR, + BNXT_RE_REMOTE_OP_ERR, + BNXT_RE_RES_EXCEED_MAX, + BNXT_RE_RES_LENGTH_MISMATCH, + BNXT_RE_RES_EXCEEDS_WQE, + BNXT_RE_RES_OPCODE_ERR, + BNXT_RE_RES_RX_INVALID_RKEY, + BNXT_RE_RES_RX_DOMAIN_ERR, + BNXT_RE_RES_RX_NO_PERM, + BNXT_RE_RES_RX_RANGE_ERR, + BNXT_RE_RES_TX_INVALID_RKEY, + BNXT_RE_RES_TX_DOMAIN_ERR, + BNXT_RE_RES_TX_NO_PERM, + BNXT_RE_RES_TX_RANGE_ERR, + BNXT_RE_RES_IRRQ_OFLOW, + BNXT_RE_RES_UNSUP_OPCODE, + BNXT_RE_RES_UNALIGNED_ATOMIC, + BNXT_RE_RES_REM_INV_ERR, + BNXT_RE_RES_MEM_ERROR64, + BNXT_RE_RES_SRQ_ERR, + BNXT_RE_RES_CMP_ERR, + BNXT_RE_RES_INVALID_DUP_RKEY, + BNXT_RE_RES_WQE_FORMAT_ERR, + BNXT_RE_RES_CQ_LOAD_ERR, + BNXT_RE_RES_SRQ_LOAD_ERR, + BNXT_RE_RES_TX_PCI_ERR, + BNXT_RE_RES_RX_PCI_ERR, + BNXT_RE_RES_OOS_DROP_COUNT, + BNXT_RE_NUM_IRQ_STARTED, + BNXT_RE_NUM_IRQ_STOPPED, + BNXT_RE_POLL_IN_INTR_EN, + BNXT_RE_POLL_IN_INTR_DIS, + BNXT_RE_CMDQ_FULL_DBG_CNT, + BNXT_RE_FW_SERVICE_PROF_TYPE_SUP, + BNXT_RE_DBQ_INT_RECV, + BNXT_RE_DBQ_INT_EN, + BNXT_RE_DBQ_PACING_RESCHED, + BNXT_RE_DBQ_PACING_CMPL, + BNXT_RE_DBQ_PACING_ALERT, + BNXT_RE_DBQ_DBR_FIFO_REG, + BNXT_RE_DBQ_NUM_EXT_COUNTERS +}; + +#define BNXT_RE_NUM_STD_COUNTERS (BNXT_RE_OUT_OF_SEQ_ERR + 1) + +struct bnxt_re_stats { + struct bnxt_qplib_roce_stats errs; + struct bnxt_qplib_ext_stat ext_stat; +}; + +struct rdma_hw_stats *bnxt_re_alloc_hw_port_stats(struct ib_device *ibdev, + u8 port_num); +int bnxt_re_get_hw_stats(struct ib_device *ibdev, + struct rdma_hw_stats *stats, + u8 port, int index); +int bnxt_re_get_device_stats(struct bnxt_re_dev *rdev); +int bnxt_re_get_flow_stats_from_service_pf(struct bnxt_re_dev *rdev, + struct bnxt_re_flow_counters *stats, + struct bnxt_qplib_query_stats_info *sinfo); +int bnxt_re_get_qos_stats(struct bnxt_re_dev *rdev); +#endif /* __STATS_H__ */ diff --git a/sys/modules/bnxt/bnxt_re/Makefile b/sys/modules/bnxt/bnxt_re/Makefile new file mode 100644 index 000000000000..d6aa701dbae6 --- /dev/null +++ b/sys/modules/bnxt/bnxt_re/Makefile @@ -0,0 +1,22 @@ +.PATH: ${SRCTOP}/sys/dev/bnxt/bnxt_re + +KMOD=bnxt_re +SRCS += ib_verbs.c ib_verbs.h +SRCS += qplib_fp.c qplib_fp.h +SRCS += qplib_sp.c qplib_sp.h +SRCS += qplib_res.c qplib_res.h +SRCS += qplib_rcfw.c qplib_rcfw.h +SRCS += stats.c stats.h +SRCS += main.c bnxt_re.h +SRCS += opt_inet.h opt_inet6.h opt_ratelimit.h +SRCS += ${LINUXKPI_GENSRCS} + +CFLAGS+= -I${SRCTOP}/sys/dev/bnxt/bnxt_en +CFLAGS+= -I${SRCTOP}/sys/ofed/include +CFLAGS+= -I${SRCTOP}/sys/ofed/include/uapi +CFLAGS+= ${LINUXKPI_INCLUDES} +CFLAGS+= -DCONFIG_INFINIBAND_USER_MEM + +.include + +CFLAGS+= -Wno-cast-qual -Wno-pointer-arith ${GCC_MS_EXTENSIONS}