diff --git a/contrib/ofed/libbnxtre/abi.h b/contrib/ofed/libbnxtre/abi.h index 390605edb40b..e3533c122688 100644 --- a/contrib/ofed/libbnxtre/abi.h +++ b/contrib/ofed/libbnxtre/abi.h @@ -1,542 +1,543 @@ /* * Copyright (c) 2024, Broadcom. All rights reserved. The term * Broadcom refers to Broadcom Limited and/or its subsidiaries. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in * the documentation and/or other materials provided with the * distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #ifndef __BNXT_RE_ABI_H__ #define __BNXT_RE_ABI_H__ #include #include #include #include #include #include #include #include #include #include #include #include #define __aligned_u64 __attribute__((aligned(8))) u64 #define BNXT_RE_ABI_VERSION 6 #define BNXT_RE_MAX_INLINE_SIZE 0x60 #define BNXT_RE_MAX_INLINE_SIZE_VAR_WQE 0x1E0 #define BNXT_RE_MAX_PUSH_SIZE_VAR_WQE 0xD0 #define BNXT_RE_FULL_FLAG_DELTA 0x00 enum bnxt_re_wr_opcode { BNXT_RE_WR_OPCD_SEND = 0x00, BNXT_RE_WR_OPCD_SEND_IMM = 0x01, BNXT_RE_WR_OPCD_SEND_INVAL = 0x02, BNXT_RE_WR_OPCD_RDMA_WRITE = 0x04, BNXT_RE_WR_OPCD_RDMA_WRITE_IMM = 0x05, BNXT_RE_WR_OPCD_RDMA_READ = 0x06, BNXT_RE_WR_OPCD_ATOMIC_CS = 0x08, BNXT_RE_WR_OPCD_ATOMIC_FA = 0x0B, BNXT_RE_WR_OPCD_LOC_INVAL = 0x0C, BNXT_RE_WR_OPCD_BIND = 0x0E, BNXT_RE_WR_OPCD_RECV = 0x80, BNXT_RE_WR_OPCD_INVAL = 0xFF }; enum bnxt_re_wr_flags { BNXT_RE_WR_FLAGS_INLINE = 0x10, BNXT_RE_WR_FLAGS_SE = 0x08, BNXT_RE_WR_FLAGS_UC_FENCE = 0x04, BNXT_RE_WR_FLAGS_RD_FENCE = 0x02, BNXT_RE_WR_FLAGS_SIGNALED = 0x01 }; #define BNXT_RE_MEMW_TYPE_2 0x02 #define BNXT_RE_MEMW_TYPE_1 0x00 enum bnxt_re_wr_bind_acc { BNXT_RE_WR_BIND_ACC_LWR = 0x01, BNXT_RE_WR_BIND_ACC_RRD = 0x02, BNXT_RE_WR_BIND_ACC_RWR = 0x04, BNXT_RE_WR_BIND_ACC_RAT = 0x08, BNXT_RE_WR_BIND_ACC_MWB = 0x10, BNXT_RE_WR_BIND_ACC_ZBVA = 0x01, BNXT_RE_WR_BIND_ACC_SHIFT = 0x10 }; enum bnxt_re_wc_type { BNXT_RE_WC_TYPE_SEND = 0x00, BNXT_RE_WC_TYPE_RECV_RC = 0x01, BNXT_RE_WC_TYPE_RECV_UD = 0x02, BNXT_RE_WC_TYPE_RECV_RAW = 0x03, BNXT_RE_WC_TYPE_TERM = 0x0E, BNXT_RE_WC_TYPE_COFF = 0x0F }; #define BNXT_RE_WC_OPCD_RECV 0x80 enum bnxt_re_req_wc_status { BNXT_RE_REQ_ST_OK = 0x00, BNXT_RE_REQ_ST_BAD_RESP = 0x01, BNXT_RE_REQ_ST_LOC_LEN = 0x02, BNXT_RE_REQ_ST_LOC_QP_OP = 0x03, BNXT_RE_REQ_ST_PROT = 0x04, BNXT_RE_REQ_ST_MEM_OP = 0x05, BNXT_RE_REQ_ST_REM_INVAL = 0x06, BNXT_RE_REQ_ST_REM_ACC = 0x07, BNXT_RE_REQ_ST_REM_OP = 0x08, BNXT_RE_REQ_ST_RNR_NAK_XCED = 0x09, BNXT_RE_REQ_ST_TRNSP_XCED = 0x0A, BNXT_RE_REQ_ST_WR_FLUSH = 0x0B }; enum bnxt_re_rsp_wc_status { BNXT_RE_RSP_ST_OK = 0x00, BNXT_RE_RSP_ST_LOC_ACC = 0x01, BNXT_RE_RSP_ST_LOC_LEN = 0x02, BNXT_RE_RSP_ST_LOC_PROT = 0x03, BNXT_RE_RSP_ST_LOC_QP_OP = 0x04, BNXT_RE_RSP_ST_MEM_OP = 0x05, BNXT_RE_RSP_ST_REM_INVAL = 0x06, BNXT_RE_RSP_ST_WR_FLUSH = 0x07, BNXT_RE_RSP_ST_HW_FLUSH = 0x08 }; enum bnxt_re_hdr_offset { BNXT_RE_HDR_WT_MASK = 0xFF, BNXT_RE_HDR_FLAGS_MASK = 0xFF, BNXT_RE_HDR_FLAGS_SHIFT = 0x08, BNXT_RE_HDR_WS_MASK = 0xFF, BNXT_RE_HDR_WS_SHIFT = 0x10 }; enum bnxt_re_db_que_type { BNXT_RE_QUE_TYPE_SQ = 0x00, BNXT_RE_QUE_TYPE_RQ = 0x01, BNXT_RE_QUE_TYPE_SRQ = 0x02, BNXT_RE_QUE_TYPE_SRQ_ARM = 0x03, BNXT_RE_QUE_TYPE_CQ = 0x04, BNXT_RE_QUE_TYPE_CQ_ARMSE = 0x05, BNXT_RE_QUE_TYPE_CQ_ARMALL = 0x06, BNXT_RE_QUE_TYPE_CQ_ARMENA = 0x07, BNXT_RE_QUE_TYPE_SRQ_ARMENA = 0x08, BNXT_RE_QUE_TYPE_CQ_CUT_ACK = 0x09, BNXT_RE_PUSH_TYPE_START = 0x0C, BNXT_RE_PUSH_TYPE_END = 0x0D, BNXT_RE_QUE_TYPE_NULL = 0x0F }; enum bnxt_re_db_mask { BNXT_RE_DB_INDX_MASK = 0xFFFFFFUL, BNXT_RE_DB_PILO_MASK = 0x0FFUL, BNXT_RE_DB_PILO_SHIFT = 0x18, BNXT_RE_DB_QID_MASK = 0xFFFFFUL, BNXT_RE_DB_PIHI_MASK = 0xF00UL, BNXT_RE_DB_PIHI_SHIFT = 0x0C, /* Because mask is 0xF00 */ BNXT_RE_DB_TYP_MASK = 0x0FUL, BNXT_RE_DB_TYP_SHIFT = 0x1C, BNXT_RE_DB_VALID_SHIFT = 0x1A, BNXT_RE_DB_EPOCH_SHIFT = 0x18, BNXT_RE_DB_TOGGLE_SHIFT = 0x19, }; enum bnxt_re_psns_mask { BNXT_RE_PSNS_SPSN_MASK = 0xFFFFFF, BNXT_RE_PSNS_OPCD_MASK = 0xFF, BNXT_RE_PSNS_OPCD_SHIFT = 0x18, BNXT_RE_PSNS_NPSN_MASK = 0xFFFFFF, BNXT_RE_PSNS_FLAGS_MASK = 0xFF, BNXT_RE_PSNS_FLAGS_SHIFT = 0x18 }; enum bnxt_re_msns_mask { BNXT_RE_SQ_MSN_SEARCH_START_PSN_MASK = 0xFFFFFFUL, BNXT_RE_SQ_MSN_SEARCH_START_PSN_SHIFT = 0, BNXT_RE_SQ_MSN_SEARCH_NEXT_PSN_MASK = 0xFFFFFF000000ULL, BNXT_RE_SQ_MSN_SEARCH_NEXT_PSN_SHIFT = 0x18, BNXT_RE_SQ_MSN_SEARCH_START_IDX_MASK = 0xFFFF000000000000ULL, BNXT_RE_SQ_MSN_SEARCH_START_IDX_SHIFT = 0x30 }; enum bnxt_re_bcqe_mask { BNXT_RE_BCQE_PH_MASK = 0x01, BNXT_RE_BCQE_TYPE_MASK = 0x0F, BNXT_RE_BCQE_TYPE_SHIFT = 0x01, BNXT_RE_BCQE_STATUS_MASK = 0xFF, BNXT_RE_BCQE_STATUS_SHIFT = 0x08, BNXT_RE_BCQE_FLAGS_MASK = 0xFFFFU, BNXT_RE_BCQE_FLAGS_SHIFT = 0x10, BNXT_RE_BCQE_RWRID_MASK = 0xFFFFFU, BNXT_RE_BCQE_SRCQP_MASK = 0xFF, BNXT_RE_BCQE_SRCQP_SHIFT = 0x18 }; enum bnxt_re_rc_flags_mask { BNXT_RE_RC_FLAGS_SRQ_RQ_MASK = 0x01, BNXT_RE_RC_FLAGS_IMM_MASK = 0x02, BNXT_RE_RC_FLAGS_IMM_SHIFT = 0x01, BNXT_RE_RC_FLAGS_INV_MASK = 0x04, BNXT_RE_RC_FLAGS_INV_SHIFT = 0x02, BNXT_RE_RC_FLAGS_RDMA_MASK = 0x08, BNXT_RE_RC_FLAGS_RDMA_SHIFT = 0x03 }; enum bnxt_re_ud_flags_mask { BNXT_RE_UD_FLAGS_SRQ_RQ_MASK = 0x01, BNXT_RE_UD_FLAGS_SRQ_RQ_SFT = 0x00, BNXT_RE_UD_FLAGS_IMM_MASK = 0x02, BNXT_RE_UD_FLAGS_IMM_SFT = 0x01, BNXT_RE_UD_FLAGS_IP_VER_MASK = 0x30, BNXT_RE_UD_FLAGS_IP_VER_SFT = 0x4, BNXT_RE_UD_FLAGS_META_MASK = 0x3C0, BNXT_RE_UD_FLAGS_META_SFT = 0x6, BNXT_RE_UD_FLAGS_EXT_META_MASK = 0xC00, BNXT_RE_UD_FLAGS_EXT_META_SFT = 0x10, }; enum bnxt_re_ud_cqe_mask { BNXT_RE_UD_CQE_MAC_MASK = 0xFFFFFFFFFFFFULL, BNXT_RE_UD_CQE_SRCQPLO_MASK = 0xFFFF, BNXT_RE_UD_CQE_SRCQPLO_SHIFT = 0x30, BNXT_RE_UD_CQE_LEN_MASK = 0x3FFFU }; enum bnxt_re_shpg_offt { BNXT_RE_SHPG_BEG_RESV_OFFT = 0x00, BNXT_RE_SHPG_AVID_OFFT = 0x10, BNXT_RE_SHPG_AVID_SIZE = 0x04, BNXT_RE_SHPG_END_RESV_OFFT = 0xFF0 }; enum bnxt_re_que_flags_mask { BNXT_RE_FLAG_EPOCH_TAIL_SHIFT = 0x0UL, BNXT_RE_FLAG_EPOCH_HEAD_SHIFT = 0x1UL, BNXT_RE_FLAG_EPOCH_TAIL_MASK = 0x1UL, BNXT_RE_FLAG_EPOCH_HEAD_MASK = 0x2UL, }; enum bnxt_re_db_epoch_flag_shift { BNXT_RE_DB_EPOCH_TAIL_SHIFT = BNXT_RE_DB_EPOCH_SHIFT, BNXT_RE_DB_EPOCH_HEAD_SHIFT = (BNXT_RE_DB_EPOCH_SHIFT - 1) }; enum bnxt_re_ppp_st_en_mask { BNXT_RE_PPP_ENABLED_MASK = 0x1UL, BNXT_RE_PPP_STATE_MASK = 0x2UL, }; enum bnxt_re_ppp_st_shift { BNXT_RE_PPP_ST_SHIFT = 0x1UL }; struct bnxt_re_db_hdr { __u64 typ_qid_indx; /* typ: 4, qid:20, indx:24 */ }; #define BNXT_RE_CHIP_ID0_CHIP_NUM_SFT 0x00 #define BNXT_RE_CHIP_ID0_CHIP_REV_SFT 0x10 #define BNXT_RE_CHIP_ID0_CHIP_MET_SFT 0x18 enum { BNXT_RE_COMP_MASK_UCNTX_WC_DPI_ENABLED = 0x01, BNXT_RE_COMP_MASK_UCNTX_POW2_DISABLED = 0x02, BNXT_RE_COMP_MASK_UCNTX_RSVD_WQE_DISABLED = 0x04, BNXT_RE_COMP_MASK_UCNTX_MQP_EX_SUPPORTED = 0x8, BNXT_RE_COMP_MASK_UCNTX_DBR_PACING_ENABLED = 0x10, BNXT_RE_COMP_MASK_UCNTX_DBR_RECOVERY_ENABLED = 0x20, BNXT_RE_COMP_MASK_UCNTX_HW_RETX_ENABLED = 0x40 }; enum bnxt_re_req_to_drv { BNXT_RE_COMP_MASK_REQ_UCNTX_POW2_SUPPORT = 0x01, BNXT_RE_COMP_MASK_REQ_UCNTX_RSVD_WQE = 0x02 }; #define BNXT_RE_WQE_MODES_WQE_MODE_MASK 0x01 /* bit wise modes can be extended here. */ enum bnxt_re_modes { BNXT_RE_WQE_MODE_STATIC = 0x00, BNXT_RE_WQE_MODE_VARIABLE = 0x01 /* Other modes can be here */ }; struct bnxt_re_cntx_req { struct ibv_get_context cmd; __aligned_u64 comp_mask; }; struct bnxt_re_cntx_resp { struct ibv_get_context_resp resp; __u32 dev_id; __u32 max_qp; /* To allocate qp-table */ __u32 pg_size; __u32 cqe_size; __u32 max_cqd; __u32 chip_id0; __u32 chip_id1; __u32 modes; __aligned_u64 comp_mask; } __attribute__((packed)); enum { BNXT_RE_COMP_MASK_PD_HAS_WC_DPI = 0x01, BNXT_RE_COMP_MASK_PD_HAS_DBR_BAR_ADDR = 0x02, }; struct bnxt_re_pd_resp { struct ibv_alloc_pd_resp resp; __u32 pdid; __u32 dpi; __u64 dbr; __u64 comp_mask; __u32 wcdpi; __u64 dbr_bar_map; } __attribute__((packed)); struct bnxt_re_mr_resp { struct ibv_reg_mr_resp resp; } __attribute__((packed)); /* CQ */ enum { BNXT_RE_COMP_MASK_CQ_HAS_DB_INFO = 0x01, BNXT_RE_COMP_MASK_CQ_HAS_WC_DPI = 0x02, BNXT_RE_COMP_MASK_CQ_HAS_CQ_PAGE = 0x04 }; enum { BNXT_RE_COMP_MASK_CQ_REQ_HAS_CAP_MASK = 0x1 }; enum { BNXT_RE_COMP_MASK_CQ_REQ_CAP_DBR_RECOVERY = 0x1 }; struct bnxt_re_cq_req { struct ibv_create_cq cmd; __u64 cq_va; __u64 cq_handle; __aligned_u64 comp_mask; __u16 cq_capab; } __attribute__((packed)); struct bnxt_re_cq_resp { struct ibv_create_cq_resp resp; __u32 cqid; __u32 tail; __u32 phase; __u32 rsvd; __aligned_u64 comp_mask; __u32 dpi; __u64 dbr; __u32 wcdpi; __u64 cq_page; } __attribute__((packed)); struct bnxt_re_resize_cq_req { struct ibv_resize_cq cmd; __u64 cq_va; } __attribute__((packed)); struct bnxt_re_bcqe { __u32 flg_st_typ_ph; __u32 qphi_rwrid; } __attribute__((packed)); struct bnxt_re_req_cqe { __u64 qp_handle; __u32 con_indx; /* 16 bits valid. */ __u32 rsvd1; __u64 rsvd2; } __attribute__((packed)); struct bnxt_re_rc_cqe { __u32 length; __u32 imm_key; __u64 qp_handle; __u64 mr_handle; } __attribute__((packed)); struct bnxt_re_ud_cqe { __u32 length; /* 14 bits */ __u32 immd; __u64 qp_handle; __u64 qplo_mac; /* 16:48*/ } __attribute__((packed)); struct bnxt_re_term_cqe { __u64 qp_handle; __u32 rq_sq_cidx; __u32 rsvd; __u64 rsvd1; } __attribute__((packed)); struct bnxt_re_cutoff_cqe { __u64 rsvd1; __u64 rsvd2; __u64 rsvd3; __u8 cqe_type_toggle; __u8 status; __u16 rsvd4; __u32 rsvd5; } __attribute__((packed)); /* QP */ struct bnxt_re_qp_req { struct ibv_create_qp cmd; __u64 qpsva; __u64 qprva; __u64 qp_handle; } __attribute__((packed)); struct bnxt_re_qp_resp { struct ibv_create_qp_resp resp; __u32 qpid; } __attribute__((packed)); enum bnxt_re_modify_ex_mask { BNXT_RE_MQP_PPP_REQ_EN_MASK = 0x1UL, BNXT_RE_MQP_PPP_REQ_EN = 0x1UL, BNXT_RE_MQP_PATH_MTU_MASK = 0x2UL, BNXT_RE_MQP_PPP_IDX_MASK = 0x7UL, BNXT_RE_MQP_PPP_STATE = 0x10UL }; /* Modify QP */ struct bnxt_re_modify_ex_req { struct ibv_modify_qp_ex cmd; __aligned_u64 comp_mask; __u32 dpi; __u32 rsvd; }; struct bnxt_re_modify_ex_resp { struct ibv_modify_qp_resp_ex resp; __aligned_u64 comp_mask; __u32 ppp_st_idx; __u32 path_mtu; }; union lower_shdr { __u64 qkey_len; __u64 lkey_plkey; __u64 rva; }; struct bnxt_re_bsqe { __u32 rsv_ws_fl_wt; __u32 key_immd; union lower_shdr lhdr; } __attribute__((packed)); struct bnxt_re_psns_ext { __u32 opc_spsn; __u32 flg_npsn; __u16 st_slot_idx; __u16 rsvd0; __u32 rsvd1; } __attribute__((packed)); /* sq_msn_search (size:64b/8B) */ struct bnxt_re_msns { __u64 start_idx_next_psn_start_psn; } __attribute__((packed)); struct bnxt_re_psns { __u32 opc_spsn; __u32 flg_npsn; } __attribute__((packed)); struct bnxt_re_sge { __u64 pa; __u32 lkey; __u32 length; } __attribute__((packed)); struct bnxt_re_send { __u32 dst_qp; __u32 avid; __u64 rsvd; } __attribute__((packed)); struct bnxt_re_raw { __u32 cfa_meta; __u32 rsvd2; __u64 rsvd3; } __attribute__((packed)); struct bnxt_re_rdma { __u64 rva; __u32 rkey; __u32 rsvd2; } __attribute__((packed)); struct bnxt_re_atomic { __u64 swp_dt; __u64 cmp_dt; } __attribute__((packed)); struct bnxt_re_inval { __u64 rsvd[2]; } __attribute__((packed)); struct bnxt_re_bind { __u64 va; __u64 len; /* only 40 bits are valid */ } __attribute__((packed)); struct bnxt_re_brqe { __u32 rsv_ws_fl_wt; __u32 rsvd; __u32 wrid; __u32 rsvd1; } __attribute__((packed)); struct bnxt_re_rqe { __u64 rsvd[2]; } __attribute__((packed)); /* SRQ */ struct bnxt_re_srq_req { struct ibv_create_srq cmd; __u64 srqva; __u64 srq_handle; } __attribute__((packed)); struct bnxt_re_srq_resp { struct ibv_create_srq_resp resp; __u32 srqid; + __u64 srq_page; } __attribute__((packed)); struct bnxt_re_srqe { __u64 rsvd[2]; } __attribute__((packed)); struct bnxt_re_push_wqe { __u64 addr[32]; } __attribute__((packed));; #endif diff --git a/contrib/ofed/libbnxtre/db.c b/contrib/ofed/libbnxtre/db.c index 8751297c9218..3d416795679c 100644 --- a/contrib/ofed/libbnxtre/db.c +++ b/contrib/ofed/libbnxtre/db.c @@ -1,566 +1,571 @@ /* * Copyright (c) 2024, Broadcom. All rights reserved. The term * Broadcom refers to Broadcom Limited and/or its subsidiaries. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in * the documentation and/or other materials provided with the * distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * * Description: Doorbell handling functions. */ #include #include #include "abi.h" #include "main.h" #define BNXT_RE_DB_FIFO_ROOM_MASK_P5 0x1FFF8000 #define BNXT_RE_MAX_FIFO_DEPTH_P5 0x2c00 #define BNXT_RE_DB_FIFO_ROOM_MASK_P7 0x3FFF8000 -#define BNXT_RE_MAX_FIFO_DEPTH_P7 0x8000 +#define BNXT_RE_MAX_FIFO_DEPTH_P7 0x7fff #define BNXT_RE_DB_FIFO_ROOM_SHIFT 15 #define BNXT_RE_DB_THRESHOLD 20 #define BNXT_RE_DB_FIFO_ROOM_MASK(ctx) \ (_is_chip_thor2((ctx)) ? \ BNXT_RE_DB_FIFO_ROOM_MASK_P7 :\ BNXT_RE_DB_FIFO_ROOM_MASK_P5) #define BNXT_RE_MAX_FIFO_DEPTH(ctx) \ (_is_chip_thor2((ctx)) ? \ BNXT_RE_MAX_FIFO_DEPTH_P7 :\ BNXT_RE_MAX_FIFO_DEPTH_P5) static uint32_t xorshift32(struct xorshift32_state *state) { /* Algorithm "xor" from p. 4 of Marsaglia, "Xorshift RNGs" */ uint32_t x = state->seed; x ^= x << 13; x ^= x >> 17; x ^= x << 5; return state->seed = x; } static uint16_t rnd(struct xorshift32_state *state, uint16_t range) { /* range must be a power of 2 - 1 */ return (xorshift32(state) & range); } static int calculate_fifo_occupancy(struct bnxt_re_context *cntx) { uint32_t *dbr_map = cntx->bar_map + 0x1a8; uint32_t read_val, fifo_occup; read_val = *dbr_map; fifo_occup = BNXT_RE_MAX_FIFO_DEPTH(cntx->cctx) - ((read_val & BNXT_RE_DB_FIFO_ROOM_MASK(cntx->cctx)) >> BNXT_RE_DB_FIFO_ROOM_SHIFT); return fifo_occup; } static inline uint32_t find_min(uint32_t x, uint32_t y) { return (y > x ? x : y); } int bnxt_re_do_pacing(struct bnxt_re_context *cntx, struct xorshift32_state *state) { /* First 4 bytes of shared page (pacing_info) contains the DBR * pacing information. Second 4 bytes (pacing_th) contains * the pacing threshold value to determine whether to * add delay or not */ struct bnxt_re_pacing_data *pacing_data = (struct bnxt_re_pacing_data *)cntx->dbr_page; uint32_t wait_time = 1; uint32_t fifo_occup; if (!pacing_data) return 0; /* If the device in error recovery state, return error to * not to ring new doorbells in this state. */ if (pacing_data->dev_err_state) return -EFAULT; if (rnd(state, BNXT_RE_MAX_DO_PACING) < pacing_data->do_pacing) { while ((fifo_occup = calculate_fifo_occupancy(cntx)) > pacing_data->pacing_th) { struct bnxt_re_cq *cq; uint32_t usec_wait; if (pacing_data->alarm_th && fifo_occup > pacing_data->alarm_th) { cq = container_of(cntx->dbr_cq, struct bnxt_re_cq, ibvcq); bnxt_re_poll_kernel_cq(cq); } usec_wait = rnd(state, wait_time - 1); if (usec_wait) bnxt_re_sub_sec_busy_wait(usec_wait * 1000); /* wait time capped at 128 us */ wait_time = find_min(wait_time * 2, 128); } } return 0; } static inline void bnxt_re_ring_db(struct bnxt_re_dpi *dpi, __u64 key, uint64_t *db_key, uint8_t *lock) { while (1) { if (__sync_bool_compare_and_swap(lock, 0, 1)) { *db_key = key; bnxt_re_wm_barrier(); iowrite64(dpi->dbpage, key); bnxt_re_wm_barrier(); *lock = 0; break; } } } static inline void bnxt_re_init_push_hdr(struct bnxt_re_db_hdr *hdr, uint32_t indx, uint32_t qid, uint32_t typ, uint32_t pidx) { __u64 key_lo, key_hi; key_lo = (((pidx & BNXT_RE_DB_PILO_MASK) << BNXT_RE_DB_PILO_SHIFT) | (indx & BNXT_RE_DB_INDX_MASK)); key_hi = ((((pidx & BNXT_RE_DB_PIHI_MASK) << BNXT_RE_DB_PIHI_SHIFT) | (qid & BNXT_RE_DB_QID_MASK)) | ((typ & BNXT_RE_DB_TYP_MASK) << BNXT_RE_DB_TYP_SHIFT) | (0x1UL << BNXT_RE_DB_VALID_SHIFT)); hdr->typ_qid_indx = htole64((key_lo | (key_hi << 32))); } static inline void bnxt_re_init_db_hdr(struct bnxt_re_db_hdr *hdr, uint32_t indx, uint32_t toggle, uint32_t qid, uint32_t typ) { __u64 key_lo, key_hi; key_lo = htole32(indx | toggle); key_hi = ((qid & BNXT_RE_DB_QID_MASK) | ((typ & BNXT_RE_DB_TYP_MASK) << BNXT_RE_DB_TYP_SHIFT) | (0x1UL << BNXT_RE_DB_VALID_SHIFT)); hdr->typ_qid_indx = htole64((key_lo | (key_hi << 32))); } static inline void __bnxt_re_ring_pend_db(__u64 *ucdb, __u64 key, struct bnxt_re_qp *qp) { struct bnxt_re_db_hdr hdr; bnxt_re_init_db_hdr(&hdr, (*qp->jsqq->hwque->dbtail | ((qp->jsqq->hwque->flags & BNXT_RE_FLAG_EPOCH_TAIL_MASK) << BNXT_RE_DB_EPOCH_TAIL_SHIFT)), 0, qp->qpid, BNXT_RE_QUE_TYPE_SQ); while (1) { if (__sync_bool_compare_and_swap(&qp->sq_dbr_lock, 0, 1)) { qp->sq_shadow_db_key = hdr.typ_qid_indx; bnxt_re_wm_barrier(); iowrite64(ucdb, key); bnxt_re_wm_barrier(); qp->sq_dbr_lock = 0; break; } } } void bnxt_re_ring_rq_db(struct bnxt_re_qp *qp) { struct bnxt_re_db_hdr hdr; if (bnxt_re_do_pacing(qp->cntx, &qp->rand)) return; bnxt_re_init_db_hdr(&hdr, (*qp->jrqq->hwque->dbtail | ((qp->jrqq->hwque->flags & BNXT_RE_FLAG_EPOCH_TAIL_MASK) << BNXT_RE_DB_EPOCH_TAIL_SHIFT)), 0, qp->qpid, BNXT_RE_QUE_TYPE_RQ); bnxt_re_ring_db(qp->udpi, hdr.typ_qid_indx, &qp->rq_shadow_db_key, &qp->rq_dbr_lock); } void bnxt_re_ring_sq_db(struct bnxt_re_qp *qp) { struct bnxt_re_db_hdr hdr; if (bnxt_re_do_pacing(qp->cntx, &qp->rand)) return; bnxt_re_init_db_hdr(&hdr, (*qp->jsqq->hwque->dbtail | ((qp->jsqq->hwque->flags & BNXT_RE_FLAG_EPOCH_TAIL_MASK) << BNXT_RE_DB_EPOCH_TAIL_SHIFT)), 0, qp->qpid, BNXT_RE_QUE_TYPE_SQ); bnxt_re_ring_db(qp->udpi, hdr.typ_qid_indx, &qp->sq_shadow_db_key, &qp->sq_dbr_lock); } void bnxt_re_ring_srq_db(struct bnxt_re_srq *srq) { struct bnxt_re_db_hdr hdr; if (bnxt_re_do_pacing(srq->uctx, &srq->rand)) return; bnxt_re_init_db_hdr(&hdr, (srq->srqq->tail | ((srq->srqq->flags & BNXT_RE_FLAG_EPOCH_TAIL_MASK) << BNXT_RE_DB_EPOCH_TAIL_SHIFT)), 0, srq->srqid, BNXT_RE_QUE_TYPE_SRQ); bnxt_re_ring_db(srq->udpi, hdr.typ_qid_indx, &srq->shadow_db_key, &srq->dbr_lock); } void bnxt_re_ring_srq_arm(struct bnxt_re_srq *srq) { struct bnxt_re_db_hdr hdr; + uint32_t toggle = 0; + + if (srq->srq_page) + toggle = *(uint32_t *)srq->srq_page; if (bnxt_re_do_pacing(srq->uctx, &srq->rand)) return; - bnxt_re_init_db_hdr(&hdr, srq->cap.srq_limit, 0, srq->srqid, + bnxt_re_init_db_hdr(&hdr, srq->cap.srq_limit, + toggle << BNXT_RE_DB_TOGGLE_SHIFT, srq->srqid, BNXT_RE_QUE_TYPE_SRQ_ARM); bnxt_re_ring_db(srq->udpi, hdr.typ_qid_indx, &srq->shadow_db_key, &srq->dbr_lock); } void bnxt_re_ring_cq_db(struct bnxt_re_cq *cq) { struct bnxt_re_db_hdr hdr; if (bnxt_re_do_pacing(cq->cntx, &cq->rand)) return; bnxt_re_init_db_hdr(&hdr, (cq->cqq->head | ((cq->cqq->flags & BNXT_RE_FLAG_EPOCH_HEAD_MASK) << BNXT_RE_DB_EPOCH_HEAD_SHIFT)), 0, cq->cqid, BNXT_RE_QUE_TYPE_CQ); bnxt_re_ring_db(cq->udpi, hdr.typ_qid_indx, &cq->shadow_db_key, &cq->dbr_lock); } void bnxt_re_ring_cq_arm_db(struct bnxt_re_cq *cq, uint8_t aflag) { uint32_t *cq_page = cq->cq_page; struct bnxt_re_db_hdr hdr; uint32_t toggle = 0; if (cq_page) toggle = *cq_page; if (bnxt_re_do_pacing(cq->cntx, &cq->rand)) return; bnxt_re_init_db_hdr(&hdr, (cq->cqq->head | ((cq->cqq->flags & BNXT_RE_FLAG_EPOCH_HEAD_MASK) << BNXT_RE_DB_EPOCH_HEAD_SHIFT)), toggle << BNXT_RE_DB_TOGGLE_SHIFT, cq->cqid, aflag); bnxt_re_ring_db(cq->udpi, hdr.typ_qid_indx, &cq->shadow_db_key, &cq->dbr_lock); } void bnxt_re_ring_pstart_db(struct bnxt_re_qp *qp, struct bnxt_re_push_buffer *pbuf) { __u64 key; if (bnxt_re_do_pacing(qp->cntx, &qp->rand)) return; key = ((((pbuf->wcdpi & BNXT_RE_DB_PIHI_MASK) << BNXT_RE_DB_PIHI_SHIFT) | (pbuf->qpid & BNXT_RE_DB_QID_MASK)) | ((BNXT_RE_PUSH_TYPE_START & BNXT_RE_DB_TYP_MASK) << BNXT_RE_DB_TYP_SHIFT) | (0x1UL << BNXT_RE_DB_VALID_SHIFT)); key <<= 32; key |= ((((__u32)pbuf->wcdpi & BNXT_RE_DB_PILO_MASK) << BNXT_RE_DB_PILO_SHIFT) | (pbuf->st_idx & BNXT_RE_DB_INDX_MASK)); bnxt_re_wm_barrier(); iowrite64(pbuf->ucdb, key); } void bnxt_re_ring_pend_db(struct bnxt_re_qp *qp, struct bnxt_re_push_buffer *pbuf) { __u64 key; if (bnxt_re_do_pacing(qp->cntx, &qp->rand)) return; key = ((((pbuf->wcdpi & BNXT_RE_DB_PIHI_MASK) << BNXT_RE_DB_PIHI_SHIFT) | (pbuf->qpid & BNXT_RE_DB_QID_MASK)) | ((BNXT_RE_PUSH_TYPE_END & BNXT_RE_DB_TYP_MASK) << BNXT_RE_DB_TYP_SHIFT) | (0x1UL << BNXT_RE_DB_VALID_SHIFT)); key <<= 32; key |= ((((__u32)pbuf->wcdpi & BNXT_RE_DB_PILO_MASK) << BNXT_RE_DB_PILO_SHIFT) | (pbuf->tail & BNXT_RE_DB_INDX_MASK)); __bnxt_re_ring_pend_db(pbuf->ucdb, key, qp); } void bnxt_re_fill_ppp(struct bnxt_re_push_buffer *pbuf, struct bnxt_re_qp *qp, uint8_t len, uint32_t idx) { struct bnxt_re_db_ppp_hdr phdr = {}; __u64 *dst, *src; __u8 plen; int indx; src = (__u64 *)&phdr; plen = len + sizeof(phdr) + bnxt_re_get_sqe_hdr_sz(); bnxt_re_init_db_hdr(&phdr.db_hdr, (*qp->jsqq->hwque->dbtail | ((qp->jsqq->hwque->flags & BNXT_RE_FLAG_EPOCH_TAIL_MASK) << BNXT_RE_DB_EPOCH_TAIL_SHIFT)), 0, qp->qpid, BNXT_RE_QUE_TYPE_SQ); phdr.rsv_psz_pidx = ((pbuf->st_idx & BNXT_RE_DB_INDX_MASK) | (((plen % 8 ? (plen / 8) + 1 : plen / 8) & BNXT_RE_PUSH_SIZE_MASK) << BNXT_RE_PUSH_SIZE_SHIFT)); bnxt_re_wm_barrier(); for (indx = 0; indx < 2; indx++) { dst = (__u64 *)(pbuf->pbuf + indx); iowrite64(dst, *src); src++; } bnxt_re_copy_data_to_pb(pbuf, 1, idx); mmio_flush_writes(); } void bnxt_re_fill_push_wcb(struct bnxt_re_qp *qp, struct bnxt_re_push_buffer *pbuf, uint32_t idx) { bnxt_re_ring_pstart_db(qp, pbuf); mmio_wc_start(); bnxt_re_copy_data_to_pb(pbuf, 0, idx); /* Flush WQE write before push end db. */ mmio_flush_writes(); bnxt_re_ring_pend_db(qp, pbuf); } int bnxt_re_init_pbuf_list(struct bnxt_re_context *ucntx) { struct bnxt_re_push_buffer *pbuf; int indx, wqesz; int size, offt; __u64 wcpage; __u64 dbpage; void *base; size = (sizeof(*ucntx->pbrec) + 16 * (sizeof(*ucntx->pbrec->pbuf) + sizeof(struct bnxt_re_push_wqe))); ucntx->pbrec = calloc(1, size); if (!ucntx->pbrec) goto out; offt = sizeof(*ucntx->pbrec); base = ucntx->pbrec; ucntx->pbrec->pbuf = (base + offt); ucntx->pbrec->pbmap = ~0x00; ucntx->pbrec->pbmap &= ~0x7fff; /* 15 bits */ ucntx->pbrec->udpi = &ucntx->udpi; wqesz = sizeof(struct bnxt_re_push_wqe); wcpage = (__u64)ucntx->udpi.wcdbpg; dbpage = (__u64)ucntx->udpi.dbpage; offt = sizeof(*ucntx->pbrec->pbuf) * 16; base = (char *)ucntx->pbrec->pbuf + offt; for (indx = 0; indx < 16; indx++) { pbuf = &ucntx->pbrec->pbuf[indx]; pbuf->wqe = base + indx * wqesz; pbuf->pbuf = (__u64 *)(wcpage + indx * wqesz); pbuf->ucdb = (__u64 *)(dbpage + (indx + 1) * sizeof(__u64)); pbuf->wcdpi = ucntx->udpi.wcdpi; } return 0; out: return -ENOMEM; } struct bnxt_re_push_buffer *bnxt_re_get_pbuf(uint8_t *push_st_en, uint8_t ppp_idx, struct bnxt_re_context *cntx) { struct bnxt_re_push_buffer *pbuf = NULL; uint8_t buf_state = 0; __u32 old; int bit; if (_is_chip_thor2(cntx->cctx)) { buf_state = !!(*push_st_en & BNXT_RE_PPP_STATE_MASK); pbuf = &cntx->pbrec->pbuf[(ppp_idx * 2) + buf_state]; /* Flip */ *push_st_en ^= 1UL << BNXT_RE_PPP_ST_SHIFT; } else { old = cntx->pbrec->pbmap; while ((bit = __builtin_ffs(~cntx->pbrec->pbmap)) != 0) { if (__sync_bool_compare_and_swap (&cntx->pbrec->pbmap, old, (old | 0x01 << (bit - 1)))) break; old = cntx->pbrec->pbmap; } if (bit) { pbuf = &cntx->pbrec->pbuf[bit]; pbuf->nbit = bit; } } return pbuf; } void bnxt_re_put_pbuf(struct bnxt_re_context *cntx, struct bnxt_re_push_buffer *pbuf) { struct bnxt_re_push_rec *pbrec; __u32 old; int bit; if (_is_chip_thor2(cntx->cctx)) return; pbrec = cntx->pbrec; if (pbuf->nbit) { bit = pbuf->nbit; pbuf->nbit = 0; old = pbrec->pbmap; while (!__sync_bool_compare_and_swap(&pbrec->pbmap, old, (old & (~(0x01 << (bit - 1)))))) old = pbrec->pbmap; } } void bnxt_re_destroy_pbuf_list(struct bnxt_re_context *cntx) { free(cntx->pbrec); } void bnxt_re_replay_db(struct bnxt_re_context *cntx, struct xorshift32_state *state, struct bnxt_re_dpi *dpi, uint64_t *shadow_key, uint8_t *dbr_lock) { if (bnxt_re_do_pacing(cntx, state)) return; cntx->replay_cnt++; if (cntx->replay_cnt % BNXT_RE_DB_REPLAY_YIELD_CNT == 0) pthread_yield(); if (__sync_bool_compare_and_swap(dbr_lock, 0, 1)) { bnxt_re_wm_barrier(); if (*shadow_key == BNXT_RE_DB_KEY_INVALID) { *dbr_lock = 0; return; } iowrite64(dpi->dbpage, *shadow_key); bnxt_re_wm_barrier(); *dbr_lock = 0; } } void bnxt_re_db_recovery(struct bnxt_re_context *cntx) { struct bnxt_re_list_node *cur, *tmp; struct bnxt_re_qp *qp; struct bnxt_re_cq *cq; struct bnxt_re_srq *srq; pthread_spin_lock(&cntx->qp_dbr_res.lock); list_for_each_node_safe(cur, tmp, &cntx->qp_dbr_res.head) { qp = list_node(cur, struct bnxt_re_qp, dbnode); bnxt_re_replay_db(cntx, &qp->rand, qp->udpi, &qp->sq_shadow_db_key, &qp->sq_dbr_lock); bnxt_re_replay_db(cntx, &qp->rand, qp->udpi, &qp->rq_shadow_db_key, &qp->rq_dbr_lock); } pthread_spin_unlock(&cntx->qp_dbr_res.lock); pthread_spin_lock(&cntx->cq_dbr_res.lock); list_for_each_node_safe(cur, tmp, &cntx->cq_dbr_res.head) { cq = list_node(cur, struct bnxt_re_cq, dbnode); bnxt_re_replay_db(cntx, &cq->rand, cq->udpi, &cq->shadow_db_key, &cq->dbr_lock); } pthread_spin_unlock(&cntx->cq_dbr_res.lock); pthread_spin_lock(&cntx->srq_dbr_res.lock); list_for_each_node_safe(cur, tmp, &cntx->srq_dbr_res.head) { srq = list_node(cur, struct bnxt_re_srq, dbnode); bnxt_re_replay_db(cntx, &srq->rand, srq->udpi, &srq->shadow_db_key, &srq->dbr_lock); } pthread_spin_unlock(&cntx->srq_dbr_res.lock); } void *bnxt_re_dbr_thread(void *arg) { uint32_t *epoch, *epoch_ack, usr_epoch; struct bnxt_re_context *cntx = arg; struct ibv_cq *ev_cq; void *ev_ctx; int ret; while (1) { ret = ibv_get_cq_event(cntx->dbr_ev_chan, &ev_cq, &ev_ctx); if (ret) { fprintf(stderr, "Failed to get cq_event\n"); pthread_exit(NULL); } epoch = cntx->db_recovery_page; epoch_ack = epoch + 1; if (!epoch || !epoch_ack) { fprintf(stderr, "DB reovery page is NULL\n"); pthread_exit(NULL); } if (*epoch == *epoch_ack) { ibv_ack_cq_events(ev_cq, 1); continue; } usr_epoch = *epoch; bnxt_re_db_recovery(cntx); *epoch_ack = usr_epoch; ibv_ack_cq_events(ev_cq, 1); } } diff --git a/contrib/ofed/libbnxtre/main.h b/contrib/ofed/libbnxtre/main.h index ea258de22abd..06ed4647d68e 100644 --- a/contrib/ofed/libbnxtre/main.h +++ b/contrib/ofed/libbnxtre/main.h @@ -1,538 +1,539 @@ /* * Copyright (c) 2024, Broadcom. All rights reserved. The term * Broadcom refers to Broadcom Limited and/or its subsidiaries. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in * the documentation and/or other materials provided with the * distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * */ #ifndef __BNXT_RE_MAIN_H__ #define __BNXT_RE_MAIN_H__ #include #include #include #include #include #include #include #include #include #include #include #include #include #include "abi.h" #include "list.h" #include "memory.h" #define DEV "bnxt_re : " #define BNXT_RE_UD_QP_STALL 0x400000 #define CHIP_NUM_57508 0x1750 #define CHIP_NUM_57504 0x1751 #define CHIP_NUM_57502 0x1752 #define CHIP_NUM_58818 0xd818 #define CHIP_NUM_57608 0x1760 #define BNXT_NSEC_PER_SEC 1000000000UL struct bnxt_re_chip_ctx { __u16 chip_num; __u8 chip_rev; __u8 chip_metal; bool chip_is_gen_p5_thor2; }; #define BNXT_RE_MAP_WC 0x1000 #define BNXT_RE_DBR_PAGE 0x2000 #define BNXT_RE_DB_RECOVERY_PAGE 0x3000 #define BNXT_RE_DB_REPLAY_YIELD_CNT 256 #define BNXT_RE_DB_KEY_INVALID -1 #define BNXT_RE_MAX_DO_PACING 0xFFFF #define bnxt_re_wm_barrier() udma_to_device_barrier() #define unlikely(x) __builtin_expect(!!(x), 0) #define likely(x) __builtin_expect(!!(x), 1) #define CNA(v, d) \ { .vendor = PCI_VENDOR_ID_##v, \ .device = d } #define BNXT_RE_DEFINE_CNA_TABLE(_name) \ static const struct { \ unsigned vendor; \ unsigned device; \ } _name[] struct bnxt_re_dpi { __u32 dpindx; __u32 wcdpi; __u64 *dbpage; __u64 *wcdbpg; }; struct bnxt_re_pd { struct ibv_pd ibvpd; uint32_t pdid; }; struct xorshift32_state { uint32_t seed; }; struct bnxt_re_cq { struct ibv_cq ibvcq; struct bnxt_re_list_head sfhead; struct bnxt_re_list_head rfhead; struct bnxt_re_list_head prev_cq_head; struct bnxt_re_context *cntx; struct bnxt_re_queue *cqq; struct bnxt_re_dpi *udpi; struct bnxt_re_mem *resize_mem; struct bnxt_re_mem *mem; struct bnxt_re_list_node dbnode; uint64_t shadow_db_key; uint32_t cqe_sz; uint32_t cqid; struct xorshift32_state rand; int deferred_arm_flags; bool first_arm; bool deferred_arm; bool phase; uint8_t dbr_lock; void *cq_page; }; struct bnxt_re_push_buffer { __u64 *pbuf; /*push wc buffer */ __u64 *wqe; /* hwqe addresses */ __u64 *ucdb; uint32_t st_idx; uint32_t qpid; uint16_t wcdpi; uint16_t nbit; uint32_t tail; }; enum bnxt_re_push_info_mask { BNXT_RE_PUSH_SIZE_MASK = 0x1FUL, BNXT_RE_PUSH_SIZE_SHIFT = 0x18UL }; struct bnxt_re_db_ppp_hdr { struct bnxt_re_db_hdr db_hdr; __u64 rsv_psz_pidx; }; struct bnxt_re_push_rec { struct bnxt_re_dpi *udpi; struct bnxt_re_push_buffer *pbuf; __u32 pbmap; /* only 16 bits in use */ }; struct bnxt_re_wrid { uint64_t wrid; int next_idx; uint32_t bytes; uint8_t sig; uint8_t slots; uint8_t wc_opcd; }; struct bnxt_re_qpcap { uint32_t max_swr; uint32_t max_rwr; uint32_t max_ssge; uint32_t max_rsge; uint32_t max_inline; uint8_t sqsig; uint8_t is_atomic_cap; }; struct bnxt_re_srq { struct ibv_srq ibvsrq; struct ibv_srq_attr cap; uint32_t srqid; struct bnxt_re_context *uctx; struct bnxt_re_queue *srqq; struct bnxt_re_wrid *srwrid; struct bnxt_re_dpi *udpi; struct bnxt_re_mem *mem; int start_idx; int last_idx; struct bnxt_re_list_node dbnode; uint64_t shadow_db_key; struct xorshift32_state rand; uint8_t dbr_lock; bool arm_req; + void *srq_page; }; struct bnxt_re_joint_queue { struct bnxt_re_context *cntx; struct bnxt_re_queue *hwque; struct bnxt_re_wrid *swque; uint32_t start_idx; uint32_t last_idx; }; struct bnxt_re_qp { struct ibv_qp ibvqp; struct bnxt_re_qpcap cap; struct bnxt_re_context *cntx; struct bnxt_re_chip_ctx *cctx; struct bnxt_re_joint_queue *jsqq; struct bnxt_re_joint_queue *jrqq; struct bnxt_re_dpi *udpi; uint64_t wqe_cnt; uint16_t mtu; uint16_t qpst; uint8_t qptyp; uint8_t qpmode; uint8_t push_st_en; uint8_t ppp_idx; uint32_t sq_psn; uint32_t sq_msn; uint32_t qpid; uint16_t max_push_sz; uint8_t sq_dbr_lock; uint8_t rq_dbr_lock; struct xorshift32_state rand; struct bnxt_re_list_node snode; struct bnxt_re_list_node rnode; struct bnxt_re_srq *srq; struct bnxt_re_cq *rcq; struct bnxt_re_cq *scq; struct bnxt_re_mem *mem;/* at cl 6 */ struct bnxt_re_list_node dbnode; uint64_t sq_shadow_db_key; uint64_t rq_shadow_db_key; }; struct bnxt_re_mr { struct ibv_mr vmr; }; struct bnxt_re_ah { struct ibv_ah ibvah; struct bnxt_re_pd *pd; uint32_t avid; }; struct bnxt_re_dev { struct verbs_device vdev; struct ibv_device_attr devattr; uint32_t pg_size; uint32_t cqe_size; uint32_t max_cq_depth; uint8_t abi_version; }; struct bnxt_re_res_list { struct bnxt_re_list_head head; pthread_spinlock_t lock; }; struct bnxt_re_context { struct ibv_context ibvctx; struct bnxt_re_dev *rdev; struct bnxt_re_chip_ctx *cctx; uint64_t comp_mask; struct bnxt_re_dpi udpi; uint32_t dev_id; uint32_t max_qp; uint32_t max_srq; uint32_t modes; void *shpg; pthread_mutex_t shlock; struct bnxt_re_push_rec *pbrec; void *dbr_page; void *bar_map; struct bnxt_re_res_list qp_dbr_res; struct bnxt_re_res_list cq_dbr_res; struct bnxt_re_res_list srq_dbr_res; void *db_recovery_page; struct ibv_comp_channel *dbr_ev_chan; struct ibv_cq *dbr_cq; pthread_t dbr_thread; uint64_t replay_cnt; }; struct bnxt_re_pacing_data { uint32_t do_pacing; uint32_t pacing_th; uint32_t dev_err_state; uint32_t alarm_th; }; /* Chip context related functions */ bool _is_chip_gen_p5(struct bnxt_re_chip_ctx *cctx); bool _is_chip_a0(struct bnxt_re_chip_ctx *cctx); bool _is_chip_thor2(struct bnxt_re_chip_ctx *cctx); bool _is_chip_gen_p5_thor2(struct bnxt_re_chip_ctx *cctx); /* DB ring functions used internally*/ void bnxt_re_ring_rq_db(struct bnxt_re_qp *qp); void bnxt_re_ring_sq_db(struct bnxt_re_qp *qp); void bnxt_re_ring_srq_arm(struct bnxt_re_srq *srq); void bnxt_re_ring_srq_db(struct bnxt_re_srq *srq); void bnxt_re_ring_cq_db(struct bnxt_re_cq *cq); void bnxt_re_ring_cq_arm_db(struct bnxt_re_cq *cq, uint8_t aflag); void bnxt_re_ring_pstart_db(struct bnxt_re_qp *qp, struct bnxt_re_push_buffer *pbuf); void bnxt_re_ring_pend_db(struct bnxt_re_qp *qp, struct bnxt_re_push_buffer *pbuf); void bnxt_re_fill_push_wcb(struct bnxt_re_qp *qp, struct bnxt_re_push_buffer *pbuf, uint32_t idx); void bnxt_re_fill_ppp(struct bnxt_re_push_buffer *pbuf, struct bnxt_re_qp *qp, uint8_t len, uint32_t idx); int bnxt_re_init_pbuf_list(struct bnxt_re_context *cntx); void bnxt_re_destroy_pbuf_list(struct bnxt_re_context *cntx); struct bnxt_re_push_buffer *bnxt_re_get_pbuf(uint8_t *push_st_en, uint8_t ppp_idx, struct bnxt_re_context *cntx); void bnxt_re_put_pbuf(struct bnxt_re_context *cntx, struct bnxt_re_push_buffer *pbuf); void bnxt_re_db_recovery(struct bnxt_re_context *cntx); void *bnxt_re_dbr_thread(void *arg); bool _is_db_drop_recovery_enable(struct bnxt_re_context *cntx); int bnxt_re_poll_kernel_cq(struct bnxt_re_cq *cq); extern int bnxt_single_threaded; extern int bnxt_dyn_debug; #define bnxt_re_trace(fmt, ...) \ { \ if (bnxt_dyn_debug) \ fprintf(stderr, fmt, ##__VA_ARGS__); \ } /* pointer conversion functions*/ static inline struct bnxt_re_dev *to_bnxt_re_dev(struct ibv_device *ibvdev) { return container_of(ibvdev, struct bnxt_re_dev, vdev); } static inline struct bnxt_re_context *to_bnxt_re_context( struct ibv_context *ibvctx) { return container_of(ibvctx, struct bnxt_re_context, ibvctx); } static inline struct bnxt_re_pd *to_bnxt_re_pd(struct ibv_pd *ibvpd) { return container_of(ibvpd, struct bnxt_re_pd, ibvpd); } static inline struct bnxt_re_cq *to_bnxt_re_cq(struct ibv_cq *ibvcq) { return container_of(ibvcq, struct bnxt_re_cq, ibvcq); } static inline struct bnxt_re_qp *to_bnxt_re_qp(struct ibv_qp *ibvqp) { return container_of(ibvqp, struct bnxt_re_qp, ibvqp); } static inline struct bnxt_re_srq *to_bnxt_re_srq(struct ibv_srq *ibvsrq) { return container_of(ibvsrq, struct bnxt_re_srq, ibvsrq); } static inline struct bnxt_re_ah *to_bnxt_re_ah(struct ibv_ah *ibvah) { return container_of(ibvah, struct bnxt_re_ah, ibvah); } /* CQE manipulations */ #define bnxt_re_get_cqe_sz() (sizeof(struct bnxt_re_req_cqe) + \ sizeof(struct bnxt_re_bcqe)) #define bnxt_re_get_sqe_hdr_sz() (sizeof(struct bnxt_re_bsqe) + \ sizeof(struct bnxt_re_send)) #define bnxt_re_get_srqe_hdr_sz() (sizeof(struct bnxt_re_brqe) + \ sizeof(struct bnxt_re_srqe)) #define bnxt_re_get_srqe_sz() (sizeof(struct bnxt_re_brqe) + \ sizeof(struct bnxt_re_srqe) + \ BNXT_RE_MAX_INLINE_SIZE) #define bnxt_re_is_cqe_valid(valid, phase) \ (((valid) & BNXT_RE_BCQE_PH_MASK) == (phase)) static inline void bnxt_re_change_cq_phase(struct bnxt_re_cq *cq) { if (!cq->cqq->head) cq->phase = !(cq->phase & BNXT_RE_BCQE_PH_MASK); } static inline void *bnxt_re_get_swqe(struct bnxt_re_joint_queue *jqq, uint32_t *wqe_idx) { if (wqe_idx) *wqe_idx = jqq->start_idx; return &jqq->swque[jqq->start_idx]; } static inline void bnxt_re_jqq_mod_start(struct bnxt_re_joint_queue *jqq, uint32_t idx) { jqq->start_idx = jqq->swque[idx].next_idx; } static inline void bnxt_re_jqq_mod_last(struct bnxt_re_joint_queue *jqq, uint32_t idx) { jqq->last_idx = jqq->swque[idx].next_idx; } static inline uint32_t bnxt_re_init_depth(uint32_t ent, uint64_t cmask) { return cmask & BNXT_RE_COMP_MASK_UCNTX_POW2_DISABLED ? ent : roundup_pow_of_two(ent); } static inline uint32_t bnxt_re_get_diff(uint64_t cmask) { return cmask & BNXT_RE_COMP_MASK_UCNTX_RSVD_WQE_DISABLED ? 0 : BNXT_RE_FULL_FLAG_DELTA; } static inline int bnxt_re_calc_wqe_sz(int nsge) { /* This is used for both sq and rq. In case hdr size differs * in future move to individual functions. */ return sizeof(struct bnxt_re_sge) * nsge + bnxt_re_get_sqe_hdr_sz(); } /* Helper function to copy to push buffers */ static inline void bnxt_re_copy_data_to_pb(struct bnxt_re_push_buffer *pbuf, uint8_t offset, uint32_t idx) { __u64 *src; __u64 *dst; int indx; for (indx = 0; indx < idx; indx++) { dst = (__u64 *)(pbuf->pbuf + 2 * (indx + offset)); src = (__u64 *)pbuf->wqe[indx]; iowrite64(dst, *src); dst++; src++; iowrite64(dst, *src); } } static inline int bnxt_re_dp_spin_init(struct bnxt_spinlock *lock, int pshared, int need_lock) { lock->in_use = 0; lock->need_lock = need_lock; return pthread_spin_init(&lock->lock, PTHREAD_PROCESS_PRIVATE); } static inline int bnxt_re_dp_spin_destroy(struct bnxt_spinlock *lock) { return pthread_spin_destroy(&lock->lock); } static inline int bnxt_spin_lock(struct bnxt_spinlock *lock) { if (lock->need_lock) return pthread_spin_lock(&lock->lock); if (unlikely(lock->in_use)) { fprintf(stderr, "*** ERROR: multithreading violation ***\n" "You are running a multithreaded application but\n" "you set BNXT_SINGLE_THREADED=1. Please unset it.\n"); abort(); } else { lock->in_use = 1; /* This fence is not at all correct, but it increases the */ /* chance that in_use is detected by another thread without */ /* much runtime cost. */ atomic_thread_fence(memory_order_acq_rel); } return 0; } static inline int bnxt_spin_unlock(struct bnxt_spinlock *lock) { if (lock->need_lock) return pthread_spin_unlock(&lock->lock); lock->in_use = 0; return 0; } static void timespec_sub(const struct timespec *a, const struct timespec *b, struct timespec *res) { res->tv_sec = a->tv_sec - b->tv_sec; res->tv_nsec = a->tv_nsec - b->tv_nsec; if (res->tv_nsec < 0) { res->tv_sec--; res->tv_nsec += BNXT_NSEC_PER_SEC; } } /* * Function waits in a busy loop for a given nano seconds * The maximum wait period allowed is less than one second */ static inline void bnxt_re_sub_sec_busy_wait(uint32_t nsec) { struct timespec start, cur, res; if (nsec >= BNXT_NSEC_PER_SEC) return; if (clock_gettime(CLOCK_REALTIME, &start)) { fprintf(stderr, "%s: failed to get time : %d", __func__, errno); return; } while (1) { if (clock_gettime(CLOCK_REALTIME, &cur)) { fprintf(stderr, "%s: failed to get time : %d", __func__, errno); return; } timespec_sub(&cur, &start, &res); if (res.tv_nsec >= nsec) break; } } #define BNXT_RE_HW_RETX(a) ((a)->comp_mask & BNXT_RE_COMP_MASK_UCNTX_HW_RETX_ENABLED) #define bnxt_re_dp_spin_lock(lock) bnxt_spin_lock(lock) #define bnxt_re_dp_spin_unlock(lock) bnxt_spin_unlock(lock) #endif diff --git a/contrib/ofed/libbnxtre/verbs.c b/contrib/ofed/libbnxtre/verbs.c index 7054af34b0c7..320b374abe7e 100644 --- a/contrib/ofed/libbnxtre/verbs.c +++ b/contrib/ofed/libbnxtre/verbs.c @@ -1,2557 +1,2566 @@ /* * Copyright (c) 2024, Broadcom. All rights reserved. The term * Broadcom refers to Broadcom Limited and/or its subsidiaries. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in * the documentation and/or other materials provided with the * distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * */ #include #include #include #include #include #include #include #include #include #include #include #include "main.h" #include "verbs.h" static int ibv_to_bnxt_re_wr_opcd[11] = { BNXT_RE_WR_OPCD_RDMA_WRITE, BNXT_RE_WR_OPCD_RDMA_WRITE_IMM, BNXT_RE_WR_OPCD_SEND, BNXT_RE_WR_OPCD_SEND_IMM, BNXT_RE_WR_OPCD_RDMA_READ, BNXT_RE_WR_OPCD_ATOMIC_CS, BNXT_RE_WR_OPCD_ATOMIC_FA, BNXT_RE_WR_OPCD_INVAL, BNXT_RE_WR_OPCD_INVAL, BNXT_RE_WR_OPCD_INVAL, BNXT_RE_WR_OPCD_INVAL }; static int ibv_wr_to_wc_opcd[11] = { IBV_WC_RDMA_WRITE, IBV_WC_RDMA_WRITE, IBV_WC_SEND, IBV_WC_SEND, IBV_WC_RDMA_READ, IBV_WC_COMP_SWAP, IBV_WC_FETCH_ADD, 0xFF, 0xFF, 0xFF, 0xFF }; static int bnxt_re_req_to_ibv_status [12] = { IBV_WC_SUCCESS, IBV_WC_BAD_RESP_ERR, IBV_WC_LOC_LEN_ERR, IBV_WC_LOC_QP_OP_ERR, IBV_WC_LOC_PROT_ERR, IBV_WC_MW_BIND_ERR, IBV_WC_REM_INV_REQ_ERR, IBV_WC_REM_ACCESS_ERR, IBV_WC_REM_OP_ERR, IBV_WC_RNR_RETRY_EXC_ERR, IBV_WC_RETRY_EXC_ERR, IBV_WC_WR_FLUSH_ERR }; static int bnxt_re_res_to_ibv_status [9] = { IBV_WC_SUCCESS, IBV_WC_LOC_ACCESS_ERR, IBV_WC_LOC_LEN_ERR, IBV_WC_LOC_PROT_ERR, IBV_WC_LOC_QP_OP_ERR, IBV_WC_MW_BIND_ERR, IBV_WC_REM_INV_REQ_ERR, IBV_WC_WR_FLUSH_ERR, IBV_WC_FATAL_ERR }; static int bnxt_re_poll_one(struct bnxt_re_cq *cq, int nwc, struct ibv_wc *wc, uint32_t *resize); int bnxt_single_threaded; int bnxt_dyn_debug; int bnxt_re_query_device(struct ibv_context *ibvctx, struct ibv_device_attr *dev_attr) { struct ibv_query_device cmd = {}; uint8_t fw_ver[8]; int status; memset(dev_attr, 0, sizeof(struct ibv_device_attr)); status = ibv_cmd_query_device(ibvctx, dev_attr, (uint64_t *)&fw_ver, &cmd, sizeof(cmd)); snprintf(dev_attr->fw_ver, 64, "%d.%d.%d.%d", fw_ver[0], fw_ver[1], fw_ver[2], fw_ver[3]); return status; } int bnxt_re_query_device_compat(struct ibv_context *ibvctx, struct ibv_device_attr *dev_attr) { int rc = 0; rc = bnxt_re_query_device(ibvctx, dev_attr); return rc; } int bnxt_re_query_port(struct ibv_context *ibvctx, uint8_t port, struct ibv_port_attr *port_attr) { struct ibv_query_port cmd = {}; return ibv_cmd_query_port(ibvctx, port, port_attr, &cmd, sizeof(cmd)); } static inline bool bnxt_re_is_wcdpi_enabled(struct bnxt_re_context *cntx) { return cntx->comp_mask & BNXT_RE_COMP_MASK_UCNTX_WC_DPI_ENABLED; } static int bnxt_re_map_db_page(struct ibv_context *ibvctx, uint64_t dbr, uint32_t dpi, uint32_t wcdpi) { struct bnxt_re_context *cntx = to_bnxt_re_context(ibvctx); struct bnxt_re_dev *dev = to_bnxt_re_dev(ibvctx->device); cntx->udpi.dpindx = dpi; cntx->udpi.dbpage = mmap(NULL, dev->pg_size, PROT_WRITE, MAP_SHARED, ibvctx->cmd_fd, dbr); if (cntx->udpi.dbpage == MAP_FAILED) return -ENOMEM; if (wcdpi) { cntx->udpi.wcdbpg = mmap(NULL, dev->pg_size, PROT_WRITE, MAP_SHARED, ibvctx->cmd_fd, BNXT_RE_MAP_WC); if (cntx->udpi.wcdbpg == MAP_FAILED) return -ENOMEM; cntx->udpi.wcdpi = wcdpi; } return 0; } struct ibv_pd *bnxt_re_alloc_pd(struct ibv_context *ibvctx) { struct bnxt_re_context *cntx = to_bnxt_re_context(ibvctx); struct bnxt_re_pd_resp resp = {}; struct ibv_alloc_pd cmd = {}; struct bnxt_re_pd *pd; uint64_t dbr_map; pd = calloc(1, sizeof(*pd)); if (!pd) return NULL; if (ibv_cmd_alloc_pd(ibvctx, &pd->ibvpd, &cmd, sizeof(cmd), &resp.resp, sizeof(resp))) goto out; pd->pdid = resp.pdid; /* Map DB page now. */ if (!cntx->udpi.dbpage) { uint32_t wcdpi = 0; if (bnxt_re_is_wcdpi_enabled(cntx) && resp.comp_mask & BNXT_RE_COMP_MASK_PD_HAS_WC_DPI) wcdpi = resp.wcdpi; if (bnxt_re_map_db_page(ibvctx, resp.dbr, resp.dpi, wcdpi)) goto fail; if (cntx->cctx->chip_is_gen_p5_thor2 && cntx->udpi.wcdpi) bnxt_re_init_pbuf_list(cntx); } if (resp.comp_mask & BNXT_RE_COMP_MASK_PD_HAS_DBR_BAR_ADDR) { dbr_map = resp.dbr_bar_map & 0xFFFFFFFFFFFFF000; cntx->bar_map = mmap(NULL, 4096, PROT_READ, MAP_SHARED, ibvctx->cmd_fd, dbr_map); if (cntx->bar_map == MAP_FAILED) goto fail; } return &pd->ibvpd; fail: ibv_cmd_dealloc_pd(&pd->ibvpd); out: free(pd); return NULL; } int bnxt_re_free_pd(struct ibv_pd *ibvpd) { struct bnxt_re_pd *pd = to_bnxt_re_pd(ibvpd); int status; status = ibv_cmd_dealloc_pd(ibvpd); if (status) return status; /* DPI un-mapping will be done during uninit_ucontext */ free(pd); return 0; } struct ibv_mr *get_ibv_mr_from_bnxt_re_mr(struct bnxt_re_mr *mr) { return &mr->vmr; } struct ibv_mr *bnxt_re_reg_mr(struct ibv_pd *ibvpd, void *sva, size_t len, int access) { struct bnxt_re_mr_resp resp = {}; struct ibv_reg_mr cmd = {}; struct bnxt_re_mr *mr; uint64_t hw_va; hw_va = (uint64_t) sva; mr = calloc(1, sizeof(*mr)); if (!mr) return NULL; if (ibv_cmd_reg_mr(ibvpd, sva, len, hw_va, access, &mr->vmr, &cmd, sizeof(cmd), &resp.resp, sizeof(resp))) { free(mr); return NULL; } return get_ibv_mr_from_bnxt_re_mr(mr); } int bnxt_re_dereg_mr(VERBS_MR *ibvmr) { struct bnxt_re_mr *mr = (struct bnxt_re_mr *)ibvmr; int status; status = ibv_cmd_dereg_mr(ibvmr); if (status) return status; free(mr); return 0; } void *bnxt_re_alloc_cqslab(struct bnxt_re_context *cntx, uint32_t ncqe, uint32_t cur) { struct bnxt_re_mem *mem; uint32_t depth, sz; depth = bnxt_re_init_depth(ncqe + 1, cntx->comp_mask); if (depth > cntx->rdev->max_cq_depth + 1) depth = cntx->rdev->max_cq_depth + 1; if (depth == cur) return NULL; sz = get_aligned((depth * cntx->rdev->cqe_size), cntx->rdev->pg_size); mem = bnxt_re_alloc_mem(sz, cntx->rdev->pg_size); if (mem) mem->pad = depth; return mem; } struct ibv_cq *_bnxt_re_create_cq(struct ibv_context *ibvctx, int ncqe, struct ibv_comp_channel *channel, int vec, bool soft_cq) { struct bnxt_re_context *cntx = to_bnxt_re_context(ibvctx); struct bnxt_re_dev *dev = to_bnxt_re_dev(ibvctx->device); struct bnxt_re_cq_resp resp = {}; struct bnxt_re_cq_req cmd = {}; struct bnxt_re_cq *cq; bool has_dpi; if (ncqe > dev->max_cq_depth) return NULL; cq = calloc(1, (sizeof(*cq) + sizeof(struct bnxt_re_queue))); if (!cq) return NULL; cq->cqq = (void *)((char *)cq + sizeof(*cq)); if (!cq->cqq) goto mem; cq->mem = bnxt_re_alloc_cqslab(cntx, ncqe, 0); if (!cq->mem) goto mem; cq->cqq->depth = cq->mem->pad; cq->cqq->stride = dev->cqe_size; /* As an exception no need to call get_ring api we know * this is the only consumer */ cq->cqq->va = cq->mem->va_head; if (!cq->cqq->va) goto fail; cmd.cq_va = (uint64_t)cq->cqq->va; cmd.cq_handle = (uint64_t)cq; if (soft_cq) { cmd.comp_mask |= BNXT_RE_COMP_MASK_CQ_REQ_HAS_CAP_MASK; cmd.cq_capab |= BNXT_RE_COMP_MASK_CQ_REQ_CAP_DBR_RECOVERY; } if (ibv_cmd_create_cq(ibvctx, ncqe, channel, vec, &cq->ibvcq, &cmd.cmd, sizeof(cmd), &resp.resp, sizeof(resp))) goto fail; has_dpi = resp.comp_mask & BNXT_RE_COMP_MASK_CQ_HAS_DB_INFO; if (!cntx->udpi.dbpage && has_dpi) { uint32_t wcdpi = 0; if (bnxt_re_is_wcdpi_enabled(cntx) && resp.comp_mask & BNXT_RE_COMP_MASK_CQ_HAS_WC_DPI) wcdpi = resp.wcdpi; if (bnxt_re_map_db_page(ibvctx, resp.dbr, resp.dpi, wcdpi)) goto fail; if (cntx->cctx->chip_is_gen_p5_thor2 && cntx->udpi.wcdpi) bnxt_re_init_pbuf_list(cntx); } if (resp.comp_mask & BNXT_RE_COMP_MASK_CQ_HAS_CQ_PAGE) { - cq->cq_page = mmap(NULL, dev->pg_size, PROT_WRITE, MAP_SHARED, + cq->cq_page = mmap(NULL, dev->pg_size, PROT_READ | PROT_WRITE, MAP_SHARED, ibvctx->cmd_fd, resp.cq_page); - if (!cq->cq_page) + if (cq->cq_page == MAP_FAILED) fprintf(stderr, DEV "Valid cq_page not mapped\n"); } cq->cqid = resp.cqid; cq->phase = resp.phase; cq->cqq->tail = resp.tail; cq->udpi = &cntx->udpi; cq->first_arm = true; cq->cntx = cntx; cq->rand.seed = cq->cqid; cq->shadow_db_key = BNXT_RE_DB_KEY_INVALID; bnxt_re_dp_spin_init(&cq->cqq->qlock, PTHREAD_PROCESS_PRIVATE, !bnxt_single_threaded); INIT_DBLY_LIST_HEAD(&cq->sfhead); INIT_DBLY_LIST_HEAD(&cq->rfhead); INIT_DBLY_LIST_HEAD(&cq->prev_cq_head); if (_is_db_drop_recovery_enable(cntx) && !soft_cq) { INIT_DBLY_LIST_NODE(&cq->dbnode); pthread_spin_lock(&cntx->cq_dbr_res.lock); bnxt_re_list_add_node(&cq->dbnode, &cntx->cq_dbr_res.head); pthread_spin_unlock(&cntx->cq_dbr_res.lock); } return &cq->ibvcq; fail: bnxt_re_free_mem(cq->mem); mem: free(cq); return NULL; } struct ibv_cq *bnxt_re_create_cq(struct ibv_context *ibvctx, int ncqe, struct ibv_comp_channel *channel, int vec) { struct bnxt_re_context *cntx = to_bnxt_re_context(ibvctx); struct bnxt_re_dev *dev = to_bnxt_re_dev(ibvctx->device); sigset_t block_sig_set, old_sig_set; int ret; if (_is_db_drop_recovery_enable(cntx) && !cntx->dbr_cq) { cntx->dbr_ev_chan = ibv_create_comp_channel(ibvctx); if (!cntx->dbr_ev_chan) { fprintf(stderr, DEV "Failed to create completion channel\n"); goto free; } cntx->dbr_cq = _bnxt_re_create_cq(ibvctx, 1, cntx->dbr_ev_chan, vec, 1); if (!cntx->dbr_cq) { fprintf(stderr, DEV "Couldn't create CQ\n"); goto free; } cntx->db_recovery_page = mmap(NULL, dev->pg_size, PROT_READ | PROT_WRITE, MAP_SHARED, ibvctx->cmd_fd, BNXT_RE_DB_RECOVERY_PAGE); if (cntx->db_recovery_page == MAP_FAILED) { fprintf(stderr, DEV "Couldn't map DB recovery page\n"); goto free; } /* Create pthread to handle the doorbell drop events. This thread is * not going to handle any signals. Before creation block all the * signals, and after creation restore the old signal mask. */ sigfillset(&block_sig_set); pthread_sigmask(SIG_BLOCK, &block_sig_set, &old_sig_set); ret = pthread_create(&cntx->dbr_thread, NULL, bnxt_re_dbr_thread, cntx); if (ret) { fprintf(stderr, DEV "Couldn't create pthread\n"); pthread_sigmask(SIG_SETMASK, &old_sig_set, NULL); goto free; } pthread_sigmask(SIG_SETMASK, &old_sig_set, NULL); INIT_DBLY_LIST_HEAD(&cntx->qp_dbr_res.head); pthread_spin_init(&cntx->qp_dbr_res.lock, PTHREAD_PROCESS_PRIVATE); INIT_DBLY_LIST_HEAD(&cntx->cq_dbr_res.head); pthread_spin_init(&cntx->cq_dbr_res.lock, PTHREAD_PROCESS_PRIVATE); INIT_DBLY_LIST_HEAD(&cntx->srq_dbr_res.head); pthread_spin_init(&cntx->srq_dbr_res.lock, PTHREAD_PROCESS_PRIVATE); } return(_bnxt_re_create_cq(ibvctx, ncqe, channel, vec, 0)); free: if (cntx->dbr_ev_chan) { ret = ibv_destroy_comp_channel(cntx->dbr_ev_chan); if (ret) fprintf(stderr, DEV "ibv_destroy_comp_channel error\n"); } if (cntx->dbr_cq) { if (cntx->db_recovery_page) munmap(cntx->db_recovery_page, dev->pg_size); ret = ibv_destroy_cq(cntx->dbr_cq); if (ret) fprintf(stderr, DEV "ibv_destroy_cq error\n"); } return NULL; } int bnxt_re_poll_kernel_cq(struct bnxt_re_cq *cq) { struct ibv_wc tmp_wc; int rc; rc = ibv_cmd_poll_cq(&cq->ibvcq, 1, &tmp_wc); if (unlikely(rc)) fprintf(stderr, "ibv_cmd_poll_cq failed: %d\n", rc); return rc; } #define BNXT_RE_QUEUE_START_PHASE 0x01 /* * Function to complete the last steps in CQ resize. Invoke poll function * in the kernel driver; this serves as a signal to the driver to complete CQ * resize steps required. Free memory mapped for the original CQ and switch * over to the memory mapped for CQ with the new size. Finally Ack the Cutoff * CQE. This function must be called under cq->cqq.lock. */ void bnxt_re_resize_cq_complete(struct bnxt_re_cq *cq) { struct bnxt_re_context *cntx = to_bnxt_re_context(cq->ibvcq.context); bnxt_re_poll_kernel_cq(cq); bnxt_re_free_mem(cq->mem); cq->mem = cq->resize_mem; cq->resize_mem = NULL; /* As an exception no need to call get_ring api we know * this is the only consumer */ cq->cqq->va = cq->mem->va_head; /* * We don't want to memcpy() the entire cqq structure below; otherwise * we'd end up overwriting cq->cqq.lock that is held by the caller. * So we copy the members piecemeal. cqq->head, cqq->tail implicitly * set to 0 before cutoff_ack DB. */ cq->cqq->depth = cq->mem->pad; cq->cqq->stride = cntx->rdev->cqe_size; cq->cqq->head = 0; cq->cqq->tail = 0; cq->phase = BNXT_RE_QUEUE_START_PHASE; /* Reset epoch portion of the flags */ cq->cqq->flags &= ~(BNXT_RE_FLAG_EPOCH_TAIL_MASK | BNXT_RE_FLAG_EPOCH_HEAD_MASK); bnxt_re_ring_cq_arm_db(cq, BNXT_RE_QUE_TYPE_CQ_CUT_ACK); } int bnxt_re_resize_cq(struct ibv_cq *ibvcq, int ncqe) { struct bnxt_re_context *cntx = to_bnxt_re_context(ibvcq->context); struct bnxt_re_dev *dev = to_bnxt_re_dev(ibvcq->context->device); struct bnxt_re_cq *cq = to_bnxt_re_cq(ibvcq); struct bnxt_re_resize_cq_req req = {}; uint32_t exit_cnt = 20; struct ibv_resize_cq_resp resp = {}; int rc = 0; if (ncqe > dev->max_cq_depth) return -EINVAL; bnxt_re_dp_spin_lock(&cq->cqq->qlock); cq->resize_mem = bnxt_re_alloc_cqslab(cntx, ncqe, cq->cqq->depth); if (unlikely(!cq->resize_mem)) { rc = -ENOMEM; goto done; } /* As an exception no need to call get_ring api we know * this is the only consumer */ req.cq_va = (uint64_t)cq->resize_mem->va_head; rc = ibv_cmd_resize_cq(ibvcq, ncqe, &req.cmd, sizeof(req), &resp, sizeof(resp)); if (unlikely(rc)) { bnxt_re_free_mem(cq->resize_mem); goto done; } while(true) { struct ibv_wc tmp_wc = {0}; uint32_t resize = 0; int dqed = 0; struct bnxt_re_work_compl *compl = NULL; dqed = bnxt_re_poll_one(cq, 1, &tmp_wc, &resize); if (resize) { break; } if (dqed) { compl = calloc(1, sizeof(*compl)); if (unlikely(!compl)) { fprintf(stderr, "%s: No Memory.. Continue\n", __func__); break; } memcpy(&compl->wc, &tmp_wc, sizeof(tmp_wc)); bnxt_re_list_add_node(&compl->cnode, &cq->prev_cq_head); compl = NULL; memset(&tmp_wc, 0, sizeof(tmp_wc)); } else { exit_cnt--; if (unlikely(!exit_cnt)) { rc = -EIO; break; } else { /* wait for 100 milli seconds */ bnxt_re_sub_sec_busy_wait(100 * 1000000); } } } done: bnxt_re_dp_spin_unlock(&cq->cqq->qlock); return rc; } static void bnxt_re_destroy_resize_cq_list(struct bnxt_re_cq *cq) { struct bnxt_re_list_node *cur, *tmp; struct bnxt_re_work_compl *compl; if (bnxt_re_list_empty(&cq->prev_cq_head)) return; list_for_each_node_safe(cur, tmp, &cq->prev_cq_head) { compl = list_node(cur, struct bnxt_re_work_compl, cnode); bnxt_re_list_del_node(&compl->cnode, &cq->prev_cq_head); free(compl); } } int bnxt_re_destroy_cq(struct ibv_cq *ibvcq) { struct bnxt_re_cq *cq = to_bnxt_re_cq(ibvcq); int status; if (_is_db_drop_recovery_enable(cq->cntx) && ibvcq != cq->cntx->dbr_cq) { pthread_spin_lock(&cq->cntx->cq_dbr_res.lock); bnxt_re_list_del_node(&cq->dbnode, &cq->cntx->cq_dbr_res.head); pthread_spin_unlock(&cq->cntx->cq_dbr_res.lock); } status = ibv_cmd_destroy_cq(ibvcq); if (status) { if (_is_db_drop_recovery_enable(cq->cntx) && ibvcq != cq->cntx->dbr_cq) { pthread_spin_lock(&cq->cntx->cq_dbr_res.lock); bnxt_re_list_add_node(&cq->dbnode, &cq->cntx->cq_dbr_res.head); pthread_spin_unlock(&cq->cntx->cq_dbr_res.lock); } return status; } bnxt_re_destroy_resize_cq_list(cq); bnxt_re_free_mem(cq->mem); free(cq); return 0; } static uint8_t bnxt_re_poll_err_scqe(struct bnxt_re_qp *qp, struct ibv_wc *ibvwc, struct bnxt_re_req_cqe *scqe, uint32_t flg_val, int *cnt) { struct bnxt_re_queue *sq = qp->jsqq->hwque; struct bnxt_re_wrid *swrid; struct bnxt_re_cq *scq; uint8_t status; uint32_t head; scq = to_bnxt_re_cq(qp->ibvqp.send_cq); head = qp->jsqq->last_idx; swrid = &qp->jsqq->swque[head]; *cnt = 1; status = (flg_val >> BNXT_RE_BCQE_STATUS_SHIFT) & BNXT_RE_BCQE_STATUS_MASK; ibvwc->status = bnxt_re_req_to_ibv_status[status]; ibvwc->wc_flags = 0; ibvwc->wr_id = swrid->wrid; ibvwc->qp_num = qp->qpid; ibvwc->opcode = swrid->wc_opcd; ibvwc->byte_len = 0; bnxt_re_incr_head(sq, swrid->slots); bnxt_re_jqq_mod_last(qp->jsqq, head); if (qp->qpst != IBV_QPS_ERR) qp->qpst = IBV_QPS_ERR; bnxt_re_list_add_node(&qp->snode, &scq->sfhead); bnxt_re_trace("%s: qp_num = 0x%x status = %d\n", __func__, ibvwc->qp_num, ibvwc->status) return false; } static uint8_t bnxt_re_poll_success_scqe(struct bnxt_re_qp *qp, struct ibv_wc *ibvwc, struct bnxt_re_req_cqe *scqe, int *cnt) { struct bnxt_re_queue *sq = qp->jsqq->hwque; struct bnxt_re_wrid *swrid; uint8_t pcqe = false; uint32_t cindx, head; head = qp->jsqq->last_idx; swrid = &qp->jsqq->swque[head]; cindx = le32toh(scqe->con_indx) % qp->cap.max_swr; if (!(swrid->sig & IBV_SEND_SIGNALED)) { *cnt = 0; } else { ibvwc->status = IBV_WC_SUCCESS; ibvwc->wc_flags = 0; ibvwc->qp_num = qp->qpid; ibvwc->wr_id = swrid->wrid; ibvwc->opcode = swrid->wc_opcd; if (ibvwc->opcode == IBV_WC_RDMA_READ || ibvwc->opcode == IBV_WC_COMP_SWAP || ibvwc->opcode == IBV_WC_FETCH_ADD) ibvwc->byte_len = swrid->bytes; *cnt = 1; } bnxt_re_incr_head(sq, swrid->slots); bnxt_re_jqq_mod_last(qp->jsqq, head); if (qp->jsqq->last_idx != cindx) pcqe = true; return pcqe; } static uint8_t bnxt_re_poll_scqe(struct bnxt_re_qp *qp, struct ibv_wc *ibvwc, void *cqe, uint32_t flg_val, int *cnt) { uint8_t status, pcqe = false; status = (flg_val >> BNXT_RE_BCQE_STATUS_SHIFT) & BNXT_RE_BCQE_STATUS_MASK; if (status == BNXT_RE_REQ_ST_OK) pcqe = bnxt_re_poll_success_scqe(qp, ibvwc, cqe, cnt); else pcqe = bnxt_re_poll_err_scqe(qp, ibvwc, cqe, flg_val, cnt); return pcqe; } static void bnxt_re_release_srqe(struct bnxt_re_srq *srq, int tag) { bnxt_re_dp_spin_lock(&srq->srqq->qlock); srq->srwrid[srq->last_idx].next_idx = tag; srq->last_idx = tag; srq->srwrid[srq->last_idx].next_idx = -1; bnxt_re_dp_spin_unlock(&srq->srqq->qlock); } static int bnxt_re_poll_err_rcqe(struct bnxt_re_qp *qp, struct ibv_wc *ibvwc, struct bnxt_re_bcqe *hdr, uint32_t flg_val, void *cqe) { struct bnxt_re_wrid *swque; struct bnxt_re_queue *rq; struct bnxt_re_cq *rcq; uint8_t status, cnt; uint32_t head = 0; rcq = to_bnxt_re_cq(qp->ibvqp.recv_cq); status = (flg_val >> BNXT_RE_BCQE_STATUS_SHIFT) & BNXT_RE_BCQE_STATUS_MASK; /* skip h/w flush errors */ if (status == BNXT_RE_RSP_ST_HW_FLUSH) return 0; if (!qp->srq) { rq = qp->jrqq->hwque; head = qp->jrqq->last_idx; swque = &qp->jrqq->swque[head]; ibvwc->wr_id = swque->wrid; cnt = swque->slots; } else { struct bnxt_re_srq *srq; int tag; srq = qp->srq; rq = srq->srqq; cnt = 1; tag = le32toh(hdr->qphi_rwrid) & BNXT_RE_BCQE_RWRID_MASK; ibvwc->wr_id = srq->srwrid[tag].wrid; bnxt_re_release_srqe(srq, tag); } ibvwc->status = bnxt_re_res_to_ibv_status[status]; ibvwc->qp_num = qp->qpid; ibvwc->opcode = IBV_WC_RECV; ibvwc->byte_len = 0; ibvwc->wc_flags = 0; if (qp->qptyp == IBV_QPT_UD) ibvwc->src_qp = 0; if (!qp->srq) bnxt_re_jqq_mod_last(qp->jrqq, head); bnxt_re_incr_head(rq, cnt); if (!qp->srq) bnxt_re_list_add_node(&qp->rnode, &rcq->rfhead); bnxt_re_trace("%s: qp_num = 0x%x status = %d\n", __func__, ibvwc->qp_num, ibvwc->status) return 1; } static void bnxt_re_fill_ud_cqe(struct ibv_wc *ibvwc, struct bnxt_re_bcqe *hdr, void *cqe, uint8_t flags) { struct bnxt_re_ud_cqe *ucqe = cqe; uint32_t qpid; qpid = ((le32toh(hdr->qphi_rwrid) >> BNXT_RE_BCQE_SRCQP_SHIFT) & BNXT_RE_BCQE_SRCQP_SHIFT) << 0x10; /* higher 8 bits of 24 */ qpid |= (le64toh(ucqe->qplo_mac) >> BNXT_RE_UD_CQE_SRCQPLO_SHIFT) & BNXT_RE_UD_CQE_SRCQPLO_MASK; /*lower 16 of 24 */ ibvwc->src_qp = qpid; ibvwc->wc_flags |= IBV_WC_GRH; ibvwc->sl = (flags & BNXT_RE_UD_FLAGS_IP_VER_MASK) >> BNXT_RE_UD_FLAGS_IP_VER_SFT; /*IB-stack ABI in user do not ask for MAC to be reported. */ } static void bnxt_re_poll_success_rcqe(struct bnxt_re_qp *qp, struct ibv_wc *ibvwc, struct bnxt_re_bcqe *hdr, uint32_t flg_val, void *cqe) { uint8_t flags, is_imm, is_rdma; struct bnxt_re_rc_cqe *rcqe; struct bnxt_re_wrid *swque; struct bnxt_re_queue *rq; uint32_t head = 0; uint32_t rcqe_len; uint8_t cnt; rcqe = cqe; if (!qp->srq) { rq = qp->jrqq->hwque; head = qp->jrqq->last_idx; swque = &qp->jrqq->swque[head]; cnt = swque->slots; ibvwc->wr_id = swque->wrid; } else { struct bnxt_re_srq *srq; int tag; srq = qp->srq; rq = srq->srqq; cnt = 1; tag = le32toh(hdr->qphi_rwrid) & BNXT_RE_BCQE_RWRID_MASK; ibvwc->wr_id = srq->srwrid[tag].wrid; bnxt_re_release_srqe(srq, tag); } ibvwc->status = IBV_WC_SUCCESS; ibvwc->qp_num = qp->qpid; rcqe_len = le32toh(rcqe->length); ibvwc->byte_len = (qp->qptyp == IBV_QPT_UD) ? rcqe_len & BNXT_RE_UD_CQE_LEN_MASK : rcqe_len; ibvwc->opcode = IBV_WC_RECV; flags = (flg_val >> BNXT_RE_BCQE_FLAGS_SHIFT) & BNXT_RE_BCQE_FLAGS_MASK; is_imm = (flags & BNXT_RE_RC_FLAGS_IMM_MASK) >> BNXT_RE_RC_FLAGS_IMM_SHIFT; is_rdma = (flags & BNXT_RE_RC_FLAGS_RDMA_MASK) >> BNXT_RE_RC_FLAGS_RDMA_SHIFT; ibvwc->wc_flags = 0; if (is_imm) { ibvwc->wc_flags |= IBV_WC_WITH_IMM; /* The HW is returning imm_data in little-endian format, * swap to Big Endian as expected by application */ ibvwc->imm_data = htobe32(le32toh(rcqe->imm_key)); if (is_rdma) ibvwc->opcode = IBV_WC_RECV_RDMA_WITH_IMM; } if (qp->qptyp == IBV_QPT_UD) { bnxt_re_fill_ud_cqe(ibvwc, hdr, cqe, flags); } if (!qp->srq) bnxt_re_jqq_mod_last(qp->jrqq, head); bnxt_re_incr_head(rq, cnt); } static uint8_t bnxt_re_poll_rcqe(struct bnxt_re_qp *qp, struct ibv_wc *ibvwc, void *cqe, uint32_t flg_val, int *cnt) { struct bnxt_re_bcqe *hdr; uint8_t status, pcqe = false; hdr = cqe + sizeof(struct bnxt_re_rc_cqe); status = (flg_val >> BNXT_RE_BCQE_STATUS_SHIFT) & BNXT_RE_BCQE_STATUS_MASK; *cnt = 1; if (status == BNXT_RE_RSP_ST_OK) bnxt_re_poll_success_rcqe(qp, ibvwc, hdr, flg_val, cqe); else *cnt = bnxt_re_poll_err_rcqe(qp, ibvwc, hdr, flg_val, cqe); return pcqe; } static void bnxt_re_qp_move_flush_err(struct bnxt_re_qp *qp) { struct bnxt_re_cq *scq, *rcq; scq = to_bnxt_re_cq(qp->ibvqp.send_cq); rcq = to_bnxt_re_cq(qp->ibvqp.recv_cq); if (qp->qpst != IBV_QPS_ERR) qp->qpst = IBV_QPS_ERR; bnxt_re_list_add_node(&qp->rnode, &rcq->rfhead); bnxt_re_list_add_node(&qp->snode, &scq->sfhead); } /* Always return false */ static uint8_t bnxt_re_poll_term_cqe(struct bnxt_re_qp *qp, int *cnt) { /* For now just add the QP to flush list without * considering the index reported in the CQE. * Continue reporting flush completions until the * SQ and RQ are empty. */ *cnt = 0; if (qp->qpst != IBV_QPS_RESET) bnxt_re_qp_move_flush_err(qp); return false; } static int bnxt_re_poll_one(struct bnxt_re_cq *cq, int nwc, struct ibv_wc *wc, uint32_t *resize) { int type, cnt = 0, dqed = 0, hw_polled = 0; struct bnxt_re_queue *cqq = cq->cqq; struct bnxt_re_req_cqe *scqe; struct bnxt_re_ud_cqe *rcqe; uint64_t *qp_handle = NULL; struct bnxt_re_bcqe *hdr; struct bnxt_re_qp *qp; uint8_t pcqe = false; uint32_t flg_val; void *cqe; while (nwc) { cqe = cqq->va + cqq->head * bnxt_re_get_cqe_sz(); hdr = cqe + sizeof(struct bnxt_re_req_cqe); flg_val = le32toh(hdr->flg_st_typ_ph); if (unlikely(!bnxt_re_is_cqe_valid(flg_val, cq->phase))) break; type = (flg_val >> BNXT_RE_BCQE_TYPE_SHIFT) & BNXT_RE_BCQE_TYPE_MASK; switch (type) { case BNXT_RE_WC_TYPE_SEND: scqe = cqe; qp_handle = (uint64_t *)&scqe->qp_handle; qp = (struct bnxt_re_qp *) (uintptr_t)le64toh(scqe->qp_handle); if (!qp) break; /*stale cqe. should be rung.*/ pcqe = bnxt_re_poll_scqe(qp, wc, cqe, flg_val, &cnt); break; case BNXT_RE_WC_TYPE_RECV_RC: case BNXT_RE_WC_TYPE_RECV_UD: rcqe = cqe; qp_handle = (uint64_t *)&rcqe->qp_handle; qp = (struct bnxt_re_qp *) (uintptr_t)le64toh(rcqe->qp_handle); if (!qp) break; /*stale cqe. should be rung.*/ pcqe = bnxt_re_poll_rcqe(qp, wc, cqe, flg_val, &cnt); break; case BNXT_RE_WC_TYPE_RECV_RAW: break; case BNXT_RE_WC_TYPE_TERM: scqe = cqe; qp_handle = (uint64_t *)&scqe->qp_handle; qp = (struct bnxt_re_qp *) (uintptr_t)le64toh(scqe->qp_handle); if (!qp) break; pcqe = bnxt_re_poll_term_cqe(qp, &cnt); break; case BNXT_RE_WC_TYPE_COFF: /* Stop further processing and return */ bnxt_re_resize_cq_complete(cq); if (unlikely(resize)) *resize = 1; return dqed; default: break; }; if (pcqe) goto skipp_real; hw_polled++; if (qp_handle) { *qp_handle = 0x0ULL; /* mark cqe as read */ qp_handle = NULL; } bnxt_re_incr_head(cq->cqq, 1); bnxt_re_change_cq_phase(cq); skipp_real: if (cnt) { cnt = 0; dqed++; nwc--; wc++; } } if (likely(hw_polled)) bnxt_re_ring_cq_db(cq); return dqed; } static int bnxt_re_poll_flush_wcs(struct bnxt_re_joint_queue *jqq, struct ibv_wc *ibvwc, uint32_t qpid, int nwc) { struct bnxt_re_queue *que; struct bnxt_re_wrid *wrid; uint32_t cnt = 0; que = jqq->hwque; while(nwc) { if (bnxt_re_is_que_empty(que)) break; wrid = &jqq->swque[jqq->last_idx]; ibvwc->status = IBV_WC_WR_FLUSH_ERR; ibvwc->opcode = wrid->wc_opcd; ibvwc->wr_id = wrid->wrid; ibvwc->qp_num = qpid; ibvwc->byte_len = 0; ibvwc->wc_flags = 0; bnxt_re_jqq_mod_last(jqq, jqq->last_idx); bnxt_re_incr_head(que, wrid->slots); nwc--; cnt++; ibvwc++; } return cnt; } static int bnxt_re_poll_flush_wqes(struct bnxt_re_cq *cq, struct bnxt_re_list_head *lhead, struct ibv_wc *ibvwc, uint32_t nwc) { struct bnxt_re_list_node *cur, *tmp; struct bnxt_re_joint_queue *jqq; struct bnxt_re_qp *qp; bool sq_list = false; uint32_t polled = 0; sq_list = (lhead == &cq->sfhead) ? true : false; if (!bnxt_re_list_empty(lhead)) { list_for_each_node_safe(cur, tmp, lhead) { if (sq_list) { qp = list_node(cur, struct bnxt_re_qp, snode); jqq = qp->jsqq; } else { qp = list_node(cur, struct bnxt_re_qp, rnode); jqq = qp->jrqq; if (!jqq) /* Using srq no need to flush */ goto done; } if (bnxt_re_is_que_empty(jqq->hwque)) continue; polled += bnxt_re_poll_flush_wcs(jqq, ibvwc + polled, qp->qpid, nwc - polled); if (!(nwc - polled)) break; } } done: return polled; } static int bnxt_re_poll_flush_lists(struct bnxt_re_cq *cq, uint32_t nwc, struct ibv_wc *ibvwc) { int left, polled = 0; polled = bnxt_re_poll_flush_wqes(cq, &cq->sfhead, ibvwc, nwc); left = nwc - polled; if (!left) return polled; polled += bnxt_re_poll_flush_wqes(cq, &cq->rfhead, ibvwc + polled, left); return polled; } static int bnxt_re_poll_resize_cq_list(struct bnxt_re_cq *cq, uint32_t nwc, struct ibv_wc *ibvwc) { struct bnxt_re_list_node *cur, *tmp; struct bnxt_re_work_compl *compl; int left; left = nwc; list_for_each_node_safe(cur, tmp, &cq->prev_cq_head) { compl = list_node(cur, struct bnxt_re_work_compl, cnode); if (!left) break; memcpy(ibvwc, &compl->wc, sizeof(*ibvwc)); ibvwc++; left--; bnxt_re_list_del_node(&compl->cnode, &cq->prev_cq_head); free(compl); } return nwc - left; } int bnxt_re_poll_cq(struct ibv_cq *ibvcq, int nwc, struct ibv_wc *wc) { int dqed = 0, left = 0; struct bnxt_re_cq *cq; uint32_t resize = 0; cq = container_of(ibvcq, struct bnxt_re_cq, ibvcq); bnxt_re_dp_spin_lock(&cq->cqq->qlock); left = nwc; /* Check whether we have anything to be completed from prev cq context */ if (unlikely(!bnxt_re_list_empty(&cq->prev_cq_head))) { dqed = bnxt_re_poll_resize_cq_list(cq, nwc, wc); left = nwc - dqed; if (!left) { bnxt_re_dp_spin_unlock(&cq->cqq->qlock); return dqed; } } dqed += bnxt_re_poll_one(cq, left, wc + dqed, &resize); /* Check if anything is there to flush. */ left = nwc - dqed; if (left && (!bnxt_re_list_empty(&cq->sfhead) || !bnxt_re_list_empty(&cq->rfhead))) dqed += bnxt_re_poll_flush_lists(cq, left, (wc + dqed)); bnxt_re_dp_spin_unlock(&cq->cqq->qlock); return dqed; } void bnxt_re_cleanup_cq(struct bnxt_re_qp *qp, struct bnxt_re_cq *cq) { struct bnxt_re_queue *que = cq->cqq; struct bnxt_re_req_cqe *scqe; struct bnxt_re_rc_cqe *rcqe; struct bnxt_re_bcqe *hdr; int indx, type; void *cqe; bnxt_re_dp_spin_lock(&que->qlock); for(indx = 0; indx < que->depth; indx++) { cqe = que->va + indx * bnxt_re_get_cqe_sz(); hdr = cqe + sizeof(struct bnxt_re_req_cqe); type = (hdr->flg_st_typ_ph >> BNXT_RE_BCQE_TYPE_SHIFT) & BNXT_RE_BCQE_TYPE_MASK; if (type == BNXT_RE_WC_TYPE_COFF) continue; if (type == BNXT_RE_WC_TYPE_SEND || type == BNXT_RE_WC_TYPE_TERM) { scqe = cqe; if (scqe->qp_handle == (uint64_t)qp) scqe->qp_handle = 0ULL; } else { rcqe = cqe; if (rcqe->qp_handle == (uint64_t)qp) rcqe->qp_handle = 0ULL; } } if (_is_db_drop_recovery_enable(cq->cntx)) { pthread_spin_lock(&cq->cntx->cq_dbr_res.lock); bnxt_re_list_del_node(&cq->dbnode, &cq->cntx->cq_dbr_res.head); pthread_spin_unlock(&cq->cntx->cq_dbr_res.lock); } bnxt_re_list_del_node(&qp->snode, &cq->sfhead); bnxt_re_list_del_node(&qp->rnode, &cq->rfhead); bnxt_re_dp_spin_unlock(&que->qlock); } void bnxt_re_cq_event(struct ibv_cq *ibvcq) { } int bnxt_re_arm_cq(struct ibv_cq *ibvcq, int flags) { struct bnxt_re_cq *cq = to_bnxt_re_cq(ibvcq); bnxt_re_dp_spin_lock(&cq->cqq->qlock); flags = !flags ? BNXT_RE_QUE_TYPE_CQ_ARMALL : BNXT_RE_QUE_TYPE_CQ_ARMSE; bnxt_re_ring_cq_arm_db(cq, flags); bnxt_re_dp_spin_unlock(&cq->cqq->qlock); return 0; } static int bnxt_re_check_qp_limits(struct bnxt_re_context *cntx, struct ibv_qp_init_attr *attr) { struct ibv_device_attr *devattr; struct bnxt_re_dev *rdev; rdev = cntx->rdev; devattr = &rdev->devattr; if (attr->qp_type != IBV_QPT_RC && attr->qp_type != IBV_QPT_UD) return EINVAL; if (attr->cap.max_send_sge > devattr->max_sge) return EINVAL; if (attr->cap.max_recv_sge > devattr->max_sge) return EINVAL; if (cntx->modes & BNXT_RE_WQE_MODE_VARIABLE) { if (attr->cap.max_inline_data > BNXT_RE_MAX_INLINE_SIZE_VAR_WQE) return -EINVAL; } else if (attr->cap.max_inline_data > BNXT_RE_MAX_INLINE_SIZE) { return EINVAL; } if (attr->cap.max_send_wr > devattr->max_qp_wr) attr->cap.max_send_wr = devattr->max_qp_wr; if (attr->cap.max_recv_wr > devattr->max_qp_wr) attr->cap.max_recv_wr = devattr->max_qp_wr; return 0; } static int bnxt_re_get_rq_slots(struct bnxt_re_dev *rdev, uint8_t qpmode, uint32_t nrwr, uint32_t nsge, uint32_t *esz) { uint32_t max_wqesz; uint32_t wqe_size; uint32_t stride; uint32_t slots; stride = sizeof(struct bnxt_re_sge); max_wqesz = bnxt_re_calc_wqe_sz(rdev->devattr.max_sge); wqe_size = bnxt_re_calc_wqe_sz(nsge); if (wqe_size > max_wqesz) return -EINVAL; if (qpmode == BNXT_RE_WQE_MODE_STATIC) wqe_size = bnxt_re_calc_wqe_sz(6); if (esz) *esz = wqe_size; slots = (nrwr * wqe_size) / stride; return slots; } static int bnxt_re_get_sq_slots(struct bnxt_re_dev *rdev, uint8_t qpmode, uint32_t nswr, uint32_t nsge, uint32_t ils, uint32_t *esize) { uint32_t max_wqesz; uint32_t wqe_size; uint32_t cal_ils; uint32_t stride; uint32_t ilsize; uint32_t hdr_sz; uint32_t slots; hdr_sz = bnxt_re_get_sqe_hdr_sz(); stride = sizeof(struct bnxt_re_sge); max_wqesz = bnxt_re_calc_wqe_sz(rdev->devattr.max_sge); ilsize = get_aligned(ils, hdr_sz); wqe_size = bnxt_re_calc_wqe_sz(nsge); if (ilsize) { cal_ils = hdr_sz + ilsize; wqe_size = MAX(cal_ils, wqe_size); wqe_size = get_aligned(wqe_size, hdr_sz); } if (wqe_size > max_wqesz) return -EINVAL; if (qpmode == BNXT_RE_WQE_MODE_STATIC) wqe_size = bnxt_re_calc_wqe_sz(6); if (esize) *esize = wqe_size; slots = (nswr * wqe_size) / stride; return slots; } static int bnxt_re_get_sqmem_size(struct bnxt_re_context *cntx, struct ibv_qp_init_attr *attr, struct bnxt_re_qattr *qattr) { uint32_t nsge, nswr, diff = 0; size_t bytes = 0; uint32_t npsn; uint32_t ils; uint8_t mode; uint32_t esz; int nslots; mode = cntx->modes & BNXT_RE_WQE_MODE_VARIABLE; nsge = attr->cap.max_send_sge; diff = bnxt_re_get_diff(cntx->comp_mask); nswr = attr->cap.max_send_wr + 1 + diff; nswr = bnxt_re_init_depth(nswr, cntx->comp_mask); ils = attr->cap.max_inline_data; nslots = bnxt_re_get_sq_slots(cntx->rdev, mode, nswr, nsge, ils, &esz); if (nslots < 0) return nslots; npsn = bnxt_re_get_npsn(mode, nswr, nslots); if (BNXT_RE_HW_RETX(cntx)) npsn = roundup_pow_of_two(npsn); qattr->nwr = nswr; qattr->slots = nslots; qattr->esize = esz; bytes = nslots * sizeof(struct bnxt_re_sge); /* ring */ bytes += npsn * bnxt_re_get_psne_size(cntx); /* psn */ qattr->sz_ring = get_aligned(bytes, cntx->rdev->pg_size); qattr->sz_shad = nswr * sizeof(struct bnxt_re_wrid); /* shadow */ return 0; } static int bnxt_re_get_rqmem_size(struct bnxt_re_context *cntx, struct ibv_qp_init_attr *attr, struct bnxt_re_qattr *qattr) { uint32_t nrwr, nsge; size_t bytes = 0; uint32_t esz; int nslots; nsge = attr->cap.max_recv_sge; nrwr = attr->cap.max_recv_wr + 1; nrwr = bnxt_re_init_depth(nrwr, cntx->comp_mask); nslots = bnxt_re_get_rq_slots(cntx->rdev, cntx->modes, nrwr, nsge, &esz); if (nslots < 0) return nslots; qattr->nwr = nrwr; qattr->slots = nslots; qattr->esize = esz; bytes = nslots * sizeof(struct bnxt_re_sge); qattr->sz_ring = get_aligned(bytes, cntx->rdev->pg_size); qattr->sz_shad = nrwr * sizeof(struct bnxt_re_wrid); return 0; } static int bnxt_re_get_qpmem_size(struct bnxt_re_context *cntx, struct ibv_qp_init_attr *attr, struct bnxt_re_qattr *qattr) { int size = 0; int tmp; int rc; size = sizeof(struct bnxt_re_qp); tmp = sizeof(struct bnxt_re_joint_queue); tmp += sizeof(struct bnxt_re_queue); size += tmp; rc = bnxt_re_get_sqmem_size(cntx, attr, &qattr[BNXT_RE_QATTR_SQ_INDX]); if (rc < 0) return -EINVAL; size += qattr[BNXT_RE_QATTR_SQ_INDX].sz_ring; size += qattr[BNXT_RE_QATTR_SQ_INDX].sz_shad; if (!attr->srq) { tmp = sizeof(struct bnxt_re_joint_queue); tmp += sizeof(struct bnxt_re_queue); size += tmp; rc = bnxt_re_get_rqmem_size(cntx, attr, &qattr[BNXT_RE_QATTR_RQ_INDX]); if (rc < 0) return -EINVAL; size += qattr[BNXT_RE_QATTR_RQ_INDX].sz_ring; size += qattr[BNXT_RE_QATTR_RQ_INDX].sz_shad; } return size; } static void *bnxt_re_alloc_qpslab(struct bnxt_re_context *cntx, struct ibv_qp_init_attr *attr, struct bnxt_re_qattr *qattr) { int bytes; bytes = bnxt_re_get_qpmem_size(cntx, attr, qattr); if (bytes < 0) return NULL; return bnxt_re_alloc_mem(bytes, cntx->rdev->pg_size); } static int bnxt_re_alloc_queue_ptr(struct bnxt_re_qp *qp, struct ibv_qp_init_attr *attr) { int rc = -ENOMEM; int jqsz, qsz; jqsz = sizeof(struct bnxt_re_joint_queue); qsz = sizeof(struct bnxt_re_queue); qp->jsqq = bnxt_re_get_obj(qp->mem, jqsz); if (!qp->jsqq) return rc; qp->jsqq->hwque = bnxt_re_get_obj(qp->mem, qsz); if (!qp->jsqq->hwque) goto fail; if (!attr->srq) { qp->jrqq = bnxt_re_get_obj(qp->mem, jqsz); if (!qp->jrqq) goto fail; qp->jrqq->hwque = bnxt_re_get_obj(qp->mem, qsz); if (!qp->jrqq->hwque) goto fail; } return 0; fail: return rc; } static int bnxt_re_alloc_init_swque(struct bnxt_re_joint_queue *jqq, struct bnxt_re_mem *mem, struct bnxt_re_qattr *qattr) { int indx; jqq->swque = bnxt_re_get_obj(mem, qattr->sz_shad); if (!jqq->swque) return -ENOMEM; jqq->start_idx = 0; jqq->last_idx = qattr->nwr - 1; for (indx = 0; indx < qattr->nwr; indx++) jqq->swque[indx].next_idx = indx + 1; jqq->swque[jqq->last_idx].next_idx = 0; jqq->last_idx = 0; return 0; } static inline int bnxt_log2(int n) { int t; if (n <= 0) return -1; t = 0; while ((1 << t) < n) ++t; return t; } static int bnxt_re_alloc_queues(struct bnxt_re_qp *qp, struct ibv_qp_init_attr *attr, struct bnxt_re_qattr *qattr) { struct bnxt_re_context *cntx; struct bnxt_re_queue *que; uint32_t psn_size; uint8_t indx; int ret; cntx = qp->cntx; indx = BNXT_RE_QATTR_SQ_INDX; que = qp->jsqq->hwque; que->stride = sizeof(struct bnxt_re_sge); que->depth = qattr[indx].slots; que->diff = (bnxt_re_get_diff(cntx->comp_mask) * qattr[indx].esize) / que->stride; que->va = bnxt_re_get_ring(qp->mem, qattr[indx].sz_ring); if (!que->va) return -ENOMEM; /* PSN-search memory is allocated without checking for * QP-Type. Kernel driver do not map this memory if it * is UD-qp. UD-qp use this memory to maintain WC-opcode. * See definition of bnxt_re_fill_psns() for the use case. */ que->pad = (que->va + que->depth * que->stride); psn_size = bnxt_re_get_psne_size(qp->cntx); que->pad_stride_log2 = (uint32_t)bnxt_log2((double)psn_size); ret = bnxt_re_alloc_init_swque(qp->jsqq, qp->mem, &qattr[indx]); if (ret) goto fail; qp->cap.max_swr = qattr[indx].nwr; qp->jsqq->cntx = qp->cntx; que->dbtail = (qp->qpmode == BNXT_RE_WQE_MODE_VARIABLE) ? &que->tail : &qp->jsqq->start_idx; /* Init and adjust MSN table size according to qp mode */ if (!BNXT_RE_HW_RETX(qp->cntx)) goto skip_msn; que->msn = 0; que->msn_tbl_sz = 0; if (qp->qpmode & BNXT_RE_WQE_MODE_VARIABLE) que->msn_tbl_sz = roundup_pow_of_two(qattr->slots) / 2; else que->msn_tbl_sz = roundup_pow_of_two(qattr->nwr); skip_msn: bnxt_re_dp_spin_init(&que->qlock, PTHREAD_PROCESS_PRIVATE, !bnxt_single_threaded); if (qp->jrqq) { indx = BNXT_RE_QATTR_RQ_INDX; que = qp->jrqq->hwque; que->stride = sizeof(struct bnxt_re_sge); que->depth = qattr[indx].slots; que->max_slots = qattr[indx].esize / que->stride; que->dbtail = &qp->jrqq->start_idx; que->va = bnxt_re_get_ring(qp->mem, qattr[indx].sz_ring); if (!que->va) return -ENOMEM; /* For RQ only bnxt_re_wri.wrid is used. */ ret = bnxt_re_alloc_init_swque(qp->jrqq, qp->mem, &qattr[indx]); if (ret) goto fail; bnxt_re_dp_spin_init(&que->qlock, PTHREAD_PROCESS_PRIVATE, !bnxt_single_threaded); qp->cap.max_rwr = qattr[indx].nwr; qp->jrqq->cntx = qp->cntx; } return 0; fail: return ret; } void bnxt_re_async_event(struct ibv_async_event *event) { struct ibv_qp *ibvqp; struct bnxt_re_qp *qp; switch (event->event_type) { case IBV_EVENT_CQ_ERR: break; case IBV_EVENT_SRQ_ERR: case IBV_EVENT_QP_FATAL: case IBV_EVENT_QP_REQ_ERR: case IBV_EVENT_QP_ACCESS_ERR: case IBV_EVENT_PATH_MIG_ERR: { ibvqp = event->element.qp; qp = to_bnxt_re_qp(ibvqp); bnxt_re_qp_move_flush_err(qp); break; } case IBV_EVENT_SQ_DRAINED: case IBV_EVENT_PATH_MIG: case IBV_EVENT_COMM_EST: case IBV_EVENT_QP_LAST_WQE_REACHED: case IBV_EVENT_SRQ_LIMIT_REACHED: case IBV_EVENT_PORT_ACTIVE: case IBV_EVENT_PORT_ERR: default: break; } } struct ibv_qp *bnxt_re_create_qp(struct ibv_pd *ibvpd, struct ibv_qp_init_attr *attr) { struct bnxt_re_context *cntx = to_bnxt_re_context(ibvpd->context); struct bnxt_re_qp_resp resp = {}; struct ibv_device_attr *devattr; struct bnxt_re_qp_req req = {}; struct bnxt_re_qattr qattr[2]; struct bnxt_re_qpcap *cap; struct bnxt_re_dev *rdev; struct bnxt_re_qp *qp; void *mem; if (bnxt_re_check_qp_limits(cntx, attr)) return NULL; memset(qattr, 0, (2 * sizeof(*qattr))); mem = bnxt_re_alloc_qpslab(cntx, attr, qattr); if (!mem) return NULL; qp = bnxt_re_get_obj(mem, sizeof(*qp)); if (!qp) goto fail; qp->mem = mem; qp->cctx = cntx->cctx; qp->cntx = cntx; qp->qpmode = cntx->modes & BNXT_RE_WQE_MODE_VARIABLE; /* alloc queue pointers */ if (bnxt_re_alloc_queue_ptr(qp, attr)) goto fail; /* alloc queues */ if (bnxt_re_alloc_queues(qp, attr, qattr)) goto fail; /* Fill ibv_cmd */ cap = &qp->cap; req.qpsva = (uint64_t)qp->jsqq->hwque->va; req.qprva = qp->jrqq ? (uint64_t)qp->jrqq->hwque->va : 0; req.qp_handle = (uint64_t)qp; if (ibv_cmd_create_qp(ibvpd, &qp->ibvqp, attr, &req.cmd, sizeof(req), &resp.resp, sizeof(resp))) goto fail; qp->qpid = resp.qpid; qp->qptyp = attr->qp_type; qp->qpst = IBV_QPS_RESET; qp->scq = to_bnxt_re_cq(attr->send_cq); qp->rcq = to_bnxt_re_cq(attr->recv_cq); if (attr->srq) qp->srq = to_bnxt_re_srq(attr->srq); qp->udpi = &cntx->udpi; qp->rand.seed = qp->qpid; qp->sq_shadow_db_key = BNXT_RE_DB_KEY_INVALID; qp->rq_shadow_db_key = BNXT_RE_DB_KEY_INVALID; qp->sq_msn = 0; rdev = cntx->rdev; devattr = &rdev->devattr; cap->max_ssge = attr->cap.max_send_sge; cap->max_rsge = attr->cap.max_recv_sge; cap->max_inline = attr->cap.max_inline_data; cap->sqsig = attr->sq_sig_all; cap->is_atomic_cap = devattr->atomic_cap; INIT_DBLY_LIST_NODE(&qp->snode); INIT_DBLY_LIST_NODE(&qp->rnode); INIT_DBLY_LIST_NODE(&qp->dbnode); /* For SR2, push will be negotiated at modify qp */ if (_is_chip_gen_p5(qp->cctx) && cntx->udpi.wcdpi) { qp->push_st_en = 1; qp->max_push_sz = BNXT_RE_MAX_INLINE_SIZE; } if (_is_db_drop_recovery_enable(cntx)) { pthread_spin_lock(&cntx->qp_dbr_res.lock); bnxt_re_list_add_node(&qp->dbnode, &cntx->qp_dbr_res.head); pthread_spin_unlock(&cntx->qp_dbr_res.lock); } return &qp->ibvqp; fail: bnxt_re_free_mem(mem); return NULL; } int bnxt_re_modify_qp(struct ibv_qp *ibvqp, struct ibv_qp_attr *attr, int attr_mask) { struct bnxt_re_qp *qp = to_bnxt_re_qp(ibvqp); int rc; struct bnxt_re_modify_ex_resp resp = {}; struct bnxt_re_modify_ex_req req = {}; bool can_issue_mqp_ex = false; if (bnxt_re_is_mqp_ex_supported(qp->cntx)) { can_issue_mqp_ex = true; /* Request for PPP */ if (can_request_ppp(qp, attr, attr_mask)) { req.comp_mask |= BNXT_RE_MQP_PPP_REQ_EN; req.dpi = qp->udpi->wcdpi; } if (attr_mask & IBV_QP_PATH_MTU) req.comp_mask |= BNXT_RE_MQP_PATH_MTU_MASK; } rc = ibv_cmd_modify_qp_compat(ibvqp, attr, attr_mask, can_issue_mqp_ex, &req, &resp); if (!rc) { if (attr_mask & IBV_QP_STATE) { qp->qpst = attr->qp_state; /* transition to reset */ if (qp->qpst == IBV_QPS_RESET) { qp->jsqq->hwque->head = 0; qp->jsqq->hwque->tail = 0; *qp->jsqq->hwque->dbtail = 0; qp->jsqq->start_idx = 0; qp->jsqq->last_idx = 0; bnxt_re_cleanup_cq(qp, qp->scq); if (qp->jrqq) { qp->jrqq->hwque->head = 0; qp->jrqq->hwque->tail = 0; *qp->jrqq->hwque->dbtail = 0; qp->jrqq->start_idx = 0; qp->jrqq->last_idx = 0; bnxt_re_cleanup_cq(qp, qp->rcq); } } /* Copy if PUSH was enabled */ if (resp.comp_mask & BNXT_RE_MQP_PPP_REQ_EN_MASK) { qp->push_st_en = BNXT_RE_MQP_PPP_REQ_EN; /* Set the next posting state * based on current h/w state */ qp->push_st_en |= !(!!(resp.ppp_st_idx & BNXT_RE_MQP_PPP_STATE)) << BNXT_RE_PPP_ST_SHIFT; qp->ppp_idx = (resp.ppp_st_idx & BNXT_RE_MQP_PPP_IDX_MASK); if (qp->qpmode == BNXT_RE_WQE_MODE_VARIABLE) qp->max_push_sz = BNXT_RE_MAX_PUSH_SIZE_VAR_WQE; else qp->max_push_sz = BNXT_RE_MAX_INLINE_SIZE; } } if (attr_mask & IBV_QP_SQ_PSN) qp->sq_psn = attr->sq_psn; if (resp.comp_mask & BNXT_RE_MQP_PATH_MTU_MASK) qp->mtu = resp.path_mtu; else if (attr_mask & IBV_QP_PATH_MTU) qp->mtu = (0x80 << attr->path_mtu); } return rc; } int bnxt_re_query_qp(struct ibv_qp *ibvqp, struct ibv_qp_attr *attr, int attr_mask, struct ibv_qp_init_attr *init_attr) { struct bnxt_re_qp *qp = to_bnxt_re_qp(ibvqp); struct ibv_query_qp cmd = {}; int rc; rc = ibv_cmd_query_qp(ibvqp, attr, attr_mask, init_attr, &cmd, sizeof(cmd)); if (!rc) qp->qpst = ibvqp->state; return rc; } int bnxt_re_destroy_qp(struct ibv_qp *ibvqp) { struct bnxt_re_qp *qp = to_bnxt_re_qp(ibvqp); struct bnxt_re_mem *mem; int status; qp->qpst = IBV_QPS_RESET; if (_is_db_drop_recovery_enable(qp->cntx)) { pthread_spin_lock(&qp->cntx->qp_dbr_res.lock); bnxt_re_list_del_node(&qp->dbnode, &qp->cntx->qp_dbr_res.head); pthread_spin_unlock(&qp->cntx->qp_dbr_res.lock); } status = ibv_cmd_destroy_qp(ibvqp); if (status) { if (_is_db_drop_recovery_enable(qp->cntx)) { pthread_spin_lock(&qp->cntx->qp_dbr_res.lock); bnxt_re_list_add_node(&qp->dbnode, &qp->cntx->qp_dbr_res.head); pthread_spin_unlock(&qp->cntx->qp_dbr_res.lock); } return status; } bnxt_re_cleanup_cq(qp, qp->rcq); bnxt_re_cleanup_cq(qp, qp->scq); mem = qp->mem; bnxt_re_free_mem(mem); return 0; } static void bnxt_re_put_rx_sge(struct bnxt_re_queue *que, uint32_t *idx, struct ibv_sge *sgl, int nsg) { struct bnxt_re_sge *sge; int indx; for (indx = 0; indx < nsg; indx++) { sge = bnxt_re_get_hwqe(que, (*idx)++); sge->pa = htole64(sgl[indx].addr); sge->lkey = htole32(sgl[indx].lkey); sge->length = htole32(sgl[indx].length); } } static int bnxt_re_put_tx_sge(struct bnxt_re_queue *que, uint32_t *idx, struct ibv_sge *sgl, int nsg) { struct bnxt_re_sge *sge; int indx; int len; len = 0; for (indx = 0; indx < nsg; indx++) { sge = bnxt_re_get_hwqe(que, (*idx)++); sge->pa = htole64(sgl[indx].addr); sge->lkey = htole32(sgl[indx].lkey); sge->length = htole32(sgl[indx].length); len += sgl[indx].length; } return len; } static inline int bnxt_re_calc_inline_len(struct ibv_send_wr *swr) { int illen, indx; illen = 0; for (indx = 0; indx < swr->num_sge; indx++) illen += swr->sg_list[indx].length; return get_aligned(illen, sizeof(struct bnxt_re_sge)); } static int bnxt_re_put_inline(struct bnxt_re_queue *que, uint32_t *idx, struct bnxt_re_push_buffer *pbuf, struct ibv_sge *sgl, uint32_t nsg, uint16_t max_ils) { int len, t_len, offt = 0; int t_cplen = 0, cplen; bool pull_dst = true; void *il_dst = NULL; void *il_src = NULL; int alsize; int indx; alsize = sizeof(struct bnxt_re_sge); t_len = 0; for (indx = 0; indx < nsg; indx++) { len = sgl[indx].length; il_src = (void *)sgl[indx].addr; t_len += len; if (t_len > max_ils) goto bad; while (len) { if (pull_dst) { pull_dst = false; il_dst = bnxt_re_get_hwqe(que, (*idx)++); if (pbuf) pbuf->wqe[*idx - 1] = (__u64)il_dst; t_cplen = 0; offt = 0; } cplen = MIN(len, alsize); cplen = MIN(cplen,(alsize - offt)); memcpy(il_dst, il_src, cplen); t_cplen += cplen; il_src += cplen; il_dst += cplen; offt += cplen; len -= cplen; if (t_cplen == alsize) pull_dst = true; } } return t_len; bad: return -ENOMEM; } static int bnxt_re_required_slots(struct bnxt_re_qp *qp, struct ibv_send_wr *wr, uint32_t *wqe_sz, void **pbuf) { uint32_t wqe_byte; int ilsize; if (wr->send_flags & IBV_SEND_INLINE) { ilsize = bnxt_re_calc_inline_len(wr); if (ilsize > qp->cap.max_inline) return -EINVAL; if (qp->push_st_en && ilsize <= qp->max_push_sz) *pbuf = bnxt_re_get_pbuf(&qp->push_st_en, qp->ppp_idx, qp->cntx); wqe_byte = (ilsize + bnxt_re_get_sqe_hdr_sz()); } else { wqe_byte = bnxt_re_calc_wqe_sz(wr->num_sge); } /* que->stride is always 2^4 = 16, thus using hard-coding */ *wqe_sz = wqe_byte >> 4; if (qp->qpmode == BNXT_RE_WQE_MODE_STATIC) return 8; return *wqe_sz; } static inline void bnxt_re_set_hdr_flags(struct bnxt_re_bsqe *hdr, struct ibv_send_wr *wr, uint32_t slots, uint8_t sqsig) { uint32_t send_flags; uint32_t hdrval = 0; uint8_t opcd; send_flags = wr->send_flags; if (send_flags & IBV_SEND_SIGNALED || sqsig) hdrval |= ((BNXT_RE_WR_FLAGS_SIGNALED & BNXT_RE_HDR_FLAGS_MASK) << BNXT_RE_HDR_FLAGS_SHIFT); if (send_flags & IBV_SEND_FENCE) hdrval |= ((BNXT_RE_WR_FLAGS_UC_FENCE & BNXT_RE_HDR_FLAGS_MASK) << BNXT_RE_HDR_FLAGS_SHIFT); if (send_flags & IBV_SEND_SOLICITED) hdrval |= ((BNXT_RE_WR_FLAGS_SE & BNXT_RE_HDR_FLAGS_MASK) << BNXT_RE_HDR_FLAGS_SHIFT); if (send_flags & IBV_SEND_INLINE) hdrval |= ((BNXT_RE_WR_FLAGS_INLINE & BNXT_RE_HDR_FLAGS_MASK) << BNXT_RE_HDR_FLAGS_SHIFT); hdrval |= (slots & BNXT_RE_HDR_WS_MASK) << BNXT_RE_HDR_WS_SHIFT; /* Fill opcode */ opcd = ibv_to_bnxt_re_wr_opcd[wr->opcode]; hdrval |= (opcd & BNXT_RE_HDR_WT_MASK); hdr->rsv_ws_fl_wt = htole32(hdrval); } static int bnxt_re_build_tx_sge(struct bnxt_re_queue *que, uint32_t *idx, struct bnxt_re_push_buffer *pbuf, struct ibv_send_wr *wr, uint16_t max_il) { if (wr->send_flags & IBV_SEND_INLINE) return bnxt_re_put_inline(que, idx, pbuf, wr->sg_list, wr->num_sge, max_il); return bnxt_re_put_tx_sge(que, idx, wr->sg_list, wr->num_sge); } static void *bnxt_re_pull_psn_buff(struct bnxt_re_queue *que, bool hw_retx) { if (hw_retx) return (void *)(que->pad + ((que->msn) << que->pad_stride_log2)); return (void *)(que->pad + ((*que->dbtail) << que->pad_stride_log2)); } static void bnxt_re_fill_psns_for_msntbl(struct bnxt_re_qp *qp, uint32_t len, uint32_t st_idx, uint8_t opcode) { uint32_t npsn = 0, start_psn = 0, next_psn = 0; struct bnxt_re_msns *msns; uint32_t pkt_cnt = 0; msns = bnxt_re_pull_psn_buff(qp->jsqq->hwque, true); msns->start_idx_next_psn_start_psn = 0; if (qp->qptyp == IBV_QPT_RC) { start_psn = qp->sq_psn; pkt_cnt = (len / qp->mtu); if (len % qp->mtu) pkt_cnt++; /* Increment the psn even for 0 len packets * e.g. for opcode rdma-write-with-imm-data * with length field = 0 */ if (bnxt_re_is_zero_len_pkt(len, opcode)) pkt_cnt = 1; /* make it 24 bit */ next_psn = qp->sq_psn + pkt_cnt; npsn = next_psn; qp->sq_psn = next_psn; msns->start_idx_next_psn_start_psn |= bnxt_re_update_msn_tbl(st_idx, npsn, start_psn); qp->jsqq->hwque->msn++; qp->jsqq->hwque->msn %= qp->jsqq->hwque->msn_tbl_sz; } } static void bnxt_re_fill_psns(struct bnxt_re_qp *qp, uint32_t len, uint32_t st_idx, uint8_t opcode) { uint32_t opc_spsn = 0, flg_npsn = 0; struct bnxt_re_psns_ext *psns_ext; uint32_t pkt_cnt = 0, nxt_psn = 0; struct bnxt_re_psns *psns; psns = bnxt_re_pull_psn_buff(qp->jsqq->hwque, false); psns_ext = (struct bnxt_re_psns_ext *)psns; if (qp->qptyp == IBV_QPT_RC) { opc_spsn = qp->sq_psn & BNXT_RE_PSNS_SPSN_MASK; pkt_cnt = (len / qp->mtu); if (len % qp->mtu) pkt_cnt++; /* Increment the psn even for 0 len packets * e.g. for opcode rdma-write-with-imm-data * with length field = 0 */ if (bnxt_re_is_zero_len_pkt(len, opcode)) pkt_cnt = 1; nxt_psn = ((qp->sq_psn + pkt_cnt) & BNXT_RE_PSNS_NPSN_MASK); flg_npsn = nxt_psn; qp->sq_psn = nxt_psn; } psns->opc_spsn = htole32(opc_spsn); psns->flg_npsn = htole32(flg_npsn); /* Update for Thor p5 not Thor2 */ if (!BNXT_RE_HW_RETX(qp->cntx) && qp->cctx->chip_is_gen_p5_thor2) psns_ext->st_slot_idx = st_idx; } static int bnxt_re_build_ud_sqe(struct ibv_send_wr *wr, struct bnxt_re_bsqe *hdr, struct bnxt_re_send *sqe) { struct bnxt_re_ah *ah; uint64_t qkey; ah = to_bnxt_re_ah(wr->wr.ud.ah); if (!wr->wr.ud.ah) return -EINVAL; qkey = wr->wr.ud.remote_qkey; hdr->lhdr.qkey_len |= htole64(qkey << 32); sqe->dst_qp = htole32(wr->wr.ud.remote_qpn); sqe->avid = htole32(ah->avid & 0xFFFFF); return 0; } static void bnxt_re_build_cns_sqe(struct ibv_send_wr *wr, struct bnxt_re_bsqe *hdr, void *hdr2) { struct bnxt_re_atomic *sqe = hdr2; hdr->key_immd = htole32(wr->wr.atomic.rkey); hdr->lhdr.rva = htole64(wr->wr.atomic.remote_addr); sqe->cmp_dt = htole64(wr->wr.atomic.compare_add); sqe->swp_dt = htole64(wr->wr.atomic.swap); } static void bnxt_re_build_fna_sqe(struct ibv_send_wr *wr, struct bnxt_re_bsqe *hdr, void *hdr2) { struct bnxt_re_atomic *sqe = hdr2; hdr->key_immd = htole32(wr->wr.atomic.rkey); hdr->lhdr.rva = htole64(wr->wr.atomic.remote_addr); sqe->swp_dt = htole64(wr->wr.atomic.compare_add); } void bnxt_re_force_rts2rts(struct bnxt_re_qp *qp) { struct ibv_qp_attr attr = {}; int attr_mask; attr_mask = IBV_QP_STATE; attr.qp_state = IBV_QPS_RTS; bnxt_re_modify_qp(&qp->ibvqp, &attr, attr_mask); qp->wqe_cnt = 0; } int bnxt_re_post_send(struct ibv_qp *ibvqp, struct ibv_send_wr *wr, struct ibv_send_wr **bad) { struct bnxt_re_qp *qp = to_bnxt_re_qp(ibvqp); struct bnxt_re_queue *sq = qp->jsqq->hwque; struct bnxt_re_push_buffer *pbuf = NULL; bool chip_is_not_gen_p5_thor2; int slots, ret = 0, len = 0; uint32_t swq_idx, wqe_size; struct bnxt_re_wrid *wrid; struct bnxt_re_rdma *rsqe; struct bnxt_re_bsqe *hdr; struct bnxt_re_send *sqe; bool ring_db = false; uint32_t idx; bnxt_re_dp_spin_lock(&sq->qlock); chip_is_not_gen_p5_thor2 = !qp->cctx->chip_is_gen_p5_thor2; while (wr) { slots = bnxt_re_required_slots(qp, wr, &wqe_size, (void **)&pbuf); if (unlikely(slots < 0 || bnxt_re_is_que_full(sq, slots)) || wr->num_sge > qp->cap.max_ssge) { *bad = wr; ret = ENOMEM; goto bad_wr; } if ((wr->opcode == IBV_WR_ATOMIC_CMP_AND_SWP || wr->opcode == IBV_WR_ATOMIC_FETCH_AND_ADD) && !qp->cap.is_atomic_cap) { *bad = wr; ret = EINVAL; goto bad_wr; } idx = 0; len = 0; hdr = bnxt_re_get_hwqe(sq, idx++); sqe = bnxt_re_get_hwqe(sq, idx++); /* populate push buffer */ if (pbuf) { pbuf->qpid = qp->qpid; pbuf->wqe[0] = (__u64)hdr; pbuf->wqe[1] = (__u64)sqe; pbuf->st_idx = *sq->dbtail; } if (wr->num_sge) { len = bnxt_re_build_tx_sge(sq, &idx, pbuf, wr, qp->cap.max_inline); if (unlikely(len < 0)) { ret = ENOMEM; *bad = wr; goto bad_wr; } } hdr->lhdr.qkey_len = htole32(len); bnxt_re_set_hdr_flags(hdr, wr, wqe_size, qp->cap.sqsig); switch (wr->opcode) { case IBV_WR_SEND_WITH_IMM: /* HW is swapping the immediate data before * sending it out on the wire. To workaround * this, swap the imm_data value as sent by * the application so that the value going out * on the wire is in big-endian format. */ hdr->key_immd = htole32(be32toh(wr->imm_data)); if (qp->qptyp == IBV_QPT_UD) { if (chip_is_not_gen_p5_thor2 && qp->wqe_cnt == BNXT_RE_UD_QP_STALL) bnxt_re_force_rts2rts(qp); len = bnxt_re_build_ud_sqe(wr, hdr, sqe); } break; case IBV_WR_SEND: if (qp->qptyp == IBV_QPT_UD) { if (chip_is_not_gen_p5_thor2 && qp->wqe_cnt == BNXT_RE_UD_QP_STALL) bnxt_re_force_rts2rts(qp); len = bnxt_re_build_ud_sqe(wr, hdr, sqe); } break; case IBV_WR_RDMA_WRITE_WITH_IMM: hdr->key_immd = htole32(be32toh(wr->imm_data)); case IBV_WR_RDMA_WRITE: case IBV_WR_RDMA_READ: rsqe = (struct bnxt_re_rdma *)sqe; rsqe->rva = htole64(wr->wr.rdma.remote_addr); rsqe->rkey = htole32(wr->wr.rdma.rkey); break; case IBV_WR_ATOMIC_CMP_AND_SWP: bnxt_re_build_cns_sqe(wr, hdr, sqe); break; case IBV_WR_ATOMIC_FETCH_AND_ADD: bnxt_re_build_fna_sqe(wr, hdr, sqe); break; default : len = -EINVAL; break; } if (unlikely(len < 0)) { ret = (len == -EINVAL) ? EINVAL : ENOMEM; *bad = wr; break; } if (BNXT_RE_HW_RETX(qp->cntx)) bnxt_re_fill_psns_for_msntbl(qp, len, *sq->dbtail, wr->opcode); else bnxt_re_fill_psns(qp, len, *sq->dbtail, wr->opcode); wrid = bnxt_re_get_swqe(qp->jsqq, &swq_idx); wrid->wrid = wr->wr_id; wrid->bytes = len; wrid->slots = slots; wrid->sig = (wr->send_flags & IBV_SEND_SIGNALED || qp->cap.sqsig) ? IBV_SEND_SIGNALED : 0; wrid->wc_opcd = ibv_wr_to_wc_opcd[wr->opcode]; bnxt_re_incr_tail(sq, slots); bnxt_re_jqq_mod_start(qp->jsqq, swq_idx); ring_db = true; if (pbuf) { ring_db = false; pbuf->tail = *sq->dbtail; if (_is_chip_thor2(qp->cctx)) { /* WA for SR2 A0, ring additional db */ ring_db |= _is_chip_a0(qp->cctx); bnxt_re_fill_ppp(pbuf, qp, len, idx); } else { bnxt_re_fill_push_wcb(qp, pbuf, idx); } bnxt_re_put_pbuf(qp->cntx, pbuf); pbuf = NULL; } qp->wqe_cnt++; qp->sq_msn++; wr = wr->next; } bad_wr: if (ring_db) bnxt_re_ring_sq_db(qp); if (pbuf) bnxt_re_put_pbuf(qp->cntx, pbuf); bnxt_re_dp_spin_unlock(&sq->qlock); return ret; } int bnxt_re_post_recv(struct ibv_qp *ibvqp, struct ibv_recv_wr *wr, struct ibv_recv_wr **bad) { struct bnxt_re_qp *qp = to_bnxt_re_qp(ibvqp); struct bnxt_re_queue *rq = qp->jrqq->hwque; struct bnxt_re_wrid *swque; struct bnxt_re_brqe *hdr; struct bnxt_re_sge *sge; bool ring_db = false; uint32_t swq_idx; uint32_t hdrval; uint32_t idx; int rc = 0; bnxt_re_dp_spin_lock(&rq->qlock); while (wr) { if (unlikely(bnxt_re_is_que_full(rq, rq->max_slots) || wr->num_sge > qp->cap.max_rsge)) { *bad = wr; rc = ENOMEM; break; } swque = bnxt_re_get_swqe(qp->jrqq, &swq_idx); /* * Initialize idx to 2 since the length of header wqe is 32 bytes * i.e. sizeof(struct bnxt_re_brqe) + sizeof(struct bnxt_re_send) */ idx = 2; hdr = bnxt_re_get_hwqe_hdr(rq); if (!wr->num_sge) { /* * HW needs at least one SGE for RQ Entries. * Create an entry if num_sge = 0, * update the idx and set length of sge to 0. */ sge = bnxt_re_get_hwqe(rq, idx++); sge->length = 0; } else { /* Fill SGEs */ bnxt_re_put_rx_sge(rq, &idx, wr->sg_list, wr->num_sge); } hdrval = BNXT_RE_WR_OPCD_RECV; hdrval |= ((idx & BNXT_RE_HDR_WS_MASK) << BNXT_RE_HDR_WS_SHIFT); hdr->rsv_ws_fl_wt = htole32(hdrval); hdr->wrid = htole32(swq_idx); swque->wrid = wr->wr_id; swque->slots = rq->max_slots; swque->wc_opcd = BNXT_RE_WC_OPCD_RECV; bnxt_re_jqq_mod_start(qp->jrqq, swq_idx); bnxt_re_incr_tail(rq, rq->max_slots); ring_db = true; wr = wr->next; } if (ring_db) bnxt_re_ring_rq_db(qp); bnxt_re_dp_spin_unlock(&rq->qlock); return rc; } static size_t bnxt_re_get_srqmem_size(struct bnxt_re_context *cntx, struct ibv_srq_init_attr *attr, struct bnxt_re_qattr *qattr) { uint32_t stride, nswr; size_t size = 0; size = sizeof(struct bnxt_re_srq); size += sizeof(struct bnxt_re_queue); /* allocate 1 extra to determin full condition */ nswr = attr->attr.max_wr + 1; nswr = bnxt_re_init_depth(nswr, cntx->comp_mask); stride = bnxt_re_get_srqe_sz(); qattr->nwr = nswr; qattr->slots = nswr; qattr->esize = stride; qattr->sz_ring = get_aligned((nswr * stride), cntx->rdev->pg_size); qattr->sz_shad = nswr * sizeof(struct bnxt_re_wrid); /* shadow */ size += qattr->sz_ring; size += qattr->sz_shad; return size; } static void *bnxt_re_alloc_srqslab(struct bnxt_re_context *cntx, struct ibv_srq_init_attr *attr, struct bnxt_re_qattr *qattr) { size_t bytes; bytes = bnxt_re_get_srqmem_size(cntx, attr, qattr); return bnxt_re_alloc_mem(bytes, cntx->rdev->pg_size); } static struct bnxt_re_srq *bnxt_re_srq_alloc_queue_ptr(struct bnxt_re_mem *mem) { struct bnxt_re_srq *srq; srq = bnxt_re_get_obj(mem, sizeof(*srq)); if (!srq) return NULL; srq->srqq = bnxt_re_get_obj(mem, sizeof(struct bnxt_re_queue)); if (!srq->srqq) return NULL; return srq; } static int bnxt_re_srq_alloc_queue(struct bnxt_re_srq *srq, struct ibv_srq_init_attr *attr, struct bnxt_re_qattr *qattr) { struct bnxt_re_queue *que; int ret = -ENOMEM; int idx; que = srq->srqq; que->depth = qattr->slots; que->stride = qattr->esize; que->va = bnxt_re_get_ring(srq->mem, qattr->sz_ring); if (!que->va) goto bail; bnxt_re_dp_spin_init(&que->qlock, PTHREAD_PROCESS_PRIVATE, !bnxt_single_threaded); /* For SRQ only bnxt_re_wrid.wrid is used. */ srq->srwrid = bnxt_re_get_obj(srq->mem, qattr->sz_shad); if (!srq->srwrid) goto bail; srq->start_idx = 0; srq->last_idx = que->depth - 1; for (idx = 0; idx < que->depth; idx++) srq->srwrid[idx].next_idx = idx + 1; srq->srwrid[srq->last_idx].next_idx = -1; return 0; bail: bnxt_re_dp_spin_destroy(&srq->srqq->qlock); return ret; } struct ibv_srq *bnxt_re_create_srq(struct ibv_pd *ibvpd, struct ibv_srq_init_attr *attr) { struct bnxt_re_srq_resp resp = {}; struct bnxt_re_srq_req cmd = {}; struct bnxt_re_qattr qattr = {}; struct bnxt_re_context *uctx; struct bnxt_re_srq *srq; void *mem; int ret; uctx = to_bnxt_re_context(ibvpd->context); mem = bnxt_re_alloc_srqslab(uctx, attr, &qattr); if (!mem) return NULL; srq = bnxt_re_srq_alloc_queue_ptr(mem); if (!srq) goto fail; srq->uctx = uctx; srq->mem = mem; if (bnxt_re_srq_alloc_queue(srq, attr, &qattr)) goto fail; cmd.srqva = (uint64_t)srq->srqq->va; cmd.srq_handle = (uint64_t)srq; ret = ibv_cmd_create_srq(ibvpd, &srq->ibvsrq, attr, &cmd.cmd, sizeof(cmd), &resp.resp, sizeof(resp)); if (ret) goto fail; srq->srqid = resp.srqid; + if (resp.srq_page) { + srq->srq_page = mmap(NULL, uctx->rdev->pg_size, PROT_READ, MAP_SHARED, + ibvpd->context->cmd_fd, resp.srq_page); + if (srq->srq_page == MAP_FAILED) + srq->srq_page = NULL; + } + srq->udpi = &uctx->udpi; srq->cap.max_wr = srq->srqq->depth; srq->cap.max_sge = attr->attr.max_sge; srq->cap.srq_limit = attr->attr.srq_limit; srq->arm_req = false; srq->rand.seed = srq->srqid; srq->shadow_db_key = BNXT_RE_DB_KEY_INVALID; INIT_DBLY_LIST_NODE(&srq->dbnode); if (_is_db_drop_recovery_enable(uctx)) { pthread_spin_lock(&uctx->srq_dbr_res.lock); bnxt_re_list_add_node(&srq->dbnode, &uctx->srq_dbr_res.head); pthread_spin_unlock(&uctx->srq_dbr_res.lock); } return &srq->ibvsrq; fail: bnxt_re_free_mem(mem); return NULL; } int bnxt_re_modify_srq(struct ibv_srq *ibvsrq, struct ibv_srq_attr *attr, int attr_mask) { struct bnxt_re_srq *srq = to_bnxt_re_srq(ibvsrq); struct ibv_modify_srq cmd = {}; int status = 0; status = ibv_cmd_modify_srq(ibvsrq, attr, attr_mask, &cmd, sizeof(cmd)); if (!status && ((attr_mask & IBV_SRQ_LIMIT) && (srq->cap.srq_limit != attr->srq_limit))) { srq->cap.srq_limit = attr->srq_limit; } srq->arm_req = true; return status; } int bnxt_re_destroy_srq(struct ibv_srq *ibvsrq) { struct bnxt_re_srq *srq = to_bnxt_re_srq(ibvsrq); struct bnxt_re_mem *mem; int ret; if (_is_db_drop_recovery_enable(srq->uctx)) { pthread_spin_lock(&srq->uctx->srq_dbr_res.lock); bnxt_re_list_del_node(&srq->dbnode, &srq->uctx->srq_dbr_res.head); pthread_spin_unlock(&srq->uctx->srq_dbr_res.lock); } + if (srq->srq_page) + munmap(srq->srq_page, srq->uctx->rdev->pg_size); ret = ibv_cmd_destroy_srq(ibvsrq); if (ret) { if (_is_db_drop_recovery_enable(srq->uctx)) { pthread_spin_lock(&srq->uctx->srq_dbr_res.lock); bnxt_re_list_add_node(&srq->dbnode, &srq->uctx->srq_dbr_res.head); pthread_spin_unlock(&srq->uctx->srq_dbr_res.lock); } return ret; } bnxt_re_dp_spin_destroy(&srq->srqq->qlock); mem = srq->mem; bnxt_re_free_mem(mem); return 0; } int bnxt_re_query_srq(struct ibv_srq *ibvsrq, struct ibv_srq_attr *attr) { struct ibv_query_srq cmd = {}; return ibv_cmd_query_srq(ibvsrq, attr, &cmd, sizeof cmd); } static int bnxt_re_build_srqe(struct bnxt_re_srq *srq, struct ibv_recv_wr *wr, void *srqe) { struct bnxt_re_brqe *hdr = srqe; struct bnxt_re_wrid *wrid; struct bnxt_re_sge *sge; int wqe_sz, len, next; uint32_t hdrval = 0; int indx; sge = (srqe + bnxt_re_get_srqe_hdr_sz()); next = srq->start_idx; wrid = &srq->srwrid[next]; len = 0; for (indx = 0; indx < wr->num_sge; indx++, sge++) { sge->pa = htole64(wr->sg_list[indx].addr); sge->lkey = htole32(wr->sg_list[indx].lkey); sge->length = htole32(wr->sg_list[indx].length); len += wr->sg_list[indx].length; } hdrval = BNXT_RE_WR_OPCD_RECV; wqe_sz = wr->num_sge + (bnxt_re_get_srqe_hdr_sz() >> 4); /* 16B align */ /* HW needs at least one SGE for SRQ Entries. * Increment SRQ WQE size if num_sge = 0 to * include the extra SGE. Set the sge length to * zero. */ if (!wr->num_sge) { wqe_sz++; sge->length = 0; } hdrval |= ((wqe_sz & BNXT_RE_HDR_WS_MASK) << BNXT_RE_HDR_WS_SHIFT); hdr->rsv_ws_fl_wt = htole32(hdrval); hdr->wrid = htole32((uint32_t)next); /* Fill wrid */ wrid->wrid = wr->wr_id; wrid->bytes = len; /* N.A. for RQE */ wrid->sig = 0; /* N.A. for RQE */ return len; } int bnxt_re_post_srq_recv(struct ibv_srq *ibvsrq, struct ibv_recv_wr *wr, struct ibv_recv_wr **bad) { struct bnxt_re_srq *srq = to_bnxt_re_srq(ibvsrq); struct bnxt_re_queue *rq = srq->srqq; int ret, count = 0; void *srqe; bnxt_re_dp_spin_lock(&rq->qlock); count = rq->tail > rq->head ? rq->tail - rq->head : rq->depth - rq->head + rq->tail; while (wr) { if (srq->start_idx == srq->last_idx || wr->num_sge > srq->cap.max_sge) { *bad = wr; bnxt_re_dp_spin_unlock(&rq->qlock); return ENOMEM; } srqe = (void *) (rq->va + (rq->tail * rq->stride)); memset(srqe, 0, bnxt_re_get_srqe_sz()); ret = bnxt_re_build_srqe(srq, wr, srqe); if (ret < 0) { bnxt_re_dp_spin_unlock(&rq->qlock); *bad = wr; return ENOMEM; } srq->start_idx = srq->srwrid[srq->start_idx].next_idx; bnxt_re_incr_tail(rq, 1); wr = wr->next; bnxt_re_ring_srq_db(srq); count++; if (srq->arm_req == true && count > srq->cap.srq_limit) { srq->arm_req = false; bnxt_re_ring_srq_arm(srq); } } bnxt_re_dp_spin_unlock(&rq->qlock); return 0; } struct ibv_ah *bnxt_re_create_ah(struct ibv_pd *ibvpd, struct ibv_ah_attr *attr) { struct bnxt_re_context *uctx; struct bnxt_re_pd *pd; struct bnxt_re_ah *ah; int status; struct ibv_create_ah_resp resp = {}; pd = to_bnxt_re_pd(ibvpd); uctx = to_bnxt_re_context(ibvpd->context); ah = calloc(1, sizeof(struct bnxt_re_ah)); if (!ah) { goto failed; } ah->pd = pd; pthread_mutex_lock(&uctx->shlock); status = ibv_cmd_create_ah(ibvpd, &ah->ibvah, attr, &resp, sizeof(resp)); if (status) { pthread_mutex_unlock(&uctx->shlock); free(ah); goto failed; } /* read AV ID now. */ ah->avid = *(uint32_t *)(uctx->shpg + BNXT_RE_SHPG_AVID_OFFT); pthread_mutex_unlock(&uctx->shlock); return &ah->ibvah; failed: return NULL; } int bnxt_re_destroy_ah(struct ibv_ah *ibvah) { struct bnxt_re_ah *ah; int status; ah = to_bnxt_re_ah(ibvah); status = ibv_cmd_destroy_ah(ibvah); if (status) return status; free(ah); return 0; } diff --git a/sys/dev/bnxt/bnxt_en/bnxt.h b/sys/dev/bnxt/bnxt_en/bnxt.h index 0ba7b5723b91..78e98eb23beb 100644 --- a/sys/dev/bnxt/bnxt_en/bnxt.h +++ b/sys/dev/bnxt/bnxt_en/bnxt.h @@ -1,1390 +1,1391 @@ /*- * Broadcom NetXtreme-C/E network driver. * * Copyright (c) 2016 Broadcom, All Rights Reserved. * The term Broadcom refers to Broadcom Limited and/or its subsidiaries * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS' * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF * THE POSSIBILITY OF SUCH DAMAGE. */ #include #ifndef _BNXT_H #define _BNXT_H #include #include #include #include #include #include #include #include #include #include #include #include "hsi_struct_def.h" #include "bnxt_dcb.h" #include "bnxt_auxbus_compat.h" #define DFLT_HWRM_CMD_TIMEOUT 500 /* PCI IDs */ #define BROADCOM_VENDOR_ID 0x14E4 #define BCM57301 0x16c8 #define BCM57302 0x16c9 #define BCM57304 0x16ca #define BCM57311 0x16ce #define BCM57312 0x16cf #define BCM57314 0x16df #define BCM57402 0x16d0 #define BCM57402_NPAR 0x16d4 #define BCM57404 0x16d1 #define BCM57404_NPAR 0x16e7 #define BCM57406 0x16d2 #define BCM57406_NPAR 0x16e8 #define BCM57407 0x16d5 #define BCM57407_NPAR 0x16ea #define BCM57407_SFP 0x16e9 #define BCM57412 0x16d6 #define BCM57412_NPAR1 0x16de #define BCM57412_NPAR2 0x16eb #define BCM57414 0x16d7 #define BCM57414_NPAR1 0x16ec #define BCM57414_NPAR2 0x16ed #define BCM57416 0x16d8 #define BCM57416_NPAR1 0x16ee #define BCM57416_NPAR2 0x16ef #define BCM57416_SFP 0x16e3 #define BCM57417 0x16d9 #define BCM57417_NPAR1 0x16c0 #define BCM57417_NPAR2 0x16cc #define BCM57417_SFP 0x16e2 #define BCM57454 0x1614 #define BCM58700 0x16cd #define BCM57508 0x1750 #define BCM57504 0x1751 #define BCM57504_NPAR 0x1801 #define BCM57502 0x1752 #define BCM57608 0x1760 #define BCM57604 0x1761 #define BCM57602 0x1762 #define BCM57601 0x1763 #define NETXTREME_C_VF1 0x16cb #define NETXTREME_C_VF2 0x16e1 #define NETXTREME_C_VF3 0x16e5 #define NETXTREME_E_VF1 0x16c1 #define NETXTREME_E_VF2 0x16d3 #define NETXTREME_E_VF3 0x16dc #define EVENT_DATA1_RESET_NOTIFY_FATAL(data1) \ (((data1) & \ HWRM_ASYNC_EVENT_CMPL_RESET_NOTIFY_EVENT_DATA1_REASON_CODE_MASK) ==\ HWRM_ASYNC_EVENT_CMPL_RESET_NOTIFY_EVENT_DATA1_REASON_CODE_FW_EXCEPTION_FATAL) #define BNXT_EVENT_ERROR_REPORT_TYPE(data1) \ (((data1) & \ HWRM_ASYNC_EVENT_CMPL_ERROR_REPORT_BASE_EVENT_DATA1_ERROR_TYPE_MASK) >> \ HWRM_ASYNC_EVENT_CMPL_ERROR_REPORT_BASE_EVENT_DATA1_ERROR_TYPE_SFT) #define BNXT_EVENT_INVALID_SIGNAL_DATA(data2) \ (((data2) & \ HWRM_ASYNC_EVENT_CMPL_ERROR_REPORT_INVALID_SIGNAL_EVENT_DATA2_PIN_ID_MASK) >> \ HWRM_ASYNC_EVENT_CMPL_ERROR_REPORT_INVALID_SIGNAL_EVENT_DATA2_PIN_ID_SFT) #define BNXT_EVENT_DBR_EPOCH(data) \ (((data) & HWRM_ASYNC_EVENT_CMPL_ERROR_REPORT_DOORBELL_DROP_THRESHOLD_EVENT_DATA1_EPOCH_MASK) >> \ HWRM_ASYNC_EVENT_CMPL_ERROR_REPORT_DOORBELL_DROP_THRESHOLD_EVENT_DATA1_EPOCH_SFT) #define BNXT_EVENT_THERMAL_THRESHOLD_TEMP(data2) \ (((data2) & \ HWRM_ASYNC_EVENT_CMPL_ERROR_REPORT_THERMAL_EVENT_DATA2_THRESHOLD_TEMP_MASK) >> \ HWRM_ASYNC_EVENT_CMPL_ERROR_REPORT_THERMAL_EVENT_DATA2_THRESHOLD_TEMP_SFT) #define EVENT_DATA2_NVM_ERR_ADDR(data2) \ (((data2) & \ HWRM_ASYNC_EVENT_CMPL_ERROR_REPORT_NVM_EVENT_DATA2_ERR_ADDR_MASK) >> \ HWRM_ASYNC_EVENT_CMPL_ERROR_REPORT_NVM_EVENT_DATA2_ERR_ADDR_SFT) #define EVENT_DATA1_THERMAL_THRESHOLD_DIR_INCREASING(data1) \ (((data1) & \ HWRM_ASYNC_EVENT_CMPL_ERROR_REPORT_THERMAL_EVENT_DATA1_TRANSITION_DIR) == \ HWRM_ASYNC_EVENT_CMPL_ERROR_REPORT_THERMAL_EVENT_DATA1_TRANSITION_DIR_INCREASING) #define EVENT_DATA1_NVM_ERR_TYPE_WRITE(data1) \ (((data1) & \ HWRM_ASYNC_EVENT_CMPL_ERROR_REPORT_NVM_EVENT_DATA1_NVM_ERR_TYPE_MASK) == \ HWRM_ASYNC_EVENT_CMPL_ERROR_REPORT_NVM_EVENT_DATA1_NVM_ERR_TYPE_WRITE) #define EVENT_DATA1_NVM_ERR_TYPE_ERASE(data1) \ (((data1) & \ HWRM_ASYNC_EVENT_CMPL_ERROR_REPORT_NVM_EVENT_DATA1_NVM_ERR_TYPE_MASK) == \ HWRM_ASYNC_EVENT_CMPL_ERROR_REPORT_NVM_EVENT_DATA1_NVM_ERR_TYPE_ERASE) #define EVENT_DATA1_THERMAL_THRESHOLD_TYPE(data1) \ ((data1) & HWRM_ASYNC_EVENT_CMPL_ERROR_REPORT_THERMAL_EVENT_DATA1_THRESHOLD_TYPE_MASK) #define BNXT_EVENT_THERMAL_CURRENT_TEMP(data2) \ ((data2) & HWRM_ASYNC_EVENT_CMPL_ERROR_REPORT_THERMAL_EVENT_DATA2_CURRENT_TEMP_MASK) #define EVENT_DATA1_RESET_NOTIFY_FW_ACTIVATION(data1) \ (((data1) & \ HWRM_ASYNC_EVENT_CMPL_RESET_NOTIFY_EVENT_DATA1_REASON_CODE_MASK) ==\ HWRM_ASYNC_EVENT_CMPL_RESET_NOTIFY_EVENT_DATA1_REASON_CODE_FW_ACTIVATION) #define EVENT_DATA2_RESET_NOTIFY_FW_STATUS_CODE(data2) \ ((data2) & \ HWRM_ASYNC_EVENT_CMPL_RESET_NOTIFY_EVENT_DATA2_FW_STATUS_CODE_MASK) #define EVENT_DATA1_RECOVERY_ENABLED(data1) \ !!((data1) & \ HWRM_ASYNC_EVENT_CMPL_ERROR_RECOVERY_EVENT_DATA1_FLAGS_RECOVERY_ENABLED) #define EVENT_DATA1_RECOVERY_MASTER_FUNC(data1) \ !!((data1) & \ HWRM_ASYNC_EVENT_CMPL_ERROR_RECOVERY_EVENT_DATA1_FLAGS_MASTER_FUNC) #define INVALID_STATS_CTX_ID -1 /* Maximum numbers of RX and TX descriptors. iflib requires this to be a power * of two. The hardware has no particular limitation. */ #define BNXT_MAX_RXD ((INT32_MAX >> 1) + 1) #define BNXT_MAX_TXD ((INT32_MAX >> 1) + 1) #define CSUM_OFFLOAD (CSUM_IP_TSO|CSUM_IP6_TSO|CSUM_IP| \ CSUM_IP_UDP|CSUM_IP_TCP|CSUM_IP_SCTP| \ CSUM_IP6_UDP|CSUM_IP6_TCP|CSUM_IP6_SCTP) #define BNXT_MAX_MTU 9600 #define BNXT_RSS_HASH_TYPE_TCPV4 0 #define BNXT_RSS_HASH_TYPE_UDPV4 1 #define BNXT_RSS_HASH_TYPE_IPV4 2 #define BNXT_RSS_HASH_TYPE_TCPV6 3 #define BNXT_RSS_HASH_TYPE_UDPV6 4 #define BNXT_RSS_HASH_TYPE_IPV6 5 #define BNXT_GET_RSS_PROFILE_ID(rss_hash_type) ((rss_hash_type >> 1) & 0x1F) #define BNXT_NO_MORE_WOL_FILTERS 0xFFFF #define bnxt_wol_supported(softc) (!((softc)->flags & BNXT_FLAG_VF) && \ ((softc)->flags & BNXT_FLAG_WOL_CAP )) /* 64-bit doorbell */ #define DBR_INDEX_MASK 0x0000000000ffffffULL #define DBR_PI_LO_MASK 0xff000000UL #define DBR_PI_LO_SFT 24 #define DBR_EPOCH_MASK 0x01000000UL #define DBR_EPOCH_SFT 24 #define DBR_TOGGLE_MASK 0x06000000UL #define DBR_TOGGLE_SFT 25 #define DBR_XID_MASK 0x000fffff00000000ULL #define DBR_XID_SFT 32 #define DBR_PI_HI_MASK 0xf0000000000000ULL #define DBR_PI_HI_SFT 52 #define DBR_PATH_L2 (0x1ULL << 56) #define DBR_VALID (0x1ULL << 58) #define DBR_TYPE_SQ (0x0ULL << 60) #define DBR_TYPE_RQ (0x1ULL << 60) #define DBR_TYPE_SRQ (0x2ULL << 60) #define DBR_TYPE_SRQ_ARM (0x3ULL << 60) #define DBR_TYPE_CQ (0x4ULL << 60) #define DBR_TYPE_CQ_ARMSE (0x5ULL << 60) #define DBR_TYPE_CQ_ARMALL (0x6ULL << 60) #define DBR_TYPE_CQ_ARMENA (0x7ULL << 60) #define DBR_TYPE_SRQ_ARMENA (0x8ULL << 60) #define DBR_TYPE_CQ_CUTOFF_ACK (0x9ULL << 60) #define DBR_TYPE_NQ (0xaULL << 60) #define DBR_TYPE_NQ_ARM (0xbULL << 60) #define DBR_TYPE_PUSH_START (0xcULL << 60) #define DBR_TYPE_PUSH_END (0xdULL << 60) #define DBR_TYPE_NQ_MASK (0xeULL << 60) #define DBR_TYPE_NULL (0xfULL << 60) #define BNXT_MAX_L2_QUEUES 128 #define BNXT_ROCE_IRQ_COUNT 9 #define BNXT_MAX_NUM_QUEUES (BNXT_MAX_L2_QUEUES + BNXT_ROCE_IRQ_COUNT) /* Completion related defines */ #define CMP_VALID(cmp, v_bit) \ ((!!(((struct cmpl_base *)(cmp))->info3_v & htole32(CMPL_BASE_V))) == !!(v_bit) ) /* Chip class phase 5 */ #define BNXT_CHIP_P5(sc) ((sc->flags & BNXT_FLAG_CHIP_P5)) /* Chip class phase 7 */ #define BNXT_CHIP_P7(sc) ((sc->flags & BNXT_FLAG_CHIP_P7)) /* Chip class phase 5 plus */ #define BNXT_CHIP_P5_PLUS(sc) \ (BNXT_CHIP_P5(sc) || BNXT_CHIP_P7(sc)) #define DB_PF_OFFSET_P5 0x10000 #define DB_VF_OFFSET_P5 0x4000 #define NQ_VALID(cmp, v_bit) \ ((!!(((nq_cn_t *)(cmp))->v & htole32(NQ_CN_V))) == !!(v_bit) ) #ifndef DIV_ROUND_UP #define DIV_ROUND_UP(n, d) (((n) + (d) - 1) / (d)) #endif #ifndef roundup #define roundup(x, y) ((((x) + ((y) - 1)) / (y)) * (y)) #endif #define NEXT_CP_CONS_V(ring, cons, v_bit) do { \ if (__predict_false(++(cons) == (ring)->ring_size)) \ ((cons) = 0, (v_bit) = !v_bit); \ } while (0) #define RING_NEXT(ring, idx) (__predict_false(idx + 1 == (ring)->ring_size) ? \ 0 : idx + 1) #define CMPL_PREFETCH_NEXT(cpr, idx) \ __builtin_prefetch(&((struct cmpl_base *)(cpr)->ring.vaddr)[((idx) +\ (CACHE_LINE_SIZE / sizeof(struct cmpl_base))) & \ ((cpr)->ring.ring_size - 1)]) /* Lock macros */ #define BNXT_HWRM_LOCK_INIT(_softc, _name) \ mtx_init(&(_softc)->hwrm_lock, _name, "BNXT HWRM Lock", MTX_DEF) #define BNXT_HWRM_LOCK(_softc) mtx_lock(&(_softc)->hwrm_lock) #define BNXT_HWRM_UNLOCK(_softc) mtx_unlock(&(_softc)->hwrm_lock) #define BNXT_HWRM_LOCK_DESTROY(_softc) mtx_destroy(&(_softc)->hwrm_lock) #define BNXT_HWRM_LOCK_ASSERT(_softc) mtx_assert(&(_softc)->hwrm_lock, \ MA_OWNED) #define BNXT_IS_FLOW_CTRL_CHANGED(link_info) \ ((link_info->last_flow_ctrl.tx != link_info->flow_ctrl.tx) || \ (link_info->last_flow_ctrl.rx != link_info->flow_ctrl.rx) || \ (link_info->last_flow_ctrl.autoneg != link_info->flow_ctrl.autoneg)) /* Chip info */ #define BNXT_TSO_SIZE UINT16_MAX #define min_t(type, x, y) ({ \ type __min1 = (x); \ type __min2 = (y); \ __min1 < __min2 ? __min1 : __min2; }) #define max_t(type, x, y) ({ \ type __max1 = (x); \ type __max2 = (y); \ __max1 > __max2 ? __max1 : __max2; }) #define clamp_t(type, _x, min, max) min_t(type, max_t(type, _x, min), max) #define BNXT_IFMEDIA_ADD(supported, fw_speed, ifm_speed) do { \ if ((supported) & HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_ ## fw_speed) \ ifmedia_add(softc->media, IFM_ETHER | (ifm_speed), 0, NULL); \ } while(0) #define BNXT_MIN_FRAME_SIZE 52 /* Frames must be padded to this size for some A0 chips */ #define BNXT_RX_STATS_EXT_OFFSET(counter) \ (offsetof(struct rx_port_stats_ext, counter) / 8) #define BNXT_RX_STATS_EXT_NUM_LEGACY \ BNXT_RX_STATS_EXT_OFFSET(rx_fec_corrected_blocks) #define BNXT_TX_STATS_EXT_OFFSET(counter) \ (offsetof(struct tx_port_stats_ext, counter) / 8) extern const char bnxt_driver_version[]; typedef void (*bnxt_doorbell_tx)(void *, uint16_t idx); typedef void (*bnxt_doorbell_rx)(void *, uint16_t idx); typedef void (*bnxt_doorbell_rx_cq)(void *, bool); typedef void (*bnxt_doorbell_tx_cq)(void *, bool); typedef void (*bnxt_doorbell_nq)(void *, bool); typedef struct bnxt_doorbell_ops { bnxt_doorbell_tx bnxt_db_tx; bnxt_doorbell_rx bnxt_db_rx; bnxt_doorbell_rx_cq bnxt_db_rx_cq; bnxt_doorbell_tx_cq bnxt_db_tx_cq; bnxt_doorbell_nq bnxt_db_nq; } bnxt_dooorbell_ops_t; /* NVRAM access */ enum bnxt_nvm_directory_type { BNX_DIR_TYPE_UNUSED = 0, BNX_DIR_TYPE_PKG_LOG = 1, BNX_DIR_TYPE_UPDATE = 2, BNX_DIR_TYPE_CHIMP_PATCH = 3, BNX_DIR_TYPE_BOOTCODE = 4, BNX_DIR_TYPE_VPD = 5, BNX_DIR_TYPE_EXP_ROM_MBA = 6, BNX_DIR_TYPE_AVS = 7, BNX_DIR_TYPE_PCIE = 8, BNX_DIR_TYPE_PORT_MACRO = 9, BNX_DIR_TYPE_APE_FW = 10, BNX_DIR_TYPE_APE_PATCH = 11, BNX_DIR_TYPE_KONG_FW = 12, BNX_DIR_TYPE_KONG_PATCH = 13, BNX_DIR_TYPE_BONO_FW = 14, BNX_DIR_TYPE_BONO_PATCH = 15, BNX_DIR_TYPE_TANG_FW = 16, BNX_DIR_TYPE_TANG_PATCH = 17, BNX_DIR_TYPE_BOOTCODE_2 = 18, BNX_DIR_TYPE_CCM = 19, BNX_DIR_TYPE_PCI_CFG = 20, BNX_DIR_TYPE_TSCF_UCODE = 21, BNX_DIR_TYPE_ISCSI_BOOT = 22, BNX_DIR_TYPE_ISCSI_BOOT_IPV6 = 24, BNX_DIR_TYPE_ISCSI_BOOT_IPV4N6 = 25, BNX_DIR_TYPE_ISCSI_BOOT_CFG6 = 26, BNX_DIR_TYPE_EXT_PHY = 27, BNX_DIR_TYPE_SHARED_CFG = 40, BNX_DIR_TYPE_PORT_CFG = 41, BNX_DIR_TYPE_FUNC_CFG = 42, BNX_DIR_TYPE_MGMT_CFG = 48, BNX_DIR_TYPE_MGMT_DATA = 49, BNX_DIR_TYPE_MGMT_WEB_DATA = 50, BNX_DIR_TYPE_MGMT_WEB_META = 51, BNX_DIR_TYPE_MGMT_EVENT_LOG = 52, BNX_DIR_TYPE_MGMT_AUDIT_LOG = 53 }; enum bnxnvm_pkglog_field_index { BNX_PKG_LOG_FIELD_IDX_INSTALLED_TIMESTAMP = 0, BNX_PKG_LOG_FIELD_IDX_PKG_DESCRIPTION = 1, BNX_PKG_LOG_FIELD_IDX_PKG_VERSION = 2, BNX_PKG_LOG_FIELD_IDX_PKG_TIMESTAMP = 3, BNX_PKG_LOG_FIELD_IDX_PKG_CHECKSUM = 4, BNX_PKG_LOG_FIELD_IDX_INSTALLED_ITEMS = 5, BNX_PKG_LOG_FIELD_IDX_INSTALLED_MASK = 6 }; #define BNX_DIR_ORDINAL_FIRST 0 #define BNX_DIR_EXT_NONE 0 struct bnxt_bar_info { struct resource *res; bus_space_tag_t tag; bus_space_handle_t handle; bus_size_t size; int rid; }; struct bnxt_flow_ctrl { bool rx; bool tx; bool autoneg; }; struct bnxt_link_info { uint8_t media_type; uint8_t transceiver; uint8_t phy_addr; uint8_t phy_link_status; uint8_t wire_speed; uint8_t loop_back; uint8_t link_up; uint8_t last_link_up; uint8_t duplex; uint8_t last_duplex; uint8_t last_phy_type; struct bnxt_flow_ctrl flow_ctrl; struct bnxt_flow_ctrl last_flow_ctrl; uint8_t duplex_setting; uint8_t auto_mode; #define PHY_VER_LEN 3 uint8_t phy_ver[PHY_VER_LEN]; uint8_t phy_type; #define BNXT_PHY_STATE_ENABLED 0 #define BNXT_PHY_STATE_DISABLED 1 uint8_t phy_state; uint16_t link_speed; uint16_t support_speeds; uint16_t support_speeds2; uint16_t support_pam4_speeds; uint16_t auto_link_speeds; uint16_t auto_link_speeds2; uint16_t auto_pam4_link_speeds; uint16_t force_link_speed; uint16_t force_link_speeds2; uint16_t force_pam4_link_speed; bool force_pam4_speed; bool force_speed2_nrz; bool force_pam4_56_speed2; bool force_pam4_112_speed2; uint16_t advertising; uint16_t advertising_pam4; uint32_t preemphasis; uint16_t support_auto_speeds; uint16_t support_force_speeds; uint16_t support_pam4_auto_speeds; uint16_t support_pam4_force_speeds; uint16_t support_auto_speeds2; uint16_t support_force_speeds2; #define BNXT_SIG_MODE_NRZ HWRM_PORT_PHY_QCFG_OUTPUT_SIGNAL_MODE_NRZ #define BNXT_SIG_MODE_PAM4 HWRM_PORT_PHY_QCFG_OUTPUT_SIGNAL_MODE_PAM4 #define BNXT_SIG_MODE_PAM4_112 HWRM_PORT_PHY_QCFG_OUTPUT_SIGNAL_MODE_PAM4_112 uint8_t req_signal_mode; uint8_t active_fec_sig_mode; uint8_t sig_mode; /* copy of requested setting */ uint8_t autoneg; #define BNXT_AUTONEG_SPEED 1 #define BNXT_AUTONEG_FLOW_CTRL 2 uint8_t req_duplex; uint16_t req_link_speed; uint8_t module_status; struct hwrm_port_phy_qcfg_output phy_qcfg_resp; }; enum bnxt_phy_type { BNXT_MEDIA_CR = 0, BNXT_MEDIA_LR, BNXT_MEDIA_SR, BNXT_MEDIA_ER, BNXT_MEDIA_KR, BNXT_MEDIA_AC, BNXT_MEDIA_BASECX, BNXT_MEDIA_BASET, BNXT_MEDIA_BASEKX, BNXT_MEDIA_BASESGMII, BNXT_MEDIA_END }; enum bnxt_cp_type { BNXT_DEFAULT, BNXT_TX, BNXT_RX, BNXT_SHARED }; struct bnxt_queue_info { uint8_t queue_id; uint8_t queue_profile; }; struct bnxt_func_info { uint32_t fw_fid; uint8_t mac_addr[ETHER_ADDR_LEN]; uint16_t max_rsscos_ctxs; uint16_t max_cp_rings; uint16_t max_tx_rings; uint16_t max_rx_rings; uint16_t max_hw_ring_grps; uint16_t max_irqs; uint16_t max_l2_ctxs; uint16_t max_vnics; uint16_t max_stat_ctxs; }; struct bnxt_pf_info { #define BNXT_FIRST_PF_FID 1 #define BNXT_FIRST_VF_FID 128 uint8_t port_id; uint32_t first_vf_id; uint16_t active_vfs; uint16_t max_vfs; uint32_t max_encap_records; uint32_t max_decap_records; uint32_t max_tx_em_flows; uint32_t max_tx_wm_flows; uint32_t max_rx_em_flows; uint32_t max_rx_wm_flows; unsigned long *vf_event_bmap; uint16_t hwrm_cmd_req_pages; void *hwrm_cmd_req_addr[4]; bus_addr_t hwrm_cmd_req_dma_addr[4]; }; struct bnxt_vf_info { uint16_t fw_fid; uint8_t mac_addr[ETHER_ADDR_LEN]; uint16_t max_rsscos_ctxs; uint16_t max_cp_rings; uint16_t max_tx_rings; uint16_t max_rx_rings; uint16_t max_hw_ring_grps; uint16_t max_l2_ctxs; uint16_t max_irqs; uint16_t max_vnics; uint16_t max_stat_ctxs; uint32_t vlan; #define BNXT_VF_QOS 0x1 #define BNXT_VF_SPOOFCHK 0x2 #define BNXT_VF_LINK_FORCED 0x4 #define BNXT_VF_LINK_UP 0x8 uint32_t flags; uint32_t func_flags; /* func cfg flags */ uint32_t min_tx_rate; uint32_t max_tx_rate; void *hwrm_cmd_req_addr; bus_addr_t hwrm_cmd_req_dma_addr; }; #define BNXT_PF(softc) (!((softc)->flags & BNXT_FLAG_VF)) #define BNXT_VF(softc) ((softc)->flags & BNXT_FLAG_VF) struct bnxt_vlan_tag { SLIST_ENTRY(bnxt_vlan_tag) next; uint64_t filter_id; uint16_t tag; }; struct bnxt_vnic_info { uint16_t id; uint16_t def_ring_grp; uint16_t cos_rule; uint16_t lb_rule; uint16_t mru; uint32_t rx_mask; struct iflib_dma_info mc_list; int mc_list_count; #define BNXT_MAX_MC_ADDRS 16 uint32_t flags; #define BNXT_VNIC_FLAG_DEFAULT 0x01 #define BNXT_VNIC_FLAG_BD_STALL 0x02 #define BNXT_VNIC_FLAG_VLAN_STRIP 0x04 uint64_t filter_id; uint16_t rss_id; uint32_t rss_hash_type; uint8_t rss_hash_key[HW_HASH_KEY_SIZE]; struct iflib_dma_info rss_hash_key_tbl; struct iflib_dma_info rss_grp_tbl; SLIST_HEAD(vlan_head, bnxt_vlan_tag) vlan_tags; struct iflib_dma_info vlan_tag_list; }; struct bnxt_grp_info { uint16_t stats_ctx; uint16_t grp_id; uint16_t rx_ring_id; uint16_t cp_ring_id; uint16_t ag_ring_id; }; #define EPOCH_ARR_SZ 4096 struct bnxt_ring { uint64_t paddr; vm_offset_t doorbell; caddr_t vaddr; struct bnxt_softc *softc; uint32_t ring_size; /* Must be a power of two */ uint16_t id; /* Logical ID */ uint16_t phys_id; uint16_t idx; struct bnxt_full_tpa_start *tpa_start; union { u64 db_key64; u32 db_key32; }; uint32_t db_ring_mask; uint32_t db_epoch_mask; uint8_t db_epoch_shift; uint64_t epoch_arr[EPOCH_ARR_SZ]; bool epoch_bit; }; struct bnxt_cp_ring { struct bnxt_ring ring; struct if_irq irq; uint32_t cons; uint32_t raw_cons; bool v_bit; /* Value of valid bit */ struct ctx_hw_stats *stats; uint32_t stats_ctx_id; uint32_t last_idx; /* Used by RX rings only * set to the last read pidx */ uint64_t int_count; uint8_t toggle; uint8_t type; #define Q_TYPE_TX 1 #define Q_TYPE_RX 2 }; struct bnxt_full_tpa_start { struct rx_tpa_start_cmpl low; struct rx_tpa_start_cmpl_hi high; }; /* All the version information for the part */ #define BNXT_VERSTR_SIZE (3*3+2+1) /* ie: "255.255.255\0" */ #define BNXT_NAME_SIZE 17 #define FW_VER_STR_LEN 32 #define BC_HWRM_STR_LEN 21 struct bnxt_ver_info { uint8_t hwrm_if_major; uint8_t hwrm_if_minor; uint8_t hwrm_if_update; char hwrm_if_ver[BNXT_VERSTR_SIZE]; char driver_hwrm_if_ver[BNXT_VERSTR_SIZE]; char mgmt_fw_ver[FW_VER_STR_LEN]; char netctrl_fw_ver[FW_VER_STR_LEN]; char roce_fw_ver[FW_VER_STR_LEN]; char fw_ver_str[FW_VER_STR_LEN]; char phy_ver[BNXT_VERSTR_SIZE]; char pkg_ver[64]; char hwrm_fw_name[BNXT_NAME_SIZE]; char mgmt_fw_name[BNXT_NAME_SIZE]; char netctrl_fw_name[BNXT_NAME_SIZE]; char roce_fw_name[BNXT_NAME_SIZE]; char phy_vendor[BNXT_NAME_SIZE]; char phy_partnumber[BNXT_NAME_SIZE]; uint16_t chip_num; uint8_t chip_rev; uint8_t chip_metal; uint8_t chip_bond_id; uint8_t chip_type; uint8_t hwrm_min_major; uint8_t hwrm_min_minor; uint8_t hwrm_min_update; uint64_t fw_ver_code; #define BNXT_FW_VER_CODE(maj, min, bld, rsv) \ ((uint64_t)(maj) << 48 | (uint64_t)(min) << 32 | (uint64_t)(bld) << 16 | (rsv)) #define BNXT_FW_MAJ(softc) ((softc)->ver_info->fw_ver_code >> 48) #define BNXT_FW_MIN(softc) (((softc)->ver_info->fw_ver_code >> 32) & 0xffff) #define BNXT_FW_BLD(softc) (((softc)->ver_info->fw_ver_code >> 16) & 0xffff) #define BNXT_FW_RSV(softc) (((softc)->ver_info->fw_ver_code) & 0xffff) struct sysctl_ctx_list ver_ctx; struct sysctl_oid *ver_oid; }; struct bnxt_nvram_info { uint16_t mfg_id; uint16_t device_id; uint32_t sector_size; uint32_t size; uint32_t reserved_size; uint32_t available_size; struct sysctl_ctx_list nvm_ctx; struct sysctl_oid *nvm_oid; }; struct bnxt_func_qcfg { uint16_t alloc_completion_rings; uint16_t alloc_tx_rings; uint16_t alloc_rx_rings; uint16_t alloc_vnics; }; struct bnxt_hw_lro { uint16_t enable; uint16_t is_mode_gro; uint16_t max_agg_segs; uint16_t max_aggs; uint32_t min_agg_len; }; /* The hardware supports certain page sizes. Use the supported page sizes * to allocate the rings. */ #if (PAGE_SHIFT < 12) #define BNXT_PAGE_SHIFT 12 #elif (PAGE_SHIFT <= 13) #define BNXT_PAGE_SHIFT PAGE_SHIFT #elif (PAGE_SHIFT < 16) #define BNXT_PAGE_SHIFT 13 #else #define BNXT_PAGE_SHIFT 16 #endif #define BNXT_PAGE_SIZE (1 << BNXT_PAGE_SHIFT) #define MAX_CTX_PAGES (BNXT_PAGE_SIZE / 8) #define MAX_CTX_TOTAL_PAGES (MAX_CTX_PAGES * MAX_CTX_PAGES) struct bnxt_ring_mem_info { int nr_pages; int page_size; uint16_t flags; #define BNXT_RMEM_VALID_PTE_FLAG 1 #define BNXT_RMEM_RING_PTE_FLAG 2 #define BNXT_RMEM_USE_FULL_PAGE_FLAG 4 uint16_t depth; struct bnxt_ctx_mem_type *ctx_mem; struct iflib_dma_info *pg_arr; struct iflib_dma_info pg_tbl; int vmem_size; void **vmem; }; struct bnxt_ctx_pg_info { uint32_t entries; uint32_t nr_pages; struct iflib_dma_info ctx_arr[MAX_CTX_PAGES]; struct bnxt_ring_mem_info ring_mem; struct bnxt_ctx_pg_info **ctx_pg_tbl; }; #define BNXT_MAX_TQM_SP_RINGS 1 #define BNXT_MAX_TQM_FP_LEGACY_RINGS 8 #define BNXT_MAX_TQM_FP_RINGS 9 #define BNXT_MAX_TQM_LEGACY_RINGS \ (BNXT_MAX_TQM_SP_RINGS + BNXT_MAX_TQM_FP_LEGACY_RINGS) #define BNXT_MAX_TQM_RINGS \ (BNXT_MAX_TQM_SP_RINGS + BNXT_MAX_TQM_FP_RINGS) #define BNXT_BACKING_STORE_CFG_LEGACY_LEN 256 #define BNXT_BACKING_STORE_CFG_LEN \ sizeof(struct hwrm_func_backing_store_cfg_input) #define BNXT_SET_CTX_PAGE_ATTR(attr) \ do { \ if (BNXT_PAGE_SIZE == 0x2000) \ attr = HWRM_FUNC_BACKING_STORE_CFG_INPUT_SRQ_PG_SIZE_PG_8K; \ else if (BNXT_PAGE_SIZE == 0x10000) \ attr = HWRM_FUNC_BACKING_STORE_CFG_INPUT_SRQ_PG_SIZE_PG_64K; \ else \ attr = HWRM_FUNC_BACKING_STORE_CFG_INPUT_SRQ_PG_SIZE_PG_4K; \ } while (0) struct bnxt_ctx_mem_type { u16 type; u16 entry_size; u32 flags; #define BNXT_CTX_MEM_TYPE_VALID HWRM_FUNC_BACKING_STORE_QCAPS_V2_OUTPUT_FLAGS_TYPE_VALID u32 instance_bmap; u8 init_value; u8 entry_multiple; u16 init_offset; #define BNXT_CTX_INIT_INVALID_OFFSET 0xffff u32 max_entries; u32 min_entries; u8 last:1; u8 mem_valid:1; u8 split_entry_cnt; #define BNXT_MAX_SPLIT_ENTRY 4 union { struct { u32 qp_l2_entries; u32 qp_qp1_entries; }; u32 srq_l2_entries; u32 cq_l2_entries; u32 vnic_entries; struct { u32 mrav_av_entries; u32 mrav_num_entries_units; }; u32 split[BNXT_MAX_SPLIT_ENTRY]; }; struct bnxt_ctx_pg_info *pg_info; }; #define BNXT_CTX_QP HWRM_FUNC_BACKING_STORE_CFG_V2_INPUT_TYPE_QP #define BNXT_CTX_SRQ HWRM_FUNC_BACKING_STORE_CFG_V2_INPUT_TYPE_SRQ #define BNXT_CTX_CQ HWRM_FUNC_BACKING_STORE_CFG_V2_INPUT_TYPE_CQ #define BNXT_CTX_VNIC HWRM_FUNC_BACKING_STORE_CFG_V2_INPUT_TYPE_VNIC #define BNXT_CTX_STAT HWRM_FUNC_BACKING_STORE_CFG_V2_INPUT_TYPE_STAT #define BNXT_CTX_STQM HWRM_FUNC_BACKING_STORE_CFG_V2_INPUT_TYPE_SP_TQM_RING #define BNXT_CTX_FTQM HWRM_FUNC_BACKING_STORE_CFG_V2_INPUT_TYPE_FP_TQM_RING #define BNXT_CTX_MRAV HWRM_FUNC_BACKING_STORE_CFG_V2_INPUT_TYPE_MRAV #define BNXT_CTX_TIM HWRM_FUNC_BACKING_STORE_CFG_V2_INPUT_TYPE_TIM #define BNXT_CTX_TKC HWRM_FUNC_BACKING_STORE_CFG_V2_INPUT_TYPE_TKC #define BNXT_CTX_RKC HWRM_FUNC_BACKING_STORE_CFG_V2_INPUT_TYPE_RKC #define BNXT_CTX_MTQM HWRM_FUNC_BACKING_STORE_CFG_V2_INPUT_TYPE_MP_TQM_RING #define BNXT_CTX_SQDBS HWRM_FUNC_BACKING_STORE_CFG_V2_INPUT_TYPE_SQ_DB_SHADOW #define BNXT_CTX_RQDBS HWRM_FUNC_BACKING_STORE_CFG_V2_INPUT_TYPE_RQ_DB_SHADOW #define BNXT_CTX_SRQDBS HWRM_FUNC_BACKING_STORE_CFG_V2_INPUT_TYPE_SRQ_DB_SHADOW #define BNXT_CTX_CQDBS HWRM_FUNC_BACKING_STORE_CFG_V2_INPUT_TYPE_CQ_DB_SHADOW #define BNXT_CTX_QTKC HWRM_FUNC_BACKING_STORE_CFG_V2_INPUT_TYPE_QUIC_TKC #define BNXT_CTX_QRKC HWRM_FUNC_BACKING_STORE_CFG_V2_INPUT_TYPE_QUIC_RKC #define BNXT_CTX_MAX (BNXT_CTX_TIM + 1) #define BNXT_CTX_L2_MAX (BNXT_CTX_FTQM + 1) #define BNXT_CTX_V2_MAX (HWRM_FUNC_BACKING_STORE_CFG_V2_INPUT_TYPE_ROCE_HWRM_TRACE + 1) #define BNXT_CTX_SRT_TRACE HWRM_FUNC_BACKING_STORE_QCFG_V2_OUTPUT_TYPE_SRT_TRACE #define BNXT_CTX_ROCE_HWRM_TRACE HWRM_FUNC_BACKING_STORE_CFG_V2_INPUT_TYPE_ROCE_HWRM_TRACE #define BNXT_CTX_INV ((u16)-1) struct bnxt_ctx_mem_info { u8 tqm_fp_rings_count; u32 flags; #define BNXT_CTX_FLAG_INITED 0x01 struct bnxt_ctx_mem_type ctx_arr[BNXT_CTX_V2_MAX]; }; struct bnxt_hw_resc { uint16_t min_rsscos_ctxs; uint16_t max_rsscos_ctxs; uint16_t min_cp_rings; uint16_t max_cp_rings; uint16_t resv_cp_rings; uint16_t min_tx_rings; uint16_t max_tx_rings; uint16_t resv_tx_rings; uint16_t max_tx_sch_inputs; uint16_t min_rx_rings; uint16_t max_rx_rings; uint16_t resv_rx_rings; uint16_t min_hw_ring_grps; uint16_t max_hw_ring_grps; uint16_t resv_hw_ring_grps; uint16_t min_l2_ctxs; uint16_t max_l2_ctxs; uint16_t min_vnics; uint16_t max_vnics; uint16_t resv_vnics; uint16_t min_stat_ctxs; uint16_t max_stat_ctxs; uint16_t resv_stat_ctxs; uint16_t max_nqs; uint16_t max_irqs; uint16_t resv_irqs; }; enum bnxt_type_ets { BNXT_TYPE_ETS_TSA = 0, BNXT_TYPE_ETS_PRI2TC, BNXT_TYPE_ETS_TCBW, BNXT_TYPE_ETS_MAX }; static const char *const BNXT_ETS_TYPE_STR[] = { "tsa", "pri2tc", "tcbw", }; static const char *const BNXT_ETS_HELP_STR[] = { "X is 1 (strict), 0 (ets)", "TC values for pri 0 to 7", "TC BW values for pri 0 to 7, Sum should be 100", }; #define BNXT_HWRM_MAX_REQ_LEN (softc->hwrm_max_req_len) struct bnxt_softc_list { SLIST_ENTRY(bnxt_softc_list) next; struct bnxt_softc *softc; }; #ifndef BIT_ULL #define BIT_ULL(nr) (1ULL << (nr)) #endif struct bnxt_aux_dev { struct auxiliary_device aux_dev; struct bnxt_en_dev *edev; int id; }; struct bnxt_msix_tbl { uint32_t entry; uint32_t vector; }; enum bnxt_health_severity { SEVERITY_NORMAL = 0, SEVERITY_WARNING, SEVERITY_RECOVERABLE, SEVERITY_FATAL, }; enum bnxt_health_remedy { REMEDY_DEVLINK_RECOVER, REMEDY_POWER_CYCLE_DEVICE, REMEDY_POWER_CYCLE_HOST, REMEDY_FW_UPDATE, REMEDY_HW_REPLACE, }; struct bnxt_fw_health { u32 flags; u32 polling_dsecs; u32 master_func_wait_dsecs; u32 normal_func_wait_dsecs; u32 post_reset_wait_dsecs; u32 post_reset_max_wait_dsecs; u32 regs[4]; u32 mapped_regs[4]; #define BNXT_FW_HEALTH_REG 0 #define BNXT_FW_HEARTBEAT_REG 1 #define BNXT_FW_RESET_CNT_REG 2 #define BNXT_FW_RESET_INPROG_REG 3 u32 fw_reset_inprog_reg_mask; u32 last_fw_heartbeat; u32 last_fw_reset_cnt; u8 enabled:1; u8 primary:1; u8 status_reliable:1; u8 resets_reliable:1; u8 tmr_multiplier; u8 tmr_counter; u8 fw_reset_seq_cnt; u32 fw_reset_seq_regs[16]; u32 fw_reset_seq_vals[16]; u32 fw_reset_seq_delay_msec[16]; u32 echo_req_data1; u32 echo_req_data2; struct devlink_health_reporter *fw_reporter; struct mutex lock; enum bnxt_health_severity severity; enum bnxt_health_remedy remedy; u32 arrests; u32 discoveries; u32 survivals; u32 fatalities; u32 diagnoses; }; #define BNXT_FW_HEALTH_REG_TYPE_MASK 3 #define BNXT_FW_HEALTH_REG_TYPE_CFG 0 #define BNXT_FW_HEALTH_REG_TYPE_GRC 1 #define BNXT_FW_HEALTH_REG_TYPE_BAR0 2 #define BNXT_FW_HEALTH_REG_TYPE_BAR1 3 #define BNXT_FW_HEALTH_REG_TYPE(reg) ((reg) & BNXT_FW_HEALTH_REG_TYPE_MASK) #define BNXT_FW_HEALTH_REG_OFF(reg) ((reg) & ~BNXT_FW_HEALTH_REG_TYPE_MASK) #define BNXT_FW_HEALTH_WIN_BASE 0x3000 #define BNXT_FW_HEALTH_WIN_MAP_OFF 8 #define BNXT_FW_HEALTH_WIN_OFF(reg) (BNXT_FW_HEALTH_WIN_BASE + \ ((reg) & BNXT_GRC_OFFSET_MASK)) #define BNXT_FW_STATUS_HEALTH_MSK 0xffff #define BNXT_FW_STATUS_HEALTHY 0x8000 #define BNXT_FW_STATUS_SHUTDOWN 0x100000 #define BNXT_FW_STATUS_RECOVERING 0x400000 #define BNXT_FW_IS_HEALTHY(sts) (((sts) & BNXT_FW_STATUS_HEALTH_MSK) ==\ BNXT_FW_STATUS_HEALTHY) #define BNXT_FW_IS_BOOTING(sts) (((sts) & BNXT_FW_STATUS_HEALTH_MSK) < \ BNXT_FW_STATUS_HEALTHY) #define BNXT_FW_IS_ERR(sts) (((sts) & BNXT_FW_STATUS_HEALTH_MSK) > \ BNXT_FW_STATUS_HEALTHY) #define BNXT_FW_IS_RECOVERING(sts) (BNXT_FW_IS_ERR(sts) && \ ((sts) & BNXT_FW_STATUS_RECOVERING)) #define BNXT_FW_RETRY 5 #define BNXT_FW_IF_RETRY 10 #define BNXT_FW_SLOT_RESET_RETRY 4 #define BNXT_GRCPF_REG_CHIMP_COMM 0x0 #define BNXT_GRCPF_REG_CHIMP_COMM_TRIGGER 0x100 #define BNXT_GRCPF_REG_WINDOW_BASE_OUT 0x400 #define BNXT_GRCPF_REG_SYNC_TIME 0x480 #define BNXT_GRCPF_REG_SYNC_TIME_ADJ 0x488 #define BNXT_GRCPF_REG_SYNC_TIME_ADJ_PER_MSK 0xffffffUL #define BNXT_GRCPF_REG_SYNC_TIME_ADJ_PER_SFT 0 #define BNXT_GRCPF_REG_SYNC_TIME_ADJ_VAL_MSK 0x1f000000UL #define BNXT_GRCPF_REG_SYNC_TIME_ADJ_VAL_SFT 24 #define BNXT_GRCPF_REG_SYNC_TIME_ADJ_SIGN_MSK 0x20000000UL #define BNXT_GRCPF_REG_SYNC_TIME_ADJ_SIGN_SFT 29 #define BNXT_GRC_REG_STATUS_P5 0x520 #define BNXT_GRCPF_REG_KONG_COMM 0xA00 #define BNXT_GRCPF_REG_KONG_COMM_TRIGGER 0xB00 #define BNXT_CAG_REG_LEGACY_INT_STATUS 0x4014 #define BNXT_CAG_REG_BASE 0x300000 #define BNXT_GRC_REG_CHIP_NUM 0x48 #define BNXT_GRC_REG_BASE 0x260000 #define BNXT_TS_REG_TIMESYNC_TS0_LOWER 0x640180c #define BNXT_TS_REG_TIMESYNC_TS0_UPPER 0x6401810 #define BNXT_GRC_BASE_MASK 0xfffff000 #define BNXT_GRC_OFFSET_MASK 0x00000ffc #define NQE_CN_TYPE(type) ((type) & NQ_CN_TYPE_MASK) #define NQE_CN_TOGGLE(type) (((type) & NQ_CN_TOGGLE_MASK) >> \ NQ_CN_TOGGLE_SFT) #define DB_EPOCH(ring, idx) (((idx) & (ring)->db_epoch_mask) << \ ((ring)->db_epoch_shift)) #define DB_TOGGLE(tgl) ((tgl) << DBR_TOGGLE_SFT) #define DB_RING_IDX_CMP(ring, idx) (((idx) & (ring)->db_ring_mask) | \ DB_EPOCH(ring, idx)) #define DB_RING_IDX(ring, idx, bit) (((idx) & (ring)->db_ring_mask) | \ ((bit) << (24))) struct bnxt_softc { device_t dev; if_ctx_t ctx; if_softc_ctx_t scctx; if_shared_ctx_t sctx; if_t ifp; uint32_t domain; uint32_t bus; uint32_t slot; uint32_t function; uint32_t dev_fn; struct ifmedia *media; struct bnxt_ctx_mem_info *ctx_mem; struct bnxt_hw_resc hw_resc; struct bnxt_softc_list list; struct bnxt_bar_info hwrm_bar; struct bnxt_bar_info doorbell_bar; struct bnxt_link_info link_info; #define BNXT_FLAG_VF 0x0001 #define BNXT_FLAG_NPAR 0x0002 #define BNXT_FLAG_WOL_CAP 0x0004 #define BNXT_FLAG_SHORT_CMD 0x0008 #define BNXT_FLAG_FW_CAP_NEW_RM 0x0010 #define BNXT_FLAG_CHIP_P5 0x0020 #define BNXT_FLAG_TPA 0x0040 #define BNXT_FLAG_FW_CAP_EXT_STATS 0x0080 #define BNXT_FLAG_MULTI_HOST 0x0100 #define BNXT_FLAG_MULTI_ROOT 0x0200 #define BNXT_FLAG_ROCEV1_CAP 0x0400 #define BNXT_FLAG_ROCEV2_CAP 0x0800 #define BNXT_FLAG_ROCE_CAP (BNXT_FLAG_ROCEV1_CAP | BNXT_FLAG_ROCEV2_CAP) #define BNXT_FLAG_CHIP_P7 0x1000 uint32_t flags; #define BNXT_STATE_LINK_CHANGE (0) #define BNXT_STATE_MAX (BNXT_STATE_LINK_CHANGE + 1) bitstr_t *state_bv; uint32_t total_irqs; struct bnxt_msix_tbl *irq_tbl; struct bnxt_func_info func; struct bnxt_func_qcfg fn_qcfg; struct bnxt_pf_info pf; struct bnxt_vf_info vf; uint16_t hwrm_cmd_seq; uint32_t hwrm_cmd_timeo; /* milliseconds */ struct iflib_dma_info hwrm_cmd_resp; struct iflib_dma_info hwrm_short_cmd_req_addr; /* Interrupt info for HWRM */ struct if_irq irq; struct mtx hwrm_lock; uint16_t hwrm_max_req_len; uint16_t hwrm_max_ext_req_len; uint32_t hwrm_spec_code; #define BNXT_MAX_QUEUE 8 uint8_t max_tc; uint8_t max_lltc; struct bnxt_queue_info tx_q_info[BNXT_MAX_QUEUE]; struct bnxt_queue_info rx_q_info[BNXT_MAX_QUEUE]; uint8_t tc_to_qidx[BNXT_MAX_QUEUE]; uint8_t tx_q_ids[BNXT_MAX_QUEUE]; uint8_t rx_q_ids[BNXT_MAX_QUEUE]; uint8_t tx_max_q; uint8_t rx_max_q; uint8_t is_asym_q; struct bnxt_ieee_ets *ieee_ets; struct bnxt_ieee_pfc *ieee_pfc; uint8_t dcbx_cap; uint8_t default_pri; uint8_t max_dscp_value; uint64_t admin_ticks; struct iflib_dma_info hw_rx_port_stats; struct iflib_dma_info hw_tx_port_stats; struct rx_port_stats *rx_port_stats; struct tx_port_stats *tx_port_stats; struct iflib_dma_info hw_tx_port_stats_ext; struct iflib_dma_info hw_rx_port_stats_ext; struct tx_port_stats_ext *tx_port_stats_ext; struct rx_port_stats_ext *rx_port_stats_ext; uint16_t fw_rx_stats_ext_size; uint16_t fw_tx_stats_ext_size; uint16_t hw_ring_stats_size; uint8_t tx_pri2cos_idx[8]; uint8_t rx_pri2cos_idx[8]; bool pri2cos_valid; uint64_t tx_bytes_pri[8]; uint64_t tx_packets_pri[8]; uint64_t rx_bytes_pri[8]; uint64_t rx_packets_pri[8]; uint8_t port_count; int num_cp_rings; struct bnxt_cp_ring *nq_rings; struct bnxt_ring *tx_rings; struct bnxt_cp_ring *tx_cp_rings; struct iflib_dma_info tx_stats[BNXT_MAX_NUM_QUEUES]; int ntxqsets; struct bnxt_vnic_info vnic_info; struct bnxt_ring *ag_rings; struct bnxt_ring *rx_rings; struct bnxt_cp_ring *rx_cp_rings; struct bnxt_grp_info *grp_info; struct iflib_dma_info rx_stats[BNXT_MAX_NUM_QUEUES]; int nrxqsets; uint16_t rx_buf_size; struct bnxt_cp_ring def_cp_ring; struct bnxt_cp_ring def_nq_ring; struct iflib_dma_info def_cp_ring_mem; struct iflib_dma_info def_nq_ring_mem; struct task def_cp_task; int db_size; + int db_offset; int legacy_db_size; struct bnxt_doorbell_ops db_ops; struct sysctl_ctx_list hw_stats; struct sysctl_oid *hw_stats_oid; struct sysctl_ctx_list hw_lro_ctx; struct sysctl_oid *hw_lro_oid; struct sysctl_ctx_list flow_ctrl_ctx; struct sysctl_oid *flow_ctrl_oid; struct sysctl_ctx_list dcb_ctx; struct sysctl_oid *dcb_oid; struct bnxt_ver_info *ver_info; struct bnxt_nvram_info *nvm_info; bool wol; bool is_dev_init; struct bnxt_hw_lro hw_lro; uint8_t wol_filter_id; uint16_t rx_coal_usecs; uint16_t rx_coal_usecs_irq; uint16_t rx_coal_frames; uint16_t rx_coal_frames_irq; uint16_t tx_coal_usecs; uint16_t tx_coal_usecs_irq; uint16_t tx_coal_frames; uint16_t tx_coal_frames_irq; #define BNXT_USEC_TO_COAL_TIMER(x) ((x) * 25 / 2) #define BNXT_DEF_STATS_COAL_TICKS 1000000 #define BNXT_MIN_STATS_COAL_TICKS 250000 #define BNXT_MAX_STATS_COAL_TICKS 1000000 uint64_t fw_cap; #define BNXT_FW_CAP_SHORT_CMD BIT_ULL(0) #define BNXT_FW_CAP_LLDP_AGENT BIT_ULL(1) #define BNXT_FW_CAP_DCBX_AGENT BIT_ULL(2) #define BNXT_FW_CAP_NEW_RM BIT_ULL(3) #define BNXT_FW_CAP_IF_CHANGE BIT_ULL(4) #define BNXT_FW_CAP_LINK_ADMIN BIT_ULL(5) #define BNXT_FW_CAP_VF_RES_MIN_GUARANTEED BIT_ULL(6) #define BNXT_FW_CAP_KONG_MB_CHNL BIT_ULL(7) #define BNXT_FW_CAP_ADMIN_MTU BIT_ULL(8) #define BNXT_FW_CAP_ADMIN_PF BIT_ULL(9) #define BNXT_FW_CAP_OVS_64BIT_HANDLE BIT_ULL(10) #define BNXT_FW_CAP_TRUSTED_VF BIT_ULL(11) #define BNXT_FW_CAP_VF_VNIC_NOTIFY BIT_ULL(12) #define BNXT_FW_CAP_ERROR_RECOVERY BIT_ULL(13) #define BNXT_FW_CAP_PKG_VER BIT_ULL(14) #define BNXT_FW_CAP_CFA_ADV_FLOW BIT_ULL(15) #define BNXT_FW_CAP_CFA_RFS_RING_TBL_IDX_V2 BIT_ULL(16) #define BNXT_FW_CAP_PCIE_STATS_SUPPORTED BIT_ULL(17) #define BNXT_FW_CAP_EXT_STATS_SUPPORTED BIT_ULL(18) #define BNXT_FW_CAP_SECURE_MODE BIT_ULL(19) #define BNXT_FW_CAP_ERR_RECOVER_RELOAD BIT_ULL(20) #define BNXT_FW_CAP_HOT_RESET BIT_ULL(21) #define BNXT_FW_CAP_CRASHDUMP BIT_ULL(23) #define BNXT_FW_CAP_VLAN_RX_STRIP BIT_ULL(24) #define BNXT_FW_CAP_VLAN_TX_INSERT BIT_ULL(25) #define BNXT_FW_CAP_EXT_HW_STATS_SUPPORTED BIT_ULL(26) #define BNXT_FW_CAP_CFA_EEM BIT_ULL(27) #define BNXT_FW_CAP_DBG_QCAPS BIT_ULL(29) #define BNXT_FW_CAP_RING_MONITOR BIT_ULL(30) #define BNXT_FW_CAP_ECN_STATS BIT_ULL(31) #define BNXT_FW_CAP_TRUFLOW BIT_ULL(32) #define BNXT_FW_CAP_VF_CFG_FOR_PF BIT_ULL(33) #define BNXT_FW_CAP_PTP_PPS BIT_ULL(34) #define BNXT_FW_CAP_HOT_RESET_IF BIT_ULL(35) #define BNXT_FW_CAP_LIVEPATCH BIT_ULL(36) #define BNXT_FW_CAP_NPAR_1_2 BIT_ULL(37) #define BNXT_FW_CAP_RSS_HASH_TYPE_DELTA BIT_ULL(38) #define BNXT_FW_CAP_PTP_RTC BIT_ULL(39) #define BNXT_FW_CAP_TRUFLOW_EN BIT_ULL(40) #define BNXT_TRUFLOW_EN(bp) ((bp)->fw_cap & BNXT_FW_CAP_TRUFLOW_EN) #define BNXT_FW_CAP_RX_ALL_PKT_TS BIT_ULL(41) #define BNXT_FW_CAP_BACKING_STORE_V2 BIT_ULL(42) #define BNXT_FW_CAP_DBR_SUPPORTED BIT_ULL(43) #define BNXT_FW_CAP_GENERIC_STATS BIT_ULL(44) #define BNXT_FW_CAP_DBR_PACING_SUPPORTED BIT_ULL(45) #define BNXT_FW_CAP_PTP_PTM BIT_ULL(46) #define BNXT_FW_CAP_CFA_NTUPLE_RX_EXT_IP_PROTO BIT_ULL(47) #define BNXT_FW_CAP_ENABLE_RDMA_SRIOV BIT_ULL(48) #define BNXT_FW_CAP_RSS_TCAM BIT_ULL(49) uint32_t lpi_tmr_lo; uint32_t lpi_tmr_hi; /* copied from flags and flags2 in hwrm_port_phy_qcaps_output */ uint16_t phy_flags; #define BNXT_PHY_FL_EEE_CAP HWRM_PORT_PHY_QCAPS_OUTPUT_FLAGS_EEE_SUPPORTED #define BNXT_PHY_FL_EXT_LPBK HWRM_PORT_PHY_QCAPS_OUTPUT_FLAGS_EXTERNAL_LPBK_SUPPORTED #define BNXT_PHY_FL_AN_PHY_LPBK HWRM_PORT_PHY_QCAPS_OUTPUT_FLAGS_AUTONEG_LPBK_SUPPORTED #define BNXT_PHY_FL_SHARED_PORT_CFG HWRM_PORT_PHY_QCAPS_OUTPUT_FLAGS_SHARED_PHY_CFG_SUPPORTED #define BNXT_PHY_FL_PORT_STATS_NO_RESET HWRM_PORT_PHY_QCAPS_OUTPUT_FLAGS_CUMULATIVE_COUNTERS_ON_RESET #define BNXT_PHY_FL_NO_PHY_LPBK HWRM_PORT_PHY_QCAPS_OUTPUT_FLAGS_LOCAL_LPBK_NOT_SUPPORTED #define BNXT_PHY_FL_FW_MANAGED_LKDN HWRM_PORT_PHY_QCAPS_OUTPUT_FLAGS_FW_MANAGED_LINK_DOWN #define BNXT_PHY_FL_NO_FCS HWRM_PORT_PHY_QCAPS_OUTPUT_FLAGS_NO_FCS #define BNXT_PHY_FL_NO_PAUSE (HWRM_PORT_PHY_QCAPS_OUTPUT_FLAGS2_PAUSE_UNSUPPORTED << 8) #define BNXT_PHY_FL_NO_PFC (HWRM_PORT_PHY_QCAPS_OUTPUT_FLAGS2_PFC_UNSUPPORTED << 8) #define BNXT_PHY_FL_BANK_SEL (HWRM_PORT_PHY_QCAPS_OUTPUT_FLAGS2_BANK_ADDR_SUPPORTED << 8) struct bnxt_aux_dev *aux_dev; struct net_device *net_dev; struct mtx en_ops_lock; uint8_t port_partition_type; struct bnxt_en_dev *edev; unsigned long state; #define BNXT_STATE_OPEN 0 #define BNXT_STATE_IN_SP_TASK 1 #define BNXT_STATE_READ_STATS 2 #define BNXT_STATE_FW_RESET_DET 3 #define BNXT_STATE_IN_FW_RESET 4 #define BNXT_STATE_ABORT_ERR 5 #define BNXT_STATE_FW_FATAL_COND 6 #define BNXT_STATE_DRV_REGISTERED 7 #define BNXT_STATE_PCI_CHANNEL_IO_FROZEN 8 #define BNXT_STATE_NAPI_DISABLED 9 #define BNXT_STATE_L2_FILTER_RETRY 10 #define BNXT_STATE_FW_ACTIVATE 11 #define BNXT_STATE_RECOVER 12 #define BNXT_STATE_FW_NON_FATAL_COND 13 #define BNXT_STATE_FW_ACTIVATE_RESET 14 #define BNXT_STATE_HALF_OPEN 15 #define BNXT_NO_FW_ACCESS(bp) \ test_bit(BNXT_STATE_FW_FATAL_COND, &(bp)->state) struct pci_dev *pdev; struct work_struct sp_task; unsigned long sp_event; #define BNXT_RX_MASK_SP_EVENT 0 #define BNXT_RX_NTP_FLTR_SP_EVENT 1 #define BNXT_LINK_CHNG_SP_EVENT 2 #define BNXT_HWRM_EXEC_FWD_REQ_SP_EVENT 3 #define BNXT_VXLAN_ADD_PORT_SP_EVENT 4 #define BNXT_VXLAN_DEL_PORT_SP_EVENT 5 #define BNXT_RESET_TASK_SP_EVENT 6 #define BNXT_RST_RING_SP_EVENT 7 #define BNXT_HWRM_PF_UNLOAD_SP_EVENT 8 #define BNXT_PERIODIC_STATS_SP_EVENT 9 #define BNXT_HWRM_PORT_MODULE_SP_EVENT 10 #define BNXT_RESET_TASK_SILENT_SP_EVENT 11 #define BNXT_GENEVE_ADD_PORT_SP_EVENT 12 #define BNXT_GENEVE_DEL_PORT_SP_EVENT 13 #define BNXT_LINK_SPEED_CHNG_SP_EVENT 14 #define BNXT_FLOW_STATS_SP_EVENT 15 #define BNXT_UPDATE_PHY_SP_EVENT 16 #define BNXT_RING_COAL_NOW_SP_EVENT 17 #define BNXT_FW_RESET_NOTIFY_SP_EVENT 18 #define BNXT_FW_EXCEPTION_SP_EVENT 19 #define BNXT_VF_VNIC_CHANGE_SP_EVENT 20 #define BNXT_LINK_CFG_CHANGE_SP_EVENT 21 #define BNXT_PTP_CURRENT_TIME_EVENT 22 #define BNXT_FW_ECHO_REQUEST_SP_EVENT 23 #define BNXT_VF_CFG_CHNG_SP_EVENT 24 struct delayed_work fw_reset_task; int fw_reset_state; #define BNXT_FW_RESET_STATE_POLL_VF 1 #define BNXT_FW_RESET_STATE_RESET_FW 2 #define BNXT_FW_RESET_STATE_ENABLE_DEV 3 #define BNXT_FW_RESET_STATE_POLL_FW 4 #define BNXT_FW_RESET_STATE_OPENING 5 #define BNXT_FW_RESET_STATE_POLL_FW_DOWN 6 u16 fw_reset_min_dsecs; #define BNXT_DFLT_FW_RST_MIN_DSECS 20 u16 fw_reset_max_dsecs; #define BNXT_DFLT_FW_RST_MAX_DSECS 60 unsigned long fw_reset_timestamp; struct bnxt_fw_health *fw_health; }; struct bnxt_filter_info { STAILQ_ENTRY(bnxt_filter_info) next; uint64_t fw_l2_filter_id; #define INVALID_MAC_INDEX ((uint16_t)-1) uint16_t mac_index; /* Filter Characteristics */ uint32_t flags; uint32_t enables; uint8_t l2_addr[ETHER_ADDR_LEN]; uint8_t l2_addr_mask[ETHER_ADDR_LEN]; uint16_t l2_ovlan; uint16_t l2_ovlan_mask; uint16_t l2_ivlan; uint16_t l2_ivlan_mask; uint8_t t_l2_addr[ETHER_ADDR_LEN]; uint8_t t_l2_addr_mask[ETHER_ADDR_LEN]; uint16_t t_l2_ovlan; uint16_t t_l2_ovlan_mask; uint16_t t_l2_ivlan; uint16_t t_l2_ivlan_mask; uint8_t tunnel_type; uint16_t mirror_vnic_id; uint32_t vni; uint8_t pri_hint; uint64_t l2_filter_id_hint; }; #define I2C_DEV_ADDR_A0 0xa0 #define BNXT_MAX_PHY_I2C_RESP_SIZE 64 /* Function declarations */ void bnxt_report_link(struct bnxt_softc *softc); bool bnxt_check_hwrm_version(struct bnxt_softc *softc); struct bnxt_softc *bnxt_find_dev(uint32_t domain, uint32_t bus, uint32_t dev_fn, char *name); int bnxt_read_sfp_module_eeprom_info(struct bnxt_softc *bp, uint16_t i2c_addr, uint16_t page_number, uint8_t bank, bool bank_sel_en, uint16_t start_addr, uint16_t data_length, uint8_t *buf); void bnxt_dcb_init(struct bnxt_softc *softc); void bnxt_dcb_free(struct bnxt_softc *softc); uint8_t bnxt_dcb_setdcbx(struct bnxt_softc *softc, uint8_t mode); uint8_t bnxt_dcb_getdcbx(struct bnxt_softc *softc); int bnxt_dcb_ieee_getets(struct bnxt_softc *softc, struct bnxt_ieee_ets *ets); int bnxt_dcb_ieee_setets(struct bnxt_softc *softc, struct bnxt_ieee_ets *ets); uint8_t get_phy_type(struct bnxt_softc *softc); int bnxt_dcb_ieee_getpfc(struct bnxt_softc *softc, struct bnxt_ieee_pfc *pfc); int bnxt_dcb_ieee_setpfc(struct bnxt_softc *softc, struct bnxt_ieee_pfc *pfc); int bnxt_dcb_ieee_setapp(struct bnxt_softc *softc, struct bnxt_dcb_app *app); int bnxt_dcb_ieee_delapp(struct bnxt_softc *softc, struct bnxt_dcb_app *app); int bnxt_dcb_ieee_listapp(struct bnxt_softc *softc, struct bnxt_dcb_app *app, size_t nitems, int *num_inputs); #endif /* _BNXT_H */ diff --git a/sys/dev/bnxt/bnxt_en/bnxt_hwrm.c b/sys/dev/bnxt/bnxt_en/bnxt_hwrm.c index 9e7f4614d9f9..3e5dc0bf4b6d 100644 --- a/sys/dev/bnxt/bnxt_en/bnxt_hwrm.c +++ b/sys/dev/bnxt/bnxt_en/bnxt_hwrm.c @@ -1,3191 +1,3193 @@ /*- * Broadcom NetXtreme-C/E network driver. * * Copyright (c) 2016 Broadcom, All Rights Reserved. * The term Broadcom refers to Broadcom Limited and/or its subsidiaries * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS' * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF * THE POSSIBILITY OF SUCH DAMAGE. */ #include #include #include #include "bnxt.h" #include "bnxt_hwrm.h" #include "hsi_struct_def.h" static int bnxt_hwrm_err_map(uint16_t err); static inline int _is_valid_ether_addr(uint8_t *); static inline void get_random_ether_addr(uint8_t *); static void bnxt_hwrm_set_link_common(struct bnxt_softc *softc, struct hwrm_port_phy_cfg_input *req); static void bnxt_hwrm_set_pause_common(struct bnxt_softc *softc, struct hwrm_port_phy_cfg_input *req); static void bnxt_hwrm_set_eee(struct bnxt_softc *softc, struct hwrm_port_phy_cfg_input *req); /* NVRam stuff has a five minute timeout */ #define BNXT_NVM_TIMEO (5 * 60 * 1000) #define BNXT_RX_STATS_PRI_ENTRY(counter, n) \ BNXT_RX_STATS_EXT_OFFSET(counter##_cos0) #define BNXT_TX_STATS_PRI_ENTRY(counter, n) \ BNXT_TX_STATS_EXT_OFFSET(counter##_cos0) #define BNXT_RX_STATS_PRI_ENTRIES(counter) \ BNXT_RX_STATS_PRI_ENTRY(counter, 0), \ BNXT_RX_STATS_PRI_ENTRY(counter, 1), \ BNXT_RX_STATS_PRI_ENTRY(counter, 2), \ BNXT_RX_STATS_PRI_ENTRY(counter, 3), \ BNXT_RX_STATS_PRI_ENTRY(counter, 4), \ BNXT_RX_STATS_PRI_ENTRY(counter, 5), \ BNXT_RX_STATS_PRI_ENTRY(counter, 6), \ BNXT_RX_STATS_PRI_ENTRY(counter, 7) #define BNXT_TX_STATS_PRI_ENTRIES(counter) \ BNXT_TX_STATS_PRI_ENTRY(counter, 0), \ BNXT_TX_STATS_PRI_ENTRY(counter, 1), \ BNXT_TX_STATS_PRI_ENTRY(counter, 2), \ BNXT_TX_STATS_PRI_ENTRY(counter, 3), \ BNXT_TX_STATS_PRI_ENTRY(counter, 4), \ BNXT_TX_STATS_PRI_ENTRY(counter, 5), \ BNXT_TX_STATS_PRI_ENTRY(counter, 6), \ BNXT_TX_STATS_PRI_ENTRY(counter, 7) long bnxt_rx_bytes_pri_arr_base_off[] = {BNXT_RX_STATS_PRI_ENTRIES(rx_bytes)}; long bnxt_rx_pkts_pri_arr_base_off[] = {BNXT_RX_STATS_PRI_ENTRIES(rx_packets)}; long bnxt_tx_bytes_pri_arr_base_off[] = {BNXT_TX_STATS_PRI_ENTRIES(tx_bytes)}; long bnxt_tx_pkts_pri_arr_base_off[] = {BNXT_TX_STATS_PRI_ENTRIES(tx_packets)}; static int bnxt_hwrm_err_map(uint16_t err) { int rc; switch (err) { case HWRM_ERR_CODE_SUCCESS: return 0; case HWRM_ERR_CODE_INVALID_PARAMS: case HWRM_ERR_CODE_INVALID_FLAGS: case HWRM_ERR_CODE_INVALID_ENABLES: return EINVAL; case HWRM_ERR_CODE_RESOURCE_ACCESS_DENIED: return EACCES; case HWRM_ERR_CODE_RESOURCE_ALLOC_ERROR: return ENOMEM; case HWRM_ERR_CODE_CMD_NOT_SUPPORTED: return ENOSYS; case HWRM_ERR_CODE_FAIL: return EIO; case HWRM_ERR_CODE_HWRM_ERROR: case HWRM_ERR_CODE_UNKNOWN_ERR: default: return EDOOFUS; } return rc; } int bnxt_alloc_hwrm_dma_mem(struct bnxt_softc *softc) { int rc; rc = iflib_dma_alloc(softc->ctx, PAGE_SIZE, &softc->hwrm_cmd_resp, BUS_DMA_NOWAIT); return rc; } void bnxt_free_hwrm_dma_mem(struct bnxt_softc *softc) { if (softc->hwrm_cmd_resp.idi_vaddr) iflib_dma_free(&softc->hwrm_cmd_resp); softc->hwrm_cmd_resp.idi_vaddr = NULL; return; } void bnxt_hwrm_cmd_hdr_init(struct bnxt_softc *softc, void *request, uint16_t req_type) { struct input *req = request; req->req_type = htole16(req_type); req->cmpl_ring = 0xffff; req->target_id = 0xffff; req->resp_addr = htole64(softc->hwrm_cmd_resp.idi_paddr); } int _hwrm_send_message(struct bnxt_softc *softc, void *msg, uint32_t msg_len) { struct input *req = msg; struct hwrm_err_output *resp = (void *)softc->hwrm_cmd_resp.idi_vaddr; uint32_t *data = msg; int i; uint8_t *valid; uint16_t err; uint16_t max_req_len = BNXT_HWRM_MAX_REQ_LEN; struct hwrm_short_input short_input = {0}; /* TODO: DMASYNC in here. */ req->seq_id = htole16(softc->hwrm_cmd_seq++); memset(resp, 0, PAGE_SIZE); if (BNXT_NO_FW_ACCESS(softc) && (req->req_type != HWRM_FUNC_RESET && req->req_type != HWRM_VER_GET)) return -EINVAL; if ((softc->flags & BNXT_FLAG_SHORT_CMD) || msg_len > BNXT_HWRM_MAX_REQ_LEN) { void *short_cmd_req = softc->hwrm_short_cmd_req_addr.idi_vaddr; uint16_t max_msg_len; /* Set boundary for maximum extended request length for short * cmd format. If passed up from device use the max supported * internal req length. */ max_msg_len = softc->hwrm_max_ext_req_len; memcpy(short_cmd_req, req, msg_len); if (msg_len < max_msg_len) memset((uint8_t *) short_cmd_req + msg_len, 0, max_msg_len - msg_len); short_input.req_type = req->req_type; short_input.signature = htole16(HWRM_SHORT_INPUT_SIGNATURE_SHORT_CMD); short_input.size = htole16(msg_len); short_input.req_addr = htole64(softc->hwrm_short_cmd_req_addr.idi_paddr); data = (uint32_t *)&short_input; msg_len = sizeof(short_input); /* Sync memory write before updating doorbell */ wmb(); max_req_len = BNXT_HWRM_SHORT_REQ_LEN; } /* Write request msg to hwrm channel */ for (i = 0; i < msg_len; i += 4) { bus_space_write_4(softc->hwrm_bar.tag, softc->hwrm_bar.handle, i, *data); data++; } /* Clear to the end of the request buffer */ for (i = msg_len; i < max_req_len; i += 4) bus_space_write_4(softc->hwrm_bar.tag, softc->hwrm_bar.handle, i, 0); /* Ring channel doorbell */ bus_space_write_4(softc->hwrm_bar.tag, softc->hwrm_bar.handle, 0x100, htole32(1)); /* Check if response len is updated */ for (i = 0; i < softc->hwrm_cmd_timeo; i++) { if (resp->resp_len && resp->resp_len <= 4096) break; DELAY(1000); } if (i >= softc->hwrm_cmd_timeo) { device_printf(softc->dev, "Timeout sending %s: (timeout: %u) seq: %d\n", GET_HWRM_REQ_TYPE(req->req_type), softc->hwrm_cmd_timeo, le16toh(req->seq_id)); return ETIMEDOUT; } /* Last byte of resp contains the valid key */ valid = (uint8_t *)resp + resp->resp_len - 1; for (i = 0; i < softc->hwrm_cmd_timeo; i++) { if (*valid == HWRM_RESP_VALID_KEY) break; DELAY(1000); } if (i >= softc->hwrm_cmd_timeo) { device_printf(softc->dev, "Timeout sending %s: " "(timeout: %u) msg {0x%x 0x%x} len:%d v: %d\n", GET_HWRM_REQ_TYPE(req->req_type), softc->hwrm_cmd_timeo, le16toh(req->req_type), le16toh(req->seq_id), msg_len, *valid); return ETIMEDOUT; } err = le16toh(resp->error_code); if (err) { /* HWRM_ERR_CODE_FAIL is a "normal" error, don't log */ if (err != HWRM_ERR_CODE_FAIL) { device_printf(softc->dev, "%s command returned %s error.\n", GET_HWRM_REQ_TYPE(req->req_type), GET_HWRM_ERROR_CODE(err)); } return bnxt_hwrm_err_map(err); } return 0; } int hwrm_send_message(struct bnxt_softc *softc, void *msg, uint32_t msg_len) { int rc; BNXT_HWRM_LOCK(softc); rc = _hwrm_send_message(softc, msg, msg_len); BNXT_HWRM_UNLOCK(softc); return rc; } int bnxt_hwrm_queue_qportcfg(struct bnxt_softc *softc, uint32_t path_dir) { int rc = 0; struct hwrm_queue_qportcfg_input req = {0}; struct hwrm_queue_qportcfg_output *resp = (void *)softc->hwrm_cmd_resp.idi_vaddr; uint8_t max_tc, max_lltc, *max_q; uint8_t queue_profile, queue_id; struct bnxt_queue_info *q_info; uint8_t i, j, *qptr, *q_ids; bool no_rdma; bnxt_hwrm_cmd_hdr_init(softc, &req, HWRM_QUEUE_QPORTCFG); BNXT_HWRM_LOCK(softc); rc = _hwrm_send_message(softc, &req, sizeof(req)); if (rc) goto qportcfg_exit; if (!resp->max_configurable_queues) { rc = -EINVAL; goto qportcfg_exit; } if (resp->queue_cfg_info & HWRM_QUEUE_QPORTCFG_OUTPUT_QUEUE_CFG_INFO_ASYM_CFG) { softc->is_asym_q = true; /* bnxt_init_cosq_names(softc, path_dir); */ } else { softc->is_asym_q = false; /* bnxt_free_stats_cosqnames_mem(softc); */ } max_tc = min_t(uint8_t, resp->max_configurable_queues, BNXT_MAX_QUEUE); max_lltc = resp->max_configurable_lossless_queues; /* * No RDMA support yet. * no_rdma = !(softc->flags & BNXT_FLAG_ROCE_CAP); */ no_rdma = true; qptr = &resp->queue_id0; if (path_dir == HWRM_QUEUE_QPORTCFG_INPUT_FLAGS_PATH_TX) { q_info = softc->tx_q_info; q_ids = softc->tx_q_ids; max_q = &softc->tx_max_q; } else { q_info = softc->rx_q_info; q_ids = softc->rx_q_ids; max_q = &softc->rx_max_q; } for (i = 0, j = 0; i < max_tc; i++) { queue_id = *qptr; qptr++; queue_profile = *qptr; qptr++; q_info[j].queue_id = queue_id; q_info[j].queue_profile = queue_profile; q_ids[i] = queue_id; softc->tc_to_qidx[j] = j; if (!BNXT_CNPQ(q_info[j].queue_profile) || (no_rdma && BNXT_PF(softc))) j++; } *max_q = max_tc; max_tc = max_t(uint8_t, j, 1); softc->max_tc = softc->max_tc ? min(softc->max_tc, max_tc) : max_tc; softc->max_lltc = softc->max_lltc ? min(softc->max_lltc, max_lltc) : max_lltc; if (softc->max_lltc > softc->max_tc) softc->max_lltc = softc->max_tc; qportcfg_exit: BNXT_HWRM_UNLOCK(softc); return rc; } static int bnxt_alloc_all_ctx_pg_info(struct bnxt_softc *softc, int ctx_max) { struct bnxt_ctx_mem_info *ctx = softc->ctx_mem; u16 type; for (type = 0; type < ctx_max; type++) { struct bnxt_ctx_mem_type *ctxm = &ctx->ctx_arr[type]; int n = 1; if (!ctxm->max_entries || ctxm->pg_info) continue; if (ctxm->instance_bmap) n = hweight32(ctxm->instance_bmap); ctxm->pg_info = kcalloc(n, sizeof(*ctxm->pg_info), GFP_ATOMIC); if (!ctxm->pg_info) return -ENOMEM; } return 0; } static void bnxt_init_ctx_initializer(struct bnxt_ctx_mem_type *ctxm, u8 init_val, u8 init_offset, bool init_mask_set) { ctxm->init_value = init_val; ctxm->init_offset = BNXT_CTX_INIT_INVALID_OFFSET; if (init_mask_set) ctxm->init_offset = init_offset * 4; else ctxm->init_value = 0; } #define BNXT_CTX_INIT_VALID(flags) \ (!!((flags) & \ HWRM_FUNC_BACKING_STORE_QCAPS_V2_OUTPUT_FLAGS_ENABLE_CTX_KIND_INIT)) static int bnxt_hwrm_func_backing_store_qcaps_v2(struct bnxt_softc *softc) { struct hwrm_func_backing_store_qcaps_v2_input req = {0}; struct hwrm_func_backing_store_qcaps_v2_output *resp = (void *)softc->hwrm_cmd_resp.idi_vaddr; struct bnxt_ctx_mem_info *ctx = NULL; u16 type; int rc; bnxt_hwrm_cmd_hdr_init(softc, &req, HWRM_FUNC_BACKING_STORE_QCAPS_V2); ctx = malloc(sizeof(*ctx), M_DEVBUF, M_NOWAIT | M_ZERO); if (!ctx) return -ENOMEM; softc->ctx_mem = ctx; BNXT_HWRM_LOCK(softc); for (type = 0; type < BNXT_CTX_V2_MAX; ) { struct bnxt_ctx_mem_type *ctxm = &ctx->ctx_arr[type]; u8 init_val, init_off, i; __le32 *p; u32 flags; req.type = cpu_to_le16(type); rc = _hwrm_send_message(softc, &req, sizeof(req)); if (rc) goto ctx_done; flags = le32_to_cpu(resp->flags); type = le16_to_cpu(resp->next_valid_type); if (!(flags & HWRM_FUNC_BACKING_STORE_QCAPS_V2_OUTPUT_FLAGS_TYPE_VALID)) continue; ctxm->type = le16_to_cpu(resp->type); ctxm->flags = flags; ctxm->entry_size = le16_to_cpu(resp->entry_size); ctxm->instance_bmap = le32_to_cpu(resp->instance_bit_map); ctxm->entry_multiple = resp->entry_multiple; ctxm->max_entries = le32_to_cpu(resp->max_num_entries); ctxm->min_entries = le32_to_cpu(resp->min_num_entries); init_val = resp->ctx_init_value; init_off = resp->ctx_init_offset; bnxt_init_ctx_initializer(ctxm, init_val, init_off, BNXT_CTX_INIT_VALID(flags)); ctxm->split_entry_cnt = min_t(u8, resp->subtype_valid_cnt, BNXT_MAX_SPLIT_ENTRY); for (i = 0, p = &resp->split_entry_0; i < ctxm->split_entry_cnt; i++, p++) ctxm->split[i] = le32_to_cpu(*p); } rc = bnxt_alloc_all_ctx_pg_info(softc, BNXT_CTX_V2_MAX); ctx_done: BNXT_HWRM_UNLOCK(softc); return rc; } int bnxt_hwrm_func_backing_store_qcaps(struct bnxt_softc *softc) { struct hwrm_func_backing_store_qcaps_input req = {0}; struct hwrm_func_backing_store_qcaps_output *resp = (void *)softc->hwrm_cmd_resp.idi_vaddr; int rc; if (softc->hwrm_spec_code < 0x10902 || softc->ctx_mem) return 0; if (BNXT_CHIP_P7(softc)) { if (softc->fw_cap & BNXT_FW_CAP_BACKING_STORE_V2) return bnxt_hwrm_func_backing_store_qcaps_v2(softc); } if (BNXT_VF(softc)) return 0; bnxt_hwrm_cmd_hdr_init(softc, &req, HWRM_FUNC_BACKING_STORE_QCAPS); BNXT_HWRM_LOCK(softc); rc = _hwrm_send_message(softc, &req, sizeof(req)); if (!rc) { struct bnxt_ctx_mem_type *ctxm; struct bnxt_ctx_mem_info *ctx; u8 init_val, init_idx = 0; u16 init_mask; ctx = softc->ctx_mem; if (!ctx) { ctx = malloc(sizeof(*ctx), M_DEVBUF, M_NOWAIT | M_ZERO); if (!ctx) { rc = -ENOMEM; goto ctx_err; } softc->ctx_mem = ctx; } init_val = resp->ctx_kind_initializer; init_mask = le16_to_cpu(resp->ctx_init_mask); ctxm = &ctx->ctx_arr[BNXT_CTX_QP]; ctxm->max_entries = le32_to_cpu(resp->qp_max_entries); ctxm->qp_qp1_entries = le16_to_cpu(resp->qp_min_qp1_entries); ctxm->qp_l2_entries = le16_to_cpu(resp->qp_max_l2_entries); ctxm->entry_size = le16_to_cpu(resp->qp_entry_size); bnxt_init_ctx_initializer(ctxm, init_val, resp->qp_init_offset, (init_mask & (1 << init_idx++)) != 0); ctxm = &ctx->ctx_arr[BNXT_CTX_SRQ]; ctxm->srq_l2_entries = le16_to_cpu(resp->srq_max_l2_entries); ctxm->max_entries = le32_to_cpu(resp->srq_max_entries); ctxm->entry_size = le16_to_cpu(resp->srq_entry_size); bnxt_init_ctx_initializer(ctxm, init_val, resp->srq_init_offset, (init_mask & (1 << init_idx++)) != 0); ctxm = &ctx->ctx_arr[BNXT_CTX_CQ]; ctxm->cq_l2_entries = le16_to_cpu(resp->cq_max_l2_entries); ctxm->max_entries = le32_to_cpu(resp->cq_max_entries); ctxm->entry_size = le16_to_cpu(resp->cq_entry_size); bnxt_init_ctx_initializer(ctxm, init_val, resp->cq_init_offset, (init_mask & (1 << init_idx++)) != 0); ctxm = &ctx->ctx_arr[BNXT_CTX_VNIC]; ctxm->vnic_entries = le32_to_cpu(resp->vnic_max_vnic_entries); ctxm->max_entries = ctxm->vnic_entries + le16_to_cpu(resp->vnic_max_ring_table_entries); ctxm->entry_size = le16_to_cpu(resp->vnic_entry_size); bnxt_init_ctx_initializer(ctxm, init_val, resp->vnic_init_offset, (init_mask & (1 << init_idx++)) != 0); ctxm = &ctx->ctx_arr[BNXT_CTX_STAT]; ctxm->max_entries = le32_to_cpu(resp->stat_max_entries); ctxm->entry_size = le16_to_cpu(resp->stat_entry_size); bnxt_init_ctx_initializer(ctxm, init_val, resp->stat_init_offset, (init_mask & (1 << init_idx++)) != 0); ctxm = &ctx->ctx_arr[BNXT_CTX_STQM]; ctxm->entry_size = le16_to_cpu(resp->tqm_entry_size); ctxm->min_entries = le32_to_cpu(resp->tqm_min_entries_per_ring); ctxm->max_entries = le32_to_cpu(resp->tqm_max_entries_per_ring); ctxm->entry_multiple = resp->tqm_entries_multiple; if (!ctxm->entry_multiple) ctxm->entry_multiple = 1; memcpy(&ctx->ctx_arr[BNXT_CTX_FTQM], ctxm, sizeof(*ctxm)); ctxm = &ctx->ctx_arr[BNXT_CTX_MRAV]; ctxm->max_entries = le32_to_cpu(resp->mrav_max_entries); ctxm->entry_size = le16_to_cpu(resp->mrav_entry_size); ctxm->mrav_num_entries_units = le16_to_cpu(resp->mrav_num_entries_units); bnxt_init_ctx_initializer(ctxm, init_val, resp->mrav_init_offset, (init_mask & (1 << init_idx++)) != 0); ctxm = &ctx->ctx_arr[BNXT_CTX_TIM]; ctxm->entry_size = le16_to_cpu(resp->tim_entry_size); ctxm->max_entries = le32_to_cpu(resp->tim_max_entries); ctx->tqm_fp_rings_count = resp->tqm_fp_rings_count; if (!ctx->tqm_fp_rings_count) ctx->tqm_fp_rings_count = softc->tx_max_q; else if (ctx->tqm_fp_rings_count > BNXT_MAX_TQM_FP_LEGACY_RINGS) ctx->tqm_fp_rings_count = BNXT_MAX_TQM_FP_LEGACY_RINGS; if (ctx->tqm_fp_rings_count == BNXT_MAX_TQM_FP_LEGACY_RINGS && softc->hwrm_max_ext_req_len >= BNXT_BACKING_STORE_CFG_LEN) { ctx->tqm_fp_rings_count += resp->tqm_fp_rings_count_ext; if (ctx->tqm_fp_rings_count > BNXT_MAX_TQM_FP_RINGS) ctx->tqm_fp_rings_count = BNXT_MAX_TQM_FP_RINGS; } ctxm = &ctx->ctx_arr[BNXT_CTX_FTQM]; memcpy(ctxm, &ctx->ctx_arr[BNXT_CTX_STQM], sizeof(*ctxm)); ctxm->instance_bmap = (1 << ctx->tqm_fp_rings_count) - 1; rc = bnxt_alloc_all_ctx_pg_info(softc, BNXT_CTX_MAX); } else { rc = 0; } ctx_err: BNXT_HWRM_UNLOCK(softc); return rc; } #define HWRM_FUNC_BACKING_STORE_CFG_INPUT_DFLT_ENABLES \ (HWRM_FUNC_BACKING_STORE_CFG_INPUT_ENABLES_QP | \ HWRM_FUNC_BACKING_STORE_CFG_INPUT_ENABLES_SRQ | \ HWRM_FUNC_BACKING_STORE_CFG_INPUT_ENABLES_CQ | \ HWRM_FUNC_BACKING_STORE_CFG_INPUT_ENABLES_VNIC | \ HWRM_FUNC_BACKING_STORE_CFG_INPUT_ENABLES_STAT) static void bnxt_hwrm_set_pg_attr(struct bnxt_ring_mem_info *rmem, uint8_t *pg_attr, uint64_t *pg_dir) { if (!rmem->nr_pages) return; BNXT_SET_CTX_PAGE_ATTR(*pg_attr); if (rmem->depth >= 1) { if (rmem->depth == 2) *pg_attr |= HWRM_FUNC_BACKING_STORE_CFG_INPUT_QPC_LVL_LVL_2; else *pg_attr |= HWRM_FUNC_BACKING_STORE_CFG_INPUT_QPC_LVL_LVL_1; *pg_dir = htole64(rmem->pg_tbl.idi_paddr); } else { *pg_dir = htole64(rmem->pg_arr[0].idi_paddr); } } int bnxt_hwrm_func_backing_store_cfg(struct bnxt_softc *softc, uint32_t enables) { struct hwrm_func_backing_store_cfg_input req = {0}; struct bnxt_ctx_mem_info *ctx = softc->ctx_mem; struct bnxt_ctx_pg_info *ctx_pg; struct bnxt_ctx_mem_type *ctxm; u32 req_len = sizeof(req); __le32 *num_entries; u32 ena, flags = 0; __le64 *pg_dir; u8 *pg_attr; int i; if (!ctx) return 0; if (req_len > softc->hwrm_max_ext_req_len) req_len = BNXT_BACKING_STORE_CFG_LEGACY_LEN; bnxt_hwrm_cmd_hdr_init(softc, &req, HWRM_FUNC_BACKING_STORE_CFG); req.enables = htole32(enables); if (enables & HWRM_FUNC_BACKING_STORE_CFG_INPUT_ENABLES_QP) { ctxm = &ctx->ctx_arr[BNXT_CTX_QP]; ctx_pg = ctxm->pg_info; req.qp_num_entries = cpu_to_le32(ctx_pg->entries); req.qp_num_qp1_entries = cpu_to_le16(ctxm->qp_qp1_entries); req.qp_num_l2_entries = cpu_to_le16(ctxm->qp_l2_entries); req.qp_entry_size = cpu_to_le16(ctxm->entry_size); bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem, &req.qpc_pg_size_qpc_lvl, &req.qpc_page_dir); } if (enables & HWRM_FUNC_BACKING_STORE_CFG_INPUT_ENABLES_SRQ) { ctxm = &ctx->ctx_arr[BNXT_CTX_SRQ]; ctx_pg = ctxm->pg_info; req.srq_num_entries = cpu_to_le32(ctx_pg->entries); req.srq_num_l2_entries = cpu_to_le16(ctxm->srq_l2_entries); req.srq_entry_size = cpu_to_le16(ctxm->entry_size); bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem, &req.srq_pg_size_srq_lvl, &req.srq_page_dir); } if (enables & HWRM_FUNC_BACKING_STORE_CFG_INPUT_ENABLES_CQ) { ctxm = &ctx->ctx_arr[BNXT_CTX_CQ]; ctx_pg = ctxm->pg_info; req.cq_num_entries = cpu_to_le32(ctx_pg->entries); req.cq_num_l2_entries = cpu_to_le16(ctxm->cq_l2_entries); req.cq_entry_size = cpu_to_le16(ctxm->entry_size); bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem, &req.cq_pg_size_cq_lvl, &req.cq_page_dir); } if (enables & HWRM_FUNC_BACKING_STORE_CFG_INPUT_ENABLES_MRAV) { ctxm = &ctx->ctx_arr[BNXT_CTX_MRAV]; ctx_pg = ctxm->pg_info; req.mrav_num_entries = cpu_to_le32(ctx_pg->entries); if (ctxm->mrav_num_entries_units) flags |= HWRM_FUNC_BACKING_STORE_CFG_INPUT_FLAGS_MRAV_RESERVATION_SPLIT; req.mrav_entry_size = cpu_to_le16(ctxm->entry_size); bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem, &req.mrav_pg_size_mrav_lvl, &req.mrav_page_dir); } if (enables & HWRM_FUNC_BACKING_STORE_CFG_INPUT_ENABLES_TIM) { ctxm = &ctx->ctx_arr[BNXT_CTX_TIM]; ctx_pg = ctxm->pg_info; req.tim_num_entries = cpu_to_le32(ctx_pg->entries); req.tim_entry_size = cpu_to_le16(ctxm->entry_size); bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem, &req.tim_pg_size_tim_lvl, &req.tim_page_dir); } if (enables & HWRM_FUNC_BACKING_STORE_CFG_INPUT_ENABLES_VNIC) { ctxm = &ctx->ctx_arr[BNXT_CTX_VNIC]; ctx_pg = ctxm->pg_info; req.vnic_num_vnic_entries = cpu_to_le16(ctxm->vnic_entries); req.vnic_num_ring_table_entries = cpu_to_le16(ctxm->max_entries - ctxm->vnic_entries); req.vnic_entry_size = cpu_to_le16(ctxm->entry_size); bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem, &req.vnic_pg_size_vnic_lvl, &req.vnic_page_dir); } if (enables & HWRM_FUNC_BACKING_STORE_CFG_INPUT_ENABLES_STAT) { ctxm = &ctx->ctx_arr[BNXT_CTX_STAT]; ctx_pg = ctxm->pg_info; req.stat_num_entries = cpu_to_le32(ctxm->max_entries); req.stat_entry_size = cpu_to_le16(ctxm->entry_size); bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem, &req.stat_pg_size_stat_lvl, &req.stat_page_dir); } ctxm = &ctx->ctx_arr[BNXT_CTX_STQM]; for (i = 0, num_entries = &req.tqm_sp_num_entries, pg_attr = &req.tqm_sp_pg_size_tqm_sp_lvl, pg_dir = &req.tqm_sp_page_dir, ena = HWRM_FUNC_BACKING_STORE_CFG_INPUT_ENABLES_TQM_SP, ctx_pg = ctxm->pg_info; i < BNXT_MAX_TQM_LEGACY_RINGS; ctx_pg = &ctx->ctx_arr[BNXT_CTX_FTQM].pg_info[i], i++, num_entries++, pg_attr++, pg_dir++, ena <<= 1) { if (!(enables & ena)) continue; req.tqm_entry_size = cpu_to_le16(ctxm->entry_size); *num_entries = cpu_to_le32(ctx_pg->entries); bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem, pg_attr, pg_dir); } if (enables & HWRM_FUNC_BACKING_STORE_CFG_INPUT_ENABLES_TQM_RING8) { pg_attr = &req.tqm_ring8_pg_size_tqm_ring_lvl; pg_dir = &req.tqm_ring8_page_dir; ctx_pg = &ctx->ctx_arr[BNXT_CTX_FTQM].pg_info[8]; req.tqm_ring8_num_entries = cpu_to_le32(ctx_pg->entries); bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem, pg_attr, pg_dir); } req.flags = cpu_to_le32(flags); return hwrm_send_message(softc, &req, req_len); } int bnxt_hwrm_func_resc_qcaps(struct bnxt_softc *softc, bool all) { struct hwrm_func_resource_qcaps_output *resp = (void *)softc->hwrm_cmd_resp.idi_vaddr; struct hwrm_func_resource_qcaps_input req = {0}; struct bnxt_hw_resc *hw_resc = &softc->hw_resc; int rc; bnxt_hwrm_cmd_hdr_init(softc, &req, HWRM_FUNC_RESOURCE_QCAPS); req.fid = htole16(0xffff); BNXT_HWRM_LOCK(softc); rc = _hwrm_send_message(softc, &req, sizeof(req)); if (rc) { rc = -EIO; goto hwrm_func_resc_qcaps_exit; } hw_resc->max_tx_sch_inputs = le16toh(resp->max_tx_scheduler_inputs); if (!all) goto hwrm_func_resc_qcaps_exit; hw_resc->min_rsscos_ctxs = le16toh(resp->min_rsscos_ctx); hw_resc->max_rsscos_ctxs = le16toh(resp->max_rsscos_ctx); hw_resc->min_cp_rings = le16toh(resp->min_cmpl_rings); hw_resc->max_cp_rings = le16toh(resp->max_cmpl_rings); hw_resc->min_tx_rings = le16toh(resp->min_tx_rings); hw_resc->max_tx_rings = le16toh(resp->max_tx_rings); hw_resc->min_rx_rings = le16toh(resp->min_rx_rings); hw_resc->max_rx_rings = le16toh(resp->max_rx_rings); hw_resc->min_hw_ring_grps = le16toh(resp->min_hw_ring_grps); hw_resc->max_hw_ring_grps = le16toh(resp->max_hw_ring_grps); hw_resc->min_l2_ctxs = le16toh(resp->min_l2_ctxs); hw_resc->max_l2_ctxs = le16toh(resp->max_l2_ctxs); hw_resc->min_vnics = le16toh(resp->min_vnics); hw_resc->max_vnics = le16toh(resp->max_vnics); hw_resc->min_stat_ctxs = le16toh(resp->min_stat_ctx); hw_resc->max_stat_ctxs = le16toh(resp->max_stat_ctx); if (BNXT_CHIP_P5_PLUS(softc)) { hw_resc->max_nqs = le16toh(resp->max_msix); hw_resc->max_hw_ring_grps = hw_resc->max_rx_rings; } hwrm_func_resc_qcaps_exit: BNXT_HWRM_UNLOCK(softc); return rc; } int bnxt_hwrm_func_backing_store_cfg_v2(struct bnxt_softc *softc, struct bnxt_ctx_mem_type *ctxm, bool last) { struct hwrm_func_backing_store_cfg_v2_input req = {0}; u32 instance_bmap = ctxm->instance_bmap; int i, j, rc = 0, n = 1; __le32 *p; if (!(ctxm->flags & BNXT_CTX_MEM_TYPE_VALID) || !ctxm->pg_info) return 0; if (instance_bmap) n = hweight32(ctxm->instance_bmap); else instance_bmap = 1; bnxt_hwrm_cmd_hdr_init(softc, &req, HWRM_FUNC_BACKING_STORE_CFG_V2); BNXT_HWRM_LOCK(softc); req.type = cpu_to_le16(ctxm->type); req.entry_size = cpu_to_le16(ctxm->entry_size); req.subtype_valid_cnt = ctxm->split_entry_cnt; for (i = 0, p = &req.split_entry_0; i < ctxm->split_entry_cnt; i++) p[i] = cpu_to_le32(ctxm->split[i]); for (i = 0, j = 0; j < n && !rc; i++) { struct bnxt_ctx_pg_info *ctx_pg; if (!(instance_bmap & (1 << i))) continue; req.instance = cpu_to_le16(i); ctx_pg = &ctxm->pg_info[j++]; if (!ctx_pg->entries) continue; req.num_entries = cpu_to_le32(ctx_pg->entries); bnxt_hwrm_set_pg_attr(&ctx_pg->ring_mem, &req.page_size_pbl_level, &req.page_dir); if (last && j == n) req.flags = cpu_to_le32(HWRM_FUNC_BACKING_STORE_CFG_V2_INPUT_FLAGS_BS_CFG_ALL_DONE); rc = _hwrm_send_message(softc, &req, sizeof(req)); } BNXT_HWRM_UNLOCK(softc); return rc; } int bnxt_hwrm_passthrough(struct bnxt_softc *softc, void *req, uint32_t req_len, void *resp, uint32_t resp_len, uint32_t app_timeout) { int rc = 0; void *output = (void *)softc->hwrm_cmd_resp.idi_vaddr; struct input *input = req; uint32_t old_timeo; input->resp_addr = htole64(softc->hwrm_cmd_resp.idi_paddr); BNXT_HWRM_LOCK(softc); old_timeo = softc->hwrm_cmd_timeo; if (input->req_type == HWRM_NVM_INSTALL_UPDATE) softc->hwrm_cmd_timeo = BNXT_NVM_TIMEO; else softc->hwrm_cmd_timeo = max(app_timeout, softc->hwrm_cmd_timeo); rc = _hwrm_send_message(softc, req, req_len); softc->hwrm_cmd_timeo = old_timeo; if (rc) { device_printf(softc->dev, "%s: %s command failed with rc: 0x%x\n", __FUNCTION__, GET_HWRM_REQ_TYPE(input->req_type), rc); goto fail; } memcpy(resp, output, resp_len); fail: BNXT_HWRM_UNLOCK(softc); return rc; } int bnxt_hwrm_ver_get(struct bnxt_softc *softc) { struct hwrm_ver_get_input req = {0}; struct hwrm_ver_get_output *resp = (void *)softc->hwrm_cmd_resp.idi_vaddr; int rc; const char nastr[] = ""; const char naver[] = ""; uint32_t dev_caps_cfg; uint16_t fw_maj, fw_min, fw_bld, fw_rsv, len; softc->hwrm_max_req_len = HWRM_MAX_REQ_LEN; softc->hwrm_cmd_timeo = 1000; bnxt_hwrm_cmd_hdr_init(softc, &req, HWRM_VER_GET); req.hwrm_intf_maj = HWRM_VERSION_MAJOR; req.hwrm_intf_min = HWRM_VERSION_MINOR; req.hwrm_intf_upd = HWRM_VERSION_UPDATE; BNXT_HWRM_LOCK(softc); rc = _hwrm_send_message(softc, &req, sizeof(req)); if (rc) goto fail; snprintf(softc->ver_info->hwrm_if_ver, BNXT_VERSTR_SIZE, "%d.%d.%d", resp->hwrm_intf_maj_8b, resp->hwrm_intf_min_8b, resp->hwrm_intf_upd_8b); softc->ver_info->hwrm_if_major = resp->hwrm_intf_maj_8b; softc->ver_info->hwrm_if_minor = resp->hwrm_intf_min_8b; softc->ver_info->hwrm_if_update = resp->hwrm_intf_upd_8b; strlcpy(softc->ver_info->driver_hwrm_if_ver, HWRM_VERSION_STR, BNXT_VERSTR_SIZE); strlcpy(softc->ver_info->hwrm_fw_name, resp->hwrm_fw_name, BNXT_NAME_SIZE); softc->hwrm_spec_code = resp->hwrm_intf_maj_8b << 16 | resp->hwrm_intf_min_8b << 8 | resp->hwrm_intf_upd_8b; if (resp->hwrm_intf_maj_8b < 1) { device_printf(softc->dev, "HWRM interface %d.%d.%d is older " "than 1.0.0.\n", resp->hwrm_intf_maj_8b, resp->hwrm_intf_min_8b, resp->hwrm_intf_upd_8b); device_printf(softc->dev, "Please update firmware with HWRM " "interface 1.0.0 or newer.\n"); } if (resp->mgmt_fw_major == 0 && resp->mgmt_fw_minor == 0 && resp->mgmt_fw_build == 0) { strlcpy(softc->ver_info->mgmt_fw_ver, naver, BNXT_VERSTR_SIZE); strlcpy(softc->ver_info->mgmt_fw_name, nastr, BNXT_NAME_SIZE); } else { snprintf(softc->ver_info->mgmt_fw_ver, FW_VER_STR_LEN, "%d.%d.%d.%d", resp->mgmt_fw_major, resp->mgmt_fw_minor, resp->mgmt_fw_build, resp->mgmt_fw_patch); strlcpy(softc->ver_info->mgmt_fw_name, resp->mgmt_fw_name, BNXT_NAME_SIZE); } if (resp->netctrl_fw_major == 0 && resp->netctrl_fw_minor == 0 && resp->netctrl_fw_build == 0) { strlcpy(softc->ver_info->netctrl_fw_ver, naver, BNXT_VERSTR_SIZE); strlcpy(softc->ver_info->netctrl_fw_name, nastr, BNXT_NAME_SIZE); } else { snprintf(softc->ver_info->netctrl_fw_ver, FW_VER_STR_LEN, "%d.%d.%d.%d", resp->netctrl_fw_major, resp->netctrl_fw_minor, resp->netctrl_fw_build, resp->netctrl_fw_patch); strlcpy(softc->ver_info->netctrl_fw_name, resp->netctrl_fw_name, BNXT_NAME_SIZE); } if (resp->roce_fw_major == 0 && resp->roce_fw_minor == 0 && resp->roce_fw_build == 0) { strlcpy(softc->ver_info->roce_fw_ver, naver, BNXT_VERSTR_SIZE); strlcpy(softc->ver_info->roce_fw_name, nastr, BNXT_NAME_SIZE); } else { snprintf(softc->ver_info->roce_fw_ver, BNXT_VERSTR_SIZE, "%d.%d.%d.%d", resp->roce_fw_major, resp->roce_fw_minor, resp->roce_fw_build, resp->roce_fw_patch); strlcpy(softc->ver_info->roce_fw_name, resp->roce_fw_name, BNXT_NAME_SIZE); } fw_maj = le32toh(resp->hwrm_fw_major); if (softc->hwrm_spec_code > 0x10803 && fw_maj) { fw_min = le16toh(resp->hwrm_fw_minor); fw_bld = le16toh(resp->hwrm_fw_build); fw_rsv = le16toh(resp->hwrm_fw_patch); len = FW_VER_STR_LEN; } else { fw_maj = resp->hwrm_fw_maj_8b; fw_min = resp->hwrm_fw_min_8b; fw_bld = resp->hwrm_fw_bld_8b; fw_rsv = resp->hwrm_fw_rsvd_8b; len = BC_HWRM_STR_LEN; } softc->ver_info->fw_ver_code = BNXT_FW_VER_CODE(fw_maj, fw_min, fw_bld, fw_rsv); snprintf (softc->ver_info->fw_ver_str, len, "%d.%d.%d.%d", fw_maj, fw_min, fw_bld, fw_rsv); if (strlen(resp->active_pkg_name)) { int fw_ver_len = strlen (softc->ver_info->fw_ver_str); snprintf(softc->ver_info->fw_ver_str + fw_ver_len, FW_VER_STR_LEN - fw_ver_len - 1, "/pkg %s", resp->active_pkg_name); softc->fw_cap |= BNXT_FW_CAP_PKG_VER; } softc->ver_info->chip_num = le16toh(resp->chip_num); softc->ver_info->chip_rev = resp->chip_rev; softc->ver_info->chip_metal = resp->chip_metal; softc->ver_info->chip_bond_id = resp->chip_bond_id; softc->ver_info->chip_type = resp->chip_platform_type; if (resp->hwrm_intf_maj_8b >= 1) { softc->hwrm_max_req_len = le16toh(resp->max_req_win_len); softc->hwrm_max_ext_req_len = le16toh(resp->max_ext_req_len); } softc->hwrm_cmd_timeo = le16toh(resp->def_req_timeout); if (!softc->hwrm_cmd_timeo) softc->hwrm_cmd_timeo = DFLT_HWRM_CMD_TIMEOUT; dev_caps_cfg = le32toh(resp->dev_caps_cfg); if ((dev_caps_cfg & HWRM_VER_GET_OUTPUT_DEV_CAPS_CFG_SHORT_CMD_SUPPORTED) && (dev_caps_cfg & HWRM_VER_GET_OUTPUT_DEV_CAPS_CFG_SHORT_CMD_REQUIRED)) softc->flags |= BNXT_FLAG_SHORT_CMD; if ((dev_caps_cfg & HWRM_VER_GET_OUTPUT_DEV_CAPS_CFG_SHORT_CMD_SUPPORTED) && (dev_caps_cfg & HWRM_VER_GET_OUTPUT_DEV_CAPS_CFG_SHORT_CMD_REQUIRED)) softc->fw_cap |= BNXT_FW_CAP_SHORT_CMD; if (dev_caps_cfg & HWRM_VER_GET_OUTPUT_DEV_CAPS_CFG_KONG_MB_CHNL_SUPPORTED) softc->fw_cap |= BNXT_FW_CAP_KONG_MB_CHNL; if (dev_caps_cfg & HWRM_VER_GET_OUTPUT_DEV_CAPS_CFG_FLOW_HANDLE_64BIT_SUPPORTED) softc->fw_cap |= BNXT_FW_CAP_OVS_64BIT_HANDLE; if (dev_caps_cfg & HWRM_VER_GET_OUTPUT_DEV_CAPS_CFG_TRUSTED_VF_SUPPORTED) softc->fw_cap |= BNXT_FW_CAP_TRUSTED_VF; if (dev_caps_cfg & HWRM_VER_GET_OUTPUT_DEV_CAPS_CFG_CFA_ADV_FLOW_MGNT_SUPPORTED) softc->fw_cap |= BNXT_FW_CAP_CFA_ADV_FLOW; if (dev_caps_cfg & HWRM_VER_GET_OUTPUT_DEV_CAPS_CFG_CFA_EEM_SUPPORTED) softc->fw_cap |= BNXT_FW_CAP_CFA_EEM; if (dev_caps_cfg & HWRM_VER_GET_OUTPUT_DEV_CAPS_CFG_CFA_TRUFLOW_SUPPORTED) softc->fw_cap |= BNXT_FW_CAP_TRUFLOW_EN; fail: BNXT_HWRM_UNLOCK(softc); return rc; } static const u16 bnxt_async_events_arr[] = { HWRM_ASYNC_EVENT_CMPL_EVENT_ID_LINK_STATUS_CHANGE, HWRM_ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CHANGE, HWRM_ASYNC_EVENT_CMPL_EVENT_ID_PF_DRVR_UNLOAD, HWRM_ASYNC_EVENT_CMPL_EVENT_ID_PORT_CONN_NOT_ALLOWED, HWRM_ASYNC_EVENT_CMPL_EVENT_ID_VF_CFG_CHANGE, HWRM_ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CFG_CHANGE, HWRM_ASYNC_EVENT_CMPL_EVENT_ID_PORT_PHY_CFG_CHANGE, HWRM_ASYNC_EVENT_CMPL_EVENT_ID_RESET_NOTIFY, HWRM_ASYNC_EVENT_CMPL_EVENT_ID_ERROR_RECOVERY, HWRM_ASYNC_EVENT_CMPL_EVENT_ID_RING_MONITOR_MSG, HWRM_ASYNC_EVENT_CMPL_EVENT_ID_DEFAULT_VNIC_CHANGE, HWRM_ASYNC_EVENT_CMPL_EVENT_ID_DEBUG_NOTIFICATION, HWRM_ASYNC_EVENT_CMPL_EVENT_ID_DEFERRED_RESPONSE, HWRM_ASYNC_EVENT_CMPL_EVENT_ID_ECHO_REQUEST, HWRM_ASYNC_EVENT_CMPL_EVENT_ID_PPS_TIMESTAMP, HWRM_ASYNC_EVENT_CMPL_EVENT_ID_ERROR_REPORT, HWRM_ASYNC_EVENT_CMPL_EVENT_ID_PHC_UPDATE, }; int bnxt_hwrm_func_drv_rgtr(struct bnxt_softc *bp, unsigned long *bmap, int bmap_size, bool async_only) { DECLARE_BITMAP(async_events_bmap, 256); u32 *events = (u32 *)async_events_bmap; struct hwrm_func_drv_rgtr_output *resp = (void *)bp->hwrm_cmd_resp.idi_vaddr; struct hwrm_func_drv_rgtr_input req = {0}; u32 flags = 0; int rc; int i; bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FUNC_DRV_RGTR); req.ver_maj = HWRM_VERSION_MAJOR; req.ver_min = HWRM_VERSION_MINOR; req.ver_upd = HWRM_VERSION_UPDATE; req.enables = htole32(HWRM_FUNC_DRV_RGTR_INPUT_ENABLES_OS_TYPE | HWRM_FUNC_DRV_RGTR_INPUT_ENABLES_VER | HWRM_FUNC_DRV_RGTR_INPUT_ENABLES_ASYNC_EVENT_FWD); if (bp->fw_cap & BNXT_FW_CAP_HOT_RESET) flags |= HWRM_FUNC_DRV_RGTR_INPUT_FLAGS_HOT_RESET_SUPPORT; if (bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY) flags |= HWRM_FUNC_DRV_RGTR_INPUT_FLAGS_ERROR_RECOVERY_SUPPORT | HWRM_FUNC_DRV_RGTR_INPUT_FLAGS_MASTER_SUPPORT; if (bp->fw_cap & BNXT_FW_CAP_NPAR_1_2) flags |= HWRM_FUNC_DRV_RGTR_INPUT_FLAGS_NPAR_1_2_SUPPORT; flags |= HWRM_FUNC_DRV_RGTR_INPUT_FLAGS_ASYM_QUEUE_CFG_SUPPORT; req.flags = htole32(flags); req.os_type = htole16(HWRM_FUNC_DRV_RGTR_INPUT_OS_TYPE_FREEBSD); if (BNXT_PF(bp)) { req.enables |= htole32(HWRM_FUNC_DRV_RGTR_INPUT_ENABLES_VF_REQ_FWD); } if (bp->fw_cap & BNXT_FW_CAP_OVS_64BIT_HANDLE) req.flags |= cpu_to_le32(HWRM_FUNC_DRV_RGTR_INPUT_FLAGS_FLOW_HANDLE_64BIT_MODE); memset(async_events_bmap, 0, sizeof(async_events_bmap)); for (i = 0; i < ARRAY_SIZE(bnxt_async_events_arr); i++) { u16 event_id = bnxt_async_events_arr[i]; if (event_id == HWRM_ASYNC_EVENT_CMPL_EVENT_ID_ERROR_RECOVERY && !(bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY)) { continue; } __set_bit(bnxt_async_events_arr[i], async_events_bmap); } if (bmap && bmap_size) { for (i = 0; i < bmap_size; i++) { if (test_bit(i, bmap)) __set_bit(i, async_events_bmap); } } for (i = 0; i < 8; i++) req.async_event_fwd[i] |= htole32(events[i]); if (async_only) req.enables = htole32(HWRM_FUNC_DRV_RGTR_INPUT_ENABLES_ASYNC_EVENT_FWD); rc = hwrm_send_message(bp, &req, sizeof(req)); if (!rc) { if (resp->flags & le32toh(HWRM_FUNC_DRV_RGTR_OUTPUT_FLAGS_IF_CHANGE_SUPPORTED)) bp->fw_cap |= BNXT_FW_CAP_IF_CHANGE; } return rc; } int bnxt_hwrm_func_drv_unrgtr(struct bnxt_softc *softc, bool shutdown) { struct hwrm_func_drv_unrgtr_input req = {0}; bnxt_hwrm_cmd_hdr_init(softc, &req, HWRM_FUNC_DRV_UNRGTR); if (shutdown == true) req.flags |= HWRM_FUNC_DRV_UNRGTR_INPUT_FLAGS_PREPARE_FOR_SHUTDOWN; return hwrm_send_message(softc, &req, sizeof(req)); } static inline int _is_valid_ether_addr(uint8_t *addr) { char zero_addr[6] = { 0, 0, 0, 0, 0, 0 }; if ((addr[0] & 1) || (!bcmp(addr, zero_addr, ETHER_ADDR_LEN))) return (FALSE); return (TRUE); } static inline void get_random_ether_addr(uint8_t *addr) { uint8_t temp[ETHER_ADDR_LEN]; arc4rand(&temp, sizeof(temp), 0); temp[0] &= 0xFE; temp[0] |= 0x02; bcopy(temp, addr, sizeof(temp)); } int bnxt_hwrm_func_qcaps(struct bnxt_softc *softc) { int rc = 0; struct hwrm_func_qcaps_input req = {0}; struct hwrm_func_qcaps_output *resp = (void *)softc->hwrm_cmd_resp.idi_vaddr; struct bnxt_func_info *func = &softc->func; uint32_t flags, flags_ext, flags_ext2; bnxt_hwrm_cmd_hdr_init(softc, &req, HWRM_FUNC_QCAPS); req.fid = htole16(0xffff); BNXT_HWRM_LOCK(softc); rc = _hwrm_send_message(softc, &req, sizeof(req)); if (rc) goto fail; flags = htole32(resp->flags); if (flags & HWRM_FUNC_QCAPS_OUTPUT_FLAGS_WOL_MAGICPKT_SUPPORTED) softc->flags |= BNXT_FLAG_WOL_CAP; if (flags & HWRM_FUNC_QCAPS_OUTPUT_FLAGS_EXT_STATS_SUPPORTED) softc->flags |= BNXT_FLAG_FW_CAP_EXT_STATS; /* Enable RoCE only on Thor devices */ if (BNXT_CHIP_P5_PLUS(softc)) { if (flags & HWRM_FUNC_QCAPS_OUTPUT_FLAGS_ROCE_V1_SUPPORTED) softc->flags |= BNXT_FLAG_ROCEV1_CAP; if (flags & HWRM_FUNC_QCAPS_OUTPUT_FLAGS_ROCE_V2_SUPPORTED) softc->flags |= BNXT_FLAG_ROCEV2_CAP; } if (flags & HWRM_FUNC_QCAPS_OUTPUT_FLAGS_LINK_ADMIN_STATUS_SUPPORTED) softc->fw_cap |= BNXT_FW_CAP_LINK_ADMIN; if (flags & HWRM_FUNC_QCAPS_OUTPUT_FLAGS_ADMIN_PF_SUPPORTED) softc->fw_cap |= BNXT_FW_CAP_ADMIN_PF; if (flags & HWRM_FUNC_QCAPS_OUTPUT_FLAGS_HOT_RESET_CAPABLE) softc->fw_cap |= BNXT_FW_CAP_HOT_RESET; if (flags & HWRM_FUNC_QCAPS_OUTPUT_FLAGS_ERROR_RECOVERY_CAPABLE) softc->fw_cap |= BNXT_FW_CAP_ERROR_RECOVERY; if (flags & HWRM_FUNC_QCAPS_OUTPUT_FLAGS_PCIE_STATS_SUPPORTED) softc->fw_cap |= BNXT_FW_CAP_PCIE_STATS_SUPPORTED; if (flags & HWRM_FUNC_QCAPS_OUTPUT_FLAGS_EXT_STATS_SUPPORTED) softc->fw_cap |= BNXT_FW_CAP_EXT_STATS_SUPPORTED; if (flags & HWRM_FUNC_QCAPS_OUTPUT_FLAGS_ERR_RECOVER_RELOAD) softc->fw_cap |= BNXT_FW_CAP_ERR_RECOVER_RELOAD; if (flags & HWRM_FUNC_QCAPS_OUTPUT_FLAGS_NOTIFY_VF_DEF_VNIC_CHNG_SUPPORTED) softc->fw_cap |= BNXT_FW_CAP_VF_VNIC_NOTIFY; if (flags & HWRM_FUNC_QCAPS_OUTPUT_FLAGS_CRASHDUMP_CMD_SUPPORTED) softc->fw_cap |= BNXT_FW_CAP_CRASHDUMP; if (!(flags & HWRM_FUNC_QCAPS_OUTPUT_FLAGS_VLAN_ACCELERATION_TX_DISABLED)) softc->fw_cap |= BNXT_FW_CAP_VLAN_TX_INSERT; if (flags & HWRM_FUNC_QCAPS_OUTPUT_FLAGS_DBG_QCAPS_CMD_SUPPORTED) softc->fw_cap |= BNXT_FW_CAP_DBG_QCAPS; flags_ext = htole32(resp->flags_ext); if (flags_ext & HWRM_FUNC_QCAPS_OUTPUT_FLAGS_EXT_EXT_HW_STATS_SUPPORTED) softc->fw_cap |= BNXT_FW_CAP_EXT_HW_STATS_SUPPORTED; if (BNXT_PF(softc) && (flags_ext & HWRM_FUNC_QCAPS_OUTPUT_FLAGS_EXT_ECN_STATS_SUPPORTED)) softc->fw_cap |= BNXT_FW_CAP_ECN_STATS; if (flags_ext & HWRM_FUNC_QCAPS_OUTPUT_FLAGS_EXT_PTP_PPS_SUPPORTED) softc->fw_cap |= BNXT_FW_CAP_PTP_PPS; if (flags_ext & HWRM_FUNC_QCAPS_OUTPUT_FLAGS_EXT_PTP_PTM_SUPPORTED) softc->fw_cap |= BNXT_FW_CAP_PTP_PTM; if (flags_ext & HWRM_FUNC_QCAPS_OUTPUT_FLAGS_EXT_PTP_64BIT_RTC_SUPPORTED) softc->fw_cap |= BNXT_FW_CAP_PTP_RTC; if (BNXT_PF(softc) && (flags_ext & HWRM_FUNC_QCAPS_OUTPUT_FLAGS_EXT_HOT_RESET_IF_SUPPORT)) softc->fw_cap |= BNXT_FW_CAP_HOT_RESET_IF; if (BNXT_PF(softc) && (flags_ext & HWRM_FUNC_QCAPS_OUTPUT_FLAGS_EXT_FW_LIVEPATCH_SUPPORTED)) softc->fw_cap |= BNXT_FW_CAP_LIVEPATCH; if (flags_ext & HWRM_FUNC_QCAPS_OUTPUT_FLAGS_EXT_NPAR_1_2_SUPPORTED) softc->fw_cap |= BNXT_FW_CAP_NPAR_1_2; if (flags_ext & HWRM_FUNC_QCAPS_OUTPUT_FLAGS_EXT_BS_V2_SUPPORTED) softc->fw_cap |= BNXT_FW_CAP_BACKING_STORE_V2; if (BNXT_PF(softc) && (flags_ext & HWRM_FUNC_QCAPS_OUTPUT_FLAGS_EXT_VF_CFG_ASYNC_FOR_PF_SUPPORTED)) softc->fw_cap |= BNXT_FW_CAP_VF_CFG_FOR_PF; flags_ext2 = htole32(resp->flags_ext2); if (flags_ext2 & HWRM_FUNC_QCAPS_OUTPUT_FLAGS_EXT2_RX_ALL_PKTS_TIMESTAMPS_SUPPORTED) softc->fw_cap |= BNXT_FW_CAP_RX_ALL_PKT_TS; if (flags_ext2 & HWRM_FUNC_QCAPS_OUTPUT_FLAGS_EXT2_SW_DBR_DROP_RECOVERY_SUPPORTED) softc->fw_cap |= BNXT_FW_CAP_DBR_SUPPORTED; if (flags_ext2 & HWRM_FUNC_QCAPS_OUTPUT_FLAGS_EXT2_DBR_PACING_EXT_SUPPORTED || flags_ext & HWRM_FUNC_QCAPS_OUTPUT_FLAGS_EXT_DBR_PACING_SUPPORTED) softc->fw_cap |= BNXT_FW_CAP_DBR_PACING_SUPPORTED; if (flags_ext2 & HWRM_FUNC_QCAPS_OUTPUT_FLAGS_EXT2_GENERIC_STATS_SUPPORTED) softc->fw_cap |= BNXT_FW_CAP_GENERIC_STATS; func->fw_fid = le16toh(resp->fid); memcpy(func->mac_addr, resp->mac_address, ETHER_ADDR_LEN); func->max_rsscos_ctxs = le16toh(resp->max_rsscos_ctx); func->max_cp_rings = le16toh(resp->max_cmpl_rings); func->max_tx_rings = le16toh(resp->max_tx_rings); func->max_rx_rings = le16toh(resp->max_rx_rings); func->max_hw_ring_grps = le32toh(resp->max_hw_ring_grps); if (!func->max_hw_ring_grps) func->max_hw_ring_grps = func->max_tx_rings; func->max_l2_ctxs = le16toh(resp->max_l2_ctxs); func->max_vnics = le16toh(resp->max_vnics); func->max_stat_ctxs = le16toh(resp->max_stat_ctx); if (BNXT_PF(softc)) { struct bnxt_pf_info *pf = &softc->pf; pf->port_id = le16toh(resp->port_id); pf->first_vf_id = le16toh(resp->first_vf_id); pf->max_vfs = le16toh(resp->max_vfs); pf->max_encap_records = le32toh(resp->max_encap_records); pf->max_decap_records = le32toh(resp->max_decap_records); pf->max_tx_em_flows = le32toh(resp->max_tx_em_flows); pf->max_tx_wm_flows = le32toh(resp->max_tx_wm_flows); pf->max_rx_em_flows = le32toh(resp->max_rx_em_flows); pf->max_rx_wm_flows = le32toh(resp->max_rx_wm_flows); } if (!_is_valid_ether_addr(func->mac_addr)) { device_printf(softc->dev, "Invalid ethernet address, generating random locally administered address\n"); get_random_ether_addr(func->mac_addr); } fail: BNXT_HWRM_UNLOCK(softc); return rc; } int bnxt_hwrm_func_qcfg(struct bnxt_softc *softc) { struct hwrm_func_qcfg_input req = {0}; struct hwrm_func_qcfg_output *resp = (void *)softc->hwrm_cmd_resp.idi_vaddr; struct bnxt_func_qcfg *fn_qcfg = &softc->fn_qcfg; uint32_t min_db_offset = 0; uint16_t flags; int rc; bnxt_hwrm_cmd_hdr_init(softc, &req, HWRM_FUNC_QCFG); req.fid = htole16(0xffff); BNXT_HWRM_LOCK(softc); rc = _hwrm_send_message(softc, &req, sizeof(req)); if (rc) goto end; fn_qcfg->alloc_completion_rings = le16toh(resp->alloc_cmpl_rings); fn_qcfg->alloc_tx_rings = le16toh(resp->alloc_tx_rings); fn_qcfg->alloc_rx_rings = le16toh(resp->alloc_rx_rings); fn_qcfg->alloc_vnics = le16toh(resp->alloc_vnics); switch (resp->port_partition_type) { case HWRM_FUNC_QCFG_OUTPUT_PORT_PARTITION_TYPE_NPAR1_0: case HWRM_FUNC_QCFG_OUTPUT_PORT_PARTITION_TYPE_NPAR1_2: case HWRM_FUNC_QCFG_OUTPUT_PORT_PARTITION_TYPE_NPAR1_5: case HWRM_FUNC_QCFG_OUTPUT_PORT_PARTITION_TYPE_NPAR2_0: softc->port_partition_type = resp->port_partition_type; break; } flags = le16toh(resp->flags); if (flags & (HWRM_FUNC_QCFG_OUTPUT_FLAGS_FW_DCBX_AGENT_ENABLED | HWRM_FUNC_QCFG_OUTPUT_FLAGS_FW_LLDP_AGENT_ENABLED)) { softc->fw_cap |= BNXT_FW_CAP_LLDP_AGENT; if (flags & HWRM_FUNC_QCFG_OUTPUT_FLAGS_FW_DCBX_AGENT_ENABLED) softc->fw_cap |= BNXT_FW_CAP_DCBX_AGENT; } if (BNXT_PF(softc) && (flags & HWRM_FUNC_QCFG_OUTPUT_FLAGS_MULTI_HOST)) softc->flags |= BNXT_FLAG_MULTI_HOST; if (BNXT_PF(softc) && (flags & HWRM_FUNC_QCFG_OUTPUT_FLAGS_MULTI_ROOT)) softc->flags |= BNXT_FLAG_MULTI_ROOT; if (flags & HWRM_FUNC_QCFG_OUTPUT_FLAGS_SECURE_MODE_ENABLED) softc->fw_cap |= BNXT_FW_CAP_SECURE_MODE; if (flags & HWRM_FUNC_QCFG_OUTPUT_FLAGS_RING_MONITOR_ENABLED) softc->fw_cap |= BNXT_FW_CAP_RING_MONITOR; if (flags & HWRM_FUNC_QCFG_OUTPUT_FLAGS_ENABLE_RDMA_SRIOV) softc->fw_cap |= BNXT_FW_CAP_ENABLE_RDMA_SRIOV; if (softc->db_size) goto end; softc->legacy_db_size = le16_to_cpu(resp->legacy_l2_db_size_kb) * 1024; + softc->db_offset = le16toh(resp->legacy_l2_db_size_kb) * 1024; if (BNXT_CHIP_P5(softc)) { if (BNXT_PF(softc)) min_db_offset = DB_PF_OFFSET_P5; else min_db_offset = DB_VF_OFFSET_P5; softc->legacy_db_size = min_db_offset; + softc->db_offset = min_db_offset; } softc->db_size = roundup2(le16_to_cpu(resp->l2_doorbell_bar_size_kb) * 1024, PAGE_SIZE); if (!softc->db_size || softc->db_size > pci_resource_len(softc->pdev, 2) || softc->db_size <= min_db_offset) softc->db_size = pci_resource_len(softc->pdev, 2); end: BNXT_HWRM_UNLOCK(softc); return rc; } int bnxt_hwrm_func_reset(struct bnxt_softc *softc) { struct hwrm_func_reset_input req = {0}; bnxt_hwrm_cmd_hdr_init(softc, &req, HWRM_FUNC_RESET); req.enables = 0; return hwrm_send_message(softc, &req, sizeof(req)); } static void bnxt_hwrm_set_link_common(struct bnxt_softc *softc, struct hwrm_port_phy_cfg_input *req) { struct bnxt_link_info *link_info = &softc->link_info; uint8_t autoneg = softc->link_info.autoneg; uint16_t fw_link_speed = softc->link_info.req_link_speed; if (autoneg & BNXT_AUTONEG_SPEED) { uint8_t phy_type = get_phy_type(softc); if (phy_type == HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_1G_BASET || phy_type == HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_BASET || phy_type == HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_BASETE) { req->auto_mode |= htole32(HWRM_PORT_PHY_CFG_INPUT_AUTO_MODE_SPEED_MASK); if (link_info->advertising) { req->enables |= htole32(HWRM_PORT_PHY_CFG_INPUT_ENABLES_AUTO_LINK_SPEED_MASK); req->auto_link_speed_mask = htole16(link_info->advertising); } } else { req->auto_mode |= HWRM_PORT_PHY_CFG_INPUT_AUTO_MODE_ALL_SPEEDS; } req->enables |= htole32(HWRM_PORT_PHY_CFG_INPUT_ENABLES_AUTO_MODE); req->flags |= htole32(HWRM_PORT_PHY_CFG_INPUT_FLAGS_RESTART_AUTONEG); } else { if (link_info->force_speed2_nrz || link_info->force_pam4_56_speed2 || link_info->force_pam4_112_speed2) { req->force_link_speeds2 = htole16(fw_link_speed); req->enables |= htole32(HWRM_PORT_PHY_CFG_INPUT_ENABLES_FORCE_LINK_SPEEDS2); link_info->force_speed2_nrz = false; link_info->force_pam4_56_speed2 = false; link_info->force_pam4_112_speed2 = false; } else if (link_info->force_pam4_speed) { req->force_pam4_link_speed = htole16(fw_link_speed); req->enables |= htole32(HWRM_PORT_PHY_CFG_INPUT_ENABLES_FORCE_PAM4_LINK_SPEED); link_info->force_pam4_speed = false; } else { req->force_link_speed = htole16(fw_link_speed); } req->flags |= htole32(HWRM_PORT_PHY_CFG_INPUT_FLAGS_FORCE); } /* tell chimp that the setting takes effect immediately */ req->flags |= htole32(HWRM_PORT_PHY_CFG_INPUT_FLAGS_RESET_PHY); } static void bnxt_hwrm_set_pause_common(struct bnxt_softc *softc, struct hwrm_port_phy_cfg_input *req) { struct bnxt_link_info *link_info = &softc->link_info; if (link_info->flow_ctrl.autoneg) { req->auto_pause = HWRM_PORT_PHY_CFG_INPUT_AUTO_PAUSE_AUTONEG_PAUSE; if (link_info->flow_ctrl.rx) req->auto_pause |= HWRM_PORT_PHY_CFG_INPUT_AUTO_PAUSE_RX; if (link_info->flow_ctrl.tx) req->auto_pause |= HWRM_PORT_PHY_CFG_INPUT_AUTO_PAUSE_TX; req->enables |= htole32(HWRM_PORT_PHY_CFG_INPUT_ENABLES_AUTO_PAUSE); } else { if (link_info->flow_ctrl.rx) req->force_pause |= HWRM_PORT_PHY_CFG_INPUT_FORCE_PAUSE_RX; if (link_info->flow_ctrl.tx) req->force_pause |= HWRM_PORT_PHY_CFG_INPUT_FORCE_PAUSE_TX; req->enables |= htole32(HWRM_PORT_PHY_CFG_INPUT_ENABLES_FORCE_PAUSE); req->auto_pause = req->force_pause; req->enables |= htole32(HWRM_PORT_PHY_CFG_INPUT_ENABLES_AUTO_PAUSE); } } /* JFV this needs interface connection */ static void bnxt_hwrm_set_eee(struct bnxt_softc *softc, struct hwrm_port_phy_cfg_input *req) { /* struct ethtool_eee *eee = &softc->eee; */ bool eee_enabled = false; if (eee_enabled) { #if 0 uint16_t eee_speeds; uint32_t flags = HWRM_PORT_PHY_CFG_INPUT_FLAGS_EEE_ENABLE; if (eee->tx_lpi_enabled) flags |= HWRM_PORT_PHY_CFG_INPUT_FLAGS_EEE_TX_LPI; req->flags |= htole32(flags); eee_speeds = bnxt_get_fw_auto_link_speeds(eee->advertised); req->eee_link_speed_mask = htole16(eee_speeds); req->tx_lpi_timer = htole32(eee->tx_lpi_timer); #endif } else { req->flags |= htole32(HWRM_PORT_PHY_CFG_INPUT_FLAGS_EEE_DISABLE); } } int bnxt_hwrm_set_link_setting(struct bnxt_softc *softc, bool set_pause, bool set_eee, bool set_link) { struct hwrm_port_phy_cfg_input req = {0}; int rc; if (softc->flags & BNXT_FLAG_NPAR) return ENOTSUP; bnxt_hwrm_cmd_hdr_init(softc, &req, HWRM_PORT_PHY_CFG); if (set_pause) { bnxt_hwrm_set_pause_common(softc, &req); if (softc->link_info.flow_ctrl.autoneg) set_link = true; } if (set_link) bnxt_hwrm_set_link_common(softc, &req); if (set_eee) bnxt_hwrm_set_eee(softc, &req); BNXT_HWRM_LOCK(softc); rc = _hwrm_send_message(softc, &req, sizeof(req)); if (!rc) { if (set_pause) { /* since changing of 'force pause' setting doesn't * trigger any link change event, the driver needs to * update the current pause result upon successfully i * return of the phy_cfg command */ if (!softc->link_info.flow_ctrl.autoneg) bnxt_report_link(softc); } } BNXT_HWRM_UNLOCK(softc); return rc; } int bnxt_hwrm_vnic_set_hds(struct bnxt_softc *softc, struct bnxt_vnic_info *vnic) { struct hwrm_vnic_plcmodes_cfg_input req = {0}; if (!BNXT_CHIP_P5_PLUS(softc)) return 0; bnxt_hwrm_cmd_hdr_init(softc, &req, HWRM_VNIC_PLCMODES_CFG); req.flags = htole32(HWRM_VNIC_PLCMODES_CFG_INPUT_FLAGS_JUMBO_PLACEMENT); req.vnic_id = htole16(vnic->id); return hwrm_send_message(softc, &req, sizeof(req)); } int bnxt_hwrm_vnic_cfg(struct bnxt_softc *softc, struct bnxt_vnic_info *vnic) { struct hwrm_vnic_cfg_input req = {0}; bnxt_hwrm_cmd_hdr_init(softc, &req, HWRM_VNIC_CFG); if (vnic->flags & BNXT_VNIC_FLAG_DEFAULT) req.flags |= htole32(HWRM_VNIC_CFG_INPUT_FLAGS_DEFAULT); if (vnic->flags & BNXT_VNIC_FLAG_BD_STALL) req.flags |= htole32(HWRM_VNIC_CFG_INPUT_FLAGS_BD_STALL_MODE); if (vnic->flags & BNXT_VNIC_FLAG_VLAN_STRIP) req.flags |= htole32(HWRM_VNIC_CFG_INPUT_FLAGS_VLAN_STRIP_MODE); if (BNXT_CHIP_P5_PLUS (softc)) { req.default_rx_ring_id = htole16(softc->rx_rings[0].phys_id); req.default_cmpl_ring_id = htole16(softc->rx_cp_rings[0].ring.phys_id); req.enables |= htole32(HWRM_VNIC_CFG_INPUT_ENABLES_DEFAULT_RX_RING_ID | HWRM_VNIC_CFG_INPUT_ENABLES_DEFAULT_CMPL_RING_ID); req.vnic_id = htole16(vnic->id); } else { req.enables = htole32(HWRM_VNIC_CFG_INPUT_ENABLES_DFLT_RING_GRP | HWRM_VNIC_CFG_INPUT_ENABLES_RSS_RULE); req.vnic_id = htole16(vnic->id); req.dflt_ring_grp = htole16(vnic->def_ring_grp); } req.rss_rule = htole16(vnic->rss_id); req.cos_rule = htole16(vnic->cos_rule); req.lb_rule = htole16(vnic->lb_rule); req.enables |= htole32(HWRM_VNIC_CFG_INPUT_ENABLES_MRU); req.mru = htole16(vnic->mru); return hwrm_send_message(softc, &req, sizeof(req)); } int bnxt_hwrm_vnic_free(struct bnxt_softc *softc, struct bnxt_vnic_info *vnic) { struct hwrm_vnic_free_input req = {0}; int rc = 0; if (vnic->id == (uint16_t)HWRM_NA_SIGNATURE) return rc; bnxt_hwrm_cmd_hdr_init(softc, &req, HWRM_VNIC_FREE); req.vnic_id = htole32(vnic->id); BNXT_HWRM_LOCK(softc); rc = _hwrm_send_message(softc, &req, sizeof(req)); if (rc) goto fail; fail: BNXT_HWRM_UNLOCK(softc); return (rc); } int bnxt_hwrm_vnic_alloc(struct bnxt_softc *softc, struct bnxt_vnic_info *vnic) { struct hwrm_vnic_alloc_input req = {0}; struct hwrm_vnic_alloc_output *resp = (void *)softc->hwrm_cmd_resp.idi_vaddr; int rc; if (vnic->id != (uint16_t)HWRM_NA_SIGNATURE) { device_printf(softc->dev, "Attempt to re-allocate vnic %04x\n", vnic->id); return EDOOFUS; } bnxt_hwrm_cmd_hdr_init(softc, &req, HWRM_VNIC_ALLOC); if (vnic->flags & BNXT_VNIC_FLAG_DEFAULT) req.flags = htole32(HWRM_VNIC_ALLOC_INPUT_FLAGS_DEFAULT); BNXT_HWRM_LOCK(softc); rc = _hwrm_send_message(softc, &req, sizeof(req)); if (rc) goto fail; vnic->id = le32toh(resp->vnic_id); fail: BNXT_HWRM_UNLOCK(softc); return (rc); } int bnxt_hwrm_vnic_ctx_free(struct bnxt_softc *softc, uint16_t ctx_id) { struct hwrm_vnic_rss_cos_lb_ctx_free_input req = {0}; int rc = 0; if (ctx_id == (uint16_t)HWRM_NA_SIGNATURE) return rc; bnxt_hwrm_cmd_hdr_init(softc, &req, HWRM_VNIC_RSS_COS_LB_CTX_FREE); req.rss_cos_lb_ctx_id = htole16(ctx_id); BNXT_HWRM_LOCK(softc); rc = _hwrm_send_message(softc, &req, sizeof(req)); if (rc) goto fail; fail: BNXT_HWRM_UNLOCK(softc); return rc; } int bnxt_hwrm_vnic_ctx_alloc(struct bnxt_softc *softc, uint16_t *ctx_id) { struct hwrm_vnic_rss_cos_lb_ctx_alloc_input req = {0}; struct hwrm_vnic_rss_cos_lb_ctx_alloc_output *resp = (void *)softc->hwrm_cmd_resp.idi_vaddr; int rc; if (*ctx_id != (uint16_t)HWRM_NA_SIGNATURE) { device_printf(softc->dev, "Attempt to re-allocate vnic ctx %04x\n", *ctx_id); return EDOOFUS; } bnxt_hwrm_cmd_hdr_init(softc, &req, HWRM_VNIC_RSS_COS_LB_CTX_ALLOC); BNXT_HWRM_LOCK(softc); rc = _hwrm_send_message(softc, &req, sizeof(req)); if (rc) goto fail; *ctx_id = le32toh(resp->rss_cos_lb_ctx_id); fail: BNXT_HWRM_UNLOCK(softc); return (rc); } int bnxt_hwrm_ring_grp_alloc(struct bnxt_softc *softc, struct bnxt_grp_info *grp) { struct hwrm_ring_grp_alloc_input req = {0}; struct hwrm_ring_grp_alloc_output *resp; int rc = 0; if (grp->grp_id != (uint16_t)HWRM_NA_SIGNATURE) { device_printf(softc->dev, "Attempt to re-allocate ring group %04x\n", grp->grp_id); return EDOOFUS; } if (BNXT_CHIP_P5_PLUS (softc)) return 0; resp = (void *)softc->hwrm_cmd_resp.idi_vaddr; bnxt_hwrm_cmd_hdr_init(softc, &req, HWRM_RING_GRP_ALLOC); req.cr = htole16(grp->cp_ring_id); req.rr = htole16(grp->rx_ring_id); req.ar = htole16(grp->ag_ring_id); req.sc = htole16(grp->stats_ctx); BNXT_HWRM_LOCK(softc); rc = _hwrm_send_message(softc, &req, sizeof(req)); if (rc) goto fail; grp->grp_id = le32toh(resp->ring_group_id); fail: BNXT_HWRM_UNLOCK(softc); return rc; } int bnxt_hwrm_ring_grp_free(struct bnxt_softc *softc, struct bnxt_grp_info *grp) { struct hwrm_ring_grp_free_input req = {0}; int rc = 0; if (grp->grp_id == (uint16_t)HWRM_NA_SIGNATURE) return 0; if (BNXT_CHIP_P5_PLUS (softc)) return 0; bnxt_hwrm_cmd_hdr_init(softc, &req, HWRM_RING_GRP_FREE); req.ring_group_id = htole32(grp->grp_id); BNXT_HWRM_LOCK(softc); rc = _hwrm_send_message(softc, &req, sizeof(req)); if (rc) goto fail; fail: BNXT_HWRM_UNLOCK(softc); return rc; } int bnxt_hwrm_ring_free(struct bnxt_softc *softc, uint32_t ring_type, struct bnxt_ring *ring, int cmpl_ring_id) { struct hwrm_ring_free_input req = {0}; struct hwrm_ring_free_output *resp; int rc = 0; uint16_t error_code; if (ring->phys_id == (uint16_t)HWRM_NA_SIGNATURE) return 0; resp = (void *)softc->hwrm_cmd_resp.idi_vaddr; bnxt_hwrm_cmd_hdr_init(softc, &req, HWRM_RING_FREE); req.cmpl_ring = htole16(cmpl_ring_id); req.ring_type = ring_type; req.ring_id = htole16(ring->phys_id); BNXT_HWRM_LOCK(softc); rc = _hwrm_send_message(softc, &req, sizeof(req)); error_code = le16toh(resp->error_code); if (rc || error_code) { device_printf(softc->dev, "hwrm_ring_free type %d failed. " "rc:%x err:%x\n", ring_type, rc, error_code); if (!rc) rc = -EIO; } BNXT_HWRM_UNLOCK(softc); return rc; } /* * Ring allocation message to the firmware */ int bnxt_hwrm_ring_alloc(struct bnxt_softc *softc, uint8_t type, struct bnxt_ring *ring) { struct hwrm_ring_alloc_input req = {0}; struct hwrm_ring_alloc_output *resp; uint16_t idx = ring->idx; struct bnxt_cp_ring *cp_ring; int rc; if (ring->phys_id != (uint16_t)HWRM_NA_SIGNATURE) { device_printf(softc->dev, "Attempt to re-allocate ring %04x\n", ring->phys_id); return EDOOFUS; } resp = (void *)softc->hwrm_cmd_resp.idi_vaddr; bnxt_hwrm_cmd_hdr_init(softc, &req, HWRM_RING_ALLOC); req.enables = htole32(0); req.fbo = htole32(0); req.ring_type = type; req.page_tbl_addr = htole64(ring->paddr); req.logical_id = htole16(ring->id); req.length = htole32(ring->ring_size); switch (type) { case HWRM_RING_ALLOC_INPUT_RING_TYPE_TX: cp_ring = &softc->tx_cp_rings[idx]; req.cmpl_ring_id = htole16(cp_ring->ring.phys_id); /* queue_id - what CoS queue the TX ring is associated with */ req.queue_id = htole16(softc->tx_q_info[0].queue_id); req.stat_ctx_id = htole32(cp_ring->stats_ctx_id); req.enables |= htole32( HWRM_RING_ALLOC_INPUT_ENABLES_STAT_CTX_ID_VALID); break; case HWRM_RING_ALLOC_INPUT_RING_TYPE_RX: if (!BNXT_CHIP_P5_PLUS(softc)) break; cp_ring = &softc->rx_cp_rings[idx]; req.stat_ctx_id = htole32(cp_ring->stats_ctx_id); req.rx_buf_size = htole16(softc->rx_buf_size); req.enables |= htole32( HWRM_RING_ALLOC_INPUT_ENABLES_RX_BUF_SIZE_VALID | HWRM_RING_ALLOC_INPUT_ENABLES_STAT_CTX_ID_VALID); break; case HWRM_RING_ALLOC_INPUT_RING_TYPE_RX_AGG: if (!BNXT_CHIP_P5_PLUS(softc)) { req.ring_type = HWRM_RING_ALLOC_INPUT_RING_TYPE_RX; break; } cp_ring = &softc->rx_cp_rings[idx]; req.rx_ring_id = htole16(softc->rx_rings[idx].phys_id); req.stat_ctx_id = htole32(cp_ring->stats_ctx_id); req.rx_buf_size = htole16(softc->rx_buf_size); req.enables |= htole32( HWRM_RING_ALLOC_INPUT_ENABLES_RX_RING_ID_VALID | HWRM_RING_ALLOC_INPUT_ENABLES_RX_BUF_SIZE_VALID | HWRM_RING_ALLOC_INPUT_ENABLES_STAT_CTX_ID_VALID); break; case HWRM_RING_ALLOC_INPUT_RING_TYPE_L2_CMPL: if (!BNXT_CHIP_P5_PLUS(softc)) { req.int_mode = HWRM_RING_ALLOC_INPUT_INT_MODE_MSIX; break; } req.cq_handle = htole64(ring->id); req.nq_ring_id = htole16(softc->nq_rings[idx].ring.phys_id); req.enables |= htole32( HWRM_RING_ALLOC_INPUT_ENABLES_NQ_RING_ID_VALID); break; case HWRM_RING_ALLOC_INPUT_RING_TYPE_NQ: req.int_mode = HWRM_RING_ALLOC_INPUT_INT_MODE_MSIX; break; default: device_printf(softc->dev, "hwrm alloc invalid ring type %d\n", type); return -1; } BNXT_HWRM_LOCK(softc); rc = _hwrm_send_message(softc, &req, sizeof(req)); if (rc) goto fail; ring->phys_id = le16toh(resp->ring_id); fail: BNXT_HWRM_UNLOCK(softc); return rc; } int bnxt_hwrm_stat_ctx_free(struct bnxt_softc *softc, struct bnxt_cp_ring *cpr) { struct hwrm_stat_ctx_free_input req = {0}; int rc = 0; if (cpr->stats_ctx_id == HWRM_NA_SIGNATURE) return rc; bnxt_hwrm_cmd_hdr_init(softc, &req, HWRM_STAT_CTX_FREE); req.stat_ctx_id = htole16(cpr->stats_ctx_id); BNXT_HWRM_LOCK(softc); rc = _hwrm_send_message(softc, &req, sizeof(req)); if (rc) goto fail; fail: BNXT_HWRM_UNLOCK(softc); return rc; } int bnxt_hwrm_stat_ctx_alloc(struct bnxt_softc *softc, struct bnxt_cp_ring *cpr, uint64_t paddr) { struct hwrm_stat_ctx_alloc_input req = {0}; struct hwrm_stat_ctx_alloc_output *resp; int rc = 0; if (cpr->stats_ctx_id != HWRM_NA_SIGNATURE) { device_printf(softc->dev, "Attempt to re-allocate stats ctx %08x\n", cpr->stats_ctx_id); return EDOOFUS; } resp = (void *)softc->hwrm_cmd_resp.idi_vaddr; bnxt_hwrm_cmd_hdr_init(softc, &req, HWRM_STAT_CTX_ALLOC); req.update_period_ms = htole32(1000); req.stats_dma_addr = htole64(paddr); if (BNXT_CHIP_P7(softc)) req.stats_dma_length = htole16(sizeof(struct ctx_hw_stats_ext)); else if (BNXT_CHIP_P5(softc)) req.stats_dma_length = htole16(sizeof(struct ctx_hw_stats_ext) - 8); else req.stats_dma_length = htole16(sizeof(struct ctx_hw_stats)); BNXT_HWRM_LOCK(softc); rc = _hwrm_send_message(softc, &req, sizeof(req)); if (rc) goto fail; cpr->stats_ctx_id = le32toh(resp->stat_ctx_id); fail: BNXT_HWRM_UNLOCK(softc); return rc; } int bnxt_hwrm_port_qstats(struct bnxt_softc *softc) { struct hwrm_port_qstats_input req = {0}; int rc = 0; bnxt_hwrm_cmd_hdr_init(softc, &req, HWRM_PORT_QSTATS); req.port_id = htole16(softc->pf.port_id); req.rx_stat_host_addr = htole64(softc->hw_rx_port_stats.idi_paddr); req.tx_stat_host_addr = htole64(softc->hw_tx_port_stats.idi_paddr); BNXT_HWRM_LOCK(softc); rc = _hwrm_send_message(softc, &req, sizeof(req)); BNXT_HWRM_UNLOCK(softc); return rc; } static int bnxt_hwrm_pri2cos_idx(struct bnxt_softc *softc, uint32_t path_dir) { struct hwrm_queue_pri2cos_qcfg_input req = {0}; struct hwrm_queue_pri2cos_qcfg_output *resp; uint8_t *pri2cos_idx, *q_ids, max_q; int rc, i, j; uint8_t *pri2cos; bnxt_hwrm_cmd_hdr_init(softc, &req, HWRM_QUEUE_PRI2COS_QCFG); resp = (void *)softc->hwrm_cmd_resp.idi_vaddr; req.flags = htole32(HWRM_QUEUE_PRI2COS_QCFG_INPUT_FLAGS_IVLAN | path_dir); rc = hwrm_send_message(softc, &req, sizeof(req)); if (rc) return rc; if (path_dir == HWRM_QUEUE_PRI2COS_QCFG_INPUT_FLAGS_PATH_TX) { pri2cos_idx = softc->tx_pri2cos_idx; q_ids = softc->tx_q_ids; max_q = softc->tx_max_q; } else { pri2cos_idx = softc->rx_pri2cos_idx; q_ids = softc->rx_q_ids; max_q = softc->rx_max_q; } pri2cos = &resp->pri0_cos_queue_id; for (i = 0; i < BNXT_MAX_QUEUE; i++) { uint8_t queue_id = pri2cos[i]; uint8_t queue_idx; /* Per port queue IDs start from 0, 10, 20, etc */ queue_idx = queue_id % 10; if (queue_idx > BNXT_MAX_QUEUE) { softc->pri2cos_valid = false; rc = -EINVAL; return rc; } for (j = 0; j < max_q; j++) { if (q_ids[j] == queue_id) pri2cos_idx[i] = queue_idx; } } softc->pri2cos_valid = true; return rc; } int bnxt_hwrm_port_qstats_ext(struct bnxt_softc *softc) { struct hwrm_port_qstats_ext_input req = {0}; struct hwrm_port_qstats_ext_output *resp; int rc = 0, i; uint32_t tx_stat_size; bnxt_hwrm_cmd_hdr_init(softc, &req, HWRM_PORT_QSTATS_EXT); resp = (void *)softc->hwrm_cmd_resp.idi_vaddr; tx_stat_size = sizeof(struct tx_port_stats_ext); req.port_id = htole16(softc->pf.port_id); req.tx_stat_size = htole16(tx_stat_size); req.rx_stat_size = htole16(sizeof(struct rx_port_stats_ext)); req.rx_stat_host_addr = htole64(softc->hw_rx_port_stats_ext.idi_paddr); req.tx_stat_host_addr = htole64(softc->hw_tx_port_stats_ext.idi_paddr); rc = hwrm_send_message(softc, &req, sizeof(req)); if (!rc) { softc->fw_rx_stats_ext_size = le16toh(resp->rx_stat_size) / 8; if (BNXT_FW_MAJ(softc) < 220 && !BNXT_CHIP_P7(softc) && softc->fw_rx_stats_ext_size > BNXT_RX_STATS_EXT_NUM_LEGACY) softc->fw_rx_stats_ext_size = BNXT_RX_STATS_EXT_NUM_LEGACY; softc->fw_tx_stats_ext_size = tx_stat_size ? le16toh(resp->tx_stat_size) / 8 : 0; } else { softc->fw_rx_stats_ext_size = 0; softc->fw_tx_stats_ext_size = 0; } if (softc->fw_tx_stats_ext_size <= offsetof(struct tx_port_stats_ext, pfc_pri0_tx_duration_us) / 8) { softc->pri2cos_valid = false; return rc; } rc = bnxt_hwrm_pri2cos_idx(softc, HWRM_QUEUE_PRI2COS_QCFG_INPUT_FLAGS_PATH_TX); if (rc) return rc; if (softc->is_asym_q) { rc = bnxt_hwrm_pri2cos_idx(softc, HWRM_QUEUE_PRI2COS_QCFG_INPUT_FLAGS_PATH_RX); if (rc) return rc; } else { memcpy(softc->rx_pri2cos_idx, softc->tx_pri2cos_idx, sizeof(softc->rx_pri2cos_idx)); } u64 *rx_port_stats_ext = (u64 *)softc->hw_rx_port_stats_ext.idi_vaddr; u64 *tx_port_stats_ext = (u64 *)softc->hw_tx_port_stats_ext.idi_vaddr; if (softc->pri2cos_valid) { for (i = 0; i < 8; i++) { long n = bnxt_rx_bytes_pri_arr_base_off[i] + softc->rx_pri2cos_idx[i]; softc->rx_bytes_pri[i] = *(rx_port_stats_ext + n); } for (i = 0; i < 8; i++) { long n = bnxt_rx_pkts_pri_arr_base_off[i] + softc->rx_pri2cos_idx[i]; softc->rx_packets_pri[i] = *(rx_port_stats_ext + n); } for (i = 0; i < 8; i++) { long n = bnxt_tx_bytes_pri_arr_base_off[i] + softc->tx_pri2cos_idx[i]; softc->tx_bytes_pri[i] = *(tx_port_stats_ext + n); } for (i = 0; i < 8; i++) { long n = bnxt_tx_pkts_pri_arr_base_off[i] + softc->tx_pri2cos_idx[i]; softc->tx_packets_pri[i] = *(tx_port_stats_ext + n); } } return rc; } int bnxt_hwrm_cfa_l2_set_rx_mask(struct bnxt_softc *softc, struct bnxt_vnic_info *vnic) { struct hwrm_cfa_l2_set_rx_mask_input req = {0}; uint32_t mask = vnic->rx_mask; bnxt_hwrm_cmd_hdr_init(softc, &req, HWRM_CFA_L2_SET_RX_MASK); req.vnic_id = htole32(vnic->id); req.mask = htole32(mask); req.mc_tbl_addr = htole64(vnic->mc_list.idi_paddr); req.num_mc_entries = htole32(vnic->mc_list_count); return hwrm_send_message(softc, &req, sizeof(req)); } int bnxt_hwrm_l2_filter_free(struct bnxt_softc *softc, uint64_t filter_id) { struct hwrm_cfa_l2_filter_free_input req = {0}; int rc = 0; if (filter_id == -1) return rc; bnxt_hwrm_cmd_hdr_init(softc, &req, HWRM_CFA_L2_FILTER_FREE); req.l2_filter_id = htole64(filter_id); BNXT_HWRM_LOCK(softc); rc = _hwrm_send_message(softc, &req, sizeof(req)); if (rc) goto fail; fail: BNXT_HWRM_UNLOCK(softc); return (rc); } int bnxt_hwrm_free_filter(struct bnxt_softc *softc) { struct bnxt_vnic_info *vnic = &softc->vnic_info; struct bnxt_vlan_tag *tag; int rc = 0; rc = bnxt_hwrm_l2_filter_free(softc, softc->vnic_info.filter_id); if (rc) goto end; SLIST_FOREACH(tag, &vnic->vlan_tags, next) { rc = bnxt_hwrm_l2_filter_free(softc, tag->filter_id); if (rc) goto end; tag->filter_id = -1; } end: return rc; } int bnxt_hwrm_l2_filter_alloc(struct bnxt_softc *softc, uint16_t vlan_tag, uint64_t *filter_id) { struct hwrm_cfa_l2_filter_alloc_input req = {0}; struct hwrm_cfa_l2_filter_alloc_output *resp; struct bnxt_vnic_info *vnic = &softc->vnic_info; uint32_t enables = 0; int rc = 0; if (*filter_id != -1) { device_printf(softc->dev, "Attempt to re-allocate l2 ctx " "filter (fid: 0x%jx)\n", (uintmax_t)*filter_id); return EDOOFUS; } resp = (void *)softc->hwrm_cmd_resp.idi_vaddr; bnxt_hwrm_cmd_hdr_init(softc, &req, HWRM_CFA_L2_FILTER_ALLOC); req.flags = htole32(HWRM_CFA_L2_FILTER_ALLOC_INPUT_FLAGS_PATH_RX); enables = HWRM_CFA_L2_FILTER_ALLOC_INPUT_ENABLES_L2_ADDR | HWRM_CFA_L2_FILTER_ALLOC_INPUT_ENABLES_L2_ADDR_MASK | HWRM_CFA_L2_FILTER_ALLOC_INPUT_ENABLES_DST_ID; if (vlan_tag != 0xffff) { enables |= HWRM_CFA_L2_FILTER_ALLOC_INPUT_ENABLES_L2_IVLAN | HWRM_CFA_L2_FILTER_ALLOC_INPUT_ENABLES_L2_IVLAN_MASK | HWRM_CFA_L2_FILTER_ALLOC_INPUT_ENABLES_NUM_VLANS; req.l2_ivlan_mask = 0xffff; req.l2_ivlan = vlan_tag; req.num_vlans = 1; } req.enables = htole32(enables); req.dst_id = htole16(vnic->id); memcpy(req.l2_addr, if_getlladdr(iflib_get_ifp(softc->ctx)), ETHER_ADDR_LEN); memset(&req.l2_addr_mask, 0xff, sizeof(req.l2_addr_mask)); BNXT_HWRM_LOCK(softc); rc = _hwrm_send_message(softc, &req, sizeof(req)); if (rc) goto fail; *filter_id = le64toh(resp->l2_filter_id); fail: BNXT_HWRM_UNLOCK(softc); return (rc); } int bnxt_hwrm_set_filter(struct bnxt_softc *softc) { struct bnxt_vnic_info *vnic = &softc->vnic_info; struct bnxt_vlan_tag *tag; int rc = 0; rc = bnxt_hwrm_l2_filter_alloc(softc, 0xffff, &vnic->filter_id); if (rc) goto end; SLIST_FOREACH(tag, &vnic->vlan_tags, next) { rc = bnxt_hwrm_l2_filter_alloc(softc, tag->tag, &tag->filter_id); if (rc) goto end; } end: return rc; } int bnxt_hwrm_rss_cfg(struct bnxt_softc *softc, struct bnxt_vnic_info *vnic, uint32_t hash_type) { struct hwrm_vnic_rss_cfg_input req = {0}; bnxt_hwrm_cmd_hdr_init(softc, &req, HWRM_VNIC_RSS_CFG); if (BNXT_CHIP_P7(softc)) req.flags |= HWRM_VNIC_RSS_CFG_INPUT_FLAGS_IPSEC_HASH_TYPE_CFG_SUPPORT; req.hash_type = htole32(hash_type); req.ring_grp_tbl_addr = htole64(vnic->rss_grp_tbl.idi_paddr); req.hash_key_tbl_addr = htole64(vnic->rss_hash_key_tbl.idi_paddr); req.rss_ctx_idx = htole16(vnic->rss_id); req.hash_mode_flags = HWRM_FUNC_SPD_CFG_INPUT_HASH_MODE_FLAGS_DEFAULT; if (BNXT_CHIP_P5_PLUS(softc)) { req.vnic_id = htole16(vnic->id); req.ring_table_pair_index = 0x0; } return hwrm_send_message(softc, &req, sizeof(req)); } int bnxt_hwrm_reserve_pf_rings(struct bnxt_softc *softc) { struct hwrm_func_cfg_input req = {0}; bnxt_hwrm_cmd_hdr_init(softc, &req, HWRM_FUNC_CFG); req.fid = htole16(0xffff); req.enables |= htole32(HWRM_FUNC_CFG_INPUT_ENABLES_NUM_RSSCOS_CTXS); req.enables |= htole32(HWRM_FUNC_CFG_INPUT_ENABLES_NUM_CMPL_RINGS); req.enables |= htole32(HWRM_FUNC_CFG_INPUT_ENABLES_NUM_TX_RINGS); req.enables |= htole32(HWRM_FUNC_CFG_INPUT_ENABLES_NUM_RX_RINGS); req.enables |= htole32(HWRM_FUNC_CFG_INPUT_ENABLES_NUM_VNICS); req.enables |= htole32(HWRM_FUNC_CFG_INPUT_ENABLES_NUM_MSIX); req.enables |= htole32(HWRM_FUNC_CFG_INPUT_ENABLES_NUM_STAT_CTXS); req.num_msix = htole16(BNXT_MAX_NUM_QUEUES); req.num_rsscos_ctxs = htole16(0x8); req.num_cmpl_rings = htole16(BNXT_MAX_NUM_QUEUES * 2); req.num_tx_rings = htole16(BNXT_MAX_NUM_QUEUES); req.num_rx_rings = htole16(BNXT_MAX_NUM_QUEUES); req.num_vnics = htole16(BNXT_MAX_NUM_QUEUES); req.num_stat_ctxs = htole16(BNXT_MAX_NUM_QUEUES * 2); return hwrm_send_message(softc, &req, sizeof(req)); } int bnxt_cfg_async_cr(struct bnxt_softc *softc) { int rc = 0; struct hwrm_func_cfg_input req = {0}; if (!BNXT_PF(softc)) return 0; bnxt_hwrm_cmd_hdr_init(softc, &req, HWRM_FUNC_CFG); req.fid = htole16(0xffff); req.enables = htole32(HWRM_FUNC_CFG_INPUT_ENABLES_ASYNC_EVENT_CR); if (BNXT_CHIP_P5_PLUS(softc)) req.async_event_cr = htole16(softc->nq_rings[0].ring.phys_id); else req.async_event_cr = htole16(softc->def_cp_ring.ring.phys_id); rc = hwrm_send_message(softc, &req, sizeof(req)); return rc; } void bnxt_validate_hw_lro_settings(struct bnxt_softc *softc) { softc->hw_lro.enable = min(softc->hw_lro.enable, 1); softc->hw_lro.is_mode_gro = min(softc->hw_lro.is_mode_gro, 1); softc->hw_lro.max_agg_segs = min(softc->hw_lro.max_agg_segs, HWRM_VNIC_TPA_CFG_INPUT_MAX_AGG_SEGS_MAX); softc->hw_lro.max_aggs = min(softc->hw_lro.max_aggs, HWRM_VNIC_TPA_CFG_INPUT_MAX_AGGS_MAX); softc->hw_lro.min_agg_len = min(softc->hw_lro.min_agg_len, BNXT_MAX_MTU); } int bnxt_hwrm_vnic_tpa_cfg(struct bnxt_softc *softc) { struct hwrm_vnic_tpa_cfg_input req = {0}; uint32_t flags; if (softc->vnic_info.id == (uint16_t) HWRM_NA_SIGNATURE) { return 0; } if (!(softc->flags & BNXT_FLAG_TPA)) return 0; bnxt_hwrm_cmd_hdr_init(softc, &req, HWRM_VNIC_TPA_CFG); if (softc->hw_lro.enable) { flags = HWRM_VNIC_TPA_CFG_INPUT_FLAGS_TPA | HWRM_VNIC_TPA_CFG_INPUT_FLAGS_ENCAP_TPA | HWRM_VNIC_TPA_CFG_INPUT_FLAGS_AGG_WITH_ECN | HWRM_VNIC_TPA_CFG_INPUT_FLAGS_AGG_WITH_SAME_GRE_SEQ; if (softc->hw_lro.is_mode_gro) flags |= HWRM_VNIC_TPA_CFG_INPUT_FLAGS_GRO; else flags |= HWRM_VNIC_TPA_CFG_INPUT_FLAGS_RSC_WND_UPDATE; req.flags = htole32(flags); req.enables = htole32(HWRM_VNIC_TPA_CFG_INPUT_ENABLES_MAX_AGG_SEGS | HWRM_VNIC_TPA_CFG_INPUT_ENABLES_MAX_AGGS | HWRM_VNIC_TPA_CFG_INPUT_ENABLES_MIN_AGG_LEN); req.max_agg_segs = htole16(softc->hw_lro.max_agg_segs); req.max_aggs = htole16(softc->hw_lro.max_aggs); req.min_agg_len = htole32(softc->hw_lro.min_agg_len); } req.vnic_id = htole16(softc->vnic_info.id); return hwrm_send_message(softc, &req, sizeof(req)); } int bnxt_hwrm_nvm_find_dir_entry(struct bnxt_softc *softc, uint16_t type, uint16_t *ordinal, uint16_t ext, uint16_t *index, bool use_index, uint8_t search_opt, uint32_t *data_length, uint32_t *item_length, uint32_t *fw_ver) { struct hwrm_nvm_find_dir_entry_input req = {0}; struct hwrm_nvm_find_dir_entry_output *resp = (void *)softc->hwrm_cmd_resp.idi_vaddr; int rc = 0; uint32_t old_timeo; MPASS(ordinal); bnxt_hwrm_cmd_hdr_init(softc, &req, HWRM_NVM_FIND_DIR_ENTRY); if (use_index) { req.enables = htole32( HWRM_NVM_FIND_DIR_ENTRY_INPUT_ENABLES_DIR_IDX_VALID); req.dir_idx = htole16(*index); } req.dir_type = htole16(type); req.dir_ordinal = htole16(*ordinal); req.dir_ext = htole16(ext); req.opt_ordinal = search_opt; BNXT_HWRM_LOCK(softc); old_timeo = softc->hwrm_cmd_timeo; softc->hwrm_cmd_timeo = BNXT_NVM_TIMEO; rc = _hwrm_send_message(softc, &req, sizeof(req)); softc->hwrm_cmd_timeo = old_timeo; if (rc) goto exit; if (item_length) *item_length = le32toh(resp->dir_item_length); if (data_length) *data_length = le32toh(resp->dir_data_length); if (fw_ver) *fw_ver = le32toh(resp->fw_ver); *ordinal = le16toh(resp->dir_ordinal); if (index) *index = le16toh(resp->dir_idx); exit: BNXT_HWRM_UNLOCK(softc); return (rc); } int bnxt_hwrm_nvm_read(struct bnxt_softc *softc, uint16_t index, uint32_t offset, uint32_t length, struct iflib_dma_info *data) { struct hwrm_nvm_read_input req = {0}; int rc; uint32_t old_timeo; if (length > data->idi_size) { rc = EINVAL; goto exit; } bnxt_hwrm_cmd_hdr_init(softc, &req, HWRM_NVM_READ); req.host_dest_addr = htole64(data->idi_paddr); req.dir_idx = htole16(index); req.offset = htole32(offset); req.len = htole32(length); BNXT_HWRM_LOCK(softc); old_timeo = softc->hwrm_cmd_timeo; softc->hwrm_cmd_timeo = BNXT_NVM_TIMEO; rc = _hwrm_send_message(softc, &req, sizeof(req)); softc->hwrm_cmd_timeo = old_timeo; BNXT_HWRM_UNLOCK(softc); if (rc) goto exit; bus_dmamap_sync(data->idi_tag, data->idi_map, BUS_DMASYNC_POSTREAD); goto exit; exit: return rc; } int bnxt_hwrm_nvm_modify(struct bnxt_softc *softc, uint16_t index, uint32_t offset, void *data, bool cpyin, uint32_t length) { struct hwrm_nvm_modify_input req = {0}; struct iflib_dma_info dma_data; int rc; uint32_t old_timeo; if (length == 0 || !data) return EINVAL; rc = iflib_dma_alloc(softc->ctx, length, &dma_data, BUS_DMA_NOWAIT); if (rc) return ENOMEM; if (cpyin) { rc = copyin(data, dma_data.idi_vaddr, length); if (rc) goto exit; } else memcpy(dma_data.idi_vaddr, data, length); bus_dmamap_sync(dma_data.idi_tag, dma_data.idi_map, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); bnxt_hwrm_cmd_hdr_init(softc, &req, HWRM_NVM_MODIFY); req.host_src_addr = htole64(dma_data.idi_paddr); req.dir_idx = htole16(index); req.offset = htole32(offset); req.len = htole32(length); BNXT_HWRM_LOCK(softc); old_timeo = softc->hwrm_cmd_timeo; softc->hwrm_cmd_timeo = BNXT_NVM_TIMEO; rc = _hwrm_send_message(softc, &req, sizeof(req)); softc->hwrm_cmd_timeo = old_timeo; BNXT_HWRM_UNLOCK(softc); exit: iflib_dma_free(&dma_data); return rc; } int bnxt_hwrm_fw_reset(struct bnxt_softc *softc, uint8_t processor, uint8_t *selfreset) { struct hwrm_fw_reset_input req = {0}; struct hwrm_fw_reset_output *resp = (void *)softc->hwrm_cmd_resp.idi_vaddr; int rc; MPASS(selfreset); bnxt_hwrm_cmd_hdr_init(softc, &req, HWRM_FW_RESET); req.embedded_proc_type = processor; req.selfrst_status = *selfreset; BNXT_HWRM_LOCK(softc); rc = _hwrm_send_message(softc, &req, sizeof(req)); if (rc) goto exit; *selfreset = resp->selfrst_status; exit: BNXT_HWRM_UNLOCK(softc); return rc; } int bnxt_hwrm_fw_qstatus(struct bnxt_softc *softc, uint8_t type, uint8_t *selfreset) { struct hwrm_fw_qstatus_input req = {0}; struct hwrm_fw_qstatus_output *resp = (void *)softc->hwrm_cmd_resp.idi_vaddr; int rc; MPASS(selfreset); bnxt_hwrm_cmd_hdr_init(softc, &req, HWRM_FW_QSTATUS); req.embedded_proc_type = type; BNXT_HWRM_LOCK(softc); rc = _hwrm_send_message(softc, &req, sizeof(req)); if (rc) goto exit; *selfreset = resp->selfrst_status; exit: BNXT_HWRM_UNLOCK(softc); return rc; } int bnxt_hwrm_nvm_write(struct bnxt_softc *softc, void *data, bool cpyin, uint16_t type, uint16_t ordinal, uint16_t ext, uint16_t attr, uint16_t option, uint32_t data_length, bool keep, uint32_t *item_length, uint16_t *index) { struct hwrm_nvm_write_input req = {0}; struct hwrm_nvm_write_output *resp = (void *)softc->hwrm_cmd_resp.idi_vaddr; struct iflib_dma_info dma_data; int rc; uint32_t old_timeo; if (data_length) { rc = iflib_dma_alloc(softc->ctx, data_length, &dma_data, BUS_DMA_NOWAIT); if (rc) return ENOMEM; if (cpyin) { rc = copyin(data, dma_data.idi_vaddr, data_length); if (rc) goto early_exit; } else memcpy(dma_data.idi_vaddr, data, data_length); bus_dmamap_sync(dma_data.idi_tag, dma_data.idi_map, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); } else dma_data.idi_paddr = 0; bnxt_hwrm_cmd_hdr_init(softc, &req, HWRM_NVM_WRITE); req.host_src_addr = htole64(dma_data.idi_paddr); req.dir_type = htole16(type); req.dir_ordinal = htole16(ordinal); req.dir_ext = htole16(ext); req.dir_attr = htole16(attr); req.dir_data_length = htole32(data_length); req.option = htole16(option); if (keep) { req.flags = htole16(HWRM_NVM_WRITE_INPUT_FLAGS_KEEP_ORIG_ACTIVE_IMG); } if (item_length) req.dir_item_length = htole32(*item_length); BNXT_HWRM_LOCK(softc); old_timeo = softc->hwrm_cmd_timeo; softc->hwrm_cmd_timeo = BNXT_NVM_TIMEO; rc = _hwrm_send_message(softc, &req, sizeof(req)); softc->hwrm_cmd_timeo = old_timeo; if (rc) goto exit; if (item_length) *item_length = le32toh(resp->dir_item_length); if (index) *index = le16toh(resp->dir_idx); exit: BNXT_HWRM_UNLOCK(softc); early_exit: if (data_length) iflib_dma_free(&dma_data); return rc; } int bnxt_hwrm_nvm_erase_dir_entry(struct bnxt_softc *softc, uint16_t index) { struct hwrm_nvm_erase_dir_entry_input req = {0}; uint32_t old_timeo; int rc; bnxt_hwrm_cmd_hdr_init(softc, &req, HWRM_NVM_ERASE_DIR_ENTRY); req.dir_idx = htole16(index); BNXT_HWRM_LOCK(softc); old_timeo = softc->hwrm_cmd_timeo; softc->hwrm_cmd_timeo = BNXT_NVM_TIMEO; rc = _hwrm_send_message(softc, &req, sizeof(req)); softc->hwrm_cmd_timeo = old_timeo; BNXT_HWRM_UNLOCK(softc); return rc; } int bnxt_hwrm_nvm_get_dir_info(struct bnxt_softc *softc, uint32_t *entries, uint32_t *entry_length) { struct hwrm_nvm_get_dir_info_input req = {0}; struct hwrm_nvm_get_dir_info_output *resp = (void *)softc->hwrm_cmd_resp.idi_vaddr; int rc; uint32_t old_timeo; bnxt_hwrm_cmd_hdr_init(softc, &req, HWRM_NVM_GET_DIR_INFO); BNXT_HWRM_LOCK(softc); old_timeo = softc->hwrm_cmd_timeo; softc->hwrm_cmd_timeo = BNXT_NVM_TIMEO; rc = _hwrm_send_message(softc, &req, sizeof(req)); softc->hwrm_cmd_timeo = old_timeo; if (rc) goto exit; if (entries) *entries = le32toh(resp->entries); if (entry_length) *entry_length = le32toh(resp->entry_length); exit: BNXT_HWRM_UNLOCK(softc); return rc; } int bnxt_hwrm_nvm_get_dir_entries(struct bnxt_softc *softc, uint32_t *entries, uint32_t *entry_length, struct iflib_dma_info *dma_data) { struct hwrm_nvm_get_dir_entries_input req = {0}; uint32_t ent; uint32_t ent_len; int rc; uint32_t old_timeo; if (!entries) entries = &ent; if (!entry_length) entry_length = &ent_len; rc = bnxt_hwrm_nvm_get_dir_info(softc, entries, entry_length); if (rc) goto exit; if (*entries * *entry_length > dma_data->idi_size) { rc = EINVAL; goto exit; } /* * TODO: There's a race condition here that could blow up DMA memory... * we need to allocate the max size, not the currently in use * size. The command should totally have a max size here. */ bnxt_hwrm_cmd_hdr_init(softc, &req, HWRM_NVM_GET_DIR_ENTRIES); req.host_dest_addr = htole64(dma_data->idi_paddr); BNXT_HWRM_LOCK(softc); old_timeo = softc->hwrm_cmd_timeo; softc->hwrm_cmd_timeo = BNXT_NVM_TIMEO; rc = _hwrm_send_message(softc, &req, sizeof(req)); softc->hwrm_cmd_timeo = old_timeo; BNXT_HWRM_UNLOCK(softc); if (rc) goto exit; bus_dmamap_sync(dma_data->idi_tag, dma_data->idi_map, BUS_DMASYNC_POSTWRITE); exit: return rc; } int bnxt_hwrm_nvm_get_dev_info(struct bnxt_softc *softc, uint16_t *mfg_id, uint16_t *device_id, uint32_t *sector_size, uint32_t *nvram_size, uint32_t *reserved_size, uint32_t *available_size) { struct hwrm_nvm_get_dev_info_input req = {0}; struct hwrm_nvm_get_dev_info_output *resp = (void *)softc->hwrm_cmd_resp.idi_vaddr; int rc; uint32_t old_timeo; bnxt_hwrm_cmd_hdr_init(softc, &req, HWRM_NVM_GET_DEV_INFO); BNXT_HWRM_LOCK(softc); old_timeo = softc->hwrm_cmd_timeo; softc->hwrm_cmd_timeo = BNXT_NVM_TIMEO; rc = _hwrm_send_message(softc, &req, sizeof(req)); softc->hwrm_cmd_timeo = old_timeo; if (rc) goto exit; if (mfg_id) *mfg_id = le16toh(resp->manufacturer_id); if (device_id) *device_id = le16toh(resp->device_id); if (sector_size) *sector_size = le32toh(resp->sector_size); if (nvram_size) *nvram_size = le32toh(resp->nvram_size); if (reserved_size) *reserved_size = le32toh(resp->reserved_size); if (available_size) *available_size = le32toh(resp->available_size); exit: BNXT_HWRM_UNLOCK(softc); return rc; } int bnxt_hwrm_nvm_install_update(struct bnxt_softc *softc, uint32_t install_type, uint64_t *installed_items, uint8_t *result, uint8_t *problem_item, uint8_t *reset_required) { struct hwrm_nvm_install_update_input req = {0}; struct hwrm_nvm_install_update_output *resp = (void *)softc->hwrm_cmd_resp.idi_vaddr; int rc; uint32_t old_timeo; bnxt_hwrm_cmd_hdr_init(softc, &req, HWRM_NVM_INSTALL_UPDATE); req.install_type = htole32(install_type); BNXT_HWRM_LOCK(softc); old_timeo = softc->hwrm_cmd_timeo; softc->hwrm_cmd_timeo = BNXT_NVM_TIMEO; rc = _hwrm_send_message(softc, &req, sizeof(req)); softc->hwrm_cmd_timeo = old_timeo; if (rc) goto exit; if (installed_items) *installed_items = le32toh(resp->installed_items); if (result) *result = resp->result; if (problem_item) *problem_item = resp->problem_item; if (reset_required) *reset_required = resp->reset_required; exit: BNXT_HWRM_UNLOCK(softc); return rc; } int bnxt_hwrm_nvm_verify_update(struct bnxt_softc *softc, uint16_t type, uint16_t ordinal, uint16_t ext) { struct hwrm_nvm_verify_update_input req = {0}; uint32_t old_timeo; int rc; bnxt_hwrm_cmd_hdr_init(softc, &req, HWRM_NVM_VERIFY_UPDATE); req.dir_type = htole16(type); req.dir_ordinal = htole16(ordinal); req.dir_ext = htole16(ext); BNXT_HWRM_LOCK(softc); old_timeo = softc->hwrm_cmd_timeo; softc->hwrm_cmd_timeo = BNXT_NVM_TIMEO; rc = _hwrm_send_message(softc, &req, sizeof(req)); softc->hwrm_cmd_timeo = old_timeo; BNXT_HWRM_UNLOCK(softc); return rc; } int bnxt_hwrm_fw_get_time(struct bnxt_softc *softc, uint16_t *year, uint8_t *month, uint8_t *day, uint8_t *hour, uint8_t *minute, uint8_t *second, uint16_t *millisecond, uint16_t *zone) { struct hwrm_fw_get_time_input req = {0}; struct hwrm_fw_get_time_output *resp = (void *)softc->hwrm_cmd_resp.idi_vaddr; int rc; bnxt_hwrm_cmd_hdr_init(softc, &req, HWRM_FW_GET_TIME); BNXT_HWRM_LOCK(softc); rc = _hwrm_send_message(softc, &req, sizeof(req)); if (rc) goto exit; if (year) *year = le16toh(resp->year); if (month) *month = resp->month; if (day) *day = resp->day; if (hour) *hour = resp->hour; if (minute) *minute = resp->minute; if (second) *second = resp->second; if (millisecond) *millisecond = le16toh(resp->millisecond); if (zone) *zone = le16toh(resp->zone); exit: BNXT_HWRM_UNLOCK(softc); return rc; } int bnxt_hwrm_fw_set_time(struct bnxt_softc *softc, uint16_t year, uint8_t month, uint8_t day, uint8_t hour, uint8_t minute, uint8_t second, uint16_t millisecond, uint16_t zone) { struct hwrm_fw_set_time_input req = {0}; bnxt_hwrm_cmd_hdr_init(softc, &req, HWRM_FW_SET_TIME); req.year = htole16(year); req.month = month; req.day = day; req.hour = hour; req.minute = minute; req.second = second; req.millisecond = htole16(millisecond); req.zone = htole16(zone); return hwrm_send_message(softc, &req, sizeof(req)); } int bnxt_read_sfp_module_eeprom_info(struct bnxt_softc *softc, uint16_t i2c_addr, uint16_t page_number, uint8_t bank,bool bank_sel_en, uint16_t start_addr, uint16_t data_length, uint8_t *buf) { struct hwrm_port_phy_i2c_read_output *output = (void *)softc->hwrm_cmd_resp.idi_vaddr; struct hwrm_port_phy_i2c_read_input req = {0}; int rc = 0, byte_offset = 0; BNXT_HWRM_LOCK(softc); bnxt_hwrm_cmd_hdr_init(softc, &req, HWRM_PORT_PHY_I2C_READ); req.i2c_slave_addr = i2c_addr; req.page_number = htole16(page_number); req.port_id = htole16(softc->pf.port_id); do { uint16_t xfer_size; xfer_size = min_t(uint16_t, data_length, BNXT_MAX_PHY_I2C_RESP_SIZE); data_length -= xfer_size; req.page_offset = htole16(start_addr + byte_offset); req.data_length = xfer_size; req.bank_number = bank; req.enables = htole32((start_addr + byte_offset ? HWRM_PORT_PHY_I2C_READ_INPUT_ENABLES_PAGE_OFFSET : 0) | (bank_sel_en ? HWRM_PORT_PHY_I2C_READ_INPUT_ENABLES_BANK_NUMBER : 0)); rc = _hwrm_send_message(softc, &req, sizeof(req)); if (!rc) memcpy(buf + byte_offset, output->data, xfer_size); byte_offset += xfer_size; } while (!rc && data_length > 0); BNXT_HWRM_UNLOCK(softc); return rc; } int bnxt_hwrm_port_phy_qcfg(struct bnxt_softc *softc) { struct bnxt_link_info *link_info = &softc->link_info; struct hwrm_port_phy_qcfg_input req = {0}; struct hwrm_port_phy_qcfg_output *resp = (void *)softc->hwrm_cmd_resp.idi_vaddr; int rc = 0; bnxt_hwrm_cmd_hdr_init(softc, &req, HWRM_PORT_PHY_QCFG); BNXT_HWRM_LOCK(softc); rc = _hwrm_send_message(softc, &req, sizeof(req)); if (rc) goto exit; memcpy(&link_info->phy_qcfg_resp, resp, sizeof(*resp)); link_info->phy_link_status = resp->link; link_info->duplex = resp->duplex_cfg; link_info->auto_mode = resp->auto_mode; /* * When AUTO_PAUSE_AUTONEG_PAUSE bit is set to 1, * the advertisement of pause is enabled. * 1. When the auto_mode is not set to none and this flag is set to 1, * then the auto_pause bits on this port are being advertised and * autoneg pause results are being interpreted. * 2. When the auto_mode is not set to none and this flag is set to 0, * the pause is forced as indicated in force_pause, and also * advertised as auto_pause bits, but the autoneg results are not * interpreted since the pause configuration is being forced. * 3. When the auto_mode is set to none and this flag is set to 1, * auto_pause bits should be ignored and should be set to 0. */ link_info->flow_ctrl.autoneg = false; link_info->flow_ctrl.tx = false; link_info->flow_ctrl.rx = false; if ((resp->auto_mode) && (resp->auto_pause & BNXT_AUTO_PAUSE_AUTONEG_PAUSE)) { link_info->flow_ctrl.autoneg = true; } if (link_info->flow_ctrl.autoneg) { if (resp->auto_pause & BNXT_PAUSE_TX) link_info->flow_ctrl.tx = true; if (resp->auto_pause & BNXT_PAUSE_RX) link_info->flow_ctrl.rx = true; } else { if (resp->force_pause & BNXT_PAUSE_TX) link_info->flow_ctrl.tx = true; if (resp->force_pause & BNXT_PAUSE_RX) link_info->flow_ctrl.rx = true; } link_info->duplex_setting = resp->duplex_cfg; if (link_info->phy_link_status == HWRM_PORT_PHY_QCFG_OUTPUT_LINK_LINK) link_info->link_speed = le16toh(resp->link_speed); else link_info->link_speed = 0; link_info->force_link_speed = le16toh(resp->force_link_speed); link_info->auto_link_speeds = le16toh(resp->auto_link_speed); link_info->support_speeds = le16toh(resp->support_speeds); link_info->auto_link_speeds = le16toh(resp->auto_link_speed_mask); link_info->preemphasis = le32toh(resp->preemphasis); link_info->phy_ver[0] = resp->phy_maj; link_info->phy_ver[1] = resp->phy_min; link_info->phy_ver[2] = resp->phy_bld; snprintf(softc->ver_info->phy_ver, sizeof(softc->ver_info->phy_ver), "%d.%d.%d", link_info->phy_ver[0], link_info->phy_ver[1], link_info->phy_ver[2]); strlcpy(softc->ver_info->phy_vendor, resp->phy_vendor_name, BNXT_NAME_SIZE); strlcpy(softc->ver_info->phy_partnumber, resp->phy_vendor_partnumber, BNXT_NAME_SIZE); link_info->media_type = resp->media_type; link_info->phy_type = resp->phy_type; link_info->transceiver = resp->xcvr_pkg_type; link_info->phy_addr = resp->eee_config_phy_addr & HWRM_PORT_PHY_QCFG_OUTPUT_PHY_ADDR_MASK; link_info->module_status = resp->module_status; link_info->support_pam4_speeds = le16toh(resp->support_pam4_speeds); link_info->auto_pam4_link_speeds = le16toh(resp->auto_pam4_link_speed_mask); link_info->force_pam4_link_speed = le16toh(resp->force_pam4_link_speed); if (softc->hwrm_spec_code >= 0x10504) link_info->active_fec_sig_mode = resp->active_fec_signal_mode; link_info->support_speeds2 = le16toh(resp->support_speeds2); link_info->auto_link_speeds2 = le16toh(resp->auto_link_speeds2); link_info->force_link_speeds2 = le16toh(resp->force_link_speeds2); exit: BNXT_HWRM_UNLOCK(softc); return rc; } static bool bnxt_phy_qcaps_no_speed(struct hwrm_port_phy_qcaps_output *resp) { if (!resp->supported_speeds_auto_mode && !resp->supported_speeds_force_mode && !resp->supported_pam4_speeds_auto_mode && !resp->supported_pam4_speeds_force_mode && !resp->supported_speeds2_auto_mode && !resp->supported_speeds2_force_mode) return true; return false; } int bnxt_hwrm_phy_qcaps(struct bnxt_softc *softc) { struct bnxt_link_info *link_info = &softc->link_info; struct hwrm_port_phy_qcaps_output *resp = (void *)softc->hwrm_cmd_resp.idi_vaddr; struct hwrm_port_phy_qcaps_input req = {}; int rc; if (softc->hwrm_spec_code < 0x10201) return 0; bnxt_hwrm_cmd_hdr_init(softc, &req, HWRM_PORT_PHY_QCAPS); BNXT_HWRM_LOCK(softc); rc = _hwrm_send_message(softc, &req, sizeof(req)); if (rc) goto exit; softc->phy_flags = resp->flags | (resp->flags2 << 8); if (resp->flags & HWRM_PORT_PHY_QCAPS_OUTPUT_FLAGS_EEE_SUPPORTED) { softc->lpi_tmr_lo = le32toh(resp->tx_lpi_timer_low) & HWRM_PORT_PHY_QCAPS_OUTPUT_TX_LPI_TIMER_LOW_MASK; softc->lpi_tmr_hi = le32toh(resp->valid_tx_lpi_timer_high) & HWRM_PORT_PHY_QCAPS_OUTPUT_TX_LPI_TIMER_HIGH_MASK; } if (softc->hwrm_spec_code >= 0x10a01) { if (bnxt_phy_qcaps_no_speed(resp)) { link_info->phy_state = BNXT_PHY_STATE_DISABLED; device_printf(softc->dev, "Ethernet link disabled\n"); } else if (link_info->phy_state == BNXT_PHY_STATE_DISABLED) { link_info->phy_state = BNXT_PHY_STATE_ENABLED; device_printf(softc->dev, "Ethernet link enabled\n"); /* Phy re-enabled, reprobe the speeds */ link_info->support_auto_speeds = 0; link_info->support_pam4_auto_speeds = 0; link_info->support_auto_speeds2 = 0; } } if (resp->supported_speeds_auto_mode) link_info->support_auto_speeds = le16toh(resp->supported_speeds_auto_mode); if (resp->supported_speeds_force_mode) link_info->support_force_speeds = le16toh(resp->supported_speeds_force_mode); if (resp->supported_pam4_speeds_auto_mode) link_info->support_pam4_auto_speeds = le16toh(resp->supported_pam4_speeds_auto_mode); if (resp->supported_pam4_speeds_force_mode) link_info->support_pam4_force_speeds = le16toh(resp->supported_pam4_speeds_force_mode); if (resp->supported_speeds2_auto_mode) link_info->support_auto_speeds2 = le16toh(resp->supported_speeds2_auto_mode); if (resp->supported_speeds2_force_mode) link_info->support_force_speeds2 = le16toh(resp->supported_speeds2_force_mode); exit: BNXT_HWRM_UNLOCK(softc); return rc; } uint16_t bnxt_hwrm_get_wol_fltrs(struct bnxt_softc *softc, uint16_t handle) { struct hwrm_wol_filter_qcfg_input req = {0}; struct hwrm_wol_filter_qcfg_output *resp = (void *)softc->hwrm_cmd_resp.idi_vaddr; uint16_t next_handle = 0; int rc; bnxt_hwrm_cmd_hdr_init(softc, &req, HWRM_WOL_FILTER_QCFG); req.port_id = htole16(softc->pf.port_id); req.handle = htole16(handle); rc = hwrm_send_message(softc, &req, sizeof(req)); if (!rc) { next_handle = le16toh(resp->next_handle); if (next_handle != 0) { if (resp->wol_type == HWRM_WOL_FILTER_ALLOC_INPUT_WOL_TYPE_MAGICPKT) { softc->wol = 1; softc->wol_filter_id = resp->wol_filter_id; } } } return next_handle; } int bnxt_hwrm_alloc_wol_fltr(struct bnxt_softc *softc) { struct hwrm_wol_filter_alloc_input req = {0}; struct hwrm_wol_filter_alloc_output *resp = (void *)softc->hwrm_cmd_resp.idi_vaddr; int rc; bnxt_hwrm_cmd_hdr_init(softc, &req, HWRM_WOL_FILTER_ALLOC); req.port_id = htole16(softc->pf.port_id); req.wol_type = HWRM_WOL_FILTER_ALLOC_INPUT_WOL_TYPE_MAGICPKT; req.enables = htole32(HWRM_WOL_FILTER_ALLOC_INPUT_ENABLES_MAC_ADDRESS); memcpy(req.mac_address, softc->func.mac_addr, ETHER_ADDR_LEN); rc = hwrm_send_message(softc, &req, sizeof(req)); if (!rc) softc->wol_filter_id = resp->wol_filter_id; return rc; } int bnxt_hwrm_free_wol_fltr(struct bnxt_softc *softc) { struct hwrm_wol_filter_free_input req = {0}; bnxt_hwrm_cmd_hdr_init(softc, &req, HWRM_WOL_FILTER_FREE); req.port_id = htole16(softc->pf.port_id); req.enables = htole32(HWRM_WOL_FILTER_FREE_INPUT_ENABLES_WOL_FILTER_ID); req.wol_filter_id = softc->wol_filter_id; return hwrm_send_message(softc, &req, sizeof(req)); } static void bnxt_hwrm_set_coal_params(struct bnxt_softc *softc, uint32_t max_frames, uint32_t buf_tmrs, uint16_t flags, struct hwrm_ring_cmpl_ring_cfg_aggint_params_input *req) { req->flags = htole16(flags); req->num_cmpl_dma_aggr = htole16((uint16_t)max_frames); req->num_cmpl_dma_aggr_during_int = htole16(max_frames >> 16); req->cmpl_aggr_dma_tmr = htole16((uint16_t)buf_tmrs); req->cmpl_aggr_dma_tmr_during_int = htole16(buf_tmrs >> 16); /* Minimum time between 2 interrupts set to buf_tmr x 2 */ req->int_lat_tmr_min = htole16((uint16_t)buf_tmrs * 2); req->int_lat_tmr_max = htole16((uint16_t)buf_tmrs * 4); req->num_cmpl_aggr_int = htole16((uint16_t)max_frames * 4); } int bnxt_hwrm_set_coal(struct bnxt_softc *softc) { int i, rc = 0; struct hwrm_ring_cmpl_ring_cfg_aggint_params_input req_rx = {0}, req_tx = {0}, *req; uint16_t max_buf, max_buf_irq; uint16_t buf_tmr, buf_tmr_irq; uint32_t flags; bnxt_hwrm_cmd_hdr_init(softc, &req_rx, HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS); bnxt_hwrm_cmd_hdr_init(softc, &req_tx, HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS); /* Each rx completion (2 records) should be DMAed immediately. * DMA 1/4 of the completion buffers at a time. */ max_buf = min_t(uint16_t, softc->rx_coal_frames / 4, 2); /* max_buf must not be zero */ max_buf = clamp_t(uint16_t, max_buf, 1, 63); max_buf_irq = clamp_t(uint16_t, softc->rx_coal_frames_irq, 1, 63); buf_tmr = BNXT_USEC_TO_COAL_TIMER(softc->rx_coal_usecs); /* buf timer set to 1/4 of interrupt timer */ buf_tmr = max_t(uint16_t, buf_tmr / 4, 1); buf_tmr_irq = BNXT_USEC_TO_COAL_TIMER(softc->rx_coal_usecs_irq); buf_tmr_irq = max_t(uint16_t, buf_tmr_irq, 1); flags = HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS_INPUT_FLAGS_TIMER_RESET; /* RING_IDLE generates more IRQs for lower latency. Enable it only * if coal_usecs is less than 25 us. */ if (softc->rx_coal_usecs < 25) flags |= HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS_INPUT_FLAGS_RING_IDLE; bnxt_hwrm_set_coal_params(softc, max_buf_irq << 16 | max_buf, buf_tmr_irq << 16 | buf_tmr, flags, &req_rx); /* max_buf must not be zero */ max_buf = clamp_t(uint16_t, softc->tx_coal_frames, 1, 63); max_buf_irq = clamp_t(uint16_t, softc->tx_coal_frames_irq, 1, 63); buf_tmr = BNXT_USEC_TO_COAL_TIMER(softc->tx_coal_usecs); /* buf timer set to 1/4 of interrupt timer */ buf_tmr = max_t(uint16_t, buf_tmr / 4, 1); buf_tmr_irq = BNXT_USEC_TO_COAL_TIMER(softc->tx_coal_usecs_irq); buf_tmr_irq = max_t(uint16_t, buf_tmr_irq, 1); flags = HWRM_RING_CMPL_RING_CFG_AGGINT_PARAMS_INPUT_FLAGS_TIMER_RESET; bnxt_hwrm_set_coal_params(softc, max_buf_irq << 16 | max_buf, buf_tmr_irq << 16 | buf_tmr, flags, &req_tx); for (i = 0; i < softc->nrxqsets; i++) { req = &req_rx; req->ring_id = htole16(softc->grp_info[i].cp_ring_id); rc = hwrm_send_message(softc, req, sizeof(*req)); if (rc) break; } return rc; } void bnxt_hwrm_ring_info_get(struct bnxt_softc *softc, uint8_t ring_type, uint32_t ring_id, uint32_t *prod, uint32_t *cons) { hwrm_dbg_ring_info_get_input_t req = {0}; hwrm_dbg_ring_info_get_output_t *resp = (void *)softc->hwrm_cmd_resp.idi_vaddr; int rc = 0; *prod = *cons = 0xffffffff; bnxt_hwrm_cmd_hdr_init(softc, &req, HWRM_DBG_RING_INFO_GET); req.ring_type = le32toh(ring_type); req.fw_ring_id = le32toh(ring_id); rc = hwrm_send_message(softc, &req, sizeof(req)); if (!rc) { *prod = resp->producer_index; *cons = resp->consumer_index; } return; } diff --git a/sys/dev/bnxt/bnxt_en/bnxt_ulp.c b/sys/dev/bnxt/bnxt_en/bnxt_ulp.c index 3c1f62cb4da3..9950858f0eef 100644 --- a/sys/dev/bnxt/bnxt_en/bnxt_ulp.c +++ b/sys/dev/bnxt/bnxt_en/bnxt_ulp.c @@ -1,526 +1,527 @@ /*- * Broadcom NetXtreme-C/E network driver. * * Copyright (c) 2024 Broadcom, All Rights Reserved. * The term Broadcom refers to Broadcom Limited and/or its subsidiaries * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS' * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF * THE POSSIBILITY OF SUCH DAMAGE. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "hsi_struct_def.h" #include "bnxt.h" #include "bnxt_hwrm.h" #include "bnxt_ulp.h" void bnxt_destroy_irq(struct bnxt_softc *softc); static int bnxt_register_dev(struct bnxt_en_dev *edev, int ulp_id, struct bnxt_ulp_ops *ulp_ops, void *handle) { struct bnxt_softc *bp = edev->softc; struct bnxt_ulp *ulp; int rc = 0; if (ulp_id >= BNXT_MAX_ULP) return -EINVAL; mtx_lock(&bp->en_ops_lock); ulp = &edev->ulp_tbl[ulp_id]; if (rcu_access_pointer(ulp->ulp_ops)) { device_printf(bp->dev, "ulp id %d already registered\n", ulp_id); rc = -EBUSY; goto exit; } edev->flags &= ~BNXT_EN_FLAG_ULP_STOPPED; atomic_set(&ulp->ref_count, 0); ulp->handle = handle; rcu_assign_pointer(ulp->ulp_ops, ulp_ops); if (ulp_id == BNXT_ROCE_ULP) { if (test_bit(BNXT_STATE_OPEN, &bp->state) && bp->is_dev_init) bnxt_hwrm_vnic_cfg(bp, &bp->vnic_info); } exit: mtx_unlock(&bp->en_ops_lock); return rc; } static int bnxt_unregister_dev(struct bnxt_en_dev *edev, int ulp_id) { struct bnxt_softc *bp = edev->softc; struct bnxt_ulp *ulp; int i = 0; if (ulp_id >= BNXT_MAX_ULP) return -EINVAL; ulp = &edev->ulp_tbl[ulp_id]; if (!rcu_access_pointer(ulp->ulp_ops)) { device_printf(bp->dev, "ulp id %d not registered\n", ulp_id); return -EINVAL; } if (ulp_id == BNXT_ROCE_ULP && ulp->msix_requested) edev->en_ops->bnxt_free_msix(edev, ulp_id); mtx_lock(&bp->en_ops_lock); RCU_INIT_POINTER(ulp->ulp_ops, NULL); synchronize_rcu(); ulp->max_async_event_id = 0; ulp->async_events_bmap = NULL; while (atomic_read(&ulp->ref_count) != 0 && i < 10) { msleep(100); i++; } mtx_unlock(&bp->en_ops_lock); return 0; } static void bnxt_fill_msix_vecs(struct bnxt_softc *bp, struct bnxt_msix_entry *ent) { struct bnxt_en_dev *edev = bp->edev; int num_msix, idx, i; num_msix = edev->ulp_tbl[BNXT_ROCE_ULP].msix_requested; idx = edev->ulp_tbl[BNXT_ROCE_ULP].msix_base; for (i = 0; i < num_msix; i++) { ent[i].vector = bp->irq_tbl[idx + i].vector; ent[i].ring_idx = idx + i; if (BNXT_CHIP_P5_PLUS(bp)) - ent[i].db_offset = DB_PF_OFFSET_P5; + ent[i].db_offset = bp->db_offset; else ent[i].db_offset = (idx + i) * 0x80; } } static int bnxt_req_msix_vecs(struct bnxt_en_dev *edev, int ulp_id, struct bnxt_msix_entry *ent, int num_msix) { struct bnxt_softc *bp = edev->softc; int avail_msix, idx; if (ulp_id != BNXT_ROCE_ULP) return -EINVAL; if (edev->ulp_tbl[ulp_id].msix_requested) return -EAGAIN; idx = bp->total_irqs - BNXT_ROCE_IRQ_COUNT; avail_msix = BNXT_ROCE_IRQ_COUNT; mtx_lock(&bp->en_ops_lock); edev->ulp_tbl[ulp_id].msix_base = idx; edev->ulp_tbl[ulp_id].msix_requested = avail_msix; bnxt_fill_msix_vecs(bp, ent); edev->flags |= BNXT_EN_FLAG_MSIX_REQUESTED; mtx_unlock(&bp->en_ops_lock); return avail_msix; } static int bnxt_free_msix_vecs(struct bnxt_en_dev *edev, int ulp_id) { struct bnxt_softc *bp = edev->softc; if (ulp_id != BNXT_ROCE_ULP) return -EINVAL; if (!(edev->flags & BNXT_EN_FLAG_MSIX_REQUESTED)) return 0; mtx_lock(&bp->en_ops_lock); edev->ulp_tbl[ulp_id].msix_requested = 0; edev->flags &= ~BNXT_EN_FLAG_MSIX_REQUESTED; if (edev->flags & BNXT_EN_FLAG_ULP_STOPPED) goto stopped; stopped: mtx_unlock(&bp->en_ops_lock); return 0; } int bnxt_get_ulp_msix_num(struct bnxt_softc *bp) { if (bnxt_ulp_registered(bp->edev, BNXT_ROCE_ULP)) { struct bnxt_en_dev *edev = bp->edev; return edev->ulp_tbl[BNXT_ROCE_ULP].msix_requested; } return 0; } int bnxt_get_ulp_msix_base(struct bnxt_softc *bp) { if (bnxt_ulp_registered(bp->edev, BNXT_ROCE_ULP)) { struct bnxt_en_dev *edev = bp->edev; if (edev->ulp_tbl[BNXT_ROCE_ULP].msix_requested) return edev->ulp_tbl[BNXT_ROCE_ULP].msix_base; } return 0; } static int bnxt_send_msg(struct bnxt_en_dev *edev, int ulp_id, struct bnxt_fw_msg *fw_msg) { struct bnxt_softc *softc = edev->softc; int rc; if ((ulp_id != BNXT_ROCE_ULP) && softc->fw_reset_state) return -EBUSY; rc = bnxt_hwrm_passthrough(softc, fw_msg->msg, fw_msg->msg_len, fw_msg->resp, fw_msg->resp_max_len, fw_msg->timeout); return rc; } static void bnxt_ulp_get(struct bnxt_ulp *ulp) { atomic_inc(&ulp->ref_count); } static void bnxt_ulp_put(struct bnxt_ulp *ulp) { atomic_dec(&ulp->ref_count); } void bnxt_ulp_stop(struct bnxt_softc *bp) { struct bnxt_en_dev *edev = bp->edev; struct bnxt_ulp_ops *ops; int i; if (!edev) return; edev->flags |= BNXT_EN_FLAG_ULP_STOPPED; edev->en_state = bp->state; for (i = 0; i < BNXT_MAX_ULP; i++) { struct bnxt_ulp *ulp = &edev->ulp_tbl[i]; ops = ulp->ulp_ops; if (!ops || !ops->ulp_stop) continue; ops->ulp_stop(ulp->handle); } } void bnxt_ulp_start(struct bnxt_softc *bp, int err) { struct bnxt_en_dev *edev = bp->edev; struct bnxt_ulp_ops *ops; int i; if (!edev) return; edev->flags &= ~BNXT_EN_FLAG_ULP_STOPPED; edev->en_state = bp->state; if (err) return; for (i = 0; i < BNXT_MAX_ULP; i++) { struct bnxt_ulp *ulp = &edev->ulp_tbl[i]; ops = ulp->ulp_ops; if (!ops || !ops->ulp_start) continue; ops->ulp_start(ulp->handle); } } void bnxt_ulp_sriov_cfg(struct bnxt_softc *bp, int num_vfs) { struct bnxt_en_dev *edev = bp->edev; struct bnxt_ulp_ops *ops; int i; if (!edev) return; for (i = 0; i < BNXT_MAX_ULP; i++) { struct bnxt_ulp *ulp = &edev->ulp_tbl[i]; rcu_read_lock(); ops = rcu_dereference(ulp->ulp_ops); if (!ops || !ops->ulp_sriov_config) { rcu_read_unlock(); continue; } bnxt_ulp_get(ulp); rcu_read_unlock(); ops->ulp_sriov_config(ulp->handle, num_vfs); bnxt_ulp_put(ulp); } } void bnxt_ulp_shutdown(struct bnxt_softc *bp) { struct bnxt_en_dev *edev = bp->edev; struct bnxt_ulp_ops *ops; int i; if (!edev) return; for (i = 0; i < BNXT_MAX_ULP; i++) { struct bnxt_ulp *ulp = &edev->ulp_tbl[i]; ops = ulp->ulp_ops; if (!ops || !ops->ulp_shutdown) continue; ops->ulp_shutdown(ulp->handle); } } void bnxt_ulp_irq_stop(struct bnxt_softc *bp) { struct bnxt_en_dev *edev = bp->edev; struct bnxt_ulp_ops *ops; if (!edev || !(edev->flags & BNXT_EN_FLAG_MSIX_REQUESTED)) return; if (bnxt_ulp_registered(bp->edev, BNXT_ROCE_ULP)) { struct bnxt_ulp *ulp = &edev->ulp_tbl[BNXT_ROCE_ULP]; if (!ulp->msix_requested) return; ops = ulp->ulp_ops; if (!ops || !ops->ulp_irq_stop) return; ops->ulp_irq_stop(ulp->handle); } } void bnxt_ulp_async_events(struct bnxt_softc *bp, struct hwrm_async_event_cmpl *cmpl) { u16 event_id = le16_to_cpu(cmpl->event_id); struct bnxt_en_dev *edev = bp->edev; struct bnxt_ulp_ops *ops; int i; if (!edev) return; rcu_read_lock(); for (i = 0; i < BNXT_MAX_ULP; i++) { struct bnxt_ulp *ulp = &edev->ulp_tbl[i]; ops = rcu_dereference(ulp->ulp_ops); if (!ops || !ops->ulp_async_notifier) continue; if (!ulp->async_events_bmap || event_id > ulp->max_async_event_id) continue; /* Read max_async_event_id first before testing the bitmap. */ rmb(); if (edev->flags & BNXT_EN_FLAG_ULP_STOPPED) continue; if (test_bit(event_id, ulp->async_events_bmap)) ops->ulp_async_notifier(ulp->handle, cmpl); } rcu_read_unlock(); } static int bnxt_register_async_events(struct bnxt_en_dev *edev, int ulp_id, unsigned long *events_bmap, u16 max_id) { struct bnxt_softc *bp = edev->softc; struct bnxt_ulp *ulp; if (ulp_id >= BNXT_MAX_ULP) return -EINVAL; mtx_lock(&bp->en_ops_lock); ulp = &edev->ulp_tbl[ulp_id]; ulp->async_events_bmap = events_bmap; wmb(); ulp->max_async_event_id = max_id; bnxt_hwrm_func_drv_rgtr(bp, events_bmap, max_id + 1, true); mtx_unlock(&bp->en_ops_lock); return 0; } void bnxt_destroy_irq(struct bnxt_softc *softc) { kfree(softc->irq_tbl); } static int bnxt_populate_irq(struct bnxt_softc *softc) { struct resource_list *rl = NULL; struct resource_list_entry *rle = NULL; struct bnxt_msix_tbl *irq_tbl = NULL; struct pci_devinfo *dinfo = NULL; int i; softc->total_irqs = softc->scctx->isc_nrxqsets + BNXT_ROCE_IRQ_COUNT; irq_tbl = kzalloc(softc->total_irqs * sizeof(*softc->irq_tbl), GFP_KERNEL); if (!irq_tbl) { device_printf(softc->dev, "Failed to allocate IRQ table\n"); return -1; } dinfo = device_get_ivars(softc->pdev->dev.bsddev); rl = &dinfo->resources; rle = resource_list_find(rl, SYS_RES_IRQ, 1); softc->pdev->dev.irq_start = rle->start; softc->pdev->dev.irq_end = rle->start + softc->total_irqs; for (i = 0; i < softc->total_irqs; i++) { irq_tbl[i].entry = i; irq_tbl[i].vector = softc->pdev->dev.irq_start + i; } softc->irq_tbl = irq_tbl; return 0; } static const struct bnxt_en_ops bnxt_en_ops_tbl = { .bnxt_register_device = bnxt_register_dev, .bnxt_unregister_device = bnxt_unregister_dev, .bnxt_request_msix = bnxt_req_msix_vecs, .bnxt_free_msix = bnxt_free_msix_vecs, .bnxt_send_fw_msg = bnxt_send_msg, .bnxt_register_fw_async_events = bnxt_register_async_events, }; void bnxt_aux_dev_release(struct device *dev) { struct bnxt_aux_dev *bnxt_adev = container_of(dev, struct bnxt_aux_dev, aux_dev.dev); struct bnxt_softc *bp = bnxt_adev->edev->softc; kfree(bnxt_adev->edev); bnxt_adev->edev = NULL; bp->edev = NULL; } static inline void bnxt_set_edev_info(struct bnxt_en_dev *edev, struct bnxt_softc *bp) { edev->en_ops = &bnxt_en_ops_tbl; edev->net = bp->ifp; edev->pdev = bp->pdev; edev->softc = bp; edev->l2_db_size = bp->db_size; + edev->l2_db_offset = bp->db_offset; mtx_init(&bp->en_ops_lock, "Ethernet ops lock", NULL, MTX_DEF); if (bp->flags & BNXT_FLAG_ROCEV1_CAP) edev->flags |= BNXT_EN_FLAG_ROCEV1_CAP; if (bp->flags & BNXT_FLAG_ROCEV2_CAP) edev->flags |= BNXT_EN_FLAG_ROCEV2_CAP; if (bp->is_asym_q) edev->flags |= BNXT_EN_FLAG_ASYM_Q; edev->hwrm_bar = bp->hwrm_bar; edev->port_partition_type = bp->port_partition_type; edev->ulp_version = BNXT_ULP_VERSION; } int bnxt_rdma_aux_device_del(struct bnxt_softc *softc) { struct bnxt_aux_dev *bnxt_adev = softc->aux_dev; struct auxiliary_device *adev; adev = &bnxt_adev->aux_dev; auxiliary_device_delete(adev); auxiliary_device_uninit(adev); bnxt_destroy_irq(softc); return 0; } int bnxt_rdma_aux_device_add(struct bnxt_softc *bp) { struct bnxt_aux_dev *bnxt_adev = bp->aux_dev; struct bnxt_en_dev *edev = bnxt_adev->edev; struct auxiliary_device *aux_dev; int ret = -1; if (bnxt_populate_irq(bp)) return ret; device_printf(bp->dev, "V:D:SV:SD %x:%x:%x:%x, irq 0x%x, " "devfn 0x%x, cla 0x%x, rev 0x%x, msi_en 0x%x\n", bp->pdev->vendor, bp->pdev->device, bp->pdev->subsystem_vendor, bp->pdev->subsystem_device, bp->pdev->irq, bp->pdev->devfn, bp->pdev->class, bp->pdev->revision, bp->pdev->msi_enabled); aux_dev = &bnxt_adev->aux_dev; aux_dev->id = bnxt_adev->id; aux_dev->name = "rdma"; aux_dev->dev.parent = &bp->pdev->dev; aux_dev->dev.release = bnxt_aux_dev_release; if (!edev) { edev = kzalloc(sizeof(*edev), GFP_KERNEL); if (!edev) return -ENOMEM; } bnxt_set_edev_info(edev, bp); bnxt_adev->edev = edev; bp->edev = edev; ret = auxiliary_device_init(aux_dev); if (ret) goto err_free_edev; ret = auxiliary_device_add(aux_dev); if (ret) goto err_dev_uninit; return 0; err_dev_uninit: auxiliary_device_uninit(aux_dev); err_free_edev: kfree(edev); bnxt_adev->edev = NULL; bp->edev = NULL; return ret; } diff --git a/sys/dev/bnxt/bnxt_en/bnxt_ulp.h b/sys/dev/bnxt/bnxt_en/bnxt_ulp.h index 0108293046d7..bbf16304456a 100644 --- a/sys/dev/bnxt/bnxt_en/bnxt_ulp.h +++ b/sys/dev/bnxt/bnxt_en/bnxt_ulp.h @@ -1,161 +1,164 @@ /*- * Broadcom NetXtreme-C/E network driver. * * Copyright (c) 2024 Broadcom, All Rights Reserved. * The term Broadcom refers to Broadcom Limited and/or its subsidiaries * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS' * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF * THE POSSIBILITY OF SUCH DAMAGE. */ #ifndef BNXT_ULP_H #define BNXT_ULP_H #include #include "bnxt.h" #define BNXT_ROCE_ULP 0 #define BNXT_OTHER_ULP 1 #define BNXT_MAX_ULP 2 #define BNXT_MIN_ROCE_CP_RINGS 2 #define BNXT_MIN_ROCE_STAT_CTXS 1 struct hwrm_async_event_cmpl; struct bnxt_softc; struct bnxt_bar_info; struct bnxt_msix_entry { uint32_t vector; uint32_t ring_idx; uint32_t db_offset; }; struct bnxt_ulp_ops { void (*ulp_async_notifier)(void *, struct hwrm_async_event_cmpl *); void (*ulp_stop)(void *); void (*ulp_start)(void *); void (*ulp_sriov_config)(void *, int); void (*ulp_shutdown)(void *); void (*ulp_irq_stop)(void *); void (*ulp_irq_restart)(void *, struct bnxt_msix_entry *); }; struct bnxt_fw_msg { void *msg; int msg_len; void *resp; int resp_max_len; int timeout; }; struct bnxt_ulp { void *handle; struct bnxt_ulp_ops __rcu *ulp_ops; unsigned long *async_events_bmap; u16 max_async_event_id; u16 msix_requested; u16 msix_base; atomic_t ref_count; }; struct bnxt_en_dev { struct ifnet *net; struct pci_dev *pdev; struct bnxt_softc *softc; u32 flags; #define BNXT_EN_FLAG_ROCEV1_CAP 0x1 #define BNXT_EN_FLAG_ROCEV2_CAP 0x2 #define BNXT_EN_FLAG_ROCE_CAP (BNXT_EN_FLAG_ROCEV1_CAP | \ BNXT_EN_FLAG_ROCEV2_CAP) #define BNXT_EN_FLAG_MSIX_REQUESTED 0x4 #define BNXT_EN_FLAG_ULP_STOPPED 0x8 #define BNXT_EN_FLAG_ASYM_Q 0x10 #define BNXT_EN_FLAG_MULTI_HOST 0x20 #define BNXT_EN_ASYM_Q(edev) ((edev)->flags & BNXT_EN_FLAG_ASYM_Q) #define BNXT_EN_MH(edev) ((edev)->flags & BNXT_EN_FLAG_MULTI_HOST) const struct bnxt_en_ops *en_ops; struct bnxt_ulp ulp_tbl[BNXT_MAX_ULP]; + int l2_db_offset; /* Doorbell BAR offset + * of non-cacheable. + */ int l2_db_size; /* Doorbell BAR size in * bytes mapped by L2 * driver. */ int l2_db_size_nc; /* Doorbell BAR size in * bytes mapped as non- * cacheable. */ u32 ulp_version; /* bnxt_re checks the * ulp_version is correct * to ensure compatibility * with bnxt_en. */ #define BNXT_ULP_VERSION 0x695a0008 /* Change this when any interface * structure or API changes * between bnxt_en and bnxt_re. */ unsigned long en_state; void __iomem *bar0; u16 hw_ring_stats_size; u16 pf_port_id; u8 port_partition_type; #define BNXT_EN_NPAR(edev) ((edev)->port_partition_type) u8 port_count; struct bnxt_dbr *en_dbr; struct bnxt_bar_info hwrm_bar; u32 espeed; }; struct bnxt_en_ops { int (*bnxt_register_device)(struct bnxt_en_dev *, int, struct bnxt_ulp_ops *, void *); int (*bnxt_unregister_device)(struct bnxt_en_dev *, int); int (*bnxt_request_msix)(struct bnxt_en_dev *, int, struct bnxt_msix_entry *, int); int (*bnxt_free_msix)(struct bnxt_en_dev *, int); int (*bnxt_send_fw_msg)(struct bnxt_en_dev *, int, struct bnxt_fw_msg *); int (*bnxt_register_fw_async_events)(struct bnxt_en_dev *, int, unsigned long *, u16); int (*bnxt_dbr_complete)(struct bnxt_en_dev *, int, u32); }; static inline bool bnxt_ulp_registered(struct bnxt_en_dev *edev, int ulp_id) { if (edev && rcu_access_pointer(edev->ulp_tbl[ulp_id].ulp_ops)) return true; return false; } int bnxt_get_ulp_msix_num(struct bnxt_softc *bp); int bnxt_get_ulp_msix_base(struct bnxt_softc *bp); int bnxt_get_ulp_stat_ctxs(struct bnxt_softc *bp); void bnxt_ulp_stop(struct bnxt_softc *bp); void bnxt_ulp_start(struct bnxt_softc *bp, int err); void bnxt_ulp_sriov_cfg(struct bnxt_softc *bp, int num_vfs); void bnxt_ulp_shutdown(struct bnxt_softc *bp); void bnxt_ulp_irq_stop(struct bnxt_softc *bp); void bnxt_ulp_irq_restart(struct bnxt_softc *bp, int err); void bnxt_ulp_async_events(struct bnxt_softc *bp, struct hwrm_async_event_cmpl *cmpl); struct bnxt_en_dev *bnxt_ulp_probe(struct net_device *dev); void bnxt_aux_dev_release(struct device *dev); int bnxt_rdma_aux_device_add(struct bnxt_softc *bp); int bnxt_rdma_aux_device_del(struct bnxt_softc *bp); #endif diff --git a/sys/dev/bnxt/bnxt_re/ib_verbs.c b/sys/dev/bnxt/bnxt_re/ib_verbs.c index 0383a16757aa..dba0f348c414 100644 --- a/sys/dev/bnxt/bnxt_re/ib_verbs.c +++ b/sys/dev/bnxt/bnxt_re/ib_verbs.c @@ -1,5498 +1,5501 @@ /* * Copyright (c) 2015-2024, Broadcom. All rights reserved. The term * Broadcom refers to Broadcom Limited and/or its subsidiaries. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in * the documentation and/or other materials provided with the * distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * * Description: IB Verbs interpreter */ #include #include #include #include "bnxt_re.h" #include "ib_verbs.h" static inline struct scatterlist *get_ib_umem_sgl(struct ib_umem *umem, u32 *nmap) { *nmap = umem->nmap; return umem->sg_head.sgl; } static inline void bnxt_re_peer_mem_release(struct ib_umem *umem) { dev_dbg(NULL, "ib_umem_release getting invoked \n"); ib_umem_release(umem); } void bnxt_re_resolve_dmac_task(struct work_struct *work) { int rc = -1; struct bnxt_re_dev *rdev; struct ib_ah_attr *ah_attr; struct bnxt_re_resolve_dmac_work *dmac_work = container_of(work, struct bnxt_re_resolve_dmac_work, work); rdev = dmac_work->rdev; ah_attr = dmac_work->ah_attr; rc = ib_resolve_eth_dmac(&rdev->ibdev, ah_attr); if (rc) dev_err(rdev_to_dev(dmac_work->rdev), "Failed to resolve dest mac rc = %d\n", rc); atomic_set(&dmac_work->status_wait, rc << 8); } static int __from_ib_access_flags(int iflags) { int qflags = 0; if (iflags & IB_ACCESS_LOCAL_WRITE) qflags |= BNXT_QPLIB_ACCESS_LOCAL_WRITE; if (iflags & IB_ACCESS_REMOTE_READ) qflags |= BNXT_QPLIB_ACCESS_REMOTE_READ; if (iflags & IB_ACCESS_REMOTE_WRITE) qflags |= BNXT_QPLIB_ACCESS_REMOTE_WRITE; if (iflags & IB_ACCESS_REMOTE_ATOMIC) qflags |= BNXT_QPLIB_ACCESS_REMOTE_ATOMIC; if (iflags & IB_ACCESS_MW_BIND) qflags |= BNXT_QPLIB_ACCESS_MW_BIND; if (iflags & IB_ZERO_BASED) qflags |= BNXT_QPLIB_ACCESS_ZERO_BASED; if (iflags & IB_ACCESS_ON_DEMAND) qflags |= BNXT_QPLIB_ACCESS_ON_DEMAND; return qflags; }; static enum ib_access_flags __to_ib_access_flags(int qflags) { enum ib_access_flags iflags = 0; if (qflags & BNXT_QPLIB_ACCESS_LOCAL_WRITE) iflags |= IB_ACCESS_LOCAL_WRITE; if (qflags & BNXT_QPLIB_ACCESS_REMOTE_WRITE) iflags |= IB_ACCESS_REMOTE_WRITE; if (qflags & BNXT_QPLIB_ACCESS_REMOTE_READ) iflags |= IB_ACCESS_REMOTE_READ; if (qflags & BNXT_QPLIB_ACCESS_REMOTE_ATOMIC) iflags |= IB_ACCESS_REMOTE_ATOMIC; if (qflags & BNXT_QPLIB_ACCESS_MW_BIND) iflags |= IB_ACCESS_MW_BIND; if (qflags & BNXT_QPLIB_ACCESS_ZERO_BASED) iflags |= IB_ZERO_BASED; if (qflags & BNXT_QPLIB_ACCESS_ON_DEMAND) iflags |= IB_ACCESS_ON_DEMAND; return iflags; }; static int bnxt_re_copy_to_udata(struct bnxt_re_dev *rdev, void *data, int len, struct ib_udata *udata) { int rc; rc = ib_copy_to_udata(udata, data, len); if (rc) dev_err(rdev_to_dev(rdev), "ucontext copy failed from %ps rc %d\n", __builtin_return_address(0), rc); return rc; } struct ifnet *bnxt_re_get_netdev(struct ib_device *ibdev, u8 port_num) { struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev); struct ifnet *netdev = NULL; rcu_read_lock(); if (!rdev || !rdev->netdev) goto end; netdev = rdev->netdev; /* In case of active-backup bond mode, return active slave */ if (netdev) dev_hold(netdev); end: rcu_read_unlock(); return netdev; } int bnxt_re_query_device(struct ib_device *ibdev, struct ib_device_attr *ib_attr, struct ib_udata *udata) { struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev); struct bnxt_qplib_dev_attr *dev_attr = rdev->dev_attr; memset(ib_attr, 0, sizeof(*ib_attr)); memcpy(&ib_attr->fw_ver, dev_attr->fw_ver, 4); bnxt_qplib_get_guid(rdev->dev_addr, (u8 *)&ib_attr->sys_image_guid); ib_attr->max_mr_size = BNXT_RE_MAX_MR_SIZE; ib_attr->page_size_cap = dev_attr->page_size_cap; ib_attr->vendor_id = rdev->en_dev->pdev->vendor; ib_attr->vendor_part_id = rdev->en_dev->pdev->device; ib_attr->hw_ver = rdev->en_dev->pdev->subsystem_device; ib_attr->max_qp = dev_attr->max_qp; ib_attr->max_qp_wr = dev_attr->max_qp_wqes; /* * Read and set from the module param 'min_tx_depth' * only once after the driver load */ if (rdev->min_tx_depth == 1 && min_tx_depth < dev_attr->max_qp_wqes) rdev->min_tx_depth = min_tx_depth; ib_attr->device_cap_flags = IB_DEVICE_CURR_QP_STATE_MOD | IB_DEVICE_RC_RNR_NAK_GEN | IB_DEVICE_SHUTDOWN_PORT | IB_DEVICE_SYS_IMAGE_GUID | IB_DEVICE_LOCAL_DMA_LKEY | IB_DEVICE_RESIZE_MAX_WR | IB_DEVICE_PORT_ACTIVE_EVENT | IB_DEVICE_N_NOTIFY_CQ | IB_DEVICE_MEM_WINDOW | IB_DEVICE_MEM_WINDOW_TYPE_2B | IB_DEVICE_MEM_MGT_EXTENSIONS; ib_attr->max_send_sge = dev_attr->max_qp_sges; ib_attr->max_recv_sge = dev_attr->max_qp_sges; ib_attr->max_sge_rd = dev_attr->max_qp_sges; ib_attr->max_cq = dev_attr->max_cq; ib_attr->max_cqe = dev_attr->max_cq_wqes; ib_attr->max_mr = dev_attr->max_mr; ib_attr->max_pd = dev_attr->max_pd; ib_attr->max_qp_rd_atom = dev_attr->max_qp_rd_atom; ib_attr->max_qp_init_rd_atom = dev_attr->max_qp_init_rd_atom; if (dev_attr->is_atomic) { ib_attr->atomic_cap = IB_ATOMIC_GLOB; ib_attr->masked_atomic_cap = IB_ATOMIC_GLOB; } ib_attr->max_ee_rd_atom = 0; ib_attr->max_res_rd_atom = 0; ib_attr->max_ee_init_rd_atom = 0; ib_attr->max_ee = 0; ib_attr->max_rdd = 0; ib_attr->max_mw = dev_attr->max_mw; ib_attr->max_raw_ipv6_qp = 0; ib_attr->max_raw_ethy_qp = dev_attr->max_raw_ethy_qp; ib_attr->max_mcast_grp = 0; ib_attr->max_mcast_qp_attach = 0; ib_attr->max_total_mcast_qp_attach = 0; ib_attr->max_ah = dev_attr->max_ah; ib_attr->max_srq = dev_attr->max_srq; ib_attr->max_srq_wr = dev_attr->max_srq_wqes; ib_attr->max_srq_sge = dev_attr->max_srq_sges; ib_attr->max_fast_reg_page_list_len = MAX_PBL_LVL_1_PGS; ib_attr->max_pkeys = 1; ib_attr->local_ca_ack_delay = BNXT_RE_DEFAULT_ACK_DELAY; ib_attr->sig_prot_cap = 0; ib_attr->sig_guard_cap = 0; ib_attr->odp_caps.general_caps = 0; return 0; } int bnxt_re_modify_device(struct ib_device *ibdev, int device_modify_mask, struct ib_device_modify *device_modify) { dev_dbg(rdev_to_dev(rdev), "Modify device with mask 0x%x\n", device_modify_mask); switch (device_modify_mask) { case IB_DEVICE_MODIFY_SYS_IMAGE_GUID: /* Modify the GUID requires the modification of the GID table */ /* GUID should be made as READ-ONLY */ break; case IB_DEVICE_MODIFY_NODE_DESC: /* Node Desc should be made as READ-ONLY */ break; default: break; } return 0; } static void __to_ib_speed_width(u32 espeed, u8 *speed, u8 *width) { switch (espeed) { case SPEED_1000: *speed = IB_SPEED_SDR; *width = IB_WIDTH_1X; break; case SPEED_10000: *speed = IB_SPEED_QDR; *width = IB_WIDTH_1X; break; case SPEED_20000: *speed = IB_SPEED_DDR; *width = IB_WIDTH_4X; break; case SPEED_25000: *speed = IB_SPEED_EDR; *width = IB_WIDTH_1X; break; case SPEED_40000: *speed = IB_SPEED_QDR; *width = IB_WIDTH_4X; break; case SPEED_50000: *speed = IB_SPEED_EDR; *width = IB_WIDTH_2X; break; case SPEED_100000: *speed = IB_SPEED_EDR; *width = IB_WIDTH_4X; break; case SPEED_200000: *speed = IB_SPEED_HDR; *width = IB_WIDTH_4X; break; + case SPEED_400000: + *speed = IB_SPEED_NDR; + *width = IB_WIDTH_4X; + break; default: *speed = IB_SPEED_SDR; *width = IB_WIDTH_1X; break; } } /* Port */ int bnxt_re_query_port(struct ib_device *ibdev, u8 port_num, struct ib_port_attr *port_attr) { struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev); struct bnxt_qplib_dev_attr *dev_attr = rdev->dev_attr; u8 active_speed = 0, active_width = 0; dev_dbg(rdev_to_dev(rdev), "QUERY PORT with port_num 0x%x\n", port_num); memset(port_attr, 0, sizeof(*port_attr)); port_attr->phys_state = IB_PORT_PHYS_STATE_DISABLED; port_attr->state = bnxt_re_get_link_state(rdev); if (port_attr->state == IB_PORT_ACTIVE) port_attr->phys_state = IB_PORT_PHYS_STATE_LINK_UP; port_attr->max_mtu = IB_MTU_4096; port_attr->active_mtu = iboe_get_mtu(if_getmtu(rdev->netdev)); port_attr->gid_tbl_len = dev_attr->max_sgid; port_attr->port_cap_flags = IB_PORT_CM_SUP | IB_PORT_REINIT_SUP | IB_PORT_DEVICE_MGMT_SUP | IB_PORT_VENDOR_CLASS_SUP | IB_PORT_IP_BASED_GIDS; port_attr->max_msg_sz = (u32)BNXT_RE_MAX_MR_SIZE_LOW; port_attr->bad_pkey_cntr = 0; port_attr->qkey_viol_cntr = 0; port_attr->pkey_tbl_len = dev_attr->max_pkey; port_attr->lid = 0; port_attr->sm_lid = 0; port_attr->lmc = 0; port_attr->max_vl_num = 4; port_attr->sm_sl = 0; port_attr->subnet_timeout = 0; port_attr->init_type_reply = 0; rdev->espeed = rdev->en_dev->espeed; if (test_bit(BNXT_RE_FLAG_IBDEV_REGISTERED, &rdev->flags)) __to_ib_speed_width(rdev->espeed, &active_speed, &active_width); port_attr->active_speed = active_speed; port_attr->active_width = active_width; return 0; } int bnxt_re_modify_port(struct ib_device *ibdev, u8 port_num, int port_modify_mask, struct ib_port_modify *port_modify) { dev_dbg(rdev_to_dev(rdev), "Modify port with mask 0x%x\n", port_modify_mask); switch (port_modify_mask) { case IB_PORT_SHUTDOWN: break; case IB_PORT_INIT_TYPE: break; case IB_PORT_RESET_QKEY_CNTR: break; default: break; } return 0; } int bnxt_re_get_port_immutable(struct ib_device *ibdev, u8 port_num, struct ib_port_immutable *immutable) { struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev); struct ib_port_attr port_attr; if (bnxt_re_query_port(ibdev, port_num, &port_attr)) return -EINVAL; immutable->pkey_tbl_len = port_attr.pkey_tbl_len; immutable->gid_tbl_len = port_attr.gid_tbl_len; if (rdev->roce_mode == BNXT_RE_FLAG_ROCEV1_CAP) immutable->core_cap_flags = RDMA_CORE_PORT_IBA_ROCE; else if (rdev->roce_mode == BNXT_RE_FLAG_ROCEV2_CAP) immutable->core_cap_flags = RDMA_CORE_PORT_IBA_ROCE_UDP_ENCAP; else immutable->core_cap_flags = RDMA_CORE_PORT_IBA_ROCE | RDMA_CORE_PORT_IBA_ROCE_UDP_ENCAP; immutable->max_mad_size = IB_MGMT_MAD_SIZE; return 0; } void bnxt_re_compat_qfwstr(void) { struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev); sprintf(str, "%d.%d.%d.%d", rdev->dev_attr->fw_ver[0], rdev->dev_attr->fw_ver[1], rdev->dev_attr->fw_ver[2], rdev->dev_attr->fw_ver[3]); } int bnxt_re_query_pkey(struct ib_device *ibdev, u8 port_num, u16 index, u16 *pkey) { if (index > 0) return -EINVAL; *pkey = IB_DEFAULT_PKEY_FULL; return 0; } int bnxt_re_query_gid(struct ib_device *ibdev, u8 port_num, int index, union ib_gid *gid) { struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev); int rc = 0; /* Ignore port_num */ memset(gid, 0, sizeof(*gid)); rc = bnxt_qplib_get_sgid(&rdev->qplib_res, &rdev->qplib_res.sgid_tbl, index, (struct bnxt_qplib_gid *)gid); return rc; } int bnxt_re_del_gid(struct ib_device *ibdev, u8 port_num, unsigned int index, void **context) { int rc = 0; struct bnxt_re_gid_ctx *ctx, **ctx_tbl; struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev); struct bnxt_qplib_sgid_tbl *sgid_tbl = &rdev->qplib_res.sgid_tbl; struct bnxt_qplib_gid *gid_to_del; u16 vlan_id = 0xFFFF; /* Delete the entry from the hardware */ ctx = *context; if (!ctx) { dev_err(rdev_to_dev(rdev), "GID entry has no ctx?!\n"); return -EINVAL; } if (sgid_tbl && sgid_tbl->active) { if (ctx->idx >= sgid_tbl->max) { dev_dbg(rdev_to_dev(rdev), "GID index out of range?!\n"); return -EINVAL; } gid_to_del = &sgid_tbl->tbl[ctx->idx].gid; vlan_id = sgid_tbl->tbl[ctx->idx].vlan_id; ctx->refcnt--; /* DEL_GID is called via WQ context(netdevice_event_work_handler) * or via the ib_unregister_device path. In the former case QP1 * may not be destroyed yet, in which case just return as FW * needs that entry to be present and will fail it's deletion. * We could get invoked again after QP1 is destroyed OR get an * ADD_GID call with a different GID value for the same index * where we issue MODIFY_GID cmd to update the GID entry -- TBD */ if (ctx->idx == 0 && rdma_link_local_addr((struct in6_addr *)gid_to_del) && (rdev->gsi_ctx.gsi_sqp || rdev->gsi_ctx.gsi_qp_mode == BNXT_RE_GSI_MODE_UD)) { dev_dbg(rdev_to_dev(rdev), "Trying to delete GID0 while QP1 is alive\n"); if (!ctx->refcnt) { rdev->gid_map[index] = -1; ctx_tbl = sgid_tbl->ctx; ctx_tbl[ctx->idx] = NULL; kfree(ctx); } return 0; } rdev->gid_map[index] = -1; if (!ctx->refcnt) { rc = bnxt_qplib_del_sgid(sgid_tbl, gid_to_del, vlan_id, true); if (!rc) { dev_dbg(rdev_to_dev(rdev), "GID remove success\n"); ctx_tbl = sgid_tbl->ctx; ctx_tbl[ctx->idx] = NULL; kfree(ctx); } else { dev_err(rdev_to_dev(rdev), "Remove GID failed rc = 0x%x\n", rc); } } } else { dev_dbg(rdev_to_dev(rdev), "GID sgid_tbl does not exist!\n"); return -EINVAL; } return rc; } int bnxt_re_add_gid(struct ib_device *ibdev, u8 port_num, unsigned int index, const union ib_gid *gid, const struct ib_gid_attr *attr, void **context) { int rc; u32 tbl_idx = 0; u16 vlan_id = 0xFFFF; struct bnxt_re_gid_ctx *ctx, **ctx_tbl; struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev); struct bnxt_qplib_sgid_tbl *sgid_tbl = &rdev->qplib_res.sgid_tbl; if ((attr->ndev) && is_vlan_dev(attr->ndev)) vlan_id = vlan_dev_vlan_id(attr->ndev); rc = bnxt_qplib_add_sgid(sgid_tbl, gid, rdev->dev_addr, vlan_id, true, &tbl_idx); if (rc == -EALREADY) { dev_dbg(rdev_to_dev(rdev), "GID %pI6 is already present\n", gid); ctx_tbl = sgid_tbl->ctx; if (!ctx_tbl[tbl_idx]) { ctx = kmalloc(sizeof(*ctx), GFP_KERNEL); if (!ctx) return -ENOMEM; ctx->idx = tbl_idx; ctx->refcnt = 1; ctx_tbl[tbl_idx] = ctx; } else { ctx_tbl[tbl_idx]->refcnt++; } *context = ctx_tbl[tbl_idx]; /* tbl_idx is the HW table index and index is the stack index */ rdev->gid_map[index] = tbl_idx; return 0; } else if (rc < 0) { dev_err(rdev_to_dev(rdev), "Add GID failed rc = 0x%x\n", rc); return rc; } else { ctx = kmalloc(sizeof(*ctx), GFP_KERNEL); if (!ctx) { dev_err(rdev_to_dev(rdev), "Add GID ctx failed\n"); return -ENOMEM; } ctx_tbl = sgid_tbl->ctx; ctx->idx = tbl_idx; ctx->refcnt = 1; ctx_tbl[tbl_idx] = ctx; /* tbl_idx is the HW table index and index is the stack index */ rdev->gid_map[index] = tbl_idx; *context = ctx; } return rc; } enum rdma_link_layer bnxt_re_get_link_layer(struct ib_device *ibdev, u8 port_num) { return IB_LINK_LAYER_ETHERNET; } static void bnxt_re_legacy_create_fence_wqe(struct bnxt_re_pd *pd) { struct bnxt_re_legacy_fence_data *fence = &pd->fence; struct ib_mr *ib_mr = &fence->mr->ib_mr; struct bnxt_qplib_swqe *wqe = &fence->bind_wqe; struct bnxt_re_dev *rdev = pd->rdev; if (!_is_chip_gen_p5_p7(rdev->chip_ctx)) return; memset(wqe, 0, sizeof(*wqe)); wqe->type = BNXT_QPLIB_SWQE_TYPE_BIND_MW; wqe->wr_id = BNXT_QPLIB_FENCE_WRID; wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP; wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE; wqe->bind.zero_based = false; wqe->bind.parent_l_key = ib_mr->lkey; wqe->bind.va = (u64)fence->va; wqe->bind.length = fence->size; wqe->bind.access_cntl = __from_ib_access_flags(IB_ACCESS_REMOTE_READ); wqe->bind.mw_type = SQ_BIND_MW_TYPE_TYPE1; /* Save the initial rkey in fence structure for now; * wqe->bind.r_key will be set at (re)bind time. */ fence->bind_rkey = ib_inc_rkey(fence->mw->rkey); } static int bnxt_re_legacy_bind_fence_mw(struct bnxt_qplib_qp *qplib_qp) { struct bnxt_re_qp *qp = container_of(qplib_qp, struct bnxt_re_qp, qplib_qp); struct ib_pd *ib_pd = qp->ib_qp.pd; struct bnxt_re_pd *pd = to_bnxt_re(ib_pd, struct bnxt_re_pd, ibpd); struct bnxt_re_legacy_fence_data *fence = &pd->fence; struct bnxt_qplib_swqe *fence_wqe = &fence->bind_wqe; struct bnxt_qplib_swqe wqe; int rc; /* TODO: Need SQ locking here when Fence WQE * posting moves up into bnxt_re from bnxt_qplib. */ memcpy(&wqe, fence_wqe, sizeof(wqe)); wqe.bind.r_key = fence->bind_rkey; fence->bind_rkey = ib_inc_rkey(fence->bind_rkey); dev_dbg(rdev_to_dev(qp->rdev), "Posting bind fence-WQE: rkey: %#x QP: %d PD: %p\n", wqe.bind.r_key, qp->qplib_qp.id, pd); rc = bnxt_qplib_post_send(&qp->qplib_qp, &wqe); if (rc) { dev_err(rdev_to_dev(qp->rdev), "Failed to bind fence-WQE\n"); return rc; } bnxt_qplib_post_send_db(&qp->qplib_qp); return rc; } static int bnxt_re_legacy_create_fence_mr(struct bnxt_re_pd *pd) { int mr_access_flags = IB_ACCESS_LOCAL_WRITE | IB_ACCESS_MW_BIND; struct bnxt_re_legacy_fence_data *fence = &pd->fence; struct bnxt_re_dev *rdev = pd->rdev; struct bnxt_qplib_mrinfo mrinfo; struct bnxt_re_mr *mr = NULL; struct ib_mw *ib_mw = NULL; dma_addr_t dma_addr = 0; u32 max_mr_count; u64 pbl_tbl; int rc; if (!_is_chip_gen_p5_p7(rdev->chip_ctx)) return 0; memset(&mrinfo, 0, sizeof(mrinfo)); /* Allocate a small chunk of memory and dma-map it */ fence->va = kzalloc(BNXT_RE_LEGACY_FENCE_BYTES, GFP_KERNEL); if (!fence->va) return -ENOMEM; dma_addr = ib_dma_map_single(&rdev->ibdev, fence->va, BNXT_RE_LEGACY_FENCE_BYTES, DMA_BIDIRECTIONAL); rc = ib_dma_mapping_error(&rdev->ibdev, dma_addr); if (rc) { dev_err(rdev_to_dev(rdev), "Failed to dma-map fence-MR-mem\n"); rc = -EIO; fence->dma_addr = 0; goto free_va; } fence->dma_addr = dma_addr; /* Allocate a MR */ mr = kzalloc(sizeof(*mr), GFP_KERNEL); if (!mr) goto free_dma_addr; fence->mr = mr; mr->rdev = rdev; mr->qplib_mr.pd = &pd->qplib_pd; mr->qplib_mr.type = CMDQ_ALLOCATE_MRW_MRW_FLAGS_PMR; mr->qplib_mr.flags = __from_ib_access_flags(mr_access_flags); if (!_is_alloc_mr_unified(rdev->qplib_res.dattr)) { rc = bnxt_qplib_alloc_mrw(&rdev->qplib_res, &mr->qplib_mr); if (rc) { dev_err(rdev_to_dev(rdev), "Failed to alloc fence-HW-MR\n"); goto free_mr; } /* Register MR */ mr->ib_mr.lkey = mr->qplib_mr.lkey; } mr->qplib_mr.va = (u64)fence->va; mr->qplib_mr.total_size = BNXT_RE_LEGACY_FENCE_BYTES; pbl_tbl = dma_addr; mrinfo.mrw = &mr->qplib_mr; mrinfo.ptes = &pbl_tbl; mrinfo.sg.npages = BNXT_RE_LEGACY_FENCE_PBL_SIZE; mrinfo.sg.nmap = 0; mrinfo.sg.sghead = 0; mrinfo.sg.pgshft = PAGE_SHIFT; mrinfo.sg.pgsize = PAGE_SIZE; rc = bnxt_qplib_reg_mr(&rdev->qplib_res, &mrinfo, false); if (rc) { dev_err(rdev_to_dev(rdev), "Failed to register fence-MR\n"); goto free_mr; } mr->ib_mr.lkey = mr->qplib_mr.lkey; mr->ib_mr.rkey = mr->qplib_mr.rkey; atomic_inc(&rdev->stats.rsors.mr_count); max_mr_count = atomic_read(&rdev->stats.rsors.mr_count); if (max_mr_count > (atomic_read(&rdev->stats.rsors.max_mr_count))) atomic_set(&rdev->stats.rsors.max_mr_count, max_mr_count); ib_mw = bnxt_re_alloc_mw(&pd->ibpd, IB_MW_TYPE_1, NULL); /* Create a fence MW only for kernel consumers */ if (!ib_mw) { dev_err(rdev_to_dev(rdev), "Failed to create fence-MW for PD: %p\n", pd); rc = -EINVAL; goto free_mr; } fence->mw = ib_mw; bnxt_re_legacy_create_fence_wqe(pd); return 0; free_mr: if (mr->ib_mr.lkey) { bnxt_qplib_free_mrw(&rdev->qplib_res, &mr->qplib_mr); atomic_dec(&rdev->stats.rsors.mr_count); } kfree(mr); fence->mr = NULL; free_dma_addr: ib_dma_unmap_single(&rdev->ibdev, fence->dma_addr, BNXT_RE_LEGACY_FENCE_BYTES, DMA_BIDIRECTIONAL); fence->dma_addr = 0; free_va: kfree(fence->va); fence->va = NULL; return rc; } static void bnxt_re_legacy_destroy_fence_mr(struct bnxt_re_pd *pd) { struct bnxt_re_legacy_fence_data *fence = &pd->fence; struct bnxt_re_dev *rdev = pd->rdev; struct bnxt_re_mr *mr = fence->mr; if (!_is_chip_gen_p5_p7(rdev->chip_ctx)) return; if (fence->mw) { bnxt_re_dealloc_mw(fence->mw); fence->mw = NULL; } if (mr) { if (mr->ib_mr.rkey) bnxt_qplib_dereg_mrw(&rdev->qplib_res, &mr->qplib_mr, false); if (mr->ib_mr.lkey) bnxt_qplib_free_mrw(&rdev->qplib_res, &mr->qplib_mr); kfree(mr); fence->mr = NULL; atomic_dec(&rdev->stats.rsors.mr_count); } if (fence->dma_addr) { ib_dma_unmap_single(&rdev->ibdev, fence->dma_addr, BNXT_RE_LEGACY_FENCE_BYTES, DMA_BIDIRECTIONAL); fence->dma_addr = 0; } kfree(fence->va); fence->va = NULL; } static int bnxt_re_get_user_dpi(struct bnxt_re_dev *rdev, struct bnxt_re_ucontext *cntx) { struct bnxt_qplib_chip_ctx *cctx = rdev->chip_ctx; int ret = 0; u8 type; /* Allocate DPI in alloc_pd or in create_cq to avoid failing of * ibv_devinfo and family of application when DPIs are depleted. */ type = BNXT_QPLIB_DPI_TYPE_UC; ret = bnxt_qplib_alloc_dpi(&rdev->qplib_res, &cntx->dpi, cntx, type); if (ret) { dev_err(rdev_to_dev(rdev), "Alloc doorbell page failed!\n"); goto out; } if (cctx->modes.db_push) { type = BNXT_QPLIB_DPI_TYPE_WC; ret = bnxt_qplib_alloc_dpi(&rdev->qplib_res, &cntx->wcdpi, cntx, type); if (ret) dev_err(rdev_to_dev(rdev), "push dp alloc failed\n"); } out: return ret; } /* Protection Domains */ void bnxt_re_dealloc_pd(struct ib_pd *ib_pd, struct ib_udata *udata) { struct bnxt_re_pd *pd = to_bnxt_re(ib_pd, struct bnxt_re_pd, ibpd); struct bnxt_re_dev *rdev = pd->rdev; int rc; bnxt_re_legacy_destroy_fence_mr(pd); rc = bnxt_qplib_dealloc_pd(&rdev->qplib_res, &rdev->qplib_res.pd_tbl, &pd->qplib_pd); if (rc) dev_err_ratelimited(rdev_to_dev(rdev), "%s failed rc = %d\n", __func__, rc); atomic_dec(&rdev->stats.rsors.pd_count); return; } int bnxt_re_alloc_pd(struct ib_pd *pd_in, struct ib_udata *udata) { struct ib_pd *ibpd = pd_in; struct ib_device *ibdev = ibpd->device; struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev); struct bnxt_re_ucontext *ucntx = rdma_udata_to_drv_context(udata, struct bnxt_re_ucontext, ibucontext); u32 max_pd_count; int rc; struct bnxt_re_pd *pd = container_of(ibpd, struct bnxt_re_pd, ibpd); pd->rdev = rdev; if (bnxt_qplib_alloc_pd(&rdev->qplib_res, &pd->qplib_pd)) { dev_err(rdev_to_dev(rdev), "Allocate HW Protection Domain failed!\n"); rc = -ENOMEM; goto fail; } if (udata) { struct bnxt_re_pd_resp resp = {}; if (!ucntx->dpi.dbr) { rc = bnxt_re_get_user_dpi(rdev, ucntx); if (rc) goto dbfail; } resp.pdid = pd->qplib_pd.id; /* Still allow mapping this DBR to the new user PD. */ resp.dpi = ucntx->dpi.dpi; resp.dbr = (u64)ucntx->dpi.umdbr; /* Copy only on a valid wcpdi */ if (ucntx->wcdpi.dpi) { resp.wcdpi = ucntx->wcdpi.dpi; resp.comp_mask = BNXT_RE_COMP_MASK_PD_HAS_WC_DPI; } if (rdev->dbr_pacing) { WARN_ON(!rdev->dbr_bar_addr); resp.dbr_bar_addr = (u64)rdev->dbr_bar_addr; resp.comp_mask |= BNXT_RE_COMP_MASK_PD_HAS_DBR_BAR_ADDR; } rc = bnxt_re_copy_to_udata(rdev, &resp, min(udata->outlen, sizeof(resp)), udata); if (rc) goto dbfail; } if (!udata) if (bnxt_re_legacy_create_fence_mr(pd)) dev_warn(rdev_to_dev(rdev), "Failed to create Fence-MR\n"); atomic_inc(&rdev->stats.rsors.pd_count); max_pd_count = atomic_read(&rdev->stats.rsors.pd_count); if (max_pd_count > atomic_read(&rdev->stats.rsors.max_pd_count)) atomic_set(&rdev->stats.rsors.max_pd_count, max_pd_count); return 0; dbfail: (void)bnxt_qplib_dealloc_pd(&rdev->qplib_res, &rdev->qplib_res.pd_tbl, &pd->qplib_pd); fail: return rc; } /* Address Handles */ void bnxt_re_destroy_ah(struct ib_ah *ib_ah, u32 flags) { struct bnxt_re_ah *ah = to_bnxt_re(ib_ah, struct bnxt_re_ah, ibah); struct bnxt_re_dev *rdev = ah->rdev; int rc = 0; bool block = true; block = !(flags & RDMA_DESTROY_AH_SLEEPABLE); rc = bnxt_qplib_destroy_ah(&rdev->qplib_res, &ah->qplib_ah, block); if (rc) dev_err_ratelimited(rdev_to_dev(rdev), "%s id = %d blocking %d failed rc = %d\n", __func__, ah->qplib_ah.id, block, rc); atomic_dec(&rdev->stats.rsors.ah_count); return; } static u8 _to_bnxt_re_nw_type(enum rdma_network_type ntype) { u8 nw_type; switch (ntype) { case RDMA_NETWORK_IPV4: nw_type = CMDQ_CREATE_AH_TYPE_V2IPV4; break; case RDMA_NETWORK_IPV6: nw_type = CMDQ_CREATE_AH_TYPE_V2IPV6; break; default: nw_type = CMDQ_CREATE_AH_TYPE_V1; break; } return nw_type; } static inline int bnxt_re_get_cached_gid(struct ib_device *dev, u8 port_num, int index, union ib_gid *sgid, struct ib_gid_attr **sgid_attr, struct ib_global_route *grh, struct ib_ah *ah) { int ret = 0; ret = ib_get_cached_gid(dev, port_num, index, sgid, *sgid_attr); return ret; } static inline enum rdma_network_type bnxt_re_gid_to_network_type(struct ib_gid_attr *sgid_attr, union ib_gid *sgid) { return ib_gid_to_network_type(sgid_attr->gid_type, sgid); } static int bnxt_re_get_ah_info(struct bnxt_re_dev *rdev, struct ib_ah_attr *ah_attr, struct bnxt_re_ah_info *ah_info) { struct ib_gid_attr *gattr; enum rdma_network_type ib_ntype; u8 ntype; union ib_gid *gid; int rc = 0; gid = &ah_info->sgid; gattr = &ah_info->sgid_attr; rc = bnxt_re_get_cached_gid(&rdev->ibdev, 1, ah_attr->grh.sgid_index, gid, &gattr, &ah_attr->grh, NULL); if (rc) return rc; /* Get vlan tag */ if (gattr->ndev) { if (is_vlan_dev(gattr->ndev)) ah_info->vlan_tag = vlan_dev_vlan_id(gattr->ndev); if_rele(gattr->ndev); } /* Get network header type for this GID */ ib_ntype = bnxt_re_gid_to_network_type(gattr, gid); ntype = _to_bnxt_re_nw_type(ib_ntype); ah_info->nw_type = ntype; return rc; } static u8 _get_sgid_index(struct bnxt_re_dev *rdev, u8 gindx) { gindx = rdev->gid_map[gindx]; return gindx; } static int bnxt_re_init_dmac(struct bnxt_re_dev *rdev, struct ib_ah_attr *ah_attr, struct bnxt_re_ah_info *ah_info, bool is_user, struct bnxt_re_ah *ah) { int rc = 0; u8 *dmac; if (is_user && !rdma_is_multicast_addr((struct in6_addr *) ah_attr->grh.dgid.raw) && !rdma_link_local_addr((struct in6_addr *)ah_attr->grh.dgid.raw)) { u32 retry_count = BNXT_RE_RESOLVE_RETRY_COUNT_US; struct bnxt_re_resolve_dmac_work *resolve_dmac_work; resolve_dmac_work = kzalloc(sizeof(*resolve_dmac_work), GFP_ATOMIC); resolve_dmac_work->rdev = rdev; resolve_dmac_work->ah_attr = ah_attr; resolve_dmac_work->ah_info = ah_info; atomic_set(&resolve_dmac_work->status_wait, 1); INIT_WORK(&resolve_dmac_work->work, bnxt_re_resolve_dmac_task); queue_work(rdev->resolve_wq, &resolve_dmac_work->work); do { rc = atomic_read(&resolve_dmac_work->status_wait) & 0xFF; if (!rc) break; udelay(1); } while (--retry_count); if (atomic_read(&resolve_dmac_work->status_wait)) { INIT_LIST_HEAD(&resolve_dmac_work->list); list_add_tail(&resolve_dmac_work->list, &rdev->mac_wq_list); return -EFAULT; } kfree(resolve_dmac_work); } dmac = ROCE_DMAC(ah_attr); if (dmac) memcpy(ah->qplib_ah.dmac, dmac, ETH_ALEN); return rc; } int bnxt_re_create_ah(struct ib_ah *ah_in, struct ib_ah_attr *attr, u32 flags, struct ib_udata *udata) { struct ib_ah *ib_ah = ah_in; struct ib_pd *ib_pd = ib_ah->pd; struct bnxt_re_ah *ah = container_of(ib_ah, struct bnxt_re_ah, ibah); struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ibpd); struct bnxt_re_dev *rdev = pd->rdev; struct bnxt_re_ah_info ah_info; u32 max_ah_count; bool is_user; int rc; bool block = true; struct ib_ah_attr *ah_attr = attr; block = !(flags & RDMA_CREATE_AH_SLEEPABLE); if (!(ah_attr->ah_flags & IB_AH_GRH)) dev_err(rdev_to_dev(rdev), "ah_attr->ah_flags GRH is not set\n"); ah->rdev = rdev; ah->qplib_ah.pd = &pd->qplib_pd; is_user = ib_pd->uobject ? true : false; /* Supply the configuration for the HW */ memcpy(ah->qplib_ah.dgid.data, ah_attr->grh.dgid.raw, sizeof(union ib_gid)); ah->qplib_ah.sgid_index = _get_sgid_index(rdev, ah_attr->grh.sgid_index); if (ah->qplib_ah.sgid_index == 0xFF) { dev_err(rdev_to_dev(rdev), "invalid sgid_index!\n"); rc = -EINVAL; goto fail; } ah->qplib_ah.host_sgid_index = ah_attr->grh.sgid_index; ah->qplib_ah.traffic_class = ah_attr->grh.traffic_class; ah->qplib_ah.flow_label = ah_attr->grh.flow_label; ah->qplib_ah.hop_limit = ah_attr->grh.hop_limit; ah->qplib_ah.sl = ah_attr->sl; rc = bnxt_re_get_ah_info(rdev, ah_attr, &ah_info); if (rc) goto fail; ah->qplib_ah.nw_type = ah_info.nw_type; rc = bnxt_re_init_dmac(rdev, ah_attr, &ah_info, is_user, ah); if (rc) goto fail; rc = bnxt_qplib_create_ah(&rdev->qplib_res, &ah->qplib_ah, block); if (rc) { dev_err(rdev_to_dev(rdev), "Allocate HW Address Handle failed!\n"); goto fail; } /* Write AVID to shared page. */ if (ib_pd->uobject) { struct ib_ucontext *ib_uctx = ib_pd->uobject->context; struct bnxt_re_ucontext *uctx; unsigned long flag; u32 *wrptr; uctx = to_bnxt_re(ib_uctx, struct bnxt_re_ucontext, ibucontext); spin_lock_irqsave(&uctx->sh_lock, flag); wrptr = (u32 *)((u8 *)uctx->shpg + BNXT_RE_AVID_OFFT); *wrptr = ah->qplib_ah.id; wmb(); /* make sure cache is updated. */ spin_unlock_irqrestore(&uctx->sh_lock, flag); } atomic_inc(&rdev->stats.rsors.ah_count); max_ah_count = atomic_read(&rdev->stats.rsors.ah_count); if (max_ah_count > atomic_read(&rdev->stats.rsors.max_ah_count)) atomic_set(&rdev->stats.rsors.max_ah_count, max_ah_count); return 0; fail: return rc; } int bnxt_re_modify_ah(struct ib_ah *ib_ah, struct ib_ah_attr *ah_attr) { return 0; } int bnxt_re_query_ah(struct ib_ah *ib_ah, struct ib_ah_attr *ah_attr) { struct bnxt_re_ah *ah = to_bnxt_re(ib_ah, struct bnxt_re_ah, ibah); memcpy(ah_attr->grh.dgid.raw, ah->qplib_ah.dgid.data, sizeof(union ib_gid)); ah_attr->grh.sgid_index = ah->qplib_ah.host_sgid_index; ah_attr->grh.traffic_class = ah->qplib_ah.traffic_class; ah_attr->sl = ah->qplib_ah.sl; memcpy(ROCE_DMAC(ah_attr), ah->qplib_ah.dmac, ETH_ALEN); ah_attr->ah_flags = IB_AH_GRH; ah_attr->port_num = 1; ah_attr->static_rate = 0; return 0; } /* Shared Receive Queues */ void bnxt_re_destroy_srq(struct ib_srq *ib_srq, struct ib_udata *udata) { struct bnxt_re_srq *srq = to_bnxt_re(ib_srq, struct bnxt_re_srq, ibsrq); struct bnxt_re_dev *rdev = srq->rdev; struct bnxt_qplib_srq *qplib_srq = &srq->qplib_srq; int rc = 0; rc = bnxt_qplib_destroy_srq(&rdev->qplib_res, qplib_srq); if (rc) dev_err_ratelimited(rdev_to_dev(rdev), "%s id = %d failed rc = %d\n", __func__, qplib_srq->id, rc); if (srq->umem && !IS_ERR(srq->umem)) ib_umem_release(srq->umem); atomic_dec(&rdev->stats.rsors.srq_count); return; } static u16 _max_rwqe_sz(int nsge) { return sizeof(struct rq_wqe_hdr) + (nsge * sizeof(struct sq_sge)); } static u16 bnxt_re_get_rwqe_size(struct bnxt_qplib_qp *qplqp, int rsge, int max) { if (qplqp->wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC) rsge = max; return _max_rwqe_sz(rsge); } static inline struct ib_umem *ib_umem_get_compat(struct bnxt_re_dev *rdev, struct ib_ucontext *ucontext, struct ib_udata *udata, unsigned long addr, size_t size, int access, int dmasync) { return ib_umem_get(ucontext, addr, size, access, dmasync); } static inline struct ib_umem *ib_umem_get_flags_compat(struct bnxt_re_dev *rdev, struct ib_ucontext *ucontext, struct ib_udata *udata, unsigned long addr, size_t size, int access, int dmasync) { return ib_umem_get_compat(rdev, ucontext, udata, addr, size, access, 0); } static inline size_t ib_umem_num_pages_compat(struct ib_umem *umem) { return ib_umem_num_pages(umem); } static int bnxt_re_init_user_srq(struct bnxt_re_dev *rdev, struct bnxt_re_pd *pd, struct bnxt_re_srq *srq, struct ib_udata *udata) { struct bnxt_qplib_sg_info *sginfo; struct bnxt_qplib_srq *qplib_srq; struct bnxt_re_ucontext *cntx; struct ib_ucontext *context; struct bnxt_re_srq_req ureq; struct ib_umem *umem; int rc, bytes = 0; context = pd->ibpd.uobject->context; cntx = to_bnxt_re(context, struct bnxt_re_ucontext, ibucontext); qplib_srq = &srq->qplib_srq; sginfo = &qplib_srq->sginfo; if (udata->inlen < sizeof(ureq)) dev_warn(rdev_to_dev(rdev), "Update the library ulen %d klen %d\n", (unsigned int)udata->inlen, (unsigned int)sizeof(ureq)); rc = ib_copy_from_udata(&ureq, udata, min(udata->inlen, sizeof(ureq))); if (rc) return rc; bytes = (qplib_srq->max_wqe * qplib_srq->wqe_size); bytes = PAGE_ALIGN(bytes); umem = ib_umem_get_compat(rdev, context, udata, ureq.srqva, bytes, IB_ACCESS_LOCAL_WRITE, 1); if (IS_ERR(umem)) { dev_err(rdev_to_dev(rdev), "%s: ib_umem_get failed with %ld\n", __func__, PTR_ERR(umem)); return PTR_ERR(umem); } srq->umem = umem; sginfo->sghead = get_ib_umem_sgl(umem, &sginfo->nmap); sginfo->npages = ib_umem_num_pages_compat(umem); qplib_srq->srq_handle = ureq.srq_handle; qplib_srq->dpi = &cntx->dpi; qplib_srq->is_user = true; return 0; } int bnxt_re_create_srq(struct ib_srq *srq_in, struct ib_srq_init_attr *srq_init_attr, struct ib_udata *udata) { struct bnxt_qplib_dev_attr *dev_attr; struct bnxt_re_ucontext *cntx = NULL; struct ib_ucontext *context; struct bnxt_re_dev *rdev; struct bnxt_re_pd *pd; int rc, entries; struct ib_srq *ib_srq = srq_in; struct ib_pd *ib_pd = ib_srq->pd; struct bnxt_re_srq *srq = container_of(ib_srq, struct bnxt_re_srq, ibsrq); u32 max_srq_count; pd = to_bnxt_re(ib_pd, struct bnxt_re_pd, ibpd); rdev = pd->rdev; dev_attr = rdev->dev_attr; if (rdev->mod_exit) { dev_dbg(rdev_to_dev(rdev), "%s(): in mod_exit, just return!\n", __func__); rc = -EIO; goto exit; } if (srq_init_attr->srq_type != IB_SRQT_BASIC) { dev_err(rdev_to_dev(rdev), "SRQ type not supported\n"); rc = -ENOTSUPP; goto exit; } if (udata) { context = pd->ibpd.uobject->context; cntx = to_bnxt_re(context, struct bnxt_re_ucontext, ibucontext); } if (atomic_read(&rdev->stats.rsors.srq_count) >= dev_attr->max_srq) { dev_err(rdev_to_dev(rdev), "Create SRQ failed - max exceeded(SRQs)\n"); rc = -EINVAL; goto exit; } if (srq_init_attr->attr.max_wr >= dev_attr->max_srq_wqes) { dev_err(rdev_to_dev(rdev), "Create SRQ failed - max exceeded(SRQ_WQs)\n"); rc = -EINVAL; goto exit; } srq->rdev = rdev; srq->qplib_srq.pd = &pd->qplib_pd; srq->qplib_srq.dpi = &rdev->dpi_privileged; /* Allocate 1 more than what's provided so posting max doesn't mean empty */ entries = srq_init_attr->attr.max_wr + 1; entries = bnxt_re_init_depth(entries, cntx); if (entries > dev_attr->max_srq_wqes + 1) entries = dev_attr->max_srq_wqes + 1; srq->qplib_srq.wqe_size = _max_rwqe_sz(6); /* 128 byte wqe size */ srq->qplib_srq.max_wqe = entries; srq->qplib_srq.max_sge = srq_init_attr->attr.max_sge; srq->qplib_srq.threshold = srq_init_attr->attr.srq_limit; srq->srq_limit = srq_init_attr->attr.srq_limit; srq->qplib_srq.eventq_hw_ring_id = rdev->nqr.nq[0].ring_id; srq->qplib_srq.sginfo.pgsize = PAGE_SIZE; srq->qplib_srq.sginfo.pgshft = PAGE_SHIFT; if (udata) { rc = bnxt_re_init_user_srq(rdev, pd, srq, udata); if (rc) goto fail; } rc = bnxt_qplib_create_srq(&rdev->qplib_res, &srq->qplib_srq); if (rc) { dev_err(rdev_to_dev(rdev), "Create HW SRQ failed!\n"); goto fail; } if (udata) { struct bnxt_re_srq_resp resp; resp.srqid = srq->qplib_srq.id; rc = bnxt_re_copy_to_udata(rdev, &resp, min(udata->outlen, sizeof(resp)), udata); if (rc) { bnxt_qplib_destroy_srq(&rdev->qplib_res, &srq->qplib_srq); goto fail; } } atomic_inc(&rdev->stats.rsors.srq_count); max_srq_count = atomic_read(&rdev->stats.rsors.srq_count); if (max_srq_count > atomic_read(&rdev->stats.rsors.max_srq_count)) atomic_set(&rdev->stats.rsors.max_srq_count, max_srq_count); spin_lock_init(&srq->lock); return 0; fail: if (udata && srq->umem && !IS_ERR(srq->umem)) { ib_umem_release(srq->umem); srq->umem = NULL; } exit: return rc; } int bnxt_re_modify_srq(struct ib_srq *ib_srq, struct ib_srq_attr *srq_attr, enum ib_srq_attr_mask srq_attr_mask, struct ib_udata *udata) { struct bnxt_re_srq *srq = to_bnxt_re(ib_srq, struct bnxt_re_srq, ibsrq); struct bnxt_re_dev *rdev = srq->rdev; int rc; switch (srq_attr_mask) { case IB_SRQ_MAX_WR: /* SRQ resize is not supported */ break; case IB_SRQ_LIMIT: /* Change the SRQ threshold */ if (srq_attr->srq_limit > srq->qplib_srq.max_wqe) return -EINVAL; srq->qplib_srq.threshold = srq_attr->srq_limit; rc = bnxt_qplib_modify_srq(&rdev->qplib_res, &srq->qplib_srq); if (rc) { dev_err(rdev_to_dev(rdev), "Modify HW SRQ failed!\n"); return rc; } /* On success, update the shadow */ srq->srq_limit = srq_attr->srq_limit; if (udata) { /* Build and send response back to udata */ rc = bnxt_re_copy_to_udata(rdev, srq, 0, udata); if (rc) return rc; } break; default: dev_err(rdev_to_dev(rdev), "Unsupported srq_attr_mask 0x%x\n", srq_attr_mask); return -EINVAL; } return 0; } int bnxt_re_query_srq(struct ib_srq *ib_srq, struct ib_srq_attr *srq_attr) { struct bnxt_re_srq *srq = to_bnxt_re(ib_srq, struct bnxt_re_srq, ibsrq); struct bnxt_re_dev *rdev = srq->rdev; int rc; rc = bnxt_qplib_query_srq(&rdev->qplib_res, &srq->qplib_srq); if (rc) { dev_err(rdev_to_dev(rdev), "Query HW SRQ (0x%x) failed! rc = %d\n", srq->qplib_srq.id, rc); return rc; } srq_attr->max_wr = srq->qplib_srq.max_wqe; srq_attr->max_sge = srq->qplib_srq.max_sge; srq_attr->srq_limit = srq->qplib_srq.threshold; return 0; } int bnxt_re_post_srq_recv(struct ib_srq *ib_srq, const struct ib_recv_wr *wr, const struct ib_recv_wr **bad_wr) { struct bnxt_re_srq *srq = to_bnxt_re(ib_srq, struct bnxt_re_srq, ibsrq); struct bnxt_qplib_swqe wqe = {}; unsigned long flags; int rc = 0; spin_lock_irqsave(&srq->lock, flags); while (wr) { /* Transcribe each ib_recv_wr to qplib_swqe */ wqe.num_sge = wr->num_sge; wqe.sg_list = (struct bnxt_qplib_sge *)wr->sg_list; wqe.wr_id = wr->wr_id; wqe.type = BNXT_QPLIB_SWQE_TYPE_RECV; rc = bnxt_qplib_post_srq_recv(&srq->qplib_srq, &wqe); if (rc) { *bad_wr = wr; break; } wr = wr->next; } spin_unlock_irqrestore(&srq->lock, flags); return rc; } unsigned long bnxt_re_lock_cqs(struct bnxt_re_qp *qp) { unsigned long flags; spin_lock_irqsave(&qp->scq->cq_lock, flags); if (qp->rcq && qp->rcq != qp->scq) spin_lock(&qp->rcq->cq_lock); return flags; } void bnxt_re_unlock_cqs(struct bnxt_re_qp *qp, unsigned long flags) { if (qp->rcq && qp->rcq != qp->scq) spin_unlock(&qp->rcq->cq_lock); spin_unlock_irqrestore(&qp->scq->cq_lock, flags); } /* Queue Pairs */ static int bnxt_re_destroy_gsi_sqp(struct bnxt_re_qp *qp) { struct bnxt_re_qp *gsi_sqp; struct bnxt_re_ah *gsi_sah; struct bnxt_re_dev *rdev; unsigned long flags; int rc = 0; rdev = qp->rdev; gsi_sqp = rdev->gsi_ctx.gsi_sqp; gsi_sah = rdev->gsi_ctx.gsi_sah; /* remove from active qp list */ mutex_lock(&rdev->qp_lock); list_del(&gsi_sqp->list); mutex_unlock(&rdev->qp_lock); if (gsi_sah) { dev_dbg(rdev_to_dev(rdev), "Destroy the shadow AH\n"); rc = bnxt_qplib_destroy_ah(&rdev->qplib_res, &gsi_sah->qplib_ah, true); if (rc) dev_err(rdev_to_dev(rdev), "Destroy HW AH for shadow QP failed!\n"); atomic_dec(&rdev->stats.rsors.ah_count); } dev_dbg(rdev_to_dev(rdev), "Destroy the shadow QP\n"); rc = bnxt_qplib_destroy_qp(&rdev->qplib_res, &gsi_sqp->qplib_qp); if (rc) dev_err(rdev_to_dev(rdev), "Destroy Shadow QP failed\n"); /* Clean the CQ for shadow QP completions */ flags = bnxt_re_lock_cqs(gsi_sqp); bnxt_qplib_clean_qp(&gsi_sqp->qplib_qp); bnxt_re_unlock_cqs(gsi_sqp, flags); bnxt_qplib_free_qp_res(&rdev->qplib_res, &gsi_sqp->qplib_qp); bnxt_qplib_free_hdr_buf(&rdev->qplib_res, &gsi_sqp->qplib_qp); kfree(rdev->gsi_ctx.sqp_tbl); kfree(gsi_sah); kfree(gsi_sqp); rdev->gsi_ctx.gsi_sqp = NULL; rdev->gsi_ctx.gsi_sah = NULL; rdev->gsi_ctx.sqp_tbl = NULL; atomic_dec(&rdev->stats.rsors.qp_count); return 0; } static void bnxt_re_dump_debug_stats(struct bnxt_re_dev *rdev, u32 active_qps) { u32 total_qp = 0; u64 avg_time = 0; int i; if (!rdev->rcfw.sp_perf_stats_enabled) return; switch (active_qps) { case 1: /* Potential hint for Test Stop */ for (i = 0; i < RCFW_MAX_STAT_INDEX; i++) { if (rdev->rcfw.qp_destroy_stats[i]) { total_qp++; avg_time += rdev->rcfw.qp_destroy_stats[i]; } } if (total_qp >= 0 || avg_time >= 0) dev_dbg(rdev_to_dev(rdev), "Perf Debug: %ps Total (%d) QP destroyed in (%ld) msec\n", __builtin_return_address(0), total_qp, (long)jiffies_to_msecs(avg_time)); break; case 2: /* Potential hint for Test Start */ dev_dbg(rdev_to_dev(rdev), "Perf Debug: %ps active_qps = %d\n", __builtin_return_address(0), active_qps); break; default: /* Potential hint to know latency of QP destroy. * Average time taken for 1K QP Destroy. */ if (active_qps > 1024 && !(active_qps % 1024)) dev_dbg(rdev_to_dev(rdev), "Perf Debug: %ps Active QP (%d) Watermark (%d)\n", __builtin_return_address(0), active_qps, atomic_read(&rdev->stats.rsors.max_qp_count)); break; } } int bnxt_re_destroy_qp(struct ib_qp *ib_qp, struct ib_udata *udata) { struct bnxt_re_qp *qp = to_bnxt_re(ib_qp, struct bnxt_re_qp, ib_qp); struct bnxt_re_dev *rdev = qp->rdev; unsigned long flags; u32 active_qps; int rc; mutex_lock(&rdev->qp_lock); list_del(&qp->list); active_qps = atomic_dec_return(&rdev->stats.rsors.qp_count); if (qp->qplib_qp.type == CMDQ_CREATE_QP_TYPE_RC) atomic_dec(&rdev->stats.rsors.rc_qp_count); else if (qp->qplib_qp.type == CMDQ_CREATE_QP_TYPE_UD) atomic_dec(&rdev->stats.rsors.ud_qp_count); mutex_unlock(&rdev->qp_lock); rc = bnxt_qplib_destroy_qp(&rdev->qplib_res, &qp->qplib_qp); if (rc) dev_err_ratelimited(rdev_to_dev(rdev), "%s id = %d failed rc = %d\n", __func__, qp->qplib_qp.id, rc); if (!ib_qp->uobject) { flags = bnxt_re_lock_cqs(qp); bnxt_qplib_clean_qp(&qp->qplib_qp); bnxt_re_unlock_cqs(qp, flags); } bnxt_qplib_free_qp_res(&rdev->qplib_res, &qp->qplib_qp); if (ib_qp->qp_type == IB_QPT_GSI && rdev->gsi_ctx.gsi_qp_mode != BNXT_RE_GSI_MODE_UD) { if (rdev->gsi_ctx.gsi_qp_mode == BNXT_RE_GSI_MODE_ALL && rdev->gsi_ctx.gsi_sqp) { bnxt_re_destroy_gsi_sqp(qp); } bnxt_qplib_free_hdr_buf(&rdev->qplib_res, &qp->qplib_qp); } if (qp->rumem && !IS_ERR(qp->rumem)) ib_umem_release(qp->rumem); if (qp->sumem && !IS_ERR(qp->sumem)) ib_umem_release(qp->sumem); kfree(qp); bnxt_re_dump_debug_stats(rdev, active_qps); return 0; } static u8 __from_ib_qp_type(enum ib_qp_type type) { switch (type) { case IB_QPT_GSI: return CMDQ_CREATE_QP1_TYPE_GSI; case IB_QPT_RC: return CMDQ_CREATE_QP_TYPE_RC; case IB_QPT_UD: return CMDQ_CREATE_QP_TYPE_UD; case IB_QPT_RAW_ETHERTYPE: return CMDQ_CREATE_QP_TYPE_RAW_ETHERTYPE; default: return IB_QPT_MAX; } } static u16 _get_swqe_sz(int nsge) { return sizeof(struct sq_send_hdr) + nsge * sizeof(struct sq_sge); } static int bnxt_re_get_swqe_size(int ilsize, int nsge) { u16 wqe_size, calc_ils; wqe_size = _get_swqe_sz(nsge); if (ilsize) { calc_ils = (sizeof(struct sq_send_hdr) + ilsize); wqe_size = max_t(int, calc_ils, wqe_size); wqe_size = ALIGN(wqe_size, 32); } return wqe_size; } static int bnxt_re_setup_swqe_size(struct bnxt_re_qp *qp, struct ib_qp_init_attr *init_attr) { struct bnxt_qplib_dev_attr *dev_attr; struct bnxt_qplib_qp *qplqp; struct bnxt_re_dev *rdev; struct bnxt_qplib_q *sq; int align, ilsize; rdev = qp->rdev; qplqp = &qp->qplib_qp; sq = &qplqp->sq; dev_attr = rdev->dev_attr; align = sizeof(struct sq_send_hdr); ilsize = ALIGN(init_attr->cap.max_inline_data, align); sq->wqe_size = bnxt_re_get_swqe_size(ilsize, sq->max_sge); if (sq->wqe_size > _get_swqe_sz(dev_attr->max_qp_sges)) return -EINVAL; /* For Cu/Wh and gen p5 backward compatibility mode * wqe size is fixed to 128 bytes */ if (sq->wqe_size < _get_swqe_sz(dev_attr->max_qp_sges) && qplqp->wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC) sq->wqe_size = _get_swqe_sz(dev_attr->max_qp_sges); if (init_attr->cap.max_inline_data) { qplqp->max_inline_data = sq->wqe_size - sizeof(struct sq_send_hdr); init_attr->cap.max_inline_data = qplqp->max_inline_data; if (qplqp->wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC) sq->max_sge = qplqp->max_inline_data / sizeof(struct sq_sge); } return 0; } static int bnxt_re_init_user_qp(struct bnxt_re_dev *rdev, struct bnxt_re_pd *pd, struct bnxt_re_qp *qp, struct ib_udata *udata) { struct bnxt_qplib_sg_info *sginfo; struct bnxt_qplib_qp *qplib_qp; struct bnxt_re_ucontext *cntx; struct ib_ucontext *context; struct bnxt_re_qp_req ureq; struct ib_umem *umem; int rc, bytes = 0; int psn_nume; int psn_sz; qplib_qp = &qp->qplib_qp; context = pd->ibpd.uobject->context; cntx = to_bnxt_re(context, struct bnxt_re_ucontext, ibucontext); sginfo = &qplib_qp->sq.sginfo; if (udata->inlen < sizeof(ureq)) dev_warn(rdev_to_dev(rdev), "Update the library ulen %d klen %d\n", (unsigned int)udata->inlen, (unsigned int)sizeof(ureq)); rc = ib_copy_from_udata(&ureq, udata, min(udata->inlen, sizeof(ureq))); if (rc) return rc; bytes = (qplib_qp->sq.max_wqe * qplib_qp->sq.wqe_size); + bytes = PAGE_ALIGN(bytes); /* Consider mapping PSN search memory only for RC QPs. */ if (qplib_qp->type == CMDQ_CREATE_QP_TYPE_RC) { psn_sz = _is_chip_gen_p5_p7(rdev->chip_ctx) ? sizeof(struct sq_psn_search_ext) : sizeof(struct sq_psn_search); - if (rdev->dev_attr && BNXT_RE_HW_RETX(rdev->dev_attr->dev_cap_flags)) + if (rdev->dev_attr && _is_host_msn_table(rdev->dev_attr->dev_cap_ext_flags2)) psn_sz = sizeof(struct sq_msn_search); psn_nume = (qplib_qp->wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC) ? qplib_qp->sq.max_wqe : ((qplib_qp->sq.max_wqe * qplib_qp->sq.wqe_size) / sizeof(struct bnxt_qplib_sge)); - if (BNXT_RE_HW_RETX(rdev->dev_attr->dev_cap_flags)) + if (rdev->dev_attr && _is_host_msn_table(rdev->dev_attr->dev_cap_ext_flags2)) psn_nume = roundup_pow_of_two(psn_nume); bytes += (psn_nume * psn_sz); + bytes = PAGE_ALIGN(bytes); } - bytes = PAGE_ALIGN(bytes); umem = ib_umem_get_compat(rdev, context, udata, ureq.qpsva, bytes, IB_ACCESS_LOCAL_WRITE, 1); if (IS_ERR(umem)) { dev_err(rdev_to_dev(rdev), "%s: ib_umem_get failed with %ld\n", __func__, PTR_ERR(umem)); return PTR_ERR(umem); } qp->sumem = umem; /* pgsize and pgshft were initialize already. */ sginfo->sghead = get_ib_umem_sgl(umem, &sginfo->nmap); sginfo->npages = ib_umem_num_pages_compat(umem); qplib_qp->qp_handle = ureq.qp_handle; if (!qp->qplib_qp.srq) { sginfo = &qplib_qp->rq.sginfo; bytes = (qplib_qp->rq.max_wqe * qplib_qp->rq.wqe_size); bytes = PAGE_ALIGN(bytes); umem = ib_umem_get_compat(rdev, context, udata, ureq.qprva, bytes, IB_ACCESS_LOCAL_WRITE, 1); if (IS_ERR(umem)) { dev_err(rdev_to_dev(rdev), "%s: ib_umem_get failed ret =%ld\n", __func__, PTR_ERR(umem)); goto rqfail; } qp->rumem = umem; /* pgsize and pgshft were initialize already. */ sginfo->sghead = get_ib_umem_sgl(umem, &sginfo->nmap); sginfo->npages = ib_umem_num_pages_compat(umem); } qplib_qp->dpi = &cntx->dpi; qplib_qp->is_user = true; return 0; rqfail: ib_umem_release(qp->sumem); qp->sumem = NULL; qplib_qp->sq.sginfo.sghead = NULL; qplib_qp->sq.sginfo.nmap = 0; return PTR_ERR(umem); } static struct bnxt_re_ah *bnxt_re_create_shadow_qp_ah(struct bnxt_re_pd *pd, struct bnxt_qplib_res *qp1_res, struct bnxt_qplib_qp *qp1_qp) { struct bnxt_re_dev *rdev = pd->rdev; struct bnxt_re_ah *ah; union ib_gid sgid; int rc; ah = kzalloc(sizeof(*ah), GFP_KERNEL); if (!ah) { dev_err(rdev_to_dev(rdev), "Allocate Address Handle failed!\n"); return NULL; } memset(ah, 0, sizeof(*ah)); ah->rdev = rdev; ah->qplib_ah.pd = &pd->qplib_pd; rc = bnxt_re_query_gid(&rdev->ibdev, 1, 0, &sgid); if (rc) goto fail; /* supply the dgid data same as sgid */ memcpy(ah->qplib_ah.dgid.data, &sgid.raw, sizeof(union ib_gid)); ah->qplib_ah.sgid_index = 0; ah->qplib_ah.traffic_class = 0; ah->qplib_ah.flow_label = 0; ah->qplib_ah.hop_limit = 1; ah->qplib_ah.sl = 0; /* Have DMAC same as SMAC */ ether_addr_copy(ah->qplib_ah.dmac, rdev->dev_addr); dev_dbg(rdev_to_dev(rdev), "ah->qplib_ah.dmac = %x:%x:%x:%x:%x:%x\n", ah->qplib_ah.dmac[0], ah->qplib_ah.dmac[1], ah->qplib_ah.dmac[2], ah->qplib_ah.dmac[3], ah->qplib_ah.dmac[4], ah->qplib_ah.dmac[5]); rc = bnxt_qplib_create_ah(&rdev->qplib_res, &ah->qplib_ah, true); if (rc) { dev_err(rdev_to_dev(rdev), "Allocate HW AH for Shadow QP failed!\n"); goto fail; } dev_dbg(rdev_to_dev(rdev), "AH ID = %d\n", ah->qplib_ah.id); atomic_inc(&rdev->stats.rsors.ah_count); return ah; fail: kfree(ah); return NULL; } void bnxt_re_update_shadow_ah(struct bnxt_re_dev *rdev) { struct bnxt_re_qp *gsi_qp; struct bnxt_re_ah *sah; struct bnxt_re_pd *pd; struct ib_pd *ib_pd; int rc; if (!rdev) return; sah = rdev->gsi_ctx.gsi_sah; dev_dbg(rdev_to_dev(rdev), "Updating the AH\n"); if (sah) { /* Check if the AH created with current mac address */ if (!compare_ether_header(sah->qplib_ah.dmac, rdev->dev_addr)) { dev_dbg(rdev_to_dev(rdev), "Not modifying shadow AH during AH update\n"); return; } gsi_qp = rdev->gsi_ctx.gsi_qp; ib_pd = gsi_qp->ib_qp.pd; pd = to_bnxt_re(ib_pd, struct bnxt_re_pd, ibpd); rc = bnxt_qplib_destroy_ah(&rdev->qplib_res, &sah->qplib_ah, false); if (rc) { dev_err(rdev_to_dev(rdev), "Failed to destroy shadow AH during AH update\n"); return; } atomic_dec(&rdev->stats.rsors.ah_count); kfree(sah); rdev->gsi_ctx.gsi_sah = NULL; sah = bnxt_re_create_shadow_qp_ah(pd, &rdev->qplib_res, &gsi_qp->qplib_qp); if (!sah) { dev_err(rdev_to_dev(rdev), "Failed to update AH for ShadowQP\n"); return; } rdev->gsi_ctx.gsi_sah = sah; atomic_inc(&rdev->stats.rsors.ah_count); } } static struct bnxt_re_qp *bnxt_re_create_shadow_qp(struct bnxt_re_pd *pd, struct bnxt_qplib_res *qp1_res, struct bnxt_qplib_qp *qp1_qp) { struct bnxt_re_dev *rdev = pd->rdev; struct bnxt_re_qp *qp; int rc; qp = kzalloc(sizeof(*qp), GFP_KERNEL); if (!qp) { dev_err(rdev_to_dev(rdev), "Allocate internal UD QP failed!\n"); return NULL; } memset(qp, 0, sizeof(*qp)); qp->rdev = rdev; /* Initialize the shadow QP structure from the QP1 values */ ether_addr_copy(qp->qplib_qp.smac, rdev->dev_addr); qp->qplib_qp.pd = &pd->qplib_pd; qp->qplib_qp.qp_handle = (u64)&qp->qplib_qp; qp->qplib_qp.type = IB_QPT_UD; qp->qplib_qp.max_inline_data = 0; qp->qplib_qp.sig_type = true; /* Shadow QP SQ depth should be same as QP1 RQ depth */ qp->qplib_qp.sq.wqe_size = bnxt_re_get_swqe_size(0, 6); qp->qplib_qp.sq.max_wqe = qp1_qp->rq.max_wqe; qp->qplib_qp.sq.max_sge = 2; /* Q full delta can be 1 since it is internal QP */ qp->qplib_qp.sq.q_full_delta = 1; qp->qplib_qp.sq.sginfo.pgsize = PAGE_SIZE; qp->qplib_qp.sq.sginfo.pgshft = PAGE_SHIFT; qp->qplib_qp.scq = qp1_qp->scq; qp->qplib_qp.rcq = qp1_qp->rcq; qp->qplib_qp.rq.wqe_size = _max_rwqe_sz(6); /* 128 Byte wqe size */ qp->qplib_qp.rq.max_wqe = qp1_qp->rq.max_wqe; qp->qplib_qp.rq.max_sge = qp1_qp->rq.max_sge; qp->qplib_qp.rq.sginfo.pgsize = PAGE_SIZE; qp->qplib_qp.rq.sginfo.pgshft = PAGE_SHIFT; /* Q full delta can be 1 since it is internal QP */ qp->qplib_qp.rq.q_full_delta = 1; qp->qplib_qp.mtu = qp1_qp->mtu; qp->qplib_qp.dpi = &rdev->dpi_privileged; rc = bnxt_qplib_alloc_hdr_buf(qp1_res, &qp->qplib_qp, 0, BNXT_QPLIB_MAX_GRH_HDR_SIZE_IPV6); if (rc) goto fail; rc = bnxt_qplib_create_qp(qp1_res, &qp->qplib_qp); if (rc) { dev_err(rdev_to_dev(rdev), "create HW QP failed!\n"); goto qp_fail; } dev_dbg(rdev_to_dev(rdev), "Created shadow QP with ID = %d\n", qp->qplib_qp.id); spin_lock_init(&qp->sq_lock); INIT_LIST_HEAD(&qp->list); mutex_lock(&rdev->qp_lock); list_add_tail(&qp->list, &rdev->qp_list); atomic_inc(&rdev->stats.rsors.qp_count); mutex_unlock(&rdev->qp_lock); return qp; qp_fail: bnxt_qplib_free_hdr_buf(qp1_res, &qp->qplib_qp); fail: kfree(qp); return NULL; } static int bnxt_re_init_rq_attr(struct bnxt_re_qp *qp, struct ib_qp_init_attr *init_attr, void *cntx) { struct bnxt_qplib_dev_attr *dev_attr; struct bnxt_qplib_qp *qplqp; struct bnxt_re_dev *rdev; struct bnxt_qplib_q *rq; int entries; rdev = qp->rdev; qplqp = &qp->qplib_qp; rq = &qplqp->rq; dev_attr = rdev->dev_attr; if (init_attr->srq) { struct bnxt_re_srq *srq; srq = to_bnxt_re(init_attr->srq, struct bnxt_re_srq, ibsrq); if (!srq) { dev_err(rdev_to_dev(rdev), "SRQ not found\n"); return -EINVAL; } qplqp->srq = &srq->qplib_srq; rq->max_wqe = 0; } else { rq->max_sge = init_attr->cap.max_recv_sge; if (rq->max_sge > dev_attr->max_qp_sges) rq->max_sge = dev_attr->max_qp_sges; init_attr->cap.max_recv_sge = rq->max_sge; rq->wqe_size = bnxt_re_get_rwqe_size(qplqp, rq->max_sge, dev_attr->max_qp_sges); /* Allocate 1 more than what's provided so posting max doesn't mean empty */ entries = init_attr->cap.max_recv_wr + 1; entries = bnxt_re_init_depth(entries, cntx); rq->max_wqe = min_t(u32, entries, dev_attr->max_qp_wqes + 1); rq->q_full_delta = 0; rq->sginfo.pgsize = PAGE_SIZE; rq->sginfo.pgshft = PAGE_SHIFT; } return 0; } static void bnxt_re_adjust_gsi_rq_attr(struct bnxt_re_qp *qp) { struct bnxt_qplib_dev_attr *dev_attr; struct bnxt_qplib_qp *qplqp; struct bnxt_re_dev *rdev; rdev = qp->rdev; qplqp = &qp->qplib_qp; dev_attr = rdev->dev_attr; if (rdev->gsi_ctx.gsi_qp_mode != BNXT_RE_GSI_MODE_UD) qplqp->rq.max_sge = dev_attr->max_qp_sges; } static int bnxt_re_init_sq_attr(struct bnxt_re_qp *qp, struct ib_qp_init_attr *init_attr, void *cntx) { struct bnxt_qplib_dev_attr *dev_attr; struct bnxt_qplib_qp *qplqp; struct bnxt_re_dev *rdev; struct bnxt_qplib_q *sq; int diff = 0; int entries; int rc; rdev = qp->rdev; qplqp = &qp->qplib_qp; sq = &qplqp->sq; dev_attr = rdev->dev_attr; sq->max_sge = init_attr->cap.max_send_sge; if (sq->max_sge > dev_attr->max_qp_sges) { sq->max_sge = dev_attr->max_qp_sges; init_attr->cap.max_send_sge = sq->max_sge; } rc = bnxt_re_setup_swqe_size(qp, init_attr); if (rc) return rc; /* * Change the SQ depth if user has requested minimum using * configfs. Only supported for kernel consumers. Setting * min_tx_depth to 4096 to handle iser SQ full condition * in most of the newer OS distros */ entries = init_attr->cap.max_send_wr; if (!cntx && rdev->min_tx_depth && init_attr->qp_type != IB_QPT_GSI) { /* * If users specify any value greater than 1 use min_tx_depth * provided by user for comparison. Else, compare it with the * BNXT_RE_MIN_KERNEL_QP_TX_DEPTH and adjust it accordingly. */ if (rdev->min_tx_depth > 1 && entries < rdev->min_tx_depth) entries = rdev->min_tx_depth; else if (entries < BNXT_RE_MIN_KERNEL_QP_TX_DEPTH) entries = BNXT_RE_MIN_KERNEL_QP_TX_DEPTH; } diff = bnxt_re_get_diff(cntx, rdev->chip_ctx); entries = bnxt_re_init_depth(entries + diff + 1, cntx); sq->max_wqe = min_t(u32, entries, dev_attr->max_qp_wqes + diff + 1); sq->q_full_delta = diff + 1; /* * Reserving one slot for Phantom WQE. Application can * post one extra entry in this case. But allowing this to avoid * unexpected Queue full condition */ sq->q_full_delta -= 1; /* becomes 0 for gen-p5 */ sq->sginfo.pgsize = PAGE_SIZE; sq->sginfo.pgshft = PAGE_SHIFT; return 0; } static void bnxt_re_adjust_gsi_sq_attr(struct bnxt_re_qp *qp, struct ib_qp_init_attr *init_attr, void *cntx) { struct bnxt_qplib_dev_attr *dev_attr; struct bnxt_qplib_qp *qplqp; struct bnxt_re_dev *rdev; int entries; rdev = qp->rdev; qplqp = &qp->qplib_qp; dev_attr = rdev->dev_attr; if (rdev->gsi_ctx.gsi_qp_mode != BNXT_RE_GSI_MODE_UD) { entries = init_attr->cap.max_send_wr + 1; entries = bnxt_re_init_depth(entries, cntx); qplqp->sq.max_wqe = min_t(u32, entries, dev_attr->max_qp_wqes + 1); qplqp->sq.q_full_delta = qplqp->sq.max_wqe - init_attr->cap.max_send_wr; qplqp->sq.max_sge++; /* Need one extra sge to put UD header */ if (qplqp->sq.max_sge > dev_attr->max_qp_sges) qplqp->sq.max_sge = dev_attr->max_qp_sges; } } static int bnxt_re_init_qp_type(struct bnxt_re_dev *rdev, struct ib_qp_init_attr *init_attr) { struct bnxt_qplib_chip_ctx *chip_ctx; struct bnxt_re_gsi_context *gsi_ctx; int qptype; chip_ctx = rdev->chip_ctx; gsi_ctx = &rdev->gsi_ctx; qptype = __from_ib_qp_type(init_attr->qp_type); if (qptype == IB_QPT_MAX) { dev_err(rdev_to_dev(rdev), "QP type 0x%x not supported\n", qptype); qptype = -EINVAL; goto out; } if (_is_chip_gen_p5_p7(chip_ctx) && init_attr->qp_type == IB_QPT_GSI) { /* For Thor always force UD mode. */ qptype = CMDQ_CREATE_QP_TYPE_GSI; gsi_ctx->gsi_qp_mode = BNXT_RE_GSI_MODE_UD; } out: return qptype; } static int bnxt_re_init_qp_wqe_mode(struct bnxt_re_dev *rdev) { return rdev->chip_ctx->modes.wqe_mode; } static int bnxt_re_init_qp_attr(struct bnxt_re_qp *qp, struct bnxt_re_pd *pd, struct ib_qp_init_attr *init_attr, struct ib_udata *udata) { struct bnxt_qplib_dev_attr *dev_attr; struct bnxt_re_ucontext *cntx = NULL; struct ib_ucontext *context; struct bnxt_qplib_qp *qplqp; struct bnxt_re_dev *rdev; struct bnxt_re_cq *cq; int rc = 0, qptype; rdev = qp->rdev; qplqp = &qp->qplib_qp; dev_attr = rdev->dev_attr; if (udata) { context = pd->ibpd.uobject->context; cntx = to_bnxt_re(context, struct bnxt_re_ucontext, ibucontext); } /* Setup misc params */ qplqp->is_user = false; qplqp->pd = &pd->qplib_pd; qplqp->qp_handle = (u64)qplqp; qplqp->sig_type = ((init_attr->sq_sig_type == IB_SIGNAL_ALL_WR) ? true : false); qptype = bnxt_re_init_qp_type(rdev, init_attr); if (qptype < 0) { rc = qptype; goto out; } qplqp->type = (u8)qptype; qplqp->wqe_mode = bnxt_re_init_qp_wqe_mode(rdev); ether_addr_copy(qplqp->smac, rdev->dev_addr); if (init_attr->qp_type == IB_QPT_RC) { qplqp->max_rd_atomic = dev_attr->max_qp_rd_atom; qplqp->max_dest_rd_atomic = dev_attr->max_qp_init_rd_atom; } qplqp->mtu = ib_mtu_enum_to_int(iboe_get_mtu(if_getmtu(rdev->netdev))); qplqp->dpi = &rdev->dpi_privileged; /* Doorbell page */ if (init_attr->create_flags) { dev_dbg(rdev_to_dev(rdev), "QP create flags 0x%x not supported\n", init_attr->create_flags); return -EOPNOTSUPP; } /* Setup CQs */ if (init_attr->send_cq) { cq = to_bnxt_re(init_attr->send_cq, struct bnxt_re_cq, ibcq); if (!cq) { dev_err(rdev_to_dev(rdev), "Send CQ not found\n"); rc = -EINVAL; goto out; } qplqp->scq = &cq->qplib_cq; qp->scq = cq; } if (init_attr->recv_cq) { cq = to_bnxt_re(init_attr->recv_cq, struct bnxt_re_cq, ibcq); if (!cq) { dev_err(rdev_to_dev(rdev), "Receive CQ not found\n"); rc = -EINVAL; goto out; } qplqp->rcq = &cq->qplib_cq; qp->rcq = cq; } /* Setup RQ/SRQ */ rc = bnxt_re_init_rq_attr(qp, init_attr, cntx); if (rc) goto out; if (init_attr->qp_type == IB_QPT_GSI) bnxt_re_adjust_gsi_rq_attr(qp); /* Setup SQ */ rc = bnxt_re_init_sq_attr(qp, init_attr, cntx); if (rc) goto out; if (init_attr->qp_type == IB_QPT_GSI) bnxt_re_adjust_gsi_sq_attr(qp, init_attr, cntx); if (udata) /* This will update DPI and qp_handle */ rc = bnxt_re_init_user_qp(rdev, pd, qp, udata); out: return rc; } static int bnxt_re_create_shadow_gsi(struct bnxt_re_qp *qp, struct bnxt_re_pd *pd) { struct bnxt_re_sqp_entries *sqp_tbl = NULL; struct bnxt_re_dev *rdev; struct bnxt_re_qp *sqp; struct bnxt_re_ah *sah; int rc = 0; rdev = qp->rdev; /* Create a shadow QP to handle the QP1 traffic */ sqp_tbl = kzalloc(sizeof(*sqp_tbl) * BNXT_RE_MAX_GSI_SQP_ENTRIES, GFP_KERNEL); if (!sqp_tbl) return -ENOMEM; rdev->gsi_ctx.sqp_tbl = sqp_tbl; sqp = bnxt_re_create_shadow_qp(pd, &rdev->qplib_res, &qp->qplib_qp); if (!sqp) { rc = -ENODEV; dev_err(rdev_to_dev(rdev), "Failed to create Shadow QP for QP1\n"); goto out; } rdev->gsi_ctx.gsi_sqp = sqp; sqp->rcq = qp->rcq; sqp->scq = qp->scq; sah = bnxt_re_create_shadow_qp_ah(pd, &rdev->qplib_res, &qp->qplib_qp); if (!sah) { bnxt_qplib_destroy_qp(&rdev->qplib_res, &sqp->qplib_qp); rc = -ENODEV; dev_err(rdev_to_dev(rdev), "Failed to create AH entry for ShadowQP\n"); goto out; } rdev->gsi_ctx.gsi_sah = sah; return 0; out: kfree(sqp_tbl); return rc; } static int __get_rq_hdr_buf_size(u8 gsi_mode) { return (gsi_mode == BNXT_RE_GSI_MODE_ALL) ? BNXT_QPLIB_MAX_QP1_RQ_HDR_SIZE_V2 : BNXT_QPLIB_MAX_QP1_RQ_HDR_SIZE; } static int __get_sq_hdr_buf_size(u8 gsi_mode) { return (gsi_mode != BNXT_RE_GSI_MODE_ROCE_V1) ? BNXT_QPLIB_MAX_QP1_SQ_HDR_SIZE_V2 : BNXT_QPLIB_MAX_QP1_SQ_HDR_SIZE; } static int bnxt_re_create_gsi_qp(struct bnxt_re_qp *qp, struct bnxt_re_pd *pd) { struct bnxt_qplib_qp *qplqp; struct bnxt_qplib_res *res; struct bnxt_re_dev *rdev; u32 sstep, rstep; u8 gsi_mode; int rc = 0; rdev = qp->rdev; qplqp = &qp->qplib_qp; res = &rdev->qplib_res; gsi_mode = rdev->gsi_ctx.gsi_qp_mode; rstep = __get_rq_hdr_buf_size(gsi_mode); sstep = __get_sq_hdr_buf_size(gsi_mode); rc = bnxt_qplib_alloc_hdr_buf(res, qplqp, sstep, rstep); if (rc) goto out; rc = bnxt_qplib_create_qp1(res, qplqp); if (rc) { dev_err(rdev_to_dev(rdev), "create HW QP1 failed!\n"); goto out; } if (gsi_mode == BNXT_RE_GSI_MODE_ALL) rc = bnxt_re_create_shadow_gsi(qp, pd); out: return rc; } static bool bnxt_re_test_qp_limits(struct bnxt_re_dev *rdev, struct ib_qp_init_attr *init_attr, struct bnxt_qplib_dev_attr *dev_attr) { bool rc = true; int ilsize; ilsize = ALIGN(init_attr->cap.max_inline_data, sizeof(struct sq_sge)); if ((init_attr->cap.max_send_wr > dev_attr->max_qp_wqes) || (init_attr->cap.max_recv_wr > dev_attr->max_qp_wqes) || (init_attr->cap.max_send_sge > dev_attr->max_qp_sges) || (init_attr->cap.max_recv_sge > dev_attr->max_qp_sges) || (ilsize > dev_attr->max_inline_data)) { dev_err(rdev_to_dev(rdev), "Create QP failed - max exceeded! " "0x%x/0x%x 0x%x/0x%x 0x%x/0x%x " "0x%x/0x%x 0x%x/0x%x\n", init_attr->cap.max_send_wr, dev_attr->max_qp_wqes, init_attr->cap.max_recv_wr, dev_attr->max_qp_wqes, init_attr->cap.max_send_sge, dev_attr->max_qp_sges, init_attr->cap.max_recv_sge, dev_attr->max_qp_sges, init_attr->cap.max_inline_data, dev_attr->max_inline_data); rc = false; } return rc; } static inline struct bnxt_re_qp *__get_qp_from_qp_in(struct ib_pd *qp_in, struct bnxt_re_dev *rdev) { struct bnxt_re_qp *qp; qp = kzalloc(sizeof(*qp), GFP_KERNEL); if (!qp) dev_err(rdev_to_dev(rdev), "Allocate QP failed!\n"); return qp; } struct ib_qp *bnxt_re_create_qp(struct ib_pd *qp_in, struct ib_qp_init_attr *qp_init_attr, struct ib_udata *udata) { struct bnxt_re_pd *pd; struct ib_pd *ib_pd = qp_in; struct bnxt_qplib_dev_attr *dev_attr; struct bnxt_re_dev *rdev; u32 active_qps, tmp_qps; struct bnxt_re_qp *qp; int rc; pd = to_bnxt_re(ib_pd, struct bnxt_re_pd, ibpd); rdev = pd->rdev; dev_attr = rdev->dev_attr; if (rdev->mod_exit) { rc = -EIO; dev_dbg(rdev_to_dev(rdev), "%s(): in mod_exit, just return!\n", __func__); goto exit; } if (atomic_read(&rdev->stats.rsors.qp_count) >= dev_attr->max_qp) { dev_err(rdev_to_dev(rdev), "Create QP failed - max exceeded(QPs Alloc'd %u of max %u)\n", atomic_read(&rdev->stats.rsors.qp_count), dev_attr->max_qp); rc = -EINVAL; goto exit; } rc = bnxt_re_test_qp_limits(rdev, qp_init_attr, dev_attr); if (!rc) { rc = -EINVAL; goto exit; } qp = __get_qp_from_qp_in(qp_in, rdev); if (!qp) { rc = -ENOMEM; goto exit; } qp->rdev = rdev; rc = bnxt_re_init_qp_attr(qp, pd, qp_init_attr, udata); if (rc) goto fail; if (qp_init_attr->qp_type == IB_QPT_GSI && !_is_chip_gen_p5_p7(rdev->chip_ctx)) { rc = bnxt_re_create_gsi_qp(qp, pd); if (rc == -ENODEV) goto qp_destroy; if (rc) goto fail; } else { rc = bnxt_qplib_create_qp(&rdev->qplib_res, &qp->qplib_qp); if (rc) { dev_err(rdev_to_dev(rdev), "create HW QP failed!\n"); goto free_umem; } if (udata) { struct bnxt_re_qp_resp resp; resp.qpid = qp->qplib_qp.id; rc = bnxt_re_copy_to_udata(rdev, &resp, min(udata->outlen, sizeof(resp)), udata); if (rc) goto qp_destroy; } } qp->ib_qp.qp_num = qp->qplib_qp.id; if (qp_init_attr->qp_type == IB_QPT_GSI) rdev->gsi_ctx.gsi_qp = qp; spin_lock_init(&qp->sq_lock); spin_lock_init(&qp->rq_lock); INIT_LIST_HEAD(&qp->list); mutex_lock(&rdev->qp_lock); list_add_tail(&qp->list, &rdev->qp_list); mutex_unlock(&rdev->qp_lock); atomic_inc(&rdev->stats.rsors.qp_count); active_qps = atomic_read(&rdev->stats.rsors.qp_count); if (active_qps > atomic_read(&rdev->stats.rsors.max_qp_count)) atomic_set(&rdev->stats.rsors.max_qp_count, active_qps); bnxt_re_dump_debug_stats(rdev, active_qps); /* Get the counters for RC QPs and UD QPs */ if (qp_init_attr->qp_type == IB_QPT_RC) { tmp_qps = atomic_inc_return(&rdev->stats.rsors.rc_qp_count); if (tmp_qps > atomic_read(&rdev->stats.rsors.max_rc_qp_count)) atomic_set(&rdev->stats.rsors.max_rc_qp_count, tmp_qps); } else if (qp_init_attr->qp_type == IB_QPT_UD) { tmp_qps = atomic_inc_return(&rdev->stats.rsors.ud_qp_count); if (tmp_qps > atomic_read(&rdev->stats.rsors.max_ud_qp_count)) atomic_set(&rdev->stats.rsors.max_ud_qp_count, tmp_qps); } return &qp->ib_qp; qp_destroy: bnxt_qplib_destroy_qp(&rdev->qplib_res, &qp->qplib_qp); free_umem: if (udata) { if (qp->rumem && !IS_ERR(qp->rumem)) ib_umem_release(qp->rumem); if (qp->sumem && !IS_ERR(qp->sumem)) ib_umem_release(qp->sumem); } fail: kfree(qp); exit: return ERR_PTR(rc); } static int bnxt_re_modify_shadow_qp(struct bnxt_re_dev *rdev, struct bnxt_re_qp *qp1_qp, int qp_attr_mask) { struct bnxt_re_qp *qp = rdev->gsi_ctx.gsi_sqp; int rc = 0; if (qp_attr_mask & IB_QP_STATE) { qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_STATE; qp->qplib_qp.state = qp1_qp->qplib_qp.state; } if (qp_attr_mask & IB_QP_PKEY_INDEX) { qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_PKEY; qp->qplib_qp.pkey_index = qp1_qp->qplib_qp.pkey_index; } if (qp_attr_mask & IB_QP_QKEY) { qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_QKEY; /* Using a Random QKEY */ qp->qplib_qp.qkey = BNXT_RE_QP_RANDOM_QKEY; } if (qp_attr_mask & IB_QP_SQ_PSN) { qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_SQ_PSN; qp->qplib_qp.sq.psn = qp1_qp->qplib_qp.sq.psn; } rc = bnxt_qplib_modify_qp(&rdev->qplib_res, &qp->qplib_qp); if (rc) dev_err(rdev_to_dev(rdev), "Modify Shadow QP for QP1 failed\n"); return rc; } static u32 ipv4_from_gid(u8 *gid) { return (gid[15] << 24 | gid[14] << 16 | gid[13] << 8 | gid[12]); } static u16 get_source_port(struct bnxt_re_dev *rdev, struct bnxt_re_qp *qp) { u8 ip_off, data[48], smac[ETH_ALEN]; u16 crc = 0, buf_len = 0, i; u8 addr_len; u32 qpn; if (qp->qplib_qp.nw_type == CMDQ_MODIFY_QP_NETWORK_TYPE_ROCEV2_IPV6) { addr_len = 6; ip_off = 10; } else { addr_len = 4; ip_off = 12; } memcpy(smac, qp->qplib_qp.smac, ETH_ALEN); memset(data, 0, 48); memcpy(data, qp->qplib_qp.ah.dmac, ETH_ALEN); buf_len += ETH_ALEN; memcpy(data + buf_len, smac, ETH_ALEN); buf_len += ETH_ALEN; memcpy(data + buf_len, qp->qplib_qp.ah.dgid.data + ip_off, addr_len); buf_len += addr_len; memcpy(data + buf_len, qp->qp_info_entry.sgid.raw + ip_off, addr_len); buf_len += addr_len; qpn = htonl(qp->qplib_qp.dest_qpn); memcpy(data + buf_len, (u8 *)&qpn + 1, 3); buf_len += 3; for (i = 0; i < buf_len; i++) crc = crc16(crc, (data + i), 1); return crc; } static void bnxt_re_update_qp_info(struct bnxt_re_dev *rdev, struct bnxt_re_qp *qp) { u16 type; type = __from_hw_to_ib_qp_type(qp->qplib_qp.type); /* User-space can extract ip address with sgid_index. */ if (ipv6_addr_v4mapped((struct in6_addr *)&qp->qplib_qp.ah.dgid)) { qp->qp_info_entry.s_ip.ipv4_addr = ipv4_from_gid(qp->qp_info_entry.sgid.raw); qp->qp_info_entry.d_ip.ipv4_addr = ipv4_from_gid(qp->qplib_qp.ah.dgid.data); } else { memcpy(&qp->qp_info_entry.s_ip.ipv6_addr, qp->qp_info_entry.sgid.raw, sizeof(qp->qp_info_entry.s_ip.ipv6_addr)); memcpy(&qp->qp_info_entry.d_ip.ipv6_addr, qp->qplib_qp.ah.dgid.data, sizeof(qp->qp_info_entry.d_ip.ipv6_addr)); } if (type == IB_QPT_RC && (qp->qplib_qp.nw_type == CMDQ_MODIFY_QP_NETWORK_TYPE_ROCEV2_IPV4 || qp->qplib_qp.nw_type == CMDQ_MODIFY_QP_NETWORK_TYPE_ROCEV2_IPV6)) { qp->qp_info_entry.s_port = get_source_port(rdev, qp); } qp->qp_info_entry.d_port = BNXT_RE_QP_DEST_PORT; } static void bnxt_qplib_manage_flush_qp(struct bnxt_re_qp *qp) { struct bnxt_qplib_q *rq, *sq; unsigned long flags; if (qp->sumem) return; if (qp->qplib_qp.state == CMDQ_MODIFY_QP_NEW_STATE_ERR) { rq = &qp->qplib_qp.rq; sq = &qp->qplib_qp.sq; dev_dbg(rdev_to_dev(qp->rdev), "Move QP = %p to flush list\n", qp); flags = bnxt_re_lock_cqs(qp); bnxt_qplib_add_flush_qp(&qp->qplib_qp); bnxt_re_unlock_cqs(qp, flags); if (sq->hwq.prod != sq->hwq.cons) bnxt_re_handle_cqn(&qp->scq->qplib_cq); if (qp->rcq && (qp->rcq != qp->scq) && (rq->hwq.prod != rq->hwq.cons)) bnxt_re_handle_cqn(&qp->rcq->qplib_cq); } if (qp->qplib_qp.state == CMDQ_MODIFY_QP_NEW_STATE_RESET) { dev_dbg(rdev_to_dev(qp->rdev), "Move QP = %p out of flush list\n", qp); flags = bnxt_re_lock_cqs(qp); bnxt_qplib_clean_qp(&qp->qplib_qp); bnxt_re_unlock_cqs(qp, flags); } } bool ib_modify_qp_is_ok_compat(enum ib_qp_state cur_state, enum ib_qp_state next_state, enum ib_qp_type type, enum ib_qp_attr_mask mask) { return (ib_modify_qp_is_ok(cur_state, next_state, type, mask)); } int bnxt_re_modify_qp(struct ib_qp *ib_qp, struct ib_qp_attr *qp_attr, int qp_attr_mask, struct ib_udata *udata) { enum ib_qp_state curr_qp_state, new_qp_state; struct bnxt_re_modify_qp_ex_resp resp = {}; struct bnxt_re_modify_qp_ex_req ureq = {}; struct bnxt_qplib_dev_attr *dev_attr; struct bnxt_qplib_ppp *ppp = NULL; struct bnxt_re_dev *rdev; struct bnxt_re_qp *qp; struct ib_gid_attr *sgid_attr; struct ib_gid_attr gid_attr; union ib_gid sgid, *gid_ptr = NULL; u8 nw_type; int rc, entries, status; bool is_copy_to_udata = false; bool is_qpmtu_high = false; qp = to_bnxt_re(ib_qp, struct bnxt_re_qp, ib_qp); rdev = qp->rdev; dev_attr = rdev->dev_attr; qp->qplib_qp.modify_flags = 0; ppp = &qp->qplib_qp.ppp; if (qp_attr_mask & IB_QP_STATE) { curr_qp_state = __to_ib_qp_state(qp->qplib_qp.cur_qp_state); new_qp_state = qp_attr->qp_state; if (!ib_modify_qp_is_ok_compat(curr_qp_state, new_qp_state, ib_qp->qp_type, qp_attr_mask)) { dev_err(rdev_to_dev(rdev),"invalid attribute mask=0x%x" " specified for qpn=0x%x of type=0x%x" " current_qp_state=0x%x, new_qp_state=0x%x\n", qp_attr_mask, ib_qp->qp_num, ib_qp->qp_type, curr_qp_state, new_qp_state); return -EINVAL; } dev_dbg(rdev_to_dev(rdev), "%s:%d INFO attribute mask=0x%x qpn=0x%x " "of type=0x%x current_qp_state=0x%x, new_qp_state=0x%x\n", __func__, __LINE__, qp_attr_mask, ib_qp->qp_num, ib_qp->qp_type, curr_qp_state, new_qp_state); qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_STATE; qp->qplib_qp.state = __from_ib_qp_state(qp_attr->qp_state); if (udata && curr_qp_state == IB_QPS_RESET && new_qp_state == IB_QPS_INIT) { if (!ib_copy_from_udata(&ureq, udata, sizeof(ureq))) { if (ureq.comp_mask & BNXT_RE_COMP_MASK_MQP_EX_PPP_REQ_EN_MASK) { ppp->req = BNXT_QPLIB_PPP_REQ; ppp->dpi = ureq.dpi; } } } } if (qp_attr_mask & IB_QP_EN_SQD_ASYNC_NOTIFY) { qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_EN_SQD_ASYNC_NOTIFY; qp->qplib_qp.en_sqd_async_notify = true; } if (qp_attr_mask & IB_QP_ACCESS_FLAGS) { qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_ACCESS; qp->qplib_qp.access = __from_ib_access_flags(qp_attr->qp_access_flags); /* LOCAL_WRITE access must be set to allow RC receive */ qp->qplib_qp.access |= BNXT_QPLIB_ACCESS_LOCAL_WRITE; qp->qplib_qp.access |= CMDQ_MODIFY_QP_ACCESS_REMOTE_WRITE; qp->qplib_qp.access |= CMDQ_MODIFY_QP_ACCESS_REMOTE_READ; } if (qp_attr_mask & IB_QP_PKEY_INDEX) { qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_PKEY; qp->qplib_qp.pkey_index = qp_attr->pkey_index; } if (qp_attr_mask & IB_QP_QKEY) { qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_QKEY; qp->qplib_qp.qkey = qp_attr->qkey; } if (qp_attr_mask & IB_QP_AV) { qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_DGID | CMDQ_MODIFY_QP_MODIFY_MASK_FLOW_LABEL | CMDQ_MODIFY_QP_MODIFY_MASK_SGID_INDEX | CMDQ_MODIFY_QP_MODIFY_MASK_HOP_LIMIT | CMDQ_MODIFY_QP_MODIFY_MASK_TRAFFIC_CLASS | CMDQ_MODIFY_QP_MODIFY_MASK_DEST_MAC | CMDQ_MODIFY_QP_MODIFY_MASK_VLAN_ID; memcpy(qp->qplib_qp.ah.dgid.data, qp_attr->ah_attr.grh.dgid.raw, sizeof(qp->qplib_qp.ah.dgid.data)); qp->qplib_qp.ah.flow_label = qp_attr->ah_attr.grh.flow_label; qp->qplib_qp.ah.sgid_index = _get_sgid_index(rdev, qp_attr->ah_attr.grh.sgid_index); qp->qplib_qp.ah.host_sgid_index = qp_attr->ah_attr.grh.sgid_index; qp->qplib_qp.ah.hop_limit = qp_attr->ah_attr.grh.hop_limit; qp->qplib_qp.ah.traffic_class = qp_attr->ah_attr.grh.traffic_class; qp->qplib_qp.ah.sl = qp_attr->ah_attr.sl; ether_addr_copy(qp->qplib_qp.ah.dmac, ROCE_DMAC(&qp_attr->ah_attr)); sgid_attr = &gid_attr; status = bnxt_re_get_cached_gid(&rdev->ibdev, 1, qp_attr->ah_attr.grh.sgid_index, &sgid, &sgid_attr, &qp_attr->ah_attr.grh, NULL); if (!status) if_rele(sgid_attr->ndev); gid_ptr = &sgid; if (sgid_attr->ndev) { memcpy(qp->qplib_qp.smac, rdev->dev_addr, ETH_ALEN); nw_type = bnxt_re_gid_to_network_type(sgid_attr, &sgid); dev_dbg(rdev_to_dev(rdev), "Connection using the nw_type %d\n", nw_type); switch (nw_type) { case RDMA_NETWORK_IPV4: qp->qplib_qp.nw_type = CMDQ_MODIFY_QP_NETWORK_TYPE_ROCEV2_IPV4; break; case RDMA_NETWORK_IPV6: qp->qplib_qp.nw_type = CMDQ_MODIFY_QP_NETWORK_TYPE_ROCEV2_IPV6; break; default: qp->qplib_qp.nw_type = CMDQ_MODIFY_QP_NETWORK_TYPE_ROCEV1; break; } } memcpy(&qp->qp_info_entry.sgid, gid_ptr, sizeof(qp->qp_info_entry.sgid)); } /* MTU settings allowed only during INIT -> RTR */ if (qp_attr->qp_state == IB_QPS_RTR) { bnxt_re_init_qpmtu(qp, if_getmtu(rdev->netdev), qp_attr_mask, qp_attr, &is_qpmtu_high); if (udata && !ib_copy_from_udata(&ureq, udata, sizeof(ureq))) { if (ureq.comp_mask & BNXT_RE_COMP_MASK_MQP_EX_PATH_MTU_MASK) { resp.comp_mask |= BNXT_RE_COMP_MASK_MQP_EX_PATH_MTU_MASK; resp.path_mtu = qp->qplib_qp.mtu; is_copy_to_udata = true; } else if (is_qpmtu_high) { dev_err(rdev_to_dev(rdev), "qp %#x invalid mtu\n", qp->qplib_qp.id); return -EINVAL; } } } if (qp_attr_mask & IB_QP_TIMEOUT) { qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_TIMEOUT; qp->qplib_qp.timeout = qp_attr->timeout; } if (qp_attr_mask & IB_QP_RETRY_CNT) { qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_RETRY_CNT; qp->qplib_qp.retry_cnt = qp_attr->retry_cnt; } if (qp_attr_mask & IB_QP_RNR_RETRY) { qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_RNR_RETRY; qp->qplib_qp.rnr_retry = qp_attr->rnr_retry; } if (qp_attr_mask & IB_QP_MIN_RNR_TIMER) { qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_MIN_RNR_TIMER; qp->qplib_qp.min_rnr_timer = qp_attr->min_rnr_timer; } if (qp_attr_mask & IB_QP_RQ_PSN) { qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_RQ_PSN; qp->qplib_qp.rq.psn = qp_attr->rq_psn; } if (qp_attr_mask & IB_QP_MAX_QP_RD_ATOMIC) { qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_MAX_RD_ATOMIC; /* Cap the max_rd_atomic to device max */ if (qp_attr->max_rd_atomic > dev_attr->max_qp_rd_atom) dev_dbg(rdev_to_dev(rdev), "max_rd_atomic requested %d is > device max %d\n", qp_attr->max_rd_atomic, dev_attr->max_qp_rd_atom); qp->qplib_qp.max_rd_atomic = min_t(u32, qp_attr->max_rd_atomic, dev_attr->max_qp_rd_atom); } if (qp_attr_mask & IB_QP_SQ_PSN) { qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_SQ_PSN; qp->qplib_qp.sq.psn = qp_attr->sq_psn; } if (qp_attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) { if (qp_attr->max_dest_rd_atomic > dev_attr->max_qp_init_rd_atom) { dev_err(rdev_to_dev(rdev), "max_dest_rd_atomic requested %d is > device max %d\n", qp_attr->max_dest_rd_atomic, dev_attr->max_qp_init_rd_atom); return -EINVAL; } qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_MAX_DEST_RD_ATOMIC; qp->qplib_qp.max_dest_rd_atomic = qp_attr->max_dest_rd_atomic; } if (qp_attr_mask & IB_QP_CAP) { qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_SQ_SIZE | CMDQ_MODIFY_QP_MODIFY_MASK_RQ_SIZE | CMDQ_MODIFY_QP_MODIFY_MASK_SQ_SGE | CMDQ_MODIFY_QP_MODIFY_MASK_RQ_SGE | CMDQ_MODIFY_QP_MODIFY_MASK_MAX_INLINE_DATA; if ((qp_attr->cap.max_send_wr >= dev_attr->max_qp_wqes) || (qp_attr->cap.max_recv_wr >= dev_attr->max_qp_wqes) || (qp_attr->cap.max_send_sge >= dev_attr->max_qp_sges) || (qp_attr->cap.max_recv_sge >= dev_attr->max_qp_sges) || (qp_attr->cap.max_inline_data >= dev_attr->max_inline_data)) { dev_err(rdev_to_dev(rdev), "Create QP failed - max exceeded\n"); return -EINVAL; } entries = roundup_pow_of_two(qp_attr->cap.max_send_wr); if (entries > dev_attr->max_qp_wqes) entries = dev_attr->max_qp_wqes; entries = min_t(u32, entries, dev_attr->max_qp_wqes); qp->qplib_qp.sq.max_wqe = entries; qp->qplib_qp.sq.q_full_delta = qp->qplib_qp.sq.max_wqe - qp_attr->cap.max_send_wr; /* * Reserving one slot for Phantom WQE. Some application can * post one extra entry in this case. Allowing this to avoid * unexpected Queue full condition */ qp->qplib_qp.sq.q_full_delta -= 1; qp->qplib_qp.sq.max_sge = qp_attr->cap.max_send_sge; if (qp->qplib_qp.rq.max_wqe) { entries = roundup_pow_of_two(qp_attr->cap.max_recv_wr); if (entries > dev_attr->max_qp_wqes) entries = dev_attr->max_qp_wqes; qp->qplib_qp.rq.max_wqe = entries; qp->qplib_qp.rq.q_full_delta = qp->qplib_qp.rq.max_wqe - qp_attr->cap.max_recv_wr; qp->qplib_qp.rq.max_sge = qp_attr->cap.max_recv_sge; } else { /* SRQ was used prior, just ignore the RQ caps */ } } if (qp_attr_mask & IB_QP_DEST_QPN) { qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_DEST_QP_ID; qp->qplib_qp.dest_qpn = qp_attr->dest_qp_num; } rc = bnxt_qplib_modify_qp(&rdev->qplib_res, &qp->qplib_qp); if (rc) { dev_err(rdev_to_dev(rdev), "Modify HW QP failed!\n"); return rc; } if (qp_attr_mask & IB_QP_STATE) bnxt_qplib_manage_flush_qp(qp); if (ureq.comp_mask & BNXT_RE_COMP_MASK_MQP_EX_PPP_REQ_EN_MASK && ppp->st_idx_en & CREQ_MODIFY_QP_RESP_PINGPONG_PUSH_ENABLED) { resp.comp_mask |= BNXT_RE_COMP_MASK_MQP_EX_PPP_REQ_EN; resp.ppp_st_idx = ppp->st_idx_en >> BNXT_QPLIB_PPP_ST_IDX_SHIFT; is_copy_to_udata = true; } if (is_copy_to_udata) { rc = bnxt_re_copy_to_udata(rdev, &resp, min(udata->outlen, sizeof(resp)), udata); if (rc) return rc; } if (ib_qp->qp_type == IB_QPT_GSI && rdev->gsi_ctx.gsi_qp_mode == BNXT_RE_GSI_MODE_ALL && rdev->gsi_ctx.gsi_sqp) rc = bnxt_re_modify_shadow_qp(rdev, qp, qp_attr_mask); /* * Update info when qp_info_info */ bnxt_re_update_qp_info(rdev, qp); return rc; } int bnxt_re_query_qp(struct ib_qp *ib_qp, struct ib_qp_attr *qp_attr, int qp_attr_mask, struct ib_qp_init_attr *qp_init_attr) { struct bnxt_re_qp *qp = to_bnxt_re(ib_qp, struct bnxt_re_qp, ib_qp); struct bnxt_re_dev *rdev = qp->rdev; struct bnxt_qplib_qp *qplib_qp; int rc; qplib_qp = kcalloc(1, sizeof(*qplib_qp), GFP_KERNEL); if (!qplib_qp) return -ENOMEM; qplib_qp->id = qp->qplib_qp.id; qplib_qp->ah.host_sgid_index = qp->qplib_qp.ah.host_sgid_index; rc = bnxt_qplib_query_qp(&rdev->qplib_res, qplib_qp); if (rc) { dev_err(rdev_to_dev(rdev), "Query HW QP (0x%x) failed! rc = %d\n", qplib_qp->id, rc); goto free_mem; } qp_attr->qp_state = __to_ib_qp_state(qplib_qp->state); qp_attr->cur_qp_state = __to_ib_qp_state(qplib_qp->cur_qp_state); qp_attr->en_sqd_async_notify = qplib_qp->en_sqd_async_notify ? 1 : 0; qp_attr->qp_access_flags = __to_ib_access_flags(qplib_qp->access); qp_attr->pkey_index = qplib_qp->pkey_index; qp_attr->qkey = qplib_qp->qkey; memcpy(qp_attr->ah_attr.grh.dgid.raw, qplib_qp->ah.dgid.data, sizeof(qplib_qp->ah.dgid.data)); qp_attr->ah_attr.grh.flow_label = qplib_qp->ah.flow_label; qp_attr->ah_attr.grh.sgid_index = qplib_qp->ah.host_sgid_index; qp_attr->ah_attr.grh.hop_limit = qplib_qp->ah.hop_limit; qp_attr->ah_attr.grh.traffic_class = qplib_qp->ah.traffic_class; qp_attr->ah_attr.sl = qplib_qp->ah.sl; ether_addr_copy(ROCE_DMAC(&qp_attr->ah_attr), qplib_qp->ah.dmac); qp_attr->path_mtu = __to_ib_mtu(qplib_qp->path_mtu); qp_attr->timeout = qplib_qp->timeout; qp_attr->retry_cnt = qplib_qp->retry_cnt; qp_attr->rnr_retry = qplib_qp->rnr_retry; qp_attr->min_rnr_timer = qplib_qp->min_rnr_timer; qp_attr->rq_psn = qplib_qp->rq.psn; qp_attr->max_rd_atomic = qplib_qp->max_rd_atomic; qp_attr->sq_psn = qplib_qp->sq.psn; qp_attr->max_dest_rd_atomic = qplib_qp->max_dest_rd_atomic; qp_init_attr->sq_sig_type = qplib_qp->sig_type ? IB_SIGNAL_ALL_WR : IB_SIGNAL_REQ_WR; qp_attr->dest_qp_num = qplib_qp->dest_qpn; qp_attr->cap.max_send_wr = qp->qplib_qp.sq.max_wqe; qp_attr->cap.max_send_sge = qp->qplib_qp.sq.max_sge; qp_attr->cap.max_recv_wr = qp->qplib_qp.rq.max_wqe; qp_attr->cap.max_recv_sge = qp->qplib_qp.rq.max_sge; qp_attr->cap.max_inline_data = qp->qplib_qp.max_inline_data; qp_init_attr->cap = qp_attr->cap; free_mem: kfree(qplib_qp); return rc; } /* Builders */ /* For Raw, the application is responsible to build the entire packet */ static void bnxt_re_build_raw_send(const struct ib_send_wr *wr, struct bnxt_qplib_swqe *wqe) { switch (wr->send_flags) { case IB_SEND_IP_CSUM: wqe->rawqp1.lflags |= SQ_SEND_RAWETH_QP1_LFLAGS_IP_CHKSUM; break; default: /* Pad HW RoCE iCRC */ wqe->rawqp1.lflags |= SQ_SEND_RAWETH_QP1_LFLAGS_ROCE_CRC; break; } } /* For QP1, the driver must build the entire RoCE (v1/v2) packet hdr * as according to the sgid and AV */ static int bnxt_re_build_qp1_send(struct bnxt_re_qp *qp, const struct ib_send_wr *wr, struct bnxt_qplib_swqe *wqe, int payload_size) { struct bnxt_re_ah *ah = to_bnxt_re(ud_wr(wr)->ah, struct bnxt_re_ah, ibah); struct bnxt_qplib_ah *qplib_ah = &ah->qplib_ah; struct bnxt_qplib_sge sge; int i, rc = 0; union ib_gid sgid; u16 vlan_id; u8 *ptmac; void *buf; memset(&qp->qp1_hdr, 0, sizeof(qp->qp1_hdr)); /* Get sgid */ rc = bnxt_re_query_gid(&qp->rdev->ibdev, 1, qplib_ah->sgid_index, &sgid); if (rc) return rc; /* ETH */ qp->qp1_hdr.eth_present = 1; ptmac = ah->qplib_ah.dmac; memcpy(qp->qp1_hdr.eth.dmac_h, ptmac, 4); ptmac += 4; memcpy(qp->qp1_hdr.eth.dmac_l, ptmac, 2); ptmac = qp->qplib_qp.smac; memcpy(qp->qp1_hdr.eth.smac_h, ptmac, 2); ptmac += 2; memcpy(qp->qp1_hdr.eth.smac_l, ptmac, 4); qp->qp1_hdr.eth.type = cpu_to_be16(BNXT_QPLIB_ETHTYPE_ROCEV1); /* For vlan, check the sgid for vlan existence */ vlan_id = rdma_get_vlan_id(&sgid); if (vlan_id && vlan_id < 0x1000) { qp->qp1_hdr.vlan_present = 1; qp->qp1_hdr.eth.type = cpu_to_be16(ETH_P_8021Q); } /* GRH */ qp->qp1_hdr.grh_present = 1; qp->qp1_hdr.grh.ip_version = 6; qp->qp1_hdr.grh.payload_length = cpu_to_be16((IB_BTH_BYTES + IB_DETH_BYTES + payload_size + 7) & ~3); qp->qp1_hdr.grh.next_header = 0x1b; memcpy(qp->qp1_hdr.grh.source_gid.raw, sgid.raw, sizeof(sgid)); memcpy(qp->qp1_hdr.grh.destination_gid.raw, qplib_ah->dgid.data, sizeof(sgid)); /* BTH */ if (wr->opcode == IB_WR_SEND_WITH_IMM) { qp->qp1_hdr.bth.opcode = IB_OPCODE_UD_SEND_ONLY_WITH_IMMEDIATE; qp->qp1_hdr.immediate_present = 1; } else { qp->qp1_hdr.bth.opcode = IB_OPCODE_UD_SEND_ONLY; } if (wr->send_flags & IB_SEND_SOLICITED) qp->qp1_hdr.bth.solicited_event = 1; qp->qp1_hdr.bth.pad_count = (4 - payload_size) & 3; /* P_key for QP1 is for all members */ qp->qp1_hdr.bth.pkey = cpu_to_be16(0xFFFF); qp->qp1_hdr.bth.destination_qpn = IB_QP1; qp->qp1_hdr.bth.ack_req = 0; qp->send_psn++; qp->send_psn &= BTH_PSN_MASK; qp->qp1_hdr.bth.psn = cpu_to_be32(qp->send_psn); /* DETH */ /* Use the priviledged Q_Key for QP1 */ qp->qp1_hdr.deth.qkey = cpu_to_be32(IB_QP1_QKEY); qp->qp1_hdr.deth.source_qpn = IB_QP1; /* Pack the QP1 to the transmit buffer */ buf = bnxt_qplib_get_qp1_sq_buf(&qp->qplib_qp, &sge); if (!buf) { dev_err(rdev_to_dev(qp->rdev), "QP1 buffer is empty!\n"); rc = -ENOMEM; } for (i = wqe->num_sge; i; i--) { wqe->sg_list[i].addr = wqe->sg_list[i - 1].addr; wqe->sg_list[i].lkey = wqe->sg_list[i - 1].lkey; wqe->sg_list[i].size = wqe->sg_list[i - 1].size; } wqe->sg_list[0].addr = sge.addr; wqe->sg_list[0].lkey = sge.lkey; wqe->sg_list[0].size = sge.size; wqe->num_sge++; return rc; } static int bnxt_re_build_gsi_send(struct bnxt_re_qp *qp, const struct ib_send_wr *wr, struct bnxt_qplib_swqe *wqe) { struct bnxt_re_dev *rdev; int rc, indx, len = 0; rdev = qp->rdev; /* Mode UD is applicable to Gen P5 only */ if (rdev->gsi_ctx.gsi_qp_mode == BNXT_RE_GSI_MODE_UD) return 0; for (indx = 0; indx < wr->num_sge; indx++) { wqe->sg_list[indx].addr = wr->sg_list[indx].addr; wqe->sg_list[indx].lkey = wr->sg_list[indx].lkey; wqe->sg_list[indx].size = wr->sg_list[indx].length; len += wr->sg_list[indx].length; } rc = bnxt_re_build_qp1_send(qp, wr, wqe, len); wqe->rawqp1.lflags |= SQ_SEND_RAWETH_QP1_LFLAGS_ROCE_CRC; return rc; } /* For the MAD layer, it only provides the recv SGE the size of ib_grh + MAD datagram. No Ethernet headers, Ethertype, BTH, DETH, nor RoCE iCRC. The Cu+ solution must provide buffer for the entire receive packet (334 bytes) with no VLAN and then copy the GRH and the MAD datagram out to the provided SGE. */ static int bnxt_re_build_qp1_recv(struct bnxt_re_qp *qp, const struct ib_recv_wr *wr, struct bnxt_qplib_swqe *wqe) { struct bnxt_re_dev *rdev = qp->rdev; struct bnxt_qplib_sge ref, sge; u8 udp_hdr_size = 0; u8 ip_hdr_size = 0; int rc = 0; int size; if (bnxt_qplib_get_qp1_rq_buf(&qp->qplib_qp, &sge)) { /* Create 5 SGEs as according to the following: * Ethernet header (14) * ib_grh (40) - as provided from the wr * ib_bth + ib_deth + UDP(RoCE v2 only) (28) * MAD (256) - as provided from the wr * iCRC (4) */ /* Set RoCE v2 header size and offsets */ if (rdev->gsi_ctx.gsi_qp_mode == BNXT_RE_GSI_MODE_ROCE_V2_IPV4) ip_hdr_size = 20; if (rdev->gsi_ctx.gsi_qp_mode != BNXT_RE_GSI_MODE_ROCE_V1) udp_hdr_size = 8; /* Save the reference from ULP */ ref.addr = wr->sg_list[0].addr; ref.lkey = wr->sg_list[0].lkey; ref.size = wr->sg_list[0].length; /* SGE 1 */ size = sge.size; wqe->sg_list[0].addr = sge.addr; wqe->sg_list[0].lkey = sge.lkey; wqe->sg_list[0].size = BNXT_QPLIB_MAX_QP1_RQ_ETH_HDR_SIZE; size -= wqe->sg_list[0].size; if (size <= 0) { dev_err(rdev_to_dev(qp->rdev),"QP1 rq buffer is empty!\n"); rc = -ENOMEM; goto done; } sge.size = (u32)size; sge.addr += wqe->sg_list[0].size; /* SGE 2 */ /* In case of RoCE v2 ipv4 lower 20 bytes should have IP hdr */ wqe->sg_list[1].addr = ref.addr + ip_hdr_size; wqe->sg_list[1].lkey = ref.lkey; wqe->sg_list[1].size = sizeof(struct ib_grh) - ip_hdr_size; ref.size -= wqe->sg_list[1].size; if (ref.size <= 0) { dev_err(rdev_to_dev(qp->rdev), "QP1 ref buffer is empty!\n"); rc = -ENOMEM; goto done; } ref.addr += wqe->sg_list[1].size + ip_hdr_size; /* SGE 3 */ wqe->sg_list[2].addr = sge.addr; wqe->sg_list[2].lkey = sge.lkey; wqe->sg_list[2].size = BNXT_QPLIB_MAX_QP1_RQ_BDETH_HDR_SIZE + udp_hdr_size; size -= wqe->sg_list[2].size; if (size <= 0) { dev_err(rdev_to_dev(qp->rdev), "QP1 rq buffer is empty!\n"); rc = -ENOMEM; goto done; } sge.size = (u32)size; sge.addr += wqe->sg_list[2].size; /* SGE 4 */ wqe->sg_list[3].addr = ref.addr; wqe->sg_list[3].lkey = ref.lkey; wqe->sg_list[3].size = ref.size; ref.size -= wqe->sg_list[3].size; if (ref.size) { dev_err(rdev_to_dev(qp->rdev), "QP1 ref buffer is incorrect!\n"); rc = -ENOMEM; goto done; } /* SGE 5 */ wqe->sg_list[4].addr = sge.addr; wqe->sg_list[4].lkey = sge.lkey; wqe->sg_list[4].size = sge.size; size -= wqe->sg_list[4].size; if (size) { dev_err(rdev_to_dev(qp->rdev), "QP1 rq buffer is incorrect!\n"); rc = -ENOMEM; goto done; } sge.size = (u32)size; wqe->num_sge = 5; } else { dev_err(rdev_to_dev(qp->rdev), "QP1 buffer is empty!\n"); rc = -ENOMEM; } done: return rc; } static int bnxt_re_build_qp1_shadow_qp_recv(struct bnxt_re_qp *qp, const struct ib_recv_wr *wr, struct bnxt_qplib_swqe *wqe) { struct bnxt_re_sqp_entries *sqp_entry; struct bnxt_qplib_sge sge; struct bnxt_re_dev *rdev; u32 rq_prod_index; int rc = 0; rdev = qp->rdev; rq_prod_index = bnxt_qplib_get_rq_prod_index(&qp->qplib_qp); if (bnxt_qplib_get_qp1_rq_buf(&qp->qplib_qp, &sge)) { /* Create 1 SGE to receive the entire * ethernet packet */ /* SGE 1 */ wqe->sg_list[0].addr = sge.addr; /* TODO check the lkey to be used */ wqe->sg_list[0].lkey = sge.lkey; wqe->sg_list[0].size = BNXT_QPLIB_MAX_QP1_RQ_HDR_SIZE_V2; if (sge.size < wqe->sg_list[0].size) { dev_err(rdev_to_dev(qp->rdev), "QP1 rq buffer is empty!\n"); rc = -ENOMEM; goto done; } sqp_entry = &rdev->gsi_ctx.sqp_tbl[rq_prod_index]; sqp_entry->sge.addr = wr->sg_list[0].addr; sqp_entry->sge.lkey = wr->sg_list[0].lkey; sqp_entry->sge.size = wr->sg_list[0].length; /* Store the wrid for reporting completion */ sqp_entry->wrid = wqe->wr_id; /* change the wqe->wrid to table index */ wqe->wr_id = rq_prod_index; } done: return rc; } static bool is_ud_qp(struct bnxt_re_qp *qp) { return (qp->qplib_qp.type == CMDQ_CREATE_QP_TYPE_UD || qp->qplib_qp.type == CMDQ_CREATE_QP_TYPE_GSI); } static int bnxt_re_build_send_wqe(struct bnxt_re_qp *qp, const struct ib_send_wr *wr, struct bnxt_qplib_swqe *wqe) { struct bnxt_re_ah *ah = NULL; if(is_ud_qp(qp)) { ah = to_bnxt_re(ud_wr(wr)->ah, struct bnxt_re_ah, ibah); wqe->send.q_key = ud_wr(wr)->remote_qkey; wqe->send.dst_qp = ud_wr(wr)->remote_qpn; wqe->send.avid = ah->qplib_ah.id; } switch (wr->opcode) { case IB_WR_SEND: wqe->type = BNXT_QPLIB_SWQE_TYPE_SEND; break; case IB_WR_SEND_WITH_IMM: wqe->type = BNXT_QPLIB_SWQE_TYPE_SEND_WITH_IMM; wqe->send.imm_data = wr->ex.imm_data; break; case IB_WR_SEND_WITH_INV: wqe->type = BNXT_QPLIB_SWQE_TYPE_SEND_WITH_INV; wqe->send.inv_key = wr->ex.invalidate_rkey; break; default: dev_err(rdev_to_dev(qp->rdev), "%s Invalid opcode %d!\n", __func__, wr->opcode); return -EINVAL; } if (wr->send_flags & IB_SEND_SIGNALED) wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP; if (wr->send_flags & IB_SEND_FENCE) wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE; if (wr->send_flags & IB_SEND_SOLICITED) wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SOLICIT_EVENT; if (wr->send_flags & IB_SEND_INLINE) wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_INLINE; return 0; } static int bnxt_re_build_rdma_wqe(const struct ib_send_wr *wr, struct bnxt_qplib_swqe *wqe) { switch (wr->opcode) { case IB_WR_RDMA_WRITE: wqe->type = BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE; break; case IB_WR_RDMA_WRITE_WITH_IMM: wqe->type = BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE_WITH_IMM; wqe->rdma.imm_data = wr->ex.imm_data; break; case IB_WR_RDMA_READ: wqe->type = BNXT_QPLIB_SWQE_TYPE_RDMA_READ; wqe->rdma.inv_key = wr->ex.invalidate_rkey; break; default: return -EINVAL; } wqe->rdma.remote_va = rdma_wr(wr)->remote_addr; wqe->rdma.r_key = rdma_wr(wr)->rkey; if (wr->send_flags & IB_SEND_SIGNALED) wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP; if (wr->send_flags & IB_SEND_FENCE) wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE; if (wr->send_flags & IB_SEND_SOLICITED) wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SOLICIT_EVENT; if (wr->send_flags & IB_SEND_INLINE) wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_INLINE; return 0; } static int bnxt_re_build_atomic_wqe(const struct ib_send_wr *wr, struct bnxt_qplib_swqe *wqe) { switch (wr->opcode) { case IB_WR_ATOMIC_CMP_AND_SWP: wqe->type = BNXT_QPLIB_SWQE_TYPE_ATOMIC_CMP_AND_SWP; wqe->atomic.cmp_data = atomic_wr(wr)->compare_add; wqe->atomic.swap_data = atomic_wr(wr)->swap; break; case IB_WR_ATOMIC_FETCH_AND_ADD: wqe->type = BNXT_QPLIB_SWQE_TYPE_ATOMIC_FETCH_AND_ADD; wqe->atomic.cmp_data = atomic_wr(wr)->compare_add; break; default: return -EINVAL; } wqe->atomic.remote_va = atomic_wr(wr)->remote_addr; wqe->atomic.r_key = atomic_wr(wr)->rkey; if (wr->send_flags & IB_SEND_SIGNALED) wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP; if (wr->send_flags & IB_SEND_FENCE) wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE; if (wr->send_flags & IB_SEND_SOLICITED) wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SOLICIT_EVENT; return 0; } static int bnxt_re_build_inv_wqe(const struct ib_send_wr *wr, struct bnxt_qplib_swqe *wqe) { wqe->type = BNXT_QPLIB_SWQE_TYPE_LOCAL_INV; wqe->local_inv.inv_l_key = wr->ex.invalidate_rkey; if (wr->send_flags & IB_SEND_SIGNALED) wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP; if (wr->send_flags & IB_SEND_FENCE) wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE; if (wr->send_flags & IB_SEND_SOLICITED) wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SOLICIT_EVENT; return 0; } static int bnxt_re_build_reg_wqe(const struct ib_reg_wr *wr, struct bnxt_qplib_swqe *wqe) { struct bnxt_re_mr *mr = to_bnxt_re(wr->mr, struct bnxt_re_mr, ib_mr); struct bnxt_qplib_frpl *qplib_frpl = &mr->qplib_frpl; int reg_len, i, access = wr->access; if (mr->npages > qplib_frpl->max_pg_ptrs) { dev_err_ratelimited(rdev_to_dev(mr->rdev), " %s: failed npages %d > %d\n", __func__, mr->npages, qplib_frpl->max_pg_ptrs); return -EINVAL; } wqe->frmr.pbl_ptr = (__le64 *)qplib_frpl->hwq.pbl_ptr[0]; wqe->frmr.pbl_dma_ptr = qplib_frpl->hwq.pbl_dma_ptr[0]; wqe->frmr.levels = qplib_frpl->hwq.level; wqe->frmr.page_list = mr->pages; wqe->frmr.page_list_len = mr->npages; wqe->type = BNXT_QPLIB_SWQE_TYPE_REG_MR; if (wr->wr.send_flags & IB_SEND_SIGNALED) wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP; if (access & IB_ACCESS_LOCAL_WRITE) wqe->frmr.access_cntl |= SQ_FR_PMR_ACCESS_CNTL_LOCAL_WRITE; if (access & IB_ACCESS_REMOTE_READ) wqe->frmr.access_cntl |= SQ_FR_PMR_ACCESS_CNTL_REMOTE_READ; if (access & IB_ACCESS_REMOTE_WRITE) wqe->frmr.access_cntl |= SQ_FR_PMR_ACCESS_CNTL_REMOTE_WRITE; if (access & IB_ACCESS_REMOTE_ATOMIC) wqe->frmr.access_cntl |= SQ_FR_PMR_ACCESS_CNTL_REMOTE_ATOMIC; if (access & IB_ACCESS_MW_BIND) wqe->frmr.access_cntl |= SQ_FR_PMR_ACCESS_CNTL_WINDOW_BIND; /* TODO: OFED provides the rkey of the MR instead of the lkey */ wqe->frmr.l_key = wr->key; wqe->frmr.length = wr->mr->length; wqe->frmr.pbl_pg_sz_log = ilog2(PAGE_SIZE >> PAGE_SHIFT_4K); wqe->frmr.pg_sz_log = ilog2(wr->mr->page_size >> PAGE_SHIFT_4K); wqe->frmr.va = wr->mr->iova; reg_len = wqe->frmr.page_list_len * wr->mr->page_size; if (wqe->frmr.length > reg_len) { dev_err_ratelimited(rdev_to_dev(mr->rdev), "%s: bnxt_re_mr 0x%px len (%d > %d)\n", __func__, (void *)mr, wqe->frmr.length, reg_len); for (i = 0; i < mr->npages; i++) dev_dbg(rdev_to_dev(mr->rdev), "%s: build_reg_wqe page[%d] = 0x%llx\n", __func__, i, mr->pages[i]); return -EINVAL; } return 0; } static void bnxt_re_set_sg_list(const struct ib_send_wr *wr, struct bnxt_qplib_swqe *wqe) { wqe->sg_list = (struct bnxt_qplib_sge *)wr->sg_list; wqe->num_sge = wr->num_sge; } static void bnxt_ud_qp_hw_stall_workaround(struct bnxt_re_qp *qp) { if ((qp->ib_qp.qp_type == IB_QPT_UD || qp->ib_qp.qp_type == IB_QPT_GSI || qp->ib_qp.qp_type == IB_QPT_RAW_ETHERTYPE) && qp->qplib_qp.wqe_cnt == BNXT_RE_UD_QP_HW_STALL) { int qp_attr_mask; struct ib_qp_attr qp_attr; qp_attr_mask = IB_QP_STATE; qp_attr.qp_state = IB_QPS_RTS; bnxt_re_modify_qp(&qp->ib_qp, &qp_attr, qp_attr_mask, NULL); qp->qplib_qp.wqe_cnt = 0; } } static int bnxt_re_post_send_shadow_qp(struct bnxt_re_dev *rdev, struct bnxt_re_qp *qp, const struct ib_send_wr *wr) { struct bnxt_qplib_swqe wqe; unsigned long flags; int rc = 0; spin_lock_irqsave(&qp->sq_lock, flags); while (wr) { /* House keeping */ memset(&wqe, 0, sizeof(wqe)); /* Common */ if (wr->num_sge > qp->qplib_qp.sq.max_sge) { dev_err(rdev_to_dev(rdev), "Limit exceeded for Send SGEs\n"); rc = -EINVAL; break; } bnxt_re_set_sg_list(wr, &wqe); wqe.wr_id = wr->wr_id; wqe.type = BNXT_QPLIB_SWQE_TYPE_SEND; rc = bnxt_re_build_send_wqe(qp, wr, &wqe); if (rc) break; rc = bnxt_qplib_post_send(&qp->qplib_qp, &wqe); if (rc) { dev_err(rdev_to_dev(rdev), "bad_wr seen with opcode = 0x%x rc = %d\n", wr->opcode, rc); break; } wr = wr->next; } bnxt_qplib_post_send_db(&qp->qplib_qp); bnxt_ud_qp_hw_stall_workaround(qp); spin_unlock_irqrestore(&qp->sq_lock, flags); return rc; } static void bnxt_re_legacy_set_uc_fence(struct bnxt_qplib_swqe *wqe) { /* Need unconditional fence for non-wire memory opcode * to work as expected. */ if (wqe->type == BNXT_QPLIB_SWQE_TYPE_LOCAL_INV || wqe->type == BNXT_QPLIB_SWQE_TYPE_FAST_REG_MR || wqe->type == BNXT_QPLIB_SWQE_TYPE_REG_MR || wqe->type == BNXT_QPLIB_SWQE_TYPE_BIND_MW) wqe->flags |= BNXT_QPLIB_SWQE_FLAGS_UC_FENCE; } int bnxt_re_post_send(struct ib_qp *ib_qp, const struct ib_send_wr *wr, const struct ib_send_wr **bad_wr) { struct bnxt_re_qp *qp = to_bnxt_re(ib_qp, struct bnxt_re_qp, ib_qp); struct bnxt_qplib_sge sge[6]; struct bnxt_qplib_swqe wqe; struct bnxt_re_dev *rdev; unsigned long flags; int rc = 0; rdev = qp->rdev; spin_lock_irqsave(&qp->sq_lock, flags); while (wr) { /* House keeping */ memset(&wqe, 0, sizeof(wqe)); /* Common */ if (wr->num_sge > qp->qplib_qp.sq.max_sge) { dev_err(rdev_to_dev(rdev), "Limit exceeded for Send SGEs\n"); rc = -EINVAL; goto bad; } bnxt_re_set_sg_list(wr, &wqe); wqe.wr_id = wr->wr_id; switch (wr->opcode) { case IB_WR_SEND: case IB_WR_SEND_WITH_IMM: if (ib_qp->qp_type == IB_QPT_GSI && rdev->gsi_ctx.gsi_qp_mode != BNXT_RE_GSI_MODE_UD) { memset(sge, 0, sizeof(sge)); wqe.sg_list = sge; rc = bnxt_re_build_gsi_send(qp, wr, &wqe); if (rc) goto bad; } else if (ib_qp->qp_type == IB_QPT_RAW_ETHERTYPE) { bnxt_re_build_raw_send(wr, &wqe); } switch (wr->send_flags) { case IB_SEND_IP_CSUM: wqe.rawqp1.lflags |= SQ_SEND_RAWETH_QP1_LFLAGS_IP_CHKSUM; break; default: break; } fallthrough; case IB_WR_SEND_WITH_INV: rc = bnxt_re_build_send_wqe(qp, wr, &wqe); break; case IB_WR_RDMA_WRITE: case IB_WR_RDMA_WRITE_WITH_IMM: case IB_WR_RDMA_READ: rc = bnxt_re_build_rdma_wqe(wr, &wqe); break; case IB_WR_ATOMIC_CMP_AND_SWP: case IB_WR_ATOMIC_FETCH_AND_ADD: rc = bnxt_re_build_atomic_wqe(wr, &wqe); break; case IB_WR_RDMA_READ_WITH_INV: dev_err(rdev_to_dev(rdev), "RDMA Read with Invalidate is not supported\n"); rc = -EINVAL; goto bad; case IB_WR_LOCAL_INV: rc = bnxt_re_build_inv_wqe(wr, &wqe); break; case IB_WR_REG_MR: rc = bnxt_re_build_reg_wqe(reg_wr(wr), &wqe); break; default: /* Unsupported WRs */ dev_err(rdev_to_dev(rdev), "WR (0x%x) is not supported\n", wr->opcode); rc = -EINVAL; goto bad; } if (likely(!rc)) { if (!_is_chip_gen_p5_p7(rdev->chip_ctx)) bnxt_re_legacy_set_uc_fence(&wqe); rc = bnxt_qplib_post_send(&qp->qplib_qp, &wqe); } bad: if (unlikely(rc)) { dev_err(rdev_to_dev(rdev), "bad_wr seen with opcode = 0x%x\n", wr->opcode); *bad_wr = wr; break; } wr = wr->next; } bnxt_qplib_post_send_db(&qp->qplib_qp); if (!_is_chip_gen_p5_p7(rdev->chip_ctx)) bnxt_ud_qp_hw_stall_workaround(qp); spin_unlock_irqrestore(&qp->sq_lock, flags); return rc; } static int bnxt_re_post_recv_shadow_qp(struct bnxt_re_dev *rdev, struct bnxt_re_qp *qp, struct ib_recv_wr *wr) { struct bnxt_qplib_swqe wqe; int rc = 0; /* rq lock can be pardoned here. */ while (wr) { /* House keeping */ memset(&wqe, 0, sizeof(wqe)); /* Common */ if (wr->num_sge > qp->qplib_qp.rq.max_sge) { dev_err(rdev_to_dev(rdev), "Limit exceeded for Receive SGEs\n"); rc = -EINVAL; goto bad; } wqe.sg_list = (struct bnxt_qplib_sge *)wr->sg_list; wqe.num_sge = wr->num_sge; wqe.wr_id = wr->wr_id; wqe.type = BNXT_QPLIB_SWQE_TYPE_RECV; rc = bnxt_qplib_post_recv(&qp->qplib_qp, &wqe); bad: if (rc) { dev_err(rdev_to_dev(rdev), "bad_wr seen with RQ post\n"); break; } wr = wr->next; } bnxt_qplib_post_recv_db(&qp->qplib_qp); return rc; } static int bnxt_re_build_gsi_recv(struct bnxt_re_qp *qp, const struct ib_recv_wr *wr, struct bnxt_qplib_swqe *wqe) { struct bnxt_re_dev *rdev = qp->rdev; int rc = 0; if (rdev->gsi_ctx.gsi_qp_mode == BNXT_RE_GSI_MODE_ALL) rc = bnxt_re_build_qp1_shadow_qp_recv(qp, wr, wqe); else rc = bnxt_re_build_qp1_recv(qp, wr, wqe); return rc; } int bnxt_re_post_recv(struct ib_qp *ib_qp, const struct ib_recv_wr *wr, const struct ib_recv_wr **bad_wr) { struct bnxt_re_qp *qp = to_bnxt_re(ib_qp, struct bnxt_re_qp, ib_qp); struct bnxt_qplib_sge sge[6]; struct bnxt_qplib_swqe wqe; unsigned long flags; u32 count = 0; int rc = 0; spin_lock_irqsave(&qp->rq_lock, flags); while (wr) { memset(&wqe, 0, sizeof(wqe)); if (wr->num_sge > qp->qplib_qp.rq.max_sge) { dev_err(rdev_to_dev(qp->rdev), "Limit exceeded for Receive SGEs\n"); rc = -EINVAL; goto bad; } wqe.num_sge = wr->num_sge; wqe.sg_list = (struct bnxt_qplib_sge *)wr->sg_list; wqe.wr_id = wr->wr_id; wqe.type = BNXT_QPLIB_SWQE_TYPE_RECV; if (ib_qp->qp_type == IB_QPT_GSI && qp->rdev->gsi_ctx.gsi_qp_mode != BNXT_RE_GSI_MODE_UD) { memset(sge, 0, sizeof(sge)); wqe.sg_list = sge; rc = bnxt_re_build_gsi_recv(qp, wr, &wqe); if (rc) goto bad; } rc = bnxt_qplib_post_recv(&qp->qplib_qp, &wqe); bad: if (rc) { dev_err(rdev_to_dev(qp->rdev), "bad_wr seen with RQ post\n"); *bad_wr = wr; break; } /* Ring DB if the RQEs posted reaches a threshold value */ if (++count >= BNXT_RE_RQ_WQE_THRESHOLD) { bnxt_qplib_post_recv_db(&qp->qplib_qp); count = 0; } wr = wr->next; } if (count) bnxt_qplib_post_recv_db(&qp->qplib_qp); spin_unlock_irqrestore(&qp->rq_lock, flags); return rc; } /* Completion Queues */ void bnxt_re_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata) { struct bnxt_re_cq *cq = to_bnxt_re(ib_cq, struct bnxt_re_cq, ibcq); struct bnxt_re_dev *rdev = cq->rdev; int rc = 0; if (cq->uctx_cq_page) { BNXT_RE_CQ_PAGE_LIST_DEL(cq->uctx, cq); free_page((u64)cq->uctx_cq_page); cq->uctx_cq_page = NULL; } if (cq->is_dbr_soft_cq && cq->uctx) { void *dbr_page; if (cq->uctx->dbr_recov_cq) { dbr_page = cq->uctx->dbr_recov_cq_page; cq->uctx->dbr_recov_cq_page = NULL; cq->uctx->dbr_recov_cq = NULL; free_page((unsigned long)dbr_page); } goto end; } /* CQ getting destroyed. Set this state for cqn handler */ spin_lock_bh(&cq->qplib_cq.compl_lock); cq->qplib_cq.destroyed = true; spin_unlock_bh(&cq->qplib_cq.compl_lock); if (ib_cq->poll_ctx == IB_POLL_WORKQUEUE || ib_cq->poll_ctx == IB_POLL_UNBOUND_WORKQUEUE) cancel_work_sync(&ib_cq->work); rc = bnxt_qplib_destroy_cq(&rdev->qplib_res, &cq->qplib_cq); if (rc) dev_err_ratelimited(rdev_to_dev(rdev), "%s id = %d failed rc = %d\n", __func__, cq->qplib_cq.id, rc); bnxt_re_put_nq(rdev, cq->qplib_cq.nq); if (cq->umem && !IS_ERR(cq->umem)) ib_umem_release(cq->umem); kfree(cq->cql); atomic_dec(&rdev->stats.rsors.cq_count); end: return; } static inline struct bnxt_re_cq *__get_cq_from_cq_in(struct ib_cq *cq_in, struct bnxt_re_dev *rdev) { struct bnxt_re_cq *cq; cq = container_of(cq_in, struct bnxt_re_cq, ibcq); return cq; } int bnxt_re_create_cq(struct ib_cq *cq_in, const struct ib_cq_init_attr *attr, struct ib_udata *udata) { struct bnxt_qplib_dev_attr *dev_attr; struct bnxt_re_ucontext *uctx = NULL; struct ib_ucontext *context = NULL; struct bnxt_qplib_cq *qplcq; struct bnxt_re_cq_req ureq; struct bnxt_re_dev *rdev; int rc, entries; struct bnxt_re_cq *cq; u32 max_active_cqs; int cqe = attr->cqe; if (attr->flags) return -EOPNOTSUPP; rdev = rdev_from_cq_in(cq_in); if (rdev->mod_exit) { rc = -EIO; dev_dbg(rdev_to_dev(rdev), "%s(): in mod_exit, just return!\n", __func__); goto exit; } if (udata) { uctx = rdma_udata_to_drv_context(udata, struct bnxt_re_ucontext, ibucontext); context = &uctx->ibucontext; } dev_attr = rdev->dev_attr; if (atomic_read(&rdev->stats.rsors.cq_count) >= dev_attr->max_cq) { dev_err(rdev_to_dev(rdev), "Create CQ failed - max exceeded(CQs)\n"); rc = -EINVAL; goto exit; } /* Validate CQ fields */ if (cqe < 1 || cqe > dev_attr->max_cq_wqes) { dev_err(rdev_to_dev(rdev), "Create CQ failed - max exceeded(CQ_WQs)\n"); rc = -EINVAL; goto exit; } cq = __get_cq_from_cq_in(cq_in, rdev); if (!cq) { rc = -ENOMEM; goto exit; } cq->rdev = rdev; cq->uctx = uctx; qplcq = &cq->qplib_cq; qplcq->cq_handle = (u64)qplcq; /* * Since CQ is for QP1 is shared with Shadow CQ, the size * should be double the size. There is no way to identify * whether this CQ is for GSI QP. So assuming that the first * CQ created is for QP1 */ if (!udata && !rdev->gsi_ctx.first_cq_created && rdev->gsi_ctx.gsi_qp_mode == BNXT_RE_GSI_MODE_ALL) { rdev->gsi_ctx.first_cq_created = true; /* * Total CQE required for the CQ = CQE for QP1 RQ + * CQE for Shadow QP SQEs + CQE for Shadow QP RQEs. * Max entries of shadow QP SQ and RQ = QP1 RQEs = cqe */ cqe *= 3; } entries = bnxt_re_init_depth(cqe + 1, uctx); if (entries > dev_attr->max_cq_wqes + 1) entries = dev_attr->max_cq_wqes + 1; qplcq->sginfo.pgshft = PAGE_SHIFT; qplcq->sginfo.pgsize = PAGE_SIZE; if (udata) { if (udata->inlen < sizeof(ureq)) dev_warn(rdev_to_dev(rdev), "Update the library ulen %d klen %d\n", (unsigned int)udata->inlen, (unsigned int)sizeof(ureq)); rc = ib_copy_from_udata(&ureq, udata, min(udata->inlen, sizeof(ureq))); if (rc) goto fail; if (BNXT_RE_IS_DBR_PACING_NOTIFY_CQ(ureq)) { cq->is_dbr_soft_cq = true; goto success; } if (BNXT_RE_IS_DBR_RECOV_CQ(ureq)) { void *dbr_page; u32 *epoch; dbr_page = (void *)__get_free_page(GFP_KERNEL); if (!dbr_page) { dev_err(rdev_to_dev(rdev), "DBR recov CQ page allocation failed!"); rc = -ENOMEM; goto fail; } /* memset the epoch and epoch_ack to 0 */ epoch = dbr_page; epoch[0] = 0x0; epoch[1] = 0x0; uctx->dbr_recov_cq = cq; uctx->dbr_recov_cq_page = dbr_page; cq->is_dbr_soft_cq = true; goto success; } cq->umem = ib_umem_get_compat (rdev, context, udata, ureq.cq_va, entries * sizeof(struct cq_base), IB_ACCESS_LOCAL_WRITE, 1); if (IS_ERR(cq->umem)) { rc = PTR_ERR(cq->umem); dev_err(rdev_to_dev(rdev), "%s: ib_umem_get failed! rc = %d\n", __func__, rc); goto fail; } qplcq->sginfo.sghead = get_ib_umem_sgl(cq->umem, &qplcq->sginfo.nmap); qplcq->sginfo.npages = ib_umem_num_pages_compat(cq->umem); if (!uctx->dpi.dbr) { rc = bnxt_re_get_user_dpi(rdev, uctx); if (rc) goto c2fail; } qplcq->dpi = &uctx->dpi; } else { cq->max_cql = entries > MAX_CQL_PER_POLL ? MAX_CQL_PER_POLL : entries; cq->cql = kcalloc(cq->max_cql, sizeof(struct bnxt_qplib_cqe), GFP_KERNEL); if (!cq->cql) { dev_err(rdev_to_dev(rdev), "Allocate CQL for %d failed!\n", cq->max_cql); rc = -ENOMEM; goto fail; } qplcq->dpi = &rdev->dpi_privileged; } /* * Allocating the NQ in a round robin fashion. nq_alloc_cnt is a * used for getting the NQ index. */ qplcq->max_wqe = entries; qplcq->nq = bnxt_re_get_nq(rdev); qplcq->cnq_hw_ring_id = qplcq->nq->ring_id; rc = bnxt_qplib_create_cq(&rdev->qplib_res, qplcq); if (rc) { dev_err(rdev_to_dev(rdev), "Create HW CQ failed!\n"); goto fail; } INIT_LIST_HEAD(&cq->cq_list); cq->ibcq.cqe = entries; cq->cq_period = qplcq->period; atomic_inc(&rdev->stats.rsors.cq_count); max_active_cqs = atomic_read(&rdev->stats.rsors.cq_count); if (max_active_cqs > atomic_read(&rdev->stats.rsors.max_cq_count)) atomic_set(&rdev->stats.rsors.max_cq_count, max_active_cqs); spin_lock_init(&cq->cq_lock); if (udata) { struct bnxt_re_cq_resp resp; resp.cqid = qplcq->id; resp.tail = qplcq->hwq.cons; resp.phase = qplcq->period; resp.comp_mask = 0; resp.dbr = (u64)uctx->dpi.umdbr; resp.dpi = uctx->dpi.dpi; resp.comp_mask |= BNXT_RE_COMP_MASK_CQ_HAS_DB_INFO; /* Copy only on a valid wcpdi */ if (uctx->wcdpi.dpi) { resp.wcdpi = uctx->wcdpi.dpi; resp.comp_mask |= BNXT_RE_COMP_MASK_CQ_HAS_WC_DPI; } if (_is_chip_p7(rdev->chip_ctx)) { cq->uctx_cq_page = (void *)__get_free_page(GFP_KERNEL); if (!cq->uctx_cq_page) { dev_err(rdev_to_dev(rdev), "CQ page allocation failed!\n"); bnxt_qplib_destroy_cq(&rdev->qplib_res, qplcq); rc = -ENOMEM; goto c2fail; } resp.uctx_cq_page = (u64)cq->uctx_cq_page; resp.comp_mask |= BNXT_RE_COMP_MASK_CQ_HAS_CQ_PAGE; } rc = bnxt_re_copy_to_udata(rdev, &resp, min(udata->outlen, sizeof(resp)), udata); if (rc) { free_page((u64)cq->uctx_cq_page); cq->uctx_cq_page = NULL; bnxt_qplib_destroy_cq(&rdev->qplib_res, qplcq); goto c2fail; } if (cq->uctx_cq_page) BNXT_RE_CQ_PAGE_LIST_ADD(uctx, cq); } success: return 0; c2fail: if (udata && cq->umem && !IS_ERR(cq->umem)) ib_umem_release(cq->umem); fail: if (cq) { if (cq->cql) kfree(cq->cql); } exit: return rc; } int bnxt_re_modify_cq(struct ib_cq *ib_cq, u16 cq_count, u16 cq_period) { struct bnxt_re_cq *cq = to_bnxt_re(ib_cq, struct bnxt_re_cq, ibcq); struct bnxt_re_dev *rdev = cq->rdev; int rc; if ((cq->cq_count != cq_count) || (cq->cq_period != cq_period)) { cq->qplib_cq.count = cq_count; cq->qplib_cq.period = cq_period; rc = bnxt_qplib_modify_cq(&rdev->qplib_res, &cq->qplib_cq); if (rc) { dev_err(rdev_to_dev(rdev), "Modify HW CQ %#x failed!\n", cq->qplib_cq.id); return rc; } /* On success, update the shadow */ cq->cq_count = cq_count; cq->cq_period = cq_period; } return 0; } static void bnxt_re_resize_cq_complete(struct bnxt_re_cq *cq) { struct bnxt_re_dev *rdev = cq->rdev; bnxt_qplib_resize_cq_complete(&rdev->qplib_res, &cq->qplib_cq); cq->qplib_cq.max_wqe = cq->resize_cqe; if (cq->resize_umem) { ib_umem_release(cq->umem); cq->umem = cq->resize_umem; cq->resize_umem = NULL; cq->resize_cqe = 0; } } int bnxt_re_resize_cq(struct ib_cq *ib_cq, int cqe, struct ib_udata *udata) { struct bnxt_qplib_sg_info sginfo = {}; struct bnxt_qplib_dpi *orig_dpi = NULL; struct bnxt_qplib_dev_attr *dev_attr; struct bnxt_re_ucontext *uctx = NULL; struct bnxt_re_resize_cq_req ureq; struct ib_ucontext *context = NULL; struct bnxt_re_dev *rdev; struct bnxt_re_cq *cq; int rc, entries; /* Don't allow more than one resize request at the same time. * TODO: need a mutex here when we support kernel consumers of resize. */ cq = to_bnxt_re(ib_cq, struct bnxt_re_cq, ibcq); rdev = cq->rdev; dev_attr = rdev->dev_attr; if (ib_cq->uobject) { uctx = rdma_udata_to_drv_context(udata, struct bnxt_re_ucontext, ibucontext); context = &uctx->ibucontext; } if (cq->resize_umem) { dev_err(rdev_to_dev(rdev), "Resize CQ %#x failed - Busy\n", cq->qplib_cq.id); return -EBUSY; } /* Check the requested cq depth out of supported depth */ if (cqe < 1 || cqe > dev_attr->max_cq_wqes) { dev_err(rdev_to_dev(rdev), "Resize CQ %#x failed - max exceeded\n", cq->qplib_cq.id); return -EINVAL; } entries = bnxt_re_init_depth(cqe + 1, uctx); entries = min_t(u32, (u32)entries, dev_attr->max_cq_wqes + 1); /* Check to see if the new requested size can be handled by already * existing CQ */ if (entries == cq->ibcq.cqe) { dev_info(rdev_to_dev(rdev), "CQ is already at size %d\n", cqe); return 0; } if (ib_cq->uobject && udata) { if (udata->inlen < sizeof(ureq)) dev_warn(rdev_to_dev(rdev), "Update the library ulen %d klen %d\n", (unsigned int)udata->inlen, (unsigned int)sizeof(ureq)); rc = ib_copy_from_udata(&ureq, udata, min(udata->inlen, sizeof(ureq))); if (rc) goto fail; dev_dbg(rdev_to_dev(rdev), "%s: va %p\n", __func__, (void *)ureq.cq_va); cq->resize_umem = ib_umem_get_compat (rdev, context, udata, ureq.cq_va, entries * sizeof(struct cq_base), IB_ACCESS_LOCAL_WRITE, 1); if (IS_ERR(cq->resize_umem)) { rc = PTR_ERR(cq->resize_umem); cq->resize_umem = NULL; dev_err(rdev_to_dev(rdev), "%s: ib_umem_get failed! rc = %d\n", __func__, rc); goto fail; } cq->resize_cqe = entries; dev_dbg(rdev_to_dev(rdev), "%s: ib_umem_get() success\n", __func__); memcpy(&sginfo, &cq->qplib_cq.sginfo, sizeof(sginfo)); orig_dpi = cq->qplib_cq.dpi; cq->qplib_cq.sginfo.sghead = get_ib_umem_sgl(cq->resize_umem, &cq->qplib_cq.sginfo.nmap); cq->qplib_cq.sginfo.npages = ib_umem_num_pages_compat(cq->resize_umem); cq->qplib_cq.sginfo.pgsize = PAGE_SIZE; cq->qplib_cq.sginfo.pgshft = PAGE_SHIFT; cq->qplib_cq.dpi = &uctx->dpi; } else { /* TODO: kernel consumer */ } rc = bnxt_qplib_resize_cq(&rdev->qplib_res, &cq->qplib_cq, entries); if (rc) { dev_err(rdev_to_dev(rdev), "Resize HW CQ %#x failed!\n", cq->qplib_cq.id); goto fail; } cq->ibcq.cqe = cq->resize_cqe; /* For kernel consumers complete resize here. For uverbs consumers, * we complete it in the context of ibv_poll_cq(). */ if (!cq->resize_umem) bnxt_qplib_resize_cq_complete(&rdev->qplib_res, &cq->qplib_cq); atomic_inc(&rdev->stats.rsors.resize_count); return 0; fail: if (cq->resize_umem) { ib_umem_release(cq->resize_umem); cq->resize_umem = NULL; cq->resize_cqe = 0; memcpy(&cq->qplib_cq.sginfo, &sginfo, sizeof(sginfo)); cq->qplib_cq.dpi = orig_dpi; } return rc; } static enum ib_wc_status __req_to_ib_wc_status(u8 qstatus) { switch(qstatus) { case CQ_REQ_STATUS_OK: return IB_WC_SUCCESS; case CQ_REQ_STATUS_BAD_RESPONSE_ERR: return IB_WC_BAD_RESP_ERR; case CQ_REQ_STATUS_LOCAL_LENGTH_ERR: return IB_WC_LOC_LEN_ERR; case CQ_REQ_STATUS_LOCAL_QP_OPERATION_ERR: return IB_WC_LOC_QP_OP_ERR; case CQ_REQ_STATUS_LOCAL_PROTECTION_ERR: return IB_WC_LOC_PROT_ERR; case CQ_REQ_STATUS_MEMORY_MGT_OPERATION_ERR: return IB_WC_GENERAL_ERR; case CQ_REQ_STATUS_REMOTE_INVALID_REQUEST_ERR: return IB_WC_REM_INV_REQ_ERR; case CQ_REQ_STATUS_REMOTE_ACCESS_ERR: return IB_WC_REM_ACCESS_ERR; case CQ_REQ_STATUS_REMOTE_OPERATION_ERR: return IB_WC_REM_OP_ERR; case CQ_REQ_STATUS_RNR_NAK_RETRY_CNT_ERR: return IB_WC_RNR_RETRY_EXC_ERR; case CQ_REQ_STATUS_TRANSPORT_RETRY_CNT_ERR: return IB_WC_RETRY_EXC_ERR; case CQ_REQ_STATUS_WORK_REQUEST_FLUSHED_ERR: return IB_WC_WR_FLUSH_ERR; default: return IB_WC_GENERAL_ERR; } return 0; } static enum ib_wc_status __rawqp1_to_ib_wc_status(u8 qstatus) { switch(qstatus) { case CQ_RES_RAWETH_QP1_STATUS_OK: return IB_WC_SUCCESS; case CQ_RES_RAWETH_QP1_STATUS_LOCAL_ACCESS_ERROR: return IB_WC_LOC_ACCESS_ERR; case CQ_RES_RAWETH_QP1_STATUS_HW_LOCAL_LENGTH_ERR: return IB_WC_LOC_LEN_ERR; case CQ_RES_RAWETH_QP1_STATUS_LOCAL_PROTECTION_ERR: return IB_WC_LOC_PROT_ERR; case CQ_RES_RAWETH_QP1_STATUS_LOCAL_QP_OPERATION_ERR: return IB_WC_LOC_QP_OP_ERR; case CQ_RES_RAWETH_QP1_STATUS_MEMORY_MGT_OPERATION_ERR: return IB_WC_GENERAL_ERR; case CQ_RES_RAWETH_QP1_STATUS_WORK_REQUEST_FLUSHED_ERR: return IB_WC_WR_FLUSH_ERR; case CQ_RES_RAWETH_QP1_STATUS_HW_FLUSH_ERR: return IB_WC_WR_FLUSH_ERR; default: return IB_WC_GENERAL_ERR; } } static enum ib_wc_status __rc_to_ib_wc_status(u8 qstatus) { switch(qstatus) { case CQ_RES_RC_STATUS_OK: return IB_WC_SUCCESS; case CQ_RES_RC_STATUS_LOCAL_ACCESS_ERROR: return IB_WC_LOC_ACCESS_ERR; case CQ_RES_RC_STATUS_LOCAL_LENGTH_ERR: return IB_WC_LOC_LEN_ERR; case CQ_RES_RC_STATUS_LOCAL_PROTECTION_ERR: return IB_WC_LOC_PROT_ERR; case CQ_RES_RC_STATUS_LOCAL_QP_OPERATION_ERR: return IB_WC_LOC_QP_OP_ERR; case CQ_RES_RC_STATUS_MEMORY_MGT_OPERATION_ERR: return IB_WC_GENERAL_ERR; case CQ_RES_RC_STATUS_REMOTE_INVALID_REQUEST_ERR: return IB_WC_REM_INV_REQ_ERR; case CQ_RES_RC_STATUS_WORK_REQUEST_FLUSHED_ERR: return IB_WC_WR_FLUSH_ERR; case CQ_RES_RC_STATUS_HW_FLUSH_ERR: return IB_WC_WR_FLUSH_ERR; default: return IB_WC_GENERAL_ERR; } } static void bnxt_re_process_req_wc(struct ib_wc *wc, struct bnxt_qplib_cqe *cqe) { switch (cqe->type) { case BNXT_QPLIB_SWQE_TYPE_SEND: wc->opcode = IB_WC_SEND; break; case BNXT_QPLIB_SWQE_TYPE_SEND_WITH_IMM: wc->opcode = IB_WC_SEND; wc->wc_flags |= IB_WC_WITH_IMM; break; case BNXT_QPLIB_SWQE_TYPE_SEND_WITH_INV: wc->opcode = IB_WC_SEND; wc->wc_flags |= IB_WC_WITH_INVALIDATE; break; case BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE: wc->opcode = IB_WC_RDMA_WRITE; break; case BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE_WITH_IMM: wc->opcode = IB_WC_RDMA_WRITE; wc->wc_flags |= IB_WC_WITH_IMM; break; case BNXT_QPLIB_SWQE_TYPE_RDMA_READ: wc->opcode = IB_WC_RDMA_READ; break; case BNXT_QPLIB_SWQE_TYPE_ATOMIC_CMP_AND_SWP: wc->opcode = IB_WC_COMP_SWAP; break; case BNXT_QPLIB_SWQE_TYPE_ATOMIC_FETCH_AND_ADD: wc->opcode = IB_WC_FETCH_ADD; break; case BNXT_QPLIB_SWQE_TYPE_LOCAL_INV: wc->opcode = IB_WC_LOCAL_INV; break; case BNXT_QPLIB_SWQE_TYPE_REG_MR: wc->opcode = IB_WC_REG_MR; break; default: wc->opcode = IB_WC_SEND; break; } wc->status = __req_to_ib_wc_status(cqe->status); } static int bnxt_re_check_packet_type(u16 raweth_qp1_flags, u16 raweth_qp1_flags2) { bool is_ipv6 = false, is_ipv4 = false; /* raweth_qp1_flags Bit 9-6 indicates itype */ if ((raweth_qp1_flags & CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_ROCE) != CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS_ITYPE_ROCE) return -1; if (raweth_qp1_flags2 & CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_IP_CS_CALC && raweth_qp1_flags2 & CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_L4_CS_CALC) { /* raweth_qp1_flags2 Bit 8 indicates ip_type. 0-v4 1 - v6 */ (raweth_qp1_flags2 & CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_IP_TYPE) ? (is_ipv6 = true) : (is_ipv4 = true); return ((is_ipv6) ? BNXT_RE_ROCEV2_IPV6_PACKET : BNXT_RE_ROCEV2_IPV4_PACKET); } else { return BNXT_RE_ROCE_V1_PACKET; } } static bool bnxt_re_is_loopback_packet(struct bnxt_re_dev *rdev, void *rq_hdr_buf) { u8 *tmp_buf = NULL; struct ethhdr *eth_hdr; u16 eth_type; bool rc = false; tmp_buf = (u8 *)rq_hdr_buf; /* * If dest mac is not same as I/F mac, this could be a * loopback address or multicast address, check whether * it is a loopback packet */ if (!ether_addr_equal(tmp_buf, rdev->dev_addr)) { tmp_buf += 4; /* Check the ether type */ eth_hdr = (struct ethhdr *)tmp_buf; eth_type = ntohs(eth_hdr->h_proto); switch (eth_type) { case BNXT_QPLIB_ETHTYPE_ROCEV1: rc = true; break; default: break; } } return rc; } static bool bnxt_re_is_vlan_in_packet(struct bnxt_re_dev *rdev, void *rq_hdr_buf, struct bnxt_qplib_cqe *cqe) { struct vlan_hdr *vlan_hdr; struct ethhdr *eth_hdr; u8 *tmp_buf = NULL; u16 eth_type; tmp_buf = (u8 *)rq_hdr_buf; /* Check the ether type */ eth_hdr = (struct ethhdr *)tmp_buf; eth_type = ntohs(eth_hdr->h_proto); if (eth_type == ETH_P_8021Q) { tmp_buf += sizeof(struct ethhdr); vlan_hdr = (struct vlan_hdr *)tmp_buf; cqe->raweth_qp1_metadata = ntohs(vlan_hdr->h_vlan_TCI) | (eth_type << CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_TPID_SFT); cqe->raweth_qp1_flags2 |= CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_META_FORMAT_VLAN; return true; } return false; } static int bnxt_re_process_raw_qp_packet_receive(struct bnxt_re_qp *gsi_qp, struct bnxt_qplib_cqe *cqe) { struct bnxt_re_sqp_entries *sqp_entry = NULL; struct bnxt_qplib_hdrbuf *hdr_buf; dma_addr_t shrq_hdr_buf_map; struct ib_sge s_sge[2] = {}; struct ib_sge r_sge[2] = {}; struct ib_recv_wr rwr = {}; struct bnxt_re_ah *gsi_sah; struct bnxt_re_qp *gsi_sqp; dma_addr_t rq_hdr_buf_map; struct bnxt_re_dev *rdev; struct ib_send_wr *swr; u32 skip_bytes = 0; void *rq_hdr_buf; int pkt_type = 0; u32 offset = 0; u32 tbl_idx; int rc; struct ib_ud_wr udwr = {}; swr = &udwr.wr; rdev = gsi_qp->rdev; gsi_sqp = rdev->gsi_ctx.gsi_sqp; tbl_idx = cqe->wr_id; hdr_buf = gsi_qp->qplib_qp.rq_hdr_buf; rq_hdr_buf = (u8 *) hdr_buf->va + tbl_idx * hdr_buf->step; rq_hdr_buf_map = bnxt_qplib_get_qp_buf_from_index(&gsi_qp->qplib_qp, tbl_idx); /* Shadow QP header buffer */ shrq_hdr_buf_map = bnxt_qplib_get_qp_buf_from_index(&gsi_sqp->qplib_qp, tbl_idx); sqp_entry = &rdev->gsi_ctx.sqp_tbl[tbl_idx]; /* Find packet type from the cqe */ pkt_type = bnxt_re_check_packet_type(cqe->raweth_qp1_flags, cqe->raweth_qp1_flags2); if (pkt_type < 0) { dev_err(rdev_to_dev(rdev), "Not handling this packet\n"); return -EINVAL; } /* Adjust the offset for the user buffer and post in the rq */ if (pkt_type == BNXT_RE_ROCEV2_IPV4_PACKET) offset = 20; /* * QP1 loopback packet has 4 bytes of internal header before * ether header. Skip these four bytes. */ if (bnxt_re_is_loopback_packet(rdev, rq_hdr_buf)) skip_bytes = 4; if (bnxt_re_is_vlan_in_packet(rdev, rq_hdr_buf, cqe)) skip_bytes += VLAN_HLEN; /* Store this cqe */ memcpy(&sqp_entry->cqe, cqe, sizeof(struct bnxt_qplib_cqe)); sqp_entry->qp1_qp = gsi_qp; /* First send SGE . Skip the ether header*/ s_sge[0].addr = rq_hdr_buf_map + BNXT_QPLIB_MAX_QP1_RQ_ETH_HDR_SIZE + skip_bytes; s_sge[0].lkey = 0xFFFFFFFF; s_sge[0].length = offset ? BNXT_QPLIB_MAX_GRH_HDR_SIZE_IPV4 : BNXT_QPLIB_MAX_GRH_HDR_SIZE_IPV6; /* Second Send SGE */ s_sge[1].addr = s_sge[0].addr + s_sge[0].length + BNXT_QPLIB_MAX_QP1_RQ_BDETH_HDR_SIZE; if (pkt_type != BNXT_RE_ROCE_V1_PACKET) s_sge[1].addr += 8; s_sge[1].lkey = 0xFFFFFFFF; s_sge[1].length = 256; /* First recv SGE */ r_sge[0].addr = shrq_hdr_buf_map; r_sge[0].lkey = 0xFFFFFFFF; r_sge[0].length = 40; r_sge[1].addr = sqp_entry->sge.addr + offset; r_sge[1].lkey = sqp_entry->sge.lkey; r_sge[1].length = BNXT_QPLIB_MAX_GRH_HDR_SIZE_IPV6 + 256 - offset; /* Create receive work request */ rwr.num_sge = 2; rwr.sg_list = r_sge; rwr.wr_id = tbl_idx; rwr.next = NULL; rc = bnxt_re_post_recv_shadow_qp(rdev, gsi_sqp, &rwr); if (rc) { dev_err(rdev_to_dev(rdev), "Failed to post Rx buffers to shadow QP\n"); return -ENOMEM; } swr->num_sge = 2; swr->sg_list = s_sge; swr->wr_id = tbl_idx; swr->opcode = IB_WR_SEND; swr->next = NULL; gsi_sah = rdev->gsi_ctx.gsi_sah; udwr.ah = &gsi_sah->ibah; udwr.remote_qpn = gsi_sqp->qplib_qp.id; udwr.remote_qkey = gsi_sqp->qplib_qp.qkey; /* post data received in the send queue */ rc = bnxt_re_post_send_shadow_qp(rdev, gsi_sqp, swr); return rc; } static void bnxt_re_process_res_rawqp1_wc(struct ib_wc *wc, struct bnxt_qplib_cqe *cqe) { wc->opcode = IB_WC_RECV; wc->status = __rawqp1_to_ib_wc_status(cqe->status); wc->wc_flags |= IB_WC_GRH; } static void bnxt_re_process_res_rc_wc(struct ib_wc *wc, struct bnxt_qplib_cqe *cqe) { wc->opcode = IB_WC_RECV; wc->status = __rc_to_ib_wc_status(cqe->status); if (cqe->flags & CQ_RES_RC_FLAGS_IMM) wc->wc_flags |= IB_WC_WITH_IMM; if (cqe->flags & CQ_RES_RC_FLAGS_INV) wc->wc_flags |= IB_WC_WITH_INVALIDATE; if ((cqe->flags & (CQ_RES_RC_FLAGS_RDMA | CQ_RES_RC_FLAGS_IMM)) == (CQ_RES_RC_FLAGS_RDMA | CQ_RES_RC_FLAGS_IMM)) wc->opcode = IB_WC_RECV_RDMA_WITH_IMM; } /* Returns TRUE if pkt has valid VLAN and if VLAN id is non-zero */ static bool bnxt_re_is_nonzero_vlanid_pkt(struct bnxt_qplib_cqe *orig_cqe, u16 *vid, u8 *sl) { u32 metadata; u16 tpid; bool ret = false; metadata = orig_cqe->raweth_qp1_metadata; if (orig_cqe->raweth_qp1_flags2 & CQ_RES_RAWETH_QP1_RAWETH_QP1_FLAGS2_META_FORMAT_VLAN) { tpid = ((metadata & CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_TPID_MASK) >> CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_TPID_SFT); if (tpid == ETH_P_8021Q) { *vid = metadata & CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_VID_MASK; *sl = (metadata & CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_PRI_MASK) >> CQ_RES_RAWETH_QP1_RAWETH_QP1_METADATA_PRI_SFT; ret = !!(*vid); } } return ret; } static void bnxt_re_process_res_shadow_qp_wc(struct bnxt_re_qp *gsi_sqp, struct ib_wc *wc, struct bnxt_qplib_cqe *cqe) { u32 tbl_idx; struct bnxt_re_dev *rdev = gsi_sqp->rdev; struct bnxt_re_qp *gsi_qp = NULL; struct bnxt_qplib_cqe *orig_cqe = NULL; struct bnxt_re_sqp_entries *sqp_entry = NULL; int nw_type; u16 vlan_id; u8 sl; tbl_idx = cqe->wr_id; sqp_entry = &rdev->gsi_ctx.sqp_tbl[tbl_idx]; gsi_qp = sqp_entry->qp1_qp; orig_cqe = &sqp_entry->cqe; wc->wr_id = sqp_entry->wrid; wc->byte_len = orig_cqe->length; wc->qp = &gsi_qp->ib_qp; wc->ex.imm_data = orig_cqe->immdata; wc->src_qp = orig_cqe->src_qp; memcpy(wc->smac, orig_cqe->smac, ETH_ALEN); if (bnxt_re_is_nonzero_vlanid_pkt(orig_cqe, &vlan_id, &sl)) { if (bnxt_re_check_if_vlan_valid(rdev, vlan_id)) { wc->sl = sl; wc->vlan_id = vlan_id; wc->wc_flags |= IB_WC_WITH_VLAN; } } wc->port_num = 1; wc->vendor_err = orig_cqe->status; wc->opcode = IB_WC_RECV; wc->status = __rawqp1_to_ib_wc_status(orig_cqe->status); wc->wc_flags |= IB_WC_GRH; nw_type = bnxt_re_check_packet_type(orig_cqe->raweth_qp1_flags, orig_cqe->raweth_qp1_flags2); if(nw_type >= 0) dev_dbg(rdev_to_dev(rdev), "%s nw_type = %d\n", __func__, nw_type); } static void bnxt_re_process_res_ud_wc(struct bnxt_re_dev *rdev, struct bnxt_re_qp *qp, struct ib_wc *wc, struct bnxt_qplib_cqe *cqe) { u16 vlan_id = 0; wc->opcode = IB_WC_RECV; wc->status = __rc_to_ib_wc_status(cqe->status); if (cqe->flags & CQ_RES_UD_FLAGS_IMM) wc->wc_flags |= IB_WC_WITH_IMM; if (cqe->flags & CQ_RES_RC_FLAGS_INV) wc->wc_flags |= IB_WC_WITH_INVALIDATE; /* report only on GSI QP for Thor */ if (rdev->gsi_ctx.gsi_qp->qplib_qp.id == qp->qplib_qp.id && rdev->gsi_ctx.gsi_qp_mode == BNXT_RE_GSI_MODE_UD) { wc->wc_flags |= IB_WC_GRH; memcpy(wc->smac, cqe->smac, ETH_ALEN); wc->wc_flags |= IB_WC_WITH_SMAC; if (_is_cqe_v2_supported(rdev->dev_attr->dev_cap_flags)) { if (cqe->flags & CQ_RES_UD_V2_FLAGS_META_FORMAT_MASK) { if (cqe->cfa_meta & BNXT_QPLIB_CQE_CFA_META1_VALID) vlan_id = (cqe->cfa_meta & 0xFFF); } } else if (cqe->flags & CQ_RES_UD_FLAGS_META_FORMAT_VLAN) { vlan_id = (cqe->cfa_meta & 0xFFF); } /* Mark only if vlan_id is non zero */ if (vlan_id && bnxt_re_check_if_vlan_valid(rdev, vlan_id)) { wc->vlan_id = vlan_id; wc->wc_flags |= IB_WC_WITH_VLAN; } } } static int bnxt_re_legacy_send_phantom_wqe(struct bnxt_re_qp *qp) { struct bnxt_qplib_qp *lib_qp = &qp->qplib_qp; unsigned long flags; int rc = 0; spin_lock_irqsave(&qp->sq_lock, flags); rc = bnxt_re_legacy_bind_fence_mw(lib_qp); if (!rc) { lib_qp->sq.phantom_wqe_cnt++; dev_dbg(&lib_qp->sq.hwq.pdev->dev, "qp %#x sq->prod %#x sw_prod %#x phantom_wqe_cnt %d\n", lib_qp->id, lib_qp->sq.hwq.prod, HWQ_CMP(lib_qp->sq.hwq.prod, &lib_qp->sq.hwq), lib_qp->sq.phantom_wqe_cnt); } spin_unlock_irqrestore(&qp->sq_lock, flags); return rc; } int bnxt_re_poll_cq(struct ib_cq *ib_cq, int num_entries, struct ib_wc *wc) { struct bnxt_re_cq *cq = to_bnxt_re(ib_cq, struct bnxt_re_cq, ibcq); struct bnxt_re_dev *rdev = cq->rdev; struct bnxt_re_qp *qp; struct bnxt_qplib_cqe *cqe; int i, ncqe, budget, init_budget; struct bnxt_qplib_q *sq; struct bnxt_qplib_qp *lib_qp; u32 tbl_idx; struct bnxt_re_sqp_entries *sqp_entry = NULL; unsigned long flags; u8 gsi_mode; /* * DB recovery CQ; only process the door bell pacing alert from * the user lib */ if (cq->is_dbr_soft_cq) { bnxt_re_pacing_alert(rdev); return 0; } /* User CQ; the only processing we do is to * complete any pending CQ resize operation. */ if (cq->umem) { if (cq->resize_umem) bnxt_re_resize_cq_complete(cq); return 0; } spin_lock_irqsave(&cq->cq_lock, flags); budget = min_t(u32, num_entries, cq->max_cql); init_budget = budget; if (!cq->cql) { dev_err(rdev_to_dev(rdev), "POLL CQ no CQL to use\n"); goto exit; } cqe = &cq->cql[0]; gsi_mode = rdev->gsi_ctx.gsi_qp_mode; while (budget) { lib_qp = NULL; ncqe = bnxt_qplib_poll_cq(&cq->qplib_cq, cqe, budget, &lib_qp); if (lib_qp) { sq = &lib_qp->sq; if (sq->legacy_send_phantom == true) { qp = container_of(lib_qp, struct bnxt_re_qp, qplib_qp); if (bnxt_re_legacy_send_phantom_wqe(qp) == -ENOMEM) dev_err(rdev_to_dev(rdev), "Phantom failed! Scheduled to send again\n"); else sq->legacy_send_phantom = false; } } if (ncqe < budget) ncqe += bnxt_qplib_process_flush_list(&cq->qplib_cq, cqe + ncqe, budget - ncqe); if (!ncqe) break; for (i = 0; i < ncqe; i++, cqe++) { /* Transcribe each qplib_wqe back to ib_wc */ memset(wc, 0, sizeof(*wc)); wc->wr_id = cqe->wr_id; wc->byte_len = cqe->length; qp = to_bnxt_re((struct bnxt_qplib_qp *)cqe->qp_handle, struct bnxt_re_qp, qplib_qp); if (!qp) { dev_err(rdev_to_dev(rdev), "POLL CQ bad QP handle\n"); continue; } wc->qp = &qp->ib_qp; wc->ex.imm_data = cqe->immdata; wc->src_qp = cqe->src_qp; memcpy(wc->smac, cqe->smac, ETH_ALEN); wc->port_num = 1; wc->vendor_err = cqe->status; switch(cqe->opcode) { case CQ_BASE_CQE_TYPE_REQ: if (gsi_mode == BNXT_RE_GSI_MODE_ALL && qp->qplib_qp.id == rdev->gsi_ctx.gsi_sqp->qplib_qp.id) { /* Handle this completion with * the stored completion */ dev_dbg(rdev_to_dev(rdev), "Skipping this UD Send CQ\n"); memset(wc, 0, sizeof(*wc)); continue; } bnxt_re_process_req_wc(wc, cqe); break; case CQ_BASE_CQE_TYPE_RES_RAWETH_QP1: if (gsi_mode == BNXT_RE_GSI_MODE_ALL) { if (!cqe->status) { int rc = 0; rc = bnxt_re_process_raw_qp_packet_receive(qp, cqe); if (!rc) { memset(wc, 0, sizeof(*wc)); continue; } cqe->status = -1; } /* Errors need not be looped back. * But change the wr_id to the one * stored in the table */ tbl_idx = cqe->wr_id; sqp_entry = &rdev->gsi_ctx.sqp_tbl[tbl_idx]; wc->wr_id = sqp_entry->wrid; } bnxt_re_process_res_rawqp1_wc(wc, cqe); break; case CQ_BASE_CQE_TYPE_RES_RC: bnxt_re_process_res_rc_wc(wc, cqe); break; case CQ_BASE_CQE_TYPE_RES_UD: if (gsi_mode == BNXT_RE_GSI_MODE_ALL && qp->qplib_qp.id == rdev->gsi_ctx.gsi_sqp->qplib_qp.id) { /* Handle this completion with * the stored completion */ dev_dbg(rdev_to_dev(rdev), "Handling the UD receive CQ\n"); if (cqe->status) { /* TODO handle this completion as a failure in * loopback porocedure */ continue; } else { bnxt_re_process_res_shadow_qp_wc(qp, wc, cqe); break; } } bnxt_re_process_res_ud_wc(rdev, qp, wc, cqe); break; default: dev_err(rdev_to_dev(cq->rdev), "POLL CQ type 0x%x not handled, skip!\n", cqe->opcode); continue; } wc++; budget--; } } exit: spin_unlock_irqrestore(&cq->cq_lock, flags); return init_budget - budget; } int bnxt_re_req_notify_cq(struct ib_cq *ib_cq, enum ib_cq_notify_flags ib_cqn_flags) { struct bnxt_re_cq *cq = to_bnxt_re(ib_cq, struct bnxt_re_cq, ibcq); int type = 0, rc = 0; unsigned long flags; spin_lock_irqsave(&cq->cq_lock, flags); /* Trigger on the very next completion */ if (ib_cqn_flags & IB_CQ_NEXT_COMP) type = DBC_DBC_TYPE_CQ_ARMALL; /* Trigger on the next solicited completion */ else if (ib_cqn_flags & IB_CQ_SOLICITED) type = DBC_DBC_TYPE_CQ_ARMSE; bnxt_qplib_req_notify_cq(&cq->qplib_cq, type); /* Poll to see if there are missed events */ if ((ib_cqn_flags & IB_CQ_REPORT_MISSED_EVENTS) && !(bnxt_qplib_is_cq_empty(&cq->qplib_cq))) rc = 1; spin_unlock_irqrestore(&cq->cq_lock, flags); return rc; } /* Memory Regions */ struct ib_mr *bnxt_re_get_dma_mr(struct ib_pd *ib_pd, int mr_access_flags) { struct bnxt_qplib_mrinfo mrinfo; struct bnxt_re_dev *rdev; struct bnxt_re_mr *mr; struct bnxt_re_pd *pd; u32 max_mr_count; u64 pbl = 0; int rc; memset(&mrinfo, 0, sizeof(mrinfo)); pd = to_bnxt_re(ib_pd, struct bnxt_re_pd, ibpd); rdev = pd->rdev; mr = kzalloc(sizeof(*mr), GFP_KERNEL); if (!mr) { dev_err(rdev_to_dev(rdev), "Allocate memory for DMA MR failed!\n"); return ERR_PTR(-ENOMEM); } mr->rdev = rdev; mr->qplib_mr.pd = &pd->qplib_pd; mr->qplib_mr.flags = __from_ib_access_flags(mr_access_flags); mr->qplib_mr.type = CMDQ_ALLOCATE_MRW_MRW_FLAGS_PMR; /* Allocate and register 0 as the address */ rc = bnxt_qplib_alloc_mrw(&rdev->qplib_res, &mr->qplib_mr); if (rc) { dev_err(rdev_to_dev(rdev), "Allocate DMA MR failed!\n"); goto fail; } mr->qplib_mr.total_size = -1; /* Infinite length */ mrinfo.ptes = &pbl; mrinfo.sg.npages = 0; mrinfo.sg.pgsize = PAGE_SIZE; mrinfo.sg.pgshft = PAGE_SHIFT; mrinfo.sg.pgsize = PAGE_SIZE; mrinfo.mrw = &mr->qplib_mr; mrinfo.is_dma = true; rc = bnxt_qplib_reg_mr(&rdev->qplib_res, &mrinfo, false); if (rc) { dev_err(rdev_to_dev(rdev), "Register DMA MR failed!\n"); goto fail_mr; } mr->ib_mr.lkey = mr->qplib_mr.lkey; if (mr_access_flags & (IB_ACCESS_REMOTE_WRITE | IB_ACCESS_REMOTE_READ | IB_ACCESS_REMOTE_ATOMIC)) mr->ib_mr.rkey = mr->ib_mr.lkey; atomic_inc(&rdev->stats.rsors.mr_count); max_mr_count = atomic_read(&rdev->stats.rsors.mr_count); if (max_mr_count > atomic_read(&rdev->stats.rsors.max_mr_count)) atomic_set(&rdev->stats.rsors.max_mr_count, max_mr_count); return &mr->ib_mr; fail_mr: bnxt_qplib_free_mrw(&rdev->qplib_res, &mr->qplib_mr); fail: kfree(mr); return ERR_PTR(rc); } int bnxt_re_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata) { struct bnxt_re_mr *mr = to_bnxt_re(ib_mr, struct bnxt_re_mr, ib_mr); struct bnxt_re_dev *rdev = mr->rdev; int rc = 0; rc = bnxt_qplib_free_mrw(&rdev->qplib_res, &mr->qplib_mr); if (rc) dev_err(rdev_to_dev(rdev), "Dereg MR failed (%d): rc - %#x\n", mr->qplib_mr.lkey, rc); if (mr->pages) { bnxt_qplib_free_fast_reg_page_list(&rdev->qplib_res, &mr->qplib_frpl); kfree(mr->pages); mr->npages = 0; mr->pages = NULL; } if (!IS_ERR(mr->ib_umem) && mr->ib_umem) { mr->is_invalcb_active = false; bnxt_re_peer_mem_release(mr->ib_umem); } kfree(mr); atomic_dec(&rdev->stats.rsors.mr_count); return 0; } static int bnxt_re_set_page(struct ib_mr *ib_mr, u64 addr) { struct bnxt_re_mr *mr = to_bnxt_re(ib_mr, struct bnxt_re_mr, ib_mr); if (unlikely(mr->npages == mr->qplib_frpl.max_pg_ptrs)) return -ENOMEM; mr->pages[mr->npages++] = addr; dev_dbg(NULL, "%s: ibdev %p Set MR pages[%d] = 0x%lx\n", ROCE_DRV_MODULE_NAME, ib_mr->device, mr->npages - 1, mr->pages[mr->npages - 1]); return 0; } int bnxt_re_map_mr_sg(struct ib_mr *ib_mr, struct scatterlist *sg, int sg_nents, unsigned int *sg_offset) { struct bnxt_re_mr *mr = to_bnxt_re(ib_mr, struct bnxt_re_mr, ib_mr); mr->npages = 0; return ib_sg_to_pages(ib_mr, sg, sg_nents, sg_offset, bnxt_re_set_page); } struct ib_mr *bnxt_re_alloc_mr(struct ib_pd *ib_pd, enum ib_mr_type type, u32 max_num_sg, struct ib_udata *udata) { struct bnxt_re_pd *pd = to_bnxt_re(ib_pd, struct bnxt_re_pd, ibpd); struct bnxt_re_dev *rdev = pd->rdev; struct bnxt_re_mr *mr; u32 max_mr_count; int rc; dev_dbg(rdev_to_dev(rdev), "Alloc MR\n"); if (type != IB_MR_TYPE_MEM_REG) { dev_dbg(rdev_to_dev(rdev), "MR type 0x%x not supported\n", type); return ERR_PTR(-EINVAL); } if (max_num_sg > MAX_PBL_LVL_1_PGS) { dev_dbg(rdev_to_dev(rdev), "Max SG exceeded\n"); return ERR_PTR(-EINVAL); } mr = kzalloc(sizeof(*mr), GFP_KERNEL); if (!mr) { dev_err(rdev_to_dev(rdev), "Allocate MR mem failed!\n"); return ERR_PTR(-ENOMEM); } mr->rdev = rdev; mr->qplib_mr.pd = &pd->qplib_pd; mr->qplib_mr.flags = BNXT_QPLIB_FR_PMR; mr->qplib_mr.type = CMDQ_ALLOCATE_MRW_MRW_FLAGS_PMR; rc = bnxt_qplib_alloc_mrw(&rdev->qplib_res, &mr->qplib_mr); if (rc) { dev_err(rdev_to_dev(rdev), "Allocate MR failed!\n"); goto fail; } mr->ib_mr.lkey = mr->qplib_mr.lkey; mr->ib_mr.rkey = mr->ib_mr.lkey; mr->pages = kzalloc(sizeof(u64) * max_num_sg, GFP_KERNEL); if (!mr->pages) { dev_err(rdev_to_dev(rdev), "Allocate MR page list mem failed!\n"); rc = -ENOMEM; goto fail_mr; } rc = bnxt_qplib_alloc_fast_reg_page_list(&rdev->qplib_res, &mr->qplib_frpl, max_num_sg); if (rc) { dev_err(rdev_to_dev(rdev), "Allocate HW Fast reg page list failed!\n"); goto free_page; } dev_dbg(rdev_to_dev(rdev), "Alloc MR pages = 0x%p\n", mr->pages); atomic_inc(&rdev->stats.rsors.mr_count); max_mr_count = atomic_read(&rdev->stats.rsors.mr_count); if (max_mr_count > atomic_read(&rdev->stats.rsors.max_mr_count)) atomic_set(&rdev->stats.rsors.max_mr_count, max_mr_count); return &mr->ib_mr; free_page: kfree(mr->pages); fail_mr: bnxt_qplib_free_mrw(&rdev->qplib_res, &mr->qplib_mr); fail: kfree(mr); return ERR_PTR(rc); } /* Memory Windows */ struct ib_mw *bnxt_re_alloc_mw(struct ib_pd *ib_pd, enum ib_mw_type type, struct ib_udata *udata) { struct bnxt_re_pd *pd = to_bnxt_re(ib_pd, struct bnxt_re_pd, ibpd); struct bnxt_re_dev *rdev = pd->rdev; struct bnxt_re_mw *mw; u32 max_mw_count; int rc; mw = kzalloc(sizeof(*mw), GFP_KERNEL); if (!mw) { dev_err(rdev_to_dev(rdev), "Allocate MW failed!\n"); rc = -ENOMEM; goto exit; } mw->rdev = rdev; mw->qplib_mw.pd = &pd->qplib_pd; mw->qplib_mw.type = (type == IB_MW_TYPE_1 ? CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE1 : CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE2B); rc = bnxt_qplib_alloc_mrw(&rdev->qplib_res, &mw->qplib_mw); if (rc) { dev_err(rdev_to_dev(rdev), "Allocate MW failed!\n"); goto fail; } mw->ib_mw.rkey = mw->qplib_mw.rkey; atomic_inc(&rdev->stats.rsors.mw_count); max_mw_count = atomic_read(&rdev->stats.rsors.mw_count); if (max_mw_count > atomic_read(&rdev->stats.rsors.max_mw_count)) atomic_set(&rdev->stats.rsors.max_mw_count, max_mw_count); return &mw->ib_mw; fail: kfree(mw); exit: return ERR_PTR(rc); } int bnxt_re_dealloc_mw(struct ib_mw *ib_mw) { struct bnxt_re_mw *mw = to_bnxt_re(ib_mw, struct bnxt_re_mw, ib_mw); struct bnxt_re_dev *rdev = mw->rdev; int rc; rc = bnxt_qplib_free_mrw(&rdev->qplib_res, &mw->qplib_mw); if (rc) { dev_err(rdev_to_dev(rdev), "Free MW failed: %#x\n", rc); return rc; } kfree(mw); atomic_dec(&rdev->stats.rsors.mw_count); return rc; } static int bnxt_re_page_size_ok(int page_shift) { switch (page_shift) { case CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_4K: case CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_8K: case CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_64K: case CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_2M: case CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_256K: case CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_1M: case CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_4M: case CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_256MB: case CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_1G: return 1; default: return 0; } } static int bnxt_re_get_page_shift(struct ib_umem *umem, u64 va, u64 st, u64 cmask) { int pgshft; pgshft = ilog2(umem->page_size); return pgshft; } static int bnxt_re_get_num_pages(struct ib_umem *umem, u64 start, u64 length, int page_shift) { int npages = 0; if (page_shift == PAGE_SHIFT) { npages = ib_umem_num_pages_compat(umem); } else { npages = ALIGN(length, BIT(page_shift)) / BIT(page_shift); if (start % BIT(page_shift)) npages++; } return npages; } /* uverbs */ struct ib_mr *bnxt_re_reg_user_mr(struct ib_pd *ib_pd, u64 start, u64 length, u64 virt_addr, int mr_access_flags, struct ib_udata *udata) { struct bnxt_re_pd *pd = to_bnxt_re(ib_pd, struct bnxt_re_pd, ibpd); struct bnxt_re_dev *rdev = pd->rdev; struct bnxt_qplib_mrinfo mrinfo; int umem_pgs, page_shift, rc; struct bnxt_re_mr *mr; struct ib_umem *umem; u32 max_mr_count; int npages; dev_dbg(rdev_to_dev(rdev), "Reg user MR\n"); if (bnxt_re_get_total_mr_mw_count(rdev) >= rdev->dev_attr->max_mr) return ERR_PTR(-ENOMEM); if (rdev->mod_exit) { dev_dbg(rdev_to_dev(rdev), "%s(): in mod_exit, just return!\n", __func__); return ERR_PTR(-EIO); } memset(&mrinfo, 0, sizeof(mrinfo)); if (length > BNXT_RE_MAX_MR_SIZE) { dev_err(rdev_to_dev(rdev), "Requested MR Size: %lu " "> Max supported: %ld\n", length, BNXT_RE_MAX_MR_SIZE); return ERR_PTR(-ENOMEM); } mr = kzalloc(sizeof(*mr), GFP_KERNEL); if (!mr) { dev_err(rdev_to_dev(rdev), "Allocate MR failed!\n"); return ERR_PTR (-ENOMEM); } mr->rdev = rdev; mr->qplib_mr.pd = &pd->qplib_pd; mr->qplib_mr.flags = __from_ib_access_flags(mr_access_flags); mr->qplib_mr.type = CMDQ_ALLOCATE_MRW_MRW_FLAGS_MR; if (!_is_alloc_mr_unified(rdev->qplib_res.dattr)) { rc = bnxt_qplib_alloc_mrw(&rdev->qplib_res, &mr->qplib_mr); if (rc) { dev_err(rdev_to_dev(rdev), "Alloc MR failed!\n"); goto fail; } /* The fixed portion of the rkey is the same as the lkey */ mr->ib_mr.rkey = mr->qplib_mr.rkey; } umem = ib_umem_get_flags_compat(rdev, ib_pd->uobject->context, udata, start, length, mr_access_flags, 0); if (IS_ERR(umem)) { rc = PTR_ERR(umem); dev_err(rdev_to_dev(rdev), "%s: ib_umem_get failed! rc = %d\n", __func__, rc); goto free_mr; } mr->ib_umem = umem; mr->qplib_mr.va = virt_addr; umem_pgs = ib_umem_num_pages_compat(umem); if (!umem_pgs) { dev_err(rdev_to_dev(rdev), "umem is invalid!\n"); rc = -EINVAL; goto free_umem; } mr->qplib_mr.total_size = length; page_shift = bnxt_re_get_page_shift(umem, virt_addr, start, rdev->dev_attr->page_size_cap); if (!bnxt_re_page_size_ok(page_shift)) { dev_err(rdev_to_dev(rdev), "umem page size unsupported!\n"); rc = -EFAULT; goto free_umem; } npages = bnxt_re_get_num_pages(umem, start, length, page_shift); /* Map umem buf ptrs to the PBL */ mrinfo.sg.npages = npages; mrinfo.sg.sghead = get_ib_umem_sgl(umem, &mrinfo.sg.nmap); mrinfo.sg.pgshft = page_shift; mrinfo.sg.pgsize = BIT(page_shift); mrinfo.mrw = &mr->qplib_mr; rc = bnxt_qplib_reg_mr(&rdev->qplib_res, &mrinfo, false); if (rc) { dev_err(rdev_to_dev(rdev), "Reg user MR failed!\n"); goto free_umem; } mr->ib_mr.lkey = mr->ib_mr.rkey = mr->qplib_mr.lkey; atomic_inc(&rdev->stats.rsors.mr_count); max_mr_count = atomic_read(&rdev->stats.rsors.mr_count); if (max_mr_count > atomic_read(&rdev->stats.rsors.max_mr_count)) atomic_set(&rdev->stats.rsors.max_mr_count, max_mr_count); return &mr->ib_mr; free_umem: bnxt_re_peer_mem_release(mr->ib_umem); free_mr: if (!_is_alloc_mr_unified(rdev->qplib_res.dattr)) bnxt_qplib_free_mrw(&rdev->qplib_res, &mr->qplib_mr); fail: kfree(mr); return ERR_PTR(rc); } int bnxt_re_rereg_user_mr(struct ib_mr *ib_mr, int flags, u64 start, u64 length, u64 virt_addr, int mr_access_flags, struct ib_pd *ib_pd, struct ib_udata *udata) { struct bnxt_re_mr *mr = to_bnxt_re(ib_mr, struct bnxt_re_mr, ib_mr); struct bnxt_re_pd *pd = to_bnxt_re(ib_pd, struct bnxt_re_pd, ibpd); int umem_pgs = 0, page_shift = PAGE_SHIFT, rc; struct bnxt_re_dev *rdev = mr->rdev; struct bnxt_qplib_mrinfo mrinfo; struct ib_umem *umem; u32 npages; /* TODO: Must decipher what to modify based on the flags */ memset(&mrinfo, 0, sizeof(mrinfo)); if (flags & IB_MR_REREG_TRANS) { umem = ib_umem_get_flags_compat(rdev, ib_pd->uobject->context, udata, start, length, mr_access_flags, 0); if (IS_ERR(umem)) { rc = PTR_ERR(umem); dev_err(rdev_to_dev(rdev), "%s: ib_umem_get failed! ret = %d\n", __func__, rc); goto fail; } mr->ib_umem = umem; mr->qplib_mr.va = virt_addr; umem_pgs = ib_umem_num_pages_compat(umem); if (!umem_pgs) { dev_err(rdev_to_dev(rdev), "umem is invalid!\n"); rc = -EINVAL; goto fail_free_umem; } mr->qplib_mr.total_size = length; page_shift = bnxt_re_get_page_shift(umem, virt_addr, start, rdev->dev_attr->page_size_cap); if (!bnxt_re_page_size_ok(page_shift)) { dev_err(rdev_to_dev(rdev), "umem page size unsupported!\n"); rc = -EFAULT; goto fail_free_umem; } npages = bnxt_re_get_num_pages(umem, start, length, page_shift); /* Map umem buf ptrs to the PBL */ mrinfo.sg.npages = npages; mrinfo.sg.sghead = get_ib_umem_sgl(umem, &mrinfo.sg.nmap); mrinfo.sg.pgshft = page_shift; mrinfo.sg.pgsize = BIT(page_shift); } mrinfo.mrw = &mr->qplib_mr; if (flags & IB_MR_REREG_PD) mr->qplib_mr.pd = &pd->qplib_pd; if (flags & IB_MR_REREG_ACCESS) mr->qplib_mr.flags = __from_ib_access_flags(mr_access_flags); rc = bnxt_qplib_reg_mr(&rdev->qplib_res, &mrinfo, false); if (rc) { dev_err(rdev_to_dev(rdev), "Rereg user MR failed!\n"); goto fail_free_umem; } mr->ib_mr.rkey = mr->qplib_mr.rkey; return 0; fail_free_umem: bnxt_re_peer_mem_release(mr->ib_umem); fail: return rc; } static int bnxt_re_check_abi_version(struct bnxt_re_dev *rdev) { struct ib_device *ibdev = &rdev->ibdev; u32 uverbs_abi_ver; uverbs_abi_ver = GET_UVERBS_ABI_VERSION(ibdev); dev_dbg(rdev_to_dev(rdev), "ABI version requested %d\n", uverbs_abi_ver); if (uverbs_abi_ver != BNXT_RE_ABI_VERSION) { dev_dbg(rdev_to_dev(rdev), " is different from the device %d \n", BNXT_RE_ABI_VERSION); return -EPERM; } return 0; } int bnxt_re_alloc_ucontext(struct ib_ucontext *uctx_in, struct ib_udata *udata) { struct ib_ucontext *ctx = uctx_in; struct ib_device *ibdev = ctx->device; struct bnxt_re_ucontext *uctx = container_of(ctx, struct bnxt_re_ucontext, ibucontext); struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev); struct bnxt_qplib_dev_attr *dev_attr = rdev->dev_attr; struct bnxt_re_uctx_resp resp = {}; struct bnxt_re_uctx_req ureq = {}; struct bnxt_qplib_chip_ctx *cctx; u32 chip_met_rev_num; bool genp5 = false; int rc; cctx = rdev->chip_ctx; rc = bnxt_re_check_abi_version(rdev); if (rc) goto fail; uctx->rdev = rdev; uctx->shpg = (void *)__get_free_page(GFP_KERNEL); if (!uctx->shpg) { dev_err(rdev_to_dev(rdev), "shared memory allocation failed!\n"); rc = -ENOMEM; goto fail; } spin_lock_init(&uctx->sh_lock); if (BNXT_RE_ABI_VERSION >= 4) { chip_met_rev_num = cctx->chip_num; chip_met_rev_num |= ((u32)cctx->chip_rev & 0xFF) << BNXT_RE_CHIP_ID0_CHIP_REV_SFT; chip_met_rev_num |= ((u32)cctx->chip_metal & 0xFF) << BNXT_RE_CHIP_ID0_CHIP_MET_SFT; resp.chip_id0 = chip_met_rev_num; resp.chip_id1 = 0; /* future extension of chip info */ } if (BNXT_RE_ABI_VERSION != 4) { /*Temp, Use idr_alloc instead*/ resp.dev_id = rdev->en_dev->pdev->devfn; resp.max_qp = rdev->qplib_res.hctx->qp_ctx.max; } genp5 = _is_chip_gen_p5_p7(cctx); - if (BNXT_RE_ABI_VERSION > 5) { - resp.modes = genp5 ? cctx->modes.wqe_mode : 0; - if (rdev->dev_attr && BNXT_RE_HW_RETX(rdev->dev_attr->dev_cap_flags)) - resp.comp_mask = BNXT_RE_COMP_MASK_UCNTX_HW_RETX_ENABLED; - } + resp.modes = genp5 ? cctx->modes.wqe_mode : 0; + if (rdev->dev_attr && _is_host_msn_table(rdev->dev_attr->dev_cap_ext_flags2)) + resp.comp_mask = BNXT_RE_COMP_MASK_UCNTX_HW_RETX_ENABLED; resp.pg_size = PAGE_SIZE; resp.cqe_sz = sizeof(struct cq_base); resp.max_cqd = dev_attr->max_cq_wqes; if (genp5 && cctx->modes.db_push) { resp.comp_mask |= BNXT_RE_COMP_MASK_UCNTX_WC_DPI_ENABLED; if (_is_chip_p7(cctx) && !(dev_attr->dev_cap_flags & CREQ_QUERY_FUNC_RESP_SB_PINGPONG_PUSH_MODE)) resp.comp_mask &= ~BNXT_RE_COMP_MASK_UCNTX_WC_DPI_ENABLED; } resp.comp_mask |= BNXT_RE_COMP_MASK_UCNTX_MQP_EX_SUPPORTED; if (rdev->dbr_pacing) resp.comp_mask |= BNXT_RE_COMP_MASK_UCNTX_DBR_PACING_ENABLED; if (rdev->dbr_drop_recov && rdev->user_dbr_drop_recov) resp.comp_mask |= BNXT_RE_COMP_MASK_UCNTX_DBR_RECOVERY_ENABLED; if (udata->inlen >= sizeof(ureq)) { rc = ib_copy_from_udata(&ureq, udata, min(udata->inlen, sizeof(ureq))); if (rc) goto cfail; if (bnxt_re_init_pow2_flag(&ureq, &resp)) dev_warn(rdev_to_dev(rdev), "Enabled roundup logic. Library bug?\n"); if (bnxt_re_init_rsvd_wqe_flag(&ureq, &resp, genp5)) dev_warn(rdev_to_dev(rdev), "Rsvd wqe in use! Try the updated library.\n"); } else { dev_warn(rdev_to_dev(rdev), "Enabled roundup logic. Update the library!\n"); resp.comp_mask &= ~BNXT_RE_COMP_MASK_UCNTX_POW2_DISABLED; dev_warn(rdev_to_dev(rdev), "Rsvd wqe in use. Update the library!\n"); resp.comp_mask &= ~BNXT_RE_COMP_MASK_UCNTX_RSVD_WQE_DISABLED; } uctx->cmask = (uint64_t)resp.comp_mask; rc = bnxt_re_copy_to_udata(rdev, &resp, min(udata->outlen, sizeof(resp)), udata); if (rc) goto cfail; INIT_LIST_HEAD(&uctx->cq_list); mutex_init(&uctx->cq_lock); return 0; cfail: free_page((u64)uctx->shpg); uctx->shpg = NULL; fail: return rc; } void bnxt_re_dealloc_ucontext(struct ib_ucontext *ib_uctx) { struct bnxt_re_ucontext *uctx = to_bnxt_re(ib_uctx, struct bnxt_re_ucontext, ibucontext); struct bnxt_re_dev *rdev = uctx->rdev; int rc = 0; if (uctx->shpg) free_page((u64)uctx->shpg); if (uctx->dpi.dbr) { /* Free DPI only if this is the first PD allocated by the * application and mark the context dpi as NULL */ if (_is_chip_gen_p5_p7(rdev->chip_ctx) && uctx->wcdpi.dbr) { rc = bnxt_qplib_dealloc_dpi(&rdev->qplib_res, &uctx->wcdpi); if (rc) dev_err(rdev_to_dev(rdev), "dealloc push dp failed\n"); uctx->wcdpi.dbr = NULL; } rc = bnxt_qplib_dealloc_dpi(&rdev->qplib_res, &uctx->dpi); if (rc) dev_err(rdev_to_dev(rdev), "Deallocte HW DPI failed!\n"); /* Don't fail, continue*/ uctx->dpi.dbr = NULL; } return; } static struct bnxt_re_cq *is_bnxt_re_cq_page(struct bnxt_re_ucontext *uctx, u64 pg_off) { struct bnxt_re_cq *cq = NULL, *tmp_cq; if (!_is_chip_p7(uctx->rdev->chip_ctx)) return NULL; mutex_lock(&uctx->cq_lock); list_for_each_entry(tmp_cq, &uctx->cq_list, cq_list) { if (((u64)tmp_cq->uctx_cq_page >> PAGE_SHIFT) == pg_off) { cq = tmp_cq; break; } } mutex_unlock(&uctx->cq_lock); return cq; } /* Helper function to mmap the virtual memory from user app */ int bnxt_re_mmap(struct ib_ucontext *ib_uctx, struct vm_area_struct *vma) { struct bnxt_re_ucontext *uctx = to_bnxt_re(ib_uctx, struct bnxt_re_ucontext, ibucontext); struct bnxt_re_dev *rdev = uctx->rdev; struct bnxt_re_cq *cq = NULL; int rc = 0; u64 pfn; switch (vma->vm_pgoff) { case BNXT_RE_MAP_SH_PAGE: pfn = vtophys(uctx->shpg) >> PAGE_SHIFT; return rdma_user_mmap_io(&uctx->ibucontext, vma, pfn, PAGE_SIZE, vma->vm_page_prot, NULL); dev_dbg(rdev_to_dev(rdev), "%s:%d uctx->shpg 0x%lx, vtophys(uctx->shpg) 0x%lx, pfn = 0x%lx \n", __func__, __LINE__, (u64) uctx->shpg, vtophys(uctx->shpg), pfn); if (rc) { dev_err(rdev_to_dev(rdev), "Shared page mapping failed!\n"); rc = -EAGAIN; } return rc; case BNXT_RE_MAP_WC: vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot); pfn = (uctx->wcdpi.umdbr >> PAGE_SHIFT); if (!pfn) return -EFAULT; break; case BNXT_RE_DBR_PAGE: /* Driver doesn't expect write access request */ if (vma->vm_flags & VM_WRITE) return -EFAULT; pfn = vtophys(rdev->dbr_page) >> PAGE_SHIFT; if (!pfn) return -EFAULT; break; case BNXT_RE_MAP_DB_RECOVERY_PAGE: pfn = vtophys(uctx->dbr_recov_cq_page) >> PAGE_SHIFT; if (!pfn) return -EFAULT; break; default: cq = is_bnxt_re_cq_page(uctx, vma->vm_pgoff); if (cq) { pfn = vtophys((void *)cq->uctx_cq_page) >> PAGE_SHIFT; rc = rdma_user_mmap_io(&uctx->ibucontext, vma, pfn, PAGE_SIZE, vma->vm_page_prot, NULL); if (rc) { dev_err(rdev_to_dev(rdev), "CQ page mapping failed!\n"); rc = -EAGAIN; } goto out; } else { vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); pfn = vma->vm_pgoff; } break; } rc = rdma_user_mmap_io(&uctx->ibucontext, vma, pfn, PAGE_SIZE, vma->vm_page_prot, NULL); if (rc) { dev_err(rdev_to_dev(rdev), "DPI mapping failed!\n"); return -EAGAIN; } rc = __bnxt_re_set_vma_data(uctx, vma); out: return rc; } int bnxt_re_process_mad(struct ib_device *ibdev, int mad_flags, u8 port_num, const struct ib_wc *wc, const struct ib_grh *grh, const struct ib_mad_hdr *in_mad, size_t in_mad_size, struct ib_mad_hdr *out_mad, size_t *out_mad_size, u16 *out_mad_pkey_index) { return IB_MAD_RESULT_SUCCESS; } void bnxt_re_disassociate_ucntx(struct ib_ucontext *ib_uctx) { } diff --git a/sys/dev/bnxt/bnxt_re/ib_verbs.h b/sys/dev/bnxt/bnxt_re/ib_verbs.h index cb9f7974e92d..e8c840f8946a 100644 --- a/sys/dev/bnxt/bnxt_re/ib_verbs.h +++ b/sys/dev/bnxt/bnxt_re/ib_verbs.h @@ -1,632 +1,640 @@ /* * Copyright (c) 2015-2024, Broadcom. All rights reserved. The term * Broadcom refers to Broadcom Limited and/or its subsidiaries. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in * the documentation and/or other materials provided with the * distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * * Description: IB Verbs interpreter (header) */ #ifndef __BNXT_RE_IB_VERBS_H__ #define __BNXT_RE_IB_VERBS_H__ #include #include "bnxt_re-abi.h" #include "qplib_res.h" #include "qplib_fp.h" struct bnxt_re_dev; #define BNXT_RE_ROCE_V2_UDP_SPORT 0x8CD1 #define BNXT_RE_QP_RANDOM_QKEY 0x81818181 #ifndef IB_MTU_8192 #define IB_MTU_8192 8192 #endif #ifndef SPEED_1000 #define SPEED_1000 1000 #endif #ifndef SPEED_10000 #define SPEED_10000 10000 #endif #ifndef SPEED_20000 #define SPEED_20000 20000 #endif #ifndef SPEED_25000 #define SPEED_25000 25000 #endif #ifndef SPEED_40000 #define SPEED_40000 40000 #endif #ifndef SPEED_50000 #define SPEED_50000 50000 #endif #ifndef SPEED_100000 #define SPEED_100000 100000 #endif #ifndef SPEED_200000 #define SPEED_200000 200000 #endif +#ifndef SPEED_400000 +#define SPEED_400000 400000 +#endif + #ifndef IB_SPEED_HDR #define IB_SPEED_HDR 64 #endif +#ifndef IB_SPEED_NDR +#define IB_SPEED_NDR 128 +#endif + #define RDMA_NETWORK_IPV4 1 #define RDMA_NETWORK_IPV6 2 #define ROCE_DMAC(x) (x)->dmac #define dma_rmb() rmb() #define compat_ib_alloc_device(size) ib_alloc_device(size); #define rdev_from_cq_in(cq_in) to_bnxt_re_dev(cq_in->device, ibdev) #define GET_UVERBS_ABI_VERSION(ibdev) (ibdev->uverbs_abi_ver) #define CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_PG_256MB 0x1cUL #define IB_POLL_UNBOUND_WORKQUEUE IB_POLL_WORKQUEUE #define BNXT_RE_LEGACY_FENCE_BYTES 64 #define BNXT_RE_LEGACY_FENCE_PBL_SIZE DIV_ROUND_UP(BNXT_RE_LEGACY_FENCE_BYTES, PAGE_SIZE) static inline struct bnxt_re_cq *__get_cq_from_cq_in(struct ib_cq *cq_in, struct bnxt_re_dev *rdev); static inline struct bnxt_re_qp *__get_qp_from_qp_in(struct ib_pd *qp_in, struct bnxt_re_dev *rdev); static inline bool bnxt_re_check_if_vlan_valid(struct bnxt_re_dev *rdev, u16 vlan_id); #define bnxt_re_compat_qfwstr(void) \ bnxt_re_query_fw_str(struct ib_device *ibdev, \ char *str, size_t str_len) static inline struct scatterlist *get_ib_umem_sgl(struct ib_umem *umem, u32 *nmap); struct bnxt_re_gid_ctx { u32 idx; u32 refcnt; }; struct bnxt_re_legacy_fence_data { u32 size; void *va; dma_addr_t dma_addr; struct bnxt_re_mr *mr; struct ib_mw *mw; struct bnxt_qplib_swqe bind_wqe; u32 bind_rkey; }; struct bnxt_re_pd { struct ib_pd ibpd; struct bnxt_re_dev *rdev; struct bnxt_qplib_pd qplib_pd; struct bnxt_re_legacy_fence_data fence; }; struct bnxt_re_ah { struct ib_ah ibah; struct bnxt_re_dev *rdev; struct bnxt_qplib_ah qplib_ah; }; struct bnxt_re_srq { struct ib_srq ibsrq; struct bnxt_re_dev *rdev; u32 srq_limit; struct bnxt_qplib_srq qplib_srq; struct ib_umem *umem; spinlock_t lock; }; union ip_addr { u32 ipv4_addr; u8 ipv6_addr[16]; }; struct bnxt_re_qp_info_entry { union ib_gid sgid; union ib_gid dgid; union ip_addr s_ip; union ip_addr d_ip; u16 s_port; #define BNXT_RE_QP_DEST_PORT 4791 u16 d_port; }; struct bnxt_re_qp { struct ib_qp ib_qp; struct list_head list; struct bnxt_re_dev *rdev; spinlock_t sq_lock; spinlock_t rq_lock; struct bnxt_qplib_qp qplib_qp; struct ib_umem *sumem; struct ib_umem *rumem; /* QP1 */ u32 send_psn; struct ib_ud_header qp1_hdr; struct bnxt_re_cq *scq; struct bnxt_re_cq *rcq; struct dentry *qp_info_pdev_dentry; struct bnxt_re_qp_info_entry qp_info_entry; void *qp_data; }; struct bnxt_re_cq { struct ib_cq ibcq; struct list_head cq_list; struct bnxt_re_dev *rdev; struct bnxt_re_ucontext *uctx; spinlock_t cq_lock; u16 cq_count; u16 cq_period; struct bnxt_qplib_cq qplib_cq; struct bnxt_qplib_cqe *cql; #define MAX_CQL_PER_POLL 1024 u32 max_cql; struct ib_umem *umem; struct ib_umem *resize_umem; struct ib_ucontext *context; int resize_cqe; /* list of cq per uctx. Used only for Thor-2 */ void *uctx_cq_page; void *dbr_recov_cq_page; bool is_dbr_soft_cq; }; struct bnxt_re_mr { struct bnxt_re_dev *rdev; struct ib_mr ib_mr; struct ib_umem *ib_umem; struct bnxt_qplib_mrw qplib_mr; u32 npages; u64 *pages; struct bnxt_qplib_frpl qplib_frpl; bool is_invalcb_active; }; struct bnxt_re_frpl { struct bnxt_re_dev *rdev; struct bnxt_qplib_frpl qplib_frpl; u64 *page_list; }; struct bnxt_re_mw { struct bnxt_re_dev *rdev; struct ib_mw ib_mw; struct bnxt_qplib_mrw qplib_mw; }; struct bnxt_re_ucontext { struct ib_ucontext ibucontext; struct bnxt_re_dev *rdev; struct list_head cq_list; struct bnxt_qplib_dpi dpi; struct bnxt_qplib_dpi wcdpi; void *shpg; spinlock_t sh_lock; uint64_t cmask; struct mutex cq_lock; /* Protect cq list */ void *dbr_recov_cq_page; struct bnxt_re_cq *dbr_recov_cq; }; struct bnxt_re_ah_info { union ib_gid sgid; struct ib_gid_attr sgid_attr; u16 vlan_tag; u8 nw_type; }; struct ifnet *bnxt_re_get_netdev(struct ib_device *ibdev, u8 port_num); int bnxt_re_query_device(struct ib_device *ibdev, struct ib_device_attr *ib_attr, struct ib_udata *udata); int bnxt_re_modify_device(struct ib_device *ibdev, int device_modify_mask, struct ib_device_modify *device_modify); int bnxt_re_query_port(struct ib_device *ibdev, u8 port_num, struct ib_port_attr *port_attr); int bnxt_re_modify_port(struct ib_device *ibdev, u8 port_num, int port_modify_mask, struct ib_port_modify *port_modify); int bnxt_re_get_port_immutable(struct ib_device *ibdev, u8 port_num, struct ib_port_immutable *immutable); void bnxt_re_compat_qfwstr(void); int bnxt_re_query_pkey(struct ib_device *ibdev, u8 port_num, u16 index, u16 *pkey); int bnxt_re_del_gid(struct ib_device *ibdev, u8 port_num, unsigned int index, void **context); int bnxt_re_add_gid(struct ib_device *ibdev, u8 port_num, unsigned int index, const union ib_gid *gid, const struct ib_gid_attr *attr, void **context); int bnxt_re_query_gid(struct ib_device *ibdev, u8 port_num, int index, union ib_gid *gid); enum rdma_link_layer bnxt_re_get_link_layer(struct ib_device *ibdev, u8 port_num); int bnxt_re_alloc_pd(struct ib_pd *pd_in, struct ib_udata *udata); void bnxt_re_dealloc_pd(struct ib_pd *ib_pd, struct ib_udata *udata); int bnxt_re_create_ah(struct ib_ah *ah_in, struct ib_ah_attr *attr, u32 flags, struct ib_udata *udata); int bnxt_re_modify_ah(struct ib_ah *ah, struct ib_ah_attr *ah_attr); int bnxt_re_query_ah(struct ib_ah *ah, struct ib_ah_attr *ah_attr); void bnxt_re_destroy_ah(struct ib_ah *ib_ah, u32 flags); int bnxt_re_create_srq(struct ib_srq *srq_in, struct ib_srq_init_attr *srq_init_attr, struct ib_udata *udata); int bnxt_re_modify_srq(struct ib_srq *srq, struct ib_srq_attr *srq_attr, enum ib_srq_attr_mask srq_attr_mask, struct ib_udata *udata); int bnxt_re_query_srq(struct ib_srq *srq, struct ib_srq_attr *srq_attr); void bnxt_re_destroy_srq(struct ib_srq *ib_srq, struct ib_udata *udata); int bnxt_re_post_srq_recv(struct ib_srq *ib_srq, const struct ib_recv_wr *wr, const struct ib_recv_wr **bad_wr); struct ib_qp *bnxt_re_create_qp(struct ib_pd *qp_in, struct ib_qp_init_attr *qp_init_attr, struct ib_udata *udata); int bnxt_re_modify_qp(struct ib_qp *qp, struct ib_qp_attr *qp_attr, int qp_attr_mask, struct ib_udata *udata); int bnxt_re_query_qp(struct ib_qp *qp, struct ib_qp_attr *qp_attr, int qp_attr_mask, struct ib_qp_init_attr *qp_init_attr); int bnxt_re_destroy_qp(struct ib_qp *ib_qp, struct ib_udata *udata); int bnxt_re_post_send(struct ib_qp *ib_qp, const struct ib_send_wr *wr, const struct ib_send_wr **bad_wr); int bnxt_re_post_recv(struct ib_qp *ib_qp, const struct ib_recv_wr *wr, const struct ib_recv_wr **bad_wr); int bnxt_re_create_cq(struct ib_cq *cq_in, const struct ib_cq_init_attr *attr, struct ib_udata *udata); void bnxt_re_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata); int bnxt_re_modify_cq(struct ib_cq *cq, u16 cq_count, u16 cq_period); int bnxt_re_resize_cq(struct ib_cq *cq, int cqe, struct ib_udata *udata); int bnxt_re_poll_cq(struct ib_cq *cq, int num_entries, struct ib_wc *wc); int bnxt_re_req_notify_cq(struct ib_cq *cq, enum ib_cq_notify_flags flags); struct ib_mr *bnxt_re_get_dma_mr(struct ib_pd *pd, int mr_access_flags); int bnxt_re_map_mr_sg(struct ib_mr *ib_mr, struct scatterlist *sg, int sg_nents, unsigned int *sg_offset); struct ib_mr *bnxt_re_alloc_mr(struct ib_pd *ib_pd, enum ib_mr_type type, u32 max_num_sg, struct ib_udata *udata); int bnxt_re_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata); struct ib_mw *bnxt_re_alloc_mw(struct ib_pd *ib_pd, enum ib_mw_type type, struct ib_udata *udata); int bnxt_re_dealloc_mw(struct ib_mw *mw); struct ib_mr *bnxt_re_reg_user_mr(struct ib_pd *pd, u64 start, u64 length, u64 virt_addr, int mr_access_flags, struct ib_udata *udata); int bnxt_re_rereg_user_mr(struct ib_mr *mr, int flags, u64 start, u64 length, u64 virt_addr, int mr_access_flags, struct ib_pd *pd, struct ib_udata *udata); int bnxt_re_alloc_ucontext(struct ib_ucontext *uctx_in, struct ib_udata *udata); void bnxt_re_dealloc_ucontext(struct ib_ucontext *ib_uctx); int bnxt_re_mmap(struct ib_ucontext *context, struct vm_area_struct *vma); int bnxt_re_process_mad(struct ib_device *ibdev, int mad_flags, u8 port_num, const struct ib_wc *wc, const struct ib_grh *grh, const struct ib_mad_hdr *in_mad, size_t in_mad_size, struct ib_mad_hdr *out_mad, size_t *out_mad_size, u16 *out_mad_pkey_index); unsigned long bnxt_re_lock_cqs(struct bnxt_re_qp *qp); void bnxt_re_unlock_cqs(struct bnxt_re_qp *qp, unsigned long flags); void bnxt_re_disassociate_ucntx(struct ib_ucontext *ibcontext); static inline int __bnxt_re_set_vma_data(void *bnxt_re_uctx, struct vm_area_struct *vma); void bnxt_re_update_shadow_ah(struct bnxt_re_dev *rdev); void bnxt_re_handle_cqn(struct bnxt_qplib_cq *cq); static inline int bnxt_re_get_cached_gid(struct ib_device *dev, u8 port_num, int index, union ib_gid *sgid, struct ib_gid_attr **sgid_attr, struct ib_global_route *grh, struct ib_ah *ah); static inline enum rdma_network_type bnxt_re_gid_to_network_type(struct ib_gid_attr *sgid_attr, union ib_gid *sgid); static inline struct ib_umem *ib_umem_get_compat(struct bnxt_re_dev *rdev, struct ib_ucontext *ucontext, struct ib_udata *udata, unsigned long addr, size_t size, int access, int dmasync); static inline struct ib_umem *ib_umem_get_flags_compat(struct bnxt_re_dev *rdev, struct ib_ucontext *ucontext, struct ib_udata *udata, unsigned long addr, size_t size, int access, int dmasync); static inline size_t ib_umem_num_pages_compat(struct ib_umem *umem); static inline void bnxt_re_peer_mem_release(struct ib_umem *umem); void bnxt_re_resolve_dmac_task(struct work_struct *work); static inline enum ib_qp_type __from_hw_to_ib_qp_type(u8 type) { switch (type) { case CMDQ_CREATE_QP1_TYPE_GSI: case CMDQ_CREATE_QP_TYPE_GSI: return IB_QPT_GSI; case CMDQ_CREATE_QP_TYPE_RC: return IB_QPT_RC; case CMDQ_CREATE_QP_TYPE_UD: return IB_QPT_UD; case CMDQ_CREATE_QP_TYPE_RAW_ETHERTYPE: return IB_QPT_RAW_ETHERTYPE; default: return IB_QPT_MAX; } } static inline u8 __from_ib_qp_state(enum ib_qp_state state) { switch (state) { case IB_QPS_RESET: return CMDQ_MODIFY_QP_NEW_STATE_RESET; case IB_QPS_INIT: return CMDQ_MODIFY_QP_NEW_STATE_INIT; case IB_QPS_RTR: return CMDQ_MODIFY_QP_NEW_STATE_RTR; case IB_QPS_RTS: return CMDQ_MODIFY_QP_NEW_STATE_RTS; case IB_QPS_SQD: return CMDQ_MODIFY_QP_NEW_STATE_SQD; case IB_QPS_SQE: return CMDQ_MODIFY_QP_NEW_STATE_SQE; case IB_QPS_ERR: default: return CMDQ_MODIFY_QP_NEW_STATE_ERR; } } static inline u32 __from_ib_mtu(enum ib_mtu mtu) { switch (mtu) { case IB_MTU_256: return CMDQ_MODIFY_QP_PATH_MTU_MTU_256; case IB_MTU_512: return CMDQ_MODIFY_QP_PATH_MTU_MTU_512; case IB_MTU_1024: return CMDQ_MODIFY_QP_PATH_MTU_MTU_1024; case IB_MTU_2048: return CMDQ_MODIFY_QP_PATH_MTU_MTU_2048; case IB_MTU_4096: return CMDQ_MODIFY_QP_PATH_MTU_MTU_4096; default: return CMDQ_MODIFY_QP_PATH_MTU_MTU_2048; } } static inline enum ib_mtu __to_ib_mtu(u32 mtu) { switch (mtu & CREQ_QUERY_QP_RESP_SB_PATH_MTU_MASK) { case CMDQ_MODIFY_QP_PATH_MTU_MTU_256: return IB_MTU_256; case CMDQ_MODIFY_QP_PATH_MTU_MTU_512: return IB_MTU_512; case CMDQ_MODIFY_QP_PATH_MTU_MTU_1024: return IB_MTU_1024; case CMDQ_MODIFY_QP_PATH_MTU_MTU_2048: return IB_MTU_2048; case CMDQ_MODIFY_QP_PATH_MTU_MTU_4096: return IB_MTU_4096; case CMDQ_MODIFY_QP_PATH_MTU_MTU_8192: return IB_MTU_8192; default: return IB_MTU_2048; } } static inline enum ib_qp_state __to_ib_qp_state(u8 state) { switch (state) { case CMDQ_MODIFY_QP_NEW_STATE_RESET: return IB_QPS_RESET; case CMDQ_MODIFY_QP_NEW_STATE_INIT: return IB_QPS_INIT; case CMDQ_MODIFY_QP_NEW_STATE_RTR: return IB_QPS_RTR; case CMDQ_MODIFY_QP_NEW_STATE_RTS: return IB_QPS_RTS; case CMDQ_MODIFY_QP_NEW_STATE_SQD: return IB_QPS_SQD; case CMDQ_MODIFY_QP_NEW_STATE_SQE: return IB_QPS_SQE; case CMDQ_MODIFY_QP_NEW_STATE_ERR: default: return IB_QPS_ERR; } } static inline int bnxt_re_init_pow2_flag(struct bnxt_re_uctx_req *req, struct bnxt_re_uctx_resp *resp) { resp->comp_mask |= BNXT_RE_COMP_MASK_UCNTX_POW2_DISABLED; if (!(req->comp_mask & BNXT_RE_COMP_MASK_REQ_UCNTX_POW2_SUPPORT)) { resp->comp_mask &= ~BNXT_RE_COMP_MASK_UCNTX_POW2_DISABLED; return -EINVAL; } return 0; } static inline u32 bnxt_re_init_depth(u32 ent, struct bnxt_re_ucontext *uctx) { return uctx ? (uctx->cmask & BNXT_RE_COMP_MASK_UCNTX_POW2_DISABLED) ? ent : roundup_pow_of_two(ent) : ent; } static inline int bnxt_re_init_rsvd_wqe_flag(struct bnxt_re_uctx_req *req, struct bnxt_re_uctx_resp *resp, bool genp5) { resp->comp_mask |= BNXT_RE_COMP_MASK_UCNTX_RSVD_WQE_DISABLED; if (!(req->comp_mask & BNXT_RE_COMP_MASK_REQ_UCNTX_RSVD_WQE)) { resp->comp_mask &= ~BNXT_RE_COMP_MASK_UCNTX_RSVD_WQE_DISABLED; return -EINVAL; } else if (!genp5) { resp->comp_mask &= ~BNXT_RE_COMP_MASK_UCNTX_RSVD_WQE_DISABLED; } return 0; } static inline u32 bnxt_re_get_diff(struct bnxt_re_ucontext *uctx, struct bnxt_qplib_chip_ctx *cctx) { if (!uctx) { /* return res-wqe only for gen p4 for user resource */ return _is_chip_gen_p5_p7(cctx) ? 0 : BNXT_QPLIB_RESERVED_QP_WRS; } else if (uctx->cmask & BNXT_RE_COMP_MASK_UCNTX_RSVD_WQE_DISABLED) { return 0; } /* old lib */ return BNXT_QPLIB_RESERVED_QP_WRS; } static inline void bnxt_re_init_qpmtu(struct bnxt_re_qp *qp, int mtu, int mask, struct ib_qp_attr *qp_attr, bool *is_qpmtu_high) { int qpmtu, qpmtu_int; int ifmtu, ifmtu_int; ifmtu = iboe_get_mtu(mtu); ifmtu_int = ib_mtu_enum_to_int(ifmtu); qpmtu = ifmtu; qpmtu_int = ifmtu_int; if (mask & IB_QP_PATH_MTU) { qpmtu = qp_attr->path_mtu; qpmtu_int = ib_mtu_enum_to_int(qpmtu); if (qpmtu_int > ifmtu_int) { /* Trim the QP path mtu to interface mtu and update * the new mtu to user qp for retransmission psn * calculations. */ qpmtu = ifmtu; qpmtu_int = ifmtu_int; *is_qpmtu_high = true; } } qp->qplib_qp.path_mtu = __from_ib_mtu(qpmtu); qp->qplib_qp.mtu = qpmtu_int; qp->qplib_qp.modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU; } inline unsigned long compare_ether_header(void *a, void *b) { u32 *a32 = (u32 *)((u8 *)a + 2); u32 *b32 = (u32 *)((u8 *)b + 2); return (*(u16 *)a ^ *(u16 *)b) | (a32[0] ^ b32[0]) | (a32[1] ^ b32[1]) | (a32[2] ^ b32[2]); } struct vlan_hdr { __be16 h_vlan_TCI; __be16 h_vlan_encapsulated_proto; }; inline uint16_t crc16(uint16_t crc, const void *buffer, unsigned int len) { const unsigned char *cp = buffer; /* CRC table for the CRC-16. The poly is 0x8005 (x16 + x15 + x2 + 1). */ static uint16_t const crc16_table[256] = { 0x0000, 0xC0C1, 0xC181, 0x0140, 0xC301, 0x03C0, 0x0280, 0xC241, 0xC601, 0x06C0, 0x0780, 0xC741, 0x0500, 0xC5C1, 0xC481, 0x0440, 0xCC01, 0x0CC0, 0x0D80, 0xCD41, 0x0F00, 0xCFC1, 0xCE81, 0x0E40, 0x0A00, 0xCAC1, 0xCB81, 0x0B40, 0xC901, 0x09C0, 0x0880, 0xC841, 0xD801, 0x18C0, 0x1980, 0xD941, 0x1B00, 0xDBC1, 0xDA81, 0x1A40, 0x1E00, 0xDEC1, 0xDF81, 0x1F40, 0xDD01, 0x1DC0, 0x1C80, 0xDC41, 0x1400, 0xD4C1, 0xD581, 0x1540, 0xD701, 0x17C0, 0x1680, 0xD641, 0xD201, 0x12C0, 0x1380, 0xD341, 0x1100, 0xD1C1, 0xD081, 0x1040, 0xF001, 0x30C0, 0x3180, 0xF141, 0x3300, 0xF3C1, 0xF281, 0x3240, 0x3600, 0xF6C1, 0xF781, 0x3740, 0xF501, 0x35C0, 0x3480, 0xF441, 0x3C00, 0xFCC1, 0xFD81, 0x3D40, 0xFF01, 0x3FC0, 0x3E80, 0xFE41, 0xFA01, 0x3AC0, 0x3B80, 0xFB41, 0x3900, 0xF9C1, 0xF881, 0x3840, 0x2800, 0xE8C1, 0xE981, 0x2940, 0xEB01, 0x2BC0, 0x2A80, 0xEA41, 0xEE01, 0x2EC0, 0x2F80, 0xEF41, 0x2D00, 0xEDC1, 0xEC81, 0x2C40, 0xE401, 0x24C0, 0x2580, 0xE541, 0x2700, 0xE7C1, 0xE681, 0x2640, 0x2200, 0xE2C1, 0xE381, 0x2340, 0xE101, 0x21C0, 0x2080, 0xE041, 0xA001, 0x60C0, 0x6180, 0xA141, 0x6300, 0xA3C1, 0xA281, 0x6240, 0x6600, 0xA6C1, 0xA781, 0x6740, 0xA501, 0x65C0, 0x6480, 0xA441, 0x6C00, 0xACC1, 0xAD81, 0x6D40, 0xAF01, 0x6FC0, 0x6E80, 0xAE41, 0xAA01, 0x6AC0, 0x6B80, 0xAB41, 0x6900, 0xA9C1, 0xA881, 0x6840, 0x7800, 0xB8C1, 0xB981, 0x7940, 0xBB01, 0x7BC0, 0x7A80, 0xBA41, 0xBE01, 0x7EC0, 0x7F80, 0xBF41, 0x7D00, 0xBDC1, 0xBC81, 0x7C40, 0xB401, 0x74C0, 0x7580, 0xB541, 0x7700, 0xB7C1, 0xB681, 0x7640, 0x7200, 0xB2C1, 0xB381, 0x7340, 0xB101, 0x71C0, 0x7080, 0xB041, 0x5000, 0x90C1, 0x9181, 0x5140, 0x9301, 0x53C0, 0x5280, 0x9241, 0x9601, 0x56C0, 0x5780, 0x9741, 0x5500, 0x95C1, 0x9481, 0x5440, 0x9C01, 0x5CC0, 0x5D80, 0x9D41, 0x5F00, 0x9FC1, 0x9E81, 0x5E40, 0x5A00, 0x9AC1, 0x9B81, 0x5B40, 0x9901, 0x59C0, 0x5880, 0x9841, 0x8801, 0x48C0, 0x4980, 0x8941, 0x4B00, 0x8BC1, 0x8A81, 0x4A40, 0x4E00, 0x8EC1, 0x8F81, 0x4F40, 0x8D01, 0x4DC0, 0x4C80, 0x8C41, 0x4400, 0x84C1, 0x8581, 0x4540, 0x8701, 0x47C0, 0x4680, 0x8641, 0x8201, 0x42C0, 0x4380, 0x8341, 0x4100, 0x81C1, 0x8081, 0x4040 }; while (len--) crc = (((crc >> 8) & 0xffU) ^ crc16_table[(crc ^ *cp++) & 0xffU]) & 0x0000ffffU; return crc; } static inline int __bnxt_re_set_vma_data(void *bnxt_re_uctx, struct vm_area_struct *vma) { return 0; } static inline bool bnxt_re_check_if_vlan_valid(struct bnxt_re_dev *rdev, u16 vlan_id) { bool ret = true; /* * Check if the vlan is configured in the host. * If not configured, it can be a transparent * VLAN. So dont report the vlan id. */ return ret; } #endif diff --git a/sys/dev/bnxt/bnxt_re/main.c b/sys/dev/bnxt/bnxt_re/main.c index 3d26d21f3fc7..0e0cc32218bd 100644 --- a/sys/dev/bnxt/bnxt_re/main.c +++ b/sys/dev/bnxt/bnxt_re/main.c @@ -1,4467 +1,4474 @@ /* * Copyright (c) 2015-2024, Broadcom. All rights reserved. The term * Broadcom refers to Broadcom Limited and/or its subsidiaries. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in * the documentation and/or other materials provided with the * distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * * Description: Main component of the bnxt_re driver */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "bnxt_re.h" #include "ib_verbs.h" #include "bnxt_re-abi.h" #include "bnxt.h" static char drv_version[] = "Broadcom NetXtreme-C/E RoCE Driver " ROCE_DRV_MODULE_NAME \ " v" ROCE_DRV_MODULE_VERSION " (" ROCE_DRV_MODULE_RELDATE ")\n"; #define BNXT_RE_DESC "Broadcom NetXtreme RoCE" #define BNXT_ADEV_NAME "if_bnxt" MODULE_DESCRIPTION("Broadcom NetXtreme-C/E RoCE Driver"); MODULE_LICENSE("Dual BSD/GPL"); MODULE_DEPEND(bnxt_re, linuxkpi, 1, 1, 1); MODULE_DEPEND(bnxt_re, ibcore, 1, 1, 1); MODULE_DEPEND(bnxt_re, if_bnxt, 1, 1, 1); MODULE_VERSION(bnxt_re, 1); DEFINE_MUTEX(bnxt_re_mutex); /* mutex lock for driver */ static unsigned int restrict_mrs = 0; module_param(restrict_mrs, uint, 0); MODULE_PARM_DESC(restrict_mrs, " Restrict the no. of MRs 0 = 256K , 1 = 64K"); unsigned int restrict_stats = 0; module_param(restrict_stats, uint, 0); MODULE_PARM_DESC(restrict_stats, "Restrict stats query frequency to ethtool coalesce value. Disabled by default"); unsigned int enable_fc = 1; module_param(enable_fc, uint, 0); MODULE_PARM_DESC(enable_fc, "Enable default PFC, CC,ETS during driver load. 1 - fc enable, 0 - fc disable - Default is 1"); unsigned int min_tx_depth = 1; module_param(min_tx_depth, uint, 0); MODULE_PARM_DESC(min_tx_depth, "Minimum TX depth - Default is 1"); static uint8_t max_msix_vec[BNXT_RE_MAX_DEVICES] = {0}; static unsigned int max_msix_vec_argc; module_param_array(max_msix_vec, byte, &max_msix_vec_argc, 0444); MODULE_PARM_DESC(max_msix_vec, "Max MSI-x vectors per PF (2 - 64) - Default is 64"); unsigned int cmdq_shadow_qd = RCFW_CMD_NON_BLOCKING_SHADOW_QD; module_param_named(cmdq_shadow_qd, cmdq_shadow_qd, uint, 0644); MODULE_PARM_DESC(cmdq_shadow_qd, "Perf Stat Debug: Shadow QD Range (1-64) - Default is 64"); /* globals */ struct list_head bnxt_re_dev_list = LINUX_LIST_HEAD_INIT(bnxt_re_dev_list); static int bnxt_re_probe_count; DEFINE_MUTEX(bnxt_re_dev_lock); static u32 gmod_exit; static u32 gadd_dev_inprogress; static void bnxt_re_task(struct work_struct *work_task); static struct workqueue_struct *bnxt_re_wq; static int bnxt_re_query_hwrm_intf_version(struct bnxt_re_dev *rdev); static int bnxt_re_hwrm_qcfg(struct bnxt_re_dev *rdev, u32 *db_len, u32 *offset); static int bnxt_re_ib_init(struct bnxt_re_dev *rdev); static void bnxt_re_ib_init_2(struct bnxt_re_dev *rdev); void _bnxt_re_remove(struct auxiliary_device *adev); void writel_fbsd(struct bnxt_softc *bp, u32, u8, u32); u32 readl_fbsd(struct bnxt_softc *bp, u32, u8); static int bnxt_re_hwrm_dbr_pacing_qcfg(struct bnxt_re_dev *rdev); int bnxt_re_register_netdevice_notifier(struct notifier_block *nb) { int rc; rc = register_netdevice_notifier(nb); return rc; } int bnxt_re_unregister_netdevice_notifier(struct notifier_block *nb) { int rc; rc = unregister_netdevice_notifier(nb); return rc; } void bnxt_re_set_dma_device(struct ib_device *ibdev, struct bnxt_re_dev *rdev) { ibdev->dma_device = &rdev->en_dev->pdev->dev; } void bnxt_re_init_resolve_wq(struct bnxt_re_dev *rdev) { rdev->resolve_wq = create_singlethread_workqueue("bnxt_re_resolve_wq"); INIT_LIST_HEAD(&rdev->mac_wq_list); } void bnxt_re_uninit_resolve_wq(struct bnxt_re_dev *rdev) { struct bnxt_re_resolve_dmac_work *tmp_work = NULL, *tmp_st; if (!rdev->resolve_wq) return; flush_workqueue(rdev->resolve_wq); list_for_each_entry_safe(tmp_work, tmp_st, &rdev->mac_wq_list, list) { list_del(&tmp_work->list); kfree(tmp_work); } destroy_workqueue(rdev->resolve_wq); rdev->resolve_wq = NULL; } u32 readl_fbsd(struct bnxt_softc *bp, u32 reg_off, u8 bar_idx) { if (bar_idx) return bus_space_read_8(bp->doorbell_bar.tag, bp->doorbell_bar.handle, reg_off); else return bus_space_read_8(bp->hwrm_bar.tag, bp->hwrm_bar.handle, reg_off); } void writel_fbsd(struct bnxt_softc *bp, u32 reg_off, u8 bar_idx, u32 val) { if (bar_idx) bus_space_write_8(bp->doorbell_bar.tag, bp->doorbell_bar.handle, reg_off, htole32(val)); else bus_space_write_8(bp->hwrm_bar.tag, bp->hwrm_bar.handle, reg_off, htole32(val)); } static void bnxt_re_update_fifo_occup_slabs(struct bnxt_re_dev *rdev, u32 fifo_occup) { if (fifo_occup > rdev->dbg_stats->dbq.fifo_occup_water_mark) rdev->dbg_stats->dbq.fifo_occup_water_mark = fifo_occup; if (fifo_occup > 8 * rdev->pacing_algo_th) rdev->dbg_stats->dbq.fifo_occup_slab_4++; else if (fifo_occup > 4 * rdev->pacing_algo_th) rdev->dbg_stats->dbq.fifo_occup_slab_3++; else if (fifo_occup > 2 * rdev->pacing_algo_th) rdev->dbg_stats->dbq.fifo_occup_slab_2++; else if (fifo_occup > rdev->pacing_algo_th) rdev->dbg_stats->dbq.fifo_occup_slab_1++; } static void bnxt_re_update_do_pacing_slabs(struct bnxt_re_dev *rdev) { struct bnxt_qplib_db_pacing_data *pacing_data = rdev->qplib_res.pacing_data; if (pacing_data->do_pacing > rdev->dbg_stats->dbq.do_pacing_water_mark) rdev->dbg_stats->dbq.do_pacing_water_mark = pacing_data->do_pacing; if (pacing_data->do_pacing > 16 * rdev->dbr_def_do_pacing) rdev->dbg_stats->dbq.do_pacing_slab_5++; else if (pacing_data->do_pacing > 8 * rdev->dbr_def_do_pacing) rdev->dbg_stats->dbq.do_pacing_slab_4++; else if (pacing_data->do_pacing > 4 * rdev->dbr_def_do_pacing) rdev->dbg_stats->dbq.do_pacing_slab_3++; else if (pacing_data->do_pacing > 2 * rdev->dbr_def_do_pacing) rdev->dbg_stats->dbq.do_pacing_slab_2++; else if (pacing_data->do_pacing > rdev->dbr_def_do_pacing) rdev->dbg_stats->dbq.do_pacing_slab_1++; } static bool bnxt_re_is_qp1_qp(struct bnxt_re_qp *qp) { return qp->ib_qp.qp_type == IB_QPT_GSI; } static struct bnxt_re_qp *bnxt_re_get_qp1_qp(struct bnxt_re_dev *rdev) { struct bnxt_re_qp *qp; mutex_lock(&rdev->qp_lock); list_for_each_entry(qp, &rdev->qp_list, list) { if (bnxt_re_is_qp1_qp(qp)) { mutex_unlock(&rdev->qp_lock); return qp; } } mutex_unlock(&rdev->qp_lock); return NULL; } /* Set the maximum number of each resource that the driver actually wants * to allocate. This may be up to the maximum number the firmware has * reserved for the function. The driver may choose to allocate fewer * resources than the firmware maximum. */ static void bnxt_re_limit_pf_res(struct bnxt_re_dev *rdev) { struct bnxt_qplib_max_res dev_res = {}; struct bnxt_qplib_chip_ctx *cctx; struct bnxt_qplib_dev_attr *attr; struct bnxt_qplib_ctx *hctx; int i; attr = rdev->dev_attr; hctx = rdev->qplib_res.hctx; cctx = rdev->chip_ctx; bnxt_qplib_max_res_supported(cctx, &rdev->qplib_res, &dev_res, false); if (!_is_chip_gen_p5_p7(cctx)) { hctx->qp_ctx.max = min_t(u32, dev_res.max_qp, attr->max_qp); hctx->mrw_ctx.max = min_t(u32, dev_res.max_mr, attr->max_mr); /* To accommodate 16k MRs and 16k AHs, * driver has to allocate 32k backing store memory */ hctx->mrw_ctx.max *= 2; hctx->srq_ctx.max = min_t(u32, dev_res.max_srq, attr->max_srq); hctx->cq_ctx.max = min_t(u32, dev_res.max_cq, attr->max_cq); for (i = 0; i < MAX_TQM_ALLOC_REQ; i++) hctx->tqm_ctx.qcount[i] = attr->tqm_alloc_reqs[i]; } else { hctx->qp_ctx.max = attr->max_qp ? attr->max_qp : dev_res.max_qp; hctx->mrw_ctx.max = attr->max_mr ? attr->max_mr : dev_res.max_mr; hctx->srq_ctx.max = attr->max_srq ? attr->max_srq : dev_res.max_srq; hctx->cq_ctx.max = attr->max_cq ? attr->max_cq : dev_res.max_cq; } } static void bnxt_re_limit_vf_res(struct bnxt_re_dev *rdev, struct bnxt_qplib_vf_res *vf_res, u32 num_vf) { struct bnxt_qplib_chip_ctx *cctx = rdev->chip_ctx; struct bnxt_qplib_max_res dev_res = {}; bnxt_qplib_max_res_supported(cctx, &rdev->qplib_res, &dev_res, true); vf_res->max_qp = dev_res.max_qp / num_vf; vf_res->max_srq = dev_res.max_srq / num_vf; vf_res->max_cq = dev_res.max_cq / num_vf; /* * MR and AH shares the same backing store, the value specified * for max_mrw is split into half by the FW for MR and AH */ vf_res->max_mrw = dev_res.max_mr * 2 / num_vf; vf_res->max_gid = BNXT_RE_MAX_GID_PER_VF; } static void bnxt_re_set_resource_limits(struct bnxt_re_dev *rdev) { struct bnxt_qplib_ctx *hctx; hctx = rdev->qplib_res.hctx; memset(&hctx->vf_res, 0, sizeof(struct bnxt_qplib_vf_res)); bnxt_re_limit_pf_res(rdev); if (rdev->num_vfs) bnxt_re_limit_vf_res(rdev, &hctx->vf_res, rdev->num_vfs); } static void bnxt_re_dettach_irq(struct bnxt_re_dev *rdev) { struct bnxt_qplib_rcfw *rcfw = NULL; struct bnxt_qplib_nq *nq; int indx; rcfw = &rdev->rcfw; for (indx = 0; indx < rdev->nqr.max_init; indx++) { nq = &rdev->nqr.nq[indx]; mutex_lock(&nq->lock); bnxt_qplib_nq_stop_irq(nq, false); mutex_unlock(&nq->lock); } bnxt_qplib_rcfw_stop_irq(rcfw, false); } static void bnxt_re_detach_err_device(struct bnxt_re_dev *rdev) { /* Free the MSIx vectors only so that L2 can proceed with MSIx disable */ bnxt_re_dettach_irq(rdev); /* Set the state as detached to prevent sending any more commands */ set_bit(ERR_DEVICE_DETACHED, &rdev->rcfw.cmdq.flags); set_bit(BNXT_RE_FLAG_ERR_DEVICE_DETACHED, &rdev->flags); wake_up_all(&rdev->rcfw.cmdq.waitq); } #define MAX_DSCP_PRI_TUPLE 64 struct bnxt_re_dcb_work { struct work_struct work; struct bnxt_re_dev *rdev; struct hwrm_async_event_cmpl cmpl; }; static void bnxt_re_init_dcb_wq(struct bnxt_re_dev *rdev) { rdev->dcb_wq = create_singlethread_workqueue("bnxt_re_dcb_wq"); } static void bnxt_re_uninit_dcb_wq(struct bnxt_re_dev *rdev) { if (!rdev->dcb_wq) return; flush_workqueue(rdev->dcb_wq); destroy_workqueue(rdev->dcb_wq); rdev->dcb_wq = NULL; } static void bnxt_re_init_aer_wq(struct bnxt_re_dev *rdev) { rdev->aer_wq = create_singlethread_workqueue("bnxt_re_aer_wq"); } static void bnxt_re_uninit_aer_wq(struct bnxt_re_dev *rdev) { if (!rdev->aer_wq) return; flush_workqueue(rdev->aer_wq); destroy_workqueue(rdev->aer_wq); rdev->aer_wq = NULL; } static int bnxt_re_update_qp1_tos_dscp(struct bnxt_re_dev *rdev) { struct bnxt_re_qp *qp; if (!_is_chip_gen_p5_p7(rdev->chip_ctx)) return 0; qp = bnxt_re_get_qp1_qp(rdev); if (!qp) return 0; qp->qplib_qp.modify_flags = CMDQ_MODIFY_QP_MODIFY_MASK_TOS_DSCP; qp->qplib_qp.tos_dscp = rdev->cc_param.qp1_tos_dscp; return bnxt_qplib_modify_qp(&rdev->qplib_res, &qp->qplib_qp); } static void bnxt_re_reconfigure_dscp(struct bnxt_re_dev *rdev) { struct bnxt_qplib_cc_param *cc_param; struct bnxt_re_tc_rec *tc_rec; bool update_cc = false; u8 dscp_user; int rc; cc_param = &rdev->cc_param; tc_rec = &rdev->tc_rec[0]; if (!(cc_param->roce_dscp_user || cc_param->cnp_dscp_user)) return; if (cc_param->cnp_dscp_user) { dscp_user = (cc_param->cnp_dscp_user & 0x3f); if ((tc_rec->cnp_dscp_bv & (1ul << dscp_user)) && (cc_param->alt_tos_dscp != dscp_user)) { cc_param->alt_tos_dscp = dscp_user; cc_param->mask |= CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_ALT_TOS_DSCP; update_cc = true; } } if (cc_param->roce_dscp_user) { dscp_user = (cc_param->roce_dscp_user & 0x3f); if ((tc_rec->roce_dscp_bv & (1ul << dscp_user)) && (cc_param->tos_dscp != dscp_user)) { cc_param->tos_dscp = dscp_user; cc_param->mask |= CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_TOS_DSCP; update_cc = true; } } if (update_cc) { rc = bnxt_qplib_modify_cc(&rdev->qplib_res, cc_param); if (rc) dev_err(rdev_to_dev(rdev), "Failed to apply cc settings\n"); } } static void bnxt_re_dcb_wq_task(struct work_struct *work) { struct bnxt_qplib_cc_param *cc_param; struct bnxt_re_tc_rec *tc_rec; struct bnxt_re_dev *rdev; struct bnxt_re_dcb_work *dcb_work = container_of(work, struct bnxt_re_dcb_work, work); int rc; rdev = dcb_work->rdev; if (!rdev) goto exit; mutex_lock(&rdev->cc_lock); cc_param = &rdev->cc_param; rc = bnxt_qplib_query_cc_param(&rdev->qplib_res, cc_param); if (rc) { dev_err(rdev_to_dev(rdev), "Failed to query ccparam rc:%d", rc); goto fail; } tc_rec = &rdev->tc_rec[0]; /* * Upon the receival of DCB Async event: * If roce_dscp or cnp_dscp or both (which user configured using configfs) * is in the list, re-program the value using modify_roce_cc command */ bnxt_re_reconfigure_dscp(rdev); cc_param->roce_pri = tc_rec->roce_prio; if (cc_param->qp1_tos_dscp != cc_param->tos_dscp) { cc_param->qp1_tos_dscp = cc_param->tos_dscp; rc = bnxt_re_update_qp1_tos_dscp(rdev); if (rc) { dev_err(rdev_to_dev(rdev), "%s:Failed to modify QP1 rc:%d", __func__, rc); goto fail; } } fail: mutex_unlock(&rdev->cc_lock); exit: kfree(dcb_work); } static int bnxt_re_hwrm_dbr_pacing_broadcast_event(struct bnxt_re_dev *rdev) { struct hwrm_func_dbr_pacing_broadcast_event_output resp = {0}; struct hwrm_func_dbr_pacing_broadcast_event_input req = {0}; struct bnxt_en_dev *en_dev = rdev->en_dev; struct bnxt_fw_msg fw_msg; int rc; memset(&fw_msg, 0, sizeof(fw_msg)); bnxt_re_init_hwrm_hdr(rdev, (void *)&req, HWRM_FUNC_DBR_PACING_BROADCAST_EVENT, -1, -1); bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp, sizeof(resp), BNXT_RE_HWRM_CMD_TIMEOUT(rdev)); rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg); if (rc) { dev_dbg(rdev_to_dev(rdev), "Failed to send dbr pacing broadcast event rc:%d", rc); return rc; } return 0; } static int bnxt_re_hwrm_dbr_pacing_nqlist_query(struct bnxt_re_dev *rdev) { struct hwrm_func_dbr_pacing_nqlist_query_output resp = {0}; struct hwrm_func_dbr_pacing_nqlist_query_input req = {0}; struct bnxt_dbq_nq_list *nq_list = &rdev->nq_list; struct bnxt_en_dev *en_dev = rdev->en_dev; bool primary_found = false; struct bnxt_fw_msg fw_msg; struct bnxt_qplib_nq *nq; int rc, i, j = 1; u16 *nql_ptr; nq = &rdev->nqr.nq[0]; memset(&fw_msg, 0, sizeof(fw_msg)); bnxt_re_init_hwrm_hdr(rdev, (void *)&req, HWRM_FUNC_DBR_PACING_NQLIST_QUERY, -1, -1); bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp, sizeof(resp), BNXT_RE_HWRM_CMD_TIMEOUT(rdev)); rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg); if (rc) { dev_err(rdev_to_dev(rdev), "Failed to send dbr pacing nq list query rc:%d", rc); return rc; } nq_list->num_nql_entries = le32_to_cpu(resp.num_nqs); nql_ptr = &resp.nq_ring_id0; /* populate the nq_list of the primary function with list received * from FW. Fill the NQ IDs of secondary functions from index 1 to * num_nql_entries - 1. Fill the nq_list->nq_id[0] with the * nq_id of the primary pf */ for (i = 0; i < nq_list->num_nql_entries; i++) { u16 nq_id = *nql_ptr; dev_dbg(rdev_to_dev(rdev), "nq_list->nq_id[%d] = %d\n", i, nq_id); if (nq_id != nq->ring_id) { nq_list->nq_id[j] = nq_id; j++; } else { primary_found = true; nq_list->nq_id[0] = nq->ring_id; } nql_ptr++; } if (primary_found) bnxt_qplib_dbr_pacing_set_primary_pf(rdev->chip_ctx, 1); return 0; } static void __wait_for_fifo_occupancy_below_th(struct bnxt_re_dev *rdev) { struct bnxt_qplib_db_pacing_data *pacing_data = rdev->qplib_res.pacing_data; u32 read_val, fifo_occup; bool first_read = true; /* loop shouldn't run infintely as the occupancy usually goes * below pacing algo threshold as soon as pacing kicks in. */ while (1) { read_val = readl_fbsd(rdev->en_dev->softc, rdev->dbr_db_fifo_reg_off, 0); fifo_occup = pacing_data->fifo_max_depth - ((read_val & pacing_data->fifo_room_mask) >> pacing_data->fifo_room_shift); /* Fifo occupancy cannot be greater the MAX FIFO depth */ if (fifo_occup > pacing_data->fifo_max_depth) break; if (first_read) { bnxt_re_update_fifo_occup_slabs(rdev, fifo_occup); first_read = false; } if (fifo_occup < pacing_data->pacing_th) break; } } static void bnxt_re_set_default_pacing_data(struct bnxt_re_dev *rdev) { struct bnxt_qplib_db_pacing_data *pacing_data = rdev->qplib_res.pacing_data; pacing_data->do_pacing = rdev->dbr_def_do_pacing; pacing_data->pacing_th = rdev->pacing_algo_th; pacing_data->alarm_th = pacing_data->pacing_th * BNXT_RE_PACING_ALARM_TH_MULTIPLE(rdev->chip_ctx); } #define CAG_RING_MASK 0x7FF #define CAG_RING_SHIFT 17 #define WATERMARK_MASK 0xFFF #define WATERMARK_SHIFT 0 static bool bnxt_re_check_if_dbq_intr_triggered(struct bnxt_re_dev *rdev) { u32 read_val; int j; for (j = 0; j < 10; j++) { read_val = readl_fbsd(rdev->en_dev->softc, rdev->dbr_aeq_arm_reg_off, 0); dev_dbg(rdev_to_dev(rdev), "AEQ ARM status = 0x%x\n", read_val); if (!read_val) return true; } return false; } int bnxt_re_set_dbq_throttling_reg(struct bnxt_re_dev *rdev, u16 nq_id, u32 throttle) { u32 cag_ring_water_mark = 0, read_val; u32 throttle_val; /* Convert throttle percentage to value */ throttle_val = (rdev->qplib_res.pacing_data->fifo_max_depth * throttle) / 100; if (bnxt_qplib_dbr_pacing_ext_en(rdev->chip_ctx)) { cag_ring_water_mark = (nq_id & CAG_RING_MASK) << CAG_RING_SHIFT | (throttle_val & WATERMARK_MASK); writel_fbsd(rdev->en_dev->softc, rdev->dbr_throttling_reg_off, 0, cag_ring_water_mark); read_val = readl_fbsd(rdev->en_dev->softc , rdev->dbr_throttling_reg_off, 0); dev_dbg(rdev_to_dev(rdev), "%s: dbr_throttling_reg_off read_val = 0x%x\n", __func__, read_val); if (read_val != cag_ring_water_mark) { dev_dbg(rdev_to_dev(rdev), "nq_id = %d write_val=0x%x read_val=0x%x\n", nq_id, cag_ring_water_mark, read_val); return 1; } } writel_fbsd(rdev->en_dev->softc, rdev->dbr_aeq_arm_reg_off, 0, 1); return 0; } static void bnxt_re_set_dbq_throttling_for_non_primary(struct bnxt_re_dev *rdev) { struct bnxt_dbq_nq_list *nq_list; struct bnxt_qplib_nq *nq; int i; nq_list = &rdev->nq_list; /* Run a loop for other Active functions if this is primary function */ if (bnxt_qplib_dbr_pacing_is_primary_pf(rdev->chip_ctx)) { dev_dbg(rdev_to_dev(rdev), "%s: nq_list->num_nql_entries= %d\n", __func__, nq_list->num_nql_entries); nq = &rdev->nqr.nq[0]; for (i = nq_list->num_nql_entries - 1; i > 0; i--) { u16 nq_id = nq_list->nq_id[i]; if (nq) dev_dbg(rdev_to_dev(rdev), "%s: nq_id = %d cur_fn_ring_id = %d\n", __func__, nq_id, nq->ring_id); if (bnxt_re_set_dbq_throttling_reg (rdev, nq_id, 0)) break; bnxt_re_check_if_dbq_intr_triggered(rdev); } } } static void bnxt_re_handle_dbr_nq_pacing_notification(struct bnxt_re_dev *rdev) { struct bnxt_qplib_nq *nq; int rc = 0; nq = &rdev->nqr.nq[0]; /* Query the NQ list*/ rc = bnxt_re_hwrm_dbr_pacing_nqlist_query(rdev); if (rc) { dev_err(rdev_to_dev(rdev), "Failed to Query NQ list rc= %d", rc); return; } /*Configure GRC access for Throttling and aeq_arm register */ writel_fbsd(rdev->en_dev->softc, BNXT_GRCPF_REG_WINDOW_BASE_OUT + 28, 0, rdev->chip_ctx->dbr_aeq_arm_reg & BNXT_GRC_BASE_MASK); rdev->dbr_throttling_reg_off = (rdev->chip_ctx->dbr_throttling_reg & BNXT_GRC_OFFSET_MASK) + 0x8000; rdev->dbr_aeq_arm_reg_off = (rdev->chip_ctx->dbr_aeq_arm_reg & BNXT_GRC_OFFSET_MASK) + 0x8000; bnxt_re_set_dbq_throttling_reg(rdev, nq->ring_id, rdev->dbq_watermark); } static void bnxt_re_dbq_wq_task(struct work_struct *work) { struct bnxt_re_dbq_work *dbq_work = container_of(work, struct bnxt_re_dbq_work, work); struct bnxt_re_dev *rdev; rdev = dbq_work->rdev; if (!rdev) goto exit; switch (dbq_work->event) { case BNXT_RE_DBQ_EVENT_SCHED: dev_dbg(rdev_to_dev(rdev), "%s: Handle DBQ Pacing event\n", __func__); if (!bnxt_qplib_dbr_pacing_ext_en(rdev->chip_ctx)) bnxt_re_hwrm_dbr_pacing_broadcast_event(rdev); else bnxt_re_pacing_alert(rdev); break; case BNXT_RE_DBR_PACING_EVENT: dev_dbg(rdev_to_dev(rdev), "%s: Sched interrupt/pacing worker\n", __func__); if (_is_chip_p7(rdev->chip_ctx)) bnxt_re_pacing_alert(rdev); else if (!rdev->chip_ctx->modes.dbr_pacing_v0) bnxt_re_hwrm_dbr_pacing_qcfg(rdev); break; case BNXT_RE_DBR_NQ_PACING_NOTIFICATION: bnxt_re_handle_dbr_nq_pacing_notification(rdev); /* Issue a broadcast event to notify other functions * that primary changed */ bnxt_re_hwrm_dbr_pacing_broadcast_event(rdev); break; } exit: kfree(dbq_work); } static void bnxt_re_async_notifier(void *handle, struct hwrm_async_event_cmpl *cmpl) { struct bnxt_re_en_dev_info *en_info = auxiliary_get_drvdata(handle); struct bnxt_re_dcb_work *dcb_work; struct bnxt_re_dbq_work *dbq_work; struct bnxt_re_dev *rdev; u16 event_id; u32 data1; u32 data2 = 0; if (!cmpl) { pr_err("Async event, bad completion\n"); return; } if (!en_info || !en_info->en_dev) { pr_err("Async event, bad en_info or en_dev\n"); return; } rdev = en_info->rdev; event_id = le16_to_cpu(cmpl->event_id); data1 = le32_to_cpu(cmpl->event_data1); data2 = le32_to_cpu(cmpl->event_data2); if (!rdev || !rdev_to_dev(rdev)) { dev_dbg(NULL, "Async event, bad rdev or netdev\n"); return; } if (test_bit(BNXT_RE_FLAG_ERR_DEVICE_DETACHED, &rdev->flags) || !test_bit(BNXT_RE_FLAG_NETDEV_REGISTERED, &rdev->flags)) { dev_dbg(NULL, "Async event, device already detached\n"); return; } if (data2 >= 0) dev_dbg(rdev_to_dev(rdev), "Async event_id = %d data1 = %d data2 = %d", event_id, data1, data2); switch (event_id) { case HWRM_ASYNC_EVENT_CMPL_EVENT_ID_DCB_CONFIG_CHANGE: /* Not handling the event in older FWs */ if (!is_qport_service_type_supported(rdev)) break; if (!rdev->dcb_wq) break; dcb_work = kzalloc(sizeof(*dcb_work), GFP_ATOMIC); if (!dcb_work) break; dcb_work->rdev = rdev; memcpy(&dcb_work->cmpl, cmpl, sizeof(*cmpl)); INIT_WORK(&dcb_work->work, bnxt_re_dcb_wq_task); queue_work(rdev->dcb_wq, &dcb_work->work); break; case HWRM_ASYNC_EVENT_CMPL_EVENT_ID_RESET_NOTIFY: if (EVENT_DATA1_RESET_NOTIFY_FATAL(data1)) { /* Set rcfw flag to control commands send to Bono */ set_bit(ERR_DEVICE_DETACHED, &rdev->rcfw.cmdq.flags); /* Set bnxt_re flag to control commands send via L2 driver */ set_bit(BNXT_RE_FLAG_ERR_DEVICE_DETACHED, &rdev->flags); wake_up_all(&rdev->rcfw.cmdq.waitq); } break; case HWRM_ASYNC_EVENT_CMPL_EVENT_ID_DOORBELL_PACING_THRESHOLD: if (!rdev->dbr_pacing) break; dbq_work = kzalloc(sizeof(*dbq_work), GFP_ATOMIC); if (!dbq_work) goto unlock; dbq_work->rdev = rdev; dbq_work->event = BNXT_RE_DBR_PACING_EVENT; INIT_WORK(&dbq_work->work, bnxt_re_dbq_wq_task); queue_work(rdev->dbq_wq, &dbq_work->work); rdev->dbr_sw_stats->dbq_int_recv++; break; case HWRM_ASYNC_EVENT_CMPL_EVENT_ID_DOORBELL_PACING_NQ_UPDATE: if (!rdev->dbr_pacing) break; dbq_work = kzalloc(sizeof(*dbq_work), GFP_ATOMIC); if (!dbq_work) goto unlock; dbq_work->rdev = rdev; dbq_work->event = BNXT_RE_DBR_NQ_PACING_NOTIFICATION; INIT_WORK(&dbq_work->work, bnxt_re_dbq_wq_task); queue_work(rdev->dbq_wq, &dbq_work->work); break; default: break; } unlock: return; } static void bnxt_re_db_fifo_check(struct work_struct *work) { struct bnxt_re_dev *rdev = container_of(work, struct bnxt_re_dev, dbq_fifo_check_work); struct bnxt_qplib_db_pacing_data *pacing_data; u32 pacing_save; if (!mutex_trylock(&rdev->dbq_lock)) return; pacing_data = rdev->qplib_res.pacing_data; pacing_save = rdev->do_pacing_save; __wait_for_fifo_occupancy_below_th(rdev); cancel_delayed_work_sync(&rdev->dbq_pacing_work); if (rdev->dbr_recovery_on) goto recovery_on; if (pacing_save > rdev->dbr_def_do_pacing) { /* Double the do_pacing value during the congestion */ pacing_save = pacing_save << 1; } else { /* * when a new congestion is detected increase the do_pacing * by 8 times. And also increase the pacing_th by 4 times. The * reason to increase pacing_th is to give more space for the * queue to oscillate down without getting empty, but also more * room for the queue to increase without causing another alarm. */ pacing_save = pacing_save << 3; pacing_data->pacing_th = rdev->pacing_algo_th * 4; } if (pacing_save > BNXT_RE_MAX_DBR_DO_PACING) pacing_save = BNXT_RE_MAX_DBR_DO_PACING; pacing_data->do_pacing = pacing_save; rdev->do_pacing_save = pacing_data->do_pacing; pacing_data->alarm_th = pacing_data->pacing_th * BNXT_RE_PACING_ALARM_TH_MULTIPLE(rdev->chip_ctx); recovery_on: schedule_delayed_work(&rdev->dbq_pacing_work, msecs_to_jiffies(rdev->dbq_pacing_time)); rdev->dbr_sw_stats->dbq_pacing_alerts++; mutex_unlock(&rdev->dbq_lock); } static void bnxt_re_pacing_timer_exp(struct work_struct *work) { struct bnxt_re_dev *rdev = container_of(work, struct bnxt_re_dev, dbq_pacing_work.work); struct bnxt_qplib_db_pacing_data *pacing_data; u32 read_val, fifo_occup; struct bnxt_qplib_nq *nq; if (!mutex_trylock(&rdev->dbq_lock)) return; pacing_data = rdev->qplib_res.pacing_data; read_val = readl_fbsd(rdev->en_dev->softc , rdev->dbr_db_fifo_reg_off, 0); fifo_occup = pacing_data->fifo_max_depth - ((read_val & pacing_data->fifo_room_mask) >> pacing_data->fifo_room_shift); if (fifo_occup > pacing_data->pacing_th) goto restart_timer; /* * Instead of immediately going back to the default do_pacing * reduce it by 1/8 times and restart the timer. */ pacing_data->do_pacing = pacing_data->do_pacing - (pacing_data->do_pacing >> 3); pacing_data->do_pacing = max_t(u32, rdev->dbr_def_do_pacing, pacing_data->do_pacing); /* * If the fifo_occup is less than the interrupt enable threshold * enable the interrupt on the primary PF. */ if (rdev->dbq_int_disable && fifo_occup < rdev->pacing_en_int_th) { if (bnxt_qplib_dbr_pacing_is_primary_pf(rdev->chip_ctx)) { if (!rdev->chip_ctx->modes.dbr_pacing_v0) { nq = &rdev->nqr.nq[0]; bnxt_re_set_dbq_throttling_reg(rdev, nq->ring_id, rdev->dbq_watermark); rdev->dbr_sw_stats->dbq_int_en++; rdev->dbq_int_disable = false; } } } if (pacing_data->do_pacing <= rdev->dbr_def_do_pacing) { bnxt_re_set_default_pacing_data(rdev); rdev->dbr_sw_stats->dbq_pacing_complete++; goto dbq_unlock; } restart_timer: schedule_delayed_work(&rdev->dbq_pacing_work, msecs_to_jiffies(rdev->dbq_pacing_time)); bnxt_re_update_do_pacing_slabs(rdev); rdev->dbr_sw_stats->dbq_pacing_resched++; dbq_unlock: rdev->do_pacing_save = pacing_data->do_pacing; mutex_unlock(&rdev->dbq_lock); } void bnxt_re_pacing_alert(struct bnxt_re_dev *rdev) { struct bnxt_qplib_db_pacing_data *pacing_data; if (!rdev->dbr_pacing) return; mutex_lock(&rdev->dbq_lock); pacing_data = rdev->qplib_res.pacing_data; /* * Increase the alarm_th to max so that other user lib instances do not * keep alerting the driver. */ pacing_data->alarm_th = pacing_data->fifo_max_depth; pacing_data->do_pacing = BNXT_RE_MAX_DBR_DO_PACING; cancel_work_sync(&rdev->dbq_fifo_check_work); schedule_work(&rdev->dbq_fifo_check_work); mutex_unlock(&rdev->dbq_lock); } void bnxt_re_schedule_dbq_event(struct bnxt_qplib_res *res) { struct bnxt_re_dbq_work *dbq_work; struct bnxt_re_dev *rdev; rdev = container_of(res, struct bnxt_re_dev, qplib_res); atomic_set(&rdev->dbq_intr_running, 1); if (test_bit(BNXT_RE_FLAG_ERR_DEVICE_DETACHED, &rdev->flags)) goto exit; /* Run the loop to send dbq event to other functions * for newer FW */ if (bnxt_qplib_dbr_pacing_ext_en(rdev->chip_ctx) && !rdev->chip_ctx->modes.dbr_pacing_v0) bnxt_re_set_dbq_throttling_for_non_primary(rdev); dbq_work = kzalloc(sizeof(*dbq_work), GFP_ATOMIC); if (!dbq_work) goto exit; dbq_work->rdev = rdev; dbq_work->event = BNXT_RE_DBQ_EVENT_SCHED; INIT_WORK(&dbq_work->work, bnxt_re_dbq_wq_task); queue_work(rdev->dbq_wq, &dbq_work->work); rdev->dbr_sw_stats->dbq_int_recv++; rdev->dbq_int_disable = true; exit: atomic_set(&rdev->dbq_intr_running, 0); } static void bnxt_re_free_msix(struct bnxt_re_dev *rdev) { struct bnxt_en_dev *en_dev = rdev->en_dev; int rc; rc = en_dev->en_ops->bnxt_free_msix(rdev->en_dev, BNXT_ROCE_ULP); if (rc) dev_err(rdev_to_dev(rdev), "netdev %p free_msix failed! rc = 0x%x", rdev->netdev, rc); } static int bnxt_re_request_msix(struct bnxt_re_dev *rdev) { struct bnxt_en_dev *en_dev = rdev->en_dev; int rc = 0, num_msix_want, num_msix_got; struct bnxt_msix_entry *entry; /* * Request MSIx based on the function type. This is * a temporory solution to enable max VFs when NPAR is * enabled. * TODO - change the scheme with an adapter specific check * as the latest adapters can support more NQs. For now * this change satisfy all adapter versions. */ if (rdev->is_virtfn) num_msix_want = BNXT_RE_MAX_MSIX_VF; else if (BNXT_EN_NPAR(en_dev)) num_msix_want = BNXT_RE_MAX_MSIX_NPAR_PF; else if (_is_chip_gen_p5_p7(rdev->chip_ctx)) num_msix_want = rdev->num_msix_requested ?: BNXT_RE_MAX_MSIX_GEN_P5_PF; else num_msix_want = BNXT_RE_MAX_MSIX_PF; /* * Since MSIX vectors are used for both NQs and CREQ, we should try to * allocate num_online_cpus + 1 by taking into account the CREQ. This * leaves the number of MSIX vectors for NQs match the number of CPUs * and allows the system to be fully utilized */ num_msix_want = min_t(u32, num_msix_want, num_online_cpus() + 1); num_msix_want = min_t(u32, num_msix_want, BNXT_RE_MAX_MSIX); num_msix_want = max_t(u32, num_msix_want, BNXT_RE_MIN_MSIX); entry = rdev->nqr.msix_entries; num_msix_got = en_dev->en_ops->bnxt_request_msix(en_dev, BNXT_ROCE_ULP, entry, num_msix_want); if (num_msix_got < BNXT_RE_MIN_MSIX) { rc = -EINVAL; goto done; } if (num_msix_got != num_msix_want) dev_warn(rdev_to_dev(rdev), "bnxt_request_msix: wanted %d vectors, got %d\n", num_msix_want, num_msix_got); rdev->nqr.num_msix = num_msix_got; return 0; done: if (num_msix_got) bnxt_re_free_msix(rdev); return rc; } static int __wait_for_ib_unregister(struct bnxt_re_dev *rdev, struct bnxt_re_en_dev_info *en_info) { u64 timeout = 0; u32 cur_prod = 0, cur_cons = 0; int retry = 0, rc = 0, ret = 0; cur_prod = rdev->rcfw.cmdq.hwq.prod; cur_cons = rdev->rcfw.cmdq.hwq.cons; timeout = msecs_to_jiffies(BNXT_RE_RECOVERY_IB_UNINIT_WAIT_TIME_MS); retry = BNXT_RE_RECOVERY_IB_UNINIT_WAIT_RETRY; /* During module exit, increase timeout ten-fold to 100 mins to wait * as long as possible for ib_unregister() to complete */ if (rdev->mod_exit) retry *= 10; do { /* * Since the caller of this function invokes with bnxt_re_mutex held, * release it to avoid holding a lock while in wait / sleep mode. */ mutex_unlock(&bnxt_re_mutex); rc = wait_event_timeout(en_info->waitq, en_info->ib_uninit_done, timeout); mutex_lock(&bnxt_re_mutex); if (!bnxt_re_is_rdev_valid(rdev)) break; if (rc) break; if (!RCFW_NO_FW_ACCESS(&rdev->rcfw)) { /* No need to check for cmdq stall during module exit, * wait for ib unregister to complete */ if (!rdev->mod_exit) ret = __check_cmdq_stall(&rdev->rcfw, &cur_prod, &cur_cons); if (ret || en_info->ib_uninit_done) break; } } while (retry--); return rc; } static int bnxt_re_handle_start(struct auxiliary_device *adev) { struct bnxt_re_en_dev_info *en_info = auxiliary_get_drvdata(adev); struct bnxt_re_dev *rdev = NULL; struct ifnet *real_dev; struct bnxt_en_dev *en_dev; struct ifnet *netdev; int rc = 0; if (!en_info || !en_info->en_dev) { pr_err("Start, bad en_info or en_dev\n"); return -EINVAL; } netdev = en_info->en_dev->net; if (en_info->rdev) { dev_info(rdev_to_dev(en_info->rdev), "%s: Device is already added adev %p rdev: %p\n", __func__, adev, en_info->rdev); return 0; } en_dev = en_info->en_dev; real_dev = rdma_vlan_dev_real_dev(netdev); if (!real_dev) real_dev = netdev; rc = bnxt_re_add_device(&rdev, real_dev, en_info->gsi_mode, BNXT_RE_POST_RECOVERY_INIT, en_info->wqe_mode, en_info->num_msix_requested, adev); if (rc) { /* Add device failed. Unregister the device. * This has to be done explicitly as * bnxt_re_stop would not have unregistered */ rtnl_lock(); en_dev->en_ops->bnxt_unregister_device(en_dev, BNXT_ROCE_ULP); rtnl_unlock(); mutex_lock(&bnxt_re_dev_lock); gadd_dev_inprogress--; mutex_unlock(&bnxt_re_dev_lock); return rc; } rdev->adev = adev; rtnl_lock(); bnxt_re_get_link_speed(rdev); rtnl_unlock(); rc = bnxt_re_ib_init(rdev); if (rc) { dev_err(rdev_to_dev(rdev), "Failed ib_init\n"); return rc; } bnxt_re_ib_init_2(rdev); return rc; } static void bnxt_re_stop(void *handle) { struct bnxt_re_en_dev_info *en_info = auxiliary_get_drvdata(handle); struct ifnet *netdev; struct bnxt_re_dev *rdev; struct bnxt_en_dev *en_dev; int rc = 0; rtnl_unlock(); mutex_lock(&bnxt_re_mutex); if (!en_info || !en_info->en_dev) { pr_err("Stop, bad en_info or en_dev\n"); goto exit; } netdev = en_info->en_dev->net; rdev = en_info->rdev; if (!rdev) goto exit; if (!bnxt_re_is_rdev_valid(rdev)) goto exit; /* * Check if fw has undergone reset or is in a fatal condition. * If so, set flags so that no further commands are sent down to FW */ en_dev = rdev->en_dev; if (en_dev->en_state & BNXT_STATE_FW_FATAL_COND || en_dev->en_state & BNXT_STATE_FW_RESET_DET) { /* Set rcfw flag to control commands send to Bono */ set_bit(ERR_DEVICE_DETACHED, &rdev->rcfw.cmdq.flags); /* Set bnxt_re flag to control commands send via L2 driver */ set_bit(BNXT_RE_FLAG_ERR_DEVICE_DETACHED, &rdev->flags); wake_up_all(&rdev->rcfw.cmdq.waitq); } if (test_bit(BNXT_RE_FLAG_STOP_IN_PROGRESS, &rdev->flags)) goto exit; set_bit(BNXT_RE_FLAG_STOP_IN_PROGRESS, &rdev->flags); en_info->wqe_mode = rdev->chip_ctx->modes.wqe_mode; en_info->gsi_mode = rdev->gsi_ctx.gsi_qp_mode; en_info->num_msix_requested = rdev->num_msix_requested; en_info->ib_uninit_done = false; if (rdev->dbr_pacing) bnxt_re_set_pacing_dev_state(rdev); dev_info(rdev_to_dev(rdev), "%s: L2 driver notified to stop." "Attempting to stop and Dispatching event " "to inform the stack\n", __func__); init_waitqueue_head(&en_info->waitq); /* Schedule a work item to handle IB UNINIT for recovery */ bnxt_re_schedule_work(rdev, NETDEV_UNREGISTER, NULL, netdev, rdev->adev); rc = __wait_for_ib_unregister(rdev, en_info); if (!bnxt_re_is_rdev_valid(rdev)) goto exit; if (!rc) { dev_info(rdev_to_dev(rdev), "%s: Attempt to stop failed\n", __func__); bnxt_re_detach_err_device(rdev); goto exit; } bnxt_re_remove_device(rdev, BNXT_RE_PRE_RECOVERY_REMOVE, rdev->adev); exit: mutex_unlock(&bnxt_re_mutex); /* Take rtnl_lock before return, bnxt_re_stop is called with rtnl_lock */ rtnl_lock(); return; } static void bnxt_re_start(void *handle) { rtnl_unlock(); mutex_lock(&bnxt_re_mutex); if (bnxt_re_handle_start((struct auxiliary_device *)handle)) pr_err("Failed to start RoCE device"); mutex_unlock(&bnxt_re_mutex); /* Take rtnl_lock before return, bnxt_re_start is called with rtnl_lock */ rtnl_lock(); return; } static void bnxt_re_shutdown(void *p) { struct bnxt_re_en_dev_info *en_info = auxiliary_get_drvdata(p); struct bnxt_re_dev *rdev; if (!en_info) { pr_err("Shutdown, bad en_info\n"); return; } rtnl_unlock(); mutex_lock(&bnxt_re_mutex); rdev = en_info->rdev; if (!rdev || !bnxt_re_is_rdev_valid(rdev)) goto exit; /* rtnl_lock held by L2 before coming here */ bnxt_re_stopqps_and_ib_uninit(rdev); bnxt_re_remove_device(rdev, BNXT_RE_COMPLETE_REMOVE, rdev->adev); exit: mutex_unlock(&bnxt_re_mutex); rtnl_lock(); return; } static void bnxt_re_stop_irq(void *handle) { struct bnxt_re_en_dev_info *en_info = auxiliary_get_drvdata(handle); struct bnxt_qplib_rcfw *rcfw = NULL; struct bnxt_re_dev *rdev; struct bnxt_qplib_nq *nq; int indx; if (!en_info) { pr_err("Stop irq, bad en_info\n"); return; } rdev = en_info->rdev; if (!rdev) return; rcfw = &rdev->rcfw; for (indx = 0; indx < rdev->nqr.max_init; indx++) { nq = &rdev->nqr.nq[indx]; mutex_lock(&nq->lock); bnxt_qplib_nq_stop_irq(nq, false); mutex_unlock(&nq->lock); } if (test_bit(BNXT_RE_FLAG_ALLOC_RCFW, &rdev->flags)) bnxt_qplib_rcfw_stop_irq(rcfw, false); } static void bnxt_re_start_irq(void *handle, struct bnxt_msix_entry *ent) { struct bnxt_re_en_dev_info *en_info = auxiliary_get_drvdata(handle); struct bnxt_msix_entry *msix_ent = NULL; struct bnxt_qplib_rcfw *rcfw = NULL; struct bnxt_re_dev *rdev; struct bnxt_qplib_nq *nq; int indx, rc, vec; if (!en_info) { pr_err("Start irq, bad en_info\n"); return; } rdev = en_info->rdev; if (!rdev) return; if (test_bit(BNXT_RE_FLAG_ERR_DEVICE_DETACHED, &rdev->flags)) return; msix_ent = rdev->nqr.msix_entries; rcfw = &rdev->rcfw; if (!ent) { /* Not setting the f/w timeout bit in rcfw. * During the driver unload the first command * to f/w will timeout and that will set the * timeout bit. */ dev_err(rdev_to_dev(rdev), "Failed to re-start IRQs\n"); return; } /* Vectors may change after restart, so update with new vectors * in device structure. */ for (indx = 0; indx < rdev->nqr.num_msix; indx++) rdev->nqr.msix_entries[indx].vector = ent[indx].vector; if (test_bit(BNXT_RE_FLAG_ALLOC_RCFW, &rdev->flags)) { rc = bnxt_qplib_rcfw_start_irq(rcfw, msix_ent[BNXT_RE_AEQ_IDX].vector, false); if (rc) { dev_warn(rdev_to_dev(rdev), "Failed to reinit CREQ\n"); return; } } for (indx = 0 ; indx < rdev->nqr.max_init; indx++) { nq = &rdev->nqr.nq[indx]; vec = indx + 1; rc = bnxt_qplib_nq_start_irq(nq, indx, msix_ent[vec].vector, false); if (rc) { dev_warn(rdev_to_dev(rdev), "Failed to reinit NQ index %d\n", indx); return; } } } /* * Except for ulp_async_notifier, the remaining ulp_ops * below are called with rtnl_lock held */ static struct bnxt_ulp_ops bnxt_re_ulp_ops = { .ulp_async_notifier = bnxt_re_async_notifier, .ulp_stop = bnxt_re_stop, .ulp_start = bnxt_re_start, .ulp_shutdown = bnxt_re_shutdown, .ulp_irq_stop = bnxt_re_stop_irq, .ulp_irq_restart = bnxt_re_start_irq, }; static inline const char *bnxt_re_netevent(unsigned long event) { BNXT_RE_NETDEV_EVENT(event, NETDEV_UP); BNXT_RE_NETDEV_EVENT(event, NETDEV_DOWN); BNXT_RE_NETDEV_EVENT(event, NETDEV_CHANGE); BNXT_RE_NETDEV_EVENT(event, NETDEV_REGISTER); BNXT_RE_NETDEV_EVENT(event, NETDEV_UNREGISTER); BNXT_RE_NETDEV_EVENT(event, NETDEV_CHANGEADDR); return "Unknown"; } /* RoCE -> Net driver */ /* Driver registration routines used to let the networking driver (bnxt_en) * to know that the RoCE driver is now installed */ static void bnxt_re_unregister_netdev(struct bnxt_re_dev *rdev) { struct bnxt_en_dev *en_dev = rdev->en_dev; int rc; rtnl_lock(); rc = en_dev->en_ops->bnxt_unregister_device(rdev->en_dev, BNXT_ROCE_ULP); rtnl_unlock(); if (rc) dev_err(rdev_to_dev(rdev), "netdev %p unregister failed! rc = 0x%x", rdev->en_dev->net, rc); clear_bit(BNXT_RE_FLAG_NETDEV_REGISTERED, &rdev->flags); } static int bnxt_re_register_netdev(struct bnxt_re_dev *rdev) { struct bnxt_en_dev *en_dev = rdev->en_dev; int rc = 0; rtnl_lock(); rc = en_dev->en_ops->bnxt_register_device(en_dev, BNXT_ROCE_ULP, &bnxt_re_ulp_ops, rdev->adev); rtnl_unlock(); if (rc) { dev_err(rdev_to_dev(rdev), "netdev %p register failed! rc = 0x%x", rdev->netdev, rc); return rc; } return rc; } static void bnxt_re_set_db_offset(struct bnxt_re_dev *rdev) { struct bnxt_qplib_chip_ctx *cctx; struct bnxt_en_dev *en_dev; struct bnxt_qplib_res *res; u32 l2db_len = 0; u32 offset = 0; u32 barlen; int rc; res = &rdev->qplib_res; en_dev = rdev->en_dev; cctx = rdev->chip_ctx; /* Issue qcfg */ rc = bnxt_re_hwrm_qcfg(rdev, &l2db_len, &offset); if (rc) dev_info(rdev_to_dev(rdev), "Couldn't get DB bar size, Low latency framework is disabled\n"); /* set register offsets for both UC and WC */ - if (_is_chip_p7(cctx)) - res->dpi_tbl.ucreg.offset = offset; - else + if (_is_chip_p7(cctx)) { + res->dpi_tbl.ucreg.offset = en_dev->l2_db_offset; + res->dpi_tbl.wcreg.offset = en_dev->l2_db_size; + } else { res->dpi_tbl.ucreg.offset = res->is_vf ? BNXT_QPLIB_DBR_VF_DB_OFFSET : BNXT_QPLIB_DBR_PF_DB_OFFSET; - res->dpi_tbl.wcreg.offset = res->dpi_tbl.ucreg.offset; + res->dpi_tbl.wcreg.offset = res->dpi_tbl.ucreg.offset; + } /* If WC mapping is disabled by L2 driver then en_dev->l2_db_size * is equal to the DB-Bar actual size. This indicates that L2 * is mapping entire bar as UC-. RoCE driver can't enable WC mapping * in such cases and DB-push will be disabled. */ barlen = pci_resource_len(res->pdev, RCFW_DBR_PCI_BAR_REGION); if (cctx->modes.db_push && l2db_len && en_dev->l2_db_size != barlen) { res->dpi_tbl.wcreg.offset = en_dev->l2_db_size; dev_info(rdev_to_dev(rdev), "Low latency framework is enabled\n"); } return; } static void bnxt_re_set_drv_mode(struct bnxt_re_dev *rdev, u8 mode) { struct bnxt_qplib_chip_ctx *cctx; struct bnxt_en_dev *en_dev; en_dev = rdev->en_dev; cctx = rdev->chip_ctx; cctx->modes.wqe_mode = _is_chip_gen_p5_p7(rdev->chip_ctx) ? mode : BNXT_QPLIB_WQE_MODE_STATIC; cctx->modes.te_bypass = false; if (bnxt_re_hwrm_qcaps(rdev)) dev_err(rdev_to_dev(rdev), "Failed to query hwrm qcaps\n"); /* * TODO: Need a better mechanism for spreading of the * 512 extended PPP pages in the presence of VF and * NPAR, until then not enabling push */ if (_is_chip_p7(rdev->chip_ctx) && cctx->modes.db_push) { if (rdev->is_virtfn || BNXT_EN_NPAR(en_dev)) cctx->modes.db_push = false; } rdev->roce_mode = en_dev->flags & BNXT_EN_FLAG_ROCE_CAP; dev_dbg(rdev_to_dev(rdev), "RoCE is supported on the device - caps:0x%x", rdev->roce_mode); if (!_is_chip_gen_p5_p7(rdev->chip_ctx)) rdev->roce_mode = BNXT_RE_FLAG_ROCEV2_CAP; cctx->hw_stats_size = en_dev->hw_ring_stats_size; } static void bnxt_re_destroy_chip_ctx(struct bnxt_re_dev *rdev) { struct bnxt_qplib_chip_ctx *chip_ctx; struct bnxt_qplib_res *res; if (!rdev->chip_ctx) return; res = &rdev->qplib_res; bnxt_qplib_unmap_db_bar(res); kfree(res->hctx); res->rcfw = NULL; kfree(rdev->dev_attr); rdev->dev_attr = NULL; chip_ctx = rdev->chip_ctx; rdev->chip_ctx = NULL; res->cctx = NULL; res->hctx = NULL; res->pdev = NULL; res->netdev = NULL; kfree(chip_ctx); } static int bnxt_re_setup_chip_ctx(struct bnxt_re_dev *rdev, u8 wqe_mode) { struct bnxt_qplib_chip_ctx *chip_ctx; struct bnxt_en_dev *en_dev; int rc; en_dev = rdev->en_dev; /* Supply pci device to qplib */ rdev->qplib_res.pdev = en_dev->pdev; rdev->qplib_res.netdev = rdev->netdev; rdev->qplib_res.en_dev = en_dev; chip_ctx = kzalloc(sizeof(*chip_ctx), GFP_KERNEL); if (!chip_ctx) return -ENOMEM; rdev->chip_ctx = chip_ctx; rdev->qplib_res.cctx = chip_ctx; rc = bnxt_re_query_hwrm_intf_version(rdev); if (rc) goto fail; rdev->dev_attr = kzalloc(sizeof(*rdev->dev_attr), GFP_KERNEL); if (!rdev->dev_attr) { rc = -ENOMEM; goto fail; } rdev->qplib_res.dattr = rdev->dev_attr; rdev->qplib_res.rcfw = &rdev->rcfw; rdev->qplib_res.is_vf = rdev->is_virtfn; rdev->qplib_res.hctx = kzalloc(sizeof(*rdev->qplib_res.hctx), GFP_KERNEL); if (!rdev->qplib_res.hctx) { rc = -ENOMEM; goto fail; } bnxt_re_set_drv_mode(rdev, wqe_mode); bnxt_re_set_db_offset(rdev); rc = bnxt_qplib_map_db_bar(&rdev->qplib_res); if (rc) goto fail; rc = bnxt_qplib_enable_atomic_ops_to_root(en_dev->pdev); if (rc) dev_dbg(rdev_to_dev(rdev), "platform doesn't support global atomics"); return 0; fail: kfree(rdev->chip_ctx); rdev->chip_ctx = NULL; kfree(rdev->dev_attr); rdev->dev_attr = NULL; kfree(rdev->qplib_res.hctx); rdev->qplib_res.hctx = NULL; return rc; } static u16 bnxt_re_get_rtype(struct bnxt_re_dev *rdev) { return _is_chip_gen_p5_p7(rdev->chip_ctx) ? HWRM_RING_ALLOC_INPUT_RING_TYPE_NQ : HWRM_RING_ALLOC_INPUT_RING_TYPE_ROCE_CMPL; } static int bnxt_re_net_ring_free(struct bnxt_re_dev *rdev, u16 fw_ring_id) { int rc = -EINVAL; struct hwrm_ring_free_input req = {0}; struct hwrm_ring_free_output resp; struct bnxt_en_dev *en_dev = rdev->en_dev; struct bnxt_fw_msg fw_msg; if (!en_dev) return rc; /* To avoid unnecessary error messages during recovery. * HW is anyway in error state. So dont send down the command */ if (test_bit(BNXT_RE_FLAG_ERR_DEVICE_DETACHED, &rdev->flags)) return 0; /* allocation had failed, no need to issue hwrm */ if (fw_ring_id == 0xffff) return 0; memset(&fw_msg, 0, sizeof(fw_msg)); bnxt_re_init_hwrm_hdr(rdev, (void *)&req, HWRM_RING_FREE, -1, -1); req.ring_type = bnxt_re_get_rtype(rdev); req.ring_id = cpu_to_le16(fw_ring_id); bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp, sizeof(resp), DFLT_HWRM_CMD_TIMEOUT); rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg); if (rc) { dev_err(rdev_to_dev(rdev), "Failed to free HW ring with rc = 0x%x", rc); return rc; } dev_dbg(rdev_to_dev(rdev), "HW ring freed with id = 0x%x\n", fw_ring_id); return rc; } static int bnxt_re_net_ring_alloc(struct bnxt_re_dev *rdev, struct bnxt_re_ring_attr *ring_attr, u16 *fw_ring_id) { int rc = -EINVAL; struct hwrm_ring_alloc_input req = {0}; struct hwrm_ring_alloc_output resp; struct bnxt_en_dev *en_dev = rdev->en_dev; struct bnxt_fw_msg fw_msg; if (!en_dev) return rc; memset(&fw_msg, 0, sizeof(fw_msg)); bnxt_re_init_hwrm_hdr(rdev, (void *)&req, HWRM_RING_ALLOC, -1, -1); req.flags = cpu_to_le16(ring_attr->flags); req.enables = 0; req.page_tbl_addr = cpu_to_le64(ring_attr->dma_arr[0]); if (ring_attr->pages > 1) { /* Page size is in log2 units */ req.page_size = BNXT_PAGE_SHIFT; req.page_tbl_depth = 1; } else { req.page_size = 4; req.page_tbl_depth = 0; } req.fbo = 0; /* Association of ring index with doorbell index and MSIX number */ req.logical_id = cpu_to_le16(ring_attr->lrid); req.length = cpu_to_le32(ring_attr->depth + 1); req.ring_type = ring_attr->type; req.int_mode = ring_attr->mode; bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp, sizeof(resp), DFLT_HWRM_CMD_TIMEOUT); rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg); if (rc) { dev_err(rdev_to_dev(rdev), "Failed to allocate HW ring with rc = 0x%x", rc); return rc; } *fw_ring_id = le16_to_cpu(resp.ring_id); dev_dbg(rdev_to_dev(rdev), "HW ring allocated with id = 0x%x at slot 0x%x", resp.ring_id, ring_attr->lrid); return rc; } static int bnxt_re_net_stats_ctx_free(struct bnxt_re_dev *rdev, u32 fw_stats_ctx_id, u16 tid) { struct bnxt_en_dev *en_dev = rdev->en_dev; struct hwrm_stat_ctx_free_input req = {0}; struct hwrm_stat_ctx_free_output resp; struct bnxt_fw_msg fw_msg; int rc = -EINVAL; if (!en_dev) return rc; /* To avoid unnecessary error messages during recovery. * HW is anyway in error state. So dont send down the command */ if (test_bit(BNXT_RE_FLAG_ERR_DEVICE_DETACHED, &rdev->flags)) return 0; memset(&fw_msg, 0, sizeof(fw_msg)); bnxt_re_init_hwrm_hdr(rdev, (void *)&req, HWRM_STAT_CTX_FREE, -1, tid); req.stat_ctx_id = cpu_to_le32(fw_stats_ctx_id); bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp, sizeof(resp), DFLT_HWRM_CMD_TIMEOUT); rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg); if (rc) { dev_err(rdev_to_dev(rdev), "Failed to free HW stats ctx with rc = 0x%x", rc); return rc; } dev_dbg(rdev_to_dev(rdev), "HW stats ctx freed with id = 0x%x", fw_stats_ctx_id); return rc; } static int bnxt_re_net_stats_ctx_alloc(struct bnxt_re_dev *rdev, u16 tid) { struct hwrm_stat_ctx_alloc_output resp = {}; struct hwrm_stat_ctx_alloc_input req = {}; struct bnxt_en_dev *en_dev = rdev->en_dev; struct bnxt_qplib_stats *stat; struct bnxt_qplib_ctx *hctx; struct bnxt_fw_msg fw_msg; int rc = 0; hctx = rdev->qplib_res.hctx; stat = (tid == 0xffff) ? &hctx->stats : &hctx->stats2; stat->fw_id = INVALID_STATS_CTX_ID; if (!en_dev) return -EINVAL; memset(&fw_msg, 0, sizeof(fw_msg)); bnxt_re_init_hwrm_hdr(rdev, (void *)&req, HWRM_STAT_CTX_ALLOC, -1, tid); req.update_period_ms = cpu_to_le32(1000); req.stats_dma_length = rdev->chip_ctx->hw_stats_size; req.stats_dma_addr = cpu_to_le64(stat->dma_map); req.stat_ctx_flags = HWRM_STAT_CTX_ALLOC_INPUT_STAT_CTX_FLAGS_ROCE; bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp, sizeof(resp), DFLT_HWRM_CMD_TIMEOUT); rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg); if (rc) { dev_err(rdev_to_dev(rdev), "Failed to allocate HW stats ctx, rc = 0x%x", rc); return rc; } stat->fw_id = le32_to_cpu(resp.stat_ctx_id); dev_dbg(rdev_to_dev(rdev), "HW stats ctx allocated with id = 0x%x", stat->fw_id); return rc; } static void bnxt_re_net_unregister_async_event(struct bnxt_re_dev *rdev) { const struct bnxt_en_ops *en_ops; if (rdev->is_virtfn || test_bit(BNXT_RE_FLAG_ERR_DEVICE_DETACHED, &rdev->flags)) return; memset(rdev->event_bitmap, 0, sizeof(rdev->event_bitmap)); en_ops = rdev->en_dev->en_ops; if (en_ops->bnxt_register_fw_async_events (rdev->en_dev, BNXT_ROCE_ULP, (unsigned long *)rdev->event_bitmap, HWRM_ASYNC_EVENT_CMPL_EVENT_ID_DOORBELL_PACING_NQ_UPDATE)) dev_err(rdev_to_dev(rdev), "Failed to unregister async event"); } static void bnxt_re_net_register_async_event(struct bnxt_re_dev *rdev) { const struct bnxt_en_ops *en_ops; if (rdev->is_virtfn) return; rdev->event_bitmap[0] |= BIT(HWRM_ASYNC_EVENT_CMPL_EVENT_ID_DCB_CONFIG_CHANGE) | BIT(HWRM_ASYNC_EVENT_CMPL_EVENT_ID_RESET_NOTIFY); rdev->event_bitmap[2] |= BIT(HWRM_ASYNC_EVENT_CMPL_EVENT_ID_ERROR_REPORT - 64); rdev->event_bitmap[2] |= BIT(HWRM_ASYNC_EVENT_CMPL_EVENT_ID_DOORBELL_PACING_THRESHOLD - 64) | BIT(HWRM_ASYNC_EVENT_CMPL_EVENT_ID_DOORBELL_PACING_NQ_UPDATE - 64); en_ops = rdev->en_dev->en_ops; if (en_ops->bnxt_register_fw_async_events (rdev->en_dev, BNXT_ROCE_ULP, (unsigned long *)rdev->event_bitmap, HWRM_ASYNC_EVENT_CMPL_EVENT_ID_DOORBELL_PACING_NQ_UPDATE)) dev_err(rdev_to_dev(rdev), "Failed to reg Async event"); } static int bnxt_re_query_hwrm_intf_version(struct bnxt_re_dev *rdev) { struct bnxt_en_dev *en_dev = rdev->en_dev; struct hwrm_ver_get_output resp = {0}; struct hwrm_ver_get_input req = {0}; struct bnxt_qplib_chip_ctx *cctx; struct bnxt_fw_msg fw_msg; int rc = 0; memset(&fw_msg, 0, sizeof(fw_msg)); bnxt_re_init_hwrm_hdr(rdev, (void *)&req, HWRM_VER_GET, -1, -1); req.hwrm_intf_maj = HWRM_VERSION_MAJOR; req.hwrm_intf_min = HWRM_VERSION_MINOR; req.hwrm_intf_upd = HWRM_VERSION_UPDATE; bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp, sizeof(resp), DFLT_HWRM_CMD_TIMEOUT); rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg); if (rc) { dev_err(rdev_to_dev(rdev), "Failed to query HW version, rc = 0x%x", rc); return rc; } cctx = rdev->chip_ctx; cctx->hwrm_intf_ver = (u64) le16_to_cpu(resp.hwrm_intf_major) << 48 | (u64) le16_to_cpu(resp.hwrm_intf_minor) << 32 | (u64) le16_to_cpu(resp.hwrm_intf_build) << 16 | le16_to_cpu(resp.hwrm_intf_patch); cctx->hwrm_cmd_max_timeout = le16_to_cpu(resp.max_req_timeout); if (!cctx->hwrm_cmd_max_timeout) cctx->hwrm_cmd_max_timeout = RCFW_FW_STALL_MAX_TIMEOUT; cctx->chip_num = le16_to_cpu(resp.chip_num); cctx->chip_rev = resp.chip_rev; cctx->chip_metal = resp.chip_metal; return 0; } /* Query device config using common hwrm */ static int bnxt_re_hwrm_qcfg(struct bnxt_re_dev *rdev, u32 *db_len, u32 *offset) { struct bnxt_en_dev *en_dev = rdev->en_dev; struct hwrm_func_qcfg_output resp = {0}; struct hwrm_func_qcfg_input req = {0}; struct bnxt_fw_msg fw_msg; int rc; memset(&fw_msg, 0, sizeof(fw_msg)); bnxt_re_init_hwrm_hdr(rdev, (void *)&req, HWRM_FUNC_QCFG, -1, -1); req.fid = cpu_to_le16(0xffff); bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp, sizeof(resp), DFLT_HWRM_CMD_TIMEOUT); rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg); if (rc) { dev_err(rdev_to_dev(rdev), "Failed to query config, rc = %#x", rc); return rc; } *db_len = PAGE_ALIGN(le16_to_cpu(resp.l2_doorbell_bar_size_kb) * 1024); *offset = PAGE_ALIGN(le16_to_cpu(resp.legacy_l2_db_size_kb) * 1024); return 0; } /* Query function capabilities using common hwrm */ int bnxt_re_hwrm_qcaps(struct bnxt_re_dev *rdev) { struct bnxt_en_dev *en_dev = rdev->en_dev; struct hwrm_func_qcaps_output resp = {0}; struct hwrm_func_qcaps_input req = {0}; struct bnxt_qplib_chip_ctx *cctx; struct bnxt_fw_msg fw_msg; u8 push_enable = false; int rc; cctx = rdev->chip_ctx; memset(&fw_msg, 0, sizeof(fw_msg)); bnxt_re_init_hwrm_hdr(rdev, (void *)&req, HWRM_FUNC_QCAPS, -1, -1); req.fid = cpu_to_le16(0xffff); bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp, sizeof(resp), DFLT_HWRM_CMD_TIMEOUT); rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg); if (rc) { dev_err(rdev_to_dev(rdev), "Failed to query capabilities, rc = %#x", rc); return rc; } if (_is_chip_p7(rdev->chip_ctx)) push_enable = (resp.flags_ext & HWRM_FUNC_QCAPS_OUTPUT_FLAGS_EXT_PPP_PUSH_MODE_SUPPORTED) ? true : false; else push_enable = (resp.flags & HWRM_FUNC_QCAPS_OUTPUT_FLAGS_WCB_PUSH_MODE) ? true : false; cctx->modes.db_push = push_enable; cctx->modes.dbr_pacing = resp.flags_ext & HWRM_FUNC_QCAPS_OUTPUT_FLAGS_EXT_DBR_PACING_SUPPORTED ? true : false; cctx->modes.dbr_pacing_ext = resp.flags_ext2 & HWRM_FUNC_QCAPS_OUTPUT_FLAGS_EXT2_DBR_PACING_EXT_SUPPORTED ? true : false; cctx->modes.dbr_drop_recov = (resp.flags_ext2 & HWRM_FUNC_QCAPS_OUTPUT_FLAGS_EXT2_SW_DBR_DROP_RECOVERY_SUPPORTED) ? true : false; cctx->modes.dbr_pacing_v0 = (resp.flags_ext2 & HWRM_FUNC_QCAPS_OUTPUT_FLAGS_EXT2_DBR_PACING_V0_SUPPORTED) ? true : false; dev_dbg(rdev_to_dev(rdev), "%s: cctx->modes.dbr_pacing = %d cctx->modes.dbr_pacing_ext = %d, dbr_drop_recov %d\n", __func__, cctx->modes.dbr_pacing, cctx->modes.dbr_pacing_ext, cctx->modes.dbr_drop_recov); return 0; } static int bnxt_re_hwrm_dbr_pacing_qcfg(struct bnxt_re_dev *rdev) { struct bnxt_qplib_db_pacing_data *pacing_data = rdev->qplib_res.pacing_data; struct hwrm_func_dbr_pacing_qcfg_output resp = {0}; struct hwrm_func_dbr_pacing_qcfg_input req = {0}; struct bnxt_en_dev *en_dev = rdev->en_dev; struct bnxt_qplib_chip_ctx *cctx; struct bnxt_fw_msg fw_msg; u32 primary_nq_id; int rc; cctx = rdev->chip_ctx; memset(&fw_msg, 0, sizeof(fw_msg)); bnxt_re_init_hwrm_hdr(rdev, (void *)&req, HWRM_FUNC_DBR_PACING_QCFG, -1, -1); bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp, sizeof(resp), BNXT_RE_HWRM_CMD_TIMEOUT(rdev)); rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg); if (rc) { dev_dbg(rdev_to_dev(rdev), "Failed to query dbr pacing config, rc = %#x", rc); return rc; } primary_nq_id = le32_to_cpu(resp.primary_nq_id); if (primary_nq_id == 0xffffffff && !bnxt_qplib_dbr_pacing_ext_en(rdev->chip_ctx)) { dev_err(rdev_to_dev(rdev), "%s:%d Invoke bnxt_qplib_dbr_pacing_set_primary_pf with 1\n", __func__, __LINE__); bnxt_qplib_dbr_pacing_set_primary_pf(rdev->chip_ctx, 1); } if (bnxt_qplib_dbr_pacing_ext_en(rdev->chip_ctx)) { struct bnxt_qplib_nq *nq; nq = &rdev->nqr.nq[0]; /* Reset the primary capability */ if (nq->ring_id != primary_nq_id) bnxt_qplib_dbr_pacing_set_primary_pf(rdev->chip_ctx, 0); } if ((resp.dbr_stat_db_fifo_reg & HWRM_FUNC_DBR_PACING_QCFG_OUTPUT_DBR_STAT_DB_FIFO_REG_ADDR_SPACE_MASK) == HWRM_FUNC_DBR_PACING_QCFG_OUTPUT_DBR_STAT_DB_FIFO_REG_ADDR_SPACE_GRC) cctx->dbr_stat_db_fifo = resp.dbr_stat_db_fifo_reg & ~HWRM_FUNC_DBR_PACING_QCFG_OUTPUT_DBR_STAT_DB_FIFO_REG_ADDR_SPACE_MASK; if ((resp.dbr_throttling_aeq_arm_reg & HWRM_FUNC_DBR_PACING_QCFG_OUTPUT_DBR_THROTTLING_AEQ_ARM_REG_ADDR_SPACE_MASK) == HWRM_FUNC_DBR_PACING_QCFG_OUTPUT_DBR_THROTTLING_AEQ_ARM_REG_ADDR_SPACE_GRC) { cctx->dbr_aeq_arm_reg = resp.dbr_throttling_aeq_arm_reg & ~HWRM_FUNC_DBR_PACING_QCFG_OUTPUT_DBR_STAT_DB_FIFO_REG_ADDR_SPACE_MASK; cctx->dbr_throttling_reg = cctx->dbr_aeq_arm_reg - 4; } pacing_data->fifo_max_depth = le32_to_cpu(resp.dbr_stat_db_max_fifo_depth); if (!pacing_data->fifo_max_depth) pacing_data->fifo_max_depth = BNXT_RE_MAX_FIFO_DEPTH(cctx); pacing_data->fifo_room_mask = le32_to_cpu(resp.dbr_stat_db_fifo_reg_fifo_room_mask); pacing_data->fifo_room_shift = resp.dbr_stat_db_fifo_reg_fifo_room_shift; dev_dbg(rdev_to_dev(rdev), "%s: nq:0x%x primary_pf:%d db_fifo:0x%x aeq_arm:0x%x i" "fifo_max_depth 0x%x , resp.dbr_stat_db_max_fifo_depth 0x%x);\n", __func__, resp.primary_nq_id, cctx->modes.dbr_primary_pf, cctx->dbr_stat_db_fifo, cctx->dbr_aeq_arm_reg, pacing_data->fifo_max_depth, le32_to_cpu(resp.dbr_stat_db_max_fifo_depth)); return 0; } static int bnxt_re_hwrm_dbr_pacing_cfg(struct bnxt_re_dev *rdev, bool enable) { struct hwrm_func_dbr_pacing_cfg_output resp = {0}; struct hwrm_func_dbr_pacing_cfg_input req = {0}; struct bnxt_en_dev *en_dev = rdev->en_dev; struct bnxt_fw_msg fw_msg; int rc; if (test_bit(BNXT_RE_FLAG_ERR_DEVICE_DETACHED, &rdev->flags)) return 0; memset(&fw_msg, 0, sizeof(fw_msg)); bnxt_re_init_hwrm_hdr(rdev, (void *)&req, HWRM_FUNC_DBR_PACING_CFG, -1, -1); if (enable) { req.flags = HWRM_FUNC_DBR_PACING_CFG_INPUT_FLAGS_DBR_NQ_EVENT_ENABLE; req.enables = cpu_to_le32(HWRM_FUNC_DBR_PACING_CFG_INPUT_ENABLES_PRIMARY_NQ_ID_VALID | HWRM_FUNC_DBR_PACING_CFG_INPUT_ENABLES_PACING_THRESHOLD_VALID); } else { req.flags = HWRM_FUNC_DBR_PACING_CFG_INPUT_FLAGS_DBR_NQ_EVENT_DISABLE; } req.primary_nq_id = cpu_to_le32(rdev->dbq_nq_id); req.pacing_threshold = cpu_to_le32(rdev->dbq_watermark); dev_dbg(rdev_to_dev(rdev), "%s: nq_id = 0x%x pacing_threshold = 0x%x", __func__, req.primary_nq_id, req.pacing_threshold); bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp, sizeof(resp), BNXT_RE_HWRM_CMD_TIMEOUT(rdev)); rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg); if (rc) { dev_dbg(rdev_to_dev(rdev), "Failed to set dbr pacing config, rc = %#x", rc); return rc; } return 0; } /* Net -> RoCE driver */ /* Device */ struct bnxt_re_dev *bnxt_re_from_netdev(struct ifnet *netdev) { struct bnxt_re_dev *rdev; rcu_read_lock(); list_for_each_entry_rcu(rdev, &bnxt_re_dev_list, list) { if (rdev->netdev == netdev) { rcu_read_unlock(); dev_dbg(rdev_to_dev(rdev), "netdev (%p) found, ref_count = 0x%x", netdev, atomic_read(&rdev->ref_count)); return rdev; } } rcu_read_unlock(); return NULL; } static ssize_t show_rev(struct device *device, struct device_attribute *attr, char *buf) { struct bnxt_re_dev *rdev = to_bnxt_re_dev(device, ibdev.dev); return scnprintf(buf, PAGE_SIZE, "0x%x\n", rdev->en_dev->pdev->vendor); } static ssize_t show_hca(struct device *device, struct device_attribute *attr, char *buf) { struct bnxt_re_dev *rdev = to_bnxt_re_dev(device, ibdev.dev); return scnprintf(buf, PAGE_SIZE, "%s\n", rdev->ibdev.node_desc); } static DEVICE_ATTR(hw_rev, 0444, show_rev, NULL); static DEVICE_ATTR(hca_type, 0444, show_hca, NULL); static struct device_attribute *bnxt_re_attributes[] = { &dev_attr_hw_rev, &dev_attr_hca_type }; int ib_register_device_compat(struct bnxt_re_dev *rdev) { struct ib_device *ibdev = &rdev->ibdev; char name[IB_DEVICE_NAME_MAX]; memset(name, 0, IB_DEVICE_NAME_MAX); strlcpy(name, "bnxt_re%d", IB_DEVICE_NAME_MAX); strlcpy(ibdev->name, name, IB_DEVICE_NAME_MAX); return ib_register_device(ibdev, NULL); } static int bnxt_re_register_ib(struct bnxt_re_dev *rdev) { struct ib_device *ibdev = &rdev->ibdev; int ret = 0; /* ib device init */ ibdev->owner = THIS_MODULE; ibdev->uverbs_abi_ver = BNXT_RE_ABI_VERSION; ibdev->node_type = RDMA_NODE_IB_CA; strlcpy(ibdev->node_desc, BNXT_RE_DESC " HCA", strlen(BNXT_RE_DESC) + 5); ibdev->phys_port_cnt = 1; bnxt_qplib_get_guid(rdev->dev_addr, (u8 *)&ibdev->node_guid); /* Data path irqs is one less than the max msix vectors */ ibdev->num_comp_vectors = rdev->nqr.num_msix - 1; bnxt_re_set_dma_device(ibdev, rdev); ibdev->local_dma_lkey = BNXT_QPLIB_RSVD_LKEY; /* User space */ ibdev->uverbs_cmd_mask = (1ull << IB_USER_VERBS_CMD_GET_CONTEXT) | (1ull << IB_USER_VERBS_CMD_QUERY_DEVICE) | (1ull << IB_USER_VERBS_CMD_QUERY_PORT) | (1ull << IB_USER_VERBS_CMD_ALLOC_PD) | (1ull << IB_USER_VERBS_CMD_DEALLOC_PD) | (1ull << IB_USER_VERBS_CMD_REG_MR) | (1ull << IB_USER_VERBS_CMD_DEREG_MR) | (1ull << IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL) | (1ull << IB_USER_VERBS_CMD_CREATE_CQ) | (1ull << IB_USER_VERBS_CMD_DESTROY_CQ) | (1ull << IB_USER_VERBS_CMD_CREATE_QP) | (1ull << IB_USER_VERBS_CMD_MODIFY_QP) | (1ull << IB_USER_VERBS_CMD_QUERY_QP) | (1ull << IB_USER_VERBS_CMD_DESTROY_QP) | (1ull << IB_USER_VERBS_CMD_REREG_MR) | (1ull << IB_USER_VERBS_CMD_RESIZE_CQ) | (1ull << IB_USER_VERBS_CMD_CREATE_SRQ) | (1ull << IB_USER_VERBS_CMD_MODIFY_SRQ) | (1ull << IB_USER_VERBS_CMD_QUERY_SRQ) | (1ull << IB_USER_VERBS_CMD_DESTROY_SRQ) | (1ull << IB_USER_VERBS_CMD_ALLOC_MW) | (1ull << IB_USER_VERBS_CMD_DEALLOC_MW) | (1ull << IB_USER_VERBS_CMD_CREATE_AH) | (1ull << IB_USER_VERBS_CMD_MODIFY_AH) | (1ull << IB_USER_VERBS_CMD_QUERY_AH) | (1ull << IB_USER_VERBS_CMD_DESTROY_AH); ibdev->uverbs_ex_cmd_mask = (1ull << IB_USER_VERBS_EX_CMD_MODIFY_QP); ibdev->uverbs_cmd_mask |= (1ull << IB_USER_VERBS_CMD_POLL_CQ); #define bnxt_re_ib_ah bnxt_re_ah #define bnxt_re_ib_cq bnxt_re_cq #define bnxt_re_ib_pd bnxt_re_pd #define bnxt_re_ib_srq bnxt_re_srq #define bnxt_re_ib_ucontext bnxt_re_ucontext INIT_IB_DEVICE_OPS(&ibdev->ops, bnxt_re, BNXT_RE); ibdev->query_device = bnxt_re_query_device; ibdev->modify_device = bnxt_re_modify_device; ibdev->query_port = bnxt_re_query_port; ibdev->modify_port = bnxt_re_modify_port; ibdev->get_port_immutable = bnxt_re_get_port_immutable; ibdev->query_pkey = bnxt_re_query_pkey; ibdev->query_gid = bnxt_re_query_gid; ibdev->get_netdev = bnxt_re_get_netdev; ibdev->add_gid = bnxt_re_add_gid; ibdev->del_gid = bnxt_re_del_gid; ibdev->get_link_layer = bnxt_re_get_link_layer; ibdev->alloc_pd = bnxt_re_alloc_pd; ibdev->dealloc_pd = bnxt_re_dealloc_pd; ibdev->create_ah = bnxt_re_create_ah; ibdev->modify_ah = bnxt_re_modify_ah; ibdev->query_ah = bnxt_re_query_ah; ibdev->destroy_ah = bnxt_re_destroy_ah; ibdev->create_srq = bnxt_re_create_srq; ibdev->modify_srq = bnxt_re_modify_srq; ibdev->query_srq = bnxt_re_query_srq; ibdev->destroy_srq = bnxt_re_destroy_srq; ibdev->post_srq_recv = bnxt_re_post_srq_recv; ibdev->create_qp = bnxt_re_create_qp; ibdev->modify_qp = bnxt_re_modify_qp; ibdev->query_qp = bnxt_re_query_qp; ibdev->destroy_qp = bnxt_re_destroy_qp; ibdev->post_send = bnxt_re_post_send; ibdev->post_recv = bnxt_re_post_recv; ibdev->create_cq = bnxt_re_create_cq; ibdev->modify_cq = bnxt_re_modify_cq; ibdev->destroy_cq = bnxt_re_destroy_cq; ibdev->resize_cq = bnxt_re_resize_cq; ibdev->poll_cq = bnxt_re_poll_cq; ibdev->req_notify_cq = bnxt_re_req_notify_cq; ibdev->get_dma_mr = bnxt_re_get_dma_mr; ibdev->get_hw_stats = bnxt_re_get_hw_stats; ibdev->alloc_hw_stats = bnxt_re_alloc_hw_port_stats; ibdev->dereg_mr = bnxt_re_dereg_mr; ibdev->alloc_mr = bnxt_re_alloc_mr; ibdev->map_mr_sg = bnxt_re_map_mr_sg; ibdev->alloc_mw = bnxt_re_alloc_mw; ibdev->dealloc_mw = bnxt_re_dealloc_mw; ibdev->reg_user_mr = bnxt_re_reg_user_mr; ibdev->rereg_user_mr = bnxt_re_rereg_user_mr; ibdev->disassociate_ucontext = bnxt_re_disassociate_ucntx; ibdev->alloc_ucontext = bnxt_re_alloc_ucontext; ibdev->dealloc_ucontext = bnxt_re_dealloc_ucontext; ibdev->mmap = bnxt_re_mmap; ibdev->process_mad = bnxt_re_process_mad; ret = ib_register_device_compat(rdev); return ret; } static void bnxt_re_dev_dealloc(struct bnxt_re_dev *rdev) { int i = BNXT_RE_REF_WAIT_COUNT; dev_dbg(rdev_to_dev(rdev), "%s:Remove the device %p\n", __func__, rdev); /* Wait for rdev refcount to come down */ while ((atomic_read(&rdev->ref_count) > 1) && i--) msleep(100); if (atomic_read(&rdev->ref_count) > 1) dev_err(rdev_to_dev(rdev), "Failed waiting for ref count to deplete %d", atomic_read(&rdev->ref_count)); atomic_set(&rdev->ref_count, 0); if_rele(rdev->netdev); rdev->netdev = NULL; synchronize_rcu(); kfree(rdev->gid_map); kfree(rdev->dbg_stats); ib_dealloc_device(&rdev->ibdev); } static struct bnxt_re_dev *bnxt_re_dev_alloc(struct ifnet *netdev, struct bnxt_en_dev *en_dev) { struct bnxt_re_dev *rdev; u32 count; /* Allocate bnxt_re_dev instance here */ rdev = (struct bnxt_re_dev *)compat_ib_alloc_device(sizeof(*rdev)); if (!rdev) { pr_err("%s: bnxt_re_dev allocation failure!", ROCE_DRV_MODULE_NAME); return NULL; } /* Default values */ atomic_set(&rdev->ref_count, 0); rdev->netdev = netdev; dev_hold(rdev->netdev); rdev->en_dev = en_dev; rdev->id = rdev->en_dev->pdev->devfn; INIT_LIST_HEAD(&rdev->qp_list); mutex_init(&rdev->qp_lock); mutex_init(&rdev->cc_lock); mutex_init(&rdev->dbq_lock); bnxt_re_clear_rsors_stat(&rdev->stats.rsors); rdev->cosq[0] = rdev->cosq[1] = 0xFFFF; rdev->min_tx_depth = 1; rdev->stats.stats_query_sec = 1; /* Disable priority vlan as the default mode is DSCP based PFC */ rdev->cc_param.disable_prio_vlan_tx = 1; /* Initialize worker for DBR Pacing */ INIT_WORK(&rdev->dbq_fifo_check_work, bnxt_re_db_fifo_check); INIT_DELAYED_WORK(&rdev->dbq_pacing_work, bnxt_re_pacing_timer_exp); rdev->gid_map = kzalloc(sizeof(*(rdev->gid_map)) * BNXT_RE_MAX_SGID_ENTRIES, GFP_KERNEL); if (!rdev->gid_map) { ib_dealloc_device(&rdev->ibdev); return NULL; } for(count = 0; count < BNXT_RE_MAX_SGID_ENTRIES; count++) rdev->gid_map[count] = -1; rdev->dbg_stats = kzalloc(sizeof(*rdev->dbg_stats), GFP_KERNEL); if (!rdev->dbg_stats) { ib_dealloc_device(&rdev->ibdev); return NULL; } return rdev; } static int bnxt_re_handle_unaffi_async_event( struct creq_func_event *unaffi_async) { switch (unaffi_async->event) { case CREQ_FUNC_EVENT_EVENT_TX_WQE_ERROR: case CREQ_FUNC_EVENT_EVENT_TX_DATA_ERROR: case CREQ_FUNC_EVENT_EVENT_RX_WQE_ERROR: case CREQ_FUNC_EVENT_EVENT_RX_DATA_ERROR: case CREQ_FUNC_EVENT_EVENT_CQ_ERROR: case CREQ_FUNC_EVENT_EVENT_TQM_ERROR: case CREQ_FUNC_EVENT_EVENT_CFCQ_ERROR: case CREQ_FUNC_EVENT_EVENT_CFCS_ERROR: case CREQ_FUNC_EVENT_EVENT_CFCC_ERROR: case CREQ_FUNC_EVENT_EVENT_CFCM_ERROR: case CREQ_FUNC_EVENT_EVENT_TIM_ERROR: break; default: return -EINVAL; } return 0; } static int bnxt_re_handle_qp_async_event(void *qp_event, struct bnxt_re_qp *qp) { struct creq_qp_error_notification *err_event; struct ib_event event; unsigned int flags; if (qp->qplib_qp.state == CMDQ_MODIFY_QP_NEW_STATE_ERR && !qp->qplib_qp.is_user) { flags = bnxt_re_lock_cqs(qp); bnxt_qplib_add_flush_qp(&qp->qplib_qp); bnxt_re_unlock_cqs(qp, flags); } memset(&event, 0, sizeof(event)); event.device = &qp->rdev->ibdev; event.element.qp = &qp->ib_qp; event.event = IB_EVENT_QP_FATAL; err_event = qp_event; switch(err_event->res_err_state_reason) { case CFCQ_RES_ERR_STATE_REASON_RES_EXCEED_MAX: case CFCQ_RES_ERR_STATE_REASON_RES_PAYLOAD_LENGTH_MISMATCH: case CFCQ_RES_ERR_STATE_REASON_RES_OPCODE_ERROR: case CFCQ_RES_ERR_STATE_REASON_RES_PSN_SEQ_ERROR_RETRY_LIMIT: case CFCQ_RES_ERR_STATE_REASON_RES_RX_INVALID_R_KEY: case CFCQ_RES_ERR_STATE_REASON_RES_RX_DOMAIN_ERROR: case CFCQ_RES_ERR_STATE_REASON_RES_RX_NO_PERMISSION: case CFCQ_RES_ERR_STATE_REASON_RES_RX_RANGE_ERROR: case CFCQ_RES_ERR_STATE_REASON_RES_TX_INVALID_R_KEY: case CFCQ_RES_ERR_STATE_REASON_RES_TX_DOMAIN_ERROR: case CFCQ_RES_ERR_STATE_REASON_RES_TX_NO_PERMISSION: case CFCQ_RES_ERR_STATE_REASON_RES_TX_RANGE_ERROR: case CFCQ_RES_ERR_STATE_REASON_RES_IVALID_DUP_RKEY: case CFCQ_RES_ERR_STATE_REASON_RES_UNALIGN_ATOMIC: event.event = IB_EVENT_QP_ACCESS_ERR; break; case CFCQ_RES_ERR_STATE_REASON_RES_EXCEEDS_WQE: case CFCQ_RES_ERR_STATE_REASON_RES_WQE_FORMAT_ERROR: case CFCQ_RES_ERR_STATE_REASON_RES_SRQ_LOAD_ERROR: case CFCQ_RES_ERR_STATE_REASON_RES_UNSUPPORTED_OPCODE: case CFCQ_RES_ERR_STATE_REASON_RES_REM_INVALIDATE: event.event = IB_EVENT_QP_REQ_ERR; break; case CFCQ_RES_ERR_STATE_REASON_RES_IRRQ_OFLOW: case CFCQ_RES_ERR_STATE_REASON_RES_CMP_ERROR: case CFCQ_RES_ERR_STATE_REASON_RES_CQ_LOAD_ERROR: case CFCQ_RES_ERR_STATE_REASON_RES_TX_PCI_ERROR: case CFCQ_RES_ERR_STATE_REASON_RES_RX_PCI_ERROR: case CFCQ_RES_ERR_STATE_REASON_RES_MEMORY_ERROR: case CFCQ_RES_ERR_STATE_REASON_RES_SRQ_ERROR: event.event = IB_EVENT_QP_FATAL; break; default: if (qp->qplib_qp.srq) event.event = IB_EVENT_QP_LAST_WQE_REACHED; break; } if (err_event->res_err_state_reason) dev_err(rdev_to_dev(qp->rdev), "%s %s qp_id: %d cons (%d %d) req (%d %d) res (%d %d)\n", __func__, qp->qplib_qp.is_user ? "user" : "kernel", qp->qplib_qp.id, err_event->sq_cons_idx, err_event->rq_cons_idx, err_event->req_slow_path_state, err_event->req_err_state_reason, err_event->res_slow_path_state, err_event->res_err_state_reason); if (event.device && qp->ib_qp.event_handler) qp->ib_qp.event_handler(&event, qp->ib_qp.qp_context); return 0; } static int bnxt_re_handle_cq_async_error(void *event, struct bnxt_re_cq *cq) { struct creq_cq_error_notification *cqerr; bool send = false; cqerr = event; switch (cqerr->cq_err_reason) { case CREQ_CQ_ERROR_NOTIFICATION_CQ_ERR_REASON_REQ_CQ_INVALID_ERROR: case CREQ_CQ_ERROR_NOTIFICATION_CQ_ERR_REASON_REQ_CQ_OVERFLOW_ERROR: case CREQ_CQ_ERROR_NOTIFICATION_CQ_ERR_REASON_REQ_CQ_LOAD_ERROR: case CREQ_CQ_ERROR_NOTIFICATION_CQ_ERR_REASON_RES_CQ_INVALID_ERROR: case CREQ_CQ_ERROR_NOTIFICATION_CQ_ERR_REASON_RES_CQ_OVERFLOW_ERROR: case CREQ_CQ_ERROR_NOTIFICATION_CQ_ERR_REASON_RES_CQ_LOAD_ERROR: send = true; default: break; } if (send && cq->ibcq.event_handler) { struct ib_event ibevent = {}; ibevent.event = IB_EVENT_CQ_ERR; ibevent.element.cq = &cq->ibcq; ibevent.device = &cq->rdev->ibdev; dev_err(rdev_to_dev(cq->rdev), "%s err reason %d\n", __func__, cqerr->cq_err_reason); cq->ibcq.event_handler(&ibevent, cq->ibcq.cq_context); } cq->qplib_cq.is_cq_err_event = true; return 0; } static int bnxt_re_handle_affi_async_event(struct creq_qp_event *affi_async, void *obj) { struct bnxt_qplib_qp *qplqp; struct bnxt_qplib_cq *qplcq; struct bnxt_re_qp *qp; struct bnxt_re_cq *cq; int rc = 0; u8 event; if (!obj) return rc; /* QP was already dead, still return success */ event = affi_async->event; switch (event) { case CREQ_QP_EVENT_EVENT_QP_ERROR_NOTIFICATION: qplqp = obj; qp = container_of(qplqp, struct bnxt_re_qp, qplib_qp); rc = bnxt_re_handle_qp_async_event(affi_async, qp); break; case CREQ_QP_EVENT_EVENT_CQ_ERROR_NOTIFICATION: qplcq = obj; cq = container_of(qplcq, struct bnxt_re_cq, qplib_cq); rc = bnxt_re_handle_cq_async_error(affi_async, cq); break; default: rc = -EINVAL; } return rc; } static int bnxt_re_aeq_handler(struct bnxt_qplib_rcfw *rcfw, void *aeqe, void *obj) { struct creq_func_event *unaffi_async; struct creq_qp_event *affi_async; u8 type; int rc; type = ((struct creq_base *)aeqe)->type; if (type == CREQ_BASE_TYPE_FUNC_EVENT) { unaffi_async = aeqe; rc = bnxt_re_handle_unaffi_async_event(unaffi_async); } else { affi_async = aeqe; rc = bnxt_re_handle_affi_async_event(affi_async, obj); } return rc; } static int bnxt_re_srqn_handler(struct bnxt_qplib_nq *nq, struct bnxt_qplib_srq *handle, u8 event) { struct bnxt_re_srq *srq = to_bnxt_re(handle, struct bnxt_re_srq, qplib_srq); struct ib_event ib_event; if (srq == NULL) { pr_err("%s: SRQ is NULL, SRQN not handled", ROCE_DRV_MODULE_NAME); return -EINVAL; } ib_event.device = &srq->rdev->ibdev; ib_event.element.srq = &srq->ibsrq; if (event == NQ_SRQ_EVENT_EVENT_SRQ_THRESHOLD_EVENT) ib_event.event = IB_EVENT_SRQ_LIMIT_REACHED; else ib_event.event = IB_EVENT_SRQ_ERR; if (srq->ibsrq.event_handler) { /* Lock event_handler? */ (*srq->ibsrq.event_handler)(&ib_event, srq->ibsrq.srq_context); } return 0; } static int bnxt_re_cqn_handler(struct bnxt_qplib_nq *nq, struct bnxt_qplib_cq *handle) { struct bnxt_re_cq *cq = to_bnxt_re(handle, struct bnxt_re_cq, qplib_cq); u32 *cq_ptr; if (cq == NULL) { pr_err("%s: CQ is NULL, CQN not handled", ROCE_DRV_MODULE_NAME); return -EINVAL; } /* CQ already in destroy path. Do not handle any more events */ if (handle->destroyed || !atomic_read(&cq->ibcq.usecnt)) { if (!handle->destroyed) dev_dbg(NULL, "%s: CQ being destroyed, CQN not handled", ROCE_DRV_MODULE_NAME); return 0; } if (cq->ibcq.comp_handler) { if (cq->uctx_cq_page) { cq_ptr = (u32 *)cq->uctx_cq_page; *cq_ptr = cq->qplib_cq.toggle; } /* Lock comp_handler? */ (*cq->ibcq.comp_handler)(&cq->ibcq, cq->ibcq.cq_context); } return 0; } struct bnxt_qplib_nq *bnxt_re_get_nq(struct bnxt_re_dev *rdev) { int min, indx; mutex_lock(&rdev->nqr.load_lock); for (indx = 0, min = 0; indx < (rdev->nqr.num_msix - 1); indx++) { if (rdev->nqr.nq[min].load > rdev->nqr.nq[indx].load) min = indx; } rdev->nqr.nq[min].load++; mutex_unlock(&rdev->nqr.load_lock); return &rdev->nqr.nq[min]; } void bnxt_re_put_nq(struct bnxt_re_dev *rdev, struct bnxt_qplib_nq *nq) { mutex_lock(&rdev->nqr.load_lock); nq->load--; mutex_unlock(&rdev->nqr.load_lock); } static bool bnxt_re_check_min_attr(struct bnxt_re_dev *rdev) { struct bnxt_qplib_dev_attr *attr; bool rc = true; attr = rdev->dev_attr; if (!attr->max_cq || !attr->max_qp || !attr->max_sgid || !attr->max_mr) { dev_err(rdev_to_dev(rdev),"Insufficient RoCE resources"); dev_dbg(rdev_to_dev(rdev), "max_cq = %d, max_qp = %d, max_dpi = %d, max_sgid = %d, max_mr = %d", attr->max_cq, attr->max_qp, attr->max_dpi, attr->max_sgid, attr->max_mr); rc = false; } return rc; } static void bnxt_re_dispatch_event(struct ib_device *ibdev, struct ib_qp *qp, u8 port_num, enum ib_event_type event) { struct ib_event ib_event; ib_event.device = ibdev; if (qp) { ib_event.element.qp = qp; ib_event.event = event; if (qp->event_handler) qp->event_handler(&ib_event, qp->qp_context); } else { ib_event.element.port_num = port_num; ib_event.event = event; ib_dispatch_event(&ib_event); } dev_dbg(rdev_to_dev(to_bnxt_re_dev(ibdev, ibdev)), "ibdev %p Event 0x%x port_num 0x%x", ibdev, event, port_num); } static bool bnxt_re_is_qp1_or_shadow_qp(struct bnxt_re_dev *rdev, struct bnxt_re_qp *qp) { if (rdev->gsi_ctx.gsi_qp_mode == BNXT_RE_GSI_MODE_ALL) return (qp->ib_qp.qp_type == IB_QPT_GSI) || (qp == rdev->gsi_ctx.gsi_sqp); else return (qp->ib_qp.qp_type == IB_QPT_GSI); } static void bnxt_re_stop_all_nonqp1_nonshadow_qps(struct bnxt_re_dev *rdev) { struct bnxt_qplib_qp *qpl_qp; bool dev_detached = false; struct ib_qp_attr qp_attr; int num_qps_stopped = 0; int mask = IB_QP_STATE; struct bnxt_re_qp *qp; unsigned long flags; if (!rdev) return; restart: if (test_bit(BNXT_RE_FLAG_ERR_DEVICE_DETACHED, &rdev->flags)) dev_detached = true; qp_attr.qp_state = IB_QPS_ERR; mutex_lock(&rdev->qp_lock); list_for_each_entry(qp, &rdev->qp_list, list) { qpl_qp = &qp->qplib_qp; if (dev_detached || !bnxt_re_is_qp1_or_shadow_qp(rdev, qp)) { if (qpl_qp->state != CMDQ_MODIFY_QP_NEW_STATE_RESET && qpl_qp->state != CMDQ_MODIFY_QP_NEW_STATE_ERR) { if (dev_detached) { /* * Cant actually send the command down, * marking the state for bookkeeping */ qpl_qp->state = CMDQ_MODIFY_QP_NEW_STATE_ERR; qpl_qp->cur_qp_state = qpl_qp->state; if (!qpl_qp->is_user) { /* Add to flush list */ flags = bnxt_re_lock_cqs(qp); bnxt_qplib_add_flush_qp(qpl_qp); bnxt_re_unlock_cqs(qp, flags); } } else { num_qps_stopped++; bnxt_re_modify_qp(&qp->ib_qp, &qp_attr, mask, NULL); } bnxt_re_dispatch_event(&rdev->ibdev, &qp->ib_qp, 1, IB_EVENT_QP_FATAL); /* * 1. Release qp_lock after a budget to unblock other verb * requests (like qp_destroy) from stack. * 2. Traverse through the qp_list freshly as addition / deletion * might have happened since qp_lock is getting released here. */ if (num_qps_stopped % BNXT_RE_STOP_QPS_BUDGET == 0) { mutex_unlock(&rdev->qp_lock); goto restart; } } } } mutex_unlock(&rdev->qp_lock); } static int bnxt_re_update_gid(struct bnxt_re_dev *rdev) { struct bnxt_qplib_sgid_tbl *sgid_tbl = &rdev->qplib_res.sgid_tbl; struct bnxt_qplib_gid gid; u16 gid_idx, index; int rc = 0; if (!test_bit(BNXT_RE_FLAG_IBDEV_REGISTERED, &rdev->flags)) return 0; if (sgid_tbl == NULL) { dev_err(rdev_to_dev(rdev), "QPLIB: SGID table not allocated"); return -EINVAL; } for (index = 0; index < sgid_tbl->active; index++) { gid_idx = sgid_tbl->hw_id[index]; if (!memcmp(&sgid_tbl->tbl[index], &bnxt_qplib_gid_zero, sizeof(bnxt_qplib_gid_zero))) continue; /* Need to modify the VLAN enable setting of non VLAN GID only * as setting is done for VLAN GID while adding GID * * If disable_prio_vlan_tx is enable, then we'll need to remove the * vlan entry from the sgid_tbl. */ if (sgid_tbl->vlan[index] == true) continue; memcpy(&gid, &sgid_tbl->tbl[index], sizeof(gid)); rc = bnxt_qplib_update_sgid(sgid_tbl, &gid, gid_idx, rdev->dev_addr); } return rc; } static void bnxt_re_clear_cc(struct bnxt_re_dev *rdev) { struct bnxt_qplib_cc_param *cc_param = &rdev->cc_param; if (_is_chip_p7(rdev->chip_ctx)) { cc_param->mask = CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_TOS_DSCP; } else { cc_param->mask = (CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_CC_MODE | CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_ENABLE_CC | CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_TOS_ECN); if (!is_qport_service_type_supported(rdev)) cc_param->mask |= (CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_ALT_VLAN_PCP | CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_ALT_TOS_DSCP | CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_TOS_DSCP); } cc_param->cur_mask = cc_param->mask; if (bnxt_qplib_modify_cc(&rdev->qplib_res, cc_param)) dev_err(rdev_to_dev(rdev), "Failed to modify cc\n"); } static int bnxt_re_setup_cc(struct bnxt_re_dev *rdev) { struct bnxt_qplib_cc_param *cc_param = &rdev->cc_param; int rc; if (_is_chip_p7(rdev->chip_ctx)) { cc_param->enable = 0x0; cc_param->mask = CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_TOS_DSCP; } else { cc_param->enable = 0x1; cc_param->mask = (CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_CC_MODE | CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_ENABLE_CC | CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_TOS_ECN); if (!is_qport_service_type_supported(rdev)) cc_param->mask |= (CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_ALT_VLAN_PCP | CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_ALT_TOS_DSCP | CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_TOS_DSCP); } cc_param->cur_mask = cc_param->mask; rc = bnxt_qplib_modify_cc(&rdev->qplib_res, cc_param); if (rc) { dev_err(rdev_to_dev(rdev), "Failed to modify cc\n"); return rc; } /* Reset the programming mask */ cc_param->mask = 0; if (cc_param->qp1_tos_dscp != cc_param->tos_dscp) { cc_param->qp1_tos_dscp = cc_param->tos_dscp; rc = bnxt_re_update_qp1_tos_dscp(rdev); if (rc) { dev_err(rdev_to_dev(rdev), "%s:Failed to modify QP1:%d", __func__, rc); goto clear; } } return 0; clear: bnxt_re_clear_cc(rdev); return rc; } int bnxt_re_query_hwrm_dscp2pri(struct bnxt_re_dev *rdev, struct bnxt_re_dscp2pri *d2p, u16 *count, u16 target_id) { struct bnxt_en_dev *en_dev = rdev->en_dev; struct hwrm_queue_dscp2pri_qcfg_input req; struct hwrm_queue_dscp2pri_qcfg_output resp; struct bnxt_re_dscp2pri *dscp2pri; struct bnxt_fw_msg fw_msg; u16 in_count = *count; dma_addr_t dma_handle; int rc = 0, i; u16 data_len; u8 *kmem; data_len = *count * sizeof(*dscp2pri); memset(&fw_msg, 0, sizeof(fw_msg)); memset(&req, 0, sizeof(req)); bnxt_re_init_hwrm_hdr(rdev, (void *)&req, HWRM_QUEUE_DSCP2PRI_QCFG, -1, target_id); req.port_id = (target_id == 0xFFFF) ? en_dev->pf_port_id : 1; kmem = dma_zalloc_coherent(&en_dev->pdev->dev, data_len, &dma_handle, GFP_KERNEL); if (!kmem) { dev_err(rdev_to_dev(rdev), "dma_zalloc_coherent failure, length = %u\n", (unsigned)data_len); return -ENOMEM; } req.dest_data_addr = cpu_to_le64(dma_handle); req.dest_data_buffer_size = cpu_to_le16(data_len); bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp, sizeof(resp), DFLT_HWRM_CMD_TIMEOUT); rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg); if (rc) goto out; /* Upload the DSCP-MASK-PRI tuple(s) */ dscp2pri = (struct bnxt_re_dscp2pri *)kmem; for (i = 0; i < le16_to_cpu(resp.entry_cnt) && i < in_count; i++) { d2p[i].dscp = dscp2pri->dscp; d2p[i].mask = dscp2pri->mask; d2p[i].pri = dscp2pri->pri; dscp2pri++; } *count = le16_to_cpu(resp.entry_cnt); out: dma_free_coherent(&en_dev->pdev->dev, data_len, kmem, dma_handle); return rc; } int bnxt_re_prio_vlan_tx_update(struct bnxt_re_dev *rdev) { /* Remove the VLAN from the GID entry */ if (rdev->cc_param.disable_prio_vlan_tx) rdev->qplib_res.prio = false; else rdev->qplib_res.prio = true; return bnxt_re_update_gid(rdev); } int bnxt_re_set_hwrm_dscp2pri(struct bnxt_re_dev *rdev, struct bnxt_re_dscp2pri *d2p, u16 count, u16 target_id) { struct bnxt_en_dev *en_dev = rdev->en_dev; struct hwrm_queue_dscp2pri_cfg_input req; struct hwrm_queue_dscp2pri_cfg_output resp; struct bnxt_fw_msg fw_msg; struct bnxt_re_dscp2pri *dscp2pri; int i, rc, data_len = 3 * 256; dma_addr_t dma_handle; u8 *kmem; memset(&req, 0, sizeof(req)); memset(&fw_msg, 0, sizeof(fw_msg)); bnxt_re_init_hwrm_hdr(rdev, (void *)&req, HWRM_QUEUE_DSCP2PRI_CFG, -1, target_id); req.port_id = (target_id == 0xFFFF) ? en_dev->pf_port_id : 1; kmem = dma_alloc_coherent(&en_dev->pdev->dev, data_len, &dma_handle, GFP_KERNEL); if (!kmem) { dev_err(rdev_to_dev(rdev), "dma_alloc_coherent failure, length = %u\n", (unsigned)data_len); return -ENOMEM; } req.src_data_addr = cpu_to_le64(dma_handle); /* Download the DSCP-MASK-PRI tuple(s) */ dscp2pri = (struct bnxt_re_dscp2pri *)kmem; for (i = 0; i < count; i++) { dscp2pri->dscp = d2p[i].dscp; dscp2pri->mask = d2p[i].mask; dscp2pri->pri = d2p[i].pri; dscp2pri++; } req.entry_cnt = cpu_to_le16(count); bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp, sizeof(resp), DFLT_HWRM_CMD_TIMEOUT); rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg); dma_free_coherent(&en_dev->pdev->dev, data_len, kmem, dma_handle); return rc; } int bnxt_re_query_hwrm_qportcfg(struct bnxt_re_dev *rdev, struct bnxt_re_tc_rec *tc_rec, u16 tid) { u8 max_tc, tc, *qptr, *type_ptr0, *type_ptr1; struct hwrm_queue_qportcfg_output resp = {0}; struct hwrm_queue_qportcfg_input req = {0}; struct bnxt_en_dev *en_dev = rdev->en_dev; struct bnxt_fw_msg fw_msg; bool def_init = false; u8 *tmp_type; u8 cos_id; int rc; memset(&fw_msg, 0, sizeof(fw_msg)); bnxt_re_init_hwrm_hdr(rdev, (void *)&req, HWRM_QUEUE_QPORTCFG, -1, tid); req.port_id = (tid == 0xFFFF) ? en_dev->pf_port_id : 1; if (BNXT_EN_ASYM_Q(en_dev)) req.flags = htole32(HWRM_QUEUE_QPORTCFG_INPUT_FLAGS_PATH_RX); bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp, sizeof(resp), DFLT_HWRM_CMD_TIMEOUT); rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg); if (rc) return rc; if (!resp.max_configurable_queues) return -EINVAL; max_tc = resp.max_configurable_queues; tc_rec->max_tc = max_tc; if (resp.queue_cfg_info & HWRM_QUEUE_QPORTCFG_OUTPUT_QUEUE_CFG_INFO_USE_PROFILE_TYPE) tc_rec->serv_type_enabled = true; qptr = &resp.queue_id0; type_ptr0 = &resp.queue_id0_service_profile_type; type_ptr1 = &resp.queue_id1_service_profile_type; for (tc = 0; tc < max_tc; tc++) { tmp_type = tc ? type_ptr1 + (tc - 1) : type_ptr0; cos_id = *qptr++; /* RoCE CoS queue is the first cos queue. * For MP12 and MP17 order is 405 and 141015. */ if (is_bnxt_roce_queue(rdev, *qptr, *tmp_type)) { tc_rec->cos_id_roce = cos_id; tc_rec->tc_roce = tc; } else if (is_bnxt_cnp_queue(rdev, *qptr, *tmp_type)) { tc_rec->cos_id_cnp = cos_id; tc_rec->tc_cnp = tc; } else if (!def_init) { def_init = true; tc_rec->tc_def = tc; tc_rec->cos_id_def = cos_id; } qptr++; } return rc; } int bnxt_re_hwrm_cos2bw_qcfg(struct bnxt_re_dev *rdev, u16 target_id, struct bnxt_re_cos2bw_cfg *cfg) { struct bnxt_en_dev *en_dev = rdev->en_dev; struct hwrm_queue_cos2bw_qcfg_output resp; struct hwrm_queue_cos2bw_qcfg_input req = {0}; struct bnxt_fw_msg fw_msg; int rc, indx; void *data; memset(&fw_msg, 0, sizeof(fw_msg)); bnxt_re_init_hwrm_hdr(rdev, (void *)&req, HWRM_QUEUE_COS2BW_QCFG, -1, target_id); req.port_id = (target_id == 0xFFFF) ? en_dev->pf_port_id : 1; bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp, sizeof(resp), DFLT_HWRM_CMD_TIMEOUT); rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg); if (rc) return rc; data = &resp.queue_id0 + offsetof(struct bnxt_re_cos2bw_cfg, queue_id); for (indx = 0; indx < 8; indx++, data += (sizeof(cfg->cfg))) { memcpy(&cfg->cfg, data, sizeof(cfg->cfg)); if (indx == 0) cfg->queue_id = resp.queue_id0; cfg++; } return rc; } int bnxt_re_hwrm_cos2bw_cfg(struct bnxt_re_dev *rdev, u16 target_id, struct bnxt_re_cos2bw_cfg *cfg) { struct bnxt_en_dev *en_dev = rdev->en_dev; struct hwrm_queue_cos2bw_cfg_input req = {0}; struct hwrm_queue_cos2bw_cfg_output resp = {0}; struct bnxt_fw_msg fw_msg; void *data; int indx; int rc; memset(&fw_msg, 0, sizeof(fw_msg)); bnxt_re_init_hwrm_hdr(rdev, (void *)&req, HWRM_QUEUE_COS2BW_CFG, -1, target_id); req.port_id = (target_id == 0xFFFF) ? en_dev->pf_port_id : 1; /* Chimp wants enable bit to retain previous * config done by L2 driver */ for (indx = 0; indx < 8; indx++) { if (cfg[indx].queue_id < 40) { req.enables |= cpu_to_le32( HWRM_QUEUE_COS2BW_CFG_INPUT_ENABLES_COS_QUEUE_ID0_VALID << indx); } data = (char *)&req.unused_0 + indx * (sizeof(*cfg) - 4); memcpy(data, &cfg[indx].queue_id, sizeof(*cfg) - 4); if (indx == 0) { req.queue_id0 = cfg[0].queue_id; req.unused_0 = 0; } } memset(&resp, 0, sizeof(resp)); bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp, sizeof(resp), DFLT_HWRM_CMD_TIMEOUT); rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg); return rc; } int bnxt_re_host_pf_id_query(struct bnxt_re_dev *rdev, struct bnxt_qplib_query_fn_info *fn_info, u32 *pf_mask, u32 *first_pf) { struct hwrm_func_host_pf_ids_query_output resp = {0}; struct hwrm_func_host_pf_ids_query_input req; struct bnxt_en_dev *en_dev = rdev->en_dev; struct bnxt_fw_msg fw_msg; int rc; memset(&fw_msg, 0, sizeof(fw_msg)); memset(&req, 0, sizeof(req)); bnxt_re_init_hwrm_hdr(rdev, (void *)&req, HWRM_FUNC_HOST_PF_IDS_QUERY, -1, -1); /* To query the info from the host EPs */ switch (fn_info->host) { case HWRM_FUNC_HOST_PF_IDS_QUERY_INPUT_HOST_SOC: case HWRM_FUNC_HOST_PF_IDS_QUERY_INPUT_HOST_EP_0: case HWRM_FUNC_HOST_PF_IDS_QUERY_INPUT_HOST_EP_1: case HWRM_FUNC_HOST_PF_IDS_QUERY_INPUT_HOST_EP_2: case HWRM_FUNC_HOST_PF_IDS_QUERY_INPUT_HOST_EP_3: req.host = fn_info->host; break; default: req.host = HWRM_FUNC_HOST_PF_IDS_QUERY_INPUT_HOST_EP_0; break; } req.filter = fn_info->filter; if (req.filter > HWRM_FUNC_HOST_PF_IDS_QUERY_INPUT_FILTER_ROCE) req.filter = HWRM_FUNC_HOST_PF_IDS_QUERY_INPUT_FILTER_ALL; bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp, sizeof(resp), DFLT_HWRM_CMD_TIMEOUT); rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg); *first_pf = le16_to_cpu(resp.first_pf_id); *pf_mask = le16_to_cpu(resp.pf_ordinal_mask); return rc; } static void bnxt_re_put_stats_ctx(struct bnxt_re_dev *rdev) { struct bnxt_qplib_ctx *hctx; struct bnxt_qplib_res *res; u16 tid = 0xffff; res = &rdev->qplib_res; hctx = res->hctx; if (test_and_clear_bit(BNXT_RE_FLAG_STATS_CTX_ALLOC, &rdev->flags)) { bnxt_re_net_stats_ctx_free(rdev, hctx->stats.fw_id, tid); bnxt_qplib_free_stat_mem(res, &hctx->stats); } } static void bnxt_re_put_stats2_ctx(struct bnxt_re_dev *rdev) { test_and_clear_bit(BNXT_RE_FLAG_STATS_CTX2_ALLOC, &rdev->flags); } static int bnxt_re_get_stats_ctx(struct bnxt_re_dev *rdev) { struct bnxt_qplib_ctx *hctx; struct bnxt_qplib_res *res; u16 tid = 0xffff; int rc; res = &rdev->qplib_res; hctx = res->hctx; rc = bnxt_qplib_alloc_stat_mem(res->pdev, rdev->chip_ctx, &hctx->stats); if (rc) return -ENOMEM; rc = bnxt_re_net_stats_ctx_alloc(rdev, tid); if (rc) goto free_stat_mem; set_bit(BNXT_RE_FLAG_STATS_CTX_ALLOC, &rdev->flags); return 0; free_stat_mem: bnxt_qplib_free_stat_mem(res, &hctx->stats); return rc; } static int bnxt_re_update_dev_attr(struct bnxt_re_dev *rdev) { int rc; rc = bnxt_qplib_get_dev_attr(&rdev->rcfw); if (rc) return rc; if (!bnxt_re_check_min_attr(rdev)) return -EINVAL; return 0; } static void bnxt_re_free_tbls(struct bnxt_re_dev *rdev) { bnxt_qplib_clear_tbls(&rdev->qplib_res); bnxt_qplib_free_tbls(&rdev->qplib_res); } static int bnxt_re_alloc_init_tbls(struct bnxt_re_dev *rdev) { struct bnxt_qplib_chip_ctx *chip_ctx = rdev->chip_ctx; u8 pppp_factor = 0; int rc; /* * TODO: Need a better mechanism for spreading of the * 512 extended PPP pages. For now, spreading it * based on port_count */ if (_is_chip_p7(chip_ctx) && chip_ctx->modes.db_push) pppp_factor = rdev->en_dev->port_count; rc = bnxt_qplib_alloc_tbls(&rdev->qplib_res, pppp_factor); if (rc) return rc; bnxt_qplib_init_tbls(&rdev->qplib_res); set_bit(BNXT_RE_FLAG_TBLS_ALLOCINIT, &rdev->flags); return 0; } static void bnxt_re_clean_nqs(struct bnxt_re_dev *rdev) { struct bnxt_qplib_nq *nq; int i; if (!rdev->nqr.max_init) return; for (i = (rdev->nqr.max_init - 1) ; i >= 0; i--) { nq = &rdev->nqr.nq[i]; bnxt_qplib_disable_nq(nq); bnxt_re_net_ring_free(rdev, nq->ring_id); bnxt_qplib_free_nq_mem(nq); } rdev->nqr.max_init = 0; } static int bnxt_re_setup_nqs(struct bnxt_re_dev *rdev) { struct bnxt_re_ring_attr rattr = {}; struct bnxt_qplib_nq *nq; int rc, i; int depth; u32 offt; u16 vec; mutex_init(&rdev->nqr.load_lock); /* * TODO: Optimize the depth based on the * number of NQs. */ depth = BNXT_QPLIB_NQE_MAX_CNT; for (i = 0; i < rdev->nqr.num_msix - 1; i++) { nq = &rdev->nqr.nq[i]; vec = rdev->nqr.msix_entries[i + 1].vector; offt = rdev->nqr.msix_entries[i + 1].db_offset; nq->hwq.max_elements = depth; rc = bnxt_qplib_alloc_nq_mem(&rdev->qplib_res, nq); if (rc) { dev_err(rdev_to_dev(rdev), "Failed to get mem for NQ %d, rc = 0x%x", i, rc); goto fail_mem; } rattr.dma_arr = nq->hwq.pbl[PBL_LVL_0].pg_map_arr; rattr.pages = nq->hwq.pbl[rdev->nqr.nq[i].hwq.level].pg_count; rattr.type = bnxt_re_get_rtype(rdev); rattr.mode = HWRM_RING_ALLOC_INPUT_INT_MODE_MSIX; rattr.depth = nq->hwq.max_elements - 1; rattr.lrid = rdev->nqr.msix_entries[i + 1].ring_idx; /* Set DBR pacing capability on the first NQ ring only */ if (!i && bnxt_qplib_dbr_pacing_ext_en(rdev->chip_ctx)) rattr.flags = HWRM_RING_ALLOC_INPUT_FLAGS_NQ_DBR_PACING; else rattr.flags = 0; rc = bnxt_re_net_ring_alloc(rdev, &rattr, &nq->ring_id); if (rc) { nq->ring_id = 0xffff; /* Invalid ring-id */ dev_err(rdev_to_dev(rdev), "Failed to get fw id for NQ %d, rc = 0x%x", i, rc); goto fail_ring; } rc = bnxt_qplib_enable_nq(nq, i, vec, offt, &bnxt_re_cqn_handler, &bnxt_re_srqn_handler); if (rc) { dev_err(rdev_to_dev(rdev), "Failed to enable NQ %d, rc = 0x%x", i, rc); goto fail_en; } } rdev->nqr.max_init = i; return 0; fail_en: /* *nq was i'th nq */ bnxt_re_net_ring_free(rdev, nq->ring_id); fail_ring: bnxt_qplib_free_nq_mem(nq); fail_mem: rdev->nqr.max_init = i; return rc; } static void bnxt_re_sysfs_destroy_file(struct bnxt_re_dev *rdev) { int i; for (i = 0; i < ARRAY_SIZE(bnxt_re_attributes); i++) device_remove_file(&rdev->ibdev.dev, bnxt_re_attributes[i]); } static int bnxt_re_sysfs_create_file(struct bnxt_re_dev *rdev) { int i, j, rc = 0; for (i = 0; i < ARRAY_SIZE(bnxt_re_attributes); i++) { rc = device_create_file(&rdev->ibdev.dev, bnxt_re_attributes[i]); if (rc) { dev_err(rdev_to_dev(rdev), "Failed to create IB sysfs with rc = 0x%x", rc); /* Must clean up all created device files */ for (j = 0; j < i; j++) device_remove_file(&rdev->ibdev.dev, bnxt_re_attributes[j]); clear_bit(BNXT_RE_FLAG_IBDEV_REGISTERED, &rdev->flags); ib_unregister_device(&rdev->ibdev); return 1; } } return 0; } /* worker thread for polling periodic events. Now used for QoS programming*/ static void bnxt_re_worker(struct work_struct *work) { struct bnxt_re_dev *rdev = container_of(work, struct bnxt_re_dev, worker.work); int rc; /* QoS is in 30s cadence for PFs*/ if (!rdev->is_virtfn && !rdev->worker_30s--) rdev->worker_30s = 30; /* Use trylock for bnxt_re_dev_lock as this can be * held for long time by debugfs show path while issuing * HWRMS. If the debugfs name update is not done in this * iteration, the driver will check for the same in the * next schedule of the worker i.e after 1 sec. */ if (mutex_trylock(&bnxt_re_dev_lock)) mutex_unlock(&bnxt_re_dev_lock); if (!rdev->stats.stats_query_sec) goto resched; if (test_bit(BNXT_RE_FLAG_ISSUE_CFA_FLOW_STATS, &rdev->flags) && (rdev->is_virtfn || !_is_ext_stats_supported(rdev->dev_attr->dev_cap_flags))) { if (!(rdev->stats.stats_query_counter++ % rdev->stats.stats_query_sec)) { rc = bnxt_re_get_qos_stats(rdev); if (rc && rc != -ENOMEM) clear_bit(BNXT_RE_FLAG_ISSUE_CFA_FLOW_STATS, &rdev->flags); } } resched: schedule_delayed_work(&rdev->worker, msecs_to_jiffies(1000)); } static int bnxt_re_alloc_dbr_sw_stats_mem(struct bnxt_re_dev *rdev) { if (!(rdev->dbr_drop_recov || rdev->dbr_pacing)) return 0; rdev->dbr_sw_stats = kzalloc(sizeof(*rdev->dbr_sw_stats), GFP_KERNEL); if (!rdev->dbr_sw_stats) return -ENOMEM; return 0; } static void bnxt_re_free_dbr_sw_stats_mem(struct bnxt_re_dev *rdev) { kfree(rdev->dbr_sw_stats); rdev->dbr_sw_stats = NULL; } static int bnxt_re_initialize_dbr_drop_recov(struct bnxt_re_dev *rdev) { rdev->dbr_drop_recov_wq = create_singlethread_workqueue("bnxt_re_dbr_drop_recov"); if (!rdev->dbr_drop_recov_wq) { dev_err(rdev_to_dev(rdev), "DBR Drop Revov wq alloc failed!"); return -EINVAL; } rdev->dbr_drop_recov = true; /* Enable configfs setting dbr_drop_recov by default*/ rdev->user_dbr_drop_recov = true; rdev->user_dbr_drop_recov_timeout = BNXT_RE_DBR_RECOV_USERLAND_TIMEOUT; return 0; } static void bnxt_re_deinitialize_dbr_drop_recov(struct bnxt_re_dev *rdev) { if (rdev->dbr_drop_recov_wq) { flush_workqueue(rdev->dbr_drop_recov_wq); destroy_workqueue(rdev->dbr_drop_recov_wq); rdev->dbr_drop_recov_wq = NULL; } rdev->dbr_drop_recov = false; } static int bnxt_re_initialize_dbr_pacing(struct bnxt_re_dev *rdev) { int rc; /* Allocate a page for app use */ rdev->dbr_page = (void *)__get_free_page(GFP_KERNEL); if (!rdev->dbr_page) { dev_err(rdev_to_dev(rdev), "DBR page allocation failed!"); return -ENOMEM; } memset((u8 *)rdev->dbr_page, 0, PAGE_SIZE); rdev->qplib_res.pacing_data = (struct bnxt_qplib_db_pacing_data *)rdev->dbr_page; rc = bnxt_re_hwrm_dbr_pacing_qcfg(rdev); if (rc) { dev_err(rdev_to_dev(rdev), "Failed to query dbr pacing config %d\n", rc); goto fail; } /* Create a work queue for scheduling dbq event */ rdev->dbq_wq = create_singlethread_workqueue("bnxt_re_dbq"); if (!rdev->dbq_wq) { dev_err(rdev_to_dev(rdev), "DBQ wq alloc failed!"); rc = -ENOMEM; goto fail; } /* MAP grc window 2 for reading db fifo depth */ writel_fbsd(rdev->en_dev->softc, BNXT_GRCPF_REG_WINDOW_BASE_OUT + 4, 0, rdev->chip_ctx->dbr_stat_db_fifo & BNXT_GRC_BASE_MASK); rdev->dbr_db_fifo_reg_off = (rdev->chip_ctx->dbr_stat_db_fifo & BNXT_GRC_OFFSET_MASK) + 0x2000; rdev->qplib_res.pacing_data->grc_reg_offset = rdev->dbr_db_fifo_reg_off; rdev->dbr_bar_addr = pci_resource_start(rdev->qplib_res.pdev, 0) + rdev->dbr_db_fifo_reg_off; /* Percentage of DB FIFO */ rdev->dbq_watermark = BNXT_RE_PACING_DBQ_THRESHOLD; rdev->pacing_en_int_th = BNXT_RE_PACING_EN_INT_THRESHOLD; rdev->pacing_algo_th = BNXT_RE_PACING_ALGO_THRESHOLD; rdev->dbq_pacing_time = BNXT_RE_DBR_INT_TIME; rdev->dbr_def_do_pacing = BNXT_RE_DBR_DO_PACING_NO_CONGESTION; rdev->do_pacing_save = rdev->dbr_def_do_pacing; bnxt_re_set_default_pacing_data(rdev); dev_dbg(rdev_to_dev(rdev), "Initialized db pacing\n"); return 0; fail: free_page((u64)rdev->dbr_page); rdev->dbr_page = NULL; return rc; } static void bnxt_re_deinitialize_dbr_pacing(struct bnxt_re_dev *rdev) { if (rdev->dbq_wq) flush_workqueue(rdev->dbq_wq); cancel_work_sync(&rdev->dbq_fifo_check_work); cancel_delayed_work_sync(&rdev->dbq_pacing_work); if (rdev->dbq_wq) { destroy_workqueue(rdev->dbq_wq); rdev->dbq_wq = NULL; } if (rdev->dbr_page) free_page((u64)rdev->dbr_page); rdev->dbr_page = NULL; rdev->dbr_pacing = false; } /* enable_dbr_pacing needs to be done only for older FWs * where host selects primary function. ie. pacing_ext * flags is not set. */ int bnxt_re_enable_dbr_pacing(struct bnxt_re_dev *rdev) { struct bnxt_qplib_nq *nq; nq = &rdev->nqr.nq[0]; rdev->dbq_nq_id = nq->ring_id; if (!bnxt_qplib_dbr_pacing_ext_en(rdev->chip_ctx) && bnxt_qplib_dbr_pacing_is_primary_pf(rdev->chip_ctx)) { if (bnxt_re_hwrm_dbr_pacing_cfg(rdev, true)) { dev_err(rdev_to_dev(rdev), "Failed to set dbr pacing config\n"); return -EIO; } /* MAP grc window 8 for ARMing the NQ DBQ */ writel_fbsd(rdev->en_dev->softc, BNXT_GRCPF_REG_WINDOW_BASE_OUT + 28 , 0, rdev->chip_ctx->dbr_aeq_arm_reg & BNXT_GRC_BASE_MASK); rdev->dbr_aeq_arm_reg_off = (rdev->chip_ctx->dbr_aeq_arm_reg & BNXT_GRC_OFFSET_MASK) + 0x8000; writel_fbsd(rdev->en_dev->softc, rdev->dbr_aeq_arm_reg_off , 0, 1); } return 0; } /* disable_dbr_pacing needs to be done only for older FWs * where host selects primary function. ie. pacing_ext * flags is not set. */ int bnxt_re_disable_dbr_pacing(struct bnxt_re_dev *rdev) { int rc = 0; if (!bnxt_qplib_dbr_pacing_ext_en(rdev->chip_ctx) && bnxt_qplib_dbr_pacing_is_primary_pf(rdev->chip_ctx)) rc = bnxt_re_hwrm_dbr_pacing_cfg(rdev, false); return rc; } static void bnxt_re_ib_uninit(struct bnxt_re_dev *rdev) { if (test_bit(BNXT_RE_FLAG_IBDEV_REGISTERED, &rdev->flags)) { bnxt_re_sysfs_destroy_file(rdev); /* Cleanup ib dev */ ib_unregister_device(&rdev->ibdev); clear_bit(BNXT_RE_FLAG_IBDEV_REGISTERED, &rdev->flags); return; } } static void bnxt_re_dev_uninit(struct bnxt_re_dev *rdev, u8 op_type) { struct bnxt_qplib_dpi *kdpi; int rc, wait_count = BNXT_RE_RES_FREE_WAIT_COUNT; bnxt_re_net_unregister_async_event(rdev); bnxt_re_put_stats2_ctx(rdev); if (test_and_clear_bit(BNXT_RE_FLAG_DEV_LIST_INITIALIZED, &rdev->flags)) { /* did the caller hold the lock? */ mutex_lock(&bnxt_re_dev_lock); list_del_rcu(&rdev->list); mutex_unlock(&bnxt_re_dev_lock); } bnxt_re_uninit_resolve_wq(rdev); bnxt_re_uninit_dcb_wq(rdev); bnxt_re_uninit_aer_wq(rdev); bnxt_re_deinitialize_dbr_drop_recov(rdev); if (bnxt_qplib_dbr_pacing_en(rdev->chip_ctx)) (void)bnxt_re_disable_dbr_pacing(rdev); if (test_and_clear_bit(BNXT_RE_FLAG_WORKER_REG, &rdev->flags)) { cancel_delayed_work_sync(&rdev->worker); } /* Wait for ULPs to release references */ while (atomic_read(&rdev->stats.rsors.cq_count) && --wait_count) usleep_range(500, 1000); if (!wait_count) dev_err(rdev_to_dev(rdev), "CQ resources not freed by stack, count = 0x%x", atomic_read(&rdev->stats.rsors.cq_count)); kdpi = &rdev->dpi_privileged; if (kdpi->umdbr) { /* kernel DPI was allocated with success */ (void)bnxt_qplib_dealloc_dpi(&rdev->qplib_res, kdpi); /* * Driver just need to know no command had failed * during driver load sequence and below command is * required indeed. Piggybacking dpi allocation status. */ } /* Protect the device uninitialization and start_irq/stop_irq L2 * callbacks with rtnl lock to avoid race condition between these calls */ rtnl_lock(); if (test_and_clear_bit(BNXT_RE_FLAG_SETUP_NQ, &rdev->flags)) bnxt_re_clean_nqs(rdev); rtnl_unlock(); if (test_and_clear_bit(BNXT_RE_FLAG_TBLS_ALLOCINIT, &rdev->flags)) bnxt_re_free_tbls(rdev); if (test_and_clear_bit(BNXT_RE_FLAG_RCFW_CHANNEL_INIT, &rdev->flags)) { rc = bnxt_qplib_deinit_rcfw(&rdev->rcfw); if (rc) dev_warn(rdev_to_dev(rdev), "Failed to deinitialize fw, rc = 0x%x", rc); } bnxt_re_put_stats_ctx(rdev); if (test_and_clear_bit(BNXT_RE_FLAG_ALLOC_CTX, &rdev->flags)) bnxt_qplib_free_hwctx(&rdev->qplib_res); rtnl_lock(); if (test_and_clear_bit(BNXT_RE_FLAG_RCFW_CHANNEL_EN, &rdev->flags)) bnxt_qplib_disable_rcfw_channel(&rdev->rcfw); if (rdev->dbr_pacing) bnxt_re_deinitialize_dbr_pacing(rdev); bnxt_re_free_dbr_sw_stats_mem(rdev); if (test_and_clear_bit(BNXT_RE_FLAG_NET_RING_ALLOC, &rdev->flags)) bnxt_re_net_ring_free(rdev, rdev->rcfw.creq.ring_id); if (test_and_clear_bit(BNXT_RE_FLAG_ALLOC_RCFW, &rdev->flags)) bnxt_qplib_free_rcfw_channel(&rdev->qplib_res); if (test_and_clear_bit(BNXT_RE_FLAG_GOT_MSIX, &rdev->flags)) bnxt_re_free_msix(rdev); rtnl_unlock(); bnxt_re_destroy_chip_ctx(rdev); if (op_type != BNXT_RE_PRE_RECOVERY_REMOVE) { if (test_and_clear_bit(BNXT_RE_FLAG_NETDEV_REGISTERED, &rdev->flags)) bnxt_re_unregister_netdev(rdev); } } static int bnxt_re_dev_init(struct bnxt_re_dev *rdev, u8 op_type, u8 wqe_mode) { struct bnxt_re_ring_attr rattr = {}; struct bnxt_qplib_creq_ctx *creq; int vec, offset; int rc = 0; if (op_type != BNXT_RE_POST_RECOVERY_INIT) { /* Registered a new RoCE device instance to netdev */ rc = bnxt_re_register_netdev(rdev); if (rc) return -EINVAL; } set_bit(BNXT_RE_FLAG_NETDEV_REGISTERED, &rdev->flags); rc = bnxt_re_setup_chip_ctx(rdev, wqe_mode); if (rc) { dev_err(rdev_to_dev(rdev), "Failed to get chip context rc 0x%x", rc); bnxt_re_unregister_netdev(rdev); clear_bit(BNXT_RE_FLAG_NETDEV_REGISTERED, &rdev->flags); rc = -EINVAL; return rc; } /* Protect the device initialization and start_irq/stop_irq L2 callbacks * with rtnl lock to avoid race condition between these calls */ rtnl_lock(); rc = bnxt_re_request_msix(rdev); if (rc) { dev_err(rdev_to_dev(rdev), "Requesting MSI-X vectors failed with rc = 0x%x", rc); rc = -EINVAL; goto release_rtnl; } set_bit(BNXT_RE_FLAG_GOT_MSIX, &rdev->flags); /* Establish RCFW Communication Channel to initialize the context memory for the function and all child VFs */ rc = bnxt_qplib_alloc_rcfw_channel(&rdev->qplib_res); if (rc) { dev_err(rdev_to_dev(rdev), "Failed to alloc mem for rcfw, rc = %#x\n", rc); goto release_rtnl; } set_bit(BNXT_RE_FLAG_ALLOC_RCFW, &rdev->flags); creq = &rdev->rcfw.creq; rattr.dma_arr = creq->hwq.pbl[PBL_LVL_0].pg_map_arr; rattr.pages = creq->hwq.pbl[creq->hwq.level].pg_count; rattr.type = bnxt_re_get_rtype(rdev); rattr.mode = HWRM_RING_ALLOC_INPUT_INT_MODE_MSIX; rattr.depth = BNXT_QPLIB_CREQE_MAX_CNT - 1; rattr.lrid = rdev->nqr.msix_entries[BNXT_RE_AEQ_IDX].ring_idx; rc = bnxt_re_net_ring_alloc(rdev, &rattr, &creq->ring_id); if (rc) { creq->ring_id = 0xffff; dev_err(rdev_to_dev(rdev), "Failed to allocate CREQ fw id with rc = 0x%x", rc); goto release_rtnl; } + set_bit(BNXT_RE_FLAG_NET_RING_ALLOC, &rdev->flags); + if (!rdev->chip_ctx) goto release_rtnl; - /* Program the NQ ID for DBQ notification */ - if (rdev->chip_ctx->modes.dbr_pacing_v0 || - bnxt_qplib_dbr_pacing_en(rdev->chip_ctx) || - bnxt_qplib_dbr_pacing_ext_en(rdev->chip_ctx)) { - rc = bnxt_re_initialize_dbr_pacing(rdev); - if (!rc) - rdev->dbr_pacing = true; - else - rdev->dbr_pacing = false; - dev_dbg(rdev_to_dev(rdev), "%s: initialize db pacing ret %d\n", - __func__, rc); + + if (!(_is_chip_p7(rdev->chip_ctx))) { + /* Program the NQ ID for DBQ notification */ + if (rdev->chip_ctx->modes.dbr_pacing_v0 || + bnxt_qplib_dbr_pacing_en(rdev->chip_ctx) || + bnxt_qplib_dbr_pacing_ext_en(rdev->chip_ctx)) { + rc = bnxt_re_initialize_dbr_pacing(rdev); + if (!rc) + rdev->dbr_pacing = true; + else + rdev->dbr_pacing = false; + dev_dbg(rdev_to_dev(rdev), "%s: initialize db pacing ret %d\n", + __func__, rc); + } } vec = rdev->nqr.msix_entries[BNXT_RE_AEQ_IDX].vector; offset = rdev->nqr.msix_entries[BNXT_RE_AEQ_IDX].db_offset; rc = bnxt_qplib_enable_rcfw_channel(&rdev->rcfw, vec, offset, &bnxt_re_aeq_handler); if (rc) { dev_err(rdev_to_dev(rdev), "Failed to enable RCFW channel with rc = 0x%x", rc); goto release_rtnl; } set_bit(BNXT_RE_FLAG_RCFW_CHANNEL_EN, &rdev->flags); rc = bnxt_re_update_dev_attr(rdev); if (rc) goto release_rtnl; bnxt_re_set_resource_limits(rdev); if (!rdev->is_virtfn && !_is_chip_gen_p5_p7(rdev->chip_ctx)) { rc = bnxt_qplib_alloc_hwctx(&rdev->qplib_res); if (rc) { dev_err(rdev_to_dev(rdev), "Failed to alloc hw contexts, rc = 0x%x", rc); goto release_rtnl; } set_bit(BNXT_RE_FLAG_ALLOC_CTX, &rdev->flags); } rc = bnxt_re_get_stats_ctx(rdev); if (rc) goto release_rtnl; rc = bnxt_qplib_init_rcfw(&rdev->rcfw, rdev->is_virtfn); if (rc) { dev_err(rdev_to_dev(rdev), "Failed to initialize fw with rc = 0x%x", rc); goto release_rtnl; } set_bit(BNXT_RE_FLAG_RCFW_CHANNEL_INIT, &rdev->flags); /* Based resource count on the 'new' device caps */ rc = bnxt_re_update_dev_attr(rdev); if (rc) goto release_rtnl; rc = bnxt_re_alloc_init_tbls(rdev); if (rc) { dev_err(rdev_to_dev(rdev), "tbls alloc-init failed rc = %#x", rc); goto release_rtnl; } rc = bnxt_re_setup_nqs(rdev); if (rc) { dev_err(rdev_to_dev(rdev), "NQs alloc-init failed rc = %#x\n", rc); if (rdev->nqr.max_init == 0) goto release_rtnl; dev_warn(rdev_to_dev(rdev), "expected nqs %d available nqs %d\n", rdev->nqr.num_msix, rdev->nqr.max_init); } set_bit(BNXT_RE_FLAG_SETUP_NQ, &rdev->flags); rtnl_unlock(); rc = bnxt_qplib_alloc_dpi(&rdev->qplib_res, &rdev->dpi_privileged, rdev, BNXT_QPLIB_DPI_TYPE_KERNEL); if (rc) goto fail; if (rdev->dbr_pacing) bnxt_re_enable_dbr_pacing(rdev); if (rdev->chip_ctx->modes.dbr_drop_recov) bnxt_re_initialize_dbr_drop_recov(rdev); rc = bnxt_re_alloc_dbr_sw_stats_mem(rdev); if (rc) goto fail; /* This block of code is needed for error recovery support */ if (!rdev->is_virtfn) { struct bnxt_re_tc_rec *tc_rec; tc_rec = &rdev->tc_rec[0]; rc = bnxt_re_query_hwrm_qportcfg(rdev, tc_rec, 0xFFFF); if (rc) { dev_err(rdev_to_dev(rdev), "Failed to query port config rc:%d", rc); return rc; } /* Query f/w defaults of CC params */ rc = bnxt_qplib_query_cc_param(&rdev->qplib_res, &rdev->cc_param); if (rc) dev_warn(rdev_to_dev(rdev), "Failed to query CC defaults\n"); if (1) { rdev->num_vfs = pci_num_vf(rdev->en_dev->pdev); if (rdev->num_vfs) { bnxt_re_set_resource_limits(rdev); bnxt_qplib_set_func_resources(&rdev->qplib_res); } } } INIT_DELAYED_WORK(&rdev->worker, bnxt_re_worker); set_bit(BNXT_RE_FLAG_WORKER_REG, &rdev->flags); schedule_delayed_work(&rdev->worker, msecs_to_jiffies(1000)); bnxt_re_init_dcb_wq(rdev); bnxt_re_init_aer_wq(rdev); bnxt_re_init_resolve_wq(rdev); mutex_lock(&bnxt_re_dev_lock); list_add_tail_rcu(&rdev->list, &bnxt_re_dev_list); /* Added to the list, not in progress anymore */ gadd_dev_inprogress--; set_bit(BNXT_RE_FLAG_DEV_LIST_INITIALIZED, &rdev->flags); mutex_unlock(&bnxt_re_dev_lock); return rc; release_rtnl: rtnl_unlock(); fail: bnxt_re_dev_uninit(rdev, BNXT_RE_COMPLETE_REMOVE); return rc; } static int bnxt_re_ib_init(struct bnxt_re_dev *rdev) { int rc = 0; rc = bnxt_re_register_ib(rdev); if (rc) { dev_err(rdev_to_dev(rdev), "Register IB failed with rc = 0x%x", rc); goto fail; } if (bnxt_re_sysfs_create_file(rdev)) { bnxt_re_stopqps_and_ib_uninit(rdev); goto fail; } set_bit(BNXT_RE_FLAG_IBDEV_REGISTERED, &rdev->flags); set_bit(BNXT_RE_FLAG_ISSUE_ROCE_STATS, &rdev->flags); set_bit(BNXT_RE_FLAG_ISSUE_CFA_FLOW_STATS, &rdev->flags); bnxt_re_dispatch_event(&rdev->ibdev, NULL, 1, IB_EVENT_PORT_ACTIVE); bnxt_re_dispatch_event(&rdev->ibdev, NULL, 1, IB_EVENT_GID_CHANGE); return rc; fail: bnxt_re_dev_uninit(rdev, BNXT_RE_COMPLETE_REMOVE); return rc; } /* wrapper for ib_init funcs */ int _bnxt_re_ib_init(struct bnxt_re_dev *rdev) { return bnxt_re_ib_init(rdev); } /* wrapper for aux init funcs */ int _bnxt_re_ib_init2(struct bnxt_re_dev *rdev) { bnxt_re_ib_init_2(rdev); return 0; /* add return for future proof */ } static void bnxt_re_dev_unreg(struct bnxt_re_dev *rdev) { bnxt_re_dev_dealloc(rdev); } static int bnxt_re_dev_reg(struct bnxt_re_dev **rdev, struct ifnet *netdev, struct bnxt_en_dev *en_dev) { struct ifnet *realdev = NULL; realdev = netdev; if (realdev) dev_dbg(NULL, "%s: realdev = %p netdev = %p\n", __func__, realdev, netdev); /* * Note: * The first argument to bnxt_re_dev_alloc() is 'netdev' and * not 'realdev', since in the case of bonding we want to * register the bonded virtual netdev (master) to the ib stack. * And 'en_dev' (for L2/PCI communication) is the first slave * device (PF0 on the card). * In the case of a regular netdev, both netdev and the en_dev * correspond to the same device. */ *rdev = bnxt_re_dev_alloc(netdev, en_dev); if (!*rdev) { pr_err("%s: netdev %p not handled", ROCE_DRV_MODULE_NAME, netdev); return -ENOMEM; } bnxt_re_hold(*rdev); return 0; } void bnxt_re_get_link_speed(struct bnxt_re_dev *rdev) { rdev->espeed = rdev->en_dev->espeed; return; } void bnxt_re_stopqps_and_ib_uninit(struct bnxt_re_dev *rdev) { dev_dbg(rdev_to_dev(rdev), "%s: Stopping QPs, IB uninit on rdev: %p\n", __func__, rdev); bnxt_re_stop_all_nonqp1_nonshadow_qps(rdev); bnxt_re_ib_uninit(rdev); } void bnxt_re_remove_device(struct bnxt_re_dev *rdev, u8 op_type, struct auxiliary_device *aux_dev) { struct bnxt_re_en_dev_info *en_info; struct bnxt_qplib_cmdq_ctx *cmdq; struct bnxt_qplib_rcfw *rcfw; rcfw = &rdev->rcfw; cmdq = &rcfw->cmdq; if (test_bit(FIRMWARE_STALL_DETECTED, &cmdq->flags)) set_bit(BNXT_RE_FLAG_ERR_DEVICE_DETACHED, &rdev->flags); dev_dbg(rdev_to_dev(rdev), "%s: Removing rdev: %p\n", __func__, rdev); bnxt_re_dev_uninit(rdev, op_type); en_info = auxiliary_get_drvdata(aux_dev); if (en_info) { rtnl_lock(); en_info->rdev = NULL; rtnl_unlock(); if (op_type != BNXT_RE_PRE_RECOVERY_REMOVE) { clear_bit(BNXT_RE_FLAG_EN_DEV_PRIMARY_DEV, &en_info->flags); clear_bit(BNXT_RE_FLAG_EN_DEV_SECONDARY_DEV, &en_info->flags); clear_bit(BNXT_RE_FLAG_EN_DEV_NETDEV_REG, &en_info->flags); } } bnxt_re_dev_unreg(rdev); } int bnxt_re_add_device(struct bnxt_re_dev **rdev, struct ifnet *netdev, u8 qp_mode, u8 op_type, u8 wqe_mode, u32 num_msix_requested, struct auxiliary_device *aux_dev) { struct bnxt_re_en_dev_info *en_info; struct bnxt_en_dev *en_dev; int rc = 0; en_info = auxiliary_get_drvdata(aux_dev); en_dev = en_info->en_dev; mutex_lock(&bnxt_re_dev_lock); /* Check if driver already in mod exit and aux_dev is valid */ if (gmod_exit || !aux_dev) { mutex_unlock(&bnxt_re_dev_lock); return -ENODEV; } /* Add device in progress */ gadd_dev_inprogress++; mutex_unlock(&bnxt_re_dev_lock); rc = bnxt_re_dev_reg(rdev, netdev, en_dev); if (rc) { dev_dbg(NULL, "Failed to create add device for netdev %p\n", netdev); /* * For BNXT_RE_POST_RECOVERY_INIT special case * called from bnxt_re_start, the work is * complete only after, bnxt_re_start completes * bnxt_unregister_device in case of failure. * So bnxt_re_start will decrement gadd_dev_inprogress * in case of failure. */ if (op_type != BNXT_RE_POST_RECOVERY_INIT) { mutex_lock(&bnxt_re_dev_lock); gadd_dev_inprogress--; mutex_unlock(&bnxt_re_dev_lock); } return rc; } if (rc != 0) goto ref_error; /* * num_msix_requested = BNXT_RE_MSIX_FROM_MOD_PARAM indicates fresh driver load. * Otherwaise, this invocation can be the result of lag create / destroy, * err revovery, hot fw upgrade, etc.. */ if (num_msix_requested == BNXT_RE_MSIX_FROM_MOD_PARAM) { if (bnxt_re_probe_count < BNXT_RE_MAX_DEVICES) num_msix_requested = max_msix_vec[bnxt_re_probe_count++]; else /* Consider as default when probe_count exceeds its limit */ num_msix_requested = 0; /* if user specifies only one value, use the same for all PFs */ if (max_msix_vec_argc == 1) num_msix_requested = max_msix_vec[0]; } (*rdev)->num_msix_requested = num_msix_requested; (*rdev)->gsi_ctx.gsi_qp_mode = qp_mode; (*rdev)->adev = aux_dev; (*rdev)->dev_addr = en_dev->softc->func.mac_addr; /* Before updating the rdev pointer in bnxt_re_en_dev_info structure, * take the rtnl lock to avoid accessing invalid rdev pointer from * L2 ULP callbacks. This is applicable in all the places where rdev * pointer is updated in bnxt_re_en_dev_info. */ rtnl_lock(); en_info->rdev = *rdev; rtnl_unlock(); rc = bnxt_re_dev_init(*rdev, op_type, wqe_mode); if (rc) { ref_error: bnxt_re_dev_unreg(*rdev); *rdev = NULL; /* * For BNXT_RE_POST_RECOVERY_INIT special case * called from bnxt_re_start, the work is * complete only after, bnxt_re_start completes * bnxt_unregister_device in case of failure. * So bnxt_re_start will decrement gadd_dev_inprogress * in case of failure. */ if (op_type != BNXT_RE_POST_RECOVERY_INIT) { mutex_lock(&bnxt_re_dev_lock); gadd_dev_inprogress--; mutex_unlock(&bnxt_re_dev_lock); } } dev_dbg(rdev_to_dev(*rdev), "%s: Adding rdev: %p\n", __func__, *rdev); if (!rc) { set_bit(BNXT_RE_FLAG_EN_DEV_NETDEV_REG, &en_info->flags); } return rc; } struct bnxt_re_dev *bnxt_re_get_peer_pf(struct bnxt_re_dev *rdev) { struct pci_dev *pdev_in = rdev->en_dev->pdev; int tmp_bus_num, bus_num = pdev_in->bus->number; int tmp_dev_num, dev_num = PCI_SLOT(pdev_in->devfn); int tmp_func_num, func_num = PCI_FUNC(pdev_in->devfn); struct bnxt_re_dev *tmp_rdev; rcu_read_lock(); list_for_each_entry_rcu(tmp_rdev, &bnxt_re_dev_list, list) { tmp_bus_num = tmp_rdev->en_dev->pdev->bus->number; tmp_dev_num = PCI_SLOT(tmp_rdev->en_dev->pdev->devfn); tmp_func_num = PCI_FUNC(tmp_rdev->en_dev->pdev->devfn); if (bus_num == tmp_bus_num && dev_num == tmp_dev_num && func_num != tmp_func_num) { rcu_read_unlock(); return tmp_rdev; } } rcu_read_unlock(); return NULL; } int bnxt_re_schedule_work(struct bnxt_re_dev *rdev, unsigned long event, struct ifnet *vlan_dev, struct ifnet *netdev, struct auxiliary_device *adev) { struct bnxt_re_work *re_work; /* Allocate for the deferred task */ re_work = kzalloc(sizeof(*re_work), GFP_KERNEL); if (!re_work) return -ENOMEM; re_work->rdev = rdev; re_work->event = event; re_work->vlan_dev = vlan_dev; re_work->adev = adev; INIT_WORK(&re_work->work, bnxt_re_task); if (rdev) atomic_inc(&rdev->sched_count); re_work->netdev = netdev; queue_work(bnxt_re_wq, &re_work->work); return 0; } int bnxt_re_get_slot_pf_count(struct bnxt_re_dev *rdev) { struct pci_dev *pdev_in = rdev->en_dev->pdev; int tmp_bus_num, bus_num = pdev_in->bus->number; int tmp_dev_num, dev_num = PCI_SLOT(pdev_in->devfn); struct bnxt_re_dev *tmp_rdev; int pf_cnt = 0; rcu_read_lock(); list_for_each_entry_rcu(tmp_rdev, &bnxt_re_dev_list, list) { tmp_bus_num = tmp_rdev->en_dev->pdev->bus->number; tmp_dev_num = PCI_SLOT(tmp_rdev->en_dev->pdev->devfn); if (bus_num == tmp_bus_num && dev_num == tmp_dev_num) pf_cnt++; } rcu_read_unlock(); return pf_cnt; } /* Handle all deferred netevents tasks */ static void bnxt_re_task(struct work_struct *work) { struct bnxt_re_en_dev_info *en_info; struct auxiliary_device *aux_dev; struct bnxt_re_work *re_work; struct bnxt_re_dev *rdev; re_work = container_of(work, struct bnxt_re_work, work); mutex_lock(&bnxt_re_mutex); rdev = re_work->rdev; /* * If the previous rdev is deleted due to bond creation * do not handle the event */ if (!bnxt_re_is_rdev_valid(rdev)) goto exit; /* Ignore the event, if the device is not registred with IB stack. This * is to avoid handling any event while the device is added/removed. */ if (rdev && !test_bit(BNXT_RE_FLAG_IBDEV_REGISTERED, &rdev->flags)) { dev_dbg(rdev_to_dev(rdev), "%s: Ignoring netdev event 0x%lx", __func__, re_work->event); goto done; } /* Extra check to silence coverity. We shouldn't handle any event * when rdev is NULL. */ if (!rdev) goto exit; dev_dbg(rdev_to_dev(rdev), "Scheduled work for event 0x%lx", re_work->event); switch (re_work->event) { case NETDEV_UP: bnxt_re_dispatch_event(&rdev->ibdev, NULL, 1, IB_EVENT_PORT_ACTIVE); bnxt_re_net_register_async_event(rdev); break; case NETDEV_DOWN: bnxt_qplib_dbr_pacing_set_primary_pf(rdev->chip_ctx, 0); bnxt_re_stop_all_nonqp1_nonshadow_qps(rdev); bnxt_re_dispatch_event(&rdev->ibdev, NULL, 1, IB_EVENT_PORT_ERR); break; case NETDEV_CHANGE: if (bnxt_re_get_link_state(rdev) == IB_PORT_DOWN) { bnxt_re_stop_all_nonqp1_nonshadow_qps(rdev); bnxt_re_dispatch_event(&rdev->ibdev, NULL, 1, IB_EVENT_PORT_ERR); break; } else if (bnxt_re_get_link_state(rdev) == IB_PORT_ACTIVE) { bnxt_re_dispatch_event(&rdev->ibdev, NULL, 1, IB_EVENT_PORT_ACTIVE); } /* temporarily disable the check for SR2 */ if (!bnxt_qplib_query_cc_param(&rdev->qplib_res, &rdev->cc_param) && !_is_chip_p7(rdev->chip_ctx)) { /* * Disable CC for 10G speed * for non p5 devices */ if (rdev->sl_espeed == SPEED_10000 && !_is_chip_gen_p5_p7(rdev->chip_ctx)) { if (rdev->cc_param.enable) bnxt_re_clear_cc(rdev); } else { if (!rdev->cc_param.enable && rdev->cc_param.admin_enable) bnxt_re_setup_cc(rdev); } } break; case NETDEV_UNREGISTER: bnxt_re_stopqps_and_ib_uninit(rdev); aux_dev = rdev->adev; if (re_work->adev) goto done; bnxt_re_remove_device(rdev, BNXT_RE_COMPLETE_REMOVE, aux_dev); break; default: break; } done: if (rdev) { /* memory barrier to guarantee task completion * before decrementing sched count */ mmiowb(); atomic_dec(&rdev->sched_count); } exit: if (re_work->adev && re_work->event == NETDEV_UNREGISTER) { en_info = auxiliary_get_drvdata(re_work->adev); en_info->ib_uninit_done = true; wake_up(&en_info->waitq); } kfree(re_work); mutex_unlock(&bnxt_re_mutex); } /* "Notifier chain callback can be invoked for the same chain from different CPUs at the same time". For cases when the netdev is already present, our call to the register_netdevice_notifier() will actually get the rtnl_lock() before sending NETDEV_REGISTER and (if up) NETDEV_UP events. But for cases when the netdev is not already present, the notifier chain is subjected to be invoked from different CPUs simultaneously. This is protected by the netdev_mutex. */ static int bnxt_re_netdev_event(struct notifier_block *notifier, unsigned long event, void *ptr) { struct ifnet *real_dev, *netdev; struct bnxt_re_dev *rdev = NULL; netdev = netdev_notifier_info_to_ifp(ptr); real_dev = rdma_vlan_dev_real_dev(netdev); if (!real_dev) real_dev = netdev; /* In case of bonding,this will be bond's rdev */ rdev = bnxt_re_from_netdev(real_dev); if (!rdev) goto exit; dev_info(rdev_to_dev(rdev), "%s: Event = %s (0x%lx), rdev %s (real_dev %s)\n", __func__, bnxt_re_netevent(event), event, rdev ? rdev->netdev ? if_getdname(rdev->netdev) : "->netdev = NULL" : "= NULL", (real_dev == netdev) ? "= netdev" : if_getdname(real_dev)); if (!test_bit(BNXT_RE_FLAG_IBDEV_REGISTERED, &rdev->flags)) goto exit; bnxt_re_hold(rdev); if (real_dev != netdev) { switch (event) { case NETDEV_UP: bnxt_re_schedule_work(rdev, event, netdev, NULL, NULL); break; case NETDEV_DOWN: break; default: break; } goto done; } switch (event) { case NETDEV_CHANGEADDR: if (!_is_chip_gen_p5_p7(rdev->chip_ctx)) bnxt_re_update_shadow_ah(rdev); bnxt_qplib_get_guid(rdev->dev_addr, (u8 *)&rdev->ibdev.node_guid); break; case NETDEV_CHANGE: bnxt_re_get_link_speed(rdev); bnxt_re_schedule_work(rdev, event, NULL, NULL, NULL); break; case NETDEV_UNREGISTER: /* netdev notifier will call NETDEV_UNREGISTER again later since * we are still holding the reference to the netdev */ /* * Workaround to avoid ib_unregister hang. Check for module * reference and dont free up the device if the reference * is non zero. Checking only for PF functions. */ if (rdev) { dev_info(rdev_to_dev(rdev), "bnxt_re:Unreg recvd when module refcnt > 0"); dev_info(rdev_to_dev(rdev), "bnxt_re:Close all apps using bnxt_re devs"); dev_info(rdev_to_dev(rdev), "bnxt_re:Remove the configfs entry created for the device"); dev_info(rdev_to_dev(rdev), "bnxt_re:Refer documentation for details"); goto done; } if (atomic_read(&rdev->sched_count) > 0) goto done; if (!rdev->unreg_sched) { bnxt_re_schedule_work(rdev, NETDEV_UNREGISTER, NULL, NULL, NULL); rdev->unreg_sched = true; goto done; } break; default: break; } done: if (rdev) bnxt_re_put(rdev); exit: return NOTIFY_DONE; } static struct notifier_block bnxt_re_netdev_notifier = { .notifier_call = bnxt_re_netdev_event }; static void bnxt_re_remove_base_interface(struct bnxt_re_dev *rdev, struct auxiliary_device *adev) { bnxt_re_stopqps_and_ib_uninit(rdev); bnxt_re_remove_device(rdev, BNXT_RE_COMPLETE_REMOVE, adev); auxiliary_set_drvdata(adev, NULL); } /* * bnxt_re_remove - Removes the roce aux device * @adev - aux device pointer * * This function removes the roce device. This gets * called in the mod exit path and pci unbind path. * If the rdev is bond interace, destroys the lag * in module exit path, and in pci unbind case * destroys the lag and recreates other base interface. * If the device is already removed in error recovery * path, it just unregister with the L2. */ static void bnxt_re_remove(struct auxiliary_device *adev) { struct bnxt_re_en_dev_info *en_info = auxiliary_get_drvdata(adev); struct bnxt_en_dev *en_dev; struct bnxt_re_dev *rdev; bool primary_dev = false; bool secondary_dev = false; if (!en_info) return; mutex_lock(&bnxt_re_mutex); en_dev = en_info->en_dev; rdev = en_info->rdev; if (rdev && bnxt_re_is_rdev_valid(rdev)) { if (pci_channel_offline(rdev->rcfw.pdev)) set_bit(ERR_DEVICE_DETACHED, &rdev->rcfw.cmdq.flags); if (test_bit(BNXT_RE_FLAG_EN_DEV_PRIMARY_DEV, &en_info->flags)) primary_dev = true; if (test_bit(BNXT_RE_FLAG_EN_DEV_SECONDARY_DEV, &en_info->flags)) secondary_dev = true; /* * en_dev_info of primary device and secondary device have the * same rdev pointer when LAG is configured. This rdev pointer * is rdev of bond interface. */ if (!primary_dev && !secondary_dev) { /* removal of non bond interface */ bnxt_re_remove_base_interface(rdev, adev); } else { /* * removal of bond primary/secondary interface. In this * case bond device is already removed, so rdev->binfo * is NULL. */ auxiliary_set_drvdata(adev, NULL); } } else { /* device is removed from ulp stop, unregister the net dev */ if (test_bit(BNXT_RE_FLAG_EN_DEV_NETDEV_REG, &en_info->flags)) { rtnl_lock(); en_dev->en_ops->bnxt_unregister_device(en_dev, BNXT_ROCE_ULP); rtnl_unlock(); } } mutex_unlock(&bnxt_re_mutex); return; } /* wrapper for all external user context callers */ void _bnxt_re_remove(struct auxiliary_device *adev) { bnxt_re_remove(adev); } static void bnxt_re_ib_init_2(struct bnxt_re_dev *rdev) { int rc; rc = bnxt_re_get_device_stats(rdev); if (rc) dev_err(rdev_to_dev(rdev), "Failed initial device stat query"); bnxt_re_net_register_async_event(rdev); } static int bnxt_re_probe(struct auxiliary_device *adev, const struct auxiliary_device_id *id) { struct bnxt_aux_dev *aux_dev = container_of(adev, struct bnxt_aux_dev, aux_dev); struct bnxt_re_en_dev_info *en_info; struct bnxt_en_dev *en_dev = NULL; struct bnxt_re_dev *rdev; int rc = -ENODEV; if (aux_dev) en_dev = aux_dev->edev; if (!en_dev) return rc; if (en_dev->ulp_version != BNXT_ULP_VERSION) { pr_err("%s: probe error: bnxt_en ulp version magic %x is not compatible!\n", ROCE_DRV_MODULE_NAME, en_dev->ulp_version); return -EINVAL; } en_info = kzalloc(sizeof(*en_info), GFP_KERNEL); if (!en_info) return -ENOMEM; memset(en_info, 0, sizeof(struct bnxt_re_en_dev_info)); en_info->en_dev = en_dev; auxiliary_set_drvdata(adev, en_info); mutex_lock(&bnxt_re_mutex); rc = bnxt_re_add_device(&rdev, en_dev->net, BNXT_RE_GSI_MODE_ALL, BNXT_RE_COMPLETE_INIT, BNXT_QPLIB_WQE_MODE_STATIC, BNXT_RE_MSIX_FROM_MOD_PARAM, adev); if (rc) { mutex_unlock(&bnxt_re_mutex); return rc; } rc = bnxt_re_ib_init(rdev); if (rc) goto err; bnxt_re_ib_init_2(rdev); dev_dbg(rdev_to_dev(rdev), "%s: adev: %p\n", __func__, adev); rdev->adev = adev; mutex_unlock(&bnxt_re_mutex); return 0; err: mutex_unlock(&bnxt_re_mutex); bnxt_re_remove(adev); return rc; } static const struct auxiliary_device_id bnxt_re_id_table[] = { { .name = BNXT_ADEV_NAME ".rdma", }, {}, }; MODULE_DEVICE_TABLE(auxiliary, bnxt_re_id_table); static struct auxiliary_driver bnxt_re_driver = { .name = "rdma", .probe = bnxt_re_probe, .remove = bnxt_re_remove, .id_table = bnxt_re_id_table, }; static int __init bnxt_re_mod_init(void) { int rc = 0; pr_info("%s: %s", ROCE_DRV_MODULE_NAME, drv_version); bnxt_re_wq = create_singlethread_workqueue("bnxt_re"); if (!bnxt_re_wq) return -ENOMEM; rc = bnxt_re_register_netdevice_notifier(&bnxt_re_netdev_notifier); if (rc) { pr_err("%s: Cannot register to netdevice_notifier", ROCE_DRV_MODULE_NAME); goto err_netdev; } INIT_LIST_HEAD(&bnxt_re_dev_list); rc = auxiliary_driver_register(&bnxt_re_driver); if (rc) { pr_err("%s: Failed to register auxiliary driver\n", ROCE_DRV_MODULE_NAME); goto err_auxdrv; } return 0; err_auxdrv: bnxt_re_unregister_netdevice_notifier(&bnxt_re_netdev_notifier); err_netdev: destroy_workqueue(bnxt_re_wq); return rc; } static void __exit bnxt_re_mod_exit(void) { gmod_exit = 1; auxiliary_driver_unregister(&bnxt_re_driver); bnxt_re_unregister_netdevice_notifier(&bnxt_re_netdev_notifier); if (bnxt_re_wq) destroy_workqueue(bnxt_re_wq); } module_init(bnxt_re_mod_init); module_exit(bnxt_re_mod_exit); diff --git a/sys/dev/bnxt/bnxt_re/qplib_fp.c b/sys/dev/bnxt/bnxt_re/qplib_fp.c index 3f1b02406f7f..d4b439b05588 100644 --- a/sys/dev/bnxt/bnxt_re/qplib_fp.c +++ b/sys/dev/bnxt/bnxt_re/qplib_fp.c @@ -1,3544 +1,3563 @@ /* * Copyright (c) 2015-2024, Broadcom. All rights reserved. The term * Broadcom refers to Broadcom Limited and/or its subsidiaries. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in * the documentation and/or other materials provided with the * distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * * Description: Fast Path Operators */ #include #include #include #include #include #include #include #include #include #include "hsi_struct_def.h" #include "qplib_tlv.h" #include "qplib_res.h" #include "qplib_rcfw.h" #include "qplib_sp.h" #include "qplib_fp.h" #include "ib_verbs.h" static void __clean_cq(struct bnxt_qplib_cq *cq, u64 qp); static void bnxt_re_legacy_cancel_phantom_processing(struct bnxt_qplib_qp *qp) { qp->sq.condition = false; qp->sq.legacy_send_phantom = false; qp->sq.single = false; } static void __bnxt_qplib_add_flush_qp(struct bnxt_qplib_qp *qp) { struct bnxt_qplib_cq *scq, *rcq; scq = qp->scq; rcq = qp->rcq; if (!qp->sq.flushed) { dev_dbg(&scq->hwq.pdev->dev, "QPLIB: FP: Adding to SQ Flush list = %p\n", qp); bnxt_re_legacy_cancel_phantom_processing(qp); list_add_tail(&qp->sq_flush, &scq->sqf_head); qp->sq.flushed = true; } if (!qp->srq) { if (!qp->rq.flushed) { dev_dbg(&rcq->hwq.pdev->dev, "QPLIB: FP: Adding to RQ Flush list = %p\n", qp); list_add_tail(&qp->rq_flush, &rcq->rqf_head); qp->rq.flushed = true; } } } static void bnxt_qplib_acquire_cq_flush_locks(struct bnxt_qplib_qp *qp) __acquires(&qp->scq->flush_lock) __acquires(&qp->rcq->flush_lock) { /* Interrupts are already disabled in calling functions */ spin_lock(&qp->scq->flush_lock); if (qp->scq == qp->rcq) __acquire(&qp->rcq->flush_lock); else spin_lock(&qp->rcq->flush_lock); } static void bnxt_qplib_release_cq_flush_locks(struct bnxt_qplib_qp *qp) __releases(&qp->scq->flush_lock) __releases(&qp->rcq->flush_lock) { if (qp->scq == qp->rcq) __release(&qp->rcq->flush_lock); else spin_unlock(&qp->rcq->flush_lock); spin_unlock(&qp->scq->flush_lock); } void bnxt_qplib_add_flush_qp(struct bnxt_qplib_qp *qp) { bnxt_qplib_acquire_cq_flush_locks(qp); __bnxt_qplib_add_flush_qp(qp); bnxt_qplib_release_cq_flush_locks(qp); } static void __bnxt_qplib_del_flush_qp(struct bnxt_qplib_qp *qp) { if (qp->sq.flushed) { qp->sq.flushed = false; list_del(&qp->sq_flush); } if (!qp->srq) { if (qp->rq.flushed) { qp->rq.flushed = false; list_del(&qp->rq_flush); } } } void bnxt_qplib_clean_qp(struct bnxt_qplib_qp *qp) { bnxt_qplib_acquire_cq_flush_locks(qp); __clean_cq(qp->scq, (u64)(unsigned long)qp); qp->sq.hwq.prod = 0; qp->sq.hwq.cons = 0; qp->sq.swq_start = 0; qp->sq.swq_last = 0; __clean_cq(qp->rcq, (u64)(unsigned long)qp); qp->rq.hwq.prod = 0; qp->rq.hwq.cons = 0; qp->rq.swq_start = 0; qp->rq.swq_last = 0; __bnxt_qplib_del_flush_qp(qp); bnxt_qplib_release_cq_flush_locks(qp); } static void bnxt_qpn_cqn_sched_task(struct work_struct *work) { struct bnxt_qplib_nq_work *nq_work = container_of(work, struct bnxt_qplib_nq_work, work); struct bnxt_qplib_cq *cq = nq_work->cq; struct bnxt_qplib_nq *nq = nq_work->nq; if (cq && nq) { spin_lock_bh(&cq->compl_lock); if (nq->cqn_handler) { dev_dbg(&nq->res->pdev->dev, "%s:Trigger cq = %p event nq = %p\n", __func__, cq, nq); nq->cqn_handler(nq, cq); } spin_unlock_bh(&cq->compl_lock); } kfree(nq_work); } static void bnxt_qplib_put_hdr_buf(struct pci_dev *pdev, struct bnxt_qplib_hdrbuf *buf) { dma_free_coherent(&pdev->dev, buf->len, buf->va, buf->dma_map); kfree(buf); } static void *bnxt_qplib_get_hdr_buf(struct pci_dev *pdev, u32 step, u32 cnt) { struct bnxt_qplib_hdrbuf *hdrbuf; u32 len; hdrbuf = kmalloc(sizeof(*hdrbuf), GFP_KERNEL); if (!hdrbuf) return NULL; len = ALIGN((step * cnt), PAGE_SIZE); hdrbuf->va = dma_alloc_coherent(&pdev->dev, len, &hdrbuf->dma_map, GFP_KERNEL); if (!hdrbuf->va) goto out; hdrbuf->len = len; hdrbuf->step = step; return hdrbuf; out: kfree(hdrbuf); return NULL; } void bnxt_qplib_free_hdr_buf(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp) { if (qp->rq_hdr_buf) { bnxt_qplib_put_hdr_buf(res->pdev, qp->rq_hdr_buf); qp->rq_hdr_buf = NULL; } if (qp->sq_hdr_buf) { bnxt_qplib_put_hdr_buf(res->pdev, qp->sq_hdr_buf); qp->sq_hdr_buf = NULL; } } int bnxt_qplib_alloc_hdr_buf(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp, u32 sstep, u32 rstep) { struct pci_dev *pdev; int rc = 0; pdev = res->pdev; if (sstep) { qp->sq_hdr_buf = bnxt_qplib_get_hdr_buf(pdev, sstep, qp->sq.max_wqe); if (!qp->sq_hdr_buf) { dev_err(&pdev->dev, "QPLIB: Failed to get sq_hdr_buf\n"); return -ENOMEM; } } if (rstep) { qp->rq_hdr_buf = bnxt_qplib_get_hdr_buf(pdev, rstep, qp->rq.max_wqe); if (!qp->rq_hdr_buf) { rc = -ENOMEM; dev_err(&pdev->dev, "QPLIB: Failed to get rq_hdr_buf\n"); goto fail; } } return 0; fail: bnxt_qplib_free_hdr_buf(res, qp); return rc; } /* * clean_nq - Invalidate cqe from given nq. * @cq - Completion queue * * Traverse whole notification queue and invalidate any completion * associated cq handler provided by caller. * Note - This function traverse the hardware queue but do not update * consumer index. Invalidated cqe(marked from this function) will be * ignored from actual completion of notification queue. */ static void clean_nq(struct bnxt_qplib_cq *cq) { struct bnxt_qplib_hwq *nq_hwq = NULL; struct bnxt_qplib_nq *nq = NULL; struct nq_base *hw_nqe = NULL; struct nq_cn *nqcne = NULL; u32 peek_flags, peek_cons; u64 q_handle; u32 type; int i; nq = cq->nq; nq_hwq = &nq->hwq; spin_lock_bh(&nq_hwq->lock); peek_flags = nq->nq_db.dbinfo.flags; peek_cons = nq_hwq->cons; for (i = 0; i < nq_hwq->max_elements; i++) { hw_nqe = bnxt_qplib_get_qe(nq_hwq, peek_cons, NULL); if (!NQE_CMP_VALID(hw_nqe, peek_flags)) break; /* The valid test of the entry must be done first * before reading any further. */ dma_rmb(); type = le16_to_cpu(hw_nqe->info10_type) & NQ_BASE_TYPE_MASK; /* Processing only NQ_BASE_TYPE_CQ_NOTIFICATION */ if (type == NQ_BASE_TYPE_CQ_NOTIFICATION) { nqcne = (struct nq_cn *)hw_nqe; q_handle = le32_to_cpu(nqcne->cq_handle_low); q_handle |= (u64)le32_to_cpu(nqcne->cq_handle_high) << 32; if (q_handle == (u64)cq) { nqcne->cq_handle_low = 0; nqcne->cq_handle_high = 0; cq->cnq_events++; } } bnxt_qplib_hwq_incr_cons(nq_hwq->max_elements, &peek_cons, 1, &peek_flags); } spin_unlock_bh(&nq_hwq->lock); } /* * Wait for receiving all NQEs for this CQ. * clean_nq is tried 100 times, each time clean_cq * loops upto budget times. budget is based on the * number of CQs shared by that NQ. So any NQE from * CQ would be already in the NQ. */ static void __wait_for_all_nqes(struct bnxt_qplib_cq *cq, u16 cnq_events) { u32 retry_cnt = 100; u16 total_events; if (!cnq_events) { clean_nq(cq); return; } while (retry_cnt--) { total_events = cq->cnq_events; /* Increment total_events by 1 if any CREQ event received with CQ notification */ if (cq->is_cq_err_event) total_events++; if (cnq_events == total_events) { dev_dbg(&cq->nq->res->pdev->dev, "QPLIB: NQ cleanup - Received all NQ events\n"); return; } msleep(1); clean_nq(cq); } } static void bnxt_qplib_service_nq(unsigned long data) { struct bnxt_qplib_nq *nq = (struct bnxt_qplib_nq *)data; struct bnxt_qplib_hwq *nq_hwq = &nq->hwq; int budget = nq->budget; struct bnxt_qplib_res *res; struct bnxt_qplib_cq *cq; struct pci_dev *pdev; struct nq_base *nqe; u32 hw_polled = 0; u64 q_handle; u32 type; res = nq->res; pdev = res->pdev; spin_lock_bh(&nq_hwq->lock); /* Service the NQ until empty or budget expired */ while (budget--) { nqe = bnxt_qplib_get_qe(nq_hwq, nq_hwq->cons, NULL); if (!NQE_CMP_VALID(nqe, nq->nq_db.dbinfo.flags)) break; /* The valid test of the entry must be done first before * reading any further. */ dma_rmb(); type = le16_to_cpu(nqe->info10_type) & NQ_BASE_TYPE_MASK; switch (type) { case NQ_BASE_TYPE_CQ_NOTIFICATION: { struct nq_cn *nqcne = (struct nq_cn *)nqe; q_handle = le32_to_cpu(nqcne->cq_handle_low); q_handle |= (u64)le32_to_cpu(nqcne->cq_handle_high) << 32; cq = (struct bnxt_qplib_cq *)q_handle; if (!cq) break; cq->toggle = (le16_to_cpu(nqe->info10_type) & NQ_CN_TOGGLE_MASK) >> NQ_CN_TOGGLE_SFT; cq->dbinfo.toggle = cq->toggle; bnxt_qplib_armen_db(&cq->dbinfo, DBC_DBC_TYPE_CQ_ARMENA); spin_lock_bh(&cq->compl_lock); atomic_set(&cq->arm_state, 0) ; if (!nq->cqn_handler(nq, (cq))) nq->stats.num_cqne_processed++; else dev_warn(&pdev->dev, "QPLIB: cqn - type 0x%x not handled\n", type); cq->cnq_events++; spin_unlock_bh(&cq->compl_lock); break; } case NQ_BASE_TYPE_SRQ_EVENT: { struct bnxt_qplib_srq *srq; struct nq_srq_event *nqsrqe = (struct nq_srq_event *)nqe; + u8 toggle; q_handle = le32_to_cpu(nqsrqe->srq_handle_low); q_handle |= (u64)le32_to_cpu(nqsrqe->srq_handle_high) << 32; srq = (struct bnxt_qplib_srq *)q_handle; + toggle = (le16_to_cpu(nqe->info10_type) & NQ_CN_TOGGLE_MASK) + >> NQ_CN_TOGGLE_SFT; + srq->dbinfo.toggle = toggle; bnxt_qplib_armen_db(&srq->dbinfo, DBC_DBC_TYPE_SRQ_ARMENA); if (!nq->srqn_handler(nq, (struct bnxt_qplib_srq *)q_handle, nqsrqe->event)) nq->stats.num_srqne_processed++; else dev_warn(&pdev->dev, "QPLIB: SRQ event 0x%x not handled\n", nqsrqe->event); break; } default: dev_warn(&pdev->dev, "QPLIB: nqe with opcode = 0x%x not handled\n", type); break; } hw_polled++; bnxt_qplib_hwq_incr_cons(nq_hwq->max_elements, &nq_hwq->cons, 1, &nq->nq_db.dbinfo.flags); } nqe = bnxt_qplib_get_qe(nq_hwq, nq_hwq->cons, NULL); if (!NQE_CMP_VALID(nqe, nq->nq_db.dbinfo.flags)) { nq->stats.num_nq_rearm++; bnxt_qplib_ring_nq_db(&nq->nq_db.dbinfo, res->cctx, true); } else if (nq->requested) { bnxt_qplib_ring_nq_db(&nq->nq_db.dbinfo, res->cctx, true); nq->stats.num_tasklet_resched++; } dev_dbg(&pdev->dev, "QPLIB: cqn/srqn/dbqn \n"); if (hw_polled >= 0) dev_dbg(&pdev->dev, "QPLIB: serviced %llu/%llu/%llu budget 0x%x reaped 0x%x\n", nq->stats.num_cqne_processed, nq->stats.num_srqne_processed, nq->stats.num_dbqne_processed, budget, hw_polled); dev_dbg(&pdev->dev, "QPLIB: resched_cnt = %llu arm_count = %llu\n", nq->stats.num_tasklet_resched, nq->stats.num_nq_rearm); spin_unlock_bh(&nq_hwq->lock); } static irqreturn_t bnxt_qplib_nq_irq(int irq, void *dev_instance) { struct bnxt_qplib_nq *nq = dev_instance; struct bnxt_qplib_hwq *nq_hwq = &nq->hwq; u32 sw_cons; /* Prefetch the NQ element */ sw_cons = HWQ_CMP(nq_hwq->cons, nq_hwq); if (sw_cons >= 0) prefetch(bnxt_qplib_get_qe(nq_hwq, sw_cons, NULL)); bnxt_qplib_service_nq((unsigned long)nq); return IRQ_HANDLED; } void bnxt_qplib_nq_stop_irq(struct bnxt_qplib_nq *nq, bool kill) { struct bnxt_qplib_res *res; if (!nq->requested) return; nq->requested = false; res = nq->res; /* Mask h/w interrupt */ bnxt_qplib_ring_nq_db(&nq->nq_db.dbinfo, res->cctx, false); /* Sync with last running IRQ handler */ synchronize_irq(nq->msix_vec); free_irq(nq->msix_vec, nq); kfree(nq->name); nq->name = NULL; } void bnxt_qplib_disable_nq(struct bnxt_qplib_nq *nq) { if (nq->cqn_wq) { destroy_workqueue(nq->cqn_wq); nq->cqn_wq = NULL; } /* Make sure the HW is stopped! */ bnxt_qplib_nq_stop_irq(nq, true); nq->nq_db.reg.bar_reg = NULL; nq->nq_db.db = NULL; nq->cqn_handler = NULL; nq->srqn_handler = NULL; nq->msix_vec = 0; } int bnxt_qplib_nq_start_irq(struct bnxt_qplib_nq *nq, int nq_indx, int msix_vector, bool need_init) { struct bnxt_qplib_res *res; int rc; res = nq->res; if (nq->requested) return -EFAULT; nq->msix_vec = msix_vector; nq->name = kasprintf(GFP_KERNEL, "bnxt_re-nq-%d@pci:%s\n", nq_indx, pci_name(res->pdev)); if (!nq->name) return -ENOMEM; rc = request_irq(nq->msix_vec, bnxt_qplib_nq_irq, 0, nq->name, nq); if (rc) { kfree(nq->name); nq->name = NULL; return rc; } nq->requested = true; bnxt_qplib_ring_nq_db(&nq->nq_db.dbinfo, res->cctx, true); return rc; } static void bnxt_qplib_map_nq_db(struct bnxt_qplib_nq *nq, u32 reg_offt) { struct bnxt_qplib_reg_desc *dbreg; struct bnxt_qplib_nq_db *nq_db; struct bnxt_qplib_res *res; nq_db = &nq->nq_db; res = nq->res; dbreg = &res->dpi_tbl.ucreg; nq_db->reg.bar_id = dbreg->bar_id; nq_db->reg.bar_base = dbreg->bar_base; nq_db->reg.bar_reg = dbreg->bar_reg + reg_offt; nq_db->reg.len = _is_chip_gen_p5_p7(res->cctx) ? sizeof(u64) : sizeof(u32); nq_db->dbinfo.db = nq_db->reg.bar_reg; nq_db->dbinfo.hwq = &nq->hwq; nq_db->dbinfo.xid = nq->ring_id; nq_db->dbinfo.seed = nq->ring_id; nq_db->dbinfo.flags = 0; spin_lock_init(&nq_db->dbinfo.lock); nq_db->dbinfo.shadow_key = BNXT_QPLIB_DBR_KEY_INVALID; nq_db->dbinfo.res = nq->res; return; } int bnxt_qplib_enable_nq(struct bnxt_qplib_nq *nq, int nq_idx, int msix_vector, int bar_reg_offset, cqn_handler_t cqn_handler, srqn_handler_t srqn_handler) { struct pci_dev *pdev; int rc; pdev = nq->res->pdev; nq->cqn_handler = cqn_handler; nq->srqn_handler = srqn_handler; nq->load = 0; mutex_init(&nq->lock); /* Have a task to schedule CQ notifiers in post send case */ nq->cqn_wq = create_singlethread_workqueue("bnxt_qplib_nq\n"); if (!nq->cqn_wq) return -ENOMEM; bnxt_qplib_map_nq_db(nq, bar_reg_offset); rc = bnxt_qplib_nq_start_irq(nq, nq_idx, msix_vector, true); if (rc) { dev_err(&pdev->dev, "QPLIB: Failed to request irq for nq-idx %d\n", nq_idx); goto fail; } dev_dbg(&pdev->dev, "QPLIB: NQ max = 0x%x\n", nq->hwq.max_elements); return 0; fail: bnxt_qplib_disable_nq(nq); return rc; } void bnxt_qplib_free_nq_mem(struct bnxt_qplib_nq *nq) { if (nq->hwq.max_elements) { bnxt_qplib_free_hwq(nq->res, &nq->hwq); nq->hwq.max_elements = 0; } } int bnxt_qplib_alloc_nq_mem(struct bnxt_qplib_res *res, struct bnxt_qplib_nq *nq) { struct bnxt_qplib_hwq_attr hwq_attr = {}; struct bnxt_qplib_sg_info sginfo = {}; nq->res = res; if (!nq->hwq.max_elements || nq->hwq.max_elements > BNXT_QPLIB_NQE_MAX_CNT) nq->hwq.max_elements = BNXT_QPLIB_NQE_MAX_CNT; sginfo.pgsize = PAGE_SIZE; sginfo.pgshft = PAGE_SHIFT; hwq_attr.res = res; hwq_attr.sginfo = &sginfo; hwq_attr.depth = nq->hwq.max_elements; hwq_attr.stride = sizeof(struct nq_base); hwq_attr.type = _get_hwq_type(res); if (bnxt_qplib_alloc_init_hwq(&nq->hwq, &hwq_attr)) { dev_err(&res->pdev->dev, "QPLIB: FP NQ allocation failed\n"); return -ENOMEM; } nq->budget = 8; return 0; } /* SRQ */ static int __qplib_destroy_srq(struct bnxt_qplib_rcfw *rcfw, struct bnxt_qplib_srq *srq) { struct creq_destroy_srq_resp resp = {}; struct bnxt_qplib_cmdqmsg msg = {}; struct cmdq_destroy_srq req = {}; /* Configure the request */ req.srq_cid = cpu_to_le32(srq->id); bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_DESTROY_SRQ, sizeof(req)); bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL, sizeof(req), sizeof(resp), 0); return bnxt_qplib_rcfw_send_message(rcfw, &msg); } int bnxt_qplib_destroy_srq(struct bnxt_qplib_res *res, struct bnxt_qplib_srq *srq) { struct bnxt_qplib_rcfw *rcfw = res->rcfw; int rc; rc = __qplib_destroy_srq(rcfw, srq); if (rc) return rc; bnxt_qplib_free_hwq(res, &srq->hwq); kfree(srq->swq); return 0; } int bnxt_qplib_create_srq(struct bnxt_qplib_res *res, struct bnxt_qplib_srq *srq) { struct bnxt_qplib_hwq_attr hwq_attr = {}; struct bnxt_qplib_rcfw *rcfw = res->rcfw; struct creq_create_srq_resp resp = {}; struct bnxt_qplib_cmdqmsg msg = {}; struct cmdq_create_srq req = {}; u16 pg_sz_lvl = 0; u16 srq_size; int rc, idx; hwq_attr.res = res; hwq_attr.sginfo = &srq->sginfo; hwq_attr.depth = srq->max_wqe; hwq_attr.stride = srq->wqe_size; hwq_attr.type = HWQ_TYPE_QUEUE; rc = bnxt_qplib_alloc_init_hwq(&srq->hwq, &hwq_attr); if (rc) goto exit; /* Configure the request */ req.dpi = cpu_to_le32(srq->dpi->dpi); req.srq_handle = cpu_to_le64((uintptr_t)srq); srq_size = min_t(u32, srq->hwq.depth, U16_MAX); req.srq_size = cpu_to_le16(srq_size); pg_sz_lvl |= (_get_base_pg_size(&srq->hwq) << CMDQ_CREATE_SRQ_PG_SIZE_SFT); pg_sz_lvl |= (srq->hwq.level & CMDQ_CREATE_SRQ_LVL_MASK); req.pg_size_lvl = cpu_to_le16(pg_sz_lvl); req.pbl = cpu_to_le64(_get_base_addr(&srq->hwq)); req.pd_id = cpu_to_le32(srq->pd->id); req.eventq_id = cpu_to_le16(srq->eventq_hw_ring_id); bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_CREATE_SRQ, sizeof(req)); bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL, sizeof(req), sizeof(resp), 0); rc = bnxt_qplib_rcfw_send_message(rcfw, &msg); if (rc) goto fail; if (!srq->is_user) { srq->swq = kcalloc(srq->hwq.depth, sizeof(*srq->swq), GFP_KERNEL); if (!srq->swq) goto srq_fail; srq->start_idx = 0; srq->last_idx = srq->hwq.depth - 1; for (idx = 0; idx < srq->hwq.depth; idx++) srq->swq[idx].next_idx = idx + 1; srq->swq[srq->last_idx].next_idx = -1; } spin_lock_init(&srq->lock); srq->id = le32_to_cpu(resp.xid); srq->cctx = res->cctx; srq->dbinfo.hwq = &srq->hwq; srq->dbinfo.xid = srq->id; srq->dbinfo.db = srq->dpi->dbr; srq->dbinfo.max_slot = 1; srq->dbinfo.priv_db = res->dpi_tbl.priv_db; srq->dbinfo.flags = 0; spin_lock_init(&srq->dbinfo.lock); srq->dbinfo.shadow_key = BNXT_QPLIB_DBR_KEY_INVALID; srq->dbinfo.shadow_key_arm_ena = BNXT_QPLIB_DBR_KEY_INVALID; srq->dbinfo.res = res; srq->dbinfo.seed = srq->id; if (srq->threshold) bnxt_qplib_armen_db(&srq->dbinfo, DBC_DBC_TYPE_SRQ_ARMENA); srq->arm_req = false; return 0; srq_fail: __qplib_destroy_srq(rcfw, srq); fail: bnxt_qplib_free_hwq(res, &srq->hwq); exit: return rc; } int bnxt_qplib_modify_srq(struct bnxt_qplib_res *res, struct bnxt_qplib_srq *srq) { struct bnxt_qplib_hwq *srq_hwq = &srq->hwq; u32 avail = 0; avail = __bnxt_qplib_get_avail(srq_hwq); if (avail <= srq->threshold) { srq->arm_req = false; bnxt_qplib_srq_arm_db(&srq->dbinfo); } else { /* Deferred arming */ srq->arm_req = true; } return 0; } int bnxt_qplib_query_srq(struct bnxt_qplib_res *res, struct bnxt_qplib_srq *srq) { struct bnxt_qplib_rcfw *rcfw = res->rcfw; struct creq_query_srq_resp resp = {}; struct bnxt_qplib_cmdqmsg msg = {}; struct creq_query_srq_resp_sb *sb; struct bnxt_qplib_rcfw_sbuf sbuf; struct cmdq_query_srq req = {}; int rc = 0; bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_QUERY_SRQ, sizeof(req)); sbuf.size = ALIGN(sizeof(*sb), BNXT_QPLIB_CMDQE_UNITS); sbuf.sb = dma_zalloc_coherent(&rcfw->pdev->dev, sbuf.size, &sbuf.dma_addr, GFP_KERNEL); if (!sbuf.sb) return -ENOMEM; req.resp_size = sbuf.size / BNXT_QPLIB_CMDQE_UNITS; req.srq_cid = cpu_to_le32(srq->id); bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, &sbuf, sizeof(req), sizeof(resp), 0); rc = bnxt_qplib_rcfw_send_message(rcfw, &msg); /* TODO: What to do with the query? */ dma_free_coherent(&rcfw->pdev->dev, sbuf.size, sbuf.sb, sbuf.dma_addr); return rc; } int bnxt_qplib_post_srq_recv(struct bnxt_qplib_srq *srq, struct bnxt_qplib_swqe *wqe) { struct bnxt_qplib_hwq *srq_hwq = &srq->hwq; struct sq_sge *hw_sge; struct rq_wqe *srqe; int i, rc = 0, next; u32 avail; spin_lock(&srq_hwq->lock); if (srq->start_idx == srq->last_idx) { dev_err(&srq_hwq->pdev->dev, "QPLIB: FP: SRQ (0x%x) is full!\n", srq->id); rc = -EINVAL; spin_unlock(&srq_hwq->lock); goto done; } next = srq->start_idx; srq->start_idx = srq->swq[next].next_idx; spin_unlock(&srq_hwq->lock); srqe = bnxt_qplib_get_qe(srq_hwq, srq_hwq->prod, NULL); memset(srqe, 0, srq->wqe_size); /* Calculate wqe_size and data_len */ for (i = 0, hw_sge = (struct sq_sge *)srqe->data; i < wqe->num_sge; i++, hw_sge++) { hw_sge->va_or_pa = cpu_to_le64(wqe->sg_list[i].addr); hw_sge->l_key = cpu_to_le32(wqe->sg_list[i].lkey); hw_sge->size = cpu_to_le32(wqe->sg_list[i].size); } srqe->wqe_type = wqe->type; srqe->flags = wqe->flags; srqe->wqe_size = wqe->num_sge + ((offsetof(typeof(*srqe), data) + 15) >> 4); if (!wqe->num_sge) srqe->wqe_size++; srqe->wr_id |= cpu_to_le32((u32)next); srq->swq[next].wr_id = wqe->wr_id; bnxt_qplib_hwq_incr_prod(&srq->dbinfo, srq_hwq, srq->dbinfo.max_slot); /* retaining srq_hwq->cons for this logic actually the lock is only * required to read srq_hwq->cons. */ spin_lock(&srq_hwq->lock); avail = __bnxt_qplib_get_avail(srq_hwq); spin_unlock(&srq_hwq->lock); /* Ring DB */ bnxt_qplib_ring_prod_db(&srq->dbinfo, DBC_DBC_TYPE_SRQ); if (srq->arm_req && avail <= srq->threshold) { srq->arm_req = false; bnxt_qplib_srq_arm_db(&srq->dbinfo); } done: return rc; } /* QP */ static int __qplib_destroy_qp(struct bnxt_qplib_rcfw *rcfw, struct bnxt_qplib_qp *qp) { struct creq_destroy_qp_resp resp = {}; struct bnxt_qplib_cmdqmsg msg = {}; struct cmdq_destroy_qp req = {}; req.qp_cid = cpu_to_le32(qp->id); bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_DESTROY_QP, sizeof(req)); bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL, sizeof(req), sizeof(resp), 0); return bnxt_qplib_rcfw_send_message(rcfw, &msg); } static int bnxt_qplib_alloc_init_swq(struct bnxt_qplib_q *que) { int rc = 0; int indx; que->swq = kcalloc(que->max_wqe, sizeof(*que->swq), GFP_KERNEL); if (!que->swq) { rc = -ENOMEM; goto out; } que->swq_start = 0; que->swq_last = que->max_wqe - 1; for (indx = 0; indx < que->max_wqe; indx++) que->swq[indx].next_idx = indx + 1; que->swq[que->swq_last].next_idx = 0; /* Make it circular */ que->swq_last = 0; out: return rc; } static struct bnxt_qplib_swq *bnxt_qplib_get_swqe(struct bnxt_qplib_q *que, u32 *swq_idx) { u32 idx; idx = que->swq_start; if (swq_idx) *swq_idx = idx; return &que->swq[idx]; } static void bnxt_qplib_swq_mod_start(struct bnxt_qplib_q *que, u32 idx) { que->swq_start = que->swq[idx].next_idx; } static u32 bnxt_qplib_get_stride(void) { return sizeof(struct sq_sge); } static u32 bnxt_qplib_get_depth(struct bnxt_qplib_q *que) { u8 stride; stride = bnxt_qplib_get_stride(); return (que->wqe_size * que->max_wqe) / stride; } static u32 _set_sq_size(struct bnxt_qplib_q *que, u8 wqe_mode) { /* For Variable mode supply number of 16B slots */ return (wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC) ? que->max_wqe : bnxt_qplib_get_depth(que); } static u32 _set_sq_max_slot(u8 wqe_mode) { /* for static mode index divisor is 8 */ return (wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC) ? sizeof(struct sq_send) / sizeof(struct sq_sge) : 1; } static u32 _set_rq_max_slot(struct bnxt_qplib_q *que) { return (que->wqe_size / sizeof(struct sq_sge)); } int bnxt_qplib_create_qp1(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp) { struct bnxt_qplib_hwq_attr hwq_attr = {}; struct bnxt_qplib_rcfw *rcfw = res->rcfw; struct creq_create_qp1_resp resp = {}; struct bnxt_qplib_cmdqmsg msg = {}; struct bnxt_qplib_q *sq = &qp->sq; struct bnxt_qplib_q *rq = &qp->rq; struct cmdq_create_qp1 req = {}; struct bnxt_qplib_reftbl *tbl; unsigned long flag; u8 pg_sz_lvl = 0; u32 qp_flags = 0; int rc; /* General */ req.type = qp->type; req.dpi = cpu_to_le32(qp->dpi->dpi); req.qp_handle = cpu_to_le64(qp->qp_handle); /* SQ */ hwq_attr.res = res; hwq_attr.sginfo = &sq->sginfo; hwq_attr.stride = bnxt_qplib_get_stride(); hwq_attr.depth = bnxt_qplib_get_depth(sq); hwq_attr.type = HWQ_TYPE_QUEUE; rc = bnxt_qplib_alloc_init_hwq(&sq->hwq, &hwq_attr); if (rc) goto exit; req.sq_size = cpu_to_le32(_set_sq_size(sq, qp->wqe_mode)); req.sq_pbl = cpu_to_le64(_get_base_addr(&sq->hwq)); pg_sz_lvl = _get_base_pg_size(&sq->hwq) << CMDQ_CREATE_QP1_SQ_PG_SIZE_SFT; pg_sz_lvl |= ((sq->hwq.level & CMDQ_CREATE_QP1_SQ_LVL_MASK) << CMDQ_CREATE_QP1_SQ_LVL_SFT); req.sq_pg_size_sq_lvl = pg_sz_lvl; req.sq_fwo_sq_sge = cpu_to_le16(((0 << CMDQ_CREATE_QP1_SQ_FWO_SFT) & CMDQ_CREATE_QP1_SQ_FWO_MASK) | (sq->max_sge & CMDQ_CREATE_QP1_SQ_SGE_MASK)); req.scq_cid = cpu_to_le32(qp->scq->id); /* RQ */ if (!qp->srq) { hwq_attr.res = res; hwq_attr.sginfo = &rq->sginfo; hwq_attr.stride = bnxt_qplib_get_stride(); hwq_attr.depth = bnxt_qplib_get_depth(rq); hwq_attr.type = HWQ_TYPE_QUEUE; rc = bnxt_qplib_alloc_init_hwq(&rq->hwq, &hwq_attr); if (rc) goto fail_sq; req.rq_size = cpu_to_le32(rq->max_wqe); req.rq_pbl = cpu_to_le64(_get_base_addr(&rq->hwq)); pg_sz_lvl = _get_base_pg_size(&rq->hwq) << CMDQ_CREATE_QP1_RQ_PG_SIZE_SFT; pg_sz_lvl |= ((rq->hwq.level & CMDQ_CREATE_QP1_RQ_LVL_MASK) << CMDQ_CREATE_QP1_RQ_LVL_SFT); req.rq_pg_size_rq_lvl = pg_sz_lvl; req.rq_fwo_rq_sge = cpu_to_le16(((0 << CMDQ_CREATE_QP1_RQ_FWO_SFT) & CMDQ_CREATE_QP1_RQ_FWO_MASK) | (rq->max_sge & CMDQ_CREATE_QP1_RQ_SGE_MASK)); } else { /* SRQ */ qp_flags |= CMDQ_CREATE_QP1_QP_FLAGS_SRQ_USED; req.srq_cid = cpu_to_le32(qp->srq->id); } req.rcq_cid = cpu_to_le32(qp->rcq->id); qp_flags |= CMDQ_CREATE_QP1_QP_FLAGS_RESERVED_LKEY_ENABLE; req.qp_flags = cpu_to_le32(qp_flags); req.pd_id = cpu_to_le32(qp->pd->id); bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_CREATE_QP1, sizeof(req)); bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL, sizeof(req), sizeof(resp), 0); rc = bnxt_qplib_rcfw_send_message(rcfw, &msg); if (rc) goto fail_rq; rc = bnxt_qplib_alloc_init_swq(sq); if (rc) goto sq_swq; if (!qp->srq) { rc = bnxt_qplib_alloc_init_swq(rq); if (rc) goto rq_swq; } qp->id = le32_to_cpu(resp.xid); qp->cur_qp_state = CMDQ_MODIFY_QP_NEW_STATE_RESET; qp->cctx = res->cctx; sq->dbinfo.hwq = &sq->hwq; sq->dbinfo.xid = qp->id; sq->dbinfo.db = qp->dpi->dbr; sq->dbinfo.max_slot = _set_sq_max_slot(qp->wqe_mode); sq->dbinfo.flags = 0; spin_lock_init(&sq->dbinfo.lock); sq->dbinfo.shadow_key = BNXT_QPLIB_DBR_KEY_INVALID; sq->dbinfo.res = res; if (rq->max_wqe) { rq->dbinfo.hwq = &rq->hwq; rq->dbinfo.xid = qp->id; rq->dbinfo.db = qp->dpi->dbr; rq->dbinfo.max_slot = _set_rq_max_slot(rq); rq->dbinfo.flags = 0; spin_lock_init(&rq->dbinfo.lock); rq->dbinfo.shadow_key = BNXT_QPLIB_DBR_KEY_INVALID; rq->dbinfo.res = res; } tbl = &res->reftbl.qpref; spin_lock_irqsave(&tbl->lock, flag); tbl->rec[tbl->max].xid = qp->id; tbl->rec[tbl->max].handle = qp; spin_unlock_irqrestore(&tbl->lock, flag); return 0; rq_swq: kfree(sq->swq); sq_swq: __qplib_destroy_qp(rcfw, qp); fail_rq: bnxt_qplib_free_hwq(res, &rq->hwq); fail_sq: bnxt_qplib_free_hwq(res, &sq->hwq); exit: return rc; } static void bnxt_qplib_init_psn_ptr(struct bnxt_qplib_qp *qp, int size) { struct bnxt_qplib_hwq *sq_hwq; struct bnxt_qplib_q *sq; u64 fpsne, psn_pg; u16 indx_pad = 0; sq = &qp->sq; sq_hwq = &sq->hwq; /* First psn entry */ fpsne = (u64)bnxt_qplib_get_qe(sq_hwq, sq_hwq->depth, &psn_pg); if (!IS_ALIGNED(fpsne, PAGE_SIZE)) indx_pad = (fpsne & ~PAGE_MASK) / size; sq_hwq->pad_pgofft = indx_pad; sq_hwq->pad_pg = (u64 *)psn_pg; sq_hwq->pad_stride = size; } int bnxt_qplib_create_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp) { struct bnxt_qplib_hwq_attr hwq_attr = {}; struct bnxt_qplib_rcfw *rcfw = res->rcfw; struct bnxt_qplib_sg_info sginfo = {}; struct creq_create_qp_resp resp = {}; struct bnxt_qplib_cmdqmsg msg = {}; struct bnxt_qplib_q *sq = &qp->sq; struct bnxt_qplib_q *rq = &qp->rq; struct cmdq_create_qp req = {}; struct bnxt_qplib_reftbl *tbl; struct bnxt_qplib_hwq *xrrq; int rc, req_size, psn_sz; unsigned long flag; u8 pg_sz_lvl = 0; u32 qp_flags = 0; u32 qp_idx; u16 nsge; u32 sqsz; qp->cctx = res->cctx; - if (res->dattr) + if (res->dattr) { qp->dev_cap_flags = res->dattr->dev_cap_flags; + qp->is_host_msn_tbl = _is_host_msn_table(res->dattr->dev_cap_ext_flags2); + } + /* General */ req.type = qp->type; req.dpi = cpu_to_le32(qp->dpi->dpi); req.qp_handle = cpu_to_le64(qp->qp_handle); /* SQ */ if (qp->type == CMDQ_CREATE_QP_TYPE_RC) { psn_sz = _is_chip_gen_p5_p7(qp->cctx) ? sizeof(struct sq_psn_search_ext) : sizeof(struct sq_psn_search); - if (BNXT_RE_HW_RETX(qp->dev_cap_flags)) { + if (qp->is_host_msn_tbl) { psn_sz = sizeof(struct sq_msn_search); qp->msn = 0; } } else { psn_sz = 0; } hwq_attr.res = res; hwq_attr.sginfo = &sq->sginfo; hwq_attr.stride = bnxt_qplib_get_stride(); hwq_attr.depth = bnxt_qplib_get_depth(sq); hwq_attr.aux_stride = psn_sz; hwq_attr.aux_depth = (psn_sz) ? _set_sq_size(sq, qp->wqe_mode) : 0; /* Update msn tbl size */ - if (BNXT_RE_HW_RETX(qp->dev_cap_flags) && psn_sz) { + if (qp->is_host_msn_tbl && psn_sz) { if (qp->wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC) hwq_attr.aux_depth = roundup_pow_of_two(_set_sq_size(sq, qp->wqe_mode)); else hwq_attr.aux_depth = roundup_pow_of_two(_set_sq_size(sq, qp->wqe_mode)) / 2; qp->msn_tbl_sz = hwq_attr.aux_depth; qp->msn = 0; } hwq_attr.type = HWQ_TYPE_QUEUE; rc = bnxt_qplib_alloc_init_hwq(&sq->hwq, &hwq_attr); if (rc) goto exit; sqsz = _set_sq_size(sq, qp->wqe_mode); /* 0xffff is the max sq size hw limits to */ if (sqsz > BNXT_QPLIB_MAX_SQSZ) { pr_err("QPLIB: FP: QP (0x%x) exceeds sq size %d\n", qp->id, sqsz); goto fail_sq; } req.sq_size = cpu_to_le32(sqsz); req.sq_pbl = cpu_to_le64(_get_base_addr(&sq->hwq)); pg_sz_lvl = _get_base_pg_size(&sq->hwq) << CMDQ_CREATE_QP_SQ_PG_SIZE_SFT; pg_sz_lvl |= ((sq->hwq.level & CMDQ_CREATE_QP_SQ_LVL_MASK) << CMDQ_CREATE_QP_SQ_LVL_SFT); req.sq_pg_size_sq_lvl = pg_sz_lvl; req.sq_fwo_sq_sge = cpu_to_le16(((0 << CMDQ_CREATE_QP_SQ_FWO_SFT) & CMDQ_CREATE_QP_SQ_FWO_MASK) | - ((BNXT_RE_HW_RETX(qp->dev_cap_flags)) ? - BNXT_MSN_TBLE_SGE : sq->max_sge & + (sq->max_sge & CMDQ_CREATE_QP_SQ_SGE_MASK)); req.scq_cid = cpu_to_le32(qp->scq->id); /* RQ/SRQ */ if (!qp->srq) { hwq_attr.res = res; hwq_attr.sginfo = &rq->sginfo; hwq_attr.stride = bnxt_qplib_get_stride(); hwq_attr.depth = bnxt_qplib_get_depth(rq); hwq_attr.aux_stride = 0; hwq_attr.aux_depth = 0; hwq_attr.type = HWQ_TYPE_QUEUE; rc = bnxt_qplib_alloc_init_hwq(&rq->hwq, &hwq_attr); if (rc) goto fail_sq; req.rq_size = cpu_to_le32(rq->max_wqe); req.rq_pbl = cpu_to_le64(_get_base_addr(&rq->hwq)); pg_sz_lvl = _get_base_pg_size(&rq->hwq) << CMDQ_CREATE_QP_RQ_PG_SIZE_SFT; pg_sz_lvl |= ((rq->hwq.level & CMDQ_CREATE_QP_RQ_LVL_MASK) << CMDQ_CREATE_QP_RQ_LVL_SFT); req.rq_pg_size_rq_lvl = pg_sz_lvl; nsge = (qp->wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC) ? res->dattr->max_qp_sges : rq->max_sge; req.rq_fwo_rq_sge = cpu_to_le16(((0 << CMDQ_CREATE_QP_RQ_FWO_SFT) & CMDQ_CREATE_QP_RQ_FWO_MASK) | (nsge & CMDQ_CREATE_QP_RQ_SGE_MASK)); } else { qp_flags |= CMDQ_CREATE_QP_QP_FLAGS_SRQ_USED; req.srq_cid = cpu_to_le32(qp->srq->id); } req.rcq_cid = cpu_to_le32(qp->rcq->id); qp_flags |= CMDQ_CREATE_QP_QP_FLAGS_RESERVED_LKEY_ENABLE; qp_flags |= CMDQ_CREATE_QP_QP_FLAGS_FR_PMR_ENABLED; if (qp->sig_type) qp_flags |= CMDQ_CREATE_QP_QP_FLAGS_FORCE_COMPLETION; if (qp->wqe_mode == BNXT_QPLIB_WQE_MODE_VARIABLE) qp_flags |= CMDQ_CREATE_QP_QP_FLAGS_VARIABLE_SIZED_WQE_ENABLED; if (res->cctx->modes.te_bypass) qp_flags |= CMDQ_CREATE_QP_QP_FLAGS_OPTIMIZED_TRANSMIT_ENABLED; if (res->dattr && bnxt_ext_stats_supported(qp->cctx, res->dattr->dev_cap_flags, res->is_vf)) qp_flags |= CMDQ_CREATE_QP_QP_FLAGS_EXT_STATS_ENABLED; req.qp_flags = cpu_to_le32(qp_flags); /* ORRQ and IRRQ */ if (psn_sz) { xrrq = &qp->orrq; xrrq->max_elements = ORD_LIMIT_TO_ORRQ_SLOTS(qp->max_rd_atomic); req_size = xrrq->max_elements * BNXT_QPLIB_MAX_ORRQE_ENTRY_SIZE + PAGE_SIZE - 1; req_size &= ~(PAGE_SIZE - 1); sginfo.pgsize = req_size; sginfo.pgshft = PAGE_SHIFT; hwq_attr.res = res; hwq_attr.sginfo = &sginfo; hwq_attr.depth = xrrq->max_elements; hwq_attr.stride = BNXT_QPLIB_MAX_ORRQE_ENTRY_SIZE; hwq_attr.aux_stride = 0; hwq_attr.aux_depth = 0; hwq_attr.type = HWQ_TYPE_CTX; rc = bnxt_qplib_alloc_init_hwq(xrrq, &hwq_attr); if (rc) goto fail_rq; req.orrq_addr = cpu_to_le64(_get_base_addr(xrrq)); xrrq = &qp->irrq; xrrq->max_elements = IRD_LIMIT_TO_IRRQ_SLOTS( qp->max_dest_rd_atomic); req_size = xrrq->max_elements * BNXT_QPLIB_MAX_IRRQE_ENTRY_SIZE + PAGE_SIZE - 1; req_size &= ~(PAGE_SIZE - 1); sginfo.pgsize = req_size; hwq_attr.depth = xrrq->max_elements; hwq_attr.stride = BNXT_QPLIB_MAX_IRRQE_ENTRY_SIZE; rc = bnxt_qplib_alloc_init_hwq(xrrq, &hwq_attr); if (rc) goto fail_orrq; req.irrq_addr = cpu_to_le64(_get_base_addr(xrrq)); } req.pd_id = cpu_to_le32(qp->pd->id); bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_CREATE_QP, sizeof(req)); bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL, sizeof(req), sizeof(resp), 0); rc = bnxt_qplib_rcfw_send_message(rcfw, &msg); if (rc) goto fail; if (!qp->is_user) { rc = bnxt_qplib_alloc_init_swq(sq); if (rc) goto swq_sq; if (!qp->srq) { rc = bnxt_qplib_alloc_init_swq(rq); if (rc) goto swq_rq; } if (psn_sz) bnxt_qplib_init_psn_ptr(qp, psn_sz); } qp->id = le32_to_cpu(resp.xid); qp->cur_qp_state = CMDQ_MODIFY_QP_NEW_STATE_RESET; INIT_LIST_HEAD(&qp->sq_flush); INIT_LIST_HEAD(&qp->rq_flush); sq->dbinfo.hwq = &sq->hwq; sq->dbinfo.xid = qp->id; sq->dbinfo.db = qp->dpi->dbr; sq->dbinfo.max_slot = _set_sq_max_slot(qp->wqe_mode); sq->dbinfo.flags = 0; spin_lock_init(&sq->dbinfo.lock); sq->dbinfo.shadow_key = BNXT_QPLIB_DBR_KEY_INVALID; sq->dbinfo.res = res; sq->dbinfo.seed = qp->id; if (rq->max_wqe) { rq->dbinfo.hwq = &rq->hwq; rq->dbinfo.xid = qp->id; rq->dbinfo.db = qp->dpi->dbr; rq->dbinfo.max_slot = _set_rq_max_slot(rq); rq->dbinfo.flags = 0; spin_lock_init(&rq->dbinfo.lock); rq->dbinfo.shadow_key = BNXT_QPLIB_DBR_KEY_INVALID; rq->dbinfo.res = res; rq->dbinfo.seed = qp->id; } tbl = &res->reftbl.qpref; qp_idx = map_qp_id_to_tbl_indx(qp->id, tbl); spin_lock_irqsave(&tbl->lock, flag); tbl->rec[qp_idx].xid = qp->id; tbl->rec[qp_idx].handle = qp; spin_unlock_irqrestore(&tbl->lock, flag); return 0; swq_rq: kfree(sq->swq); swq_sq: __qplib_destroy_qp(rcfw, qp); fail: bnxt_qplib_free_hwq(res, &qp->irrq); fail_orrq: bnxt_qplib_free_hwq(res, &qp->orrq); fail_rq: bnxt_qplib_free_hwq(res, &rq->hwq); fail_sq: bnxt_qplib_free_hwq(res, &sq->hwq); exit: return rc; } static void __filter_modify_flags(struct bnxt_qplib_qp *qp) { switch (qp->cur_qp_state) { case CMDQ_MODIFY_QP_NEW_STATE_RESET: switch (qp->state) { case CMDQ_MODIFY_QP_NEW_STATE_INIT: break; default: break; } break; case CMDQ_MODIFY_QP_NEW_STATE_INIT: switch (qp->state) { case CMDQ_MODIFY_QP_NEW_STATE_RTR: if (!(qp->modify_flags & CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU)) { qp->modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU; qp->path_mtu = CMDQ_MODIFY_QP_PATH_MTU_MTU_2048; } qp->modify_flags &= ~CMDQ_MODIFY_QP_MODIFY_MASK_VLAN_ID; /* Bono FW requires the max_dest_rd_atomic to be >= 1 */ if (qp->max_dest_rd_atomic < 1) qp->max_dest_rd_atomic = 1; qp->modify_flags &= ~CMDQ_MODIFY_QP_MODIFY_MASK_SRC_MAC; /* Bono FW 20.6.5 requires SGID_INDEX to be configured */ if (!(qp->modify_flags & CMDQ_MODIFY_QP_MODIFY_MASK_SGID_INDEX)) { qp->modify_flags |= CMDQ_MODIFY_QP_MODIFY_MASK_SGID_INDEX; qp->ah.sgid_index = 0; } break; default: break; } break; case CMDQ_MODIFY_QP_NEW_STATE_RTR: switch (qp->state) { case CMDQ_MODIFY_QP_NEW_STATE_RTS: /* Bono FW requires the max_rd_atomic to be >= 1 */ if (qp->max_rd_atomic < 1) qp->max_rd_atomic = 1; qp->modify_flags &= ~(CMDQ_MODIFY_QP_MODIFY_MASK_PKEY | CMDQ_MODIFY_QP_MODIFY_MASK_DGID | CMDQ_MODIFY_QP_MODIFY_MASK_FLOW_LABEL | CMDQ_MODIFY_QP_MODIFY_MASK_SGID_INDEX | CMDQ_MODIFY_QP_MODIFY_MASK_HOP_LIMIT | CMDQ_MODIFY_QP_MODIFY_MASK_TRAFFIC_CLASS | CMDQ_MODIFY_QP_MODIFY_MASK_DEST_MAC | CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU | CMDQ_MODIFY_QP_MODIFY_MASK_RQ_PSN | CMDQ_MODIFY_QP_MODIFY_MASK_MIN_RNR_TIMER | CMDQ_MODIFY_QP_MODIFY_MASK_MAX_DEST_RD_ATOMIC | CMDQ_MODIFY_QP_MODIFY_MASK_DEST_QP_ID); break; default: break; } break; case CMDQ_MODIFY_QP_NEW_STATE_RTS: break; case CMDQ_MODIFY_QP_NEW_STATE_SQD: break; case CMDQ_MODIFY_QP_NEW_STATE_SQE: break; case CMDQ_MODIFY_QP_NEW_STATE_ERR: break; default: break; } } int bnxt_qplib_modify_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp) { struct bnxt_qplib_rcfw *rcfw = res->rcfw; struct creq_modify_qp_resp resp = {}; struct bnxt_qplib_cmdqmsg msg = {}; struct cmdq_modify_qp req = {}; bool ppp_requested = false; u32 temp32[4]; u32 bmask; int rc; bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_MODIFY_QP, sizeof(req)); /* Filter out the qp_attr_mask based on the state->new transition */ __filter_modify_flags(qp); bmask = qp->modify_flags; req.modify_mask = cpu_to_le32(qp->modify_flags); req.qp_cid = cpu_to_le32(qp->id); if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_STATE) { req.network_type_en_sqd_async_notify_new_state = (qp->state & CMDQ_MODIFY_QP_NEW_STATE_MASK) | (qp->en_sqd_async_notify == true ? CMDQ_MODIFY_QP_EN_SQD_ASYNC_NOTIFY : 0); if (__can_request_ppp(qp)) { req.path_mtu_pingpong_push_enable = CMDQ_MODIFY_QP_PINGPONG_PUSH_ENABLE; req.pingpong_push_dpi = qp->ppp.dpi; ppp_requested = true; } } req.network_type_en_sqd_async_notify_new_state |= qp->nw_type; if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_ACCESS) { req.access = qp->access; } if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_PKEY) req.pkey = IB_DEFAULT_PKEY_FULL; if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_QKEY) { req.qkey = cpu_to_le32(qp->qkey); } if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_DGID) { memcpy(temp32, qp->ah.dgid.data, sizeof(struct bnxt_qplib_gid)); req.dgid[0] = cpu_to_le32(temp32[0]); req.dgid[1] = cpu_to_le32(temp32[1]); req.dgid[2] = cpu_to_le32(temp32[2]); req.dgid[3] = cpu_to_le32(temp32[3]); } if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_FLOW_LABEL) { req.flow_label = cpu_to_le32(qp->ah.flow_label); } if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_SGID_INDEX) { req.sgid_index = cpu_to_le16(res->sgid_tbl.hw_id[qp->ah.sgid_index]); } if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_HOP_LIMIT) { req.hop_limit = qp->ah.hop_limit; } if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_TRAFFIC_CLASS) { req.traffic_class = qp->ah.traffic_class; } if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_DEST_MAC) { memcpy(req.dest_mac, qp->ah.dmac, 6); } if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_PATH_MTU) { req.path_mtu_pingpong_push_enable = qp->path_mtu; } if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_TIMEOUT) { req.timeout = qp->timeout; } if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_RETRY_CNT) { req.retry_cnt = qp->retry_cnt; } if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_RNR_RETRY) { req.rnr_retry = qp->rnr_retry; } if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_MIN_RNR_TIMER) { req.min_rnr_timer = qp->min_rnr_timer; } if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_RQ_PSN) { req.rq_psn = cpu_to_le32(qp->rq.psn); } if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_SQ_PSN) { req.sq_psn = cpu_to_le32(qp->sq.psn); } if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_MAX_RD_ATOMIC) { req.max_rd_atomic = ORD_LIMIT_TO_ORRQ_SLOTS(qp->max_rd_atomic); } if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_MAX_DEST_RD_ATOMIC) { req.max_dest_rd_atomic = IRD_LIMIT_TO_IRRQ_SLOTS(qp->max_dest_rd_atomic); } req.sq_size = cpu_to_le32(qp->sq.hwq.max_elements); req.rq_size = cpu_to_le32(qp->rq.hwq.max_elements); req.sq_sge = cpu_to_le16(qp->sq.max_sge); req.rq_sge = cpu_to_le16(qp->rq.max_sge); req.max_inline_data = cpu_to_le32(qp->max_inline_data); if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_DEST_QP_ID) req.dest_qp_id = cpu_to_le32(qp->dest_qpn); if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_ENABLE_CC) req.enable_cc = cpu_to_le16(CMDQ_MODIFY_QP_ENABLE_CC); if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_TOS_ECN) req.tos_dscp_tos_ecn = ((qp->tos_ecn << CMDQ_MODIFY_QP_TOS_ECN_SFT) & CMDQ_MODIFY_QP_TOS_ECN_MASK); if (bmask & CMDQ_MODIFY_QP_MODIFY_MASK_TOS_DSCP) req.tos_dscp_tos_ecn |= ((qp->tos_dscp << CMDQ_MODIFY_QP_TOS_DSCP_SFT) & CMDQ_MODIFY_QP_TOS_DSCP_MASK); req.vlan_pcp_vlan_dei_vlan_id = cpu_to_le16(qp->vlan_id); bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL, sizeof(req), sizeof(resp), 0); msg.qp_state = qp->state; rc = bnxt_qplib_rcfw_send_message(rcfw, &msg); if (rc == -ETIMEDOUT && (qp->state == CMDQ_MODIFY_QP_NEW_STATE_ERR)) { qp->cur_qp_state = qp->state; return 0; } else if (rc) { return rc; } if (qp->state == CMDQ_MODIFY_QP_NEW_STATE_RTR) qp->lag_src_mac = be32_to_cpu(resp.lag_src_mac); if (ppp_requested) qp->ppp.st_idx_en = resp.pingpong_push_state_index_enabled; qp->cur_qp_state = qp->state; return 0; } int bnxt_qplib_query_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp) { struct bnxt_qplib_rcfw *rcfw = res->rcfw; struct creq_query_qp_resp resp = {}; struct bnxt_qplib_cmdqmsg msg = {}; struct bnxt_qplib_rcfw_sbuf sbuf; struct creq_query_qp_resp_sb *sb; struct cmdq_query_qp req = {}; u32 temp32[4]; int i, rc; sbuf.size = ALIGN(sizeof(*sb), BNXT_QPLIB_CMDQE_UNITS); sbuf.sb = dma_zalloc_coherent(&rcfw->pdev->dev, sbuf.size, &sbuf.dma_addr, GFP_KERNEL); if (!sbuf.sb) return -ENOMEM; sb = sbuf.sb; bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_QUERY_QP, sizeof(req)); req.qp_cid = cpu_to_le32(qp->id); req.resp_size = sbuf.size / BNXT_QPLIB_CMDQE_UNITS; bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, &sbuf, sizeof(req), sizeof(resp), 0); rc = bnxt_qplib_rcfw_send_message(rcfw, &msg); if (rc) goto bail; /* Extract the context from the side buffer */ qp->state = sb->en_sqd_async_notify_state & CREQ_QUERY_QP_RESP_SB_STATE_MASK; qp->cur_qp_state = qp->state; qp->en_sqd_async_notify = sb->en_sqd_async_notify_state & CREQ_QUERY_QP_RESP_SB_EN_SQD_ASYNC_NOTIFY ? true : false; qp->access = sb->access; qp->pkey_index = le16_to_cpu(sb->pkey); qp->qkey = le32_to_cpu(sb->qkey); temp32[0] = le32_to_cpu(sb->dgid[0]); temp32[1] = le32_to_cpu(sb->dgid[1]); temp32[2] = le32_to_cpu(sb->dgid[2]); temp32[3] = le32_to_cpu(sb->dgid[3]); memcpy(qp->ah.dgid.data, temp32, sizeof(qp->ah.dgid.data)); qp->ah.flow_label = le32_to_cpu(sb->flow_label); qp->ah.sgid_index = 0; for (i = 0; i < res->sgid_tbl.max; i++) { if (res->sgid_tbl.hw_id[i] == le16_to_cpu(sb->sgid_index)) { qp->ah.sgid_index = i; break; } } if (i == res->sgid_tbl.max) dev_dbg(&res->pdev->dev, "QPLIB: SGID not found qp->id = 0x%x sgid_index = 0x%x\n", qp->id, le16_to_cpu(sb->sgid_index)); qp->ah.hop_limit = sb->hop_limit; qp->ah.traffic_class = sb->traffic_class; memcpy(qp->ah.dmac, sb->dest_mac, ETH_ALEN); qp->ah.vlan_id = le16_to_cpu(sb->path_mtu_dest_vlan_id) & CREQ_QUERY_QP_RESP_SB_VLAN_ID_MASK >> CREQ_QUERY_QP_RESP_SB_VLAN_ID_SFT; qp->path_mtu = le16_to_cpu(sb->path_mtu_dest_vlan_id) & CREQ_QUERY_QP_RESP_SB_PATH_MTU_MASK; qp->timeout = sb->timeout; qp->retry_cnt = sb->retry_cnt; qp->rnr_retry = sb->rnr_retry; qp->min_rnr_timer = sb->min_rnr_timer; qp->rq.psn = le32_to_cpu(sb->rq_psn); qp->max_rd_atomic = ORRQ_SLOTS_TO_ORD_LIMIT(sb->max_rd_atomic); qp->sq.psn = le32_to_cpu(sb->sq_psn); qp->max_dest_rd_atomic = IRRQ_SLOTS_TO_IRD_LIMIT(sb->max_dest_rd_atomic); qp->sq.max_wqe = qp->sq.hwq.max_elements; qp->rq.max_wqe = qp->rq.hwq.max_elements; qp->sq.max_sge = le16_to_cpu(sb->sq_sge); qp->rq.max_sge = le16_to_cpu(sb->rq_sge); qp->max_inline_data = le32_to_cpu(sb->max_inline_data); qp->dest_qpn = le32_to_cpu(sb->dest_qp_id); memcpy(qp->smac, sb->src_mac, ETH_ALEN); qp->vlan_id = le16_to_cpu(sb->vlan_pcp_vlan_dei_vlan_id); qp->port_id = le16_to_cpu(sb->port_id); bail: dma_free_coherent(&rcfw->pdev->dev, sbuf.size, sbuf.sb, sbuf.dma_addr); return rc; } static void __clean_cq(struct bnxt_qplib_cq *cq, u64 qp) { struct bnxt_qplib_hwq *cq_hwq = &cq->hwq; u32 peek_flags, peek_cons; struct cq_base *hw_cqe; int i; peek_flags = cq->dbinfo.flags; peek_cons = cq_hwq->cons; for (i = 0; i < cq_hwq->depth; i++) { hw_cqe = bnxt_qplib_get_qe(cq_hwq, peek_cons, NULL); if (CQE_CMP_VALID(hw_cqe, peek_flags)) { dma_rmb(); switch (hw_cqe->cqe_type_toggle & CQ_BASE_CQE_TYPE_MASK) { case CQ_BASE_CQE_TYPE_REQ: case CQ_BASE_CQE_TYPE_TERMINAL: { struct cq_req *cqe = (struct cq_req *)hw_cqe; if (qp == le64_to_cpu(cqe->qp_handle)) cqe->qp_handle = 0; break; } case CQ_BASE_CQE_TYPE_RES_RC: case CQ_BASE_CQE_TYPE_RES_UD: case CQ_BASE_CQE_TYPE_RES_RAWETH_QP1: { struct cq_res_rc *cqe = (struct cq_res_rc *)hw_cqe; if (qp == le64_to_cpu(cqe->qp_handle)) cqe->qp_handle = 0; break; } default: break; } } bnxt_qplib_hwq_incr_cons(cq_hwq->depth, &peek_cons, 1, &peek_flags); } } int bnxt_qplib_destroy_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp) { struct bnxt_qplib_rcfw *rcfw = res->rcfw; struct bnxt_qplib_reftbl *tbl; unsigned long flags; u32 qp_idx; int rc; tbl = &res->reftbl.qpref; qp_idx = map_qp_id_to_tbl_indx(qp->id, tbl); spin_lock_irqsave(&tbl->lock, flags); tbl->rec[qp_idx].xid = BNXT_QPLIB_QP_ID_INVALID; tbl->rec[qp_idx].handle = NULL; spin_unlock_irqrestore(&tbl->lock, flags); rc = __qplib_destroy_qp(rcfw, qp); if (rc) { spin_lock_irqsave(&tbl->lock, flags); tbl->rec[qp_idx].xid = qp->id; tbl->rec[qp_idx].handle = qp; spin_unlock_irqrestore(&tbl->lock, flags); return rc; } return 0; } void bnxt_qplib_free_qp_res(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp) { if (qp->irrq.max_elements) bnxt_qplib_free_hwq(res, &qp->irrq); if (qp->orrq.max_elements) bnxt_qplib_free_hwq(res, &qp->orrq); if (!qp->is_user) kfree(qp->rq.swq); bnxt_qplib_free_hwq(res, &qp->rq.hwq); if (!qp->is_user) kfree(qp->sq.swq); bnxt_qplib_free_hwq(res, &qp->sq.hwq); } void *bnxt_qplib_get_qp1_sq_buf(struct bnxt_qplib_qp *qp, struct bnxt_qplib_sge *sge) { struct bnxt_qplib_q *sq = &qp->sq; struct bnxt_qplib_hdrbuf *buf; u32 sw_prod; memset(sge, 0, sizeof(*sge)); buf = qp->sq_hdr_buf; if (buf) { sw_prod = sq->swq_start; sge->addr = (dma_addr_t)(buf->dma_map + sw_prod * buf->step); sge->lkey = 0xFFFFFFFF; sge->size = buf->step; return buf->va + sw_prod * sge->size; } return NULL; } u32 bnxt_qplib_get_rq_prod_index(struct bnxt_qplib_qp *qp) { struct bnxt_qplib_q *rq = &qp->rq; return rq->swq_start; } void *bnxt_qplib_get_qp1_rq_buf(struct bnxt_qplib_qp *qp, struct bnxt_qplib_sge *sge) { struct bnxt_qplib_q *rq = &qp->rq; struct bnxt_qplib_hdrbuf *buf; u32 sw_prod; memset(sge, 0, sizeof(*sge)); buf = qp->rq_hdr_buf; if (buf) { sw_prod = rq->swq_start; sge->addr = (dma_addr_t)(buf->dma_map + sw_prod * buf->step); sge->lkey = 0xFFFFFFFF; sge->size = buf->step; return buf->va + sw_prod * sge->size; } return NULL; } /* Fil the MSN table into the next psn row */ static void bnxt_qplib_fill_msn_search(struct bnxt_qplib_qp *qp, struct bnxt_qplib_swqe *wqe, struct bnxt_qplib_swq *swq) { struct sq_msn_search *msns; u32 start_psn, next_psn; u16 start_idx; msns = (struct sq_msn_search *)swq->psn_search; msns->start_idx_next_psn_start_psn = 0; start_psn = swq->start_psn; next_psn = swq->next_psn; start_idx = swq->slot_idx; msns->start_idx_next_psn_start_psn |= bnxt_re_update_msn_tbl(start_idx, next_psn, start_psn); pr_debug("QP_LIB MSN %d START_IDX %u NEXT_PSN %u START_PSN %u\n", qp->msn, (u16) cpu_to_le16(BNXT_RE_MSN_IDX(msns->start_idx_next_psn_start_psn)), (u32) cpu_to_le32(BNXT_RE_MSN_NPSN(msns->start_idx_next_psn_start_psn)), (u32) cpu_to_le32(BNXT_RE_MSN_SPSN(msns->start_idx_next_psn_start_psn))); qp->msn++; qp->msn %= qp->msn_tbl_sz; } static void bnxt_qplib_fill_psn_search(struct bnxt_qplib_qp *qp, struct bnxt_qplib_swqe *wqe, struct bnxt_qplib_swq *swq) { struct sq_psn_search_ext *psns_ext; struct sq_psn_search *psns; u32 flg_npsn; u32 op_spsn; if (!swq->psn_search) return; /* Handle MSN differently on cap flags */ - if (BNXT_RE_HW_RETX(qp->dev_cap_flags)) { + if (qp->is_host_msn_tbl) { bnxt_qplib_fill_msn_search(qp, wqe, swq); return; } psns = (struct sq_psn_search *)swq->psn_search; psns_ext = (struct sq_psn_search_ext *)swq->psn_search; op_spsn = ((swq->start_psn << SQ_PSN_SEARCH_START_PSN_SFT) & SQ_PSN_SEARCH_START_PSN_MASK); op_spsn |= ((wqe->type << SQ_PSN_SEARCH_OPCODE_SFT) & SQ_PSN_SEARCH_OPCODE_MASK); flg_npsn = ((swq->next_psn << SQ_PSN_SEARCH_NEXT_PSN_SFT) & SQ_PSN_SEARCH_NEXT_PSN_MASK); if (_is_chip_gen_p5_p7(qp->cctx)) { psns_ext->opcode_start_psn = cpu_to_le32(op_spsn); psns_ext->flags_next_psn = cpu_to_le32(flg_npsn); psns_ext->start_slot_idx = cpu_to_le16(swq->slot_idx); } else { psns->opcode_start_psn = cpu_to_le32(op_spsn); psns->flags_next_psn = cpu_to_le32(flg_npsn); } } static u16 _calc_ilsize(struct bnxt_qplib_swqe *wqe) { u16 size = 0; int indx; for (indx = 0; indx < wqe->num_sge; indx++) size += wqe->sg_list[indx].size; return size; } static int bnxt_qplib_put_inline(struct bnxt_qplib_qp *qp, struct bnxt_qplib_swqe *wqe, u32 *sw_prod) { struct bnxt_qplib_hwq *sq_hwq; int len, t_len, offt = 0; int t_cplen = 0, cplen; bool pull_dst = true; void *il_dst = NULL; void *il_src = NULL; int indx; sq_hwq = &qp->sq.hwq; t_len = 0; for (indx = 0; indx < wqe->num_sge; indx++) { len = wqe->sg_list[indx].size; il_src = (void *)wqe->sg_list[indx].addr; t_len += len; if (t_len > qp->max_inline_data) goto bad; while (len) { if (pull_dst) { pull_dst = false; il_dst = bnxt_qplib_get_qe(sq_hwq, ((*sw_prod) % sq_hwq->depth), NULL); (*sw_prod)++; t_cplen = 0; offt = 0; } cplen = min_t(int, len, sizeof(struct sq_sge)); cplen = min_t(int, cplen, (sizeof(struct sq_sge) - offt)); memcpy(il_dst, il_src, cplen); t_cplen += cplen; il_src += cplen; il_dst += cplen; offt += cplen; len -= cplen; if (t_cplen == sizeof(struct sq_sge)) pull_dst = true; } } return t_len; bad: return -ENOMEM; } static int bnxt_qplib_put_sges(struct bnxt_qplib_hwq *sq_hwq, struct bnxt_qplib_sge *ssge, u32 nsge, u32 *sw_prod) { struct sq_sge *dsge; int indx, len = 0; for (indx = 0; indx < nsge; indx++, (*sw_prod)++) { dsge = bnxt_qplib_get_qe(sq_hwq, ((*sw_prod) % sq_hwq->depth), NULL); dsge->va_or_pa = cpu_to_le64(ssge[indx].addr); dsge->l_key = cpu_to_le32(ssge[indx].lkey); dsge->size = cpu_to_le32(ssge[indx].size); len += ssge[indx].size; } return len; } static u16 _calculate_wqe_byte(struct bnxt_qplib_qp *qp, struct bnxt_qplib_swqe *wqe, u16 *wqe_byte) { u16 wqe_size; u32 ilsize; u16 nsge; nsge = wqe->num_sge; if (wqe->flags & BNXT_QPLIB_SWQE_FLAGS_INLINE) { ilsize = _calc_ilsize(wqe); wqe_size = (ilsize > qp->max_inline_data) ? qp->max_inline_data : ilsize; wqe_size = ALIGN(wqe_size, sizeof(struct sq_sge)); } else { wqe_size = nsge * sizeof(struct sq_sge); } /* Adding sq_send_hdr is a misnomer, for rq also hdr size is same. */ wqe_size += sizeof(struct sq_send_hdr); if (wqe_byte) *wqe_byte = wqe_size; return wqe_size / sizeof(struct sq_sge); } static u16 _translate_q_full_delta(struct bnxt_qplib_q *que, u16 wqe_bytes) { /* For Cu/Wh delta = 128, stride = 16, wqe_bytes = 128 * For Gen-p5 B/C mode delta = 0, stride = 16, wqe_bytes = 128. * For Gen-p5 delta = 0, stride = 16, 32 <= wqe_bytes <= 512. * when 8916 is disabled. */ return (que->q_full_delta * wqe_bytes) / que->hwq.element_size; } static void bnxt_qplib_pull_psn_buff(struct bnxt_qplib_qp *qp, struct bnxt_qplib_q *sq, - struct bnxt_qplib_swq *swq, bool hw_retx) + struct bnxt_qplib_swq *swq, bool is_host_msn_tbl) { struct bnxt_qplib_hwq *sq_hwq; u32 pg_num, pg_indx; void *buff; u32 tail; sq_hwq = &sq->hwq; if (!sq_hwq->pad_pg) return; tail = swq->slot_idx / sq->dbinfo.max_slot; - if (hw_retx) + if (is_host_msn_tbl) { + /* For HW retx use qp msn index */ + tail = qp->msn; tail %= qp->msn_tbl_sz; + } pg_num = (tail + sq_hwq->pad_pgofft) / (PAGE_SIZE / sq_hwq->pad_stride); pg_indx = (tail + sq_hwq->pad_pgofft) % (PAGE_SIZE / sq_hwq->pad_stride); buff = (void *)(sq_hwq->pad_pg[pg_num] + pg_indx * sq_hwq->pad_stride); /* the start ptr for buff is same ie after the SQ */ swq->psn_search = buff; } void bnxt_qplib_post_send_db(struct bnxt_qplib_qp *qp) { struct bnxt_qplib_q *sq = &qp->sq; bnxt_qplib_ring_prod_db(&sq->dbinfo, DBC_DBC_TYPE_SQ); } int bnxt_qplib_post_send(struct bnxt_qplib_qp *qp, struct bnxt_qplib_swqe *wqe) { struct bnxt_qplib_nq_work *nq_work = NULL; int i, rc = 0, data_len = 0, pkt_num = 0; struct bnxt_qplib_q *sq = &qp->sq; struct bnxt_qplib_hwq *sq_hwq; struct bnxt_qplib_swq *swq; bool sch_handler = false; u16 slots_needed; + bool msn_update; void *base_hdr; void *ext_hdr; __le32 temp32; u16 qfd_slots; u8 wqe_slots; u16 wqe_size; u32 sw_prod; u32 wqe_idx; sq_hwq = &sq->hwq; if (qp->state != CMDQ_MODIFY_QP_NEW_STATE_RTS && qp->state != CMDQ_MODIFY_QP_NEW_STATE_ERR) { dev_err(&sq_hwq->pdev->dev, "QPLIB: FP: QP (0x%x) is in the 0x%x state\n", qp->id, qp->state); rc = -EINVAL; goto done; } wqe_slots = _calculate_wqe_byte(qp, wqe, &wqe_size); slots_needed = (qp->wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC) ? sq->dbinfo.max_slot : wqe_slots; qfd_slots = _translate_q_full_delta(sq, wqe_size); if (bnxt_qplib_queue_full(sq_hwq, (slots_needed + qfd_slots))) { dev_err(&sq_hwq->pdev->dev, "QPLIB: FP: QP (0x%x) SQ is full!\n", qp->id); dev_err(&sq_hwq->pdev->dev, "QPLIB: prod = %#x cons = %#x qdepth = %#x delta = %#x slots = %#x\n", HWQ_CMP(sq_hwq->prod, sq_hwq), HWQ_CMP(sq_hwq->cons, sq_hwq), sq_hwq->max_elements, qfd_slots, slots_needed); dev_err(&sq_hwq->pdev->dev, "QPLIB: phantom_wqe_cnt: %d phantom_cqe_cnt: %d\n", sq->phantom_wqe_cnt, sq->phantom_cqe_cnt); rc = -ENOMEM; goto done; } sw_prod = sq_hwq->prod; swq = bnxt_qplib_get_swqe(sq, &wqe_idx); swq->slot_idx = sw_prod; - bnxt_qplib_pull_psn_buff(qp, sq, swq, BNXT_RE_HW_RETX(qp->dev_cap_flags)); + bnxt_qplib_pull_psn_buff(qp, sq, swq, qp->is_host_msn_tbl); swq->wr_id = wqe->wr_id; swq->type = wqe->type; swq->flags = wqe->flags; swq->slots = slots_needed; swq->start_psn = sq->psn & BTH_PSN_MASK; if (qp->sig_type || wqe->flags & BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP) swq->flags |= SQ_SEND_FLAGS_SIGNAL_COMP; dev_dbg(&sq_hwq->pdev->dev, "QPLIB: FP: QP(0x%x) post SQ wr_id[%d] = 0x%llx\n", qp->id, wqe_idx, swq->wr_id); if (qp->cur_qp_state == CMDQ_MODIFY_QP_NEW_STATE_ERR) { sch_handler = true; dev_dbg(&sq_hwq->pdev->dev, "%s Error QP. Scheduling for poll_cq\n", __func__); goto queue_err; } base_hdr = bnxt_qplib_get_qe(sq_hwq, sw_prod, NULL); sw_prod++; ext_hdr = bnxt_qplib_get_qe(sq_hwq, (sw_prod % sq_hwq->depth), NULL); sw_prod++; memset(base_hdr, 0, sizeof(struct sq_sge)); memset(ext_hdr, 0, sizeof(struct sq_sge)); if (wqe->flags & BNXT_QPLIB_SWQE_FLAGS_INLINE) data_len = bnxt_qplib_put_inline(qp, wqe, &sw_prod); else data_len = bnxt_qplib_put_sges(sq_hwq, wqe->sg_list, wqe->num_sge, &sw_prod); if (data_len < 0) goto queue_err; + /* Make sure we update MSN table only for wired wqes */ + msn_update = true; + /* Specifics */ switch (wqe->type) { case BNXT_QPLIB_SWQE_TYPE_SEND: if (qp->type == CMDQ_CREATE_QP_TYPE_RAW_ETHERTYPE || qp->type == CMDQ_CREATE_QP1_TYPE_GSI) { /* Assemble info for Raw Ethertype QPs */ struct sq_send_raweth_qp1_hdr *sqe = base_hdr; struct sq_raw_ext_hdr *ext_sqe = ext_hdr; sqe->wqe_type = wqe->type; sqe->flags = wqe->flags; sqe->wqe_size = wqe_slots; sqe->cfa_action = cpu_to_le16(wqe->rawqp1.cfa_action); sqe->lflags = cpu_to_le16(wqe->rawqp1.lflags); sqe->length = cpu_to_le32(data_len); ext_sqe->cfa_meta = cpu_to_le32((wqe->rawqp1.cfa_meta & SQ_SEND_RAWETH_QP1_CFA_META_VLAN_VID_MASK) << SQ_SEND_RAWETH_QP1_CFA_META_VLAN_VID_SFT); dev_dbg(&sq_hwq->pdev->dev, "QPLIB: FP: RAW/QP1 Send WQE:\n" "\twqe_type = 0x%x\n" "\tflags = 0x%x\n" "\twqe_size = 0x%x\n" "\tlflags = 0x%x\n" "\tcfa_action = 0x%x\n" "\tlength = 0x%x\n" "\tcfa_meta = 0x%x\n", sqe->wqe_type, sqe->flags, sqe->wqe_size, sqe->lflags, sqe->cfa_action, sqe->length, ext_sqe->cfa_meta); break; } fallthrough; case BNXT_QPLIB_SWQE_TYPE_SEND_WITH_IMM: fallthrough; case BNXT_QPLIB_SWQE_TYPE_SEND_WITH_INV: { struct sq_send_hdr *sqe = base_hdr; struct sq_ud_ext_hdr *ext_sqe = ext_hdr; sqe->wqe_type = wqe->type; sqe->flags = wqe->flags; sqe->wqe_size = wqe_slots; sqe->inv_key_or_imm_data = cpu_to_le32(wqe->send.inv_key); if (qp->type == CMDQ_CREATE_QP_TYPE_UD || qp->type == CMDQ_CREATE_QP_TYPE_GSI) { sqe->q_key = cpu_to_le32(wqe->send.q_key); sqe->length = cpu_to_le32(data_len); ext_sqe->dst_qp = cpu_to_le32( wqe->send.dst_qp & SQ_SEND_DST_QP_MASK); ext_sqe->avid = cpu_to_le32(wqe->send.avid & SQ_SEND_AVID_MASK); sq->psn = (sq->psn + 1) & BTH_PSN_MASK; + msn_update = false; } else { sqe->length = cpu_to_le32(data_len); if (qp->mtu) pkt_num = (data_len + qp->mtu - 1) / qp->mtu; if (!pkt_num) pkt_num = 1; sq->psn = (sq->psn + pkt_num) & BTH_PSN_MASK; } dev_dbg(&sq_hwq->pdev->dev, "QPLIB: FP: Send WQE:\n" "\twqe_type = 0x%x\n" "\tflags = 0x%x\n" "\twqe_size = 0x%x\n" "\tinv_key/immdata = 0x%x\n" "\tq_key = 0x%x\n" "\tdst_qp = 0x%x\n" "\tlength = 0x%x\n" "\tavid = 0x%x\n", sqe->wqe_type, sqe->flags, sqe->wqe_size, sqe->inv_key_or_imm_data, sqe->q_key, ext_sqe->dst_qp, sqe->length, ext_sqe->avid); break; } case BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE: /* fall-thru */ case BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE_WITH_IMM: /* fall-thru */ case BNXT_QPLIB_SWQE_TYPE_RDMA_READ: { struct sq_rdma_hdr *sqe = base_hdr; struct sq_rdma_ext_hdr *ext_sqe = ext_hdr; sqe->wqe_type = wqe->type; sqe->flags = wqe->flags; sqe->wqe_size = wqe_slots; sqe->imm_data = cpu_to_le32(wqe->rdma.inv_key); sqe->length = cpu_to_le32((u32)data_len); ext_sqe->remote_va = cpu_to_le64(wqe->rdma.remote_va); ext_sqe->remote_key = cpu_to_le32(wqe->rdma.r_key); if (qp->mtu) pkt_num = (data_len + qp->mtu - 1) / qp->mtu; if (!pkt_num) pkt_num = 1; sq->psn = (sq->psn + pkt_num) & BTH_PSN_MASK; dev_dbg(&sq_hwq->pdev->dev, "QPLIB: FP: RDMA WQE:\n" "\twqe_type = 0x%x\n" "\tflags = 0x%x\n" "\twqe_size = 0x%x\n" "\timmdata = 0x%x\n" "\tlength = 0x%x\n" "\tremote_va = 0x%llx\n" "\tremote_key = 0x%x\n", sqe->wqe_type, sqe->flags, sqe->wqe_size, sqe->imm_data, sqe->length, ext_sqe->remote_va, ext_sqe->remote_key); break; } case BNXT_QPLIB_SWQE_TYPE_ATOMIC_CMP_AND_SWP: /* fall-thru */ case BNXT_QPLIB_SWQE_TYPE_ATOMIC_FETCH_AND_ADD: { struct sq_atomic_hdr *sqe = base_hdr; struct sq_atomic_ext_hdr *ext_sqe = ext_hdr; sqe->wqe_type = wqe->type; sqe->flags = wqe->flags; sqe->remote_key = cpu_to_le32(wqe->atomic.r_key); sqe->remote_va = cpu_to_le64(wqe->atomic.remote_va); ext_sqe->swap_data = cpu_to_le64(wqe->atomic.swap_data); ext_sqe->cmp_data = cpu_to_le64(wqe->atomic.cmp_data); if (qp->mtu) pkt_num = (data_len + qp->mtu - 1) / qp->mtu; if (!pkt_num) pkt_num = 1; sq->psn = (sq->psn + pkt_num) & BTH_PSN_MASK; break; } case BNXT_QPLIB_SWQE_TYPE_LOCAL_INV: { struct sq_localinvalidate_hdr *sqe = base_hdr; sqe->wqe_type = wqe->type; sqe->flags = wqe->flags; sqe->inv_l_key = cpu_to_le32(wqe->local_inv.inv_l_key); dev_dbg(&sq_hwq->pdev->dev, "QPLIB: FP: LOCAL INV WQE:\n" "\twqe_type = 0x%x\n" "\tflags = 0x%x\n" "\tinv_l_key = 0x%x\n", sqe->wqe_type, sqe->flags, sqe->inv_l_key); + msn_update = false; break; } case BNXT_QPLIB_SWQE_TYPE_FAST_REG_MR: { struct sq_fr_pmr_hdr *sqe = base_hdr; struct sq_fr_pmr_ext_hdr *ext_sqe = ext_hdr; sqe->wqe_type = wqe->type; sqe->flags = wqe->flags; sqe->access_cntl = wqe->frmr.access_cntl | SQ_FR_PMR_ACCESS_CNTL_LOCAL_WRITE; sqe->zero_based_page_size_log = (wqe->frmr.pg_sz_log & SQ_FR_PMR_PAGE_SIZE_LOG_MASK) << SQ_FR_PMR_PAGE_SIZE_LOG_SFT | (wqe->frmr.zero_based == true ? SQ_FR_PMR_ZERO_BASED : 0); sqe->l_key = cpu_to_le32(wqe->frmr.l_key); /* TODO: OFED only provides length of MR up to 32-bits for FRMR */ temp32 = cpu_to_le32(wqe->frmr.length); memcpy(sqe->length, &temp32, sizeof(wqe->frmr.length)); sqe->numlevels_pbl_page_size_log = ((wqe->frmr.pbl_pg_sz_log << SQ_FR_PMR_PBL_PAGE_SIZE_LOG_SFT) & SQ_FR_PMR_PBL_PAGE_SIZE_LOG_MASK) | ((wqe->frmr.levels << SQ_FR_PMR_NUMLEVELS_SFT) & SQ_FR_PMR_NUMLEVELS_MASK); if (!wqe->frmr.levels && !wqe->frmr.pbl_ptr) { ext_sqe->pblptr = cpu_to_le64(wqe->frmr.page_list[0]); } else { for (i = 0; i < wqe->frmr.page_list_len; i++) wqe->frmr.pbl_ptr[i] = cpu_to_le64( wqe->frmr.page_list[i] | PTU_PTE_VALID); ext_sqe->pblptr = cpu_to_le64(wqe->frmr.pbl_dma_ptr); } ext_sqe->va = cpu_to_le64(wqe->frmr.va); dev_dbg(&sq_hwq->pdev->dev, "QPLIB: FP: FRMR WQE:\n" "\twqe_type = 0x%x\n" "\tflags = 0x%x\n" "\taccess_cntl = 0x%x\n" "\tzero_based_page_size_log = 0x%x\n" "\tl_key = 0x%x\n" "\tlength = 0x%x\n" "\tnumlevels_pbl_page_size_log = 0x%x\n" "\tpblptr = 0x%llx\n" "\tva = 0x%llx\n", sqe->wqe_type, sqe->flags, sqe->access_cntl, sqe->zero_based_page_size_log, sqe->l_key, *(u32 *)sqe->length, sqe->numlevels_pbl_page_size_log, ext_sqe->pblptr, ext_sqe->va); + msn_update = false; break; } case BNXT_QPLIB_SWQE_TYPE_BIND_MW: { struct sq_bind_hdr *sqe = base_hdr; struct sq_bind_ext_hdr *ext_sqe = ext_hdr; sqe->wqe_type = wqe->type; sqe->flags = wqe->flags; sqe->access_cntl = wqe->bind.access_cntl; sqe->mw_type_zero_based = wqe->bind.mw_type | (wqe->bind.zero_based == true ? SQ_BIND_ZERO_BASED : 0); sqe->parent_l_key = cpu_to_le32(wqe->bind.parent_l_key); sqe->l_key = cpu_to_le32(wqe->bind.r_key); ext_sqe->va = cpu_to_le64(wqe->bind.va); ext_sqe->length_lo = cpu_to_le32(wqe->bind.length); dev_dbg(&sq_hwq->pdev->dev, "QPLIB: FP: BIND WQE:\n" "\twqe_type = 0x%x\n" "\tflags = 0x%x\n" "\taccess_cntl = 0x%x\n" "\tmw_type_zero_based = 0x%x\n" "\tparent_l_key = 0x%x\n" "\tl_key = 0x%x\n" "\tva = 0x%llx\n" "\tlength = 0x%x\n", sqe->wqe_type, sqe->flags, sqe->access_cntl, sqe->mw_type_zero_based, sqe->parent_l_key, sqe->l_key, sqe->va, ext_sqe->length_lo); + msn_update = false; break; } default: /* Bad wqe, return error */ rc = -EINVAL; goto done; } - swq->next_psn = sq->psn & BTH_PSN_MASK; - bnxt_qplib_fill_psn_search(qp, wqe, swq); + if (!qp->is_host_msn_tbl || msn_update) { + swq->next_psn = sq->psn & BTH_PSN_MASK; + bnxt_qplib_fill_psn_search(qp, wqe, swq); + } queue_err: bnxt_qplib_swq_mod_start(sq, wqe_idx); bnxt_qplib_hwq_incr_prod(&sq->dbinfo, sq_hwq, swq->slots); qp->wqe_cnt++; done: if (sch_handler) { nq_work = kzalloc(sizeof(*nq_work), GFP_ATOMIC); if (nq_work) { nq_work->cq = qp->scq; nq_work->nq = qp->scq->nq; INIT_WORK(&nq_work->work, bnxt_qpn_cqn_sched_task); queue_work(qp->scq->nq->cqn_wq, &nq_work->work); } else { dev_err(&sq->hwq.pdev->dev, "QPLIB: FP: Failed to allocate SQ nq_work!\n"); rc = -ENOMEM; } } return rc; } void bnxt_qplib_post_recv_db(struct bnxt_qplib_qp *qp) { struct bnxt_qplib_q *rq = &qp->rq; bnxt_qplib_ring_prod_db(&rq->dbinfo, DBC_DBC_TYPE_RQ); } void bnxt_re_handle_cqn(struct bnxt_qplib_cq *cq) { struct bnxt_qplib_nq *nq; if (!(cq && cq->nq)) return; nq = cq->nq; spin_lock_bh(&cq->compl_lock); if (nq->cqn_handler) { dev_dbg(&nq->res->pdev->dev, "%s:Trigger cq = %p event nq = %p\n", __func__, cq, nq); nq->cqn_handler(nq, cq); } spin_unlock_bh(&cq->compl_lock); } int bnxt_qplib_post_recv(struct bnxt_qplib_qp *qp, struct bnxt_qplib_swqe *wqe) { struct bnxt_qplib_nq_work *nq_work = NULL; struct bnxt_qplib_q *rq = &qp->rq; struct bnxt_qplib_hwq *rq_hwq; struct bnxt_qplib_swq *swq; bool sch_handler = false; struct rq_wqe_hdr *base_hdr; struct rq_ext_hdr *ext_hdr; struct sq_sge *dsge; u8 wqe_slots; u32 wqe_idx; u32 sw_prod; int rc = 0; rq_hwq = &rq->hwq; if (qp->state == CMDQ_MODIFY_QP_NEW_STATE_RESET) { dev_err(&rq_hwq->pdev->dev, "QPLIB: FP: QP (0x%x) is in the 0x%x state\n", qp->id, qp->state); rc = -EINVAL; goto done; } wqe_slots = _calculate_wqe_byte(qp, wqe, NULL); if (bnxt_qplib_queue_full(rq_hwq, rq->dbinfo.max_slot)) { dev_err(&rq_hwq->pdev->dev, "QPLIB: FP: QP (0x%x) RQ is full!\n", qp->id); rc = -EINVAL; goto done; } swq = bnxt_qplib_get_swqe(rq, &wqe_idx); swq->wr_id = wqe->wr_id; swq->slots = rq->dbinfo.max_slot; dev_dbg(&rq_hwq->pdev->dev, "QPLIB: FP: post RQ wr_id[%d] = 0x%llx\n", wqe_idx, swq->wr_id); if (qp->cur_qp_state == CMDQ_MODIFY_QP_NEW_STATE_ERR) { sch_handler = true; dev_dbg(&rq_hwq->pdev->dev, "%s Error QP. Sched a flushed cmpl\n", __func__); goto queue_err; } sw_prod = rq_hwq->prod; base_hdr = bnxt_qplib_get_qe(rq_hwq, sw_prod, NULL); sw_prod++; ext_hdr = bnxt_qplib_get_qe(rq_hwq, (sw_prod % rq_hwq->depth), NULL); sw_prod++; memset(base_hdr, 0, sizeof(struct sq_sge)); memset(ext_hdr, 0, sizeof(struct sq_sge)); if (!wqe->num_sge) { dsge = bnxt_qplib_get_qe(rq_hwq, (sw_prod % rq_hwq->depth), NULL); dsge->size = 0; wqe_slots++; } else { bnxt_qplib_put_sges(rq_hwq, wqe->sg_list, wqe->num_sge, &sw_prod); } base_hdr->wqe_type = wqe->type; base_hdr->flags = wqe->flags; base_hdr->wqe_size = wqe_slots; base_hdr->wr_id |= cpu_to_le32(wqe_idx); queue_err: bnxt_qplib_swq_mod_start(rq, wqe_idx); bnxt_qplib_hwq_incr_prod(&rq->dbinfo, &rq->hwq, swq->slots); done: if (sch_handler) { nq_work = kzalloc(sizeof(*nq_work), GFP_ATOMIC); if (nq_work) { nq_work->cq = qp->rcq; nq_work->nq = qp->rcq->nq; INIT_WORK(&nq_work->work, bnxt_qpn_cqn_sched_task); queue_work(qp->rcq->nq->cqn_wq, &nq_work->work); } else { dev_err(&rq->hwq.pdev->dev, "QPLIB: FP: Failed to allocate RQ nq_work!\n"); rc = -ENOMEM; } } return rc; } /* CQ */ int bnxt_qplib_create_cq(struct bnxt_qplib_res *res, struct bnxt_qplib_cq *cq) { struct bnxt_qplib_hwq_attr hwq_attr = {}; struct bnxt_qplib_rcfw *rcfw = res->rcfw; struct creq_create_cq_resp resp = {}; struct bnxt_qplib_cmdqmsg msg = {}; struct cmdq_create_cq req = {}; struct bnxt_qplib_reftbl *tbl; unsigned long flag; u32 pg_sz_lvl = 0; int rc; hwq_attr.res = res; hwq_attr.depth = cq->max_wqe; hwq_attr.stride = sizeof(struct cq_base); hwq_attr.type = HWQ_TYPE_QUEUE; hwq_attr.sginfo = &cq->sginfo; rc = bnxt_qplib_alloc_init_hwq(&cq->hwq, &hwq_attr); if (rc) goto exit; bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_CREATE_CQ, sizeof(req)); if (!cq->dpi) { dev_err(&rcfw->pdev->dev, "QPLIB: FP: CREATE_CQ failed due to NULL DPI\n"); return -EINVAL; } req.dpi = cpu_to_le32(cq->dpi->dpi); req.cq_handle = cpu_to_le64(cq->cq_handle); req.cq_size = cpu_to_le32(cq->max_wqe); req.pbl = cpu_to_le64(_get_base_addr(&cq->hwq)); pg_sz_lvl = _get_base_pg_size(&cq->hwq) << CMDQ_CREATE_CQ_PG_SIZE_SFT; pg_sz_lvl |= ((cq->hwq.level & CMDQ_CREATE_CQ_LVL_MASK) << CMDQ_CREATE_CQ_LVL_SFT); req.pg_size_lvl = cpu_to_le32(pg_sz_lvl); req.cq_fco_cnq_id = cpu_to_le32( (cq->cnq_hw_ring_id & CMDQ_CREATE_CQ_CNQ_ID_MASK) << CMDQ_CREATE_CQ_CNQ_ID_SFT); bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL, sizeof(req), sizeof(resp), 0); rc = bnxt_qplib_rcfw_send_message(rcfw, &msg); if (rc) goto fail; cq->id = le32_to_cpu(resp.xid); cq->period = BNXT_QPLIB_QUEUE_START_PERIOD; init_waitqueue_head(&cq->waitq); INIT_LIST_HEAD(&cq->sqf_head); INIT_LIST_HEAD(&cq->rqf_head); spin_lock_init(&cq->flush_lock); spin_lock_init(&cq->compl_lock); /* init dbinfo */ cq->cctx = res->cctx; cq->dbinfo.hwq = &cq->hwq; cq->dbinfo.xid = cq->id; cq->dbinfo.db = cq->dpi->dbr; cq->dbinfo.priv_db = res->dpi_tbl.priv_db; cq->dbinfo.flags = 0; cq->dbinfo.toggle = 0; cq->dbinfo.res = res; cq->dbinfo.seed = cq->id; spin_lock_init(&cq->dbinfo.lock); cq->dbinfo.shadow_key = BNXT_QPLIB_DBR_KEY_INVALID; cq->dbinfo.shadow_key_arm_ena = BNXT_QPLIB_DBR_KEY_INVALID; tbl = &res->reftbl.cqref; spin_lock_irqsave(&tbl->lock, flag); tbl->rec[GET_TBL_INDEX(cq->id, tbl)].xid = cq->id; tbl->rec[GET_TBL_INDEX(cq->id, tbl)].handle = cq; spin_unlock_irqrestore(&tbl->lock, flag); bnxt_qplib_armen_db(&cq->dbinfo, DBC_DBC_TYPE_CQ_ARMENA); return 0; fail: bnxt_qplib_free_hwq(res, &cq->hwq); exit: return rc; } int bnxt_qplib_modify_cq(struct bnxt_qplib_res *res, struct bnxt_qplib_cq *cq) { /* TODO: Modify CQ threshold are passed to the HW via DBR */ return 0; } void bnxt_qplib_resize_cq_complete(struct bnxt_qplib_res *res, struct bnxt_qplib_cq *cq) { bnxt_qplib_free_hwq(res, &cq->hwq); memcpy(&cq->hwq, &cq->resize_hwq, sizeof(cq->hwq)); /* Reset only the cons bit in the flags */ cq->dbinfo.flags &= ~(1UL << BNXT_QPLIB_FLAG_EPOCH_CONS_SHIFT); /* Tell HW to switch over to the new CQ */ if (!cq->resize_hwq.is_user) bnxt_qplib_cq_coffack_db(&cq->dbinfo); } int bnxt_qplib_resize_cq(struct bnxt_qplib_res *res, struct bnxt_qplib_cq *cq, int new_cqes) { struct bnxt_qplib_hwq_attr hwq_attr = {}; struct bnxt_qplib_rcfw *rcfw = res->rcfw; struct creq_resize_cq_resp resp = {}; struct bnxt_qplib_cmdqmsg msg = {}; struct cmdq_resize_cq req = {}; u32 pgsz = 0, lvl = 0, nsz = 0; struct bnxt_qplib_pbl *pbl; u16 count = -1; int rc; bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_RESIZE_CQ, sizeof(req)); hwq_attr.sginfo = &cq->sginfo; hwq_attr.res = res; hwq_attr.depth = new_cqes; hwq_attr.stride = sizeof(struct cq_base); hwq_attr.type = HWQ_TYPE_QUEUE; rc = bnxt_qplib_alloc_init_hwq(&cq->resize_hwq, &hwq_attr); if (rc) return rc; dev_dbg(&rcfw->pdev->dev, "QPLIB: FP: %s: pbl_lvl: %d\n", __func__, cq->resize_hwq.level); req.cq_cid = cpu_to_le32(cq->id); pbl = &cq->resize_hwq.pbl[PBL_LVL_0]; pgsz = ((pbl->pg_size == ROCE_PG_SIZE_4K ? CMDQ_RESIZE_CQ_PG_SIZE_PG_4K : pbl->pg_size == ROCE_PG_SIZE_8K ? CMDQ_RESIZE_CQ_PG_SIZE_PG_8K : pbl->pg_size == ROCE_PG_SIZE_64K ? CMDQ_RESIZE_CQ_PG_SIZE_PG_64K : pbl->pg_size == ROCE_PG_SIZE_2M ? CMDQ_RESIZE_CQ_PG_SIZE_PG_2M : pbl->pg_size == ROCE_PG_SIZE_8M ? CMDQ_RESIZE_CQ_PG_SIZE_PG_8M : pbl->pg_size == ROCE_PG_SIZE_1G ? CMDQ_RESIZE_CQ_PG_SIZE_PG_1G : CMDQ_RESIZE_CQ_PG_SIZE_PG_4K) & CMDQ_RESIZE_CQ_PG_SIZE_MASK); lvl = (cq->resize_hwq.level << CMDQ_RESIZE_CQ_LVL_SFT) & CMDQ_RESIZE_CQ_LVL_MASK; nsz = (new_cqes << CMDQ_RESIZE_CQ_NEW_CQ_SIZE_SFT) & CMDQ_RESIZE_CQ_NEW_CQ_SIZE_MASK; req.new_cq_size_pg_size_lvl = cpu_to_le32(nsz|pgsz|lvl); req.new_pbl = cpu_to_le64(pbl->pg_map_arr[0]); if (!cq->resize_hwq.is_user) set_bit(CQ_FLAGS_RESIZE_IN_PROG, &cq->flags); bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL, sizeof(req), sizeof(resp), 0); rc = bnxt_qplib_rcfw_send_message(rcfw, &msg); if (rc) goto fail; if (!cq->resize_hwq.is_user) { wait: /* Wait here for the HW to switch the CQ over */ if (wait_event_interruptible_timeout(cq->waitq, !test_bit(CQ_FLAGS_RESIZE_IN_PROG, &cq->flags), msecs_to_jiffies(CQ_RESIZE_WAIT_TIME_MS)) == -ERESTARTSYS && count--) goto wait; if (test_bit(CQ_FLAGS_RESIZE_IN_PROG, &cq->flags)) { dev_err(&rcfw->pdev->dev, "QPLIB: FP: RESIZE_CQ timed out\n"); rc = -ETIMEDOUT; goto fail; } bnxt_qplib_resize_cq_complete(res, cq); } return 0; fail: if (!cq->resize_hwq.is_user) { bnxt_qplib_free_hwq(res, &cq->resize_hwq); clear_bit(CQ_FLAGS_RESIZE_IN_PROG, &cq->flags); } return rc; } void bnxt_qplib_free_cq(struct bnxt_qplib_res *res, struct bnxt_qplib_cq *cq) { bnxt_qplib_free_hwq(res, &cq->hwq); } static void bnxt_qplib_sync_cq(struct bnxt_qplib_cq *cq) { struct bnxt_qplib_nq *nq = cq->nq; /* Flush any pending work and synchronize irq */ flush_workqueue(cq->nq->cqn_wq); mutex_lock(&nq->lock); if (nq->requested) synchronize_irq(nq->msix_vec); mutex_unlock(&nq->lock); } int bnxt_qplib_destroy_cq(struct bnxt_qplib_res *res, struct bnxt_qplib_cq *cq) { struct bnxt_qplib_rcfw *rcfw = res->rcfw; struct creq_destroy_cq_resp resp = {}; struct bnxt_qplib_cmdqmsg msg = {}; struct cmdq_destroy_cq req = {}; struct bnxt_qplib_reftbl *tbl; u16 total_cnq_events; unsigned long flag; int rc; tbl = &res->reftbl.cqref; spin_lock_irqsave(&tbl->lock, flag); tbl->rec[GET_TBL_INDEX(cq->id, tbl)].handle = NULL; tbl->rec[GET_TBL_INDEX(cq->id, tbl)].xid = 0; spin_unlock_irqrestore(&tbl->lock, flag); bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_DESTROY_CQ, sizeof(req)); req.cq_cid = cpu_to_le32(cq->id); bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL, sizeof(req), sizeof(resp), 0); rc = bnxt_qplib_rcfw_send_message(rcfw, &msg); if (rc) return rc; total_cnq_events = le16_to_cpu(resp.total_cnq_events); if (total_cnq_events >= 0) dev_dbg(&rcfw->pdev->dev, "%s: cq_id = 0x%x cq = 0x%p resp.total_cnq_events = 0x%x\n", __func__, cq->id, cq, total_cnq_events); __wait_for_all_nqes(cq, total_cnq_events); bnxt_qplib_sync_cq(cq); bnxt_qplib_free_hwq(res, &cq->hwq); return 0; } static int __flush_sq(struct bnxt_qplib_q *sq, struct bnxt_qplib_qp *qp, struct bnxt_qplib_cqe **pcqe, int *budget) { struct bnxt_qplib_cqe *cqe; u32 start, last; int rc = 0; /* Now complete all outstanding SQEs with FLUSHED_ERR */ start = sq->swq_start; cqe = *pcqe; while (*budget) { last = sq->swq_last; if (start == last) { break; } /* Skip the FENCE WQE completions */ if (sq->swq[last].wr_id == BNXT_QPLIB_FENCE_WRID) { bnxt_re_legacy_cancel_phantom_processing(qp); goto skip_compl; } memset(cqe, 0, sizeof(*cqe)); cqe->status = CQ_REQ_STATUS_WORK_REQUEST_FLUSHED_ERR; cqe->opcode = CQ_BASE_CQE_TYPE_REQ; cqe->qp_handle = (u64)qp; cqe->wr_id = sq->swq[last].wr_id; cqe->src_qp = qp->id; cqe->type = sq->swq[last].type; dev_dbg(&sq->hwq.pdev->dev, "QPLIB: FP: CQ Processed terminal Req \n"); dev_dbg(&sq->hwq.pdev->dev, "QPLIB: wr_id[%d] = 0x%llx with status 0x%x\n", last, cqe->wr_id, cqe->status); cqe++; (*budget)--; skip_compl: bnxt_qplib_hwq_incr_cons(sq->hwq.depth, &sq->hwq.cons, sq->swq[last].slots, &sq->dbinfo.flags); sq->swq_last = sq->swq[last].next_idx; } *pcqe = cqe; if (!*budget && sq->swq_last != start) /* Out of budget */ rc = -EAGAIN; dev_dbg(&sq->hwq.pdev->dev, "QPLIB: FP: Flush SQ rc = 0x%x\n", rc); return rc; } static int __flush_rq(struct bnxt_qplib_q *rq, struct bnxt_qplib_qp *qp, struct bnxt_qplib_cqe **pcqe, int *budget) { struct bnxt_qplib_cqe *cqe; u32 start, last; int opcode = 0; int rc = 0; switch (qp->type) { case CMDQ_CREATE_QP1_TYPE_GSI: opcode = CQ_BASE_CQE_TYPE_RES_RAWETH_QP1; break; case CMDQ_CREATE_QP_TYPE_RC: opcode = CQ_BASE_CQE_TYPE_RES_RC; break; case CMDQ_CREATE_QP_TYPE_UD: opcode = CQ_BASE_CQE_TYPE_RES_UD; break; } /* Flush the rest of the RQ */ start = rq->swq_start; cqe = *pcqe; while (*budget) { last = rq->swq_last; if (last == start) break; memset(cqe, 0, sizeof(*cqe)); cqe->status = CQ_RES_RC_STATUS_WORK_REQUEST_FLUSHED_ERR; cqe->opcode = opcode; cqe->qp_handle = (u64)qp; cqe->wr_id = rq->swq[last].wr_id; dev_dbg(&rq->hwq.pdev->dev, "QPLIB: FP: CQ Processed Res RC \n"); dev_dbg(&rq->hwq.pdev->dev, "QPLIB: rq[%d] = 0x%llx with status 0x%x\n", last, cqe->wr_id, cqe->status); cqe++; (*budget)--; bnxt_qplib_hwq_incr_cons(rq->hwq.depth, &rq->hwq.cons, rq->swq[last].slots, &rq->dbinfo.flags); rq->swq_last = rq->swq[last].next_idx; } *pcqe = cqe; if (!*budget && rq->swq_last != start) /* Out of budget */ rc = -EAGAIN; dev_dbg(&rq->hwq.pdev->dev, "QPLIB: FP: Flush RQ rc = 0x%x\n", rc); return rc; } void bnxt_qplib_mark_qp_error(void *qp_handle) { struct bnxt_qplib_qp *qp = qp_handle; if (!qp) return; /* Must block new posting of SQ and RQ */ qp->cur_qp_state = CMDQ_MODIFY_QP_NEW_STATE_ERR; qp->state = qp->cur_qp_state; /* Add qp to flush list of the CQ */ if (!qp->is_user) bnxt_qplib_add_flush_qp(qp); } /* Note: SQE is valid from sw_sq_cons up to cqe_sq_cons (exclusive) * CQE is track from sw_cq_cons to max_element but valid only if VALID=1 */ static int bnxt_re_legacy_do_wa9060(struct bnxt_qplib_qp *qp, struct bnxt_qplib_cq *cq, u32 cq_cons, u32 swq_last, u32 cqe_sq_cons) { struct bnxt_qplib_q *sq = &qp->sq; struct bnxt_qplib_swq *swq; u32 peek_sw_cq_cons, peek_sq_cons_idx, peek_flags; struct cq_terminal *peek_term_hwcqe; struct cq_req *peek_req_hwcqe; struct bnxt_qplib_qp *peek_qp; struct bnxt_qplib_q *peek_sq; struct cq_base *peek_hwcqe; int i, rc = 0; /* Check for the psn_search marking before completing */ swq = &sq->swq[swq_last]; if (swq->psn_search && le32_to_cpu(swq->psn_search->flags_next_psn) & 0x80000000) { /* Unmark */ swq->psn_search->flags_next_psn = cpu_to_le32 (le32_to_cpu(swq->psn_search->flags_next_psn) & ~0x80000000); dev_dbg(&cq->hwq.pdev->dev, "FP: Process Req cq_cons=0x%x qp=0x%x sq cons sw=0x%x cqe=0x%x marked!\n", cq_cons, qp->id, swq_last, cqe_sq_cons); sq->condition = true; sq->legacy_send_phantom = true; /* TODO: Only ARM if the previous SQE is ARMALL */ bnxt_qplib_ring_db(&cq->dbinfo, DBC_DBC_TYPE_CQ_ARMALL); rc = -EAGAIN; goto out; } if (sq->condition == true) { /* Peek at the completions */ peek_flags = cq->dbinfo.flags; peek_sw_cq_cons = cq_cons; i = cq->hwq.depth; while (i--) { peek_hwcqe = bnxt_qplib_get_qe(&cq->hwq, peek_sw_cq_cons, NULL); /* If the next hwcqe is VALID */ if (CQE_CMP_VALID(peek_hwcqe, peek_flags)) { /* If the next hwcqe is a REQ */ dma_rmb(); switch (peek_hwcqe->cqe_type_toggle & CQ_BASE_CQE_TYPE_MASK) { case CQ_BASE_CQE_TYPE_REQ: peek_req_hwcqe = (struct cq_req *) peek_hwcqe; peek_qp = (struct bnxt_qplib_qp *) le64_to_cpu( peek_req_hwcqe->qp_handle); peek_sq = &peek_qp->sq; peek_sq_cons_idx = ((le16_to_cpu( peek_req_hwcqe->sq_cons_idx) - 1) % sq->max_wqe); /* If the hwcqe's sq's wr_id matches */ if (peek_sq == sq && sq->swq[peek_sq_cons_idx].wr_id == BNXT_QPLIB_FENCE_WRID) { /* Unbreak only if the phantom comes back */ dev_dbg(&cq->hwq.pdev->dev, "FP: Process Req qp=0x%x current sq cons sw=0x%x cqe=0x%x\n", qp->id, swq_last, cqe_sq_cons); sq->condition = false; sq->single = true; sq->phantom_cqe_cnt++; dev_dbg(&cq->hwq.pdev->dev, "qp %#x condition restored at peek cq_cons=%#x sq_cons_idx %#x, phantom_cqe_cnt: %d unmark\n", peek_qp->id, peek_sw_cq_cons, peek_sq_cons_idx, sq->phantom_cqe_cnt); rc = 0; goto out; } break; case CQ_BASE_CQE_TYPE_TERMINAL: /* In case the QP has gone into the error state */ peek_term_hwcqe = (struct cq_terminal *) peek_hwcqe; peek_qp = (struct bnxt_qplib_qp *) le64_to_cpu( peek_term_hwcqe->qp_handle); if (peek_qp == qp) { sq->condition = false; rc = 0; goto out; } break; default: break; } /* Valid but not the phantom, so keep looping */ } else { /* Not valid yet, just exit and wait */ rc = -EINVAL; goto out; } bnxt_qplib_hwq_incr_cons(cq->hwq.depth, &peek_sw_cq_cons, 1, &peek_flags); } dev_err(&cq->hwq.pdev->dev, "Should not have come here! cq_cons=0x%x qp=0x%x sq cons sw=0x%x hw=0x%x\n", cq_cons, qp->id, swq_last, cqe_sq_cons); rc = -EINVAL; } out: return rc; } static int bnxt_qplib_cq_process_req(struct bnxt_qplib_cq *cq, struct cq_req *hwcqe, struct bnxt_qplib_cqe **pcqe, int *budget, u32 cq_cons, struct bnxt_qplib_qp **lib_qp) { struct bnxt_qplib_qp *qp; struct bnxt_qplib_q *sq; struct bnxt_qplib_cqe *cqe; u32 cqe_sq_cons; struct bnxt_qplib_swq *swq; int rc = 0; qp = (struct bnxt_qplib_qp *)le64_to_cpu(hwcqe->qp_handle); dev_dbg(&cq->hwq.pdev->dev, "FP: Process Req qp=0x%p\n", qp); if (!qp) { dev_err(&cq->hwq.pdev->dev, "QPLIB: FP: Process Req qp is NULL\n"); return -EINVAL; } sq = &qp->sq; cqe_sq_cons = le16_to_cpu(hwcqe->sq_cons_idx) % sq->max_wqe; if (qp->sq.flushed) { dev_dbg(&cq->hwq.pdev->dev, "%s: QPLIB: QP in Flush QP = %p\n", __func__, qp); goto done; } /* Require to walk the sq's swq to fabricate CQEs for all previously * signaled SWQEs due to CQE aggregation from the current sq cons * to the cqe_sq_cons */ cqe = *pcqe; while (*budget) { if (sq->swq_last == cqe_sq_cons) /* Done */ break; swq = &sq->swq[sq->swq_last]; memset(cqe, 0, sizeof(*cqe)); cqe->opcode = CQ_BASE_CQE_TYPE_REQ; cqe->qp_handle = (u64)qp; cqe->src_qp = qp->id; cqe->wr_id = swq->wr_id; if (cqe->wr_id == BNXT_QPLIB_FENCE_WRID) goto skip; cqe->type = swq->type; /* For the last CQE, check for status. For errors, regardless * of the request being signaled or not, it must complete with * the hwcqe error status */ if (swq->next_idx == cqe_sq_cons && hwcqe->status != CQ_REQ_STATUS_OK) { cqe->status = hwcqe->status; dev_err(&cq->hwq.pdev->dev, "QPLIB: FP: CQ Processed Req \n"); dev_err(&cq->hwq.pdev->dev, "QPLIB: QP 0x%x wr_id[%d] = 0x%lx vendor type 0x%x with vendor status 0x%x\n", cqe->src_qp, sq->swq_last, cqe->wr_id, cqe->type, cqe->status); cqe++; (*budget)--; bnxt_qplib_mark_qp_error(qp); } else { /* Before we complete, do WA 9060 */ if (!_is_chip_gen_p5_p7(qp->cctx)) { if (bnxt_re_legacy_do_wa9060(qp, cq, cq_cons, sq->swq_last, cqe_sq_cons)) { *lib_qp = qp; goto out; } } if (swq->flags & SQ_SEND_FLAGS_SIGNAL_COMP) { dev_dbg(&cq->hwq.pdev->dev, "QPLIB: FP: CQ Processed Req \n"); dev_dbg(&cq->hwq.pdev->dev, "QPLIB: wr_id[%d] = 0x%llx \n", sq->swq_last, cqe->wr_id); dev_dbg(&cq->hwq.pdev->dev, "QPLIB: with status 0x%x\n", cqe->status); cqe->status = CQ_REQ_STATUS_OK; cqe++; (*budget)--; } } skip: bnxt_qplib_hwq_incr_cons(sq->hwq.depth, &sq->hwq.cons, swq->slots, &sq->dbinfo.flags); sq->swq_last = swq->next_idx; if (sq->single == true) break; } out: *pcqe = cqe; if (sq->swq_last != cqe_sq_cons) { /* Out of budget */ rc = -EAGAIN; goto done; } /* Back to normal completion mode only after it has completed all of the WC for this CQE */ sq->single = false; done: return rc; } static void bnxt_qplib_release_srqe(struct bnxt_qplib_srq *srq, u32 tag) { spin_lock(&srq->hwq.lock); srq->swq[srq->last_idx].next_idx = (int)tag; srq->last_idx = (int)tag; srq->swq[srq->last_idx].next_idx = -1; bnxt_qplib_hwq_incr_cons(srq->hwq.depth, &srq->hwq.cons, srq->dbinfo.max_slot, &srq->dbinfo.flags); spin_unlock(&srq->hwq.lock); } static int bnxt_qplib_cq_process_res_rc(struct bnxt_qplib_cq *cq, struct cq_res_rc *hwcqe, struct bnxt_qplib_cqe **pcqe, int *budget) { struct bnxt_qplib_srq *srq; struct bnxt_qplib_cqe *cqe; struct bnxt_qplib_qp *qp; struct bnxt_qplib_q *rq; u32 wr_id_idx; int rc = 0; qp = (struct bnxt_qplib_qp *)le64_to_cpu(hwcqe->qp_handle); if (!qp) { dev_err(&cq->hwq.pdev->dev, "QPLIB: process_cq RC qp is NULL\n"); return -EINVAL; } if (qp->rq.flushed) { dev_dbg(&cq->hwq.pdev->dev, "%s: QPLIB: QP in Flush QP = %p\n", __func__, qp); goto done; } cqe = *pcqe; cqe->opcode = hwcqe->cqe_type_toggle & CQ_BASE_CQE_TYPE_MASK; cqe->length = le32_to_cpu(hwcqe->length); cqe->invrkey = le32_to_cpu(hwcqe->imm_data_or_inv_r_key); cqe->mr_handle = le64_to_cpu(hwcqe->mr_handle); cqe->flags = le16_to_cpu(hwcqe->flags); cqe->status = hwcqe->status; cqe->qp_handle = (u64)(unsigned long)qp; wr_id_idx = le32_to_cpu(hwcqe->srq_or_rq_wr_id) & CQ_RES_RC_SRQ_OR_RQ_WR_ID_MASK; if (cqe->flags & CQ_RES_RC_FLAGS_SRQ_SRQ) { srq = qp->srq; if (!srq) { dev_err(&cq->hwq.pdev->dev, "QPLIB: FP: SRQ used but not defined??\n"); return -EINVAL; } if (wr_id_idx > srq->hwq.depth - 1) { dev_err(&cq->hwq.pdev->dev, "QPLIB: FP: CQ Process RC \n"); dev_err(&cq->hwq.pdev->dev, "QPLIB: wr_id idx 0x%x exceeded SRQ max 0x%x\n", wr_id_idx, srq->hwq.depth); return -EINVAL; } cqe->wr_id = srq->swq[wr_id_idx].wr_id; bnxt_qplib_release_srqe(srq, wr_id_idx); dev_dbg(&srq->hwq.pdev->dev, "QPLIB: FP: CQ Processed RC SRQ wr_id[%d] = 0x%llx\n", wr_id_idx, cqe->wr_id); cqe++; (*budget)--; *pcqe = cqe; } else { rq = &qp->rq; if (wr_id_idx > (rq->max_wqe - 1)) { dev_err(&cq->hwq.pdev->dev, "QPLIB: FP: CQ Process RC \n"); dev_err(&cq->hwq.pdev->dev, "QPLIB: wr_id idx 0x%x exceeded RQ max 0x%x\n", wr_id_idx, rq->hwq.depth); return -EINVAL; } if (wr_id_idx != rq->swq_last) return -EINVAL; cqe->wr_id = rq->swq[rq->swq_last].wr_id; dev_dbg(&cq->hwq.pdev->dev, "QPLIB: FP: CQ Processed RC RQ wr_id[%d] = 0x%llx\n", rq->swq_last, cqe->wr_id); cqe++; (*budget)--; bnxt_qplib_hwq_incr_cons(rq->hwq.depth, &rq->hwq.cons, rq->swq[rq->swq_last].slots, &rq->dbinfo.flags); rq->swq_last = rq->swq[rq->swq_last].next_idx; *pcqe = cqe; if (hwcqe->status != CQ_RES_RC_STATUS_OK) bnxt_qplib_mark_qp_error(qp); } done: return rc; } static int bnxt_qplib_cq_process_res_ud(struct bnxt_qplib_cq *cq, struct cq_res_ud_v2 *hwcqe, struct bnxt_qplib_cqe **pcqe, int *budget) { struct bnxt_qplib_srq *srq; struct bnxt_qplib_cqe *cqe; struct bnxt_qplib_qp *qp; struct bnxt_qplib_q *rq; u32 wr_id_idx; int rc = 0; u16 *smac; qp = (struct bnxt_qplib_qp *)le64_to_cpu(hwcqe->qp_handle); if (!qp) { dev_err(&cq->hwq.pdev->dev, "QPLIB: process_cq UD qp is NULL\n"); return -EINVAL; } if (qp->rq.flushed) { dev_dbg(&cq->hwq.pdev->dev, "%s: QPLIB: QP in Flush QP = %p\n", __func__, qp); goto done; } cqe = *pcqe; cqe->opcode = hwcqe->cqe_type_toggle & CQ_RES_UD_V2_CQE_TYPE_MASK; cqe->length = le32_to_cpu((hwcqe->length & CQ_RES_UD_V2_LENGTH_MASK)); cqe->cfa_meta = le16_to_cpu(hwcqe->cfa_metadata0); /* V2 format has metadata1 */ cqe->cfa_meta |= (((le32_to_cpu(hwcqe->src_qp_high_srq_or_rq_wr_id) & CQ_RES_UD_V2_CFA_METADATA1_MASK) >> CQ_RES_UD_V2_CFA_METADATA1_SFT) << BNXT_QPLIB_META1_SHIFT); cqe->invrkey = le32_to_cpu(hwcqe->imm_data); cqe->flags = le16_to_cpu(hwcqe->flags); cqe->status = hwcqe->status; cqe->qp_handle = (u64)(unsigned long)qp; smac = (u16 *)cqe->smac; smac[2] = ntohs(le16_to_cpu(hwcqe->src_mac[0])); smac[1] = ntohs(le16_to_cpu(hwcqe->src_mac[1])); smac[0] = ntohs(le16_to_cpu(hwcqe->src_mac[2])); wr_id_idx = le32_to_cpu(hwcqe->src_qp_high_srq_or_rq_wr_id) & CQ_RES_UD_V2_SRQ_OR_RQ_WR_ID_MASK; cqe->src_qp = le16_to_cpu(hwcqe->src_qp_low) | ((le32_to_cpu( hwcqe->src_qp_high_srq_or_rq_wr_id) & CQ_RES_UD_V2_SRC_QP_HIGH_MASK) >> 8); if (cqe->flags & CQ_RES_UD_V2_FLAGS_SRQ) { srq = qp->srq; if (!srq) { dev_err(&cq->hwq.pdev->dev, "QPLIB: FP: SRQ used but not defined??\n"); return -EINVAL; } if (wr_id_idx > srq->hwq.depth - 1) { dev_err(&cq->hwq.pdev->dev, "QPLIB: FP: CQ Process UD \n"); dev_err(&cq->hwq.pdev->dev, "QPLIB: wr_id idx 0x%x exceeded SRQ max 0x%x\n", wr_id_idx, srq->hwq.depth); return -EINVAL; } cqe->wr_id = srq->swq[wr_id_idx].wr_id; bnxt_qplib_release_srqe(srq, wr_id_idx); dev_dbg(&cq->hwq.pdev->dev, "QPLIB: FP: CQ Processed UD SRQ wr_id[%d] = 0x%llx\n", wr_id_idx, cqe->wr_id); cqe++; (*budget)--; *pcqe = cqe; } else { rq = &qp->rq; if (wr_id_idx > (rq->max_wqe - 1)) { dev_err(&cq->hwq.pdev->dev, "QPLIB: FP: CQ Process UD \n"); dev_err(&cq->hwq.pdev->dev, "QPLIB: wr_id idx 0x%x exceeded RQ max 0x%x\n", wr_id_idx, rq->hwq.depth); return -EINVAL; } if (rq->swq_last != wr_id_idx) return -EINVAL; cqe->wr_id = rq->swq[rq->swq_last].wr_id; dev_dbg(&cq->hwq.pdev->dev, "QPLIB: FP: CQ Processed UD RQ wr_id[%d] = 0x%llx\n", rq->swq_last, cqe->wr_id); cqe++; (*budget)--; bnxt_qplib_hwq_incr_cons(rq->hwq.depth, &rq->hwq.cons, rq->swq[rq->swq_last].slots, &rq->dbinfo.flags); rq->swq_last = rq->swq[rq->swq_last].next_idx; *pcqe = cqe; if (hwcqe->status != CQ_RES_UD_V2_STATUS_OK) bnxt_qplib_mark_qp_error(qp); } done: return rc; } bool bnxt_qplib_is_cq_empty(struct bnxt_qplib_cq *cq) { struct cq_base *hw_cqe; unsigned long flags; bool rc = true; spin_lock_irqsave(&cq->hwq.lock, flags); hw_cqe = bnxt_qplib_get_qe(&cq->hwq, cq->hwq.cons, NULL); /* Check for Valid bit. If the CQE is valid, return false */ rc = !CQE_CMP_VALID(hw_cqe, cq->dbinfo.flags); spin_unlock_irqrestore(&cq->hwq.lock, flags); return rc; } static int bnxt_qplib_cq_process_res_raweth_qp1(struct bnxt_qplib_cq *cq, struct cq_res_raweth_qp1 *hwcqe, struct bnxt_qplib_cqe **pcqe, int *budget) { struct bnxt_qplib_qp *qp; struct bnxt_qplib_q *rq; struct bnxt_qplib_srq *srq; struct bnxt_qplib_cqe *cqe; u32 wr_id_idx; int rc = 0; qp = (struct bnxt_qplib_qp *)le64_to_cpu(hwcqe->qp_handle); if (!qp) { dev_err(&cq->hwq.pdev->dev, "QPLIB: process_cq Raw/QP1 qp is NULL\n"); return -EINVAL; } if (qp->rq.flushed) { dev_dbg(&cq->hwq.pdev->dev, "%s: QPLIB: QP in Flush QP = %p\n", __func__, qp); goto done; } cqe = *pcqe; cqe->opcode = hwcqe->cqe_type_toggle & CQ_BASE_CQE_TYPE_MASK; cqe->flags = le16_to_cpu(hwcqe->flags); cqe->qp_handle = (u64)(unsigned long)qp; wr_id_idx = le32_to_cpu(hwcqe->raweth_qp1_payload_offset_srq_or_rq_wr_id) & CQ_RES_RAWETH_QP1_SRQ_OR_RQ_WR_ID_MASK; cqe->src_qp = qp->id; if (qp->id == 1 && !cqe->length) { /* Add workaround for the length misdetection */ cqe->length = 296; } else { cqe->length = le16_to_cpu(hwcqe->length); } cqe->pkey_index = qp->pkey_index; memcpy(cqe->smac, qp->smac, 6); cqe->raweth_qp1_flags = le16_to_cpu(hwcqe->raweth_qp1_flags); cqe->raweth_qp1_flags2 = le32_to_cpu(hwcqe->raweth_qp1_flags2); cqe->raweth_qp1_metadata = le32_to_cpu(hwcqe->raweth_qp1_metadata); dev_dbg(&cq->hwq.pdev->dev, "QPLIB: raweth_qp1_flags = 0x%x raweth_qp1_flags2 = 0x%x\n", cqe->raweth_qp1_flags, cqe->raweth_qp1_flags2); if (cqe->flags & CQ_RES_RAWETH_QP1_FLAGS_SRQ_SRQ) { srq = qp->srq; if (!srq) { dev_err(&cq->hwq.pdev->dev, "QPLIB: FP: SRQ used but not defined??\n"); return -EINVAL; } if (wr_id_idx > srq->hwq.depth - 1) { dev_err(&cq->hwq.pdev->dev, "QPLIB: FP: CQ Process Raw/QP1 \n"); dev_err(&cq->hwq.pdev->dev, "QPLIB: wr_id idx 0x%x exceeded SRQ max 0x%x\n", wr_id_idx, srq->hwq.depth); return -EINVAL; } cqe->wr_id = srq->swq[wr_id_idx].wr_id; dev_dbg(&cq->hwq.pdev->dev, "QPLIB: FP: CQ Processed Raw/QP1 SRQ \n"); dev_dbg(&cq->hwq.pdev->dev, "QPLIB: wr_id[%d] = 0x%llx with status = 0x%x\n", wr_id_idx, cqe->wr_id, hwcqe->status); cqe++; (*budget)--; srq->hwq.cons++; *pcqe = cqe; } else { rq = &qp->rq; if (wr_id_idx > (rq->max_wqe - 1)) { dev_err(&cq->hwq.pdev->dev, "QPLIB: FP: CQ Process Raw/QP1 RQ wr_id \n"); dev_err(&cq->hwq.pdev->dev, "QPLIB: ix 0x%x exceeded RQ max 0x%x\n", wr_id_idx, rq->max_wqe); return -EINVAL; } if (wr_id_idx != rq->swq_last) return -EINVAL; cqe->wr_id = rq->swq[rq->swq_last].wr_id; dev_dbg(&cq->hwq.pdev->dev, "QPLIB: FP: CQ Processed Raw/QP1 RQ \n"); dev_dbg(&cq->hwq.pdev->dev, "QPLIB: wr_id[%d] = 0x%llx with status = 0x%x\n", wr_id_idx, cqe->wr_id, hwcqe->status); cqe++; (*budget)--; bnxt_qplib_hwq_incr_cons(rq->hwq.depth, &rq->hwq.cons, rq->swq[wr_id_idx].slots, &rq->dbinfo.flags); rq->swq_last = rq->swq[rq->swq_last].next_idx; *pcqe = cqe; if (hwcqe->status != CQ_RES_RC_STATUS_OK) bnxt_qplib_mark_qp_error(qp); } done: return rc; } static int bnxt_qplib_cq_process_terminal(struct bnxt_qplib_cq *cq, struct cq_terminal *hwcqe, struct bnxt_qplib_cqe **pcqe, int *budget) { struct bnxt_qplib_q *sq, *rq; struct bnxt_qplib_cqe *cqe; struct bnxt_qplib_qp *qp; u32 swq_last; u32 cqe_cons; int rc = 0; /* Check the Status */ if (hwcqe->status != CQ_TERMINAL_STATUS_OK) dev_warn(&cq->hwq.pdev->dev, "QPLIB: FP: CQ Process Terminal Error status = 0x%x\n", hwcqe->status); qp = (struct bnxt_qplib_qp *)le64_to_cpu(hwcqe->qp_handle); if (!qp) return -EINVAL; dev_dbg(&cq->hwq.pdev->dev, "QPLIB: FP: CQ Process terminal for qp (0x%x)\n", qp->id); /* Terminal CQE requires all posted RQEs to complete with FLUSHED_ERR * from the current rq->cons to the rq->prod regardless what the * rq->cons the terminal CQE indicates. */ bnxt_qplib_mark_qp_error(qp); sq = &qp->sq; rq = &qp->rq; cqe_cons = le16_to_cpu(hwcqe->sq_cons_idx); if (cqe_cons == 0xFFFF) goto do_rq; cqe_cons %= sq->max_wqe; if (qp->sq.flushed) { dev_dbg(&cq->hwq.pdev->dev, "%s: QPLIB: QP in Flush QP = %p\n", __func__, qp); goto sq_done; } /* Terminal CQE can also include aggregated successful CQEs prior. So we must complete all CQEs from the current sq's cons to the cq_cons with status OK */ cqe = *pcqe; while (*budget) { /*sw_cons = HWQ_CMP(sq->hwq.cons, &sq->hwq);*/ swq_last = sq->swq_last; if (swq_last == cqe_cons) break; if (sq->swq[swq_last].flags & SQ_SEND_FLAGS_SIGNAL_COMP) { memset(cqe, 0, sizeof(*cqe)); cqe->status = CQ_REQ_STATUS_OK; cqe->opcode = CQ_BASE_CQE_TYPE_REQ; cqe->qp_handle = (u64)qp; cqe->src_qp = qp->id; cqe->wr_id = sq->swq[swq_last].wr_id; cqe->type = sq->swq[swq_last].type; dev_dbg(&cq->hwq.pdev->dev, "QPLIB: FP: CQ Processed terminal Req \n"); dev_dbg(&cq->hwq.pdev->dev, "QPLIB: wr_id[%d] = 0x%llx with status 0x%x\n", swq_last, cqe->wr_id, cqe->status); cqe++; (*budget)--; } bnxt_qplib_hwq_incr_cons(sq->hwq.depth, &sq->hwq.cons, sq->swq[swq_last].slots, &sq->dbinfo.flags); sq->swq_last = sq->swq[swq_last].next_idx; } *pcqe = cqe; if (!*budget && swq_last != cqe_cons) { /* Out of budget */ rc = -EAGAIN; goto sq_done; } sq_done: if (rc) return rc; do_rq: cqe_cons = le16_to_cpu(hwcqe->rq_cons_idx); if (cqe_cons == 0xFFFF) { goto done; } else if (cqe_cons > (rq->max_wqe - 1)) { dev_err(&cq->hwq.pdev->dev, "QPLIB: FP: CQ Processed terminal \n"); dev_err(&cq->hwq.pdev->dev, "QPLIB: reported rq_cons_idx 0x%x exceeds max 0x%x\n", cqe_cons, rq->hwq.depth); goto done; } if (qp->rq.flushed) { dev_dbg(&cq->hwq.pdev->dev, "%s: QPLIB: QP in Flush QP = %p\n", __func__, qp); rc = 0; goto rq_done; } rq_done: done: return rc; } static int bnxt_qplib_cq_process_cutoff(struct bnxt_qplib_cq *cq, struct cq_cutoff *hwcqe) { /* Check the Status */ if (hwcqe->status != CQ_CUTOFF_STATUS_OK) { dev_err(&cq->hwq.pdev->dev, "QPLIB: FP: CQ Process Cutoff Error status = 0x%x\n", hwcqe->status); return -EINVAL; } clear_bit(CQ_FLAGS_RESIZE_IN_PROG, &cq->flags); wake_up_interruptible(&cq->waitq); dev_dbg(&cq->hwq.pdev->dev, "QPLIB: FP: CQ Processed Cutoff\n"); return 0; } int bnxt_qplib_process_flush_list(struct bnxt_qplib_cq *cq, struct bnxt_qplib_cqe *cqe, int num_cqes) { struct bnxt_qplib_qp *qp = NULL; u32 budget = num_cqes; unsigned long flags; spin_lock_irqsave(&cq->flush_lock, flags); list_for_each_entry(qp, &cq->sqf_head, sq_flush) { dev_dbg(&cq->hwq.pdev->dev, "QPLIB: FP: Flushing SQ QP= %p\n", qp); __flush_sq(&qp->sq, qp, &cqe, &budget); } list_for_each_entry(qp, &cq->rqf_head, rq_flush) { dev_dbg(&cq->hwq.pdev->dev, "QPLIB: FP: Flushing RQ QP= %p\n", qp); __flush_rq(&qp->rq, qp, &cqe, &budget); } spin_unlock_irqrestore(&cq->flush_lock, flags); return num_cqes - budget; } int bnxt_qplib_poll_cq(struct bnxt_qplib_cq *cq, struct bnxt_qplib_cqe *cqe, int num_cqes, struct bnxt_qplib_qp **lib_qp) { struct cq_base *hw_cqe; u32 hw_polled = 0; int budget, rc = 0; u8 type; budget = num_cqes; while (budget) { hw_cqe = bnxt_qplib_get_qe(&cq->hwq, cq->hwq.cons, NULL); /* Check for Valid bit */ if (!CQE_CMP_VALID(hw_cqe, cq->dbinfo.flags)) break; /* The valid test of the entry must be done first before * reading any further. */ dma_rmb(); /* From the device's respective CQE format to qplib_wc*/ type = hw_cqe->cqe_type_toggle & CQ_BASE_CQE_TYPE_MASK; switch (type) { case CQ_BASE_CQE_TYPE_REQ: rc = bnxt_qplib_cq_process_req(cq, (struct cq_req *)hw_cqe, &cqe, &budget, cq->hwq.cons, lib_qp); break; case CQ_BASE_CQE_TYPE_RES_RC: rc = bnxt_qplib_cq_process_res_rc(cq, (struct cq_res_rc *)hw_cqe, &cqe, &budget); break; case CQ_BASE_CQE_TYPE_RES_UD: rc = bnxt_qplib_cq_process_res_ud(cq, (struct cq_res_ud_v2 *)hw_cqe, &cqe, &budget); break; case CQ_BASE_CQE_TYPE_RES_RAWETH_QP1: rc = bnxt_qplib_cq_process_res_raweth_qp1(cq, (struct cq_res_raweth_qp1 *) hw_cqe, &cqe, &budget); break; case CQ_BASE_CQE_TYPE_TERMINAL: rc = bnxt_qplib_cq_process_terminal(cq, (struct cq_terminal *)hw_cqe, &cqe, &budget); break; case CQ_BASE_CQE_TYPE_CUT_OFF: bnxt_qplib_cq_process_cutoff(cq, (struct cq_cutoff *)hw_cqe); /* Done processing this CQ */ goto exit; default: dev_err(&cq->hwq.pdev->dev, "QPLIB: process_cq unknown type 0x%x\n", hw_cqe->cqe_type_toggle & CQ_BASE_CQE_TYPE_MASK); rc = -EINVAL; break; } if (rc < 0) { dev_dbg(&cq->hwq.pdev->dev, "QPLIB: process_cqe rc = 0x%x\n", rc); if (rc == -EAGAIN) break; /* Error while processing the CQE, just skip to the next one */ if (type != CQ_BASE_CQE_TYPE_TERMINAL) dev_err(&cq->hwq.pdev->dev, "QPLIB: process_cqe error rc = 0x%x\n", rc); } hw_polled++; bnxt_qplib_hwq_incr_cons(cq->hwq.depth, &cq->hwq.cons, 1, &cq->dbinfo.flags); } if (hw_polled) bnxt_qplib_ring_db(&cq->dbinfo, DBC_DBC_TYPE_CQ); exit: return num_cqes - budget; } void bnxt_qplib_req_notify_cq(struct bnxt_qplib_cq *cq, u32 arm_type) { cq->dbinfo.toggle = cq->toggle; if (arm_type) bnxt_qplib_ring_db(&cq->dbinfo, arm_type); /* Using cq->arm_state variable to track whether to issue cq handler */ atomic_set(&cq->arm_state, 1); } void bnxt_qplib_flush_cqn_wq(struct bnxt_qplib_qp *qp) { flush_workqueue(qp->scq->nq->cqn_wq); if (qp->scq != qp->rcq) flush_workqueue(qp->rcq->nq->cqn_wq); } diff --git a/sys/dev/bnxt/bnxt_re/qplib_fp.h b/sys/dev/bnxt/bnxt_re/qplib_fp.h index 527c377f0aa5..fd2d8a08c818 100644 --- a/sys/dev/bnxt/bnxt_re/qplib_fp.h +++ b/sys/dev/bnxt/bnxt_re/qplib_fp.h @@ -1,638 +1,639 @@ /* * Copyright (c) 2015-2024, Broadcom. All rights reserved. The term * Broadcom refers to Broadcom Limited and/or its subsidiaries. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in * the documentation and/or other materials provided with the * distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * * Description: Fast Path Operators (header) */ #ifndef __BNXT_QPLIB_FP_H__ #define __BNXT_QPLIB_FP_H__ /* Temp header structures for SQ */ struct sq_ud_ext_hdr { __le32 dst_qp; __le32 avid; __le64 rsvd; }; struct sq_raw_ext_hdr { __le32 cfa_meta; __le32 rsvd0; __le64 rsvd1; }; struct sq_rdma_ext_hdr { __le64 remote_va; __le32 remote_key; __le32 rsvd; }; struct sq_atomic_ext_hdr { __le64 swap_data; __le64 cmp_data; }; struct sq_fr_pmr_ext_hdr { __le64 pblptr; __le64 va; }; struct sq_bind_ext_hdr { __le64 va; __le32 length_lo; __le32 length_hi; }; struct rq_ext_hdr { __le64 rsvd1; __le64 rsvd2; }; #define BNXT_QPLIB_ETHTYPE_ROCEV1 0x8915 struct bnxt_qplib_srq { struct bnxt_qplib_pd *pd; struct bnxt_qplib_dpi *dpi; struct bnxt_qplib_chip_ctx *cctx; struct bnxt_qplib_cq *cq; struct bnxt_qplib_swq *swq; struct bnxt_qplib_hwq hwq; struct bnxt_qplib_db_info dbinfo; struct bnxt_qplib_sg_info sginfo; u64 srq_handle; u32 id; u16 wqe_size; u32 max_wqe; u32 max_sge; u32 threshold; bool arm_req; int start_idx; int last_idx; u16 eventq_hw_ring_id; bool is_user; spinlock_t lock; }; struct bnxt_qplib_sge { u64 addr; u32 size; u32 lkey; }; /* * Buffer space for ETH(14), IP or GRH(40), UDP header(8) * and ib_bth + ib_deth (20). * Max required is 82 when RoCE V2 is enabled */ /* * RoCE V1 (38 bytes needed) * +------------+----------+--------+--------+-------+ * |Eth-hdr(14B)| GRH (40B)|bth+deth| Mad | iCRC | * | | supplied | 20B |payload | 4B | * | | by user |supplied| 256B | | * | | mad | |by user | | * | | | | | | * | sge 1 | sge 2 | sge 3 | sge 4 | sge 5 | * +------------+----------+--------+--------+-------+ */ /* * RoCE V2-IPv4 (46 Bytes needed) * +------------+----------+--------+--------+-------+ * |Eth-hdr(14B)| IP-hdr |UDP-hdr | Mad | iCRC | * | | supplied | 8B |payload | 4B | * | | by user |bth+deth| 256B | | * | | mad lower| 20B |supplied| | * | | 20B out | (sge 3)|by user | | * | | of 40B | | | | * | | grh space| | | | * | sge 1 | sge 2 | sge 3 | sge 4 | sge 5 | * +------------+----------+--------+--------+-------+ */ /* * RoCE V2-IPv6 (46 Bytes needed) * +------------+----------+--------+--------+-------+ * |Eth-hdr(14B)| IPv6 |UDP-hdr | Mad | iCRC | * | | supplied | 8B |payload | 4B | * | | by user |bth+deth| 256B | | * | | mad lower| 20B |supplied| | * | | 40 bytes | |by user | | * | | grh space| | | | * | | | | | | * | sge 1 | sge 2 | sge 3 | sge 4 | sge 5 | * +------------+----------+--------+--------+-------+ */ #define BNXT_QPLIB_MAX_QP1_SQ_HDR_SIZE 74 #define BNXT_QPLIB_MAX_QP1_SQ_HDR_SIZE_V2 86 #define BNXT_QPLIB_MAX_QP1_RQ_HDR_SIZE 46 #define BNXT_QPLIB_MAX_QP1_RQ_ETH_HDR_SIZE 14 #define BNXT_QPLIB_MAX_QP1_RQ_HDR_SIZE_V2 512 #define BNXT_QPLIB_MAX_GRH_HDR_SIZE_IPV4 20 #define BNXT_QPLIB_MAX_GRH_HDR_SIZE_IPV6 40 #define BNXT_QPLIB_MAX_QP1_RQ_BDETH_HDR_SIZE 20 #define BNXT_QPLIB_MAX_SQSZ 0xFFFF struct bnxt_qplib_hdrbuf { dma_addr_t dma_map; void *va; u32 len; u32 step; }; struct bnxt_qplib_swq { u64 wr_id; int next_idx; u8 type; u8 flags; u32 start_psn; u32 next_psn; u32 slot_idx; u8 slots; /* WIP: make it void * to handle legacy also */ struct sq_psn_search *psn_search; void *inline_data; }; struct bnxt_qplib_swqe { /* General */ #define BNXT_QPLIB_FENCE_WRID 0x46454E43 /* "FENC" */ #define BNXT_QPLIB_QP1_DUMMY_WRID 0x44554D59 /* "DUMY" */ u64 wr_id; u8 reqs_type; u8 type; #define BNXT_QPLIB_SWQE_TYPE_SEND 0 #define BNXT_QPLIB_SWQE_TYPE_SEND_WITH_IMM 1 #define BNXT_QPLIB_SWQE_TYPE_SEND_WITH_INV 2 #define BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE 4 #define BNXT_QPLIB_SWQE_TYPE_RDMA_WRITE_WITH_IMM 5 #define BNXT_QPLIB_SWQE_TYPE_RDMA_READ 6 #define BNXT_QPLIB_SWQE_TYPE_ATOMIC_CMP_AND_SWP 8 #define BNXT_QPLIB_SWQE_TYPE_ATOMIC_FETCH_AND_ADD 11 #define BNXT_QPLIB_SWQE_TYPE_LOCAL_INV 12 #define BNXT_QPLIB_SWQE_TYPE_FAST_REG_MR 13 #define BNXT_QPLIB_SWQE_TYPE_REG_MR 13 #define BNXT_QPLIB_SWQE_TYPE_BIND_MW 14 #define BNXT_QPLIB_SWQE_TYPE_RECV 128 #define BNXT_QPLIB_SWQE_TYPE_RECV_RDMA_IMM 129 u8 flags; #define BNXT_QPLIB_SWQE_FLAGS_SIGNAL_COMP (1 << 0) #define BNXT_QPLIB_SWQE_FLAGS_RD_ATOMIC_FENCE (1 << 1) #define BNXT_QPLIB_SWQE_FLAGS_UC_FENCE (1 << 2) #define BNXT_QPLIB_SWQE_FLAGS_SOLICIT_EVENT (1 << 3) #define BNXT_QPLIB_SWQE_FLAGS_INLINE (1 << 4) struct bnxt_qplib_sge *sg_list; int num_sge; union { /* Send, with imm, inval key */ struct { union { __be32 imm_data; u32 inv_key; }; u32 q_key; u32 dst_qp; u16 avid; } send; /* Send Raw Ethernet and QP1 */ struct { u16 lflags; u16 cfa_action; u32 cfa_meta; } rawqp1; /* RDMA write, with imm, read */ struct { union { __be32 imm_data; u32 inv_key; }; u64 remote_va; u32 r_key; } rdma; /* Atomic cmp/swap, fetch/add */ struct { u64 remote_va; u32 r_key; u64 swap_data; u64 cmp_data; } atomic; /* Local Invalidate */ struct { u32 inv_l_key; } local_inv; /* FR-PMR */ struct { u8 access_cntl; u8 pg_sz_log; bool zero_based; u32 l_key; u32 length; u8 pbl_pg_sz_log; #define BNXT_QPLIB_SWQE_PAGE_SIZE_4K 0 #define BNXT_QPLIB_SWQE_PAGE_SIZE_8K 1 #define BNXT_QPLIB_SWQE_PAGE_SIZE_64K 4 #define BNXT_QPLIB_SWQE_PAGE_SIZE_256K 6 #define BNXT_QPLIB_SWQE_PAGE_SIZE_1M 8 #define BNXT_QPLIB_SWQE_PAGE_SIZE_2M 9 #define BNXT_QPLIB_SWQE_PAGE_SIZE_4M 10 #define BNXT_QPLIB_SWQE_PAGE_SIZE_1G 18 u8 levels; #define PAGE_SHIFT_4K 12 __le64 *pbl_ptr; dma_addr_t pbl_dma_ptr; u64 *page_list; u16 page_list_len; u64 va; } frmr; /* Bind */ struct { u8 access_cntl; #define BNXT_QPLIB_BIND_SWQE_ACCESS_LOCAL_WRITE (1 << 0) #define BNXT_QPLIB_BIND_SWQE_ACCESS_REMOTE_READ (1 << 1) #define BNXT_QPLIB_BIND_SWQE_ACCESS_REMOTE_WRITE (1 << 2) #define BNXT_QPLIB_BIND_SWQE_ACCESS_REMOTE_ATOMIC (1 << 3) #define BNXT_QPLIB_BIND_SWQE_ACCESS_WINDOW_BIND (1 << 4) bool zero_based; u8 mw_type; u32 parent_l_key; u32 r_key; u64 va; u32 length; } bind; }; }; struct bnxt_qplib_q { struct bnxt_qplib_swq *swq; struct bnxt_qplib_db_info dbinfo; struct bnxt_qplib_sg_info sginfo; struct bnxt_qplib_hwq hwq; u32 max_wqe; u16 max_sge; u16 wqe_size; u16 q_full_delta; u32 psn; bool condition; bool single; bool legacy_send_phantom; u32 phantom_wqe_cnt; u32 phantom_cqe_cnt; u32 next_cq_cons; bool flushed; u32 swq_start; u32 swq_last; }; #define BNXT_QPLIB_PPP_REQ 0x1 #define BNXT_QPLIB_PPP_ST_IDX_SHIFT 0x1 struct bnxt_qplib_ppp { u32 dpi; u8 req; u8 st_idx_en; }; struct bnxt_qplib_qp { struct bnxt_qplib_pd *pd; struct bnxt_qplib_dpi *dpi; struct bnxt_qplib_chip_ctx *cctx; u64 qp_handle; #define BNXT_QPLIB_QP_ID_INVALID 0xFFFFFFFF u32 id; u8 type; u8 sig_type; u8 wqe_mode; u8 state; u8 cur_qp_state; u8 is_user; u64 modify_flags; u32 max_inline_data; u32 mtu; u32 path_mtu; bool en_sqd_async_notify; u16 pkey_index; u32 qkey; u32 dest_qp_id; u8 access; u8 timeout; u8 retry_cnt; u8 rnr_retry; u64 wqe_cnt; u32 min_rnr_timer; u32 max_rd_atomic; u32 max_dest_rd_atomic; u32 dest_qpn; u8 smac[6]; u16 vlan_id; u8 nw_type; u16 port_id; struct bnxt_qplib_ah ah; struct bnxt_qplib_ppp ppp; #define BTH_PSN_MASK ((1 << 24) - 1) /* SQ */ struct bnxt_qplib_q sq; /* RQ */ struct bnxt_qplib_q rq; /* SRQ */ struct bnxt_qplib_srq *srq; /* CQ */ struct bnxt_qplib_cq *scq; struct bnxt_qplib_cq *rcq; /* IRRQ and ORRQ */ struct bnxt_qplib_hwq irrq; struct bnxt_qplib_hwq orrq; /* Header buffer for QP1 */ struct bnxt_qplib_hdrbuf *sq_hdr_buf; struct bnxt_qplib_hdrbuf *rq_hdr_buf; /* ToS */ u8 tos_ecn; u8 tos_dscp; /* To track the SQ and RQ flush list */ struct list_head sq_flush; struct list_head rq_flush; /* 4 bytes of QP's scrabled mac received from FW */ u32 lag_src_mac; u32 msn; u32 msn_tbl_sz; /* get devflags in PI code */ u16 dev_cap_flags; + bool is_host_msn_tbl; }; #define CQE_CMP_VALID(hdr, pass) \ (!!((hdr)->cqe_type_toggle & CQ_BASE_TOGGLE) == \ !(pass & BNXT_QPLIB_FLAG_EPOCH_CONS_MASK)) static inline u32 __bnxt_qplib_get_avail(struct bnxt_qplib_hwq *hwq) { int cons, prod, avail; /* False full is possible retrying post-send makes sense */ cons = hwq->cons; prod = hwq->prod; avail = cons - prod; if (cons <= prod) avail += hwq->depth; return avail; } static inline bool bnxt_qplib_queue_full(struct bnxt_qplib_hwq *hwq, u8 slots) { return __bnxt_qplib_get_avail(hwq) <= slots; } struct bnxt_qplib_cqe { u8 status; u8 type; u8 opcode; u32 length; /* Lower 16 is cfa_metadata0, Upper 16 is cfa_metadata1 */ u32 cfa_meta; #define BNXT_QPLIB_META1_SHIFT 16 #define BNXT_QPLIB_CQE_CFA_META1_VALID 0x80000UL u64 wr_id; union { __be32 immdata; u32 invrkey; }; u64 qp_handle; u64 mr_handle; u16 flags; u8 smac[6]; u32 src_qp; u16 raweth_qp1_flags; u16 raweth_qp1_errors; u16 raweth_qp1_cfa_code; u32 raweth_qp1_flags2; u32 raweth_qp1_metadata; u8 raweth_qp1_payload_offset; u16 pkey_index; }; #define BNXT_QPLIB_QUEUE_START_PERIOD 0x01 struct bnxt_qplib_cq { struct bnxt_qplib_dpi *dpi; struct bnxt_qplib_chip_ctx *cctx; struct bnxt_qplib_nq *nq; struct bnxt_qplib_db_info dbinfo; struct bnxt_qplib_sg_info sginfo; struct bnxt_qplib_hwq hwq; struct bnxt_qplib_hwq resize_hwq; struct list_head sqf_head; struct list_head rqf_head; u32 max_wqe; u32 id; u16 count; u16 period; u32 cnq_hw_ring_id; u64 cq_handle; atomic_t arm_state; #define CQ_RESIZE_WAIT_TIME_MS 500 unsigned long flags; #define CQ_FLAGS_RESIZE_IN_PROG 1 wait_queue_head_t waitq; spinlock_t flush_lock; /* lock flush queue list */ spinlock_t compl_lock; /* synch CQ handlers */ u16 cnq_events; bool is_cq_err_event; bool destroyed; u8 toggle; }; #define BNXT_QPLIB_MAX_IRRQE_ENTRY_SIZE sizeof(struct xrrq_irrq) #define BNXT_QPLIB_MAX_ORRQE_ENTRY_SIZE sizeof(struct xrrq_orrq) #define IRD_LIMIT_TO_IRRQ_SLOTS(x) (2 * x + 2) #define IRRQ_SLOTS_TO_IRD_LIMIT(s) ((s >> 1) - 1) #define ORD_LIMIT_TO_ORRQ_SLOTS(x) (x + 1) #define ORRQ_SLOTS_TO_ORD_LIMIT(s) (s - 1) #define NQE_CMP_VALID(hdr, pass) \ (!!(le32_to_cpu((hdr)->info63_v & 0xffffffff) & NQ_BASE_V) == \ !(pass & BNXT_QPLIB_FLAG_EPOCH_CONS_MASK)) #define BNXT_QPLIB_NQE_MAX_CNT (128 * 1024) /* MSN table print macros for debugging */ #define BNXT_RE_MSN_IDX(m) (((m) & SQ_MSN_SEARCH_START_IDX_MASK) >> \ SQ_MSN_SEARCH_START_IDX_SFT) #define BNXT_RE_MSN_NPSN(m) (((m) & SQ_MSN_SEARCH_NEXT_PSN_MASK) >> \ SQ_MSN_SEARCH_NEXT_PSN_SFT) #define BNXT_RE_MSN_SPSN(m) (((m) & SQ_MSN_SEARCH_START_PSN_MASK) >> \ SQ_MSN_SEARCH_START_PSN_SFT) #define BNXT_MSN_TBLE_SGE 6 struct bnxt_qplib_nq_stats { u64 num_dbqne_processed; u64 num_srqne_processed; u64 num_cqne_processed; u64 num_tasklet_resched; u64 num_nq_rearm; }; struct bnxt_qplib_nq_db { struct bnxt_qplib_reg_desc reg; void __iomem *db; struct bnxt_qplib_db_info dbinfo; }; typedef int (*cqn_handler_t)(struct bnxt_qplib_nq *nq, struct bnxt_qplib_cq *cq); typedef int (*srqn_handler_t)(struct bnxt_qplib_nq *nq, struct bnxt_qplib_srq *srq, u8 event); struct bnxt_qplib_nq { struct bnxt_qplib_res *res; struct bnxt_qplib_hwq hwq; struct bnxt_qplib_nq_db nq_db; char *name; u16 ring_id; int msix_vec; bool requested; int budget; u32 load; struct mutex lock; cqn_handler_t cqn_handler; srqn_handler_t srqn_handler; struct workqueue_struct *cqn_wq; struct bnxt_qplib_nq_stats stats; }; struct bnxt_qplib_nq_work { struct work_struct work; struct bnxt_qplib_nq *nq; struct bnxt_qplib_cq *cq; }; static inline dma_addr_t bnxt_qplib_get_qp_buf_from_index(struct bnxt_qplib_qp *qp, u32 index) { struct bnxt_qplib_hdrbuf *buf; buf = qp->rq_hdr_buf; return (buf->dma_map + index * buf->step); } void bnxt_qplib_nq_stop_irq(struct bnxt_qplib_nq *nq, bool kill); void bnxt_qplib_disable_nq(struct bnxt_qplib_nq *nq); int bnxt_qplib_nq_start_irq(struct bnxt_qplib_nq *nq, int nq_indx, int msix_vector, bool need_init); int bnxt_qplib_enable_nq(struct bnxt_qplib_nq *nq, int nq_idx, int msix_vector, int bar_reg_offset, cqn_handler_t cqn_handler, srqn_handler_t srq_handler); int bnxt_qplib_create_srq(struct bnxt_qplib_res *res, struct bnxt_qplib_srq *srq); int bnxt_qplib_modify_srq(struct bnxt_qplib_res *res, struct bnxt_qplib_srq *srq); int bnxt_qplib_query_srq(struct bnxt_qplib_res *res, struct bnxt_qplib_srq *srq); int bnxt_qplib_destroy_srq(struct bnxt_qplib_res *res, struct bnxt_qplib_srq *srq); int bnxt_qplib_post_srq_recv(struct bnxt_qplib_srq *srq, struct bnxt_qplib_swqe *wqe); int bnxt_qplib_create_qp1(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp); int bnxt_qplib_create_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp); int bnxt_qplib_modify_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp); int bnxt_qplib_query_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp); int bnxt_qplib_destroy_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp); void bnxt_qplib_clean_qp(struct bnxt_qplib_qp *qp); void bnxt_qplib_free_qp_res(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp); void *bnxt_qplib_get_qp1_sq_buf(struct bnxt_qplib_qp *qp, struct bnxt_qplib_sge *sge); void *bnxt_qplib_get_qp1_rq_buf(struct bnxt_qplib_qp *qp, struct bnxt_qplib_sge *sge); u32 bnxt_qplib_get_rq_prod_index(struct bnxt_qplib_qp *qp); void bnxt_qplib_post_send_db(struct bnxt_qplib_qp *qp); int bnxt_qplib_post_send(struct bnxt_qplib_qp *qp, struct bnxt_qplib_swqe *wqe); void bnxt_qplib_post_recv_db(struct bnxt_qplib_qp *qp); int bnxt_qplib_post_recv(struct bnxt_qplib_qp *qp, struct bnxt_qplib_swqe *wqe); int bnxt_qplib_create_cq(struct bnxt_qplib_res *res, struct bnxt_qplib_cq *cq); int bnxt_qplib_modify_cq(struct bnxt_qplib_res *res, struct bnxt_qplib_cq *cq); int bnxt_qplib_resize_cq(struct bnxt_qplib_res *res, struct bnxt_qplib_cq *cq, int new_cqes); void bnxt_qplib_resize_cq_complete(struct bnxt_qplib_res *res, struct bnxt_qplib_cq *cq); int bnxt_qplib_destroy_cq(struct bnxt_qplib_res *res, struct bnxt_qplib_cq *cq); void bnxt_qplib_free_cq(struct bnxt_qplib_res *res, struct bnxt_qplib_cq *cq); int bnxt_qplib_poll_cq(struct bnxt_qplib_cq *cq, struct bnxt_qplib_cqe *cqe, int num, struct bnxt_qplib_qp **qp); bool bnxt_qplib_is_cq_empty(struct bnxt_qplib_cq *cq); void bnxt_qplib_req_notify_cq(struct bnxt_qplib_cq *cq, u32 arm_type); void bnxt_qplib_free_nq_mem(struct bnxt_qplib_nq *nq); int bnxt_qplib_alloc_nq_mem(struct bnxt_qplib_res *res, struct bnxt_qplib_nq *nq); void bnxt_qplib_add_flush_qp(struct bnxt_qplib_qp *qp); void bnxt_qplib_del_flush_qp(struct bnxt_qplib_qp *qp); int bnxt_qplib_process_flush_list(struct bnxt_qplib_cq *cq, struct bnxt_qplib_cqe *cqe, int num_cqes); void bnxt_qplib_flush_cqn_wq(struct bnxt_qplib_qp *qp); void bnxt_qplib_free_hdr_buf(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp); int bnxt_qplib_alloc_hdr_buf(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp, u32 slen, u32 rlen); static inline bool __can_request_ppp(struct bnxt_qplib_qp *qp) { bool can_request = false; if (qp->cur_qp_state == CMDQ_MODIFY_QP_NEW_STATE_RESET && qp->state == CMDQ_MODIFY_QP_NEW_STATE_INIT && qp->ppp.req && !(qp->ppp.st_idx_en & CREQ_MODIFY_QP_RESP_PINGPONG_PUSH_ENABLED)) can_request = true; return can_request; } /* MSN table update inlin */ static inline uint64_t bnxt_re_update_msn_tbl(uint32_t st_idx, uint32_t npsn, uint32_t start_psn) { return cpu_to_le64((((u64)(st_idx) << SQ_MSN_SEARCH_START_IDX_SFT) & SQ_MSN_SEARCH_START_IDX_MASK) | (((u64)(npsn) << SQ_MSN_SEARCH_NEXT_PSN_SFT) & SQ_MSN_SEARCH_NEXT_PSN_MASK) | (((start_psn) << SQ_MSN_SEARCH_START_PSN_SFT) & SQ_MSN_SEARCH_START_PSN_MASK)); } void bnxt_re_schedule_dbq_event(struct bnxt_qplib_res *res); #endif diff --git a/sys/dev/bnxt/bnxt_re/qplib_res.c b/sys/dev/bnxt/bnxt_re/qplib_res.c index f527af031176..9051f4c9f2b7 100644 --- a/sys/dev/bnxt/bnxt_re/qplib_res.c +++ b/sys/dev/bnxt/bnxt_re/qplib_res.c @@ -1,1226 +1,1227 @@ /* * Copyright (c) 2015-2024, Broadcom. All rights reserved. The term * Broadcom refers to Broadcom Limited and/or its subsidiaries. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in * the documentation and/or other materials provided with the * distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * * Description: QPLib resource manager */ #include #include #include #include #include #include #include #include #include #include "hsi_struct_def.h" #include "qplib_res.h" #include "qplib_sp.h" #include "qplib_rcfw.h" #include "bnxt.h" #include "bnxt_ulp.h" uint8_t _get_chip_gen_p5_type(struct bnxt_qplib_chip_ctx *cctx) { /* Extend this for granular type */ return(BNXT_RE_DEFAULT); } inline bool _is_alloc_mr_unified(struct bnxt_qplib_dev_attr *dattr) { return dattr->dev_cap_flags & CREQ_QUERY_FUNC_RESP_SB_MR_REGISTER_ALLOC; } /* PBL */ static void __free_pbl(struct bnxt_qplib_res *res, struct bnxt_qplib_pbl *pbl, bool is_umem) { struct pci_dev *pdev; int i; pdev = res->pdev; if (is_umem == false) { for (i = 0; i < pbl->pg_count; i++) { if (pbl->pg_arr[i]) { dma_free_coherent(&pdev->dev, pbl->pg_size, (void *)((u64)pbl->pg_arr[i] & PAGE_MASK), pbl->pg_map_arr[i]); } else dev_warn(&pdev->dev, "QPLIB: PBL free pg_arr[%d] empty?!\n", i); pbl->pg_arr[i] = NULL; } } if (pbl->pg_arr) { vfree(pbl->pg_arr); pbl->pg_arr = NULL; } if (pbl->pg_map_arr) { vfree(pbl->pg_map_arr); pbl->pg_map_arr = NULL; } pbl->pg_count = 0; pbl->pg_size = 0; } struct qplib_sg { dma_addr_t pg_map_arr; u32 size; }; static int __fill_user_dma_pages(struct bnxt_qplib_pbl *pbl, struct bnxt_qplib_sg_info *sginfo) { int sg_indx, pg_indx, tmp_size, offset; struct qplib_sg *tmp_sg = NULL; struct scatterlist *sg; u64 pmask, addr; tmp_sg = vzalloc(sginfo->nmap * sizeof(struct qplib_sg)); if (!tmp_sg) return -ENOMEM; pmask = BIT_ULL(sginfo->pgshft) - 1; sg_indx = 0; for_each_sg(sginfo->sghead, sg, sginfo->nmap, sg_indx) { tmp_sg[sg_indx].pg_map_arr = sg_dma_address(sg); tmp_sg[sg_indx].size = sg_dma_len(sg); } pg_indx = 0; for (sg_indx = 0; sg_indx < sginfo->nmap; sg_indx++) { tmp_size = tmp_sg[sg_indx].size; offset = 0; while (tmp_size > 0) { addr = tmp_sg[sg_indx].pg_map_arr + offset; if ((!sg_indx && !pg_indx) || !(addr & pmask)) { pbl->pg_map_arr[pg_indx] = addr &(~pmask); pbl->pg_count++; pg_indx++; } offset += sginfo->pgsize; tmp_size -= sginfo->pgsize; } } vfree(tmp_sg); return 0; } static int bnxt_qplib_fill_user_dma_pages(struct bnxt_qplib_pbl *pbl, struct bnxt_qplib_sg_info *sginfo) { int rc = 0; rc = __fill_user_dma_pages(pbl, sginfo); return rc; } static int __alloc_pbl(struct bnxt_qplib_res *res, struct bnxt_qplib_pbl *pbl, struct bnxt_qplib_sg_info *sginfo) { struct pci_dev *pdev; bool is_umem = false; int i; if (sginfo->nopte) return 0; pdev = res->pdev; /* page ptr arrays */ pbl->pg_arr = vmalloc(sginfo->npages * sizeof(void *)); if (!pbl->pg_arr) return -ENOMEM; pbl->pg_map_arr = vmalloc(sginfo->npages * sizeof(dma_addr_t)); if (!pbl->pg_map_arr) { vfree(pbl->pg_arr); return -ENOMEM; } pbl->pg_count = 0; pbl->pg_size = sginfo->pgsize; if (!sginfo->sghead) { for (i = 0; i < sginfo->npages; i++) { pbl->pg_arr[i] = dma_zalloc_coherent(&pdev->dev, pbl->pg_size, &pbl->pg_map_arr[i], GFP_KERNEL); if (!pbl->pg_arr[i]) goto fail; pbl->pg_count++; } } else { is_umem = true; if (bnxt_qplib_fill_user_dma_pages(pbl, sginfo)) goto fail; } return 0; fail: __free_pbl(res, pbl, is_umem); return -ENOMEM; } /* HWQ */ void bnxt_qplib_free_hwq(struct bnxt_qplib_res *res, struct bnxt_qplib_hwq *hwq) { int i; if (!hwq->max_elements) return; if (hwq->level >= PBL_LVL_MAX) return; for (i = 0; i < hwq->level + 1; i++) { if (i == hwq->level) __free_pbl(res, &hwq->pbl[i], hwq->is_user); else __free_pbl(res, &hwq->pbl[i], false); } hwq->level = PBL_LVL_MAX; hwq->max_elements = 0; hwq->element_size = 0; hwq->prod = hwq->cons = 0; hwq->cp_bit = 0; } /* All HWQs are power of 2 in size */ int bnxt_qplib_alloc_init_hwq(struct bnxt_qplib_hwq *hwq, struct bnxt_qplib_hwq_attr *hwq_attr) { u32 npages = 0, depth, stride, aux_pages = 0; dma_addr_t *src_phys_ptr, **dst_virt_ptr; struct bnxt_qplib_sg_info sginfo = {}; u32 aux_size = 0, npbl, npde; void *umem; struct bnxt_qplib_res *res; u32 aux_slots, pg_size; struct pci_dev *pdev; int i, rc, lvl; res = hwq_attr->res; pdev = res->pdev; umem = hwq_attr->sginfo->sghead; pg_size = hwq_attr->sginfo->pgsize; hwq->level = PBL_LVL_MAX; depth = roundup_pow_of_two(hwq_attr->depth); stride = roundup_pow_of_two(hwq_attr->stride); if (hwq_attr->aux_depth) { aux_slots = hwq_attr->aux_depth; aux_size = roundup_pow_of_two(hwq_attr->aux_stride); aux_pages = (aux_slots * aux_size) / pg_size; if ((aux_slots * aux_size) % pg_size) aux_pages++; } if (!umem) { hwq->is_user = false; npages = (depth * stride) / pg_size + aux_pages; if ((depth * stride) % pg_size) npages++; if (!npages) return -EINVAL; hwq_attr->sginfo->npages = npages; } else { hwq->is_user = true; npages = hwq_attr->sginfo->npages; npages = (npages * (u64)pg_size) / BIT_ULL(hwq_attr->sginfo->pgshft); if ((hwq_attr->sginfo->npages * (u64)pg_size) % BIT_ULL(hwq_attr->sginfo->pgshft)) npages++; } if (npages == MAX_PBL_LVL_0_PGS && !hwq_attr->sginfo->nopte) { /* This request is Level 0, map PTE */ rc = __alloc_pbl(res, &hwq->pbl[PBL_LVL_0], hwq_attr->sginfo); if (rc) goto fail; hwq->level = PBL_LVL_0; goto done; } if (npages >= MAX_PBL_LVL_0_PGS) { if (npages > MAX_PBL_LVL_1_PGS) { u32 flag = (hwq_attr->type == HWQ_TYPE_L2_CMPL) ? 0 : PTU_PTE_VALID; /* 2 levels of indirection */ npbl = npages >> MAX_PBL_LVL_1_PGS_SHIFT; if (npages % BIT(MAX_PBL_LVL_1_PGS_SHIFT)) npbl++; npde = npbl >> MAX_PDL_LVL_SHIFT; if(npbl % BIT(MAX_PDL_LVL_SHIFT)) npde++; /* Alloc PDE pages */ sginfo.pgsize = npde * PAGE_SIZE; sginfo.npages = 1; rc = __alloc_pbl(res, &hwq->pbl[PBL_LVL_0], &sginfo); /* Alloc PBL pages */ sginfo.npages = npbl; sginfo.pgsize = PAGE_SIZE; rc = __alloc_pbl(res, &hwq->pbl[PBL_LVL_1], &sginfo); if (rc) goto fail; /* Fill PDL with PBL page pointers */ dst_virt_ptr = (dma_addr_t **)hwq->pbl[PBL_LVL_0].pg_arr; src_phys_ptr = hwq->pbl[PBL_LVL_1].pg_map_arr; if (hwq_attr->type == HWQ_TYPE_MR) { /* For MR it is expected that we supply only 1 contigous * page i.e only 1 entry in the PDL that will contain * all the PBLs for the user supplied memory region */ for (i = 0; i < hwq->pbl[PBL_LVL_1].pg_count; i++) dst_virt_ptr[0][i] = src_phys_ptr[i] | flag; } else { for (i = 0; i < hwq->pbl[PBL_LVL_1].pg_count; i++) dst_virt_ptr[PTR_PG(i)][PTR_IDX(i)] = src_phys_ptr[i] | PTU_PDE_VALID; } /* Alloc or init PTEs */ rc = __alloc_pbl(res, &hwq->pbl[PBL_LVL_2], hwq_attr->sginfo); if (rc) goto fail; hwq->level = PBL_LVL_2; if (hwq_attr->sginfo->nopte) goto done; /* Fill PBLs with PTE pointers */ dst_virt_ptr = (dma_addr_t **)hwq->pbl[PBL_LVL_1].pg_arr; src_phys_ptr = hwq->pbl[PBL_LVL_2].pg_map_arr; for (i = 0; i < hwq->pbl[PBL_LVL_2].pg_count; i++) { dst_virt_ptr[PTR_PG(i)][PTR_IDX(i)] = src_phys_ptr[i] | PTU_PTE_VALID; } if (hwq_attr->type == HWQ_TYPE_QUEUE) { /* Find the last pg of the size */ i = hwq->pbl[PBL_LVL_2].pg_count; dst_virt_ptr[PTR_PG(i - 1)][PTR_IDX(i - 1)] |= PTU_PTE_LAST; if (i > 1) dst_virt_ptr[PTR_PG(i - 2)] [PTR_IDX(i - 2)] |= PTU_PTE_NEXT_TO_LAST; } } else { /* pages < 512 npbl = 1, npde = 0 */ u32 flag = (hwq_attr->type == HWQ_TYPE_L2_CMPL) ? 0 : PTU_PTE_VALID; /* 1 level of indirection */ npbl = npages >> MAX_PBL_LVL_1_PGS_SHIFT; if (npages % BIT(MAX_PBL_LVL_1_PGS_SHIFT)) npbl++; sginfo.npages = npbl; sginfo.pgsize = PAGE_SIZE; /* Alloc PBL page */ rc = __alloc_pbl(res, &hwq->pbl[PBL_LVL_0], &sginfo); if (rc) goto fail; /* Alloc or init PTEs */ rc = __alloc_pbl(res, &hwq->pbl[PBL_LVL_1], hwq_attr->sginfo); if (rc) goto fail; hwq->level = PBL_LVL_1; if (hwq_attr->sginfo->nopte) goto done; /* Fill PBL with PTE pointers */ dst_virt_ptr = (dma_addr_t **)hwq->pbl[PBL_LVL_0].pg_arr; src_phys_ptr = hwq->pbl[PBL_LVL_1].pg_map_arr; for (i = 0; i < hwq->pbl[PBL_LVL_1].pg_count; i++) dst_virt_ptr[PTR_PG(i)][PTR_IDX(i)] = src_phys_ptr[i] | flag; if (hwq_attr->type == HWQ_TYPE_QUEUE) { /* Find the last pg of the size */ i = hwq->pbl[PBL_LVL_1].pg_count; dst_virt_ptr[PTR_PG(i - 1)][PTR_IDX(i - 1)] |= PTU_PTE_LAST; if (i > 1) dst_virt_ptr[PTR_PG(i - 2)] [PTR_IDX(i - 2)] |= PTU_PTE_NEXT_TO_LAST; } } } done: hwq->prod = 0; hwq->cons = 0; hwq->pdev = pdev; hwq->depth = hwq_attr->depth; hwq->max_elements = depth; hwq->element_size = stride; hwq->qe_ppg = (pg_size/stride); if (hwq->level >= PBL_LVL_MAX) goto fail; /* For direct access to the elements */ lvl = hwq->level; if (hwq_attr->sginfo->nopte && hwq->level) lvl = hwq->level - 1; hwq->pbl_ptr = hwq->pbl[lvl].pg_arr; hwq->pbl_dma_ptr = hwq->pbl[lvl].pg_map_arr; spin_lock_init(&hwq->lock); return 0; fail: bnxt_qplib_free_hwq(res, hwq); return -ENOMEM; } /* Context Tables */ void bnxt_qplib_free_hwctx(struct bnxt_qplib_res *res) { struct bnxt_qplib_ctx *hctx; int i; hctx = res->hctx; bnxt_qplib_free_hwq(res, &hctx->qp_ctx.hwq); bnxt_qplib_free_hwq(res, &hctx->mrw_ctx.hwq); bnxt_qplib_free_hwq(res, &hctx->srq_ctx.hwq); bnxt_qplib_free_hwq(res, &hctx->cq_ctx.hwq); bnxt_qplib_free_hwq(res, &hctx->tim_ctx.hwq); for (i = 0; i < MAX_TQM_ALLOC_REQ; i++) bnxt_qplib_free_hwq(res, &hctx->tqm_ctx.qtbl[i]); /* restor original pde level before destroy */ hctx->tqm_ctx.pde.level = hctx->tqm_ctx.pde_level; bnxt_qplib_free_hwq(res, &hctx->tqm_ctx.pde); } static int bnxt_qplib_alloc_tqm_rings(struct bnxt_qplib_res *res, struct bnxt_qplib_ctx *hctx) { struct bnxt_qplib_hwq_attr hwq_attr = {}; struct bnxt_qplib_sg_info sginfo = {}; struct bnxt_qplib_tqm_ctx *tqmctx; int rc = 0; int i; tqmctx = &hctx->tqm_ctx; sginfo.pgsize = PAGE_SIZE; sginfo.pgshft = PAGE_SHIFT; hwq_attr.sginfo = &sginfo; hwq_attr.res = res; hwq_attr.type = HWQ_TYPE_CTX; hwq_attr.depth = 512; hwq_attr.stride = sizeof(u64); /* Alloc pdl buffer */ rc = bnxt_qplib_alloc_init_hwq(&tqmctx->pde, &hwq_attr); if (rc) goto out; /* Save original pdl level */ tqmctx->pde_level = tqmctx->pde.level; hwq_attr.stride = 1; for (i = 0; i < MAX_TQM_ALLOC_REQ; i++) { if (!tqmctx->qcount[i]) continue; hwq_attr.depth = hctx->qp_ctx.max * tqmctx->qcount[i]; rc = bnxt_qplib_alloc_init_hwq(&tqmctx->qtbl[i], &hwq_attr); if (rc) goto out; } out: return rc; } static void bnxt_qplib_map_tqm_pgtbl(struct bnxt_qplib_tqm_ctx *ctx) { struct bnxt_qplib_hwq *qtbl_hwq; dma_addr_t *dma_ptr; __le64 **pbl_ptr, *ptr; int i, j, k; int fnz_idx = -1; int pg_count; pbl_ptr = (__le64 **)ctx->pde.pbl_ptr; for (i = 0, j = 0; i < MAX_TQM_ALLOC_REQ; i++, j += MAX_TQM_ALLOC_BLK_SIZE) { qtbl_hwq = &ctx->qtbl[i]; if (!qtbl_hwq->max_elements) continue; if (fnz_idx == -1) fnz_idx = i; /* first non-zero index */ switch (qtbl_hwq->level) { case PBL_LVL_2: pg_count = qtbl_hwq->pbl[PBL_LVL_1].pg_count; for (k = 0; k < pg_count; k++) { ptr = &pbl_ptr[PTR_PG(j + k)][PTR_IDX(j + k)]; dma_ptr = &qtbl_hwq->pbl[PBL_LVL_1].pg_map_arr[k]; *ptr = cpu_to_le64(*dma_ptr | PTU_PTE_VALID); } break; case PBL_LVL_1: case PBL_LVL_0: default: ptr = &pbl_ptr[PTR_PG(j)][PTR_IDX(j)]; *ptr = cpu_to_le64(qtbl_hwq->pbl[PBL_LVL_0].pg_map_arr[0] | PTU_PTE_VALID); break; } } if (fnz_idx == -1) fnz_idx = 0; /* update pde level as per page table programming */ ctx->pde.level = (ctx->qtbl[fnz_idx].level == PBL_LVL_2) ? PBL_LVL_2 : ctx->qtbl[fnz_idx].level + 1; } static int bnxt_qplib_setup_tqm_rings(struct bnxt_qplib_res *res, struct bnxt_qplib_ctx *hctx) { int rc = 0; rc = bnxt_qplib_alloc_tqm_rings(res, hctx); if (rc) goto fail; bnxt_qplib_map_tqm_pgtbl(&hctx->tqm_ctx); fail: return rc; } /* * Routine: bnxt_qplib_alloc_hwctx * Description: * Context tables are memories which are used by the chip. * The 6 tables defined are: * QPC ctx - holds QP states * MRW ctx - holds memory region and window * SRQ ctx - holds shared RQ states * CQ ctx - holds completion queue states * TQM ctx - holds Tx Queue Manager context * TIM ctx - holds timer context * Depending on the size of the tbl requested, either a 1 Page Buffer List * or a 1-to-2-stage indirection Page Directory List + 1 PBL is used * instead. * Table might be employed as follows: * For 0 < ctx size <= 1 PAGE, 0 level of ind is used * For 1 PAGE < ctx size <= 512 entries size, 1 level of ind is used * For 512 < ctx size <= MAX, 2 levels of ind is used * Returns: * 0 if success, else -ERRORS */ int bnxt_qplib_alloc_hwctx(struct bnxt_qplib_res *res) { struct bnxt_qplib_hwq_attr hwq_attr = {}; struct bnxt_qplib_sg_info sginfo = {}; struct bnxt_qplib_ctx *hctx; struct bnxt_qplib_hwq *hwq; int rc = 0; hctx = res->hctx; /* QPC Tables */ sginfo.pgsize = PAGE_SIZE; sginfo.pgshft = PAGE_SHIFT; hwq_attr.sginfo = &sginfo; hwq_attr.res = res; hwq_attr.depth = hctx->qp_ctx.max; hwq_attr.stride = BNXT_QPLIB_MAX_QP_CTX_ENTRY_SIZE; hwq_attr.type = HWQ_TYPE_CTX; hwq = &hctx->qp_ctx.hwq; rc = bnxt_qplib_alloc_init_hwq(hwq, &hwq_attr); if (rc) goto fail; /* MRW Tables */ hwq_attr.depth = hctx->mrw_ctx.max; hwq_attr.stride = BNXT_QPLIB_MAX_MRW_CTX_ENTRY_SIZE; hwq = &hctx->mrw_ctx.hwq; rc = bnxt_qplib_alloc_init_hwq(hwq, &hwq_attr); if (rc) goto fail; /* SRQ Tables */ hwq_attr.depth = hctx->srq_ctx.max; hwq_attr.stride = BNXT_QPLIB_MAX_SRQ_CTX_ENTRY_SIZE; hwq = &hctx->srq_ctx.hwq; rc = bnxt_qplib_alloc_init_hwq(hwq, &hwq_attr); if (rc) goto fail; /* CQ Tables */ hwq_attr.depth = hctx->cq_ctx.max; hwq_attr.stride = BNXT_QPLIB_MAX_CQ_CTX_ENTRY_SIZE; hwq = &hctx->cq_ctx.hwq; rc = bnxt_qplib_alloc_init_hwq(hwq, &hwq_attr); if (rc) goto fail; /* TQM Buffer */ rc = bnxt_qplib_setup_tqm_rings(res, hctx); if (rc) goto fail; /* TIM Buffer */ hwq_attr.depth = hctx->qp_ctx.max * 16; hwq_attr.stride = 1; hwq = &hctx->tim_ctx.hwq; rc = bnxt_qplib_alloc_init_hwq(hwq, &hwq_attr); if (rc) goto fail; return 0; fail: bnxt_qplib_free_hwctx(res); return rc; } /* GUID */ void bnxt_qplib_get_guid(const u8 *dev_addr, u8 *guid) { u8 mac[ETH_ALEN]; /* MAC-48 to EUI-64 mapping */ memcpy(mac, dev_addr, ETH_ALEN); guid[0] = mac[0] ^ 2; guid[1] = mac[1]; guid[2] = mac[2]; guid[3] = 0xff; guid[4] = 0xfe; guid[5] = mac[3]; guid[6] = mac[4]; guid[7] = mac[5]; } static void bnxt_qplib_free_sgid_tbl(struct bnxt_qplib_res *res) { struct bnxt_qplib_sgid_tbl *sgid_tbl; sgid_tbl = &res->sgid_tbl; if (sgid_tbl->tbl) { kfree(sgid_tbl->tbl); sgid_tbl->tbl = NULL; kfree(sgid_tbl->hw_id); sgid_tbl->hw_id = NULL; kfree(sgid_tbl->ctx); sgid_tbl->ctx = NULL; kfree(sgid_tbl->vlan); sgid_tbl->vlan = NULL; } else { dev_dbg(&res->pdev->dev, "QPLIB: SGID tbl not present"); } sgid_tbl->max = 0; sgid_tbl->active = 0; } static void bnxt_qplib_free_reftbls(struct bnxt_qplib_res *res) { struct bnxt_qplib_reftbl *tbl; tbl = &res->reftbl.srqref; vfree(tbl->rec); tbl = &res->reftbl.cqref; vfree(tbl->rec); tbl = &res->reftbl.qpref; vfree(tbl->rec); } static int bnxt_qplib_alloc_reftbl(struct bnxt_qplib_reftbl *tbl, u32 max) { tbl->max = max; tbl->rec = vzalloc(sizeof(*tbl->rec) * max); if (!tbl->rec) return -ENOMEM; spin_lock_init(&tbl->lock); return 0; } static int bnxt_qplib_alloc_reftbls(struct bnxt_qplib_res *res, struct bnxt_qplib_dev_attr *dattr) { u32 max_cq = BNXT_QPLIB_MAX_CQ_COUNT; struct bnxt_qplib_reftbl *tbl; u32 res_cnt; int rc; /* * Allocating one extra entry to hold QP1 info. * Store QP1 info at the last entry of the table. * Decrement the tbl->max by one so that modulo * operation to get the qp table index from qp id * returns any value between 0 and max_qp-1 */ res_cnt = max_t(u32, BNXT_QPLIB_MAX_QPC_COUNT + 1, dattr->max_qp); tbl = &res->reftbl.qpref; rc = bnxt_qplib_alloc_reftbl(tbl, res_cnt); if (rc) goto fail; tbl->max--; if (_is_chip_gen_p5_p7(res->cctx)) max_cq = BNXT_QPLIB_MAX_CQ_COUNT_P5; res_cnt = max_t(u32, max_cq, dattr->max_cq); tbl = &res->reftbl.cqref; rc = bnxt_qplib_alloc_reftbl(tbl, res_cnt); if (rc) goto fail; res_cnt = max_t(u32, BNXT_QPLIB_MAX_SRQC_COUNT, dattr->max_cq); tbl = &res->reftbl.srqref; rc = bnxt_qplib_alloc_reftbl(tbl, BNXT_QPLIB_MAX_SRQC_COUNT); if (rc) goto fail; return 0; fail: return rc; } static int bnxt_qplib_alloc_sgid_tbl(struct bnxt_qplib_res *res, u16 max) { struct bnxt_qplib_sgid_tbl *sgid_tbl; sgid_tbl = &res->sgid_tbl; sgid_tbl->tbl = kcalloc(max, sizeof(*sgid_tbl->tbl), GFP_KERNEL); if (!sgid_tbl->tbl) return -ENOMEM; sgid_tbl->hw_id = kcalloc(max, sizeof(u32), GFP_KERNEL); if (!sgid_tbl->hw_id) goto free_tbl; sgid_tbl->ctx = kcalloc(max, sizeof(void *), GFP_KERNEL); if (!sgid_tbl->ctx) goto free_hw_id; sgid_tbl->vlan = kcalloc(max, sizeof(u8), GFP_KERNEL); if (!sgid_tbl->vlan) goto free_ctx; sgid_tbl->max = max; return 0; free_ctx: kfree(sgid_tbl->ctx); free_hw_id: kfree(sgid_tbl->hw_id); free_tbl: kfree(sgid_tbl->tbl); return -ENOMEM; }; static void bnxt_qplib_cleanup_sgid_tbl(struct bnxt_qplib_res *res, struct bnxt_qplib_sgid_tbl *sgid_tbl) { int i; for (i = 0; i < sgid_tbl->max; i++) { if (memcmp(&sgid_tbl->tbl[i], &bnxt_qplib_gid_zero, sizeof(bnxt_qplib_gid_zero))) bnxt_qplib_del_sgid(sgid_tbl, &sgid_tbl->tbl[i].gid, sgid_tbl->tbl[i].vlan_id, true); } memset(sgid_tbl->tbl, 0, sizeof(*sgid_tbl->tbl) * sgid_tbl->max); memset(sgid_tbl->hw_id, -1, sizeof(u16) * sgid_tbl->max); memset(sgid_tbl->vlan, 0, sizeof(u8) * sgid_tbl->max); sgid_tbl->active = 0; } static void bnxt_qplib_init_sgid_tbl(struct bnxt_qplib_sgid_tbl *sgid_tbl, struct ifnet *netdev) { u32 i; for (i = 0; i < sgid_tbl->max; i++) sgid_tbl->tbl[i].vlan_id = 0xffff; memset(sgid_tbl->hw_id, -1, sizeof(u16) * sgid_tbl->max); } /* PDs */ int bnxt_qplib_alloc_pd(struct bnxt_qplib_res *res, struct bnxt_qplib_pd *pd) { u32 bit_num; int rc = 0; struct bnxt_qplib_pd_tbl *pdt = &res->pd_tbl; mutex_lock(&res->pd_tbl_lock); bit_num = find_first_bit(pdt->tbl, pdt->max); if (bit_num == pdt->max - 1) {/* Last bit is reserved */ rc = -ENOMEM; goto fail; } /* Found unused PD */ clear_bit(bit_num, pdt->tbl); pd->id = bit_num; fail: mutex_unlock(&res->pd_tbl_lock); return rc; } int bnxt_qplib_dealloc_pd(struct bnxt_qplib_res *res, struct bnxt_qplib_pd_tbl *pdt, struct bnxt_qplib_pd *pd) { mutex_lock(&res->pd_tbl_lock); if (test_and_set_bit(pd->id, pdt->tbl)) { dev_warn(&res->pdev->dev, "Freeing an unused PD? pdn = %d\n", pd->id); mutex_unlock(&res->pd_tbl_lock); return -EINVAL; } /* Reset to reserved pdid. */ pd->id = pdt->max - 1; mutex_unlock(&res->pd_tbl_lock); return 0; } static void bnxt_qplib_free_pd_tbl(struct bnxt_qplib_pd_tbl *pdt) { if (pdt->tbl) { kfree(pdt->tbl); pdt->tbl = NULL; } pdt->max = 0; } static int bnxt_qplib_alloc_pd_tbl(struct bnxt_qplib_res *res, u32 max) { struct bnxt_qplib_pd_tbl *pdt; u32 bytes; pdt = &res->pd_tbl; max++; /* One extra for reserved pdid. */ bytes = DIV_ROUND_UP(max, 8); if (!bytes) bytes = 1; pdt->tbl = kmalloc(bytes, GFP_KERNEL); if (!pdt->tbl) { dev_err(&res->pdev->dev, "QPLIB: PD tbl allocation failed for size = %d\n", bytes); return -ENOMEM; } pdt->max = max; memset((u8 *)pdt->tbl, 0xFF, bytes); mutex_init(&res->pd_tbl_lock); return 0; } /* DPIs */ int bnxt_qplib_alloc_dpi(struct bnxt_qplib_res *res, struct bnxt_qplib_dpi *dpi, void *app, u8 type) { struct bnxt_qplib_dpi_tbl *dpit = &res->dpi_tbl; struct bnxt_qplib_reg_desc *reg; u32 bit_num; u64 umaddr; int rc = 0; reg = &dpit->wcreg; mutex_lock(&res->dpi_tbl_lock); if (type == BNXT_QPLIB_DPI_TYPE_WC && _is_chip_p7(res->cctx) && !dpit->avail_ppp) { rc = -ENOMEM; goto fail; } bit_num = find_first_bit(dpit->tbl, dpit->max); if (bit_num == dpit->max) { rc = -ENOMEM; goto fail; } /* Found unused DPI */ clear_bit(bit_num, dpit->tbl); dpit->app_tbl[bit_num] = app; dpi->bit = bit_num; dpi->dpi = bit_num + (reg->offset - dpit->ucreg.offset) / PAGE_SIZE; umaddr = reg->bar_base + reg->offset + bit_num * PAGE_SIZE; dpi->umdbr = umaddr; switch (type) { case BNXT_QPLIB_DPI_TYPE_KERNEL: /* privileged dbr was already mapped just initialize it. */ dpi->umdbr = dpit->ucreg.bar_base + dpit->ucreg.offset + bit_num * PAGE_SIZE; dpi->dbr = dpit->priv_db; dpi->dpi = dpi->bit; break; case BNXT_QPLIB_DPI_TYPE_WC: dpi->dbr = ioremap_wc(umaddr, PAGE_SIZE); if (_is_chip_p7(res->cctx) && dpi->dbr) dpit->avail_ppp--; break; default: dpi->dbr = ioremap(umaddr, PAGE_SIZE); } if (!dpi->dbr) { dev_err(&res->pdev->dev, "QPLIB: DB remap failed, type = %d\n", type); rc = -ENOMEM; } dpi->type = type; fail: mutex_unlock(&res->dpi_tbl_lock); return rc; } int bnxt_qplib_dealloc_dpi(struct bnxt_qplib_res *res, struct bnxt_qplib_dpi *dpi) { struct bnxt_qplib_dpi_tbl *dpit = &res->dpi_tbl; int rc = 0; mutex_lock(&res->dpi_tbl_lock); if (dpi->bit >= dpit->max) { dev_warn(&res->pdev->dev, "Invalid DPI? dpi = %d, bit = %d\n", dpi->dpi, dpi->bit); rc = -EINVAL; goto fail; } if (dpi->dpi && dpi->type != BNXT_QPLIB_DPI_TYPE_KERNEL) { if (dpi->type == BNXT_QPLIB_DPI_TYPE_WC && _is_chip_p7(res->cctx) && dpi->dbr) dpit->avail_ppp++; pci_iounmap(res->pdev, dpi->dbr); } if (test_and_set_bit(dpi->bit, dpit->tbl)) { dev_warn(&res->pdev->dev, "Freeing an unused DPI? dpi = %d, bit = %d\n", dpi->dpi, dpi->bit); rc = -EINVAL; goto fail; } if (dpit->app_tbl) dpit->app_tbl[dpi->bit] = NULL; memset(dpi, 0, sizeof(*dpi)); fail: mutex_unlock(&res->dpi_tbl_lock); return rc; } static void bnxt_qplib_free_dpi_tbl(struct bnxt_qplib_dpi_tbl *dpit) { kfree(dpit->tbl); kfree(dpit->app_tbl); dpit->tbl = NULL; dpit->app_tbl = NULL; dpit->max = 0; } static int bnxt_qplib_alloc_dpi_tbl(struct bnxt_qplib_res *res, struct bnxt_qplib_dev_attr *dev_attr, u8 pppp_factor) { struct bnxt_qplib_dpi_tbl *dpit; struct bnxt_qplib_reg_desc *reg; unsigned long bar_len; u32 dbr_offset; u32 bytes; dpit = &res->dpi_tbl; reg = &dpit->wcreg; if (!_is_chip_gen_p5_p7(res->cctx)) { /* Offest should come from L2 driver */ dbr_offset = dev_attr->l2_db_size; dpit->ucreg.offset = dbr_offset; dpit->wcreg.offset = dbr_offset; } bar_len = pci_resource_len(res->pdev, reg->bar_id); dpit->max = (bar_len - reg->offset) / PAGE_SIZE; if (dev_attr->max_dpi) dpit->max = min_t(u32, dpit->max, dev_attr->max_dpi); dpit->app_tbl = kzalloc(dpit->max * sizeof(void*), GFP_KERNEL); if (!dpit->app_tbl) { dev_err(&res->pdev->dev, "QPLIB: DPI app tbl allocation failed"); return -ENOMEM; } bytes = dpit->max >> 3; if (!bytes) bytes = 1; dpit->tbl = kmalloc(bytes, GFP_KERNEL); if (!dpit->tbl) { kfree(dpit->app_tbl); dev_err(&res->pdev->dev, "QPLIB: DPI tbl allocation failed for size = %d\n", bytes); return -ENOMEM; } memset((u8 *)dpit->tbl, 0xFF, bytes); /* * On SR2, 2nd doorbell page of each function * is reserved for L2 PPP. Now that the tbl is * initialized, mark it as unavailable. By default * RoCE can make use of the 512 extended pages for * PPP. */ if (_is_chip_p7(res->cctx)) { clear_bit(1, dpit->tbl); if (pppp_factor) dpit->avail_ppp = BNXT_QPLIB_MAX_EXTENDED_PPP_PAGES / pppp_factor; } mutex_init(&res->dpi_tbl_lock); dpit->priv_db = dpit->ucreg.bar_reg + dpit->ucreg.offset; return 0; } /* Stats */ void bnxt_qplib_free_stat_mem(struct bnxt_qplib_res *res, struct bnxt_qplib_stats *stats) { struct pci_dev *pdev; pdev = res->pdev; if (stats->dma) dma_free_coherent(&pdev->dev, stats->size, stats->dma, stats->dma_map); memset(stats, 0, sizeof(*stats)); stats->fw_id = -1; } int bnxt_qplib_alloc_stat_mem(struct pci_dev *pdev, struct bnxt_qplib_chip_ctx *cctx, struct bnxt_qplib_stats *stats) { cctx->hw_stats_size = 168; memset(stats, 0, sizeof(*stats)); stats->fw_id = -1; stats->size = cctx->hw_stats_size; stats->dma = dma_alloc_coherent(&pdev->dev, stats->size, &stats->dma_map, GFP_KERNEL); if (!stats->dma) { dev_err(&pdev->dev, "QPLIB: Stats DMA allocation failed"); return -ENOMEM; } return 0; } /* Resource */ int bnxt_qplib_stop_res(struct bnxt_qplib_res *res) { struct bnxt_qplib_rcfw *rcfw = res->rcfw; struct creq_stop_func_resp resp = {}; struct bnxt_qplib_cmdqmsg msg = {}; struct cmdq_stop_func req = {}; int rc; bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_STOP_FUNC, sizeof(req)); bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL, sizeof(req), sizeof(resp), 0); rc = bnxt_qplib_rcfw_send_message(rcfw, &msg); return rc; } void bnxt_qplib_clear_tbls(struct bnxt_qplib_res *res) { bnxt_qplib_cleanup_sgid_tbl(res, &res->sgid_tbl); } int bnxt_qplib_init_tbls(struct bnxt_qplib_res *res) { bnxt_qplib_init_sgid_tbl(&res->sgid_tbl, res->netdev); return 0; } void bnxt_qplib_free_tbls(struct bnxt_qplib_res *res) { bnxt_qplib_free_sgid_tbl(res); bnxt_qplib_free_pd_tbl(&res->pd_tbl); bnxt_qplib_free_dpi_tbl(&res->dpi_tbl); bnxt_qplib_free_reftbls(res); } int bnxt_qplib_alloc_tbls(struct bnxt_qplib_res *res, u8 pppp_factor) { struct bnxt_qplib_dev_attr *dev_attr; int rc = 0; dev_attr = res->dattr; rc = bnxt_qplib_alloc_reftbls(res, dev_attr); if (rc) goto fail; rc = bnxt_qplib_alloc_sgid_tbl(res, dev_attr->max_sgid); if (rc) goto fail; rc = bnxt_qplib_alloc_pd_tbl(res, dev_attr->max_pd); if (rc) goto fail; rc = bnxt_qplib_alloc_dpi_tbl(res, dev_attr, pppp_factor); if (rc) goto fail; return 0; fail: bnxt_qplib_free_tbls(res); return rc; } void bnxt_qplib_unmap_db_bar(struct bnxt_qplib_res *res) { struct bnxt_qplib_reg_desc *reg; reg = &res->dpi_tbl.ucreg; if (reg->bar_reg) pci_iounmap(res->pdev, reg->bar_reg); reg->bar_reg = NULL; reg->bar_base = 0; reg->len = 0; reg->bar_id = 0; /* Zero? or ff */ } int bnxt_qplib_map_db_bar(struct bnxt_qplib_res *res) { struct bnxt_qplib_reg_desc *ucreg; struct bnxt_qplib_reg_desc *wcreg; wcreg = &res->dpi_tbl.wcreg; wcreg->bar_id = RCFW_DBR_PCI_BAR_REGION; if (!res || !res->pdev || !wcreg) return -1; wcreg->bar_base = pci_resource_start(res->pdev, wcreg->bar_id); /* No need to set the wcreg->len here */ ucreg = &res->dpi_tbl.ucreg; ucreg->bar_id = RCFW_DBR_PCI_BAR_REGION; ucreg->bar_base = pci_resource_start(res->pdev, ucreg->bar_id); - ucreg->offset = 65536; + if (_is_chip_gen_p5(res->cctx)) + ucreg->offset = 65536; ucreg->len = ucreg->offset + PAGE_SIZE; if (!ucreg->len || ((ucreg->len & (PAGE_SIZE - 1)) != 0)) { dev_err(&res->pdev->dev, "QPLIB: invalid dbr length %d\n", (int)ucreg->len); return -EINVAL; } ucreg->bar_reg = ioremap(ucreg->bar_base, ucreg->len); if (!ucreg->bar_reg) { dev_err(&res->pdev->dev, "privileged dpi map failed!\n"); return -ENOMEM; } return 0; } /** * pci_enable_atomic_ops_to_root - enable AtomicOp requests to root port * @dev: the PCI device * @cap_mask: mask of desired AtomicOp sizes, including one or more of: * PCI_EXP_DEVCAP2_ATOMIC_COMP32 * PCI_EXP_DEVCAP2_ATOMIC_COMP64 * PCI_EXP_DEVCAP2_ATOMIC_COMP128 * * Return 0 if all upstream bridges support AtomicOp routing, egress * blocking is disabled on all upstream ports, and the root port supports * the requested completion capabilities (32-bit, 64-bit and/or 128-bit * AtomicOp completion), or negative otherwise. */ int pci_enable_atomic_ops_to_root(struct pci_dev *dev, u32 cap_mask) { struct pci_bus *bus = dev->bus; struct pci_dev *bridge; u32 cap; if (!pci_is_pcie(dev)) return -EINVAL; /* * Per PCIe r4.0, sec 6.15, endpoints and root ports may be * AtomicOp requesters. For now, we only support endpoints as * requesters and root ports as completers. No endpoints as * completers, and no peer-to-peer. */ switch (pci_pcie_type(dev)) { case PCI_EXP_TYPE_ENDPOINT: case PCI_EXP_TYPE_LEG_END: break; default: return -EINVAL; } bridge = bus->self; pcie_capability_read_dword(bridge, PCI_EXP_DEVCAP2, &cap); switch (pci_pcie_type(bridge)) { case PCI_EXP_TYPE_DOWNSTREAM: if (!(cap & PCI_EXP_DEVCAP2_ATOMIC_ROUTE)) return -EINVAL; break; /* Ensure root port supports all the sizes we care about */ case PCI_EXP_TYPE_ROOT_PORT: if ((cap & cap_mask) != cap_mask) return -EINVAL; break; } return 0; } int bnxt_qplib_enable_atomic_ops_to_root(struct pci_dev *dev) { u16 ctl2; if(pci_enable_atomic_ops_to_root(dev, PCI_EXP_DEVCAP2_ATOMIC_COMP32) && pci_enable_atomic_ops_to_root(dev, PCI_EXP_DEVCAP2_ATOMIC_COMP64)) return -EOPNOTSUPP; pcie_capability_read_word(dev, PCI_EXP_DEVCTL2, &ctl2); return !(ctl2 & PCI_EXP_DEVCTL2_ATOMIC_REQ); } diff --git a/sys/dev/bnxt/bnxt_re/qplib_res.h b/sys/dev/bnxt/bnxt_re/qplib_res.h index 6468207a49aa..914694ee3b8c 100644 --- a/sys/dev/bnxt/bnxt_re/qplib_res.h +++ b/sys/dev/bnxt/bnxt_re/qplib_res.h @@ -1,840 +1,846 @@ /* * Copyright (c) 2015-2024, Broadcom. All rights reserved. The term * Broadcom refers to Broadcom Limited and/or its subsidiaries. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in * the documentation and/or other materials provided with the * distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * * Description: QPLib resource manager (header) */ #ifndef __BNXT_QPLIB_RES_H__ #define __BNXT_QPLIB_RES_H__ #include "hsi_struct_def.h" extern const struct bnxt_qplib_gid bnxt_qplib_gid_zero; #define CHIP_NUM_57508 0x1750 #define CHIP_NUM_57504 0x1751 #define CHIP_NUM_57502 0x1752 #define CHIP_NUM_58818 0xd818 #define CHIP_NUM_57608 0x1760 #define BNXT_QPLIB_MAX_QPC_COUNT (64 * 1024) #define BNXT_QPLIB_MAX_SRQC_COUNT (64 * 1024) #define BNXT_QPLIB_MAX_CQ_COUNT (64 * 1024) #define BNXT_QPLIB_MAX_CQ_COUNT_P5 (128 * 1024) #define BNXT_QPLIB_DBR_VALID (0x1UL << 26) #define BNXT_QPLIB_DBR_EPOCH_SHIFT 24 #define BNXT_QPLIB_DBR_TOGGLE_SHIFT 25 #define BNXT_QPLIB_DBR_PF_DB_OFFSET 0x10000 #define BNXT_QPLIB_DBR_VF_DB_OFFSET 0x4000 #define BNXT_QPLIB_DBR_KEY_INVALID -1 /* chip gen type */ #define BNXT_RE_DEFAULT 0xf enum bnxt_qplib_wqe_mode { BNXT_QPLIB_WQE_MODE_STATIC = 0x00, BNXT_QPLIB_WQE_MODE_VARIABLE = 0x01, BNXT_QPLIB_WQE_MODE_INVALID = 0x02 }; #define BNXT_RE_PUSH_MODE_NONE 0 #define BNXT_RE_PUSH_MODE_WCB 1 #define BNXT_RE_PUSH_MODE_PPP 2 #define BNXT_RE_PUSH_ENABLED(mode) ((mode) == BNXT_RE_PUSH_MODE_WCB ||\ (mode) == BNXT_RE_PUSH_MODE_PPP) #define BNXT_RE_PPP_ENABLED(cctx) ((cctx)->modes.db_push_mode ==\ BNXT_RE_PUSH_MODE_PPP) #define PCI_EXP_DEVCAP2_ATOMIC_ROUTE 0x00000040 /* Atomic Op routing */ #define PCI_EXP_DEVCAP2_ATOMIC_COMP32 0x00000080 /* 32b AtomicOp completion */ #define PCI_EXP_DEVCAP2_ATOMIC_COMP64 0x00000100 /* 64b AtomicOp completion */ #define PCI_EXP_DEVCTL2_ATOMIC_EGRESS_BLOCK 0x0080 /* Block atomic egress */ #define PCI_EXP_DEVCTL2_ATOMIC_REQ 0x0040 /* Set Atomic requests */ int pci_enable_atomic_ops_to_root(struct pci_dev *dev, u32 cap_mask); struct bnxt_qplib_drv_modes { u8 wqe_mode; u8 te_bypass; u8 db_push; /* To control advanced cc params display in configfs */ u8 cc_pr_mode; /* Other modes to follow here e.g. GSI QP mode */ u8 dbr_pacing; u8 dbr_pacing_ext; u8 dbr_drop_recov; u8 dbr_primary_pf; u8 dbr_pacing_v0; }; struct bnxt_qplib_chip_ctx { u16 chip_num; u8 chip_rev; u8 chip_metal; u64 hwrm_intf_ver; struct bnxt_qplib_drv_modes modes; u32 dbr_stat_db_fifo; u32 dbr_aeq_arm_reg; u32 dbr_throttling_reg; u16 hw_stats_size; u16 hwrm_cmd_max_timeout; }; static inline bool _is_chip_num_p7(u16 chip_num) { return (chip_num == CHIP_NUM_58818 || chip_num == CHIP_NUM_57608); } static inline bool _is_chip_p7(struct bnxt_qplib_chip_ctx *cctx) { return _is_chip_num_p7(cctx->chip_num); } /* SR2 is Gen P5 */ static inline bool _is_chip_gen_p5(struct bnxt_qplib_chip_ctx *cctx) { return (cctx->chip_num == CHIP_NUM_57508 || cctx->chip_num == CHIP_NUM_57504 || cctx->chip_num == CHIP_NUM_57502); } static inline bool _is_chip_gen_p5_p7(struct bnxt_qplib_chip_ctx *cctx) { return (_is_chip_gen_p5(cctx) || _is_chip_p7(cctx)); } static inline bool _is_wqe_mode_variable(struct bnxt_qplib_chip_ctx *cctx) { return cctx->modes.wqe_mode == BNXT_QPLIB_WQE_MODE_VARIABLE; } struct bnxt_qplib_db_pacing_data { u32 do_pacing; u32 pacing_th; u32 dev_err_state; u32 alarm_th; u32 grc_reg_offset; u32 fifo_max_depth; u32 fifo_room_mask; u8 fifo_room_shift; }; static inline u8 bnxt_qplib_dbr_pacing_en(struct bnxt_qplib_chip_ctx *cctx) { return cctx->modes.dbr_pacing; } static inline u8 bnxt_qplib_dbr_pacing_ext_en(struct bnxt_qplib_chip_ctx *cctx) { return cctx->modes.dbr_pacing_ext; } static inline u8 bnxt_qplib_dbr_pacing_is_primary_pf(struct bnxt_qplib_chip_ctx *cctx) { return cctx->modes.dbr_primary_pf; } static inline void bnxt_qplib_dbr_pacing_set_primary_pf (struct bnxt_qplib_chip_ctx *cctx, u8 val) { cctx->modes.dbr_primary_pf = val; } /* Defines for handling the HWRM version check */ #define HWRM_VERSION_DEV_ATTR_MAX_DPI 0x1000A0000000D #define HWRM_VERSION_ROCE_STATS_FN_ID 0x1000A00000045 #define PTR_CNT_PER_PG (PAGE_SIZE / sizeof(void *)) #define PTR_MAX_IDX_PER_PG (PTR_CNT_PER_PG - 1) #define PTR_PG(x) (((x) & ~PTR_MAX_IDX_PER_PG) / PTR_CNT_PER_PG) #define PTR_IDX(x) ((x) & PTR_MAX_IDX_PER_PG) #define HWQ_CMP(idx, hwq) ((idx) & ((hwq)->max_elements - 1)) #define HWQ_FREE_SLOTS(hwq) (hwq->max_elements - \ ((HWQ_CMP(hwq->prod, hwq)\ - HWQ_CMP(hwq->cons, hwq))\ & (hwq->max_elements - 1))) enum bnxt_qplib_hwq_type { HWQ_TYPE_CTX, HWQ_TYPE_QUEUE, HWQ_TYPE_L2_CMPL, HWQ_TYPE_MR }; #define MAX_PBL_LVL_0_PGS 1 #define MAX_PBL_LVL_1_PGS 512 #define MAX_PBL_LVL_1_PGS_SHIFT 9 #define MAX_PDL_LVL_SHIFT 9 enum bnxt_qplib_pbl_lvl { PBL_LVL_0, PBL_LVL_1, PBL_LVL_2, PBL_LVL_MAX }; #define ROCE_PG_SIZE_4K (4 * 1024) #define ROCE_PG_SIZE_8K (8 * 1024) #define ROCE_PG_SIZE_64K (64 * 1024) #define ROCE_PG_SIZE_2M (2 * 1024 * 1024) #define ROCE_PG_SIZE_8M (8 * 1024 * 1024) #define ROCE_PG_SIZE_1G (1024 * 1024 * 1024) enum bnxt_qplib_hwrm_pg_size { BNXT_QPLIB_HWRM_PG_SIZE_4K = 0, BNXT_QPLIB_HWRM_PG_SIZE_8K = 1, BNXT_QPLIB_HWRM_PG_SIZE_64K = 2, BNXT_QPLIB_HWRM_PG_SIZE_2M = 3, BNXT_QPLIB_HWRM_PG_SIZE_8M = 4, BNXT_QPLIB_HWRM_PG_SIZE_1G = 5, }; struct bnxt_qplib_reg_desc { u8 bar_id; resource_size_t bar_base; unsigned long offset; void __iomem *bar_reg; size_t len; }; struct bnxt_qplib_pbl { u32 pg_count; u32 pg_size; void **pg_arr; dma_addr_t *pg_map_arr; }; struct bnxt_qplib_sg_info { struct scatterlist *sghead; u32 nmap; u32 npages; u32 pgshft; u32 pgsize; bool nopte; }; struct bnxt_qplib_hwq_attr { struct bnxt_qplib_res *res; struct bnxt_qplib_sg_info *sginfo; enum bnxt_qplib_hwq_type type; u32 depth; u32 stride; u32 aux_stride; u32 aux_depth; }; struct bnxt_qplib_hwq { struct pci_dev *pdev; spinlock_t lock; struct bnxt_qplib_pbl pbl[PBL_LVL_MAX]; enum bnxt_qplib_pbl_lvl level; /* 0, 1, or 2 */ void **pbl_ptr; /* ptr for easy access to the PBL entries */ dma_addr_t *pbl_dma_ptr; /* ptr for easy access to the dma_addr */ u32 max_elements; u32 depth; /* original requested depth */ u16 element_size; /* Size of each entry */ u16 qe_ppg; /* queue entry per page */ u32 prod; /* raw */ u32 cons; /* raw */ u8 cp_bit; u8 is_user; u64 *pad_pg; u32 pad_stride; u32 pad_pgofft; }; struct bnxt_qplib_db_info { void __iomem *db; void __iomem *priv_db; struct bnxt_qplib_hwq *hwq; struct bnxt_qplib_res *res; u32 xid; u32 max_slot; u32 flags; u8 toggle; spinlock_t lock; u64 shadow_key; u64 shadow_key_arm_ena; u32 seed; /* For DB pacing */ }; enum bnxt_qplib_db_info_flags_mask { BNXT_QPLIB_FLAG_EPOCH_CONS_SHIFT = 0x0UL, BNXT_QPLIB_FLAG_EPOCH_PROD_SHIFT = 0x1UL, BNXT_QPLIB_FLAG_EPOCH_CONS_MASK = 0x1UL, BNXT_QPLIB_FLAG_EPOCH_PROD_MASK = 0x2UL, }; enum bnxt_qplib_db_epoch_flag_shift { BNXT_QPLIB_DB_EPOCH_CONS_SHIFT = BNXT_QPLIB_DBR_EPOCH_SHIFT, BNXT_QPLIB_DB_EPOCH_PROD_SHIFT = (BNXT_QPLIB_DBR_EPOCH_SHIFT - 1) }; /* Tables */ struct bnxt_qplib_pd_tbl { unsigned long *tbl; u32 max; }; struct bnxt_qplib_sgid_tbl { struct bnxt_qplib_gid_info *tbl; u16 *hw_id; u16 max; u16 active; void *ctx; bool *vlan; }; enum { BNXT_QPLIB_DPI_TYPE_KERNEL = 0, BNXT_QPLIB_DPI_TYPE_UC = 1, BNXT_QPLIB_DPI_TYPE_WC = 2 }; struct bnxt_qplib_dpi { u32 dpi; u32 bit; void __iomem *dbr; u64 umdbr; u8 type; }; #define BNXT_QPLIB_MAX_EXTENDED_PPP_PAGES 512 struct bnxt_qplib_dpi_tbl { void **app_tbl; unsigned long *tbl; u16 max; u16 avail_ppp; struct bnxt_qplib_reg_desc ucreg; /* Hold entire DB bar. */ struct bnxt_qplib_reg_desc wcreg; void __iomem *priv_db; }; struct bnxt_qplib_stats { dma_addr_t dma_map; void *dma; u32 size; u32 fw_id; }; struct bnxt_qplib_vf_res { u32 max_qp; u32 max_mrw; u32 max_srq; u32 max_cq; u32 max_gid; }; #define BNXT_QPLIB_MAX_QP_CTX_ENTRY_SIZE 448 #define BNXT_QPLIB_MAX_SRQ_CTX_ENTRY_SIZE 64 #define BNXT_QPLIB_MAX_CQ_CTX_ENTRY_SIZE 64 #define BNXT_QPLIB_MAX_MRW_CTX_ENTRY_SIZE 128 #define MAX_TQM_ALLOC_REQ 48 #define MAX_TQM_ALLOC_BLK_SIZE 8 struct bnxt_qplib_tqm_ctx { struct bnxt_qplib_hwq pde; enum bnxt_qplib_pbl_lvl pde_level; /* Original level */ struct bnxt_qplib_hwq qtbl[MAX_TQM_ALLOC_REQ]; u8 qcount[MAX_TQM_ALLOC_REQ]; }; struct bnxt_qplib_hctx { struct bnxt_qplib_hwq hwq; u32 max; }; struct bnxt_qplib_refrec { void *handle; u32 xid; }; struct bnxt_qplib_reftbl { struct bnxt_qplib_refrec *rec; u32 max; spinlock_t lock; /* reftbl lock */ }; struct bnxt_qplib_reftbls { struct bnxt_qplib_reftbl qpref; struct bnxt_qplib_reftbl cqref; struct bnxt_qplib_reftbl srqref; }; #define GET_TBL_INDEX(id, tbl) ((id) % (((tbl)->max) - 1)) static inline u32 map_qp_id_to_tbl_indx(u32 qid, struct bnxt_qplib_reftbl *tbl) { return (qid == 1) ? tbl->max : GET_TBL_INDEX(qid, tbl); } /* * This structure includes the number of various roce resource table sizes * actually allocated by the driver. May be less than the maximums the firmware * allows if the driver imposes lower limits than the firmware. */ struct bnxt_qplib_ctx { struct bnxt_qplib_hctx qp_ctx; struct bnxt_qplib_hctx mrw_ctx; struct bnxt_qplib_hctx srq_ctx; struct bnxt_qplib_hctx cq_ctx; struct bnxt_qplib_hctx tim_ctx; struct bnxt_qplib_tqm_ctx tqm_ctx; struct bnxt_qplib_stats stats; struct bnxt_qplib_stats stats2; struct bnxt_qplib_vf_res vf_res; }; struct bnxt_qplib_res { struct pci_dev *pdev; struct bnxt_qplib_chip_ctx *cctx; struct bnxt_qplib_dev_attr *dattr; struct bnxt_qplib_ctx *hctx; struct ifnet *netdev; struct bnxt_en_dev *en_dev; struct bnxt_qplib_rcfw *rcfw; struct bnxt_qplib_pd_tbl pd_tbl; struct mutex pd_tbl_lock; struct bnxt_qplib_sgid_tbl sgid_tbl; struct bnxt_qplib_dpi_tbl dpi_tbl; struct mutex dpi_tbl_lock; struct bnxt_qplib_reftbls reftbl; bool prio; bool is_vf; struct bnxt_qplib_db_pacing_data *pacing_data; }; struct bnxt_qplib_query_stats_info { u32 function_id; u8 collection_id; bool vf_valid; }; struct bnxt_qplib_query_qp_info { u32 function_id; u32 num_qps; u32 start_index; bool vf_valid; }; struct bnxt_qplib_query_fn_info { bool vf_valid; u32 host; u32 filter; }; #define to_bnxt_qplib(ptr, type, member) \ container_of(ptr, type, member) struct bnxt_qplib_pd; struct bnxt_qplib_dev_attr; bool _is_chip_gen_p5(struct bnxt_qplib_chip_ctx *cctx); bool _is_chip_gen_p5_p7(struct bnxt_qplib_chip_ctx *cctx); bool _is_chip_a0(struct bnxt_qplib_chip_ctx *cctx); bool _is_chip_p7(struct bnxt_qplib_chip_ctx *cctx); bool _is_alloc_mr_unified(struct bnxt_qplib_dev_attr *dattr); void bnxt_qplib_free_hwq(struct bnxt_qplib_res *res, struct bnxt_qplib_hwq *hwq); int bnxt_qplib_alloc_init_hwq(struct bnxt_qplib_hwq *hwq, struct bnxt_qplib_hwq_attr *hwq_attr); void bnxt_qplib_get_guid(const u8 *dev_addr, u8 *guid); int bnxt_qplib_alloc_pd(struct bnxt_qplib_res *res, struct bnxt_qplib_pd *pd); int bnxt_qplib_dealloc_pd(struct bnxt_qplib_res *res, struct bnxt_qplib_pd_tbl *pd_tbl, struct bnxt_qplib_pd *pd); int bnxt_qplib_alloc_dpi(struct bnxt_qplib_res *res, struct bnxt_qplib_dpi *dpi, void *app, u8 type); int bnxt_qplib_dealloc_dpi(struct bnxt_qplib_res *res, struct bnxt_qplib_dpi *dpi); int bnxt_qplib_stop_res(struct bnxt_qplib_res *res); void bnxt_qplib_clear_tbls(struct bnxt_qplib_res *res); int bnxt_qplib_init_tbls(struct bnxt_qplib_res *res); void bnxt_qplib_free_tbls(struct bnxt_qplib_res *res); int bnxt_qplib_alloc_tbls(struct bnxt_qplib_res *res, u8 pppp_factor); void bnxt_qplib_free_hwctx(struct bnxt_qplib_res *res); int bnxt_qplib_alloc_hwctx(struct bnxt_qplib_res *res); int bnxt_qplib_alloc_stat_mem(struct pci_dev *pdev, struct bnxt_qplib_chip_ctx *cctx, struct bnxt_qplib_stats *stats); void bnxt_qplib_free_stat_mem(struct bnxt_qplib_res *res, struct bnxt_qplib_stats *stats); int bnxt_qplib_map_db_bar(struct bnxt_qplib_res *res); void bnxt_qplib_unmap_db_bar(struct bnxt_qplib_res *res); int bnxt_qplib_enable_atomic_ops_to_root(struct pci_dev *dev); u8 _get_chip_gen_p5_type(struct bnxt_qplib_chip_ctx *cctx); static inline void *bnxt_qplib_get_qe(struct bnxt_qplib_hwq *hwq, u32 indx, u64 *pg) { u32 pg_num, pg_idx; pg_num = (indx / hwq->qe_ppg); pg_idx = (indx % hwq->qe_ppg); if (pg) *pg = (u64)&hwq->pbl_ptr[pg_num]; return (void *)((u8 *)hwq->pbl_ptr[pg_num] + hwq->element_size * pg_idx); } static inline void bnxt_qplib_hwq_incr_prod(struct bnxt_qplib_db_info *dbinfo, struct bnxt_qplib_hwq *hwq, u32 cnt) { /* move prod and update toggle/epoch if wrap around */ hwq->prod += cnt; if (hwq->prod >= hwq->depth) { hwq->prod %= hwq->depth; dbinfo->flags ^= 1UL << BNXT_QPLIB_FLAG_EPOCH_PROD_SHIFT; } } static inline void bnxt_qplib_hwq_incr_cons(u32 max_elements, u32 *cons, u32 cnt, u32 *dbinfo_flags) { /* move cons and update toggle/epoch if wrap around */ *cons += cnt; if (*cons >= max_elements) { *cons %= max_elements; *dbinfo_flags ^= 1UL << BNXT_QPLIB_FLAG_EPOCH_CONS_SHIFT; } } static inline u8 _get_pte_pg_size(struct bnxt_qplib_hwq *hwq) { u8 pg_size = BNXT_QPLIB_HWRM_PG_SIZE_4K; struct bnxt_qplib_pbl *pbl; pbl = &hwq->pbl[hwq->level]; switch (pbl->pg_size) { case ROCE_PG_SIZE_4K: pg_size = BNXT_QPLIB_HWRM_PG_SIZE_4K; break; case ROCE_PG_SIZE_8K: pg_size = BNXT_QPLIB_HWRM_PG_SIZE_8K; break; case ROCE_PG_SIZE_64K: pg_size = BNXT_QPLIB_HWRM_PG_SIZE_64K; break; case ROCE_PG_SIZE_2M: pg_size = BNXT_QPLIB_HWRM_PG_SIZE_2M; break; case ROCE_PG_SIZE_8M: pg_size = BNXT_QPLIB_HWRM_PG_SIZE_8M; break; case ROCE_PG_SIZE_1G: pg_size = BNXT_QPLIB_HWRM_PG_SIZE_1G; break; default: break; } return pg_size; } static inline u64 _get_base_addr(struct bnxt_qplib_hwq *hwq) { return hwq->pbl[PBL_LVL_0].pg_map_arr[0]; } static inline u8 _get_base_pg_size(struct bnxt_qplib_hwq *hwq) { u8 pg_size = BNXT_QPLIB_HWRM_PG_SIZE_4K; struct bnxt_qplib_pbl *pbl; pbl = &hwq->pbl[PBL_LVL_0]; switch (pbl->pg_size) { case ROCE_PG_SIZE_4K: pg_size = BNXT_QPLIB_HWRM_PG_SIZE_4K; break; case ROCE_PG_SIZE_8K: pg_size = BNXT_QPLIB_HWRM_PG_SIZE_8K; break; case ROCE_PG_SIZE_64K: pg_size = BNXT_QPLIB_HWRM_PG_SIZE_64K; break; case ROCE_PG_SIZE_2M: pg_size = BNXT_QPLIB_HWRM_PG_SIZE_2M; break; case ROCE_PG_SIZE_8M: pg_size = BNXT_QPLIB_HWRM_PG_SIZE_8M; break; case ROCE_PG_SIZE_1G: pg_size = BNXT_QPLIB_HWRM_PG_SIZE_1G; break; default: break; } return pg_size; } static inline enum bnxt_qplib_hwq_type _get_hwq_type(struct bnxt_qplib_res *res) { return _is_chip_gen_p5_p7(res->cctx) ? HWQ_TYPE_QUEUE : HWQ_TYPE_L2_CMPL; } static inline bool _is_ext_stats_supported(u16 dev_cap_flags) { return dev_cap_flags & CREQ_QUERY_FUNC_RESP_SB_EXT_STATS; } static inline int bnxt_ext_stats_supported(struct bnxt_qplib_chip_ctx *ctx, u16 flags, bool virtfn) { return (_is_ext_stats_supported(flags) && ((virtfn && _is_chip_p7(ctx)) || (!virtfn))); } static inline bool _is_hw_retx_supported(u16 dev_cap_flags) { return dev_cap_flags & (CREQ_QUERY_FUNC_RESP_SB_HW_REQUESTER_RETX_ENABLED | CREQ_QUERY_FUNC_RESP_SB_HW_RESPONDER_RETX_ENABLED); } /* Disable HW_RETX */ #define BNXT_RE_HW_RETX(a) _is_hw_retx_supported((a)) +static inline bool _is_host_msn_table(u16 dev_cap_ext_flags2) +{ + return (dev_cap_ext_flags2 & CREQ_QUERY_FUNC_RESP_SB_REQ_RETRANSMISSION_SUPPORT_MASK) == + CREQ_QUERY_FUNC_RESP_SB_REQ_RETRANSMISSION_SUPPORT_HOST_MSN_TABLE; +} + static inline bool _is_cqe_v2_supported(u16 dev_cap_flags) { return dev_cap_flags & CREQ_QUERY_FUNC_RESP_SB_CQE_V2; } #define BNXT_DB_FIFO_ROOM_MASK 0x1fff8000 #define BNXT_DB_FIFO_ROOM_SHIFT 15 #define BNXT_MAX_FIFO_DEPTH 0x2c00 #define BNXT_DB_PACING_ALGO_THRESHOLD 250 #define BNXT_DEFAULT_PACING_PROBABILITY 0xFFFF #define BNXT_DBR_PACING_WIN_BASE 0x2000 #define BNXT_DBR_PACING_WIN_MAP_OFF 4 #define BNXT_DBR_PACING_WIN_OFF(reg) (BNXT_DBR_PACING_WIN_BASE + \ static inline void bnxt_qplib_ring_db32(struct bnxt_qplib_db_info *info, bool arm) { u32 key = 0; key = info->hwq->cons | (CMPL_DOORBELL_IDX_VALID | (CMPL_DOORBELL_KEY_CMPL & CMPL_DOORBELL_KEY_MASK)); if (!arm) key |= CMPL_DOORBELL_MASK; /* memory barrier */ wmb(); writel(key, info->db); } #define BNXT_QPLIB_INIT_DBHDR(xid, type, indx, toggle) \ (((u64)(((xid) & DBC_DBC_XID_MASK) | DBC_DBC_PATH_ROCE | \ (type) | BNXT_QPLIB_DBR_VALID) << 32) | (indx) | \ - ((toggle) << (BNXT_QPLIB_DBR_TOGGLE_SHIFT))) + (((u32)(toggle)) << (BNXT_QPLIB_DBR_TOGGLE_SHIFT))) static inline void bnxt_qplib_write_db(struct bnxt_qplib_db_info *info, u64 key, void __iomem *db, u64 *shadow_key) { unsigned long flags; spin_lock_irqsave(&info->lock, flags); *shadow_key = key; writeq(key, db); spin_unlock_irqrestore(&info->lock, flags); } static inline void __replay_writeq(u64 key, void __iomem *db) { /* No need to replay uninitialised shadow_keys */ if (key != BNXT_QPLIB_DBR_KEY_INVALID) writeq(key, db); } static inline void bnxt_qplib_replay_db(struct bnxt_qplib_db_info *info, bool is_arm_ena) { if (!spin_trylock_irq(&info->lock)) return; if (is_arm_ena) __replay_writeq(info->shadow_key_arm_ena, info->priv_db); else __replay_writeq(info->shadow_key, info->db); spin_unlock_irq(&info->lock); } static inline void bnxt_qplib_ring_db(struct bnxt_qplib_db_info *info, u32 type) { u64 key = 0; u32 indx; u8 toggle = 0; if (type == DBC_DBC_TYPE_CQ_ARMALL || type == DBC_DBC_TYPE_CQ_ARMSE) toggle = info->toggle; indx = ((info->hwq->cons & DBC_DBC_INDEX_MASK) | ((info->flags & BNXT_QPLIB_FLAG_EPOCH_CONS_MASK) << BNXT_QPLIB_DB_EPOCH_CONS_SHIFT)); key = BNXT_QPLIB_INIT_DBHDR(info->xid, type, indx, toggle); bnxt_qplib_write_db(info, key, info->db, &info->shadow_key); } static inline void bnxt_qplib_ring_prod_db(struct bnxt_qplib_db_info *info, u32 type) { u64 key = 0; u32 indx; indx = (((info->hwq->prod / info->max_slot) & DBC_DBC_INDEX_MASK) | ((info->flags & BNXT_QPLIB_FLAG_EPOCH_PROD_MASK) << BNXT_QPLIB_DB_EPOCH_PROD_SHIFT)); key = BNXT_QPLIB_INIT_DBHDR(info->xid, type, indx, 0); bnxt_qplib_write_db(info, key, info->db, &info->shadow_key); } static inline void bnxt_qplib_armen_db(struct bnxt_qplib_db_info *info, u32 type) { u64 key = 0; u8 toggle = 0; - if (type == DBC_DBC_TYPE_CQ_ARMENA) + if (type == DBC_DBC_TYPE_CQ_ARMENA || type == DBC_DBC_TYPE_SRQ_ARMENA) toggle = info->toggle; /* Index always at 0 */ key = BNXT_QPLIB_INIT_DBHDR(info->xid, type, 0, toggle); bnxt_qplib_write_db(info, key, info->priv_db, &info->shadow_key_arm_ena); } static inline void bnxt_qplib_cq_coffack_db(struct bnxt_qplib_db_info *info) { u64 key = 0; /* Index always at 0 */ key = BNXT_QPLIB_INIT_DBHDR(info->xid, DBC_DBC_TYPE_CQ_CUTOFF_ACK, 0, 0); bnxt_qplib_write_db(info, key, info->priv_db, &info->shadow_key); } static inline void bnxt_qplib_srq_arm_db(struct bnxt_qplib_db_info *info) { u64 key = 0; /* Index always at 0 */ - key = BNXT_QPLIB_INIT_DBHDR(info->xid, DBC_DBC_TYPE_SRQ_ARM, 0, 0); + key = BNXT_QPLIB_INIT_DBHDR(info->xid, DBC_DBC_TYPE_SRQ_ARM, 0, info->toggle); bnxt_qplib_write_db(info, key, info->priv_db, &info->shadow_key); } static inline void bnxt_qplib_ring_nq_db(struct bnxt_qplib_db_info *info, struct bnxt_qplib_chip_ctx *cctx, bool arm) { u32 type; type = arm ? DBC_DBC_TYPE_NQ_ARM : DBC_DBC_TYPE_NQ; if (_is_chip_gen_p5_p7(cctx)) bnxt_qplib_ring_db(info, type); else bnxt_qplib_ring_db32(info, arm); } struct bnxt_qplib_max_res { u32 max_qp; u32 max_mr; u32 max_cq; u32 max_srq; u32 max_ah; u32 max_pd; }; /* * Defines for maximum resources supported for chip revisions * Maximum PDs supported are restricted to Max QPs * GENP4 - Wh+ * DEFAULT - Thor */ #define BNXT_QPLIB_GENP4_PF_MAX_QP (16 * 1024) #define BNXT_QPLIB_GENP4_PF_MAX_MRW (16 * 1024) #define BNXT_QPLIB_GENP4_PF_MAX_CQ (16 * 1024) #define BNXT_QPLIB_GENP4_PF_MAX_SRQ (1 * 1024) #define BNXT_QPLIB_GENP4_PF_MAX_AH (16 * 1024) #define BNXT_QPLIB_GENP4_PF_MAX_PD BNXT_QPLIB_GENP4_PF_MAX_QP #define BNXT_QPLIB_DEFAULT_PF_MAX_QP (64 * 1024) #define BNXT_QPLIB_DEFAULT_PF_MAX_MRW (256 * 1024) #define BNXT_QPLIB_DEFAULT_PF_MAX_CQ (64 * 1024) #define BNXT_QPLIB_DEFAULT_PF_MAX_SRQ (4 * 1024) #define BNXT_QPLIB_DEFAULT_PF_MAX_AH (64 * 1024) #define BNXT_QPLIB_DEFAULT_PF_MAX_PD BNXT_QPLIB_DEFAULT_PF_MAX_QP #define BNXT_QPLIB_DEFAULT_VF_MAX_QP (6 * 1024) #define BNXT_QPLIB_DEFAULT_VF_MAX_MRW (6 * 1024) #define BNXT_QPLIB_DEFAULT_VF_MAX_CQ (6 * 1024) #define BNXT_QPLIB_DEFAULT_VF_MAX_SRQ (4 * 1024) #define BNXT_QPLIB_DEFAULT_VF_MAX_AH (6 * 1024) #define BNXT_QPLIB_DEFAULT_VF_MAX_PD BNXT_QPLIB_DEFAULT_VF_MAX_QP static inline void bnxt_qplib_max_res_supported(struct bnxt_qplib_chip_ctx *cctx, struct bnxt_qplib_res *qpl_res, struct bnxt_qplib_max_res *max_res, bool vf_res_limit) { switch (cctx->chip_num) { case CHIP_NUM_57608: case CHIP_NUM_58818: case CHIP_NUM_57504: case CHIP_NUM_57502: case CHIP_NUM_57508: if (!qpl_res->is_vf) { max_res->max_qp = BNXT_QPLIB_DEFAULT_PF_MAX_QP; max_res->max_mr = BNXT_QPLIB_DEFAULT_PF_MAX_MRW; max_res->max_cq = BNXT_QPLIB_DEFAULT_PF_MAX_CQ; max_res->max_srq = BNXT_QPLIB_DEFAULT_PF_MAX_SRQ; max_res->max_ah = BNXT_QPLIB_DEFAULT_PF_MAX_AH; max_res->max_pd = BNXT_QPLIB_DEFAULT_PF_MAX_PD; } else { max_res->max_qp = BNXT_QPLIB_DEFAULT_VF_MAX_QP; max_res->max_mr = BNXT_QPLIB_DEFAULT_VF_MAX_MRW; max_res->max_cq = BNXT_QPLIB_DEFAULT_VF_MAX_CQ; max_res->max_srq = BNXT_QPLIB_DEFAULT_VF_MAX_SRQ; max_res->max_ah = BNXT_QPLIB_DEFAULT_VF_MAX_AH; max_res->max_pd = BNXT_QPLIB_DEFAULT_VF_MAX_PD; } break; default: /* Wh+/Stratus max resources */ max_res->max_qp = BNXT_QPLIB_GENP4_PF_MAX_QP; max_res->max_mr = BNXT_QPLIB_GENP4_PF_MAX_MRW; max_res->max_cq = BNXT_QPLIB_GENP4_PF_MAX_CQ; max_res->max_srq = BNXT_QPLIB_GENP4_PF_MAX_SRQ; max_res->max_ah = BNXT_QPLIB_GENP4_PF_MAX_AH; max_res->max_pd = BNXT_QPLIB_GENP4_PF_MAX_PD; break; } } #endif diff --git a/sys/dev/bnxt/bnxt_re/qplib_sp.c b/sys/dev/bnxt/bnxt_re/qplib_sp.c index c414718a816f..31a0878e421b 100644 --- a/sys/dev/bnxt/bnxt_re/qplib_sp.c +++ b/sys/dev/bnxt/bnxt_re/qplib_sp.c @@ -1,1234 +1,1235 @@ /* * Copyright (c) 2015-2024, Broadcom. All rights reserved. The term * Broadcom refers to Broadcom Limited and/or its subsidiaries. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in * the documentation and/or other materials provided with the * distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * * Description: Slow Path Operators */ #include #include #include #include #include #include #include "hsi_struct_def.h" #include "qplib_tlv.h" #include "qplib_res.h" #include "qplib_rcfw.h" #include "qplib_sp.h" const struct bnxt_qplib_gid bnxt_qplib_gid_zero = {{ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }}; /* Device */ static u8 bnxt_qplib_is_atomic_cap(struct bnxt_qplib_rcfw *rcfw) { u16 pcie_ctl2 = 0; if (!_is_chip_gen_p5_p7(rcfw->res->cctx)) return false; pcie_capability_read_word(rcfw->pdev, PCI_EXP_DEVCTL2, &pcie_ctl2); return (pcie_ctl2 & PCI_EXP_DEVCTL2_ATOMIC_REQ); } static void bnxt_qplib_query_version(struct bnxt_qplib_rcfw *rcfw, char *fw_ver) { struct creq_query_version_resp resp = {}; struct bnxt_qplib_cmdqmsg msg = {}; struct cmdq_query_version req = {}; int rc = 0; bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_QUERY_VERSION, sizeof(req)); bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL, sizeof(req), sizeof(resp), 0); rc = bnxt_qplib_rcfw_send_message(rcfw, &msg); if (rc) { dev_err(&rcfw->pdev->dev, "QPLIB: Failed to query version\n"); return; } fw_ver[0] = resp.fw_maj; fw_ver[1] = resp.fw_minor; fw_ver[2] = resp.fw_bld; fw_ver[3] = resp.fw_rsvd; } int bnxt_qplib_get_dev_attr(struct bnxt_qplib_rcfw *rcfw) { struct creq_query_func_resp resp = {}; struct bnxt_qplib_cmdqmsg msg = {}; struct creq_query_func_resp_sb *sb; struct bnxt_qplib_rcfw_sbuf sbuf; struct bnxt_qplib_dev_attr *attr; struct bnxt_qplib_chip_ctx *cctx; struct cmdq_query_func req = {}; u8 *tqm_alloc; int i, rc = 0; u32 temp; u8 chip_gen = BNXT_RE_DEFAULT; cctx = rcfw->res->cctx; attr = rcfw->res->dattr; bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_QUERY_FUNC, sizeof(req)); sbuf.size = sizeof(*sb); sbuf.sb = dma_zalloc_coherent(&rcfw->pdev->dev, sbuf.size, &sbuf.dma_addr, GFP_KERNEL); if (!sbuf.sb) return -ENOMEM; sb = sbuf.sb; req.resp_size = sbuf.size / BNXT_QPLIB_CMDQE_UNITS; bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, &sbuf, sizeof(req), sizeof(resp), 0); rc = bnxt_qplib_rcfw_send_message(rcfw, &msg); if (rc) goto bail; /* Extract the context from the side buffer */ chip_gen = _get_chip_gen_p5_type(cctx); attr->max_qp = le32_to_cpu(sb->max_qp); attr->max_qp = min_t(u32, attr->max_qp, BNXT_RE_MAX_QP_SUPPORTED(chip_gen)); /* max_qp value reported by FW does not include the QP1 */ attr->max_qp += 1; attr->max_qp_rd_atom = sb->max_qp_rd_atom > BNXT_QPLIB_MAX_OUT_RD_ATOM ? BNXT_QPLIB_MAX_OUT_RD_ATOM : sb->max_qp_rd_atom; attr->max_qp_init_rd_atom = sb->max_qp_init_rd_atom > BNXT_QPLIB_MAX_OUT_RD_ATOM ? BNXT_QPLIB_MAX_OUT_RD_ATOM : sb->max_qp_init_rd_atom; /* Report 1 less than the max_qp_wqes reported by FW as driver adds * one extra entry while creating the qp */ attr->max_qp_wqes = le16_to_cpu(sb->max_qp_wr) - 1; /* Adjust for max_qp_wqes for variable wqe */ if (cctx->modes.wqe_mode == BNXT_QPLIB_WQE_MODE_VARIABLE) { attr->max_qp_wqes = (BNXT_MAX_SQ_SIZE) / (BNXT_MAX_VAR_WQE_SIZE / BNXT_SGE_SIZE) - 1; } if (!_is_chip_gen_p5_p7(cctx)) { /* * 128 WQEs needs to be reserved for the HW (8916). Prevent * reporting the max number for gen-p4 only. */ attr->max_qp_wqes -= BNXT_QPLIB_RESERVED_QP_WRS; } attr->max_qp_sges = sb->max_sge; if (_is_chip_gen_p5_p7(cctx) && cctx->modes.wqe_mode == BNXT_QPLIB_WQE_MODE_VARIABLE) attr->max_qp_sges = sb->max_sge_var_wqe; attr->max_cq = le32_to_cpu(sb->max_cq); attr->max_cq = min_t(u32, attr->max_cq, BNXT_RE_MAX_CQ_SUPPORTED(chip_gen)); attr->max_cq_wqes = le32_to_cpu(sb->max_cqe); attr->max_cq_wqes = min_t(u32, BNXT_QPLIB_MAX_CQ_WQES, attr->max_cq_wqes); attr->max_cq_sges = attr->max_qp_sges; attr->max_mr = le32_to_cpu(sb->max_mr); attr->max_mr = min_t(u32, attr->max_mr, BNXT_RE_MAX_MRW_SUPPORTED(chip_gen)); attr->max_mw = le32_to_cpu(sb->max_mw); attr->max_mw = min_t(u32, attr->max_mw, BNXT_RE_MAX_MRW_SUPPORTED(chip_gen)); attr->max_mr_size = le64_to_cpu(sb->max_mr_size); attr->max_pd = BNXT_QPLIB_MAX_PD; attr->max_raw_ethy_qp = le32_to_cpu(sb->max_raw_eth_qp); attr->max_ah = le32_to_cpu(sb->max_ah); attr->max_ah = min_t(u32, attr->max_ah, BNXT_RE_MAX_AH_SUPPORTED(chip_gen)); attr->max_fmr = le32_to_cpu(sb->max_fmr); attr->max_map_per_fmr = sb->max_map_per_fmr; attr->max_srq = le16_to_cpu(sb->max_srq); attr->max_srq = min_t(u32, attr->max_srq, BNXT_RE_MAX_SRQ_SUPPORTED(chip_gen)); attr->max_srq_wqes = le32_to_cpu(sb->max_srq_wr) - 1; attr->max_srq_sges = sb->max_srq_sge; attr->max_pkey = 1; attr->max_inline_data = !cctx->modes.wqe_mode ? le32_to_cpu(sb->max_inline_data) : le16_to_cpu(sb->max_inline_data_var_wqe); if (!_is_chip_p7(cctx)) { attr->l2_db_size = (sb->l2_db_space_size + 1) * (0x01 << RCFW_DBR_BASE_PAGE_SHIFT); } attr->max_sgid = le32_to_cpu(sb->max_gid); /* TODO: remove this hack for statically allocated gid_map */ bnxt_re_set_max_gid(&attr->max_sgid); attr->dev_cap_flags = le16_to_cpu(sb->dev_cap_flags); attr->page_size_cap = BIT_ULL(28) | BIT_ULL(21) | BIT_ULL(12); bnxt_qplib_query_version(rcfw, attr->fw_ver); + attr->dev_cap_ext_flags2 = le16_to_cpu(sb->dev_cap_ext_flags_2); for (i = 0; i < MAX_TQM_ALLOC_REQ / 4; i++) { temp = le32_to_cpu(sb->tqm_alloc_reqs[i]); tqm_alloc = (u8 *)&temp; attr->tqm_alloc_reqs[i * 4] = *tqm_alloc; attr->tqm_alloc_reqs[i * 4 + 1] = *(++tqm_alloc); attr->tqm_alloc_reqs[i * 4 + 2] = *(++tqm_alloc); attr->tqm_alloc_reqs[i * 4 + 3] = *(++tqm_alloc); } if (rcfw->res->cctx->hwrm_intf_ver >= HWRM_VERSION_DEV_ATTR_MAX_DPI) attr->max_dpi = le32_to_cpu(sb->max_dpi); attr->is_atomic = bnxt_qplib_is_atomic_cap(rcfw); bail: dma_free_coherent(&rcfw->pdev->dev, sbuf.size, sbuf.sb, sbuf.dma_addr); return rc; } int bnxt_qplib_set_func_resources(struct bnxt_qplib_res *res) { struct creq_set_func_resources_resp resp = {}; struct cmdq_set_func_resources req = {}; struct bnxt_qplib_cmdqmsg msg = {}; struct bnxt_qplib_rcfw *rcfw; struct bnxt_qplib_ctx *hctx; int rc = 0; rcfw = res->rcfw; hctx = res->hctx; bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_SET_FUNC_RESOURCES, sizeof(req)); req.number_of_qp = cpu_to_le32(hctx->qp_ctx.max); req.number_of_mrw = cpu_to_le32(hctx->mrw_ctx.max); req.number_of_srq = cpu_to_le32(hctx->srq_ctx.max); req.number_of_cq = cpu_to_le32(hctx->cq_ctx.max); req.max_qp_per_vf = cpu_to_le32(hctx->vf_res.max_qp); req.max_mrw_per_vf = cpu_to_le32(hctx->vf_res.max_mrw); req.max_srq_per_vf = cpu_to_le32(hctx->vf_res.max_srq); req.max_cq_per_vf = cpu_to_le32(hctx->vf_res.max_cq); req.max_gid_per_vf = cpu_to_le32(hctx->vf_res.max_gid); /* Keep the old stats context id of PF */ req.stat_ctx_id = cpu_to_le32(hctx->stats.fw_id); bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL, sizeof(req), sizeof(resp), 0); rc = bnxt_qplib_rcfw_send_message(rcfw, &msg); if (rc) dev_err(&res->pdev->dev, "QPLIB: Failed to set function resources\n"); return rc; } int bnxt_qplib_update_sgid(struct bnxt_qplib_sgid_tbl *sgid_tbl, struct bnxt_qplib_gid *gid, u16 gid_idx, const u8 *smac) { struct bnxt_qplib_res *res = to_bnxt_qplib(sgid_tbl, struct bnxt_qplib_res, sgid_tbl); struct bnxt_qplib_rcfw *rcfw = res->rcfw; struct creq_modify_gid_resp resp = {}; struct bnxt_qplib_cmdqmsg msg = {}; struct cmdq_modify_gid req = {}; int rc; bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_MODIFY_GID, sizeof(req)); req.gid[0] = cpu_to_be32(((u32 *)gid->data)[3]); req.gid[1] = cpu_to_be32(((u32 *)gid->data)[2]); req.gid[2] = cpu_to_be32(((u32 *)gid->data)[1]); req.gid[3] = cpu_to_be32(((u32 *)gid->data)[0]); if (res->prio) { req.vlan |= cpu_to_le16(CMDQ_ADD_GID_VLAN_TPID_TPID_8100 | CMDQ_ADD_GID_VLAN_VLAN_EN); } /* MAC in network format */ req.src_mac[0] = cpu_to_be16(((u16 *)smac)[0]); req.src_mac[1] = cpu_to_be16(((u16 *)smac)[1]); req.src_mac[2] = cpu_to_be16(((u16 *)smac)[2]); req.gid_index = cpu_to_le16(gid_idx); bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL, sizeof(req), sizeof(resp), 0); rc = bnxt_qplib_rcfw_send_message(rcfw, &msg); if (rc) { dev_err(&res->pdev->dev, "QPLIB: update SGID table failed\n"); return rc; } return 0; } /* SGID */ int bnxt_qplib_get_sgid(struct bnxt_qplib_res *res, struct bnxt_qplib_sgid_tbl *sgid_tbl, int index, struct bnxt_qplib_gid *gid) { if (index > sgid_tbl->max) { dev_err(&res->pdev->dev, "QPLIB: Index %d exceeded SGID table max (%d)\n", index, sgid_tbl->max); return -EINVAL; } memcpy(gid, &sgid_tbl->tbl[index].gid, sizeof(*gid)); return 0; } int bnxt_qplib_del_sgid(struct bnxt_qplib_sgid_tbl *sgid_tbl, struct bnxt_qplib_gid *gid, u16 vlan_id, bool update) { struct bnxt_qplib_res *res = to_bnxt_qplib(sgid_tbl, struct bnxt_qplib_res, sgid_tbl); struct bnxt_qplib_rcfw *rcfw = res->rcfw; int index; if (sgid_tbl == NULL) { dev_err(&res->pdev->dev, "QPLIB: SGID table not allocated\n"); return -EINVAL; } /* Do we need a sgid_lock here? */ if (!sgid_tbl->active) { dev_err(&res->pdev->dev, "QPLIB: SGID table has no active entries\n"); return -ENOMEM; } for (index = 0; index < sgid_tbl->max; index++) { if (!memcmp(&sgid_tbl->tbl[index].gid, gid, sizeof(*gid)) && vlan_id == sgid_tbl->tbl[index].vlan_id) break; } if (index == sgid_tbl->max) { dev_warn(&res->pdev->dev, "GID not found in the SGID table\n"); return 0; } if (update) { struct creq_delete_gid_resp resp = {}; struct bnxt_qplib_cmdqmsg msg = {}; struct cmdq_delete_gid req = {}; int rc; bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_DELETE_GID, sizeof(req)); if (sgid_tbl->hw_id[index] == 0xFFFF) { dev_err(&res->pdev->dev, "QPLIB: GID entry contains an invalid HW id"); return -EINVAL; } req.gid_index = cpu_to_le16(sgid_tbl->hw_id[index]); bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL, sizeof(req), sizeof(resp), 0); rc = bnxt_qplib_rcfw_send_message(rcfw, &msg); if (rc) return rc; } memcpy(&sgid_tbl->tbl[index].gid, &bnxt_qplib_gid_zero, sizeof(bnxt_qplib_gid_zero)); sgid_tbl->tbl[index].vlan_id = 0xFFFF; sgid_tbl->vlan[index] = false; sgid_tbl->active--; dev_dbg(&res->pdev->dev, "QPLIB: SGID deleted hw_id[0x%x] = 0x%x active = 0x%x\n", index, sgid_tbl->hw_id[index], sgid_tbl->active); sgid_tbl->hw_id[index] = (u16)-1; return 0; } int bnxt_qplib_add_sgid(struct bnxt_qplib_sgid_tbl *sgid_tbl, const union ib_gid *gid, const u8 *smac, u16 vlan_id, bool update, u32 *index) { struct bnxt_qplib_res *res = to_bnxt_qplib(sgid_tbl, struct bnxt_qplib_res, sgid_tbl); struct bnxt_qplib_rcfw *rcfw = res->rcfw; int i, free_idx; if (sgid_tbl == NULL) { dev_err(&res->pdev->dev, "QPLIB: SGID table not allocated\n"); return -EINVAL; } /* Do we need a sgid_lock here? */ if (sgid_tbl->active == sgid_tbl->max) { dev_err(&res->pdev->dev, "QPLIB: SGID table is full\n"); return -ENOMEM; } free_idx = sgid_tbl->max; for (i = 0; i < sgid_tbl->max; i++) { if (!memcmp(&sgid_tbl->tbl[i], gid, sizeof(*gid)) && sgid_tbl->tbl[i].vlan_id == vlan_id) { dev_dbg(&res->pdev->dev, "QPLIB: SGID entry already exist in entry %d!\n", i); *index = i; return -EALREADY; } else if (!memcmp(&sgid_tbl->tbl[i], &bnxt_qplib_gid_zero, sizeof(bnxt_qplib_gid_zero)) && free_idx == sgid_tbl->max) { free_idx = i; } } if (free_idx == sgid_tbl->max) { dev_err(&res->pdev->dev, "QPLIB: SGID table is FULL but count is not MAX??\n"); return -ENOMEM; } if (update) { struct creq_add_gid_resp resp = {}; struct bnxt_qplib_cmdqmsg msg = {}; struct cmdq_add_gid req = {}; int rc; bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_ADD_GID, sizeof(req)); req.gid[0] = cpu_to_be32(((u32 *)gid->raw)[3]); req.gid[1] = cpu_to_be32(((u32 *)gid->raw)[2]); req.gid[2] = cpu_to_be32(((u32 *)gid->raw)[1]); req.gid[3] = cpu_to_be32(((u32 *)gid->raw)[0]); /* driver should ensure that all RoCE traffic is always VLAN tagged * if RoCE traffic is running on non-zero VLAN ID or * RoCE traffic is running on non-zero Priority. */ if ((vlan_id != 0xFFFF) || res->prio) { if (vlan_id != 0xFFFF) req.vlan = cpu_to_le16(vlan_id & CMDQ_ADD_GID_VLAN_VLAN_ID_MASK); req.vlan |= cpu_to_le16(CMDQ_ADD_GID_VLAN_TPID_TPID_8100 | CMDQ_ADD_GID_VLAN_VLAN_EN); } /* MAC in network format */ req.src_mac[0] = cpu_to_be16(((u16 *)smac)[0]); req.src_mac[1] = cpu_to_be16(((u16 *)smac)[1]); req.src_mac[2] = cpu_to_be16(((u16 *)smac)[2]); bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL, sizeof(req), sizeof(resp), 0); rc = bnxt_qplib_rcfw_send_message(rcfw, &msg); if (rc) return rc; sgid_tbl->hw_id[free_idx] = le32_to_cpu(resp.xid); } if (vlan_id != 0xFFFF) sgid_tbl->vlan[free_idx] = true; memcpy(&sgid_tbl->tbl[free_idx], gid, sizeof(*gid)); sgid_tbl->tbl[free_idx].vlan_id = vlan_id; sgid_tbl->active++; dev_dbg(&res->pdev->dev, "QPLIB: SGID added hw_id[0x%x] = 0x%x active = 0x%x\n", free_idx, sgid_tbl->hw_id[free_idx], sgid_tbl->active); *index = free_idx; /* unlock */ return 0; } /* AH */ int bnxt_qplib_create_ah(struct bnxt_qplib_res *res, struct bnxt_qplib_ah *ah, bool block) { struct bnxt_qplib_rcfw *rcfw = res->rcfw; struct creq_create_ah_resp resp = {}; struct bnxt_qplib_cmdqmsg msg = {}; struct cmdq_create_ah req = {}; u32 temp32[4]; u16 temp16[3]; int rc; bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_CREATE_AH, sizeof(req)); memcpy(temp32, ah->dgid.data, sizeof(struct bnxt_qplib_gid)); req.dgid[0] = cpu_to_le32(temp32[0]); req.dgid[1] = cpu_to_le32(temp32[1]); req.dgid[2] = cpu_to_le32(temp32[2]); req.dgid[3] = cpu_to_le32(temp32[3]); req.type = ah->nw_type; req.hop_limit = ah->hop_limit; req.sgid_index = cpu_to_le16(res->sgid_tbl.hw_id[ah->sgid_index]); req.dest_vlan_id_flow_label = cpu_to_le32((ah->flow_label & CMDQ_CREATE_AH_FLOW_LABEL_MASK) | CMDQ_CREATE_AH_DEST_VLAN_ID_MASK); req.pd_id = cpu_to_le32(ah->pd->id); req.traffic_class = ah->traffic_class; /* MAC in network format */ memcpy(temp16, ah->dmac, ETH_ALEN); req.dest_mac[0] = cpu_to_le16(temp16[0]); req.dest_mac[1] = cpu_to_le16(temp16[1]); req.dest_mac[2] = cpu_to_le16(temp16[2]); bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL, sizeof(req), sizeof(resp), block); rc = bnxt_qplib_rcfw_send_message(rcfw, &msg); if (rc) return rc; ah->id = le32_to_cpu(resp.xid); /* for Cu/Wh AHID 0 is not valid */ if (!_is_chip_gen_p5_p7(res->cctx) && !ah->id) rc = -EINVAL; return rc; } int bnxt_qplib_destroy_ah(struct bnxt_qplib_res *res, struct bnxt_qplib_ah *ah, bool block) { struct bnxt_qplib_rcfw *rcfw = res->rcfw; struct creq_destroy_ah_resp resp = {}; struct bnxt_qplib_cmdqmsg msg = {}; struct cmdq_destroy_ah req = {}; int rc; bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_DESTROY_AH, sizeof(req)); req.ah_cid = cpu_to_le32(ah->id); bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL, sizeof(req), sizeof(resp), block); rc = bnxt_qplib_rcfw_send_message(rcfw, &msg); return rc; } /* MRW */ int bnxt_qplib_free_mrw(struct bnxt_qplib_res *res, struct bnxt_qplib_mrw *mrw) { struct creq_deallocate_key_resp resp = {}; struct bnxt_qplib_rcfw *rcfw = res->rcfw; struct cmdq_deallocate_key req = {}; struct bnxt_qplib_cmdqmsg msg = {}; int rc; if (mrw->lkey == 0xFFFFFFFF) { dev_info(&res->pdev->dev, "QPLIB: SP: Free a reserved lkey MRW\n"); return 0; } bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_DEALLOCATE_KEY, sizeof(req)); req.mrw_flags = mrw->type; if ((mrw->type == CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE1) || (mrw->type == CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE2A) || (mrw->type == CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE2B)) req.key = cpu_to_le32(mrw->rkey); else req.key = cpu_to_le32(mrw->lkey); bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL, sizeof(req), sizeof(resp), 0); rc = bnxt_qplib_rcfw_send_message(rcfw, &msg); if (rc) return rc; if (mrw->hwq.max_elements) bnxt_qplib_free_hwq(res, &mrw->hwq); return 0; } int bnxt_qplib_alloc_mrw(struct bnxt_qplib_res *res, struct bnxt_qplib_mrw *mrw) { struct bnxt_qplib_rcfw *rcfw = res->rcfw; struct creq_allocate_mrw_resp resp = {}; struct bnxt_qplib_cmdqmsg msg = {}; struct cmdq_allocate_mrw req = {}; int rc; bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_ALLOCATE_MRW, sizeof(req)); req.pd_id = cpu_to_le32(mrw->pd->id); req.mrw_flags = mrw->type; if ((mrw->type == CMDQ_ALLOCATE_MRW_MRW_FLAGS_PMR && mrw->flags & BNXT_QPLIB_FR_PMR) || mrw->type == CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE2A || mrw->type == CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE2B) req.access = CMDQ_ALLOCATE_MRW_ACCESS_CONSUMER_OWNED_KEY; req.mrw_handle = cpu_to_le64((uintptr_t)mrw); bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL, sizeof(req), sizeof(resp), 0); rc = bnxt_qplib_rcfw_send_message(rcfw, &msg); if (rc) return rc; if ((mrw->type == CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE1) || (mrw->type == CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE2A) || (mrw->type == CMDQ_ALLOCATE_MRW_MRW_FLAGS_MW_TYPE2B)) mrw->rkey = le32_to_cpu(resp.xid); else mrw->lkey = le32_to_cpu(resp.xid); return 0; } int bnxt_qplib_dereg_mrw(struct bnxt_qplib_res *res, struct bnxt_qplib_mrw *mrw, bool block) { struct bnxt_qplib_rcfw *rcfw = res->rcfw; struct creq_deregister_mr_resp resp = {}; struct bnxt_qplib_cmdqmsg msg = {}; struct cmdq_deregister_mr req = {}; int rc; bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_DEREGISTER_MR, sizeof(req)); req.lkey = cpu_to_le32(mrw->lkey); bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL, sizeof(req), sizeof(resp), block); rc = bnxt_qplib_rcfw_send_message(rcfw, &msg); if (rc) return rc; if (mrw->hwq.max_elements) { mrw->va = 0; mrw->total_size = 0; bnxt_qplib_free_hwq(res, &mrw->hwq); } return 0; } int bnxt_qplib_reg_mr(struct bnxt_qplib_res *res, struct bnxt_qplib_mrinfo *mrinfo, bool block) { struct bnxt_qplib_hwq_attr hwq_attr = {}; struct bnxt_qplib_rcfw *rcfw = res->rcfw; struct creq_register_mr_resp resp = {}; struct bnxt_qplib_cmdqmsg msg = {}; struct cmdq_register_mr req = {}; struct bnxt_qplib_mrw *mr; u32 buf_pg_size; u32 pg_size; u16 level; u16 flags; int rc; mr = mrinfo->mrw; buf_pg_size = 0x01ULL << mrinfo->sg.pgshft; if (mrinfo->sg.npages) { /* Free the hwq if it already exist, must be a rereg */ if (mr->hwq.max_elements) bnxt_qplib_free_hwq(res, &mr->hwq); /* Use system PAGE_SIZE */ hwq_attr.res = res; hwq_attr.depth = mrinfo->sg.npages; hwq_attr.stride = PAGE_SIZE; hwq_attr.type = HWQ_TYPE_MR; hwq_attr.sginfo = &mrinfo->sg; rc = bnxt_qplib_alloc_init_hwq(&mr->hwq, &hwq_attr); if (rc) { dev_err(&res->pdev->dev, "SP: Reg MR memory allocation failed\n"); return -ENOMEM; } } bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_REGISTER_MR, sizeof(req)); /* Configure the request */ if (mrinfo->is_dma) { /* No PBL provided, just use system PAGE_SIZE */ level = 0; req.pbl = 0; pg_size = PAGE_SIZE; } else { level = mr->hwq.level; req.pbl = cpu_to_le64(mr->hwq.pbl[PBL_LVL_0].pg_map_arr[0]); } pg_size = buf_pg_size ? buf_pg_size : PAGE_SIZE; req.log2_pg_size_lvl = (level << CMDQ_REGISTER_MR_LVL_SFT) | ((ilog2(pg_size) << CMDQ_REGISTER_MR_LOG2_PG_SIZE_SFT) & CMDQ_REGISTER_MR_LOG2_PG_SIZE_MASK); req.log2_pbl_pg_size = cpu_to_le16(((ilog2(PAGE_SIZE) << CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_SFT) & CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_MASK)); req.access = (mr->flags & 0xFFFF); req.va = cpu_to_le64(mr->va); req.key = cpu_to_le32(mr->lkey); if (_is_alloc_mr_unified(res->dattr)) { flags = 0; req.key = cpu_to_le32(mr->pd->id); flags |= CMDQ_REGISTER_MR_FLAGS_ALLOC_MR; req.flags = cpu_to_le16(flags); } req.mr_size = cpu_to_le64(mr->total_size); bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL, sizeof(req), sizeof(resp), block); rc = bnxt_qplib_rcfw_send_message(rcfw, &msg); if (rc) goto fail; if (_is_alloc_mr_unified(res->dattr)) { mr->lkey = le32_to_cpu(resp.xid); mr->rkey = mr->lkey; } return 0; fail: if (mr->hwq.max_elements) bnxt_qplib_free_hwq(res, &mr->hwq); return rc; } int bnxt_qplib_alloc_fast_reg_page_list(struct bnxt_qplib_res *res, struct bnxt_qplib_frpl *frpl, int max_pg_ptrs) { struct bnxt_qplib_hwq_attr hwq_attr = {}; struct bnxt_qplib_sg_info sginfo = {}; int pg_ptrs, rc; /* Re-calculate the max to fit the HWQ allocation model */ pg_ptrs = roundup_pow_of_two(max_pg_ptrs); sginfo.pgsize = PAGE_SIZE; sginfo.nopte = true; hwq_attr.res = res; hwq_attr.depth = pg_ptrs; hwq_attr.stride = PAGE_SIZE; hwq_attr.sginfo = &sginfo; hwq_attr.type = HWQ_TYPE_CTX; rc = bnxt_qplib_alloc_init_hwq(&frpl->hwq, &hwq_attr); if (!rc) frpl->max_pg_ptrs = pg_ptrs; return rc; } void bnxt_qplib_free_fast_reg_page_list(struct bnxt_qplib_res *res, struct bnxt_qplib_frpl *frpl) { bnxt_qplib_free_hwq(res, &frpl->hwq); } int bnxt_qplib_map_tc2cos(struct bnxt_qplib_res *res, u16 *cids) { struct bnxt_qplib_rcfw *rcfw = res->rcfw; struct creq_map_tc_to_cos_resp resp = {}; struct bnxt_qplib_cmdqmsg msg = {}; struct cmdq_map_tc_to_cos req = {}; int rc; bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_MAP_TC_TO_COS, sizeof(req)); req.cos0 = cpu_to_le16(cids[0]); req.cos1 = cpu_to_le16(cids[1]); bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL, sizeof(req), sizeof(resp), 0); rc = bnxt_qplib_rcfw_send_message(rcfw, &msg); return rc; } static void bnxt_qplib_fill_cc_gen1(struct cmdq_modify_roce_cc_gen1_tlv *ext_req, struct bnxt_qplib_cc_param_ext *cc_ext) { ext_req->modify_mask = cpu_to_le64(cc_ext->ext_mask); cc_ext->ext_mask = 0; ext_req->inactivity_th_hi = cpu_to_le16(cc_ext->inact_th_hi); ext_req->min_time_between_cnps = cpu_to_le16(cc_ext->min_delta_cnp); ext_req->init_cp = cpu_to_le16(cc_ext->init_cp); ext_req->tr_update_mode = cc_ext->tr_update_mode; ext_req->tr_update_cycles = cc_ext->tr_update_cyls; ext_req->fr_num_rtts = cc_ext->fr_rtt; ext_req->ai_rate_increase = cc_ext->ai_rate_incr; ext_req->reduction_relax_rtts_th = cpu_to_le16(cc_ext->rr_rtt_th); ext_req->additional_relax_cr_th = cpu_to_le16(cc_ext->ar_cr_th); ext_req->cr_min_th = cpu_to_le16(cc_ext->cr_min_th); ext_req->bw_avg_weight = cc_ext->bw_avg_weight; ext_req->actual_cr_factor = cc_ext->cr_factor; ext_req->max_cp_cr_th = cpu_to_le16(cc_ext->cr_th_max_cp); ext_req->cp_bias_en = cc_ext->cp_bias_en; ext_req->cp_bias = cc_ext->cp_bias; ext_req->cnp_ecn = cc_ext->cnp_ecn; ext_req->rtt_jitter_en = cc_ext->rtt_jitter_en; ext_req->link_bytes_per_usec = cpu_to_le16(cc_ext->bytes_per_usec); ext_req->reset_cc_cr_th = cpu_to_le16(cc_ext->cc_cr_reset_th); ext_req->cr_width = cc_ext->cr_width; ext_req->quota_period_min = cc_ext->min_quota; ext_req->quota_period_max = cc_ext->max_quota; ext_req->quota_period_abs_max = cc_ext->abs_max_quota; ext_req->tr_lower_bound = cpu_to_le16(cc_ext->tr_lb); ext_req->cr_prob_factor = cc_ext->cr_prob_fac; ext_req->tr_prob_factor = cc_ext->tr_prob_fac; ext_req->fairness_cr_th = cpu_to_le16(cc_ext->fair_cr_th); ext_req->red_div = cc_ext->red_div; ext_req->cnp_ratio_th = cc_ext->cnp_ratio_th; ext_req->exp_ai_rtts = cpu_to_le16(cc_ext->ai_ext_rtt); ext_req->exp_ai_cr_cp_ratio = cc_ext->exp_crcp_ratio; ext_req->use_rate_table = cc_ext->low_rate_en; ext_req->cp_exp_update_th = cpu_to_le16(cc_ext->cpcr_update_th); ext_req->high_exp_ai_rtts_th1 = cpu_to_le16(cc_ext->ai_rtt_th1); ext_req->high_exp_ai_rtts_th2 = cpu_to_le16(cc_ext->ai_rtt_th2); ext_req->actual_cr_cong_free_rtts_th = cpu_to_le16(cc_ext->cf_rtt_th); ext_req->severe_cong_cr_th1 = cpu_to_le16(cc_ext->sc_cr_th1); ext_req->severe_cong_cr_th2 = cpu_to_le16(cc_ext->sc_cr_th2); ext_req->link64B_per_rtt = cpu_to_le32(cc_ext->l64B_per_rtt); ext_req->cc_ack_bytes = cc_ext->cc_ack_bytes; ext_req->reduce_init_cong_free_rtts_th = cpu_to_le16(cc_ext->reduce_cf_rtt_th); } int bnxt_qplib_modify_cc(struct bnxt_qplib_res *res, struct bnxt_qplib_cc_param *cc_param) { struct bnxt_qplib_tlv_modify_cc_req tlv_req = {}; struct creq_modify_roce_cc_resp resp = {}; struct bnxt_qplib_cmdqmsg msg = {}; struct cmdq_modify_roce_cc *req; int req_size; void *cmd; int rc; /* Prepare the older base command */ req = &tlv_req.base_req; cmd = req; req_size = sizeof(*req); bnxt_qplib_rcfw_cmd_prep(req, CMDQ_BASE_OPCODE_MODIFY_ROCE_CC, sizeof(*req)); req->modify_mask = cpu_to_le32(cc_param->mask); req->enable_cc = cc_param->enable; req->g = cc_param->g; req->num_phases_per_state = cc_param->nph_per_state; req->time_per_phase = cc_param->time_pph; req->pkts_per_phase = cc_param->pkts_pph; req->init_cr = cpu_to_le16(cc_param->init_cr); req->init_tr = cpu_to_le16(cc_param->init_tr); req->tos_dscp_tos_ecn = (cc_param->tos_dscp << CMDQ_MODIFY_ROCE_CC_TOS_DSCP_SFT) | (cc_param->tos_ecn & CMDQ_MODIFY_ROCE_CC_TOS_ECN_MASK); req->alt_vlan_pcp = cc_param->alt_vlan_pcp; req->alt_tos_dscp = cpu_to_le16(cc_param->alt_tos_dscp); req->rtt = cpu_to_le16(cc_param->rtt); req->tcp_cp = cpu_to_le16(cc_param->tcp_cp); req->cc_mode = cc_param->cc_mode; req->inactivity_th = cpu_to_le16(cc_param->inact_th); /* For chip gen P5 onwards fill extended cmd and header */ if (_is_chip_gen_p5_p7(res->cctx)) { struct roce_tlv *hdr; u32 payload; u32 chunks; cmd = &tlv_req; req_size = sizeof(tlv_req); /* Prepare primary tlv header */ hdr = &tlv_req.tlv_hdr; chunks = CHUNKS(sizeof(struct bnxt_qplib_tlv_modify_cc_req)); payload = sizeof(struct cmdq_modify_roce_cc); ROCE_1ST_TLV_PREP(hdr, chunks, payload, true); /* Prepare secondary tlv header */ hdr = (struct roce_tlv *)&tlv_req.ext_req; payload = sizeof(struct cmdq_modify_roce_cc_gen1_tlv) - sizeof(struct roce_tlv); ROCE_EXT_TLV_PREP(hdr, TLV_TYPE_MODIFY_ROCE_CC_GEN1, payload, false, true); bnxt_qplib_fill_cc_gen1(&tlv_req.ext_req, &cc_param->cc_ext); } bnxt_qplib_fill_cmdqmsg(&msg, cmd, &resp, NULL, req_size, sizeof(resp), 0); rc = bnxt_qplib_rcfw_send_message(res->rcfw, &msg); return rc; } static void bnxt_qplib_read_cc_gen1(struct bnxt_qplib_cc_param_ext *cc_ext, struct creq_query_roce_cc_gen1_resp_sb_tlv *sb) { cc_ext->inact_th_hi = le16_to_cpu(sb->inactivity_th_hi); cc_ext->min_delta_cnp = le16_to_cpu(sb->min_time_between_cnps); cc_ext->init_cp = le16_to_cpu(sb->init_cp); cc_ext->tr_update_mode = sb->tr_update_mode; cc_ext->tr_update_cyls = sb->tr_update_cycles; cc_ext->fr_rtt = sb->fr_num_rtts; cc_ext->ai_rate_incr = sb->ai_rate_increase; cc_ext->rr_rtt_th = le16_to_cpu(sb->reduction_relax_rtts_th); cc_ext->ar_cr_th = le16_to_cpu(sb->additional_relax_cr_th); cc_ext->cr_min_th = le16_to_cpu(sb->cr_min_th); cc_ext->bw_avg_weight = sb->bw_avg_weight; cc_ext->cr_factor = sb->actual_cr_factor; cc_ext->cr_th_max_cp = le16_to_cpu(sb->max_cp_cr_th); cc_ext->cp_bias_en = sb->cp_bias_en; cc_ext->cp_bias = sb->cp_bias; cc_ext->cnp_ecn = sb->cnp_ecn; cc_ext->rtt_jitter_en = sb->rtt_jitter_en; cc_ext->bytes_per_usec = le16_to_cpu(sb->link_bytes_per_usec); cc_ext->cc_cr_reset_th = le16_to_cpu(sb->reset_cc_cr_th); cc_ext->cr_width = sb->cr_width; cc_ext->min_quota = sb->quota_period_min; cc_ext->max_quota = sb->quota_period_max; cc_ext->abs_max_quota = sb->quota_period_abs_max; cc_ext->tr_lb = le16_to_cpu(sb->tr_lower_bound); cc_ext->cr_prob_fac = sb->cr_prob_factor; cc_ext->tr_prob_fac = sb->tr_prob_factor; cc_ext->fair_cr_th = le16_to_cpu(sb->fairness_cr_th); cc_ext->red_div = sb->red_div; cc_ext->cnp_ratio_th = sb->cnp_ratio_th; cc_ext->ai_ext_rtt = le16_to_cpu(sb->exp_ai_rtts); cc_ext->exp_crcp_ratio = sb->exp_ai_cr_cp_ratio; cc_ext->low_rate_en = sb->use_rate_table; cc_ext->cpcr_update_th = le16_to_cpu(sb->cp_exp_update_th); cc_ext->ai_rtt_th1 = le16_to_cpu(sb->high_exp_ai_rtts_th1); cc_ext->ai_rtt_th2 = le16_to_cpu(sb->high_exp_ai_rtts_th2); cc_ext->cf_rtt_th = le16_to_cpu(sb->actual_cr_cong_free_rtts_th); cc_ext->sc_cr_th1 = le16_to_cpu(sb->severe_cong_cr_th1); cc_ext->sc_cr_th2 = le16_to_cpu(sb->severe_cong_cr_th2); cc_ext->l64B_per_rtt = le32_to_cpu(sb->link64B_per_rtt); cc_ext->cc_ack_bytes = sb->cc_ack_bytes; cc_ext->reduce_cf_rtt_th = le16_to_cpu(sb->reduce_init_cong_free_rtts_th); } int bnxt_qplib_query_cc_param(struct bnxt_qplib_res *res, struct bnxt_qplib_cc_param *cc_param) { struct creq_query_roce_cc_gen1_resp_sb_tlv *gen1_sb; struct bnxt_qplib_tlv_query_rcc_sb *ext_sb; struct bnxt_qplib_rcfw *rcfw = res->rcfw; struct creq_query_roce_cc_resp resp = {}; struct creq_query_roce_cc_resp_sb *sb; struct bnxt_qplib_cmdqmsg msg = {}; struct cmdq_query_roce_cc req = {}; struct bnxt_qplib_rcfw_sbuf sbuf; size_t resp_size; int rc; /* Query the parameters from chip */ bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_QUERY_ROCE_CC, sizeof(req)); if (_is_chip_gen_p5_p7(res->cctx)) resp_size = sizeof(*ext_sb); else resp_size = sizeof(*sb); sbuf.size = ALIGN(resp_size, BNXT_QPLIB_CMDQE_UNITS); sbuf.sb = dma_zalloc_coherent(&rcfw->pdev->dev, sbuf.size, &sbuf.dma_addr, GFP_KERNEL); if (!sbuf.sb) return -ENOMEM; req.resp_size = sbuf.size / BNXT_QPLIB_CMDQE_UNITS; bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, &sbuf, sizeof(req), sizeof(resp), 0); rc = bnxt_qplib_rcfw_send_message(res->rcfw, &msg); if (rc) { dev_dbg(&res->pdev->dev, "%s:Query CC param failed:0x%x\n", __func__, rc); goto out; } ext_sb = sbuf.sb; gen1_sb = &ext_sb->gen1_sb; sb = _is_chip_gen_p5_p7(res->cctx) ? &ext_sb->base_sb : (struct creq_query_roce_cc_resp_sb *)ext_sb; cc_param->enable = sb->enable_cc & CREQ_QUERY_ROCE_CC_RESP_SB_ENABLE_CC; cc_param->tos_ecn = (sb->tos_dscp_tos_ecn & CREQ_QUERY_ROCE_CC_RESP_SB_TOS_ECN_MASK) >> CREQ_QUERY_ROCE_CC_RESP_SB_TOS_ECN_SFT; cc_param->tos_dscp = (sb->tos_dscp_tos_ecn & CREQ_QUERY_ROCE_CC_RESP_SB_TOS_DSCP_MASK) >> CREQ_QUERY_ROCE_CC_RESP_SB_TOS_DSCP_SFT; cc_param->alt_tos_dscp = sb->alt_tos_dscp; cc_param->alt_vlan_pcp = sb->alt_vlan_pcp; cc_param->g = sb->g; cc_param->nph_per_state = sb->num_phases_per_state; cc_param->init_cr = le16_to_cpu(sb->init_cr); cc_param->init_tr = le16_to_cpu(sb->init_tr); cc_param->cc_mode = sb->cc_mode; cc_param->inact_th = le16_to_cpu(sb->inactivity_th); cc_param->rtt = le16_to_cpu(sb->rtt); cc_param->tcp_cp = le16_to_cpu(sb->tcp_cp); cc_param->time_pph = sb->time_per_phase; cc_param->pkts_pph = sb->pkts_per_phase; if (_is_chip_gen_p5_p7(res->cctx)) bnxt_qplib_read_cc_gen1(&cc_param->cc_ext, gen1_sb); out: dma_free_coherent(&rcfw->pdev->dev, sbuf.size, sbuf.sb, sbuf.dma_addr); return rc; } int bnxt_qplib_get_roce_error_stats(struct bnxt_qplib_rcfw *rcfw, struct bnxt_qplib_roce_stats *stats, struct bnxt_qplib_query_stats_info *sinfo) { struct creq_query_roce_stats_resp resp = {}; struct creq_query_roce_stats_resp_sb *sb; struct cmdq_query_roce_stats req = {}; struct bnxt_qplib_cmdqmsg msg = {}; struct bnxt_qplib_rcfw_sbuf sbuf; u16 cmd_flags = 0; u32 fn_id = 0; int rc = 0; bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_QUERY_ROCE_STATS, sizeof(req)); sbuf.size = sizeof(*sb); sbuf.sb = dma_zalloc_coherent(&rcfw->pdev->dev, sbuf.size, &sbuf.dma_addr, GFP_KERNEL); if (!sbuf.sb) return -ENOMEM; sb = sbuf.sb; if (rcfw->res->cctx->hwrm_intf_ver >= HWRM_VERSION_ROCE_STATS_FN_ID) { if (sinfo->function_id != 0xFFFFFFFF) { cmd_flags = CMDQ_QUERY_ROCE_STATS_FLAGS_FUNCTION_ID; if (sinfo->vf_valid) { fn_id = CMDQ_QUERY_ROCE_STATS_VF_VALID; fn_id |= (sinfo->function_id << CMDQ_QUERY_ROCE_STATS_VF_NUM_SFT) & CMDQ_QUERY_ROCE_STATS_VF_NUM_MASK; } else { fn_id = sinfo->function_id & CMDQ_QUERY_ROCE_STATS_PF_NUM_MASK; } } req.flags = cpu_to_le16(cmd_flags); req.function_id = cpu_to_le32(fn_id); if (sinfo->collection_id != 0xFF) { cmd_flags |= CMDQ_QUERY_ROCE_STATS_FLAGS_COLLECTION_ID; req.collection_id = sinfo->collection_id; } } else { /* For older HWRM version, the command length has to be * adjusted. 8 bytes are more in the newer command. * So subtract these 8 bytes for older HWRM version. * command units are adjusted inside * bnxt_qplib_rcfw_send_message. */ req.cmd_size -= 8; } req.resp_size = sbuf.size / BNXT_QPLIB_CMDQE_UNITS; bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, &sbuf, sizeof(req), sizeof(resp), 0); rc = bnxt_qplib_rcfw_send_message(rcfw, &msg); if (rc) goto bail; /* Extract the context from the side buffer */ stats->to_retransmits = le64_to_cpu(sb->to_retransmits); stats->seq_err_naks_rcvd = le64_to_cpu(sb->seq_err_naks_rcvd); stats->max_retry_exceeded = le64_to_cpu(sb->max_retry_exceeded); stats->rnr_naks_rcvd = le64_to_cpu(sb->rnr_naks_rcvd); stats->missing_resp = le64_to_cpu(sb->missing_resp); stats->unrecoverable_err = le64_to_cpu(sb->unrecoverable_err); stats->bad_resp_err = le64_to_cpu(sb->bad_resp_err); stats->local_qp_op_err = le64_to_cpu(sb->local_qp_op_err); stats->local_protection_err = le64_to_cpu(sb->local_protection_err); stats->mem_mgmt_op_err = le64_to_cpu(sb->mem_mgmt_op_err); stats->remote_invalid_req_err = le64_to_cpu(sb->remote_invalid_req_err); stats->remote_access_err = le64_to_cpu(sb->remote_access_err); stats->remote_op_err = le64_to_cpu(sb->remote_op_err); stats->dup_req = le64_to_cpu(sb->dup_req); stats->res_exceed_max = le64_to_cpu(sb->res_exceed_max); stats->res_length_mismatch = le64_to_cpu(sb->res_length_mismatch); stats->res_exceeds_wqe = le64_to_cpu(sb->res_exceeds_wqe); stats->res_opcode_err = le64_to_cpu(sb->res_opcode_err); stats->res_rx_invalid_rkey = le64_to_cpu(sb->res_rx_invalid_rkey); stats->res_rx_domain_err = le64_to_cpu(sb->res_rx_domain_err); stats->res_rx_no_perm = le64_to_cpu(sb->res_rx_no_perm); stats->res_rx_range_err = le64_to_cpu(sb->res_rx_range_err); stats->res_tx_invalid_rkey = le64_to_cpu(sb->res_tx_invalid_rkey); stats->res_tx_domain_err = le64_to_cpu(sb->res_tx_domain_err); stats->res_tx_no_perm = le64_to_cpu(sb->res_tx_no_perm); stats->res_tx_range_err = le64_to_cpu(sb->res_tx_range_err); stats->res_irrq_oflow = le64_to_cpu(sb->res_irrq_oflow); stats->res_unsup_opcode = le64_to_cpu(sb->res_unsup_opcode); stats->res_unaligned_atomic = le64_to_cpu(sb->res_unaligned_atomic); stats->res_rem_inv_err = le64_to_cpu(sb->res_rem_inv_err); stats->res_mem_error = le64_to_cpu(sb->res_mem_error); stats->res_srq_err = le64_to_cpu(sb->res_srq_err); stats->res_cmp_err = le64_to_cpu(sb->res_cmp_err); stats->res_invalid_dup_rkey = le64_to_cpu(sb->res_invalid_dup_rkey); stats->res_wqe_format_err = le64_to_cpu(sb->res_wqe_format_err); stats->res_cq_load_err = le64_to_cpu(sb->res_cq_load_err); stats->res_srq_load_err = le64_to_cpu(sb->res_srq_load_err); stats->res_tx_pci_err = le64_to_cpu(sb->res_tx_pci_err); stats->res_rx_pci_err = le64_to_cpu(sb->res_rx_pci_err); if (!rcfw->init_oos_stats) { rcfw->oos_prev = le64_to_cpu(sb->res_oos_drop_count); rcfw->init_oos_stats = true; } else { stats->res_oos_drop_count += (le64_to_cpu(sb->res_oos_drop_count) - rcfw->oos_prev) & BNXT_QPLIB_OOS_COUNT_MASK; rcfw->oos_prev = le64_to_cpu(sb->res_oos_drop_count); } stats->active_qp_count_p0 = le64_to_cpu(sb->active_qp_count_p0); stats->active_qp_count_p1 = le64_to_cpu(sb->active_qp_count_p1); stats->active_qp_count_p2 = le64_to_cpu(sb->active_qp_count_p2); stats->active_qp_count_p3 = le64_to_cpu(sb->active_qp_count_p3); bail: dma_free_coherent(&rcfw->pdev->dev, sbuf.size, sbuf.sb, sbuf.dma_addr); return rc; } int bnxt_qplib_set_link_aggr_mode(struct bnxt_qplib_res *res, u8 aggr_mode, u8 member_port_map, u8 active_port_map, bool aggr_en, u32 stats_fw_id) { struct creq_set_link_aggr_mode_resources_resp resp = {}; struct cmdq_set_link_aggr_mode_cc req = {}; struct bnxt_qplib_rcfw *rcfw = res->rcfw; struct bnxt_qplib_cmdqmsg msg = {}; int rc = 0; bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_SET_LINK_AGGR_MODE, sizeof(req)); req.aggr_enable = aggr_en; req.active_port_map = active_port_map; req.member_port_map = member_port_map; req.link_aggr_mode = aggr_mode; /* need to specify only second port stats ctx id for now */ req.stat_ctx_id[1] = cpu_to_le16((u16)(stats_fw_id)); req.modify_mask = cpu_to_le32(CMDQ_SET_LINK_AGGR_MODE_MODIFY_MASK_AGGR_EN | CMDQ_SET_LINK_AGGR_MODE_MODIFY_MASK_ACTIVE_PORT_MAP | CMDQ_SET_LINK_AGGR_MODE_MODIFY_MASK_MEMBER_PORT_MAP | CMDQ_SET_LINK_AGGR_MODE_MODIFY_MASK_AGGR_MODE | CMDQ_SET_LINK_AGGR_MODE_MODIFY_MASK_STAT_CTX_ID); bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL, sizeof(req), sizeof(resp), 0); rc = bnxt_qplib_rcfw_send_message(rcfw, &msg); if (rc) dev_err(&res->pdev->dev, "QPLIB: Failed to set link aggr mode, %#x\n", rc); return rc; } int bnxt_qplib_qext_stat(struct bnxt_qplib_rcfw *rcfw, u32 fid, struct bnxt_qplib_ext_stat *estat, struct bnxt_qplib_query_stats_info *sinfo) { struct creq_query_roce_stats_ext_resp resp = {}; struct creq_query_roce_stats_ext_resp_sb *sb; struct cmdq_query_roce_stats_ext req = {}; struct bnxt_qplib_cmdqmsg msg = {}; struct bnxt_qplib_rcfw_sbuf sbuf; int rc; sbuf.size = sizeof(*sb); sbuf.sb = dma_zalloc_coherent(&rcfw->pdev->dev, sbuf.size, &sbuf.dma_addr, GFP_KERNEL); if (!sbuf.sb) { dev_err(&rcfw->pdev->dev, "QPLIB: SP: QUERY_ROCE_STATS_EXT alloc sb failed\n"); return -ENOMEM; } sb = sbuf.sb; bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_QUERY_ROCE_STATS_EXT_OPCODE_QUERY_ROCE_STATS, sizeof(req)); req.resp_size = sbuf.size; req.resp_addr = cpu_to_le64(sbuf.dma_addr); req.flags = cpu_to_le16(CMDQ_QUERY_ROCE_STATS_EXT_FLAGS_FUNCTION_ID); if (_is_chip_p7(rcfw->res->cctx) && rcfw->res->is_vf) { if (sinfo->vf_valid) req.function_id = cpu_to_le32(CMDQ_QUERY_ROCE_STATS_EXT_VF_VALID | (fid << CMDQ_QUERY_ROCE_STATS_EXT_VF_NUM_SFT)); else req.flags = cpu_to_le16(0); } else { req.function_id = cpu_to_le32(fid); } bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, &sbuf, sizeof(req), sizeof(resp), 0); rc = bnxt_qplib_rcfw_send_message(rcfw, &msg); if (rc) goto bail; /* dump when dyndbg is enabled */ print_hex_dump_bytes("", DUMP_PREFIX_OFFSET, sb, sizeof(*sb)); estat->tx_atomic_req = le64_to_cpu(sb->tx_atomic_req_pkts); estat->tx_read_req = le64_to_cpu(sb->tx_read_req_pkts); estat->tx_read_res = le64_to_cpu(sb->tx_read_res_pkts); estat->tx_write_req = le64_to_cpu(sb->tx_write_req_pkts); estat->tx_send_req = le64_to_cpu(sb->tx_send_req_pkts); estat->tx_roce_pkts = le64_to_cpu(sb->tx_roce_pkts); estat->tx_roce_bytes = le64_to_cpu(sb->tx_roce_bytes); estat->rx_atomic_req = le64_to_cpu(sb->rx_atomic_req_pkts); estat->rx_read_req = le64_to_cpu(sb->rx_read_req_pkts); estat->rx_read_res = le64_to_cpu(sb->rx_read_res_pkts); estat->rx_write_req = le64_to_cpu(sb->rx_write_req_pkts); estat->rx_send_req = le64_to_cpu(sb->rx_send_req_pkts); estat->rx_roce_pkts = le64_to_cpu(sb->rx_roce_pkts); estat->rx_roce_bytes = le64_to_cpu(sb->rx_roce_bytes); estat->rx_roce_good_pkts = le64_to_cpu(sb->rx_roce_good_pkts); estat->rx_roce_good_bytes = le64_to_cpu(sb->rx_roce_good_bytes); estat->rx_out_of_buffer = le64_to_cpu(sb->rx_out_of_buffer_pkts); estat->rx_out_of_sequence = le64_to_cpu(sb->rx_out_of_sequence_pkts); estat->tx_cnp = le64_to_cpu(sb->tx_cnp_pkts); estat->rx_cnp = le64_to_cpu(sb->rx_cnp_pkts); estat->rx_ecn_marked = le64_to_cpu(sb->rx_ecn_marked_pkts); estat->seq_err_naks_rcvd = le64_to_cpu(sb->seq_err_naks_rcvd); estat->rnr_naks_rcvd = le64_to_cpu(sb->rnr_naks_rcvd); estat->missing_resp = le64_to_cpu(sb->missing_resp); estat->to_retransmits = le64_to_cpu(sb->to_retransmit); estat->dup_req = le64_to_cpu(sb->dup_req); estat->rx_dcn_payload_cut = le64_to_cpu(sb->rx_dcn_payload_cut); estat->te_bypassed = le64_to_cpu(sb->te_bypassed); bail: dma_free_coherent(&rcfw->pdev->dev, sbuf.size, sbuf.sb, sbuf.dma_addr); return rc; } diff --git a/sys/dev/bnxt/bnxt_re/qplib_sp.h b/sys/dev/bnxt/bnxt_re/qplib_sp.h index e306db3b9d8e..fa7dd890501c 100644 --- a/sys/dev/bnxt/bnxt_re/qplib_sp.h +++ b/sys/dev/bnxt/bnxt_re/qplib_sp.h @@ -1,432 +1,433 @@ /* * Copyright (c) 2015-2024, Broadcom. All rights reserved. The term * Broadcom refers to Broadcom Limited and/or its subsidiaries. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in * the documentation and/or other materials provided with the * distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * * Description: Slow Path Operators (header) */ #ifndef __BNXT_QPLIB_SP_H__ #define __BNXT_QPLIB_SP_H__ #include #define BNXT_QPLIB_RESERVED_QP_WRS 128 /* Resource maximums reported by the firmware */ struct bnxt_qplib_dev_attr { #define FW_VER_ARR_LEN 4 u8 fw_ver[FW_VER_ARR_LEN]; u16 max_sgid; u16 max_mrw; u32 max_qp; #define BNXT_QPLIB_MAX_OUT_RD_ATOM 126 u32 max_qp_rd_atom; u32 max_qp_init_rd_atom; u32 max_qp_wqes; u32 max_qp_sges; u32 max_cq; /* HW supports only 8K entries in PBL. * So max CQEs that can be supported per CQ is 1M. */ #define BNXT_QPLIB_MAX_CQ_WQES 0xfffff u32 max_cq_wqes; u32 max_cq_sges; u32 max_mr; u64 max_mr_size; #define BNXT_QPLIB_MAX_PD (64 * 1024) u32 max_pd; u32 max_mw; u32 max_raw_ethy_qp; u32 max_ah; u32 max_fmr; u32 max_map_per_fmr; u32 max_srq; u32 max_srq_wqes; u32 max_srq_sges; u32 max_pkey; u32 max_inline_data; u32 l2_db_size; u8 tqm_alloc_reqs[MAX_TQM_ALLOC_REQ]; u8 is_atomic; + u16 dev_cap_ext_flags2; u16 dev_cap_flags; u64 page_size_cap; u32 max_dpi; }; struct bnxt_qplib_pd { u32 id; }; struct bnxt_qplib_gid { u8 data[16]; }; struct bnxt_qplib_gid_info { struct bnxt_qplib_gid gid; u16 vlan_id; }; struct bnxt_qplib_ah { struct bnxt_qplib_gid dgid; struct bnxt_qplib_pd *pd; u32 id; u8 sgid_index; u8 host_sgid_index; /* For Query AH if the hw table and SW table are differnt */ u8 traffic_class; u32 flow_label; u8 hop_limit; u8 sl; u8 dmac[6]; u16 vlan_id; u8 nw_type; u8 enable_cc; }; struct bnxt_qplib_mrw { struct bnxt_qplib_pd *pd; int type; u32 flags; #define BNXT_QPLIB_FR_PMR 0x80000000 u32 lkey; u32 rkey; #define BNXT_QPLIB_RSVD_LKEY 0xFFFFFFFF u64 va; u64 total_size; u32 npages; u64 mr_handle; struct bnxt_qplib_hwq hwq; }; struct bnxt_qplib_mrinfo { struct bnxt_qplib_mrw *mrw; struct bnxt_qplib_sg_info sg; u64 *ptes; bool is_dma; }; struct bnxt_qplib_frpl { int max_pg_ptrs; struct bnxt_qplib_hwq hwq; }; struct bnxt_qplib_cc_param_ext { u64 ext_mask; u16 inact_th_hi; u16 min_delta_cnp; u16 init_cp; u8 tr_update_mode; u8 tr_update_cyls; u8 fr_rtt; u8 ai_rate_incr; u16 rr_rtt_th; u16 ar_cr_th; u16 cr_min_th; u8 bw_avg_weight; u8 cr_factor; u16 cr_th_max_cp; u8 cp_bias_en; u8 cp_bias; u8 cnp_ecn; u8 rtt_jitter_en; u16 bytes_per_usec; u16 cc_cr_reset_th; u8 cr_width; u8 min_quota; u8 max_quota; u8 abs_max_quota; u16 tr_lb; u8 cr_prob_fac; u8 tr_prob_fac; u16 fair_cr_th; u8 red_div; u8 cnp_ratio_th; u16 ai_ext_rtt; u8 exp_crcp_ratio; u8 low_rate_en; u16 cpcr_update_th; u16 ai_rtt_th1; u16 ai_rtt_th2; u16 cf_rtt_th; u16 sc_cr_th1; /* severe congestion cr threshold 1 */ u16 sc_cr_th2; /* severe congestion cr threshold 2 */ u32 l64B_per_rtt; u8 cc_ack_bytes; u16 reduce_cf_rtt_th; }; struct bnxt_qplib_cc_param { u8 alt_vlan_pcp; u16 alt_tos_dscp; #define BNXT_QPLIB_USER_DSCP_VALID 0x80 u8 cnp_dscp_user; u8 roce_dscp_user; u8 cc_mode; u8 enable; u16 inact_th; u16 init_cr; u16 init_tr; u16 rtt; u8 g; u8 nph_per_state; u8 time_pph; u8 pkts_pph; u8 tos_ecn; u8 tos_dscp; u8 qp1_tos_dscp; u16 tcp_cp; struct bnxt_qplib_cc_param_ext cc_ext; u8 disable_prio_vlan_tx; /* Mask used while programming the configfs values */ u32 mask; /* Mask used while displaying the configfs values */ u32 cur_mask; u8 roce_pri; #define BNXT_QPLIB_CC_PARAM_MASK_VLAN_TX_DISABLE 0x40000 #define BNXT_QPLIB_CC_PARAM_MASK_ROCE_PRI 0x80000 /* prev value to clear dscp table */ u8 prev_roce_pri; u8 prev_alt_vlan_pcp; u8 prev_tos_dscp; u16 prev_alt_tos_dscp; /* To track if admin has enabled ECN explicitly */ u8 admin_enable; }; struct bnxt_qplib_roce_stats { u64 to_retransmits; u64 seq_err_naks_rcvd; /* seq_err_naks_rcvd is 64 b */ u64 max_retry_exceeded; /* max_retry_exceeded is 64 b */ u64 rnr_naks_rcvd; /* rnr_naks_rcvd is 64 b */ u64 missing_resp; u64 unrecoverable_err; /* unrecoverable_err is 64 b */ u64 bad_resp_err; /* bad_resp_err is 64 b */ u64 local_qp_op_err; /* local_qp_op_err is 64 b */ u64 local_protection_err; /* local_protection_err is 64 b */ u64 mem_mgmt_op_err; /* mem_mgmt_op_err is 64 b */ u64 remote_invalid_req_err; /* remote_invalid_req_err is 64 b */ u64 remote_access_err; /* remote_access_err is 64 b */ u64 remote_op_err; /* remote_op_err is 64 b */ u64 dup_req; /* dup_req is 64 b */ u64 res_exceed_max; /* res_exceed_max is 64 b */ u64 res_length_mismatch; /* res_length_mismatch is 64 b */ u64 res_exceeds_wqe; /* res_exceeds_wqe is 64 b */ u64 res_opcode_err; /* res_opcode_err is 64 b */ u64 res_rx_invalid_rkey; /* res_rx_invalid_rkey is 64 b */ u64 res_rx_domain_err; /* res_rx_domain_err is 64 b */ u64 res_rx_no_perm; /* res_rx_no_perm is 64 b */ u64 res_rx_range_err; /* res_rx_range_err is 64 b */ u64 res_tx_invalid_rkey; /* res_tx_invalid_rkey is 64 b */ u64 res_tx_domain_err; /* res_tx_domain_err is 64 b */ u64 res_tx_no_perm; /* res_tx_no_perm is 64 b */ u64 res_tx_range_err; /* res_tx_range_err is 64 b */ u64 res_irrq_oflow; /* res_irrq_oflow is 64 b */ u64 res_unsup_opcode; /* res_unsup_opcode is 64 b */ u64 res_unaligned_atomic; /* res_unaligned_atomic is 64 b */ u64 res_rem_inv_err; /* res_rem_inv_err is 64 b */ u64 res_mem_error; /* res_mem_error is 64 b */ u64 res_srq_err; /* res_srq_err is 64 b */ u64 res_cmp_err; /* res_cmp_err is 64 b */ u64 res_invalid_dup_rkey; /* res_invalid_dup_rkey is 64 b */ u64 res_wqe_format_err; /* res_wqe_format_err is 64 b */ u64 res_cq_load_err; /* res_cq_load_err is 64 b */ u64 res_srq_load_err; /* res_srq_load_err is 64 b */ u64 res_tx_pci_err; /* res_tx_pci_err is 64 b */ u64 res_rx_pci_err; /* res_rx_pci_err is 64 b */ u64 res_oos_drop_count; /* res_oos_drop_count */ u64 active_qp_count_p0; /* port 0 active qps */ u64 active_qp_count_p1; /* port 1 active qps */ u64 active_qp_count_p2; /* port 2 active qps */ u64 active_qp_count_p3; /* port 3 active qps */ }; struct bnxt_qplib_ext_stat { u64 tx_atomic_req; u64 tx_read_req; u64 tx_read_res; u64 tx_write_req; u64 tx_send_req; u64 tx_roce_pkts; u64 tx_roce_bytes; u64 rx_atomic_req; u64 rx_read_req; u64 rx_read_res; u64 rx_write_req; u64 rx_send_req; u64 rx_roce_pkts; u64 rx_roce_bytes; u64 rx_roce_good_pkts; u64 rx_roce_good_bytes; u64 rx_out_of_buffer; u64 rx_out_of_sequence; u64 tx_cnp; u64 rx_cnp; u64 rx_ecn_marked; u64 seq_err_naks_rcvd; u64 rnr_naks_rcvd; u64 missing_resp; u64 to_retransmits; u64 dup_req; u64 rx_dcn_payload_cut; u64 te_bypassed; }; #define BNXT_QPLIB_ACCESS_LOCAL_WRITE (1 << 0) #define BNXT_QPLIB_ACCESS_REMOTE_READ (1 << 1) #define BNXT_QPLIB_ACCESS_REMOTE_WRITE (1 << 2) #define BNXT_QPLIB_ACCESS_REMOTE_ATOMIC (1 << 3) #define BNXT_QPLIB_ACCESS_MW_BIND (1 << 4) #define BNXT_QPLIB_ACCESS_ZERO_BASED (1 << 5) #define BNXT_QPLIB_ACCESS_ON_DEMAND (1 << 6) int bnxt_qplib_get_sgid(struct bnxt_qplib_res *res, struct bnxt_qplib_sgid_tbl *sgid_tbl, int index, struct bnxt_qplib_gid *gid); int bnxt_qplib_del_sgid(struct bnxt_qplib_sgid_tbl *sgid_tbl, struct bnxt_qplib_gid *gid, u16 vlan_id, bool update); int bnxt_qplib_add_sgid(struct bnxt_qplib_sgid_tbl *sgid_tbl, const union ib_gid *gid, const u8 *mac, u16 vlan_id, bool update, u32 *index); int bnxt_qplib_update_sgid(struct bnxt_qplib_sgid_tbl *sgid_tbl, struct bnxt_qplib_gid *gid, u16 gid_idx, const u8 *smac); int bnxt_qplib_get_dev_attr(struct bnxt_qplib_rcfw *rcfw); int bnxt_qplib_set_func_resources(struct bnxt_qplib_res *res); int bnxt_qplib_create_ah(struct bnxt_qplib_res *res, struct bnxt_qplib_ah *ah, bool block); int bnxt_qplib_destroy_ah(struct bnxt_qplib_res *res, struct bnxt_qplib_ah *ah, bool block); int bnxt_qplib_alloc_mrw(struct bnxt_qplib_res *res, struct bnxt_qplib_mrw *mrw); int bnxt_qplib_dereg_mrw(struct bnxt_qplib_res *res, struct bnxt_qplib_mrw *mrw, bool block); int bnxt_qplib_reg_mr(struct bnxt_qplib_res *res, struct bnxt_qplib_mrinfo *mrinfo, bool block); int bnxt_qplib_free_mrw(struct bnxt_qplib_res *res, struct bnxt_qplib_mrw *mr); int bnxt_qplib_alloc_fast_reg_mr(struct bnxt_qplib_res *res, struct bnxt_qplib_mrw *mr, int max); int bnxt_qplib_alloc_fast_reg_page_list(struct bnxt_qplib_res *res, struct bnxt_qplib_frpl *frpl, int max); void bnxt_qplib_free_fast_reg_page_list(struct bnxt_qplib_res *res, struct bnxt_qplib_frpl *frpl); int bnxt_qplib_map_tc2cos(struct bnxt_qplib_res *res, u16 *cids); int bnxt_qplib_modify_cc(struct bnxt_qplib_res *res, struct bnxt_qplib_cc_param *cc_param); int bnxt_qplib_query_cc_param(struct bnxt_qplib_res *res, struct bnxt_qplib_cc_param *cc_param); int bnxt_qplib_set_link_aggr_mode(struct bnxt_qplib_res *res, u8 aggr_mode, u8 member_port_map, u8 active_port_map, bool aggr_en, u32 stats_fw_id); int bnxt_qplib_get_roce_error_stats(struct bnxt_qplib_rcfw *rcfw, struct bnxt_qplib_roce_stats *stats, struct bnxt_qplib_query_stats_info *sinfo); int bnxt_qplib_qext_stat(struct bnxt_qplib_rcfw *rcfw, u32 fid, struct bnxt_qplib_ext_stat *estat, struct bnxt_qplib_query_stats_info *sinfo); static inline void bnxt_re_set_max_gid(u16 *max_sgid); bool ib_modify_qp_is_ok_compat(enum ib_qp_state cur_state, enum ib_qp_state next_state, enum ib_qp_type type, enum ib_qp_attr_mask mask); #define BNXT_MAX_SQ_SIZE 0xFFFF #define BNXT_MAX_VAR_WQE_SIZE 512 #define BNXT_SGE_SIZE 16 /* PF defines */ #define BNXT_RE_MAX_QP_SUPPORTED(chip_gen) \ chip_gen == BNXT_RE_DEFAULT ? (64 * 1024) : 0 #define BNXT_RE_MAX_MRW_SUPPORTED(chip_gen) \ chip_gen == BNXT_RE_DEFAULT ? (256 * 1024) : 0 #define BNXT_RE_MAX_CQ_SUPPORTED(chip_gen) \ chip_gen == BNXT_RE_DEFAULT ? (64 * 1024) : 0 #define BNXT_RE_MAX_SRQ_SUPPORTED(chip_gen) \ chip_gen == BNXT_RE_DEFAULT ? (4 * 1024) : 0 #define BNXT_RE_MAX_AH_SUPPORTED(chip_gen) \ chip_gen == BNXT_RE_DEFAULT ? (64 * 1024) : 0 /* VF defines */ #define BNXT_RE_VF_MAX_QP_SUPPORTED(chip_gen) \ chip_gen == BNXT_RE_DEFAULT ? (6 * 1024) : 0 #define BNXT_RE_VF_MAX_MRW_SUPPORTED(chip_gen) \ chip_gen == BNXT_RE_DEFAULT ? (6 * 1024) : 0 #define BNXT_RE_VF_MAX_CQ_SUPPORTED(chip_gen) \ chip_gen == BNXT_RE_DEFAULT ? (6 * 1024) : 0 #define BNXT_RE_VF_MAX_SRQ_SUPPORTED(chip_gen) \ chip_gen == BNXT_RE_DEFAULT ? (4 * 1024) : 0 static inline void bnxt_re_set_max_gid(u16 *max_sgid) { *max_sgid = max_t(u32, 256, *max_sgid); *max_sgid = min_t(u32, 256, *max_sgid); } #endif