diff --git a/contrib/ofed/libirdma/irdma_defs.h b/contrib/ofed/libirdma/irdma_defs.h --- a/contrib/ofed/libirdma/irdma_defs.h +++ b/contrib/ofed/libirdma/irdma_defs.h @@ -1,7 +1,7 @@ /*- * SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB * - * Copyright (c) 2015 - 2022 Intel Corporation + * Copyright (c) 2015 - 2023 Intel Corporation * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU @@ -312,7 +312,7 @@ #define IRDMA_GET_CQ_ELEM_AT_OFFSET(_cq, _i, _cqe) \ { \ - register __u32 offset; \ + __u32 offset; \ offset = IRDMA_GET_RING_OFFSET((_cq)->cq_ring, _i); \ (_cqe) = (_cq)->cq_base[offset].buf; \ } @@ -338,7 +338,7 @@ #define IRDMA_RING_MOVE_HEAD(_ring, _retcode) \ { \ - register u32 size; \ + u32 size; \ size = (_ring).size; \ if (!IRDMA_RING_FULL_ERR(_ring)) { \ (_ring).head = ((_ring).head + 1) % size; \ @@ -349,7 +349,7 @@ } #define IRDMA_RING_MOVE_HEAD_BY_COUNT(_ring, _count, _retcode) \ { \ - register u32 size; \ + u32 size; \ size = (_ring).size; \ if ((IRDMA_RING_USED_QUANTA(_ring) + (_count)) < size) { \ (_ring).head = ((_ring).head + (_count)) % size; \ @@ -360,7 +360,7 @@ } #define IRDMA_SQ_RING_MOVE_HEAD(_ring, _retcode) \ { \ - register u32 size; \ + u32 size; \ size = (_ring).size; \ if (!IRDMA_SQ_RING_FULL_ERR(_ring)) { \ (_ring).head = ((_ring).head + 1) % size; \ @@ -371,7 +371,7 @@ } #define IRDMA_SQ_RING_MOVE_HEAD_BY_COUNT(_ring, _count, _retcode) \ { \ - register u32 size; \ + u32 size; \ size = (_ring).size; \ if ((IRDMA_RING_USED_QUANTA(_ring) + (_count)) < (size - 256)) { \ (_ring).head = ((_ring).head + (_count)) % size; \ @@ -457,6 +457,19 @@ IRDMA_WQE_SIZE_256 = 256, }; +enum irdma_ws_op_type { + IRDMA_WS_OP_TYPE_NODE = 0, + IRDMA_WS_OP_TYPE_LEAF_NODE_GROUP, +}; + +enum irdma_ws_rate_limit_flags { + IRDMA_WS_RATE_LIMIT_FLAGS_VALID = 0x1, + IRDMA_WS_NO_RDMA_RATE_LIMIT = 0x2, + IRDMA_WS_LEAF_NODE_IS_PART_GROUP = 0x4, + IRDMA_WS_TREE_RATE_LIMITING = 0x8, + IRDMA_WS_PACING_CONTROL = 0x10, +}; + /** * set_64bit_val - set 64 bit value to hw wqe * @wqe_words: wqe addr to write diff --git a/contrib/ofed/libirdma/irdma_uk.c b/contrib/ofed/libirdma/irdma_uk.c --- a/contrib/ofed/libirdma/irdma_uk.c +++ b/contrib/ofed/libirdma/irdma_uk.c @@ -1,7 +1,7 @@ /*- * SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB * - * Copyright (c) 2015 - 2022 Intel Corporation + * Copyright (c) 2015 - 2023 Intel Corporation * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU @@ -275,7 +275,8 @@ if (qp->uk_attrs->hw_rev == IRDMA_GEN_1 && wqe_quanta == 1 && (IRDMA_RING_CURRENT_HEAD(qp->sq_ring) & 1)) { wqe_0 = qp->sq_base[IRDMA_RING_CURRENT_HEAD(qp->sq_ring)].elem; - wqe_0[3] = htole64(FIELD_PREP(IRDMAQPSQ_VALID, !qp->swqe_polarity)); + wqe_0[3] = htole64(FIELD_PREP(IRDMAQPSQ_VALID, + qp->swqe_polarity ? 0 : 1)); } qp->sq_wrtrk_array[*wqe_idx].wrid = info->wr_id; qp->sq_wrtrk_array[*wqe_idx].wr_len = total_size; @@ -683,8 +684,8 @@ * @polarity: polarity of wqe valid bit */ static void -irdma_copy_inline_data(u8 *wqe, struct irdma_sge *sge_list, u32 num_sges, - u8 polarity) +irdma_copy_inline_data(u8 *wqe, struct irdma_sge *sge_list, + u32 num_sges, u8 polarity) { u8 inline_valid = polarity << IRDMA_INLINE_VALID_S; u32 quanta_bytes_remaining = 8; @@ -1173,7 +1174,7 @@ u32 end_idx) { __le64 *dst_wqe, *src_wqe; - u32 wqe_idx; + u32 wqe_idx = 0; u8 wqe_quanta = qp->rq_wqe_size_multiplier; bool flip_polarity; u64 val; @@ -1480,7 +1481,8 @@ sw_wqe = qp->sq_base[tail].elem; get_64bit_val(sw_wqe, IRDMA_BYTE_24, &wqe_qword); - info->op_type = (u8)FIELD_GET(IRDMAQPSQ_OPCODE, wqe_qword); + info->op_type = (u8)FIELD_GET(IRDMAQPSQ_OPCODE, + wqe_qword); IRDMA_RING_SET_TAIL(qp->sq_ring, tail + qp->sq_wrtrk_array[tail].quanta); if (info->op_type != IRDMAQP_OP_NOP) { @@ -1834,6 +1836,9 @@ if (polarity != temp) break; + /* Ensure CQE contents are read after valid bit is checked */ + udma_from_device_barrier(); + get_64bit_val(cqe, IRDMA_BYTE_8, &comp_ctx); if ((void *)(irdma_uintptr) comp_ctx == q) set_64bit_val(cqe, IRDMA_BYTE_8, 0); @@ -1845,48 +1850,6 @@ return 0; } -/** - * irdma_nop - post a nop - * @qp: hw qp ptr - * @wr_id: work request id - * @signaled: signaled for completion - * @post_sq: ring doorbell - */ -int -irdma_nop(struct irdma_qp_uk *qp, u64 wr_id, bool signaled, bool post_sq) -{ - __le64 *wqe; - u64 hdr; - u32 wqe_idx; - struct irdma_post_sq_info info = {0}; - u16 quanta = IRDMA_QP_WQE_MIN_QUANTA; - - info.push_wqe = qp->push_db ? true : false; - info.wr_id = wr_id; - wqe = irdma_qp_get_next_send_wqe(qp, &wqe_idx, &quanta, 0, &info); - if (!wqe) - return ENOSPC; - - set_64bit_val(wqe, IRDMA_BYTE_0, 0); - set_64bit_val(wqe, IRDMA_BYTE_8, 0); - set_64bit_val(wqe, IRDMA_BYTE_16, 0); - - hdr = FIELD_PREP(IRDMAQPSQ_OPCODE, IRDMAQP_OP_NOP) | - FIELD_PREP(IRDMAQPSQ_SIGCOMPL, signaled) | - FIELD_PREP(IRDMAQPSQ_VALID, qp->swqe_polarity); - - udma_to_device_barrier(); /* make sure WQE is populated before valid bit is set */ - - set_64bit_val(wqe, IRDMA_BYTE_24, hdr); - - if (info.push_wqe) - irdma_qp_push_wqe(qp, wqe, quanta, wqe_idx, post_sq); - else if (post_sq) - irdma_uk_qp_post_wr(qp); - - return 0; -} - /** * irdma_fragcnt_to_quanta_sq - calculate quanta based on fragment count for SQ * @frag_cnt: number of fragments diff --git a/contrib/ofed/libirdma/irdma_umain.c b/contrib/ofed/libirdma/irdma_umain.c --- a/contrib/ofed/libirdma/irdma_umain.c +++ b/contrib/ofed/libirdma/irdma_umain.c @@ -1,7 +1,7 @@ /*- * SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB * - * Copyright (c) 2021 - 2022 Intel Corporation + * Copyright (c) 2021 - 2023 Intel Corporation * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU @@ -49,7 +49,7 @@ /** * Driver version */ -char libirdma_version[] = "1.1.11-k"; +char libirdma_version[] = "1.2.17-k"; unsigned int irdma_dbg; diff --git a/contrib/ofed/libirdma/irdma_user.h b/contrib/ofed/libirdma/irdma_user.h --- a/contrib/ofed/libirdma/irdma_user.h +++ b/contrib/ofed/libirdma/irdma_user.h @@ -1,7 +1,7 @@ /*- * SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB * - * Copyright (c) 2015 - 2022 Intel Corporation + * Copyright (c) 2015 - 2023 Intel Corporation * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU @@ -572,7 +572,6 @@ u32 inline_data, u8 *shift); int irdma_get_sqdepth(struct irdma_uk_attrs *uk_attrs, u32 sq_size, u8 shift, u32 *sqdepth); int irdma_get_rqdepth(struct irdma_uk_attrs *uk_attrs, u32 rq_size, u8 shift, u32 *rqdepth); -int irdma_get_srqdepth(struct irdma_uk_attrs *uk_attrs, u32 srq_size, u8 shift, u32 *srqdepth); void irdma_qp_push_wqe(struct irdma_qp_uk *qp, __le64 *wqe, u16 quanta, u32 wqe_idx, bool post_sq); void irdma_clr_wqes(struct irdma_qp_uk *qp, u32 qp_wqe_idx); diff --git a/contrib/ofed/libirdma/irdma_uverbs.c b/contrib/ofed/libirdma/irdma_uverbs.c --- a/contrib/ofed/libirdma/irdma_uverbs.c +++ b/contrib/ofed/libirdma/irdma_uverbs.c @@ -1,7 +1,7 @@ /*- * SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB * - * Copyright (C) 2019 - 2022 Intel Corporation + * Copyright (C) 2019 - 2023 Intel Corporation * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU @@ -266,11 +266,13 @@ if (!mw) return NULL; - if (ibv_cmd_alloc_mw(pd, type, mw, &cmd, sizeof(cmd), &resp, - sizeof(resp))) { + err = ibv_cmd_alloc_mw(pd, type, mw, &cmd, sizeof(cmd), &resp, + sizeof(resp)); + if (err) { printf("%s: Failed to alloc memory window\n", __func__); free(mw); + errno = err; return NULL; } @@ -446,8 +448,10 @@ if (!iwucq) return NULL; - if (pthread_spin_init(&iwucq->lock, PTHREAD_PROCESS_PRIVATE)) { + ret = pthread_spin_init(&iwucq->lock, PTHREAD_PROCESS_PRIVATE); + if (ret) { free(iwucq); + errno = ret; return NULL; } @@ -464,8 +468,10 @@ iwucq->buf_size = total_size; info.cq_base = irdma_alloc_hw_buf(total_size); - if (!info.cq_base) + if (!info.cq_base) { + ret = ENOMEM; goto err_cq_base; + } memset(info.cq_base, 0, total_size); reg_mr_cmd.reg_type = IRDMA_MEMREG_TYPE_CQ; @@ -476,17 +482,17 @@ IBV_ACCESS_LOCAL_WRITE, &iwucq->vmr.ibv_mr, ®_mr_cmd.ibv_cmd, sizeof(reg_mr_cmd), ®_mr_resp, sizeof(reg_mr_resp)); - if (ret) { - errno = ret; + if (ret) goto err_dereg_mr; - } iwucq->vmr.ibv_mr.pd = &iwvctx->iwupd->ibv_pd; if (uk_attrs->feature_flags & IRDMA_FEATURE_CQ_RESIZE) { info.shadow_area = irdma_alloc_hw_buf(IRDMA_DB_SHADOW_AREA_SIZE); - if (!info.shadow_area) + if (!info.shadow_area) { + ret = ENOMEM; goto err_alloc_shadow; + } memset(info.shadow_area, 0, IRDMA_DB_SHADOW_AREA_SIZE); reg_mr_shadow_cmd.reg_type = IRDMA_MEMREG_TYPE_CQ; @@ -499,7 +505,6 @@ ®_mr_shadow_resp, sizeof(reg_mr_shadow_resp)); if (ret) { irdma_free_hw_buf(info.shadow_area, IRDMA_DB_SHADOW_AREA_SIZE); - errno = ret; goto err_alloc_shadow; } @@ -517,10 +522,8 @@ &cmd.ibv_cmd, sizeof(cmd.ibv_cmd), sizeof(cmd), &resp.ibv_resp, sizeof(resp.ibv_resp), sizeof(resp)); attr_ex->cqe = ncqe; - if (ret) { - errno = ret; + if (ret) goto err_create_cq; - } if (ext_cq) irdma_ibvcq_ex_fill_priv_funcs(iwucq, attr_ex); @@ -548,6 +551,7 @@ free(iwucq); + errno = ret; return NULL; } @@ -1560,7 +1564,8 @@ memset(iwuqp, 0, sizeof(*iwuqp)); - if (pthread_spin_init(&iwuqp->lock, PTHREAD_PROCESS_PRIVATE)) + status = pthread_spin_init(&iwuqp->lock, PTHREAD_PROCESS_PRIVATE); + if (status) goto err_free_qp; info.sq_size = info.sq_depth >> info.sq_shift; @@ -1575,35 +1580,37 @@ } iwuqp->recv_sges = calloc(attr->cap.max_recv_sge, sizeof(*iwuqp->recv_sges)); - if (!iwuqp->recv_sges) + if (!iwuqp->recv_sges) { + status = errno; /* preserve errno */ goto err_destroy_lock; + } info.wqe_alloc_db = (u32 *)iwvctx->db; info.legacy_mode = iwvctx->legacy_mode; info.sq_wrtrk_array = calloc(info.sq_depth, sizeof(*info.sq_wrtrk_array)); - if (!info.sq_wrtrk_array) + if (!info.sq_wrtrk_array) { + status = errno; /* preserve errno */ goto err_free_rsges; + } info.rq_wrid_array = calloc(info.rq_depth, sizeof(*info.rq_wrid_array)); - if (!info.rq_wrid_array) + if (!info.rq_wrid_array) { + status = errno; /* preserve errno */ goto err_free_sq_wrtrk; + } iwuqp->sq_sig_all = attr->sq_sig_all; iwuqp->qp_type = attr->qp_type; status = irdma_vmapped_qp(iwuqp, pd, attr, &info, iwvctx->legacy_mode); - if (status) { - errno = status; + if (status) goto err_free_rq_wrid; - } iwuqp->qp.back_qp = iwuqp; iwuqp->qp.lock = &iwuqp->lock; status = irdma_uk_qp_init(&iwuqp->qp, &info); - if (status) { - errno = status; + if (status) goto err_free_vmap_qp; - } attr->cap.max_send_wr = (info.sq_depth - IRDMA_SQ_RSVD) >> info.sq_shift; attr->cap.max_recv_wr = (info.rq_depth - IRDMA_RQ_RSVD) >> info.rq_shift; @@ -1625,6 +1632,7 @@ printf("%s: failed to create QP\n", __func__); free(iwuqp); + errno = status; return NULL; } @@ -2081,11 +2089,10 @@ struct irdma_ucreate_ah_resp resp = {}; int err; - err = ibv_query_gid(ibpd->context, attr->port_num, attr->grh.sgid_index, - &sgid); - if (err) { + if (ibv_query_gid(ibpd->context, attr->port_num, attr->grh.sgid_index, + &sgid)) { fprintf(stderr, "irdma: Error from ibv_query_gid.\n"); - errno = err; + errno = ENOENT; return NULL; } diff --git a/contrib/ofed/libirdma/osdep.h b/contrib/ofed/libirdma/osdep.h --- a/contrib/ofed/libirdma/osdep.h +++ b/contrib/ofed/libirdma/osdep.h @@ -1,7 +1,7 @@ /*- * SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB * - * Copyright (c) 2021 - 2022 Intel Corporation + * Copyright (c) 2021 - 2023 Intel Corporation * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU @@ -46,7 +46,6 @@ #include #include -#define ATOMIC atomic_t #define IOMEM #define IRDMA_NTOHL(a) ntohl(a) #define IRDMA_NTOHS(a) ntohs(a) @@ -80,23 +79,13 @@ #define STATS_TIMER_DELAY 60000 /* a couple of linux size defines */ -#define SZ_128 128 -#define SZ_2K SZ_128 * 16 -#define SZ_1G (SZ_1K * SZ_1K * SZ_1K) -#define SPEED_1000 1000 -#define SPEED_10000 10000 -#define SPEED_20000 20000 -#define SPEED_25000 25000 -#define SPEED_40000 40000 -#define SPEED_100000 100000 - #define BIT_ULL(a) (1ULL << (a)) #define min(a, b) ((a) > (b) ? (b) : (a)) #ifndef likely -#define likely(x) __builtin_expect((x), 1) +#define likely(x) __builtin_expect((x), 1) #endif #ifndef unlikely -#define unlikely(x) __builtin_expect((x), 0) +#define unlikely(x) __builtin_expect((x), 0) #endif #define __aligned_u64 uint64_t __aligned(8) @@ -112,7 +101,7 @@ #define irdma_print(S, ...) printf("%s:%d "S, __FUNCTION__, __LINE__, ##__VA_ARGS__) #define irdma_debug_buf(dev, mask, desc, buf, size) \ do { \ - u32 i; \ + u32 i; \ if (!((mask) & (dev)->debug_mask)) { \ break; \ } \ @@ -122,20 +111,20 @@ irdma_debug(dev, mask, "index %03d val: %016lx\n", i, ((unsigned long *)(buf))[i / 8]); \ } while(0) -#define irdma_debug(h, m, s, ...) \ -do { \ - if (!(h)) { \ - if ((m) == IRDMA_DEBUG_INIT) \ +#define irdma_debug(h, m, s, ...) \ +do { \ + if (!(h)) { \ + if ((m) == IRDMA_DEBUG_INIT) \ printf("irdma INIT " s, ##__VA_ARGS__); \ - } else if (((m) & (h)->debug_mask)) { \ - printf("irdma " s, ##__VA_ARGS__); \ - } \ + } else if (((m) & (h)->debug_mask)) { \ + printf("irdma " s, ##__VA_ARGS__); \ + } \ } while (0) extern unsigned int irdma_dbg; -#define libirdma_debug(fmt, args...) \ -do { \ - if (irdma_dbg) \ - printf("libirdma-%s: " fmt, __func__, ##args); \ +#define libirdma_debug(fmt, args...) \ +do { \ + if (irdma_dbg) \ + printf("libirdma-%s: " fmt, __func__, ##args); \ } while (0) #define irdma_dev_err(ibdev, fmt, ...) \ pr_err("%s:%s:%d ERR "fmt, (ibdev)->name, __func__, __LINE__, ##__VA_ARGS__) @@ -144,18 +133,6 @@ #define irdma_dev_info(a, b, ...) printf(b, ##__VA_ARGS__) #define irdma_pr_warn printf -#define dump_struct(s, sz, name) \ -do { \ - unsigned char *a; \ - printf("%s %u", (name), (unsigned int)(sz)); \ - for (a = (unsigned char*)(s); a < (unsigned char *)(s) + (sz) ; a ++) { \ - if ((u64)a % 8 == 0) \ - printf("\n%p ", a); \ - printf("%2x ", *a); \ - } \ - printf("\n"); \ -}while(0) - /* * debug definition end */ @@ -179,13 +156,6 @@ #define rt_tos2priority(tos) (tos >> 5) #define ah_attr_to_dmac(attr) ((attr).dmac) -#define kc_ib_modify_qp_is_ok(cur_state, next_state, type, mask, ll) \ - ib_modify_qp_is_ok(cur_state, next_state, type, mask) -#define kc_typeq_ib_wr const -#define kc_ifp_find ip_ifp_find -#define kc_ifp6_find ip6_ifp_find -#define kc_rdma_gid_attr_network_type(sgid_attr, gid_type, gid) \ - ib_gid_to_network_type(gid_type, gid) #define irdma_del_timer_compat(tt) del_timer((tt)) #define IRDMA_TAILQ_FOREACH CK_STAILQ_FOREACH #define IRDMA_TAILQ_FOREACH_SAFE CK_STAILQ_FOREACH_SAFE @@ -193,36 +163,36 @@ static inline void db_wr32(__u32 val, __u32 *wqe_word) { - *wqe_word = val; + *wqe_word = val; } void *hw_to_dev(struct irdma_hw *hw); struct irdma_dma_mem { - void *va; - u64 pa; + void *va; + u64 pa; bus_dma_tag_t tag; bus_dmamap_t map; bus_dma_segment_t seg; bus_size_t size; - int nseg; - int flags; + int nseg; + int flags; }; struct irdma_virt_mem { - void *va; - u32 size; + void *va; + u32 size; }; #ifndef verbs_mr enum ibv_mr_type { - IBV_MR_TYPE_MR, - IBV_MR_TYPE_NULL_MR, + IBV_MR_TYPE_MR, + IBV_MR_TYPE_NULL_MR, }; struct verbs_mr { - struct ibv_mr ibv_mr; - enum ibv_mr_type mr_type; + struct ibv_mr ibv_mr; + enum ibv_mr_type mr_type; int access; }; #define verbs_get_mr(mr) container_of((mr), struct verbs_mr, ibv_mr) diff --git a/sys/dev/ice/ice_rdma.h b/sys/dev/ice/ice_rdma.h --- a/sys/dev/ice/ice_rdma.h +++ b/sys/dev/ice/ice_rdma.h @@ -65,7 +65,7 @@ * considered stable. */ #define ICE_RDMA_MAJOR_VERSION 1 -#define ICE_RDMA_MINOR_VERSION 0 +#define ICE_RDMA_MINOR_VERSION 1 #define ICE_RDMA_PATCH_VERSION 0 /** @@ -274,18 +274,19 @@ uint64_t baudrate; }; /* MTU change event */ - struct { - int mtu; - }; + int mtu; /* * TC/QoS/DCB change event - * RESET event use prep variable only * prep: if true, this is a pre-event, post-event otherwise */ struct { struct ice_qos_params port_qos; bool prep; }; + /* + * CRIT_ERR event + */ + uint32_t oicr_reg; }; }; diff --git a/sys/dev/ice/ice_rdma.c b/sys/dev/ice/ice_rdma.c --- a/sys/dev/ice/ice_rdma.c +++ b/sys/dev/ice/ice_rdma.c @@ -122,11 +122,10 @@ { struct ice_softc *sc = ice_rdma_peer_to_sc(peer); - /* - * Request that the driver re-initialize by bringing the interface - * down and up. - */ - ice_request_stack_reinit(sc); + /* Tell the base driver that RDMA is requesting a PFR */ + ice_set_state(&sc->state, ICE_STATE_RESET_PFR_REQ); + + /* XXX: Base driver will notify RDMA when it's done */ return (0); } @@ -332,6 +331,7 @@ switch(req->type) { case ICE_RDMA_EVENT_RESET: + ice_rdma_pf_reset(peer); break; case ICE_RDMA_EVENT_QSET_REGISTER: ice_rdma_qset_register_request(peer, &req->res); @@ -866,3 +866,52 @@ IRDMA_EVENT_HANDLER(peer, &event); sx_xunlock(&ice_rdma.mtx); } + +/** + * ice_rdma_notify_pe_intr - notify irdma on incoming interrupts regarding PE + * @sc: the ice driver softc + * @oicr: interrupt cause + * + * Pass the information about received interrupt to RDMA driver if it was + * relating to PE. Specifically PE_CRITERR and HMC_ERR. + * The irdma driver shall decide what should be done upon these interrupts. + */ +void +ice_rdma_notify_pe_intr(struct ice_softc *sc, uint32_t oicr) +{ + struct ice_rdma_peer *peer = &sc->rdma_entry.peer; + struct ice_rdma_event event; + + memset(&event, 0, sizeof(struct ice_rdma_event)); + event.type = ICE_RDMA_EVENT_CRIT_ERR; + event.oicr_reg = oicr; + + sx_xlock(&ice_rdma.mtx); + if (sc->rdma_entry.attached && ice_rdma.registered) + IRDMA_EVENT_HANDLER(peer, &event); + sx_xunlock(&ice_rdma.mtx); +} + +/** + * ice_rdma_notify_reset - notify irdma on incoming pf-reset + * @sc: the ice driver softc + * + * Inform irdma driver of an incoming PF reset. + * The irdma driver shall set its state to reset, and avoid using CQP + * anymore. Next step should be to call ice_rdma_pf_stop in order to + * remove resources. + */ +void +ice_rdma_notify_reset(struct ice_softc *sc) +{ + struct ice_rdma_peer *peer = &sc->rdma_entry.peer; + struct ice_rdma_event event; + + memset(&event, 0, sizeof(struct ice_rdma_event)); + event.type = ICE_RDMA_EVENT_RESET; + + sx_xlock(&ice_rdma.mtx); + if (sc->rdma_entry.attached && ice_rdma.registered) + IRDMA_EVENT_HANDLER(peer, &event); + sx_xunlock(&ice_rdma.mtx); +} diff --git a/sys/dev/ice/ice_rdma_internal.h b/sys/dev/ice/ice_rdma_internal.h --- a/sys/dev/ice/ice_rdma_internal.h +++ b/sys/dev/ice/ice_rdma_internal.h @@ -99,4 +99,6 @@ void ice_rdma_link_change(struct ice_softc *sc, int linkstate, uint64_t baudrate); void ice_rdma_notify_dcb_qos_change(struct ice_softc *sc); void ice_rdma_dcb_qos_update(struct ice_softc *sc, struct ice_port_info *pi); +void ice_rdma_notify_pe_intr(struct ice_softc *sc, uint32_t oicr); +void ice_rdma_notify_reset(struct ice_softc *sc); #endif diff --git a/sys/dev/ice/if_ice_iflib.c b/sys/dev/ice/if_ice_iflib.c --- a/sys/dev/ice/if_ice_iflib.c +++ b/sys/dev/ice/if_ice_iflib.c @@ -1279,9 +1279,11 @@ ice_set_state(&sc->state, ICE_STATE_RESET_PFR_REQ); } - if (oicr & PFINT_OICR_PE_CRITERR_M) { - device_printf(dev, "Critical Protocol Engine Error detected!\n"); - ice_set_state(&sc->state, ICE_STATE_RESET_PFR_REQ); + if (oicr & (PFINT_OICR_PE_CRITERR_M | PFINT_OICR_HMC_ERR_M)) { + if (oicr & PFINT_OICR_HMC_ERR_M) + /* Log the HMC errors */ + ice_log_hmc_error(hw, dev); + ice_rdma_notify_pe_intr(sc, oicr); } if (oicr & PFINT_OICR_PCI_EXCEPTION_M) { @@ -1289,11 +1291,6 @@ ice_set_state(&sc->state, ICE_STATE_RESET_PFR_REQ); } - if (oicr & PFINT_OICR_HMC_ERR_M) { - /* Log the HMC errors, but don't disable the interrupt cause */ - ice_log_hmc_error(hw, dev); - } - return (FILTER_SCHEDULE_THREAD); } @@ -2299,6 +2296,8 @@ if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE)) return; + /* inform the RDMA client */ + ice_rdma_notify_reset(sc); /* stop the RDMA client */ ice_rdma_pf_stop(sc); diff --git a/sys/dev/irdma/fbsd_kcompat.h b/sys/dev/irdma/fbsd_kcompat.h --- a/sys/dev/irdma/fbsd_kcompat.h +++ b/sys/dev/irdma/fbsd_kcompat.h @@ -1,7 +1,7 @@ /*- * SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB * - * Copyright (c) 2021 - 2022 Intel Corporation + * Copyright (c) 2021 - 2023 Intel Corporation * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU @@ -63,21 +63,12 @@ ibdev.dma_device = (dev) #define set_max_sge(props, rf) \ ((props)->max_sge = (rf)->sc_dev.hw_attrs.uk_attrs.max_hw_wq_frags) -#define kc_set_props_ip_gid_caps(props) \ - ((props)->port_cap_flags |= IB_PORT_IP_BASED_GIDS) #define rdma_query_gid(ibdev, port, index, gid) \ ib_get_cached_gid(ibdev, port, index, gid, NULL) #define kmap(pg) page_address(pg) #define kmap_local_page(pg) page_address(pg) #define kunmap(pg) #define kunmap_local(pg) -#if __FreeBSD_version >= 1400026 -#define kc_free_lsmm_dereg_mr(iwdev, iwqp) \ - ((iwdev)->ibdev.dereg_mr((iwqp)->lsmm_mr, NULL)) -#else -#define kc_free_lsmm_dereg_mr(iwdev, iwqp) \ - ((iwdev)->ibdev.dereg_mr((iwqp)->lsmm_mr)) -#endif #define IB_UVERBS_CQ_FLAGS_TIMESTAMP_COMPLETION IB_CQ_FLAGS_TIMESTAMP_COMPLETION #if __FreeBSD_version < 1400026 @@ -100,6 +91,7 @@ struct irdma_tunable_info { struct sysctl_ctx_list irdma_sysctl_ctx; struct sysctl_oid *irdma_sysctl_tree; + struct sysctl_oid *sws_sysctl_tree; char drv_ver[IRDMA_VER_LEN]; u8 roce_ena; }; @@ -211,8 +203,6 @@ int kc_irdma_set_roce_cm_info(struct irdma_qp *iwqp, struct ib_qp_attr *attr, u16 *vlan_id); -struct irdma_device *kc_irdma_get_device(struct ifnet *netdev); -void kc_irdma_put_device(struct irdma_device *iwdev); void kc_set_loc_seq_num_mss(struct irdma_cm_node *cm_node); u16 kc_rdma_get_udp_sport(u32 fl, u32 lqpn, u32 rqpn); @@ -230,6 +220,8 @@ int irdma_addr_resolve_neigh_ipv6(struct irdma_cm_node *cm_node, u32 *dest, int arpindex); void irdma_dcqcn_tunables_init(struct irdma_pci_f *rf); +void irdma_sysctl_settings(struct irdma_pci_f *rf); +void irdma_sw_stats_tunables_init(struct irdma_pci_f *rf); u32 irdma_create_stag(struct irdma_device *iwdev); void irdma_free_stag(struct irdma_device *iwdev, u32 stag); diff --git a/sys/dev/irdma/fbsd_kcompat.c b/sys/dev/irdma/fbsd_kcompat.c --- a/sys/dev/irdma/fbsd_kcompat.c +++ b/sys/dev/irdma/fbsd_kcompat.c @@ -1,7 +1,7 @@ /*- * SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB * - * Copyright (c) 2021 - 2022 Intel Corporation + * Copyright (c) 2021 - 2023 Intel Corporation * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU @@ -274,12 +274,15 @@ { struct ifnet *ifp = iwdev->netdev; struct ifnet *ifv; + struct epoch_tracker et; int i; irdma_add_ipv4_addr(iwdev, ifp); irdma_add_ipv6_addr(iwdev, ifp); for (i = 0; if_getvlantrunk(ifp) != NULL && i < VLAN_N_VID; ++i) { + NET_EPOCH_ENTER(et); ifv = VLAN_DEVAT(ifp, i); + NET_EPOCH_EXIT(et); if (!ifv) continue; irdma_add_ipv4_addr(iwdev, ifv); @@ -362,7 +365,8 @@ struct nhop_object *nh; if (dst_sin->sa_family == AF_INET6) - nh = fib6_lookup(RT_DEFAULT_FIB, &((struct sockaddr_in6 *)dst_sin)->sin6_addr, 0, NHR_NONE, 0); + nh = fib6_lookup(RT_DEFAULT_FIB, &((struct sockaddr_in6 *)dst_sin)->sin6_addr, + ((struct sockaddr_in6 *)dst_sin)->sin6_scope_id, NHR_NONE, 0); else nh = fib4_lookup(RT_DEFAULT_FIB, ((struct sockaddr_in *)dst_sin)->sin_addr, 0, NHR_NONE, 0); if (!nh || (nh->nh_ifp != netdev && @@ -592,6 +596,188 @@ return 0; } +enum irdma_cqp_stats_info { + IRDMA_CQP_REQ_CMDS = 28, + IRDMA_CQP_CMPL_CMDS = 29 +}; + +static int +irdma_sysctl_cqp_stats(SYSCTL_HANDLER_ARGS) +{ + struct irdma_sc_cqp *cqp = (struct irdma_sc_cqp *)arg1; + char rslt[192] = "no cqp available yet"; + int rslt_size = sizeof(rslt) - 1; + int option = (int)arg2; + + if (!cqp) { + return sysctl_handle_string(oidp, rslt, sizeof(rslt), req); + } + + snprintf(rslt, sizeof(rslt), ""); + switch (option) { + case IRDMA_CQP_REQ_CMDS: + snprintf(rslt, rslt_size, "%lu", cqp->requested_ops); + break; + case IRDMA_CQP_CMPL_CMDS: + snprintf(rslt, rslt_size, "%lu", atomic64_read(&cqp->completed_ops)); + break; + } + + return sysctl_handle_string(oidp, rslt, sizeof(rslt), req); +} + +struct irdma_sw_stats_tunable_info { + u8 op_type; + const char name[32]; + const char desc[32]; + uintptr_t value; +}; + +static const struct irdma_sw_stats_tunable_info irdma_sws_list[] = { + {IRDMA_OP_CEQ_DESTROY, "ceq_destroy", "ceq_destroy", 0}, + {IRDMA_OP_AEQ_DESTROY, "aeq_destroy", "aeq_destroy", 0}, + {IRDMA_OP_DELETE_ARP_CACHE_ENTRY, "delete_arp_cache_entry", + "delete_arp_cache_entry", 0}, + {IRDMA_OP_MANAGE_APBVT_ENTRY, "manage_apbvt_entry", + "manage_apbvt_entry", 0}, + {IRDMA_OP_CEQ_CREATE, "ceq_create", "ceq_create", 0}, + {IRDMA_OP_AEQ_CREATE, "aeq_create", "aeq_create", 0}, + {IRDMA_OP_MANAGE_QHASH_TABLE_ENTRY, "manage_qhash_table_entry", + "manage_qhash_table_entry", 0}, + {IRDMA_OP_QP_MODIFY, "qp_modify", "qp_modify", 0}, + {IRDMA_OP_QP_UPLOAD_CONTEXT, "qp_upload_context", "qp_upload_context", + 0}, + {IRDMA_OP_CQ_CREATE, "cq_create", "cq_create", 0}, + {IRDMA_OP_CQ_DESTROY, "cq_destroy", "cq_destroy", 0}, + {IRDMA_OP_QP_CREATE, "qp_create", "qp_create", 0}, + {IRDMA_OP_QP_DESTROY, "qp_destroy", "qp_destroy", 0}, + {IRDMA_OP_ALLOC_STAG, "alloc_stag", "alloc_stag", 0}, + {IRDMA_OP_MR_REG_NON_SHARED, "mr_reg_non_shared", "mr_reg_non_shared", + 0}, + {IRDMA_OP_DEALLOC_STAG, "dealloc_stag", "dealloc_stag", 0}, + {IRDMA_OP_MW_ALLOC, "mw_alloc", "mw_alloc", 0}, + {IRDMA_OP_QP_FLUSH_WQES, "qp_flush_wqes", "qp_flush_wqes", 0}, + {IRDMA_OP_ADD_ARP_CACHE_ENTRY, "add_arp_cache_entry", + "add_arp_cache_entry", 0}, + {IRDMA_OP_MANAGE_PUSH_PAGE, "manage_push_page", "manage_push_page", 0}, + {IRDMA_OP_UPDATE_PE_SDS, "update_pe_sds", "update_pe_sds", 0}, + {IRDMA_OP_MANAGE_HMC_PM_FUNC_TABLE, "manage_hmc_pm_func_table", + "manage_hmc_pm_func_table", 0}, + {IRDMA_OP_SUSPEND, "suspend", "suspend", 0}, + {IRDMA_OP_RESUME, "resume", "resume", 0}, + {IRDMA_OP_MANAGE_VCHNL_REQ_PBLE_BP, "manage_vchnl_req_pble_bp", + "manage_vchnl_req_pble_bp", 0}, + {IRDMA_OP_QUERY_FPM_VAL, "query_fpm_val", "query_fpm_val", 0}, + {IRDMA_OP_COMMIT_FPM_VAL, "commit_fpm_val", "commit_fpm_val", 0}, + {IRDMA_OP_AH_CREATE, "ah_create", "ah_create", 0}, + {IRDMA_OP_AH_MODIFY, "ah_modify", "ah_modify", 0}, + {IRDMA_OP_AH_DESTROY, "ah_destroy", "ah_destroy", 0}, + {IRDMA_OP_MC_CREATE, "mc_create", "mc_create", 0}, + {IRDMA_OP_MC_DESTROY, "mc_destroy", "mc_destroy", 0}, + {IRDMA_OP_MC_MODIFY, "mc_modify", "mc_modify", 0}, + {IRDMA_OP_STATS_ALLOCATE, "stats_allocate", "stats_allocate", 0}, + {IRDMA_OP_STATS_FREE, "stats_free", "stats_free", 0}, + {IRDMA_OP_STATS_GATHER, "stats_gather", "stats_gather", 0}, + {IRDMA_OP_WS_ADD_NODE, "ws_add_node", "ws_add_node", 0}, + {IRDMA_OP_WS_MODIFY_NODE, "ws_modify_node", "ws_modify_node", 0}, + {IRDMA_OP_WS_DELETE_NODE, "ws_delete_node", "ws_delete_node", 0}, + {IRDMA_OP_WS_FAILOVER_START, "ws_failover_start", "ws_failover_start", + 0}, + {IRDMA_OP_WS_FAILOVER_COMPLETE, "ws_failover_complete", + "ws_failover_complete", 0}, + {IRDMA_OP_SET_UP_MAP, "set_up_map", "set_up_map", 0}, + {IRDMA_OP_GEN_AE, "gen_ae", "gen_ae", 0}, + {IRDMA_OP_QUERY_RDMA_FEATURES, "query_rdma_features", + "query_rdma_features", 0}, + {IRDMA_OP_ALLOC_LOCAL_MAC_ENTRY, "alloc_local_mac_entry", + "alloc_local_mac_entry", 0}, + {IRDMA_OP_ADD_LOCAL_MAC_ENTRY, "add_local_mac_entry", + "add_local_mac_entry", 0}, + {IRDMA_OP_DELETE_LOCAL_MAC_ENTRY, "delete_local_mac_entry", + "delete_local_mac_entry", 0}, + {IRDMA_OP_CQ_MODIFY, "cq_modify", "cq_modify", 0} +}; + +static const struct irdma_sw_stats_tunable_info irdma_cmcs_list[] = { + {0, "cm_nodes_created", "cm_nodes_created", + offsetof(struct irdma_cm_core, stats_nodes_created)}, + {0, "cm_nodes_destroyed", "cm_nodes_destroyed", + offsetof(struct irdma_cm_core, stats_nodes_destroyed)}, + {0, "cm_listen_created", "cm_listen_created", + offsetof(struct irdma_cm_core, stats_listen_created)}, + {0, "cm_listen_destroyed", "cm_listen_destroyed", + offsetof(struct irdma_cm_core, stats_listen_destroyed)}, + {0, "cm_listen_nodes_created", "cm_listen_nodes_created", + offsetof(struct irdma_cm_core, stats_listen_nodes_created)}, + {0, "cm_listen_nodes_destroyed", "cm_listen_nodes_destroyed", + offsetof(struct irdma_cm_core, stats_listen_nodes_destroyed)}, + {0, "cm_lpbs", "cm_lpbs", offsetof(struct irdma_cm_core, stats_lpbs)}, + {0, "cm_accepts", "cm_accepts", offsetof(struct irdma_cm_core, + stats_accepts)}, + {0, "cm_rejects", "cm_rejects", offsetof(struct irdma_cm_core, + stats_rejects)}, + {0, "cm_connect_errs", "cm_connect_errs", + offsetof(struct irdma_cm_core, stats_connect_errs)}, + {0, "cm_passive_errs", "cm_passive_errs", + offsetof(struct irdma_cm_core, stats_passive_errs)}, + {0, "cm_pkt_retrans", "cm_pkt_retrans", offsetof(struct irdma_cm_core, + stats_pkt_retrans)}, + {0, "cm_backlog_drops", "cm_backlog_drops", + offsetof(struct irdma_cm_core, stats_backlog_drops)}, +}; + +static const struct irdma_sw_stats_tunable_info irdma_ilqs32_list[] = { + {0, "ilq_avail_buf_count", "ilq_avail_buf_count", + offsetof(struct irdma_puda_rsrc, avail_buf_count)}, + {0, "ilq_alloc_buf_count", "ilq_alloc_buf_count", + offsetof(struct irdma_puda_rsrc, alloc_buf_count)} +}; + +static const struct irdma_sw_stats_tunable_info irdma_ilqs_list[] = { + {0, "ilq_stats_buf_alloc_fail", "ilq_stats_buf_alloc_fail", + offsetof(struct irdma_puda_rsrc, stats_buf_alloc_fail)}, + {0, "ilq_stats_pkt_rcvd", "ilq_stats_pkt_rcvd", + offsetof(struct irdma_puda_rsrc, stats_pkt_rcvd)}, + {0, "ilq_stats_pkt_sent", "ilq_stats_pkt_sent", + offsetof(struct irdma_puda_rsrc, stats_pkt_sent)}, + {0, "ilq_stats_rcvd_pkt_err", "ilq_stats_rcvd_pkt_err", + offsetof(struct irdma_puda_rsrc, stats_rcvd_pkt_err)}, + {0, "ilq_stats_sent_pkt_q", "ilq_stats_sent_pkt_q", + offsetof(struct irdma_puda_rsrc, stats_sent_pkt_q)} +}; + +static const struct irdma_sw_stats_tunable_info irdma_ieqs32_list[] = { + {0, "ieq_avail_buf_count", "ieq_avail_buf_count", + offsetof(struct irdma_puda_rsrc, avail_buf_count)}, + {0, "ieq_alloc_buf_count", "ieq_alloc_buf_count", + offsetof(struct irdma_puda_rsrc, alloc_buf_count)} +}; + +static const struct irdma_sw_stats_tunable_info irdma_ieqs_list[] = { + {0, "ieq_stats_buf_alloc_fail", "ieq_stats_buf_alloc_fail", + offsetof(struct irdma_puda_rsrc, stats_buf_alloc_fail)}, + {0, "ieq_stats_pkt_rcvd", "ieq_stats_pkt_rcvd", + offsetof(struct irdma_puda_rsrc, stats_pkt_rcvd)}, + {0, "ieq_stats_pkt_sent", "ieq_stats_pkt_sent", + offsetof(struct irdma_puda_rsrc, stats_pkt_sent)}, + {0, "ieq_stats_rcvd_pkt_err", "ieq_stats_rcvd_pkt_err", + offsetof(struct irdma_puda_rsrc, stats_rcvd_pkt_err)}, + {0, "ieq_stats_sent_pkt_q", "ieq_stats_sent_pkt_q", + offsetof(struct irdma_puda_rsrc, stats_sent_pkt_q)}, + {0, "ieq_stats_bad_qp_id", "ieq_stats_bad_qp_id", + offsetof(struct irdma_puda_rsrc, stats_bad_qp_id)}, + {0, "ieq_fpdu_processed", "ieq_fpdu_processed", + offsetof(struct irdma_puda_rsrc, fpdu_processed)}, + {0, "ieq_bad_seq_num", "ieq_bad_seq_num", + offsetof(struct irdma_puda_rsrc, bad_seq_num)}, + {0, "ieq_crc_err", "ieq_crc_err", offsetof(struct irdma_puda_rsrc, + crc_err)}, + {0, "ieq_pmode_count", "ieq_pmode_count", + offsetof(struct irdma_puda_rsrc, pmode_count)}, + {0, "ieq_partials_handled", "ieq_partials_handled", + offsetof(struct irdma_puda_rsrc, partials_handled)}, +}; + /** * irdma_dcqcn_tunables_init - create tunables for dcqcn settings * @rf: RDMA PCI function @@ -638,7 +824,7 @@ rf->dcqcn_params.dcqcn_t = 0x37; SYSCTL_ADD_U16(&rf->tun_info.irdma_sysctl_ctx, irdma_sysctl_oid_list, OID_AUTO, "dcqcn_T", CTLFLAG_RDTUN, &rf->dcqcn_params.dcqcn_t, 0, - "set number of usecs that should elapse before increasing the CWND in DCQCN mode, default=0x37"); + "number of us to elapse before increasing the CWND in DCQCN mode, default=0x37"); rf->dcqcn_params.dcqcn_b = 0x249f0; SYSCTL_ADD_U32(&rf->tun_info.irdma_sysctl_ctx, irdma_sysctl_oid_list, @@ -664,6 +850,90 @@ "set minimum time between 2 consecutive rate reductions for a single flow, default=50"); } +/** + * irdma_sysctl_settings - sysctl runtime settings init + * @rf: RDMA PCI function + */ +void +irdma_sysctl_settings(struct irdma_pci_f *rf) +{ + struct sysctl_oid_list *irdma_sysctl_oid_list; + + irdma_sysctl_oid_list = SYSCTL_CHILDREN(rf->tun_info.irdma_sysctl_tree); + + SYSCTL_ADD_BOOL(&rf->tun_info.irdma_sysctl_ctx, irdma_sysctl_oid_list, + OID_AUTO, "upload_context", CTLFLAG_RWTUN, + &irdma_upload_context, 0, + "allow for generating QP's upload context, default=0"); +} + +void +irdma_sw_stats_tunables_init(struct irdma_pci_f *rf) +{ + struct sysctl_oid_list *sws_oid_list; + struct sysctl_ctx_list *irdma_ctx = &rf->tun_info.irdma_sysctl_ctx; + struct irdma_sc_dev *dev = &rf->sc_dev; + struct irdma_cm_core *cm_core = &rf->iwdev->cm_core; + struct irdma_puda_rsrc *ilq = rf->iwdev->vsi.ilq; + struct irdma_puda_rsrc *ieq = rf->iwdev->vsi.ieq; + u64 *ll_ptr; + u32 *l_ptr; + int cqp_stat_cnt = sizeof(irdma_sws_list) / sizeof(struct irdma_sw_stats_tunable_info); + int cmcore_stat_cnt = sizeof(irdma_cmcs_list) / sizeof(struct irdma_sw_stats_tunable_info); + int ilqs_stat_cnt = sizeof(irdma_ilqs_list) / sizeof(struct irdma_sw_stats_tunable_info); + int ilqs32_stat_cnt = sizeof(irdma_ilqs32_list) / sizeof(struct irdma_sw_stats_tunable_info); + int ieqs_stat_cnt = sizeof(irdma_ieqs_list) / sizeof(struct irdma_sw_stats_tunable_info); + int ieqs32_stat_cnt = sizeof(irdma_ieqs32_list) / sizeof(struct irdma_sw_stats_tunable_info); + int i; + + sws_oid_list = SYSCTL_CHILDREN(rf->tun_info.sws_sysctl_tree); + + for (i = 0; i < cqp_stat_cnt; ++i) { + SYSCTL_ADD_U64(irdma_ctx, sws_oid_list, OID_AUTO, + irdma_sws_list[i].name, CTLFLAG_RD, + &dev->cqp_cmd_stats[irdma_sws_list[i].op_type], + 0, irdma_sws_list[i].desc); + } + SYSCTL_ADD_PROC(irdma_ctx, sws_oid_list, OID_AUTO, + "req_cmds", CTLFLAG_RD | CTLTYPE_STRING, + dev->cqp, IRDMA_CQP_REQ_CMDS, irdma_sysctl_cqp_stats, "A", + "req_cmds"); + SYSCTL_ADD_PROC(irdma_ctx, sws_oid_list, OID_AUTO, + "cmpl_cmds", CTLFLAG_RD | CTLTYPE_STRING, + dev->cqp, IRDMA_CQP_CMPL_CMDS, irdma_sysctl_cqp_stats, "A", + "cmpl_cmds"); + for (i = 0; i < cmcore_stat_cnt; ++i) { + ll_ptr = (u64 *)((uintptr_t)cm_core + irdma_cmcs_list[i].value); + SYSCTL_ADD_U64(irdma_ctx, sws_oid_list, OID_AUTO, + irdma_cmcs_list[i].name, CTLFLAG_RD, ll_ptr, + 0, irdma_cmcs_list[i].desc); + } + for (i = 0; ilq && i < ilqs_stat_cnt; ++i) { + ll_ptr = (u64 *)((uintptr_t)ilq + irdma_ilqs_list[i].value); + SYSCTL_ADD_U64(irdma_ctx, sws_oid_list, OID_AUTO, + irdma_ilqs_list[i].name, CTLFLAG_RD, ll_ptr, + 0, irdma_ilqs_list[i].desc); + } + for (i = 0; ilq && i < ilqs32_stat_cnt; ++i) { + l_ptr = (u32 *)((uintptr_t)ilq + irdma_ilqs32_list[i].value); + SYSCTL_ADD_U32(irdma_ctx, sws_oid_list, OID_AUTO, + irdma_ilqs32_list[i].name, CTLFLAG_RD, l_ptr, + 0, irdma_ilqs32_list[i].desc); + } + for (i = 0; ieq && i < ieqs_stat_cnt; ++i) { + ll_ptr = (u64 *)((uintptr_t)ieq + irdma_ieqs_list[i].value); + SYSCTL_ADD_U64(irdma_ctx, sws_oid_list, OID_AUTO, + irdma_ieqs_list[i].name, CTLFLAG_RD, ll_ptr, + 0, irdma_ieqs_list[i].desc); + } + for (i = 0; ieq && i < ieqs32_stat_cnt; ++i) { + l_ptr = (u32 *)((uintptr_t)ieq + irdma_ieqs32_list[i].value); + SYSCTL_ADD_U32(irdma_ctx, sws_oid_list, OID_AUTO, + irdma_ieqs32_list[i].name, CTLFLAG_RD, l_ptr, + 0, irdma_ieqs32_list[i].desc); + } +} + /** * irdma_dmamap_cb - callback for bus_dmamap_load */ @@ -763,12 +1033,6 @@ return 0; } -inline void -irdma_prm_rem_bitmapmem(struct irdma_hw *hw, struct irdma_chunk *chunk) -{ - kfree(chunk->bitmapmem.va); -} - void irdma_cleanup_dead_qps(struct irdma_sc_vsi *vsi) { diff --git a/sys/dev/irdma/icrdma.c b/sys/dev/irdma/icrdma.c --- a/sys/dev/irdma/icrdma.c +++ b/sys/dev/irdma/icrdma.c @@ -1,7 +1,7 @@ /*- * SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB * - * Copyright (c) 2021 - 2022 Intel Corporation + * Copyright (c) 2021 - 2023 Intel Corporation * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU @@ -53,7 +53,7 @@ /** * Driver version */ -char irdma_driver_version[] = "1.1.11-k"; +char irdma_driver_version[] = "1.2.17-k"; /** * irdma_init_tunable - prepare tunables @@ -63,50 +63,56 @@ static void irdma_init_tunable(struct irdma_pci_f *rf, uint8_t pf_id) { - struct sysctl_oid_list *irdma_sysctl_oid_list; + struct sysctl_oid_list *irdma_oid_list; + struct irdma_tunable_info *t_info = &rf->tun_info; char pf_name[16]; snprintf(pf_name, 15, "irdma%d", pf_id); - sysctl_ctx_init(&rf->tun_info.irdma_sysctl_ctx); + sysctl_ctx_init(&t_info->irdma_sysctl_ctx); - rf->tun_info.irdma_sysctl_tree = SYSCTL_ADD_NODE(&rf->tun_info.irdma_sysctl_ctx, - SYSCTL_STATIC_CHILDREN(_dev), - OID_AUTO, pf_name, CTLFLAG_RD, - NULL, ""); + t_info->irdma_sysctl_tree = SYSCTL_ADD_NODE(&t_info->irdma_sysctl_ctx, + SYSCTL_STATIC_CHILDREN(_dev), + OID_AUTO, pf_name, + CTLFLAG_RD, NULL, ""); - irdma_sysctl_oid_list = SYSCTL_CHILDREN(rf->tun_info.irdma_sysctl_tree); + irdma_oid_list = SYSCTL_CHILDREN(t_info->irdma_sysctl_tree); + t_info->sws_sysctl_tree = SYSCTL_ADD_NODE(&t_info->irdma_sysctl_ctx, + irdma_oid_list, OID_AUTO, + "sw_stats", CTLFLAG_RD, + NULL, ""); /* * debug mask setting */ - SYSCTL_ADD_S32(&rf->tun_info.irdma_sysctl_ctx, irdma_sysctl_oid_list, + SYSCTL_ADD_S32(&t_info->irdma_sysctl_ctx, irdma_oid_list, OID_AUTO, "debug", CTLFLAG_RWTUN, &rf->sc_dev.debug_mask, 0, "irdma debug"); /* * RoCEv2/iWARP setting RoCEv2 the default mode */ - rf->tun_info.roce_ena = 1; - SYSCTL_ADD_U8(&rf->tun_info.irdma_sysctl_ctx, irdma_sysctl_oid_list, OID_AUTO, - "roce_enable", CTLFLAG_RDTUN, &rf->tun_info.roce_ena, 0, + t_info->roce_ena = 1; + SYSCTL_ADD_U8(&t_info->irdma_sysctl_ctx, irdma_oid_list, OID_AUTO, + "roce_enable", CTLFLAG_RDTUN, &t_info->roce_ena, 0, "RoCEv2 mode enable"); rf->protocol_used = IRDMA_IWARP_PROTOCOL_ONLY; - if (rf->tun_info.roce_ena == 1) + if (t_info->roce_ena == 1) rf->protocol_used = IRDMA_ROCE_PROTOCOL_ONLY; - else if (rf->tun_info.roce_ena != 0) + else if (t_info->roce_ena != 0) printf("%s:%d wrong roce_enable value (%d), using iWARP\n", - __func__, __LINE__, rf->tun_info.roce_ena); + __func__, __LINE__, t_info->roce_ena); printf("%s:%d protocol: %s, roce_enable value: %d\n", __func__, __LINE__, (rf->protocol_used == IRDMA_IWARP_PROTOCOL_ONLY) ? "iWARP" : "RoCEv2", - rf->tun_info.roce_ena); + t_info->roce_ena); - snprintf(rf->tun_info.drv_ver, IRDMA_VER_LEN, "%s", irdma_driver_version); - SYSCTL_ADD_STRING(&rf->tun_info.irdma_sysctl_ctx, irdma_sysctl_oid_list, - OID_AUTO, "drv_ver", CTLFLAG_RDTUN, rf->tun_info.drv_ver, + snprintf(t_info->drv_ver, IRDMA_VER_LEN, "%s", irdma_driver_version); + SYSCTL_ADD_STRING(&t_info->irdma_sysctl_ctx, irdma_oid_list, + OID_AUTO, "drv_ver", CTLFLAG_RDTUN, t_info->drv_ver, IRDMA_VER_LEN, "driver version"); irdma_dcqcn_tunables_init(rf); + irdma_sysctl_settings(rf); } /** @@ -121,8 +127,6 @@ spin_lock_irqsave(&irdma_handler_lock, flags); list_for_each_entry(hdl, &irdma_handlers, list) { - if (!hdl) - continue; if (!hdl->iwdev->rf->peer_info) continue; if (hdl->iwdev->rf->peer_info->dev == p_dev->dev) { @@ -159,9 +163,12 @@ * @qos_info: source, DCB settings structure */ static void -irdma_get_qos_info(struct irdma_l2params *l2params, struct ice_qos_params *qos_info) +irdma_get_qos_info(struct irdma_pci_f *rf, struct irdma_l2params *l2params, + struct ice_qos_params *qos_info) { int i; + char txt[7][128] = {"", "", "", "", "", "", ""}; + u8 len; l2params->num_tc = qos_info->num_tc; l2params->num_apps = qos_info->num_apps; @@ -183,33 +190,46 @@ l2params->dscp_mode = true; memcpy(l2params->dscp_map, qos_info->dscp_map, sizeof(l2params->dscp_map)); } - printf("%s:%d: l2params settings:\n num_tc %d,\n num_apps %d,\n", - __func__, __LINE__, l2params->num_tc, l2params->num_apps); - printf(" vsi_prio_type %d,\n vsi_rel_bw %d,\n egress_virt_up:", - l2params->vsi_prio_type, l2params->vsi_rel_bw); - for (i = 0; i < l2params->num_tc; i++) - printf(" %d", l2params->tc_info[i].egress_virt_up); - printf("\n ingress_virt_up:"); - for (i = 0; i < l2params->num_tc; i++) - printf(" %d", l2params->tc_info[i].ingress_virt_up); - printf("\n prio_type:"); - for (i = 0; i < l2params->num_tc; i++) - printf(" %d", l2params->tc_info[i].prio_type); - printf("\n rel_bw:"); - for (i = 0; i < l2params->num_tc; i++) - printf(" %d", l2params->tc_info[i].rel_bw); - printf("\n tc_ctx:"); - for (i = 0; i < l2params->num_tc; i++) - printf(" %lu", l2params->tc_info[i].tc_ctx); - printf("\n up2tc:"); + if (!(rf->sc_dev.debug_mask & IRDMA_DEBUG_DCB)) + return; + for (i = 0; i < l2params->num_tc; i++) { + len = strlen(txt[0]); + snprintf(txt[0] + len, sizeof(txt[0]) - 5, " %d", + l2params->tc_info[i].egress_virt_up); + len = strlen(txt[1]); + snprintf(txt[1] + len, sizeof(txt[1]) - 5, " %d", + l2params->tc_info[i].ingress_virt_up); + len = strlen(txt[2]); + snprintf(txt[2] + len, sizeof(txt[2]) - 5, " %d", + l2params->tc_info[i].prio_type); + len = strlen(txt[3]); + snprintf(txt[3] + len, sizeof(txt[3]) - 5, " %d", + l2params->tc_info[i].rel_bw); + len = strlen(txt[4]); + snprintf(txt[4] + len, sizeof(txt[4]) - 5, " %lu", + l2params->tc_info[i].tc_ctx); + } + len = strlen(txt[5]); for (i = 0; i < IRDMA_MAX_USER_PRIORITY; i++) - printf(" %d", l2params->up2tc[i]); - printf(" dscp_mode: %d,\n", l2params->dscp_mode); + len += snprintf(txt[5] + len, sizeof(txt[5]) - 5, " %d", + l2params->up2tc[i]); + len = strlen(txt[6]); for (i = 0; i < IRDMA_DSCP_NUM_VAL; i++) - printf(" %d", l2params->dscp_map[i]); - printf("\n"); - - dump_struct(l2params, sizeof(*l2params), "l2params"); + len += snprintf(txt[6] + len, sizeof(txt[6]) - 5, " %d", + l2params->dscp_map[i]); + irdma_debug(&rf->sc_dev, IRDMA_DEBUG_DCB, "num_tc: %d\n", l2params->num_tc); + irdma_debug(&rf->sc_dev, IRDMA_DEBUG_DCB, "num_apps: %d\n", l2params->num_apps); + irdma_debug(&rf->sc_dev, IRDMA_DEBUG_DCB, "vsi_prio_type: %d\n", l2params->vsi_prio_type); + irdma_debug(&rf->sc_dev, IRDMA_DEBUG_DCB, "vsi_rel_bw: %d\n", l2params->vsi_rel_bw); + irdma_debug(&rf->sc_dev, IRDMA_DEBUG_DCB, "egress_virt_up: %s\n", txt[0]); + irdma_debug(&rf->sc_dev, IRDMA_DEBUG_DCB, "ingress_virt_up:%s\n", txt[1]); + irdma_debug(&rf->sc_dev, IRDMA_DEBUG_DCB, "prio_type: %s\n", txt[2]); + irdma_debug(&rf->sc_dev, IRDMA_DEBUG_DCB, "rel_bw: %s\n", txt[3]); + irdma_debug(&rf->sc_dev, IRDMA_DEBUG_DCB, "tc_ctx: %s\n", txt[4]); + irdma_debug(&rf->sc_dev, IRDMA_DEBUG_DCB, "up2tc: %s\n", txt[5]); + irdma_debug(&rf->sc_dev, IRDMA_DEBUG_DCB, "dscp_mode: %s\n", txt[6]); + + irdma_debug_buf(&rf->sc_dev, IRDMA_DEBUG_DCB, "l2params", l2params, sizeof(*l2params)); } /** @@ -230,6 +250,35 @@ mtu); } +/** + * irdma_get_event_name - convert type enum to string + * @type: event type enum + */ +static const char * +irdma_get_event_name(enum ice_rdma_event_type type) +{ + switch (type) { + case ICE_RDMA_EVENT_LINK_CHANGE: + return "LINK CHANGE"; + case ICE_RDMA_EVENT_MTU_CHANGE: + return "MTU CHANGE"; + case ICE_RDMA_EVENT_TC_CHANGE: + return "TC CHANGE"; + case ICE_RDMA_EVENT_API_CHANGE: + return "API CHANGE"; + case ICE_RDMA_EVENT_CRIT_ERR: + return "CRITICAL ERROR"; + case ICE_RDMA_EVENT_RESET: + return "RESET"; + case ICE_RDMA_EVENT_QSET_REGISTER: + return "QSET REGISTER"; + case ICE_RDMA_EVENT_VSI_FILTER_UPDATE: + return "VSI FILTER UPDATE"; + default: + return "UNKNOWN"; + } +} + /** * irdma_event_handler - handling events from lan driver * @peer: the peer interface structure @@ -242,9 +291,7 @@ struct irdma_l2params l2params = {}; printf("%s:%d event_handler %s (%x) on pf %d (%d)\n", __func__, __LINE__, - (event->type == 1) ? "LINK CHANGE" : - (event->type == 2) ? "MTU CHANGE" : - (event->type == 3) ? "TC CHANGE" : "UNKNOWN", + irdma_get_event_name(event->type), event->type, peer->pf_id, if_getdunit(peer->ifp)); iwdev = peer_to_iwdev(peer); if (!iwdev) { @@ -277,6 +324,12 @@ event->prep ? " " : "not "); goto done; } + if (!atomic_inc_not_zero(&iwdev->rf->dev_ctx.event_rfcnt)) { + printf("%s:%d (%d) EVENT_TC_CHANGE received, but not processed %d\n", + __func__, __LINE__, if_getdunit(peer->ifp), + atomic_read(&iwdev->rf->dev_ctx.event_rfcnt)); + break; + } if (event->prep) { iwdev->vsi.tc_change_pending = true; irdma_sc_suspend_resume_qps(&iwdev->vsi, IRDMA_OP_SUSPEND); @@ -287,7 +340,7 @@ printf("%s:%d TC change preparation done\n", __func__, __LINE__); } else { l2params.tc_changed = true; - irdma_get_qos_info(&l2params, &event->port_qos); + irdma_get_qos_info(iwdev->rf, &l2params, &event->port_qos); if (iwdev->rf->protocol_used != IRDMA_IWARP_PROTOCOL_ONLY) iwdev->dcb_vlan_mode = l2params.num_tc > 1 && !l2params.dscp_mode; @@ -295,9 +348,32 @@ irdma_change_l2params(&iwdev->vsi, &l2params); printf("%s:%d TC change done\n", __func__, __LINE__); } + atomic_dec(&iwdev->rf->dev_ctx.event_rfcnt); break; case ICE_RDMA_EVENT_CRIT_ERR: - printf("%s:%d event type received: %d\n", __func__, __LINE__, event->type); + if (event->oicr_reg & IRDMAPFINT_OICR_PE_CRITERR_M) { + u32 pe_criterr; + +#define IRDMA_Q1_RESOURCE_ERR 0x0001024d + pe_criterr = readl(iwdev->rf->sc_dev.hw_regs[IRDMA_GLPE_CRITERR]); + if (pe_criterr != IRDMA_Q1_RESOURCE_ERR) { + irdma_pr_err("critical PE Error, GLPE_CRITERR=0x%08x\n", + pe_criterr); + iwdev->rf->reset = true; + } else { + irdma_dev_warn(to_ibdev(&iwdev->rf->sc_dev), + "Q1 Resource Check\n"); + } + } + if (event->oicr_reg & IRDMAPFINT_OICR_HMC_ERR_M) { + irdma_pr_err("HMC Error\n"); + iwdev->rf->reset = true; + } + if (iwdev->rf->reset) + iwdev->rf->gen_ops.request_reset(iwdev->rf); + break; + case ICE_RDMA_EVENT_RESET: + iwdev->rf->reset = true; break; default: printf("%s:%d event type unsupported: %d\n", __func__, __LINE__, event->type); @@ -342,6 +418,15 @@ irdma_debug(&rf->sc_dev, IRDMA_DEBUG_INIT, "Starting deferred closing %d (%d)\n", rf->peer_info->pf_id, if_getdunit(peer->ifp)); + atomic_dec(&rf->dev_ctx.event_rfcnt); + wait_event_timeout(iwdev->suspend_wq, + !atomic_read(&rf->dev_ctx.event_rfcnt), + IRDMA_MAX_TIMEOUT); + if (atomic_read(&rf->dev_ctx.event_rfcnt) != 0) { + printf("%s:%d (%d) waiting for event_rfcnt (%d) timeout, proceed with unload\n", + __func__, __LINE__, if_getdunit(peer->ifp), + atomic_read(&rf->dev_ctx.event_rfcnt)); + } irdma_dereg_ipaddr_event_cb(rf); irdma_ib_unregister_device(iwdev); req.type = ICE_RDMA_EVENT_VSI_FILTER_UPDATE; @@ -353,7 +438,7 @@ irdma_debug(&rf->sc_dev, IRDMA_DEBUG_INIT, "Starting deferred opening %d (%d)\n", rf->peer_info->pf_id, if_getdunit(peer->ifp)); - irdma_get_qos_info(&l2params, &peer->initial_qos_info); + irdma_get_qos_info(iwdev->rf, &l2params, &peer->initial_qos_info); if (iwdev->rf->protocol_used != IRDMA_IWARP_PROTOCOL_ONLY) iwdev->dcb_vlan_mode = l2params.num_tc > 1 && !l2params.dscp_mode; @@ -370,53 +455,18 @@ irdma_rt_deinit_hw(iwdev); ib_dealloc_device(&iwdev->ibdev); } + irdma_sw_stats_tunables_init(rf); req.type = ICE_RDMA_EVENT_VSI_FILTER_UPDATE; req.enable_filter = true; IRDMA_DI_REQ_HANDLER(peer, &req); irdma_reg_ipaddr_event_cb(rf); + atomic_inc(&rf->dev_ctx.event_rfcnt); irdma_debug(&rf->sc_dev, IRDMA_DEBUG_INIT, "Deferred opening finished %d (%d)\n", rf->peer_info->pf_id, if_getdunit(peer->ifp)); } } -/** - * irdma_open - Callback for operation open for RDMA device - * @peer: the new peer interface structure - * - * Callback implementing the RDMA_OPEN function. Called by the ice driver to - * notify the RDMA client driver that a new device has been initialized. - */ -static int -irdma_open(struct ice_rdma_peer *peer) -{ - struct ice_rdma_event event = {0}; - - event.type = ICE_RDMA_EVENT_MTU_CHANGE; - event.mtu = peer->mtu; - - irdma_event_handler(peer, &event); - - return 0; -} - -/** - * irdma_close - Callback to notify that a peer device is down - * @peer: the RDMA peer device being stopped - * - * Callback implementing the RDMA_CLOSE function. Called by the ice driver to - * notify the RDMA client driver that a peer device is being stopped. - */ -static int -irdma_close(struct ice_rdma_peer *peer) -{ - /* - * This is called when ifconfig down. Keeping it for compatibility with ice. This event might be usefull for - * future. - */ - return 0; -} - /** * irdma_alloc_pcidev - allocate memory for pcidev and populate data * @peer: the new peer interface structure @@ -615,6 +665,7 @@ sysctl_ctx_free(&iwdev->rf->tun_info.irdma_sysctl_ctx); hdl->iwdev->rf->tun_info.irdma_sysctl_tree = NULL; + hdl->iwdev->rf->tun_info.sws_sysctl_tree = NULL; irdma_ctrl_deinit_hw(iwdev->rf); @@ -630,6 +681,54 @@ return 0; } +/** + * irdma_open - Callback for operation open for RDMA device + * @peer: the new peer interface structure + * + * Callback implementing the RDMA_OPEN function. Called by the ice driver to + * notify the RDMA client driver that a new device has been initialized. + */ +static int +irdma_open(struct ice_rdma_peer *peer) +{ + struct irdma_device *iwdev; + struct ice_rdma_event event = {0}; + + iwdev = peer_to_iwdev(peer); + if (iwdev) { + event.type = ICE_RDMA_EVENT_MTU_CHANGE; + event.mtu = peer->mtu; + + irdma_event_handler(peer, &event); + } else { + irdma_probe(peer); + } + + return 0; +} + +/** + * irdma_close - Callback to notify that a peer device is down + * @peer: the RDMA peer device being stopped + * + * Callback implementing the RDMA_CLOSE function. Called by the ice driver to + * notify the RDMA client driver that a peer device is being stopped. + */ +static int +irdma_close(struct ice_rdma_peer *peer) +{ + /* + * This is called when ifconfig down or pf-reset is about to happen. + */ + struct irdma_device *iwdev; + + iwdev = peer_to_iwdev(peer); + if (iwdev && iwdev->rf->reset) + irdma_remove(peer); + + return 0; +} + /** * irdma_prep_for_unregister - ensure the driver is ready to unregister */ @@ -644,8 +743,6 @@ hdl_valid = false; spin_lock_irqsave(&irdma_handler_lock, flags); list_for_each_entry(hdl, &irdma_handlers, list) { - if (!hdl) - continue; if (!hdl->iwdev->rf->peer_info) continue; hdl_valid = true; diff --git a/sys/dev/irdma/irdma.h b/sys/dev/irdma/irdma.h --- a/sys/dev/irdma/irdma.h +++ b/sys/dev/irdma/irdma.h @@ -1,7 +1,7 @@ /*- * SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB * - * Copyright (c) 2017 - 2022 Intel Corporation + * Copyright (c) 2017 - 2023 Intel Corporation * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU @@ -167,7 +167,7 @@ }; struct irdma_mcast_grp_info { - u8 dest_mac_addr[ETH_ALEN]; + u8 dest_mac_addr[ETHER_ADDR_LEN]; u16 vlan_id; u16 hmc_fcn_id; bool ipv4_valid:1; diff --git a/sys/dev/irdma/irdma_cm.h b/sys/dev/irdma/irdma_cm.h --- a/sys/dev/irdma/irdma_cm.h +++ b/sys/dev/irdma/irdma_cm.h @@ -1,7 +1,7 @@ /*- * SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB * - * Copyright (c) 2015 - 2022 Intel Corporation + * Copyright (c) 2015 - 2023 Intel Corporation * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU @@ -288,7 +288,7 @@ int backlog; u16 loc_port; u16 vlan_id; - u8 loc_mac[ETH_ALEN]; + u8 loc_mac[ETHER_ADDR_LEN]; u8 user_pri; u8 tos; bool qhash_set:1; @@ -341,8 +341,8 @@ u16 mpav2_ird_ord; u16 lsmm_size; u8 pdata_buf[IETF_MAX_PRIV_DATA_LEN]; - u8 loc_mac[ETH_ALEN]; - u8 rem_mac[ETH_ALEN]; + u8 loc_mac[ETHER_ADDR_LEN]; + u8 rem_mac[ETHER_ADDR_LEN]; u8 user_pri; u8 tos; bool ack_rcvd:1; @@ -358,6 +358,7 @@ /* Used by internal CM APIs to pass CM information*/ struct irdma_cm_info { struct iw_cm_id *cm_id; + struct irdma_cqp_request *cqp_request; u16 loc_port; u16 rem_port; u32 loc_addr[4]; @@ -410,6 +411,12 @@ void (*cm_free_ah)(struct irdma_cm_node *cm_node); }; +struct irdma_add_mqh_cbs { + struct irdma_device *iwdev; + struct irdma_cm_info *cm_info; + struct irdma_cm_listener *cm_listen_node; +}; + int irdma_schedule_cm_timer(struct irdma_cm_node *cm_node, struct irdma_puda_buf *sqbuf, enum irdma_timer_type type, int send_retrans, diff --git a/sys/dev/irdma/irdma_cm.c b/sys/dev/irdma/irdma_cm.c --- a/sys/dev/irdma/irdma_cm.c +++ b/sys/dev/irdma/irdma_cm.c @@ -1,7 +1,7 @@ /*- * SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB * - * Copyright (c) 2015 - 2022 Intel Corporation + * Copyright (c) 2015 - 2023 Intel Corporation * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU @@ -290,10 +290,9 @@ event->cm_info.loc_port = cm_node->loc_port; event->cm_info.cm_id = cm_node->cm_id; irdma_debug(&cm_node->iwdev->rf->sc_dev, IRDMA_DEBUG_CM, - "node=%p event=%p type=%u dst=%pI4 src=%pI4\n", - cm_node, - event, type, event->cm_info.loc_addr, - event->cm_info.rem_addr); + "node=%p event=%p type=%u dst=%x src=%x\n", cm_node, event, + type, event->cm_info.loc_addr[0], + event->cm_info.rem_addr[0]); irdma_cm_post_event(event); return event; @@ -646,10 +645,11 @@ return -ENOMEM; irdma_debug(&cm_node->iwdev->rf->sc_dev, IRDMA_DEBUG_CM, - "caller: %pS cm_node %p cm_id=%p accel=%d state=%d rem_port=0x%04x, loc_port=0x%04x rem_addr=%pI4 loc_addr=%pI4\n", + "caller: %pS cm_node %p cm_id=%p accel=%d state=%d rem_port=0x%04x, loc_port=0x%04x rem_addr=%x loc_addr=%x\n", __builtin_return_address(0), cm_node, cm_node->cm_id, cm_node->accelerated, cm_node->state, cm_node->rem_port, - cm_node->loc_port, cm_node->rem_addr, cm_node->loc_addr); + cm_node->loc_port, cm_node->rem_addr[0], + cm_node->loc_addr[0]); return irdma_schedule_cm_timer(cm_node, sqbuf, IRDMA_TIMER_TYPE_SEND, 0, 1); @@ -666,9 +666,8 @@ irdma_cleanup_retrans_entry(cm_node); cm_node->cm_core->stats_connect_errs++; if (reset) { - irdma_debug(&cm_node->iwdev->rf->sc_dev, - IRDMA_DEBUG_CM, "cm_node=%p state=%d\n", cm_node, - cm_node->state); + irdma_debug(&cm_node->iwdev->rf->sc_dev, IRDMA_DEBUG_CM, + "cm_node=%p state=%d\n", cm_node, cm_node->state); atomic_inc(&cm_node->refcnt); irdma_send_reset(cm_node); } @@ -689,8 +688,7 @@ cm_node->cm_core->stats_passive_errs++; cm_node->state = IRDMA_CM_STATE_CLOSED; irdma_debug(&cm_node->iwdev->rf->sc_dev, IRDMA_DEBUG_CM, - "cm_node=%p state =%d\n", - cm_node, cm_node->state); + "cm_node=%p state=%d\n", cm_node, cm_node->state); if (reset) irdma_send_reset(cm_node); else @@ -802,8 +800,7 @@ (u32)tcph->th_flags & TH_SYN); if (ret) { irdma_debug(&cm_node->iwdev->rf->sc_dev, IRDMA_DEBUG_CM, - "Node %p, Sending Reset\n", - cm_node); + "Node %p, Sending Reset\n", cm_node); if (passive) irdma_passive_open_err(cm_node, true); else @@ -951,8 +948,7 @@ MPA_KEY_REQUEST); if (!cm_node->mpa_hdr.size) { irdma_debug(&cm_node->iwdev->rf->sc_dev, IRDMA_DEBUG_CM, - "mpa size = %d\n", - cm_node->mpa_hdr.size); + "mpa size = %d\n", cm_node->mpa_hdr.size); return -EINVAL; } @@ -1062,9 +1058,9 @@ /* Not supported RDMA0 operation */ return -EOPNOTSUPP; - irdma_debug(&cm_node->iwdev->rf->sc_dev, - IRDMA_DEBUG_CM, "MPAV2 Negotiated ORD: %d, IRD: %d\n", - cm_node->ord_size, cm_node->ird_size); + irdma_debug(&cm_node->iwdev->rf->sc_dev, IRDMA_DEBUG_CM, + "MPAV2 Negotiated ORD: %d, IRD: %d\n", cm_node->ord_size, + cm_node->ird_size); return 0; } @@ -1085,8 +1081,8 @@ *type = IRDMA_MPA_REQUEST_ACCEPT; if (len < sizeof(struct ietf_mpa_v1)) { - irdma_debug(&cm_node->iwdev->rf->sc_dev, - IRDMA_DEBUG_CM, "ietf buffer small (%x)\n", len); + irdma_debug(&cm_node->iwdev->rf->sc_dev, IRDMA_DEBUG_CM, + "ietf buffer small (%x)\n", len); return -EINVAL; } @@ -1096,22 +1092,19 @@ if (priv_data_len > IETF_MAX_PRIV_DATA_LEN) { irdma_debug(&cm_node->iwdev->rf->sc_dev, IRDMA_DEBUG_CM, - "private_data too big %d\n", - priv_data_len); + "private_data too big %d\n", priv_data_len); return -EOVERFLOW; } if (mpa_frame->rev != IETF_MPA_V1 && mpa_frame->rev != IETF_MPA_V2) { irdma_debug(&cm_node->iwdev->rf->sc_dev, IRDMA_DEBUG_CM, - "unsupported mpa rev = %d\n", - mpa_frame->rev); + "unsupported mpa rev = %d\n", mpa_frame->rev); return -EINVAL; } if (mpa_frame->rev > cm_node->mpa_frame_rev) { irdma_debug(&cm_node->iwdev->rf->sc_dev, IRDMA_DEBUG_CM, - "rev %d\n", - mpa_frame->rev); + "rev %d\n", mpa_frame->rev); return -EINVAL; } @@ -1119,29 +1112,29 @@ if (cm_node->state != IRDMA_CM_STATE_MPAREQ_SENT) { if (memcmp(mpa_frame->key, IEFT_MPA_KEY_REQ, IETF_MPA_KEY_SIZE)) { - irdma_debug(&cm_node->iwdev->rf->sc_dev, - IRDMA_DEBUG_CM, "Unexpected MPA Key received\n"); + irdma_debug(&cm_node->iwdev->rf->sc_dev, IRDMA_DEBUG_CM, + "Unexpected MPA Key received\n"); return -EINVAL; } } else { if (memcmp(mpa_frame->key, IEFT_MPA_KEY_REP, IETF_MPA_KEY_SIZE)) { - irdma_debug(&cm_node->iwdev->rf->sc_dev, - IRDMA_DEBUG_CM, "Unexpected MPA Key received\n"); + irdma_debug(&cm_node->iwdev->rf->sc_dev, IRDMA_DEBUG_CM, + "Unexpected MPA Key received\n"); return -EINVAL; } } if (priv_data_len + mpa_hdr_len > len) { - irdma_debug(&cm_node->iwdev->rf->sc_dev, - IRDMA_DEBUG_CM, "ietf buffer len(%x + %x != %x)\n", - priv_data_len, mpa_hdr_len, len); + irdma_debug(&cm_node->iwdev->rf->sc_dev, IRDMA_DEBUG_CM, + "ietf buffer len(%x + %x != %x)\n", priv_data_len, + mpa_hdr_len, len); return -EOVERFLOW; } if (len > IRDMA_MAX_CM_BUF) { - irdma_debug(&cm_node->iwdev->rf->sc_dev, - IRDMA_DEBUG_CM, "ietf buffer large len = %d\n", len); + irdma_debug(&cm_node->iwdev->rf->sc_dev, IRDMA_DEBUG_CM, + "ietf buffer large len = %d\n", len); return -EOVERFLOW; } @@ -1213,8 +1206,8 @@ new_send->timetosend += (HZ / 10); if (cm_node->close_entry) { kfree(new_send); - irdma_debug(&cm_node->iwdev->rf->sc_dev, - IRDMA_DEBUG_CM, "already close entry\n"); + irdma_debug(&cm_node->iwdev->rf->sc_dev, IRDMA_DEBUG_CM, + "already close entry\n"); return -EINVAL; } @@ -1583,14 +1576,14 @@ child_listen_list); if (child_listen_node->ipv4) irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_CM, - "removing child listen for IP=%pI4, port=%d, vlan=%d\n", - child_listen_node->loc_addr, + "removing child listen for IP=%x, port=%d, vlan=%d\n", + child_listen_node->loc_addr[0], child_listen_node->loc_port, child_listen_node->vlan_id); else irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_CM, - "removing child listen for IP=%pI6, port=%d, vlan=%d\n", - child_listen_node->loc_addr, + "removing child listen for IP=%x:%x:%x:%x, port=%d, vlan=%d\n", + IRDMA_PRINT_IP6(child_listen_node->loc_addr), child_listen_node->loc_port, child_listen_node->vlan_id); list_del(pos); @@ -1606,8 +1599,8 @@ } else { ret = 0; } - irdma_debug(&iwdev->rf->sc_dev, - IRDMA_DEBUG_CM, "Child listen node freed = %p\n", + irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_CM, + "Child listen node freed = %p\n", child_listen_node); kfree(child_listen_node); cm_parent_listen_node->cm_core->stats_listen_nodes_destroyed++; @@ -1617,7 +1610,8 @@ return ret; } -static u8 irdma_get_egress_vlan_prio(u32 *loc_addr, u8 prio, bool ipv4){ +static u8 irdma_iw_get_vlan_prio(u32 *loc_addr, u8 prio, bool ipv4) +{ return prio; } @@ -1679,14 +1673,33 @@ return vlan_id; } -struct irdma_add_mqh_cbs { - struct irdma_device *iwdev; - struct irdma_cm_info *cm_info; - struct irdma_cm_listener *cm_listen_node; -}; +static int +irdma_manage_qhash_wait(struct irdma_pci_f *rf, struct irdma_cm_info *cm_info) +{ + struct irdma_cqp_request *cqp_request = cm_info->cqp_request; + int cnt = rf->sc_dev.hw_attrs.max_cqp_compl_wait_time_ms * CQP_TIMEOUT_THRESHOLD; + u32 ret_val; + + if (!cqp_request) + return -ENOMEM; + do { + irdma_cqp_ce_handler(rf, &rf->ccq.sc_cq); + mdelay(1); + } while (!READ_ONCE(cqp_request->request_done) && --cnt); + + ret_val = cqp_request->compl_info.op_ret_val; + irdma_put_cqp_request(&rf->cqp, cqp_request); + if (cnt) { + if (!ret_val) + return 0; + return -EINVAL; + } + + return -ETIMEDOUT; +} /** - * irdma_add_mqh_ifa_cb - Adds multiple qhashes for IPV4/IPv6 + * irdma_add_mqh_ifa_cb - Adds multiple qhashes for IPv4/IPv6 * @arg: Calback argument structure from irdma_add_mqh * @ifa: Current address to compute against * @count: Current cumulative output of all callbacks in this iteration @@ -1709,24 +1722,9 @@ if (count) return 0; - if (cm_info->ipv4) - irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_CM, - "Allocating child CM Listener forIP=%pI4, vlan_id=%d, MAC=%pM\n", - &ifa->ifa_addr, - rdma_vlan_dev_vlan_id(ip_dev), if_getlladdr(ip_dev)); - else - irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_CM, - "IP=%pI6, vlan_id=%d, MAC=%pM\n", - &((struct sockaddr_in6 *)ifa->ifa_addr)->sin6_addr, - rdma_vlan_dev_vlan_id(ip_dev), - if_getlladdr(ip_dev)); - child_listen_node = kzalloc(sizeof(*child_listen_node), GFP_KERNEL); - irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_CM, - "Allocating child listener %p\n", - child_listen_node); + child_listen_node = kzalloc(sizeof(*child_listen_node), GFP_ATOMIC); if (!child_listen_node) { - irdma_debug(&iwdev->rf->sc_dev, - IRDMA_DEBUG_CM, + irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_CM, "listener memory allocation\n"); return -ENOMEM; } @@ -1735,23 +1733,44 @@ sizeof(*child_listen_node)); cm_info->vlan_id = rdma_vlan_dev_vlan_id(ip_dev); child_listen_node->vlan_id = cm_info->vlan_id; - if (cm_info->ipv4) + if (cm_info->ipv4) { + irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_CM, + "Allocating child CM Listener forIP=%x, vlan_id=%d, MAC=%x:%x:%x:%x:%x:%x\n", + ((struct sockaddr_in *)&ifa->ifa_addr)->sin_addr.s_addr, + rdma_vlan_dev_vlan_id(ip_dev), + if_getlladdr(ip_dev)[0], if_getlladdr(ip_dev)[1], + if_getlladdr(ip_dev)[2], if_getlladdr(ip_dev)[3], + if_getlladdr(ip_dev)[4], if_getlladdr(ip_dev)[5]); child_listen_node->loc_addr[0] = ntohl(((struct sockaddr_in *)ifa->ifa_addr)->sin_addr.s_addr); - else + } else { + irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_CM, + "IP=%x:%x:%x:%x, vlan_id=%d, MAC=%x:%x:%x:%x:%x:%x\n", + IRDMA_PRINT_IP6(&((struct sockaddr_in6 *)ifa->ifa_addr)->sin6_addr), + rdma_vlan_dev_vlan_id(ip_dev), + if_getlladdr(ip_dev)[0], if_getlladdr(ip_dev)[1], + if_getlladdr(ip_dev)[2], if_getlladdr(ip_dev)[3], + if_getlladdr(ip_dev)[4], if_getlladdr(ip_dev)[5]); irdma_copy_ip_ntohl(child_listen_node->loc_addr, ((struct sockaddr_in6 *)ifa->ifa_addr)->sin6_addr.__u6_addr.__u6_addr32); + } memcpy(cm_info->loc_addr, child_listen_node->loc_addr, sizeof(cm_info->loc_addr)); if (!iwdev->vsi.dscp_mode) cm_info->user_pri = - irdma_get_egress_vlan_prio(child_listen_node->loc_addr, - cm_info->user_pri, - false); + irdma_iw_get_vlan_prio(child_listen_node->loc_addr, + cm_info->user_pri, + cm_info->ipv4); ret = irdma_manage_qhash(iwdev, cm_info, IRDMA_QHASH_TYPE_TCP_SYN, IRDMA_QHASH_MANAGE_TYPE_ADD, - NULL, true); + NULL, false); + if (ret) { + kfree(child_listen_node); + return ret; + } + /* wait for qhash finish */ + ret = irdma_manage_qhash_wait(iwdev->rf, cm_info); if (ret) { kfree(child_listen_node); return ret; @@ -1782,7 +1801,7 @@ struct irdma_add_mqh_cbs cbs; struct if_iter iter; if_t ifp; - int err; + int err = -ENOENT; cbs.iwdev = iwdev; cbs.cm_info = cm_info; @@ -1882,8 +1901,8 @@ err = irdma_send_reset(cm_node); if (err) { cm_node->state = IRDMA_CM_STATE_CLOSED; - irdma_debug(&cm_node->iwdev->rf->sc_dev, - IRDMA_DEBUG_CM, "send reset failed\n"); + irdma_debug(&cm_node->iwdev->rf->sc_dev, IRDMA_DEBUG_CM, + "send reset failed\n"); } else { old_state = cm_node->state; cm_node->state = IRDMA_CM_STATE_LISTENER_DESTROYED; @@ -1922,8 +1941,8 @@ cm_core->stats_listen_destroyed++; cm_core->stats_listen_nodes_destroyed++; irdma_debug(&listener->iwdev->rf->sc_dev, IRDMA_DEBUG_CM, - "loc_port=0x%04x loc_addr=%pI4 cm_listen_node=%p cm_id=%p qhash_set=%d vlan_id=%d apbvt_del=%d\n", - listener->loc_port, listener->loc_addr, listener, + "loc_port=0x%04x loc_addr=%x cm_listen_node=%p cm_id=%p qhash_set=%d vlan_id=%d apbvt_del=%d\n", + listener->loc_port, listener->loc_addr[0], listener, listener->cm_id, listener->qhash_set, listener->vlan_id, apbvt_del); kfree(listener); @@ -2139,13 +2158,12 @@ cm_node->tos = max(listener->tos, cm_info->tos); cm_node->user_pri = rt_tos2priority(cm_node->tos); cm_node->user_pri = - irdma_get_egress_vlan_prio(cm_info->loc_addr, - cm_node->user_pri, - cm_info->ipv4); + irdma_iw_get_vlan_prio(cm_info->loc_addr, + cm_node->user_pri, + cm_info->ipv4); } irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_DCB, - "listener: TOS:[%d] UP:[%d]\n", - cm_node->tos, + "listener: TOS:[%d] UP:[%d]\n", cm_node->tos, cm_node->user_pri); } memcpy(cm_node->loc_addr, cm_info->loc_addr, sizeof(cm_node->loc_addr)); @@ -2200,8 +2218,8 @@ /* if the node is destroyed before connection was accelerated */ if (!cm_node->accelerated && cm_node->accept_pend) { - irdma_debug(&cm_node->iwdev->rf->sc_dev, - IRDMA_DEBUG_CM, "node destroyed before established\n"); + irdma_debug(&cm_node->iwdev->rf->sc_dev, IRDMA_DEBUG_CM, + "node destroyed before established\n"); atomic_dec(&cm_node->listener->pend_accepts_cnt); } if (cm_node->close_entry) @@ -2325,8 +2343,7 @@ case IRDMA_CM_STATE_OFFLOADED: default: irdma_debug(&cm_node->iwdev->rf->sc_dev, IRDMA_DEBUG_CM, - "bad state node state = %d\n", - cm_node->state); + "bad state node state = %d\n", cm_node->state); break; } } @@ -2341,10 +2358,10 @@ struct irdma_puda_buf *rbuf) { irdma_debug(&cm_node->iwdev->rf->sc_dev, IRDMA_DEBUG_CM, - "caller: %pS cm_node=%p state=%d rem_port=0x%04x loc_port=0x%04x rem_addr=%pI4 loc_addr=%pI4\n", + "caller: %pS cm_node=%p state=%d rem_port=0x%04x loc_port=0x%04x rem_addr=%x loc_addr=%x\n", __builtin_return_address(0), cm_node, cm_node->state, - cm_node->rem_port, cm_node->loc_port, cm_node->rem_addr, - cm_node->loc_addr); + cm_node->rem_port, cm_node->loc_port, cm_node->rem_addr[0], + cm_node->loc_addr[0]); irdma_cleanup_retrans_entry(cm_node); switch (cm_node->state) { @@ -2418,8 +2435,8 @@ switch (cm_node->state) { case IRDMA_CM_STATE_ESTABLISHED: if (res_type == IRDMA_MPA_REQUEST_REJECT) - irdma_debug(&cm_node->iwdev->rf->sc_dev, - IRDMA_DEBUG_CM, "state for reject\n"); + irdma_debug(&cm_node->iwdev->rf->sc_dev, IRDMA_DEBUG_CM, + "state for reject\n"); cm_node->state = IRDMA_CM_STATE_MPAREQ_RCVD; type = IRDMA_CM_EVENT_MPA_REQ; irdma_send_ack(cm_node); /* ACK received MPA request */ @@ -2439,8 +2456,7 @@ break; default: irdma_debug(&cm_node->iwdev->rf->sc_dev, IRDMA_DEBUG_CM, - "wrong cm_node state =%d\n", - cm_node->state); + "wrong cm_node state=%d\n", cm_node->state); break; } irdma_create_event(cm_node, type); @@ -2484,8 +2500,8 @@ !between(seq, rcv_nxt, (rcv_nxt + rcv_wnd))) err = -1; if (err) - irdma_debug(&cm_node->iwdev->rf->sc_dev, - IRDMA_DEBUG_CM, "seq number err\n"); + irdma_debug(&cm_node->iwdev->rf->sc_dev, IRDMA_DEBUG_CM, + "seq number err\n"); return err; } @@ -2592,8 +2608,8 @@ irdma_cleanup_retrans_entry(cm_node); /* active open */ if (irdma_check_syn(cm_node, tcph)) { - irdma_debug(&cm_node->iwdev->rf->sc_dev, - IRDMA_DEBUG_CM, "check syn fail\n"); + irdma_debug(&cm_node->iwdev->rf->sc_dev, IRDMA_DEBUG_CM, + "check syn fail\n"); return; } cm_node->tcp_cntxt.rem_ack_num = ntohl(tcph->th_ack); @@ -2601,8 +2617,7 @@ err = irdma_handle_tcp_options(cm_node, tcph, optionsize, 0); if (err) { irdma_debug(&cm_node->iwdev->rf->sc_dev, IRDMA_DEBUG_CM, - "cm_node=%p tcp_options failed\n", - cm_node); + "cm_node=%p tcp_options failed\n", cm_node); break; } irdma_cleanup_retrans_entry(cm_node); @@ -2915,8 +2930,8 @@ cm_node->state = IRDMA_CM_STATE_CLOSED; if (irdma_send_reset(cm_node)) - irdma_debug(&cm_node->iwdev->rf->sc_dev, - IRDMA_DEBUG_CM, "send reset failed\n"); + irdma_debug(&cm_node->iwdev->rf->sc_dev, IRDMA_DEBUG_CM, + "send reset failed\n"); return ret; } @@ -2962,8 +2977,8 @@ break; case IRDMA_CM_STATE_OFFLOADED: if (cm_node->send_entry) - irdma_debug(&cm_node->iwdev->rf->sc_dev, - IRDMA_DEBUG_CM, "CM send_entry in OFFLOADED state\n"); + irdma_debug(&cm_node->iwdev->rf->sc_dev, IRDMA_DEBUG_CM, + "CM send_entry in OFFLOADED state\n"); irdma_rem_ref_cm_node(cm_node); break; } @@ -3013,8 +3028,7 @@ VLAN_PRIO_SHIFT; cm_info.vlan_id = vtag & EVL_VLID_MASK; irdma_debug(&cm_core->iwdev->rf->sc_dev, IRDMA_DEBUG_CM, - "vlan_id=%d\n", - cm_info.vlan_id); + "vlan_id=%d\n", cm_info.vlan_id); } else { cm_info.vlan_id = 0xFFFF; } @@ -3055,8 +3069,8 @@ IRDMA_CM_LISTENER_ACTIVE_STATE); if (!listener) { cm_info.cm_id = NULL; - irdma_debug(&cm_core->iwdev->rf->sc_dev, - IRDMA_DEBUG_CM, "no listener found\n"); + irdma_debug(&cm_core->iwdev->rf->sc_dev, IRDMA_DEBUG_CM, + "no listener found\n"); return; } @@ -3064,8 +3078,8 @@ cm_node = irdma_make_cm_node(cm_core, iwdev, &cm_info, listener); if (!cm_node) { - irdma_debug(&cm_core->iwdev->rf->sc_dev, - IRDMA_DEBUG_CM, "allocate node failed\n"); + irdma_debug(&cm_core->iwdev->rf->sc_dev, IRDMA_DEBUG_CM, + "allocate node failed\n"); atomic_dec(&listener->refcnt); return; } @@ -3293,9 +3307,8 @@ spin_lock_irqsave(&iwdev->rf->qptable_lock, flags); if (!iwdev->rf->qp_table[iwqp->ibqp.qp_num]) { spin_unlock_irqrestore(&iwdev->rf->qptable_lock, flags); - irdma_debug(&iwdev->rf->sc_dev, - IRDMA_DEBUG_CM, "qp_id %d is already freed\n", - iwqp->ibqp.qp_num); + irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_CM, + "qp_id %d is already freed\n", iwqp->ibqp.qp_num); kfree(work); return; } @@ -3467,7 +3480,7 @@ if (iwqp->ietf_mem.va) { if (iwqp->lsmm_mr) - kc_free_lsmm_dereg_mr(iwdev, iwqp); + iwdev->ibdev.dereg_mr(iwqp->lsmm_mr, NULL); irdma_free_dma_mem(iwdev->rf->sc_dev.hw, &iwqp->ietf_mem); iwqp->ietf_mem.va = NULL; @@ -3614,9 +3627,9 @@ } irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_CM, - "rem_port=0x%04x, loc_port=0x%04x rem_addr=%pI4 loc_addr=%pI4 cm_node=%p cm_id=%p qp_id = %d\n\n", - cm_node->rem_port, cm_node->loc_port, cm_node->rem_addr, - cm_node->loc_addr, cm_node, cm_id, ibqp->qp_num); + "rem_port=0x%04x, loc_port=0x%04x rem_addr=%x loc_addr=%x cm_node=%p cm_id=%p qp_id=%d\n\n", + cm_node->rem_port, cm_node->loc_port, cm_node->rem_addr[0], + cm_node->loc_addr[0], cm_node, cm_id, ibqp->qp_num); cm_node->cm_core->stats_accepts++; return 0; @@ -3729,9 +3742,10 @@ iwqp->sc_qp.vsi->dscp_map[irdma_tos2dscp(cm_info.tos)]; } else { cm_info.user_pri = rt_tos2priority(cm_id->tos); - cm_info.user_pri = irdma_get_egress_vlan_prio(cm_info.loc_addr, - cm_info.user_pri, - cm_info.ipv4); + cm_info.user_pri = + irdma_iw_get_vlan_prio(cm_info.loc_addr, + cm_info.user_pri, + cm_info.ipv4); } if (iwqp->sc_qp.dev->ws_add(iwqp->sc_qp.vsi, cm_info.user_pri)) @@ -3740,9 +3754,8 @@ irdma_qp_add_qos(&iwqp->sc_qp); if (iwdev->rf->sc_dev.hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_2) iwdev->rf->check_fc(&iwdev->vsi, &iwqp->sc_qp); - irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_DCB, - "TOS:[%d] UP:[%d]\n", cm_id->tos, - cm_info.user_pri); + irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_DCB, "TOS:[%d] UP:[%d]\n", + cm_id->tos, cm_info.user_pri); ret = irdma_create_cm_node(&iwdev->cm_core, iwdev, conn_param, &cm_info, &cm_node); @@ -3780,21 +3793,21 @@ } irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_CM, - "rem_port=0x%04x, loc_port=0x%04x rem_addr=%pI4 loc_addr=%pI4 cm_node=%p cm_id=%p qp_id = %d\n\n", - cm_node->rem_port, cm_node->loc_port, cm_node->rem_addr, - cm_node->loc_addr, cm_node, cm_id, ibqp->qp_num); + "rem_port=0x%04x, loc_port=0x%04x rem_addr=%x loc_addr=%x cm_node=%p cm_id=%p qp_id = %d\n\n", + cm_node->rem_port, cm_node->loc_port, cm_node->rem_addr[0], + cm_node->loc_addr[0], cm_node, cm_id, ibqp->qp_num); return 0; err: if (cm_info.ipv4) - irdma_debug(&iwdev->rf->sc_dev, - IRDMA_DEBUG_CM, "connect() FAILED: dest addr=%pI4", - cm_info.rem_addr); + irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_CM, + "connect() FAILED: dest addr=%x", + cm_info.rem_addr[0]); else - irdma_debug(&iwdev->rf->sc_dev, - IRDMA_DEBUG_CM, "connect() FAILED: dest addr=%pI6", - cm_info.rem_addr); + irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_CM, + "connect() FAILED: dest addr=%x:%x:%x:%x", + IRDMA_PRINT_IP6(cm_info.rem_addr)); irdma_rem_ref_cm_node(cm_node); iwdev->cm_core.stats_connect_errs++; @@ -3864,8 +3877,8 @@ cm_listen_node = irdma_make_listen_node(&iwdev->cm_core, iwdev, &cm_info); if (!cm_listen_node) { - irdma_debug(&iwdev->rf->sc_dev, - IRDMA_DEBUG_CM, "cm_listen_node == NULL\n"); + irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_CM, + "cm_listen_node == NULL\n"); return -ENOMEM; } @@ -3886,9 +3899,9 @@ } else { if (!iwdev->vsi.dscp_mode) cm_info.user_pri = cm_listen_node->user_pri = - irdma_get_egress_vlan_prio(cm_info.loc_addr, - cm_info.user_pri, - cm_info.ipv4); + irdma_iw_get_vlan_prio(cm_info.loc_addr, + cm_info.user_pri, + cm_info.ipv4); err = irdma_manage_qhash(iwdev, &cm_info, IRDMA_QHASH_TYPE_TCP_SYN, IRDMA_QHASH_MANAGE_TYPE_ADD, @@ -3907,8 +3920,8 @@ cm_id->add_ref(cm_id); cm_listen_node->cm_core->stats_listen_created++; irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_CM, - "loc_port=0x%04x loc_addr=%pI4 cm_listen_node=%p cm_id=%p qhash_set=%d vlan_id=%d\n", - cm_listen_node->loc_port, cm_listen_node->loc_addr, + "loc_port=0x%04x loc_addr=%x cm_listen_node=%p cm_id=%p qhash_set=%d vlan_id=%d\n", + cm_listen_node->loc_port, cm_listen_node->loc_addr[0], cm_listen_node, cm_listen_node->cm_id, cm_listen_node->qhash_set, cm_listen_node->vlan_id); @@ -3935,8 +3948,8 @@ irdma_cm_del_listen(&iwdev->cm_core, cm_id->provider_data, true); else - irdma_debug(&iwdev->rf->sc_dev, - IRDMA_DEBUG_CM, "cm_id->provider_data was NULL\n"); + irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_CM, + "cm_id->provider_data was NULL\n"); cm_id->rem_ref(cm_id); @@ -4122,8 +4135,7 @@ return; irdma_debug(&cm_node->iwdev->rf->sc_dev, IRDMA_DEBUG_CM, - "reset event %p - cm_id = %p\n", - event->cm_node, cm_id); + "reset event %p - cm_id = %p\n", event->cm_node, cm_id); iwqp->cm_id = NULL; irdma_send_cm_event(cm_node, cm_node->cm_id, IW_CM_EVENT_DISCONNECT, @@ -4175,8 +4187,7 @@ break; default: irdma_debug(&cm_node->iwdev->rf->sc_dev, IRDMA_DEBUG_CM, - "bad event type = %d\n", - event->type); + "bad event type = %d\n", event->type); break; } diff --git a/sys/dev/irdma/irdma_ctrl.c b/sys/dev/irdma/irdma_ctrl.c --- a/sys/dev/irdma/irdma_ctrl.c +++ b/sys/dev/irdma/irdma_ctrl.c @@ -1,7 +1,7 @@ /*- * SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB * - * Copyright (c) 2015 - 2022 Intel Corporation + * Copyright (c) 2015 - 2023 Intel Corporation * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU @@ -179,7 +179,8 @@ irdma_debug(qp->dev, IRDMA_DEBUG_DCB, "DCB: Remove qp[%d] UP[%d] qset[%d] on_qoslist[%d]\n", - qp->qp_uk.qp_id, qp->user_pri, qp->qs_handle, qp->on_qoslist); + qp->qp_uk.qp_id, qp->user_pri, qp->qs_handle, + qp->on_qoslist); mutex_lock(&vsi->qos[qp->user_pri].qos_mutex); if (qp->on_qoslist) { qp->on_qoslist = false; @@ -199,7 +200,8 @@ irdma_debug(qp->dev, IRDMA_DEBUG_DCB, "DCB: Add qp[%d] UP[%d] qset[%d] on_qoslist[%d]\n", - qp->qp_uk.qp_id, qp->user_pri, qp->qs_handle, qp->on_qoslist); + qp->qp_uk.qp_id, qp->user_pri, qp->qs_handle, + qp->on_qoslist); mutex_lock(&vsi->qos[qp->user_pri].qos_mutex); if (!qp->on_qoslist) { list_add(&qp->list, &vsi->qos[qp->user_pri].qplist); @@ -238,17 +240,14 @@ u64 scratch, bool post_sq) { __le64 *wqe; - u64 temp, hdr; + u64 hdr; wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); if (!wqe) return -ENOSPC; set_64bit_val(wqe, IRDMA_BYTE_8, info->reach_max); - temp = info->mac_addr[5] | LS_64_1(info->mac_addr[4], 8) | - LS_64_1(info->mac_addr[3], 16) | LS_64_1(info->mac_addr[2], 24) | - LS_64_1(info->mac_addr[1], 32) | LS_64_1(info->mac_addr[0], 40); - set_64bit_val(wqe, IRDMA_BYTE_16, temp); + set_64bit_val(wqe, IRDMA_BYTE_16, irdma_mac_to_u64(info->mac_addr)); hdr = info->arp_index | FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_MANAGE_ARP) | @@ -369,10 +368,7 @@ wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); if (!wqe) return -ENOSPC; - temp = info->mac_addr[5] | LS_64_1(info->mac_addr[4], 8) | - LS_64_1(info->mac_addr[3], 16) | LS_64_1(info->mac_addr[2], 24) | - LS_64_1(info->mac_addr[1], 32) | LS_64_1(info->mac_addr[0], 40); - set_64bit_val(wqe, IRDMA_BYTE_0, temp); + set_64bit_val(wqe, IRDMA_BYTE_0, irdma_mac_to_u64(info->mac_addr)); qw1 = FIELD_PREP(IRDMA_CQPSQ_QHASH_QPN, info->qp_num) | FIELD_PREP(IRDMA_CQPSQ_QHASH_DEST_PORT, info->dest_port); @@ -711,18 +707,10 @@ struct irdma_udp_offload_info *udp; u8 push_mode_en; u32 push_idx; - u64 mac; roce_info = info->roce_info; udp = info->udp_info; - mac = LS_64_1(roce_info->mac_addr[5], 16) | - LS_64_1(roce_info->mac_addr[4], 24) | - LS_64_1(roce_info->mac_addr[3], 32) | - LS_64_1(roce_info->mac_addr[2], 40) | - LS_64_1(roce_info->mac_addr[1], 48) | - LS_64_1(roce_info->mac_addr[0], 56); - qp->user_pri = info->user_pri; if (qp->push_idx == IRDMA_INVALID_PUSH_PAGE_INDEX) { push_mode_en = 0; @@ -797,7 +785,8 @@ FIELD_PREP(IRDMAQPC_RXCQNUM, info->rcv_cq_num)); set_64bit_val(qp_ctx, IRDMA_BYTE_144, FIELD_PREP(IRDMAQPC_STAT_INDEX, info->stats_idx)); - set_64bit_val(qp_ctx, IRDMA_BYTE_152, mac); + set_64bit_val(qp_ctx, IRDMA_BYTE_152, + FIELD_PREP(IRDMAQPC_MACADDRESS, irdma_mac_to_u64(roce_info->mac_addr))); set_64bit_val(qp_ctx, IRDMA_BYTE_160, FIELD_PREP(IRDMAQPC_ORDSIZE, roce_info->ord_size) | FIELD_PREP(IRDMAQPC_IRDSIZE, irdma_sc_get_encoded_ird_size(roce_info->ird_size)) | @@ -878,16 +867,13 @@ u64 scratch, bool post_sq) { __le64 *wqe; - u64 temp, header; + u64 header; wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); if (!wqe) return -ENOSPC; - temp = info->mac_addr[5] | LS_64_1(info->mac_addr[4], 8) | - LS_64_1(info->mac_addr[3], 16) | LS_64_1(info->mac_addr[2], 24) | - LS_64_1(info->mac_addr[1], 32) | LS_64_1(info->mac_addr[0], 40); - set_64bit_val(wqe, IRDMA_BYTE_32, temp); + set_64bit_val(wqe, IRDMA_BYTE_32, irdma_mac_to_u64(info->mac_addr)); header = FIELD_PREP(IRDMA_CQPSQ_MLM_TABLEIDX, info->entry_idx) | FIELD_PREP(IRDMA_CQPSQ_OPCODE, @@ -1019,14 +1005,9 @@ FIELD_PREP(IRDMAQPC_Q2ADDR, qp->q2_pa >> 8) | FIELD_PREP(IRDMAQPC_STAT_INDEX, info->stats_idx)); - if (dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2) { - mac = LS_64_1(iw->mac_addr[5], 16) | - LS_64_1(iw->mac_addr[4], 24) | - LS_64_1(iw->mac_addr[3], 32) | - LS_64_1(iw->mac_addr[2], 40) | - LS_64_1(iw->mac_addr[1], 48) | - LS_64_1(iw->mac_addr[0], 56); - } + if (dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2) + mac = FIELD_PREP(IRDMAQPC_MACADDRESS, + irdma_mac_to_u64(iw->mac_addr)); set_64bit_val(qp_ctx, IRDMA_BYTE_152, mac | FIELD_PREP(IRDMAQPC_LASTBYTESENT, iw->last_byte_sent)); @@ -1411,8 +1392,9 @@ qp->qp_uk.sq_wrtrk_array[wqe_idx].signaled = info->signaled; irdma_debug(qp->dev, IRDMA_DEBUG_MR, - "wr_id[%llxh] wqe_idx[%04d] location[%p]\n", (unsigned long long)info->wr_id, - wqe_idx, &qp->qp_uk.sq_wrtrk_array[wqe_idx].wrid); + "wr_id[%llxh] wqe_idx[%04d] location[%p]\n", + (unsigned long long)info->wr_id, wqe_idx, + &qp->qp_uk.sq_wrtrk_array[wqe_idx].wrid); temp = (info->addr_type == IRDMA_ADDR_TYPE_VA_BASED) ? (uintptr_t)info->va : info->fbo; @@ -1533,45 +1515,6 @@ irdma_sc_gen_rts_ae(qp); } -/** - * irdma_sc_send_lsmm_nostag - for privilege qp - * @qp: sc qp struct - * @lsmm_buf: buffer with lsmm message - * @size: size of lsmm buffer - */ -void -irdma_sc_send_lsmm_nostag(struct irdma_sc_qp *qp, void *lsmm_buf, u32 size) -{ - __le64 *wqe; - u64 hdr; - struct irdma_qp_uk *qp_uk; - - qp_uk = &qp->qp_uk; - wqe = qp_uk->sq_base->elem; - - set_64bit_val(wqe, IRDMA_BYTE_0, (uintptr_t)lsmm_buf); - - if (qp->qp_uk.uk_attrs->hw_rev == IRDMA_GEN_1) - set_64bit_val(wqe, IRDMA_BYTE_8, - FIELD_PREP(IRDMAQPSQ_GEN1_FRAG_LEN, size)); - else - set_64bit_val(wqe, IRDMA_BYTE_8, - FIELD_PREP(IRDMAQPSQ_FRAG_LEN, size) | - FIELD_PREP(IRDMAQPSQ_VALID, qp->qp_uk.swqe_polarity)); - set_64bit_val(wqe, IRDMA_BYTE_16, 0); - - hdr = FIELD_PREP(IRDMAQPSQ_OPCODE, IRDMAQP_OP_RDMA_SEND) | - FIELD_PREP(IRDMAQPSQ_STREAMMODE, 1) | - FIELD_PREP(IRDMAQPSQ_WAITFORRCVPDU, 1) | - FIELD_PREP(IRDMAQPSQ_VALID, qp->qp_uk.swqe_polarity); - irdma_wmb(); /* make sure WQE is written before valid bit is set */ - - set_64bit_val(wqe, IRDMA_BYTE_24, hdr); - - irdma_debug_buf(qp->dev, IRDMA_DEBUG_WQE, "SEND_LSMM_NOSTAG WQE", wqe, - IRDMA_QP_WQE_MIN_SIZE); -} - /** * irdma_sc_send_rtt - send last read0 or write0 * @qp: sc qp struct @@ -2239,16 +2182,15 @@ struct irdma_up_info *info, u64 scratch) { __le64 *wqe; - u64 temp; + u64 temp = 0; + int i; wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); if (!wqe) return -ENOSPC; - temp = info->map[0] | LS_64_1(info->map[1], 8) | - LS_64_1(info->map[2], 16) | LS_64_1(info->map[3], 24) | - LS_64_1(info->map[4], 32) | LS_64_1(info->map[5], 40) | - LS_64_1(info->map[6], 48) | LS_64_1(info->map[7], 56); + for (i = 0; i < IRDMA_MAX_USER_PRIORITY; i++) + temp |= (u64)info->map[i] << (i * 8); set_64bit_val(wqe, IRDMA_BYTE_0, temp); set_64bit_val(wqe, IRDMA_BYTE_40, @@ -2341,7 +2283,8 @@ if (!flush_sq && !flush_rq) { irdma_debug(qp->dev, IRDMA_DEBUG_CQP, - "Additional flush request ignored for qp %x\n", qp->qp_uk.qp_id); + "Additional flush request ignored for qp %x\n", + qp->qp_uk.qp_id); return -EALREADY; } @@ -2649,11 +2592,11 @@ return -EINVAL; ceq = cq->dev->ceq[cq->ceq_id]; - if (ceq && ceq->reg_cq) + if (ceq && ceq->reg_cq) { ret_code = irdma_sc_add_cq_ctx(ceq, cq); - - if (ret_code) - return ret_code; + if (ret_code) + return ret_code; + } wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); if (!wqe) { @@ -2837,11 +2780,12 @@ irdma_check_cqp_progress(struct irdma_cqp_timeout *timeout, struct irdma_sc_dev *dev) { - if (timeout->compl_cqp_cmds != dev->cqp_cmd_stats[IRDMA_OP_CMPL_CMDS]) { - timeout->compl_cqp_cmds = dev->cqp_cmd_stats[IRDMA_OP_CMPL_CMDS]; + u64 completed_ops = atomic64_read(&dev->cqp->completed_ops); + + if (timeout->compl_cqp_cmds != completed_ops) { + timeout->compl_cqp_cmds = completed_ops; timeout->count = 0; - } else if (timeout->compl_cqp_cmds != - dev->cqp_cmd_stats[IRDMA_OP_REQ_CMDS]) { + } else if (timeout->compl_cqp_cmds != dev->cqp->requested_ops) { timeout->count++; } } @@ -2886,7 +2830,7 @@ if (newtail != tail) { /* SUCCESS */ IRDMA_RING_MOVE_TAIL(cqp->sq_ring); - cqp->dev->cqp_cmd_stats[IRDMA_OP_CMPL_CMDS]++; + atomic64_inc(&cqp->completed_ops); return 0; } irdma_usec_delay(cqp->dev->hw_attrs.max_sleep_count); @@ -3252,8 +3196,8 @@ info->dev->cqp = cqp; IRDMA_RING_INIT(cqp->sq_ring, cqp->sq_size); - cqp->dev->cqp_cmd_stats[IRDMA_OP_REQ_CMDS] = 0; - cqp->dev->cqp_cmd_stats[IRDMA_OP_CMPL_CMDS] = 0; + cqp->requested_ops = 0; + atomic64_set(&cqp->completed_ops, 0); /* for the cqp commands backlog. */ INIT_LIST_HEAD(&cqp->dev->cqp_cmd_head); @@ -3263,8 +3207,8 @@ irdma_debug(cqp->dev, IRDMA_DEBUG_WQE, "sq_size[%04d] hw_sq_size[%04d] sq_base[%p] sq_pa[%llxh] cqp[%p] polarity[x%04x]\n", - cqp->sq_size, cqp->hw_sq_size, cqp->sq_base, (unsigned long long)cqp->sq_pa, cqp, - cqp->polarity); + cqp->sq_size, cqp->hw_sq_size, cqp->sq_base, + (unsigned long long)cqp->sq_pa, cqp, cqp->polarity); return 0; } @@ -3409,7 +3353,7 @@ if (ret_code) return NULL; - cqp->dev->cqp_cmd_stats[IRDMA_OP_REQ_CMDS]++; + cqp->requested_ops++; if (!*wqe_idx) cqp->polarity = !cqp->polarity; wqe = cqp->sq_base[*wqe_idx].elem; @@ -3504,6 +3448,9 @@ if (polarity != ccq->cq_uk.polarity) return -ENOENT; + /* Ensure CEQE contents are read after valid bit is checked */ + rmb(); + get_64bit_val(cqe, IRDMA_BYTE_8, &qp_ctx); cqp = (struct irdma_sc_cqp *)(irdma_uintptr) qp_ctx; info->error = (bool)FIELD_GET(IRDMA_CQ_ERROR, temp); @@ -3539,7 +3486,7 @@ irdma_wmb(); /* make sure shadow area is updated before moving tail */ IRDMA_RING_MOVE_TAIL(cqp->sq_ring); - ccq->dev->cqp_cmd_stats[IRDMA_OP_CMPL_CMDS]++; + atomic64_inc(&cqp->completed_ops); return ret_code; } @@ -4159,13 +4106,17 @@ u8 polarity; aeqe = IRDMA_GET_CURRENT_AEQ_ELEM(aeq); - get_64bit_val(aeqe, IRDMA_BYTE_0, &compl_ctx); get_64bit_val(aeqe, IRDMA_BYTE_8, &temp); polarity = (u8)FIELD_GET(IRDMA_AEQE_VALID, temp); if (aeq->polarity != polarity) return -ENOENT; + /* Ensure AEQE contents are read after valid bit is checked */ + rmb(); + + get_64bit_val(aeqe, IRDMA_BYTE_0, &compl_ctx); + irdma_debug_buf(aeq->dev, IRDMA_DEBUG_WQE, "AEQ_ENTRY WQE", aeqe, 16); ae_src = (u8)FIELD_GET(IRDMA_AEQE_AESRC, temp); @@ -4752,16 +4703,18 @@ */ static u32 irdma_est_sd(struct irdma_sc_dev *dev, struct irdma_hmc_info *hmc_info){ - int i; + struct irdma_hmc_obj_info *pble_info; u64 size = 0; u64 sd; + int i; for (i = IRDMA_HMC_IW_QP; i < IRDMA_HMC_IW_MAX; i++) if (i != IRDMA_HMC_IW_PBLE) size += round_up(hmc_info->hmc_obj[i].cnt * hmc_info->hmc_obj[i].size, 512); - size += round_up(hmc_info->hmc_obj[IRDMA_HMC_IW_PBLE].cnt * - hmc_info->hmc_obj[IRDMA_HMC_IW_PBLE].size, 512); + + pble_info = &hmc_info->hmc_obj[IRDMA_HMC_IW_PBLE]; + size += round_up(pble_info->cnt * pble_info->size, 512); if (size & 0x1FFFFF) sd = (size >> 21) + 1; /* add 1 for remainder */ else @@ -4846,8 +4799,7 @@ goto exit; } else if (feat_cnt > IRDMA_MAX_FEATURES) { irdma_debug(dev, IRDMA_DEBUG_DEV, - "feature buf size insufficient," - "retrying with larger buffer\n"); + "feature buf size insufficient, retrying with larger buffer\n"); irdma_free_dma_mem(dev->hw, &feat_buf); feat_buf.size = 8 * feat_cnt; feat_buf.va = irdma_allocate_dma_mem(dev->hw, &feat_buf, @@ -5062,9 +5014,9 @@ pblewanted > (512 * FPM_MULTIPLIER * sd_diff)) { pblewanted -= 256 * FPM_MULTIPLIER * sd_diff; continue; - } else if (pblewanted > (100 * FPM_MULTIPLIER)) { + } else if (pblewanted > 100 * FPM_MULTIPLIER) { pblewanted -= 10 * FPM_MULTIPLIER; - } else if (pblewanted > FPM_MULTIPLIER) { + } else if (pblewanted > 16 * FPM_MULTIPLIER) { pblewanted -= FPM_MULTIPLIER; } else if (qpwanted <= 128) { if (hmc_info->hmc_obj[IRDMA_HMC_IW_FSIMC].cnt > 256) diff --git a/sys/dev/irdma/irdma_defs.h b/sys/dev/irdma/irdma_defs.h --- a/sys/dev/irdma/irdma_defs.h +++ b/sys/dev/irdma/irdma_defs.h @@ -1,7 +1,7 @@ /*- * SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB * - * Copyright (c) 2015 - 2022 Intel Corporation + * Copyright (c) 2015 - 2023 Intel Corporation * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU @@ -250,40 +250,38 @@ IRDMA_OP_MANAGE_HMC_PM_FUNC_TABLE = 22, IRDMA_OP_SUSPEND = 23, IRDMA_OP_RESUME = 24, - IRDMA_OP_MANAGE_VF_PBLE_BP = 25, + IRDMA_OP_MANAGE_VCHNL_REQ_PBLE_BP = 25, IRDMA_OP_QUERY_FPM_VAL = 26, IRDMA_OP_COMMIT_FPM_VAL = 27, - IRDMA_OP_REQ_CMDS = 28, - IRDMA_OP_CMPL_CMDS = 29, - IRDMA_OP_AH_CREATE = 30, - IRDMA_OP_AH_MODIFY = 31, - IRDMA_OP_AH_DESTROY = 32, - IRDMA_OP_MC_CREATE = 33, - IRDMA_OP_MC_DESTROY = 34, - IRDMA_OP_MC_MODIFY = 35, - IRDMA_OP_STATS_ALLOCATE = 36, - IRDMA_OP_STATS_FREE = 37, - IRDMA_OP_STATS_GATHER = 38, - IRDMA_OP_WS_ADD_NODE = 39, - IRDMA_OP_WS_MODIFY_NODE = 40, - IRDMA_OP_WS_DELETE_NODE = 41, - IRDMA_OP_WS_FAILOVER_START = 42, - IRDMA_OP_WS_FAILOVER_COMPLETE = 43, - IRDMA_OP_SET_UP_MAP = 44, - IRDMA_OP_GEN_AE = 45, - IRDMA_OP_QUERY_RDMA_FEATURES = 46, - IRDMA_OP_ALLOC_LOCAL_MAC_ENTRY = 47, - IRDMA_OP_ADD_LOCAL_MAC_ENTRY = 48, - IRDMA_OP_DELETE_LOCAL_MAC_ENTRY = 49, - IRDMA_OP_CQ_MODIFY = 50, + IRDMA_OP_AH_CREATE = 28, + IRDMA_OP_AH_MODIFY = 29, + IRDMA_OP_AH_DESTROY = 30, + IRDMA_OP_MC_CREATE = 31, + IRDMA_OP_MC_DESTROY = 32, + IRDMA_OP_MC_MODIFY = 33, + IRDMA_OP_STATS_ALLOCATE = 34, + IRDMA_OP_STATS_FREE = 35, + IRDMA_OP_STATS_GATHER = 36, + IRDMA_OP_WS_ADD_NODE = 37, + IRDMA_OP_WS_MODIFY_NODE = 38, + IRDMA_OP_WS_DELETE_NODE = 39, + IRDMA_OP_WS_FAILOVER_START = 40, + IRDMA_OP_WS_FAILOVER_COMPLETE = 41, + IRDMA_OP_SET_UP_MAP = 42, + IRDMA_OP_GEN_AE = 43, + IRDMA_OP_QUERY_RDMA_FEATURES = 44, + IRDMA_OP_ALLOC_LOCAL_MAC_ENTRY = 45, + IRDMA_OP_ADD_LOCAL_MAC_ENTRY = 46, + IRDMA_OP_DELETE_LOCAL_MAC_ENTRY = 47, + IRDMA_OP_CQ_MODIFY = 48, /* Must be last entry */ - IRDMA_MAX_CQP_OPS = 51, + IRDMA_MAX_CQP_OPS = 49, }; /* CQP SQ WQES */ -#define IRDMA_CQP_OP_CREATE_QP 0 -#define IRDMA_CQP_OP_MODIFY_QP 0x1 +#define IRDMA_CQP_OP_CREATE_QP 0x00 +#define IRDMA_CQP_OP_MODIFY_QP 0x01 #define IRDMA_CQP_OP_DESTROY_QP 0x02 #define IRDMA_CQP_OP_CREATE_CQ 0x03 #define IRDMA_CQP_OP_MODIFY_CQ 0x04 @@ -295,12 +293,11 @@ #define IRDMA_CQP_OP_DEALLOC_STAG 0x0d #define IRDMA_CQP_OP_MANAGE_LOC_MAC_TABLE 0x0e #define IRDMA_CQP_OP_MANAGE_ARP 0x0f -#define IRDMA_CQP_OP_MANAGE_VF_PBLE_BP 0x10 +#define IRDMA_CQP_OP_MANAGE_VCHNL_REQ_PBLE_BP 0x10 #define IRDMA_CQP_OP_MANAGE_PUSH_PAGES 0x11 #define IRDMA_CQP_OP_QUERY_RDMA_FEATURES 0x12 #define IRDMA_CQP_OP_UPLOAD_CONTEXT 0x13 #define IRDMA_CQP_OP_ALLOCATE_LOC_MAC_TABLE_ENTRY 0x14 -#define IRDMA_CQP_OP_UPLOAD_CONTEXT 0x13 #define IRDMA_CQP_OP_MANAGE_HMC_PM_FUNC_TABLE 0x15 #define IRDMA_CQP_OP_CREATE_CEQ 0x16 #define IRDMA_CQP_OP_DESTROY_CEQ 0x18 @@ -700,7 +697,6 @@ #define IRDMA_CQPSQ_QP_MACVALID BIT_ULL(51) #define IRDMA_CQPSQ_QP_MSSCHANGE_S 52 #define IRDMA_CQPSQ_QP_MSSCHANGE BIT_ULL(52) - #define IRDMA_CQPSQ_QP_IGNOREMWBOUND_S 54 #define IRDMA_CQPSQ_QP_IGNOREMWBOUND BIT_ULL(54) #define IRDMA_CQPSQ_QP_REMOVEHASHENTRY_S 55 @@ -1403,7 +1399,7 @@ #define IRDMA_GET_CQ_ELEM_AT_OFFSET(_cq, _i, _cqe) \ { \ - register __u32 offset; \ + __u32 offset; \ offset = IRDMA_GET_RING_OFFSET((_cq)->cq_ring, _i); \ (_cqe) = (_cq)->cq_base[offset].buf; \ } @@ -1429,7 +1425,7 @@ #define IRDMA_RING_MOVE_HEAD(_ring, _retcode) \ { \ - register u32 size; \ + u32 size; \ size = (_ring).size; \ if (!IRDMA_RING_FULL_ERR(_ring)) { \ (_ring).head = ((_ring).head + 1) % size; \ @@ -1440,7 +1436,7 @@ } #define IRDMA_RING_MOVE_HEAD_BY_COUNT(_ring, _count, _retcode) \ { \ - register u32 size; \ + u32 size; \ size = (_ring).size; \ if ((IRDMA_RING_USED_QUANTA(_ring) + (_count)) < size) { \ (_ring).head = ((_ring).head + (_count)) % size; \ @@ -1451,7 +1447,7 @@ } #define IRDMA_SQ_RING_MOVE_HEAD(_ring, _retcode) \ { \ - register u32 size; \ + u32 size; \ size = (_ring).size; \ if (!IRDMA_SQ_RING_FULL_ERR(_ring)) { \ (_ring).head = ((_ring).head + 1) % size; \ @@ -1462,7 +1458,7 @@ } #define IRDMA_SQ_RING_MOVE_HEAD_BY_COUNT(_ring, _count, _retcode) \ { \ - register u32 size; \ + u32 size; \ size = (_ring).size; \ if ((IRDMA_RING_USED_QUANTA(_ring) + (_count)) < (size - 256)) { \ (_ring).head = ((_ring).head + (_count)) % size; \ @@ -1554,6 +1550,19 @@ IRDMA_WQE_SIZE_256 = 256, }; +enum irdma_ws_op_type { + IRDMA_WS_OP_TYPE_NODE = 0, + IRDMA_WS_OP_TYPE_LEAF_NODE_GROUP, +}; + +enum irdma_ws_rate_limit_flags { + IRDMA_WS_RATE_LIMIT_FLAGS_VALID = 0x1, + IRDMA_WS_NO_RDMA_RATE_LIMIT = 0x2, + IRDMA_WS_LEAF_NODE_IS_PART_GROUP = 0x4, + IRDMA_WS_TREE_RATE_LIMITING = 0x8, + IRDMA_WS_PACING_CONTROL = 0x10, +}; + enum irdma_ws_node_op { IRDMA_ADD_NODE = 0, IRDMA_MODIFY_NODE, @@ -1580,12 +1589,6 @@ IRDMA_FEATURE_BUF_ALIGNMENT = 0x10, }; -enum icrdma_protocol_used { - ICRDMA_ANY_PROTOCOL = 0, - ICRDMA_IWARP_PROTOCOL_ONLY = 1, - ICRDMA_ROCE_PROTOCOL_ONLY = 2, -}; - /** * set_64bit_val - set 64 bit value to hw wqe * @wqe_words: wqe addr to write diff --git a/sys/dev/irdma/irdma_hmc.h b/sys/dev/irdma/irdma_hmc.h --- a/sys/dev/irdma/irdma_hmc.h +++ b/sys/dev/irdma/irdma_hmc.h @@ -1,7 +1,7 @@ /*- * SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB * - * Copyright (c) 2015 - 2022 Intel Corporation + * Copyright (c) 2015 - 2023 Intel Corporation * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU @@ -185,10 +185,6 @@ bool setsd); int irdma_update_sds_noccq(struct irdma_sc_dev *dev, struct irdma_update_sds_info *info); -struct irdma_vfdev *irdma_vfdev_from_fpm(struct irdma_sc_dev *dev, - u16 hmc_fn_id); -struct irdma_hmc_info *irdma_vf_hmcinfo_from_fpm(struct irdma_sc_dev *dev, - u16 hmc_fn_id); int irdma_add_sd_table_entry(struct irdma_hw *hw, struct irdma_hmc_info *hmc_info, u32 sd_index, enum irdma_sd_entry_type type, u64 direct_mode_sz); diff --git a/sys/dev/irdma/irdma_hmc.c b/sys/dev/irdma/irdma_hmc.c --- a/sys/dev/irdma/irdma_hmc.c +++ b/sys/dev/irdma/irdma_hmc.c @@ -1,7 +1,7 @@ /*- * SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB * - * Copyright (c) 2015 - 2022 Intel Corporation + * Copyright (c) 2015 - 2023 Intel Corporation * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU @@ -422,7 +422,7 @@ if (info->start_idx >= info->hmc_info->hmc_obj[info->rsrc_type].cnt) { irdma_debug(dev, IRDMA_DEBUG_HMC, - "error start_idx[%04d] >= [type %04d].cnt[%04d]\n", + "error start_idx[%04d] >= [type %04d].cnt[%04d]\n", info->start_idx, info->rsrc_type, info->hmc_info->hmc_obj[info->rsrc_type].cnt); return -EINVAL; @@ -431,7 +431,7 @@ if ((info->start_idx + info->count) > info->hmc_info->hmc_obj[info->rsrc_type].cnt) { irdma_debug(dev, IRDMA_DEBUG_HMC, - "error start_idx[%04d] + count %04d >= [type %04d].cnt[%04d]\n", + "error start_idx[%04d] + count %04d >= [type %04d].cnt[%04d]\n", info->start_idx, info->count, info->rsrc_type, info->hmc_info->hmc_obj[info->rsrc_type].cnt); return -EINVAL; diff --git a/sys/dev/irdma/irdma_hw.c b/sys/dev/irdma/irdma_hw.c --- a/sys/dev/irdma/irdma_hw.c +++ b/sys/dev/irdma/irdma_hw.c @@ -1,7 +1,7 @@ /*- * SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB * - * Copyright (c) 2015 - 2022 Intel Corporation + * Copyright (c) 2015 - 2023 Intel Corporation * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU @@ -124,8 +124,7 @@ } if (compl_error) { irdma_debug(dev, IRDMA_DEBUG_ERR, - "puda compl_err =0x%x\n", - compl_error); + "puda compl_err = 0x%x\n", compl_error); break; } } while (1); @@ -194,12 +193,11 @@ irdma_complete_cqp_request(struct irdma_cqp *cqp, struct irdma_cqp_request *cqp_request) { - if (cqp_request->waiting) { - cqp_request->request_done = true; + WRITE_ONCE(cqp_request->request_done, true); + if (cqp_request->waiting) wake_up(&cqp_request->waitq); - } else if (cqp_request->callback_fcn) { + else if (cqp_request->callback_fcn) cqp_request->callback_fcn(cqp_request); - } irdma_put_cqp_request(cqp, cqp_request); } @@ -236,8 +234,9 @@ aeqcnt++; irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_AEQ, - "ae_id = 0x%x bool qp=%d qp_id = %d tcp_state=%d iwarp_state=%d ae_src=%d\n", - info->ae_id, info->qp, info->qp_cq_id, info->tcp_state, + "ae_id = 0x%x (%s), is_qp = %d, qp_id = %d, tcp_state = %d, iwarp_state = %d, ae_src = %d\n", + info->ae_id, irdma_get_ae_desc(info->ae_id), + info->qp, info->qp_cq_id, info->tcp_state, info->iwarp_state, info->ae_src); if (info->qp) { @@ -323,7 +322,11 @@ break; case IRDMA_AE_QP_SUSPEND_COMPLETE: if (iwqp->iwdev->vsi.tc_change_pending) { - atomic_dec(&iwqp->sc_qp.vsi->qp_suspend_reqs); + if (!atomic_dec_return(&iwqp->sc_qp.vsi->qp_suspend_reqs)) + wake_up(&iwqp->iwdev->suspend_wq); + } + if (iwqp->suspend_pending) { + iwqp->suspend_pending = false; wake_up(&iwqp->iwdev->suspend_wq); } break; @@ -393,8 +396,10 @@ case IRDMA_AE_LCE_FUNCTION_CATASTROPHIC: case IRDMA_AE_UDA_XMIT_DGRAM_TOO_LONG: default: - irdma_dev_err(&iwdev->ibdev, "abnormal ae_id = 0x%x bool qp=%d qp_id = %d ae_source=%d\n", - info->ae_id, info->qp, info->qp_cq_id, info->ae_src); + irdma_dev_err(&iwdev->ibdev, + "AEQ: abnormal ae_id = 0x%x (%s), is_qp = %d, qp_id = %d, ae_source = %d\n", + info->ae_id, irdma_get_ae_desc(info->ae_id), + info->qp, info->qp_cq_id, info->ae_src); if (rdma_protocol_roce(&iwqp->iwdev->ibdev, 1)) { ctx_info->roce_info->err_rq_idx_valid = info->err_rq_idx_valid; if (info->rq) { @@ -698,8 +703,7 @@ status = irdma_sc_cceq_destroy_done(&iwceq->sc_ceq); if (status) irdma_debug(dev, IRDMA_DEBUG_ERR, - "CEQ destroy completion failed %d\n", - status); + "CEQ destroy completion failed %d\n", status); exit: spin_lock_destroy(&iwceq->ce_lock); spin_lock_destroy(&iwceq->sc_ceq.req_cq_lock); @@ -811,8 +815,7 @@ info.privileged = privileged; if (irdma_sc_del_hmc_obj(dev, &info, reset)) irdma_debug(dev, IRDMA_DEBUG_ERR, - "del HMC obj of type %d failed\n", - obj_type); + "del HMC obj of type %d failed\n", obj_type); } /** @@ -1026,7 +1029,7 @@ irdma_debug(dev, IRDMA_DEBUG_ERR, "cqp create failed - status %d maj_err %d min_err %d\n", status, maj_err, min_err); - goto err_create; + goto err_ctx; } INIT_LIST_HEAD(&cqp->cqp_avail_reqs); @@ -1040,7 +1043,6 @@ init_waitqueue_head(&cqp->remove_wq); return 0; -err_create: err_ctx: irdma_free_dma_mem(dev->hw, &cqp->sq); err_sq: @@ -1155,16 +1157,14 @@ msix_vec->res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE); if (!msix_vec->res) { irdma_debug(&rf->sc_dev, IRDMA_DEBUG_ERR, - "Unable to allocate bus resource int[%d]\n", - rid); + "Unable to allocate bus resource int[%d]\n", rid); return -EINVAL; } err = bus_setup_intr(dev, msix_vec->res, INTR_TYPE_NET | INTR_MPSAFE, NULL, handler, argument, &msix_vec->tag); if (err) { irdma_debug(&rf->sc_dev, IRDMA_DEBUG_ERR, - "Unable to register handler with %x status\n", - err); + "Unable to register handler with %x status\n", err); status = -EINVAL; goto fail_intr; } @@ -1230,20 +1230,21 @@ irdma_cfg_aeq_vector(struct irdma_pci_f *rf) { struct irdma_msix_vector *msix_vec = rf->iw_msixtbl; - u32 ret = 0; + int status = 0; if (!rf->msix_shared) { snprintf(msix_vec->name, sizeof(msix_vec->name) - 1, "irdma-%s-AEQ", dev_name(&rf->pcidev->dev)); tasklet_setup(&rf->dpc_tasklet, irdma_dpc); - ret = irdma_irq_request(rf, msix_vec, irdma_irq_handler, rf); - if (ret) - return ret; + status = irdma_irq_request(rf, msix_vec, irdma_irq_handler, rf); + if (status) + return status; bus_describe_intr(rf->dev_ctx.dev, msix_vec->res, msix_vec->tag, "%s", msix_vec->name); } - if (ret) { + + if (status) { irdma_debug(&rf->sc_dev, IRDMA_DEBUG_ERR, "aeq irq config fail\n"); - return -EINVAL; + return status; } rf->sc_dev.irq_ops->irdma_cfg_aeq(&rf->sc_dev, msix_vec->idx, true); @@ -1338,8 +1339,7 @@ status = irdma_create_ceq(rf, iwceq, 0, &rf->default_vsi); if (status) { irdma_debug(&rf->sc_dev, IRDMA_DEBUG_ERR, - "create ceq status = %d\n", - status); + "create ceq status = %d\n", status); goto exit; } @@ -1394,8 +1394,7 @@ status = irdma_create_ceq(rf, iwceq, ceq_id, vsi); if (status) { irdma_debug(&rf->sc_dev, IRDMA_DEBUG_ERR, - "create ceq status = %d\n", - status); + "create ceq status = %d\n", status); goto del_ceqs; } spin_lock_init(&iwceq->ce_lock); @@ -1634,8 +1633,8 @@ static int irdma_hmc_setup(struct irdma_pci_f *rf) { - int status; struct irdma_sc_dev *dev = &rf->sc_dev; + int status; u32 qpcnt; qpcnt = rsrc_limits_table[rf->limits_sel].qplimit; @@ -2429,8 +2428,9 @@ cqp_info->post_sq = 1; cqp_info->in.u.manage_apbvt_entry.cqp = &iwdev->rf->cqp.sc_cqp; cqp_info->in.u.manage_apbvt_entry.scratch = (uintptr_t)cqp_request; - irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_DEV, "%s: port=0x%04x\n", - (!add_port) ? "DELETE" : "ADD", accel_local_port); + irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_DEV, + "%s: port=0x%04x\n", (!add_port) ? "DELETE" : "ADD", + accel_local_port); status = irdma_handle_cqp_op(iwdev->rf, cqp_request); irdma_put_cqp_request(&iwdev->rf->cqp, cqp_request); @@ -2591,6 +2591,9 @@ if (!cqp_request) return -ENOMEM; + cminfo->cqp_request = cqp_request; + if (!wait) + atomic_inc(&cqp_request->refcnt); cqp_info = &cqp_request->info; info = &cqp_info->in.u.manage_qhash_table_entry.info; memset(info, 0, sizeof(*info)); @@ -2631,19 +2634,24 @@ } if (info->ipv4_valid) irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_CM, - "%s caller: %pS loc_port=0x%04x rem_port=0x%04x loc_addr=%pI4 rem_addr=%pI4 mac=%pM, vlan_id=%d cm_node=%p\n", + "%s caller: %pS loc_port=0x%04x rem_port=0x%04x loc_addr=%x rem_addr=%x mac=%x:%x:%x:%x:%x:%x, vlan_id=%d cm_node=%p\n", (!mtype) ? "DELETE" : "ADD", - __builtin_return_address(0), info->dest_port, - info->src_port, info->dest_ip, info->src_ip, - info->mac_addr, cminfo->vlan_id, - cmnode ? cmnode : NULL); + __builtin_return_address(0), info->src_port, + info->dest_port, info->src_ip[0], info->dest_ip[0], + info->mac_addr[0], info->mac_addr[1], + info->mac_addr[2], info->mac_addr[3], + info->mac_addr[4], info->mac_addr[5], + cminfo->vlan_id, cmnode ? cmnode : NULL); else irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_CM, - "%s caller: %pS loc_port=0x%04x rem_port=0x%04x loc_addr=%pI6 rem_addr=%pI6 mac=%pM, vlan_id=%d cm_node=%p\n", + "%s caller: %pS loc_port=0x%04x rem_port=0x%04x loc_addr=%x:%x:%x:%x rem_addr=%x:%x:%x:%x mac=%x:%x:%x:%x:%x:%x, vlan_id=%d cm_node=%p\n", (!mtype) ? "DELETE" : "ADD", - __builtin_return_address(0), info->dest_port, - info->src_port, info->dest_ip, info->src_ip, - info->mac_addr, cminfo->vlan_id, + __builtin_return_address(0), info->src_port, + info->dest_port, IRDMA_PRINT_IP6(info->src_ip), + IRDMA_PRINT_IP6(info->dest_ip), info->mac_addr[0], + info->mac_addr[1], info->mac_addr[2], + info->mac_addr[3], info->mac_addr[4], + info->mac_addr[5], cminfo->vlan_id, cmnode ? cmnode : NULL); cqp_info->in.u.manage_qhash_table_entry.cqp = &iwdev->rf->cqp.sc_cqp; @@ -2717,7 +2725,8 @@ "qp_id=%d qp_type=%d qpstate=%d ibqpstate=%d last_aeq=%d hw_iw_state=%d maj_err_code=%d min_err_code=%d\n", iwqp->ibqp.qp_num, rf->protocol_used, iwqp->iwarp_state, iwqp->ibqp_state, iwqp->last_aeq, iwqp->hw_iwarp_state, - cqp_request->compl_info.maj_err_code, cqp_request->compl_info.min_err_code); + cqp_request->compl_info.maj_err_code, + cqp_request->compl_info.min_err_code); put_cqp: irdma_put_cqp_request(&rf->cqp, cqp_request); diff --git a/sys/dev/irdma/irdma_kcompat.c b/sys/dev/irdma/irdma_kcompat.c --- a/sys/dev/irdma/irdma_kcompat.c +++ b/sys/dev/irdma/irdma_kcompat.c @@ -1,7 +1,7 @@ /*- * SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB * - * Copyright (c) 2018 - 2022 Intel Corporation + * Copyright (c) 2018 - 2023 Intel Corporation * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU @@ -609,36 +609,37 @@ static void irdma_fill_ah_info(struct vnet *vnet, struct irdma_ah_info *ah_info, const struct ib_gid_attr *sgid_attr, - struct sockaddr *sgid_addr, struct sockaddr *dgid_addr, + union irdma_sockaddr *sgid_addr, + union irdma_sockaddr *dgid_addr, u8 *dmac, u8 net_type) { if (net_type == RDMA_NETWORK_IPV4) { ah_info->ipv4_valid = true; ah_info->dest_ip_addr[0] = - ntohl(((struct sockaddr_in *)dgid_addr)->sin_addr.s_addr); + ntohl(dgid_addr->saddr_in.sin_addr.s_addr); ah_info->src_ip_addr[0] = - ntohl(((struct sockaddr_in *)sgid_addr)->sin_addr.s_addr); + ntohl(sgid_addr->saddr_in.sin_addr.s_addr); CURVNET_SET_QUIET(vnet); ah_info->do_lpbk = irdma_ipv4_is_lpb(ah_info->src_ip_addr[0], ah_info->dest_ip_addr[0]); CURVNET_RESTORE(); - if (ipv4_is_multicast(((struct sockaddr_in *)dgid_addr)->sin_addr.s_addr)) { + if (ipv4_is_multicast(dgid_addr->saddr_in.sin_addr.s_addr)) { irdma_mcast_mac_v4(ah_info->dest_ip_addr, dmac); } } else { irdma_copy_ip_ntohl(ah_info->dest_ip_addr, - ((struct sockaddr_in6 *)dgid_addr)->sin6_addr.__u6_addr.__u6_addr32); + dgid_addr->saddr_in6.sin6_addr.__u6_addr.__u6_addr32); irdma_copy_ip_ntohl(ah_info->src_ip_addr, - ((struct sockaddr_in6 *)sgid_addr)->sin6_addr.__u6_addr.__u6_addr32); + sgid_addr->saddr_in6.sin6_addr.__u6_addr.__u6_addr32); ah_info->do_lpbk = irdma_ipv6_is_lpb(ah_info->src_ip_addr, ah_info->dest_ip_addr); - if (rdma_is_multicast_addr(&((struct sockaddr_in6 *)dgid_addr)->sin6_addr)) { + if (rdma_is_multicast_addr(&dgid_addr->saddr_in6.sin6_addr)) { irdma_mcast_mac_v6(ah_info->dest_ip_addr, dmac); } } } -static inline u8 irdma_get_vlan_ndev_prio(if_t ndev, u8 prio) +static inline u8 irdma_roce_get_vlan_prio(if_t ndev, u8 prio) { return prio; } @@ -666,10 +667,9 @@ ah_info->vlan_tag = 0; if (ah_info->vlan_tag < VLAN_N_VID) { - if_t ndev = sgid_attr->ndev; - ah_info->insert_vlan_tag = true; - vlan_prio = (u16)irdma_get_vlan_ndev_prio(ndev, rt_tos2priority(ah_info->tc_tos)); + vlan_prio = (u16)irdma_roce_get_vlan_prio(sgid_attr->ndev, + rt_tos2priority(ah_info->tc_tos)); ah_info->vlan_tag |= vlan_prio << VLAN_PRIO_SHIFT; irdma_find_qp_update_qs(iwdev->rf, pd, vlan_prio); } @@ -726,14 +726,10 @@ struct irdma_sc_ah *sc_ah; u32 ah_id = 0; struct irdma_ah_info *ah_info; - struct irdma_create_ah_resp uresp; - union { - struct sockaddr saddr; - struct sockaddr_in saddr_in; - struct sockaddr_in6 saddr_in6; - } sgid_addr, dgid_addr; + struct irdma_create_ah_resp uresp = {}; + union irdma_sockaddr sgid_addr, dgid_addr; int err; - u8 dmac[ETH_ALEN]; + u8 dmac[ETHER_ADDR_LEN]; bool sleep = (flags & RDMA_CREATE_AH_SLEEPABLE) != 0; if (udata && udata->outlen < IRDMA_CREATE_AH_MIN_RESP_LEN) @@ -766,15 +762,11 @@ rdma_gid2ip((struct sockaddr *)&sgid_addr, &sgid); rdma_gid2ip((struct sockaddr *)&dgid_addr, &attr->grh.dgid); ah->av.attrs = *attr; - ah->av.net_type = kc_rdma_gid_attr_network_type(sgid_attr, - sgid_attr.gid_type, - &sgid); + ah->av.net_type = ib_gid_to_network_type(sgid_attr.gid_type, &sgid); if (sgid_attr.ndev) dev_put(sgid_attr.ndev); - ah->av.sgid_addr.saddr = sgid_addr.saddr; - ah->av.dgid_addr.saddr = dgid_addr.saddr; ah_info = &sc_ah->ah_info; ah_info->ah_idx = ah_id; ah_info->pd_idx = pd->sc_pd.pd_id; @@ -788,7 +780,7 @@ ether_addr_copy(dmac, attr->dmac); - irdma_fill_ah_info(if_getvnet(iwdev->netdev), ah_info, &sgid_attr, &sgid_addr.saddr, &dgid_addr.saddr, + irdma_fill_ah_info(if_getvnet(iwdev->netdev), ah_info, &sgid_attr, &sgid_addr, &dgid_addr, dmac, ah->av.net_type); err = irdma_create_ah_vlan_tag(iwdev, pd, ah_info, &sgid_attr, dmac); @@ -810,7 +802,7 @@ if (udata) { uresp.ah_id = ah->sc_ah.ah_info.ah_idx; - err = ib_copy_to_udata(udata, &uresp, sizeof(uresp)); + err = ib_copy_to_udata(udata, &uresp, min(sizeof(uresp), udata->outlen)); if (err) { irdma_ah_cqp_op(iwdev->rf, &ah->sc_ah, IRDMA_OP_AH_DESTROY, false, NULL, ah); @@ -888,14 +880,10 @@ struct irdma_sc_ah *sc_ah; u32 ah_id = 0; struct irdma_ah_info *ah_info; - struct irdma_create_ah_resp uresp; - union { - struct sockaddr saddr; - struct sockaddr_in saddr_in; - struct sockaddr_in6 saddr_in6; - } sgid_addr, dgid_addr; + struct irdma_create_ah_resp uresp = {}; + union irdma_sockaddr sgid_addr, dgid_addr; int err; - u8 dmac[ETH_ALEN]; + u8 dmac[ETHER_ADDR_LEN]; bool sleep = udata ? true : false; if (udata && udata->outlen < IRDMA_CREATE_AH_MIN_RESP_LEN) @@ -934,15 +922,11 @@ rdma_gid2ip((struct sockaddr *)&sgid_addr, &sgid); rdma_gid2ip((struct sockaddr *)&dgid_addr, &attr->grh.dgid); ah->av.attrs = *attr; - ah->av.net_type = kc_rdma_gid_attr_network_type(sgid_attr, - sgid_attr.gid_type, - &sgid); + ah->av.net_type = ib_gid_to_network_type(sgid_attr.gid_type, &sgid); if (sgid_attr.ndev) dev_put(sgid_attr.ndev); - ah->av.sgid_addr.saddr = sgid_addr.saddr; - ah->av.dgid_addr.saddr = dgid_addr.saddr; ah_info = &sc_ah->ah_info; ah_info->ah_idx = ah_id; ah_info->pd_idx = pd->sc_pd.pd_id; @@ -958,7 +942,7 @@ ib_resolve_eth_dmac(ibpd->device, attr); irdma_ether_copy(dmac, attr); - irdma_fill_ah_info(if_getvnet(iwdev->netdev), ah_info, &sgid_attr, &sgid_addr.saddr, &dgid_addr.saddr, + irdma_fill_ah_info(if_getvnet(iwdev->netdev), ah_info, &sgid_attr, &sgid_addr, &dgid_addr, dmac, ah->av.net_type); err = irdma_create_ah_vlan_tag(iwdev, pd, ah_info, &sgid_attr, dmac); @@ -980,7 +964,7 @@ if (udata) { uresp.ah_id = ah->sc_ah.ah_info.ah_idx; - err = ib_copy_to_udata(udata, &uresp, sizeof(uresp)); + err = ib_copy_to_udata(udata, &uresp, min(sizeof(uresp), udata->outlen)); if (err) { irdma_ah_cqp_op(iwdev->rf, &ah->sc_ah, IRDMA_OP_AH_DESTROY, false, NULL, ah); @@ -1129,6 +1113,8 @@ init_waitqueue_head(&iwqp->waitq); init_waitqueue_head(&iwqp->mod_qp_waitq); + spin_lock_init(&iwqp->dwork_flush_lock); + if (udata) { init_info.qp_uk_init_info.abi_ver = iwpd->sc_pd.abi_ver; err_code = irdma_setup_umode_qp(udata, iwdev, iwqp, &init_info, init_attr); @@ -1257,15 +1243,9 @@ list_del(&iwqp->ud_list_elem); spin_unlock_irqrestore(&iwqp->iwpd->udqp_list_lock, flags); - if (iwqp->iwarp_state == IRDMA_QP_STATE_RTS) + if (iwqp->iwarp_state >= IRDMA_QP_STATE_IDLE) irdma_modify_qp_to_err(&iwqp->sc_qp); - irdma_qp_rem_ref(&iwqp->ibqp); - wait_for_completion(&iwqp->free_qp); - irdma_free_lsmm_rsrc(iwqp); - if (!iwdev->rf->reset && irdma_cqp_qp_destroy_cmd(&iwdev->rf->sc_dev, &iwqp->sc_qp)) - return (iwdev->rf->rdma_ver <= IRDMA_GEN_2 && !iwqp->user_mode) ? 0 : -ENOTRECOVERABLE; -free_rsrc: if (!iwqp->user_mode) { if (iwqp->iwscq) { irdma_clean_cqes(iwqp, iwqp->iwscq); @@ -1273,6 +1253,12 @@ irdma_clean_cqes(iwqp, iwqp->iwrcq); } } + irdma_qp_rem_ref(&iwqp->ibqp); + wait_for_completion(&iwqp->free_qp); + irdma_free_lsmm_rsrc(iwqp); + if (!iwdev->rf->reset && irdma_cqp_qp_destroy_cmd(&iwdev->rf->sc_dev, &iwqp->sc_qp)) + return (iwdev->rf->rdma_ver <= IRDMA_GEN_2 && !iwqp->user_mode) ? 0 : -ENOTRECOVERABLE; +free_rsrc: irdma_remove_push_mmap_entries(iwqp); irdma_free_qp_rsrc(iwqp); @@ -1464,7 +1450,7 @@ if (!iwcq->kmem_shadow.va) { err_code = -ENOMEM; - goto cq_free_rsrc; + goto cq_kmem_free; } info.shadow_area_pa = iwcq->kmem_shadow.pa; ukinfo->shadow_area = iwcq->kmem_shadow.va; @@ -1472,19 +1458,18 @@ info.cq_base_pa = iwcq->kmem.pa; } - if (dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2) - info.shadow_read_threshold = min(info.cq_uk_init_info.cq_size / 2, - (u32)IRDMA_MAX_CQ_READ_THRESH); + info.shadow_read_threshold = min(info.cq_uk_init_info.cq_size / 2, + (u32)IRDMA_MAX_CQ_READ_THRESH); if (irdma_sc_cq_init(cq, &info)) { irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_VERBS, "init cq fail\n"); err_code = -EPROTO; - goto cq_free_rsrc; + goto cq_kmem_free; } cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true); if (!cqp_request) { err_code = -ENOMEM; - goto cq_free_rsrc; + goto cq_kmem_free; } cqp_info = &cqp_request->info; cqp_info->cqp_cmd = IRDMA_OP_CQ_CREATE; @@ -1496,7 +1481,7 @@ irdma_put_cqp_request(&rf->cqp, cqp_request); if (status) { err_code = -ENOMEM; - goto cq_free_rsrc; + goto cq_kmem_free; } if (udata) { @@ -1522,8 +1507,13 @@ #endif cq_destroy: irdma_cq_wq_destroy(rf, cq); +cq_kmem_free: + if (!iwcq->user_mode) { + irdma_free_dma_mem(dev->hw, &iwcq->kmem); + irdma_free_dma_mem(dev->hw, &iwcq->kmem_shadow); + } cq_free_rsrc: - irdma_cq_free_rsrc(rf, iwcq); + irdma_free_rsrc(rf, rf->allocated_cqs, cq_num); #if __FreeBSD_version >= 1400026 return err_code; #else @@ -1737,9 +1727,7 @@ ether_addr_copy(iwqp->ctx_info.roce_info->mac_addr, if_getlladdr(sgid_attr.ndev)); } - av->net_type = kc_rdma_gid_attr_network_type(sgid_attr, - sgid_attr.gid_type, - &sgid); + av->net_type = ib_gid_to_network_type(sgid_attr.gid_type, &sgid); rdma_gid2ip((struct sockaddr *)&av->sgid_addr, &sgid); dev_put(sgid_attr.ndev); iwqp->sc_qp.user_pri = iwqp->ctx_info.user_pri; @@ -1837,6 +1825,9 @@ int err_code; u32 stag; + if (type != IB_MW_TYPE_1 && type != IB_MW_TYPE_2) + return ERR_PTR(-EINVAL); + iwmr = kzalloc(sizeof(*iwmr), GFP_KERNEL); if (!iwmr) return ERR_PTR(-ENOMEM); @@ -2154,7 +2145,7 @@ if (rdma_protocol_roce(ibdev, 1)) { props->gid_tbl_len = 32; - kc_set_props_ip_gid_caps(props); + props->port_cap_flags |= IB_PORT_IP_BASED_GIDS; props->pkey_tbl_len = IRDMA_PKEY_TBL_SZ; } else { props->gid_tbl_len = 1; @@ -2381,3 +2372,15 @@ return 0; } + +u64 +irdma_mac_to_u64(const u8 *eth_add) +{ + int idx; + u64 u64_eth_add; + + for (idx = 0, u64_eth_add = 0; idx < ETHER_ADDR_LEN; idx++) + u64_eth_add = u64_eth_add << 8 | eth_add[idx]; + + return u64_eth_add; +} diff --git a/sys/dev/irdma/irdma_main.h b/sys/dev/irdma/irdma_main.h --- a/sys/dev/irdma/irdma_main.h +++ b/sys/dev/irdma/irdma_main.h @@ -1,7 +1,7 @@ /*- * SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB * - * Copyright (c) 2015 - 2022 Intel Corporation + * Copyright (c) 2015 - 2023 Intel Corporation * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU @@ -152,6 +152,11 @@ IP_ADDR_REGISTERED, /* Last state of open */ }; +struct ae_desc { + u16 id; + const char *desc; +}; + struct irdma_rsrc_limits { u32 qplimit; u32 mrlimit; @@ -180,8 +185,8 @@ void (*callback_fcn)(struct irdma_cqp_request *cqp_request); void *param; struct irdma_cqp_compl_info compl_info; + bool request_done; /* READ/WRITE_ONCE macros operate on it */ bool waiting:1; - bool request_done:1; bool dynamic:1; }; @@ -224,7 +229,7 @@ struct irdma_arp_entry { u32 ip_addr[4]; - u8 mac_addr[ETH_ALEN]; + u8 mac_addr[ETHER_ADDR_LEN]; }; struct irdma_msix_vector { @@ -535,6 +540,7 @@ int irdma_alloc_local_mac_entry(struct irdma_pci_f *rf, u16 *mac_tbl_idx); int irdma_add_local_mac_entry(struct irdma_pci_f *rf, const u8 *mac_addr, u16 idx); void irdma_del_local_mac_entry(struct irdma_pci_f *rf, u16 idx); +const char *irdma_get_ae_desc(u16 ae_id); u32 irdma_initialize_hw_rsrc(struct irdma_pci_f *rf); void irdma_port_ibevent(struct irdma_device *iwdev); diff --git a/sys/dev/irdma/irdma_pble.h b/sys/dev/irdma/irdma_pble.h --- a/sys/dev/irdma/irdma_pble.h +++ b/sys/dev/irdma/irdma_pble.h @@ -1,7 +1,7 @@ /*- * SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB * - * Copyright (c) 2015 - 2022 Intel Corporation + * Copyright (c) 2015 - 2023 Intel Corporation * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU @@ -102,7 +102,7 @@ struct irdma_chunk { struct list_head list; struct irdma_dma_info dmainfo; - void *bitmapbuf; + unsigned long *bitmapbuf; u32 sizeofbitmap; u64 size; @@ -111,7 +111,6 @@ u32 pg_cnt; enum irdma_alloc_type type; struct irdma_sc_dev *dev; - struct irdma_virt_mem bitmapmem; struct irdma_virt_mem chunkmem; }; @@ -162,5 +161,4 @@ unsigned long *flags); void irdma_pble_free_paged_mem(struct irdma_chunk *chunk); int irdma_pble_get_paged_mem(struct irdma_chunk *chunk, u32 pg_cnt); -void irdma_prm_rem_bitmapmem(struct irdma_hw *hw, struct irdma_chunk *chunk); #endif /* IRDMA_PBLE_H */ diff --git a/sys/dev/irdma/irdma_pble.c b/sys/dev/irdma/irdma_pble.c --- a/sys/dev/irdma/irdma_pble.c +++ b/sys/dev/irdma/irdma_pble.c @@ -1,7 +1,7 @@ /*- * SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB * - * Copyright (c) 2015 - 2022 Intel Corporation + * Copyright (c) 2015 - 2023 Intel Corporation * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU @@ -57,8 +57,7 @@ list_del(&chunk->list); if (chunk->type == PBLE_SD_PAGED) irdma_pble_free_paged_mem(chunk); - if (chunk->bitmapbuf) - irdma_prm_rem_bitmapmem(pble_rsrc->dev->hw, chunk); + bitmap_free(chunk->bitmapbuf); kfree(chunk->chunkmem.va); } spin_lock_destroy(&pinfo->prm_lock); @@ -290,7 +289,8 @@ irdma_debug(dev, IRDMA_DEBUG_PBLE, "pages = %d, unallocated_pble[%d] current_fpm_addr = %lx\n", - pages, pble_rsrc->unallocated_pble, pble_rsrc->next_fpm_addr); + pages, pble_rsrc->unallocated_pble, + pble_rsrc->next_fpm_addr); irdma_debug(dev, IRDMA_DEBUG_PBLE, "sd_entry_type = %d\n", sd_entry_type); if (sd_entry_type == IRDMA_SD_TYPE_DIRECT) @@ -304,14 +304,14 @@ if (sd_entry_type == IRDMA_SD_TYPE_PAGED) { ret_code = add_bp_pages(pble_rsrc, &info); if (ret_code) - goto error; + goto err_bp_pages; else pble_rsrc->stats_paged_sds++; } ret_code = irdma_prm_add_pble_mem(&pble_rsrc->pinfo, chunk); if (ret_code) - goto error; + goto err_bp_pages; pble_rsrc->next_fpm_addr += chunk->size; irdma_debug(dev, IRDMA_DEBUG_PBLE, @@ -333,8 +333,8 @@ return 0; error: - if (chunk->bitmapbuf) - irdma_prm_rem_bitmapmem(pble_rsrc->dev->hw, chunk); + bitmap_free(chunk->bitmapbuf); +err_bp_pages: kfree(chunk->chunkmem.va); return ret_code; diff --git a/sys/dev/irdma/irdma_protos.h b/sys/dev/irdma/irdma_protos.h --- a/sys/dev/irdma/irdma_protos.h +++ b/sys/dev/irdma/irdma_protos.h @@ -1,7 +1,7 @@ /*- * SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB * - * Copyright (c) 2016 - 2022 Intel Corporation + * Copyright (c) 2016 - 2023 Intel Corporation * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU @@ -61,8 +61,6 @@ struct irdma_vsi_pestat *pestat, bool wait); int irdma_cqp_ws_node_cmd(struct irdma_sc_dev *dev, u8 cmd, struct irdma_ws_node_info *node_info); -int irdma_cqp_up_map_cmd(struct irdma_sc_dev *dev, u8 cmd, - struct irdma_up_info *map_info); int irdma_cqp_ceq_cmd(struct irdma_sc_dev *dev, struct irdma_sc_ceq *sc_ceq, u8 op); int irdma_cqp_aeq_cmd(struct irdma_sc_dev *dev, struct irdma_sc_aeq *sc_aeq, @@ -131,8 +129,6 @@ struct irdma_dma_mem *val_mem, u16 hmc_fn_id); int irdma_alloc_query_fpm_buf(struct irdma_sc_dev *dev, struct irdma_dma_mem *mem); -int irdma_cqp_manage_hmc_fcn_cmd(struct irdma_sc_dev *dev, - struct irdma_hmc_fcn_info *hmcfcninfo, - u16 *pmf_idx); void *irdma_remove_cqp_head(struct irdma_sc_dev *dev); +u64 irdma_mac_to_u64(const u8 *eth_add); #endif /* IRDMA_PROTOS_H */ diff --git a/sys/dev/irdma/irdma_puda.h b/sys/dev/irdma/irdma_puda.h --- a/sys/dev/irdma/irdma_puda.h +++ b/sys/dev/irdma/irdma_puda.h @@ -1,7 +1,7 @@ /*- * SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB * - * Copyright (c) 2015 - 2022 Intel Corporation + * Copyright (c) 2015 - 2023 Intel Corporation * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU @@ -72,7 +72,7 @@ bool ipv4:1; bool smac_valid:1; bool vlan_valid:1; - u8 smac[ETH_ALEN]; + u8 smac[ETHER_ADDR_LEN]; }; struct irdma_puda_send_info { @@ -109,7 +109,7 @@ bool smac_valid:1; u32 seqnum; u32 ah_id; - u8 smac[ETH_ALEN]; + u8 smac[ETHER_ADDR_LEN]; struct irdma_sc_vsi *vsi; }; diff --git a/sys/dev/irdma/irdma_puda.c b/sys/dev/irdma/irdma_puda.c --- a/sys/dev/irdma/irdma_puda.c +++ b/sys/dev/irdma/irdma_puda.c @@ -1,7 +1,7 @@ /*- * SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB * - * Copyright (c) 2015 - 2022 Intel Corporation + * Copyright (c) 2015 - 2023 Intel Corporation * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU @@ -270,6 +270,9 @@ if (valid_bit != cq_uk->polarity) return -ENOENT; + /* Ensure CQE contents are read after valid bit is checked */ + rmb(); + if (cq->dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2) ext_valid = (bool)FIELD_GET(IRDMA_CQ_EXTCQE, qword3); @@ -283,6 +286,9 @@ if (polarity != cq_uk->polarity) return -ENOENT; + /* Ensure ext CQE contents are read after ext valid bit is checked */ + rmb(); + IRDMA_RING_MOVE_HEAD_NOCHECK(cq_uk->cq_ring); if (!IRDMA_RING_CURRENT_HEAD(cq_uk->cq_ring)) cq_uk->polarity = !cq_uk->polarity; diff --git a/sys/dev/irdma/irdma_type.h b/sys/dev/irdma/irdma_type.h --- a/sys/dev/irdma/irdma_type.h +++ b/sys/dev/irdma/irdma_type.h @@ -1,7 +1,7 @@ /*- * SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB * - * Copyright (c) 2015 - 2022 Intel Corporation + * Copyright (c) 2015 - 2023 Intel Corporation * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU @@ -398,6 +398,8 @@ struct irdma_dcqcn_cc_params dcqcn_params; __le64 *host_ctx; u64 *scratch_array; + u64 requested_ops; + atomic64_t completed_ops; u32 cqp_id; u32 sq_size; u32 hw_sq_size; @@ -532,7 +534,7 @@ bool use_cnp_up_override:1; }; -#define IRDMA_MAX_WS_NODES 0x3FF +#define IRDMA_MAX_WS_NODES 0x3FF #define IRDMA_WS_NODE_INVALID 0xFFFF struct irdma_ws_node_info { @@ -598,7 +600,7 @@ bool tc_change_pending:1; bool mtu_change_pending:1; struct irdma_vsi_pestat *pestat; - ATOMIC qp_suspend_reqs; + atomic_t qp_suspend_reqs; int (*register_qset)(struct irdma_sc_vsi *vsi, struct irdma_ws_node *tc_node); void (*unregister_qset)(struct irdma_sc_vsi *vsi, @@ -628,8 +630,6 @@ u32 IOMEM *aeq_alloc_db; u32 IOMEM *cqp_db; u32 IOMEM *cq_ack_db; - u32 IOMEM *ceq_itr_mask_db; - u32 IOMEM *aeq_itr_mask_db; u32 IOMEM *hw_regs[IRDMA_MAX_REGS]; u32 ceq_itr; /* Interrupt throttle, usecs between interrupts: 0 disabled. 2 - 8160 */ u64 hw_masks[IRDMA_MAX_MASKS]; @@ -782,7 +782,6 @@ u32 first_pm_pbl_idx; struct irdma_sc_vsi *vsi; struct irdma_sc_cq **reg_cq; - u32 reg_cq_idx; }; struct irdma_aeq_init_info { @@ -869,7 +868,7 @@ u16 t_high; u16 t_low; u8 last_byte_sent; - u8 mac_addr[ETH_ALEN]; + u8 mac_addr[ETHER_ADDR_LEN]; u8 rtomin; }; @@ -901,7 +900,7 @@ u16 t_high; u16 t_low; u8 last_byte_sent; - u8 mac_addr[ETH_ALEN]; + u8 mac_addr[ETHER_ADDR_LEN]; u8 rtomin; }; @@ -968,7 +967,6 @@ u32 rcv_cq_num; u32 rem_endpoint_idx; u16 stats_idx; - bool srq_valid:1; bool tcp_info_valid:1; bool iwarp_info_valid:1; bool stats_idx_valid:1; @@ -1134,7 +1132,7 @@ }; struct irdma_add_arp_cache_entry_info { - u8 mac_addr[ETH_ALEN]; + u8 mac_addr[ETHER_ADDR_LEN]; u32 reach_max; u16 arp_index; bool permanent; @@ -1151,7 +1149,7 @@ enum irdma_quad_entry_type entry_type; bool vlan_valid:1; bool ipv4_valid:1; - u8 mac_addr[ETH_ALEN]; + u8 mac_addr[ETHER_ADDR_LEN]; u16 vlan_id; u8 user_pri; u32 qp_num; @@ -1249,7 +1247,6 @@ bool post_sq); void irdma_sc_send_lsmm(struct irdma_sc_qp *qp, void *lsmm_buf, u32 size, irdma_stag stag); -void irdma_sc_send_lsmm_nostag(struct irdma_sc_qp *qp, void *lsmm_buf, u32 size); void irdma_sc_send_rtt(struct irdma_sc_qp *qp, bool read); void irdma_sc_qp_setctx(struct irdma_sc_qp *qp, __le64 *qp_ctx, struct irdma_qp_host_ctx_info *info); diff --git a/sys/dev/irdma/irdma_uda.h b/sys/dev/irdma/irdma_uda.h --- a/sys/dev/irdma/irdma_uda.h +++ b/sys/dev/irdma/irdma_uda.h @@ -1,7 +1,7 @@ /*- * SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB * - * Copyright (c) 2016 - 2021 Intel Corporation + * Copyright (c) 2016 - 2023 Intel Corporation * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU @@ -54,7 +54,7 @@ u8 insert_vlan_tag; u8 tc_tos; u8 hop_ttl; - u8 mac_addr[ETH_ALEN]; + u8 mac_addr[ETHER_ADDR_LEN]; bool ah_valid:1; bool ipv4_valid:1; bool do_lpbk:1; diff --git a/sys/dev/irdma/irdma_uda.c b/sys/dev/irdma/irdma_uda.c --- a/sys/dev/irdma/irdma_uda.c +++ b/sys/dev/irdma/irdma_uda.c @@ -1,7 +1,7 @@ /*- * SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB * - * Copyright (c) 2016 - 2022 Intel Corporation + * Copyright (c) 2016 - 2023 Intel Corporation * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU @@ -59,12 +59,8 @@ if (!wqe) return -ENOSPC; - set_64bit_val(wqe, IRDMA_BYTE_0, LS_64_1(info->mac_addr[5], 16) | - LS_64_1(info->mac_addr[4], 24) | - LS_64_1(info->mac_addr[3], 32) | - LS_64_1(info->mac_addr[2], 40) | - LS_64_1(info->mac_addr[1], 48) | - LS_64_1(info->mac_addr[0], 56)); + set_64bit_val(wqe, IRDMA_BYTE_0, + FIELD_PREP(IRDMAQPC_MACADDRESS, irdma_mac_to_u64(info->mac_addr))); qw1 = FIELD_PREP(IRDMA_UDA_CQPSQ_MAV_PDINDEXLO, info->pd_idx) | FIELD_PREP(IRDMA_UDA_CQPSQ_MAV_TC, info->tc_tos) | @@ -175,12 +171,7 @@ set_64bit_val(wqe, IRDMA_BYTE_16, FIELD_PREP(IRDMA_UDA_CQPSQ_MG_VLANID, info->vlan_id) | FIELD_PREP(IRDMA_UDA_CQPSQ_QS_HANDLE, info->qs_handle)); - set_64bit_val(wqe, IRDMA_BYTE_0, LS_64_1(info->dest_mac_addr[5], 0) | - LS_64_1(info->dest_mac_addr[4], 8) | - LS_64_1(info->dest_mac_addr[3], 16) | - LS_64_1(info->dest_mac_addr[2], 24) | - LS_64_1(info->dest_mac_addr[1], 32) | - LS_64_1(info->dest_mac_addr[0], 40)); + set_64bit_val(wqe, IRDMA_BYTE_0, irdma_mac_to_u64(info->dest_mac_addr)); set_64bit_val(wqe, IRDMA_BYTE_8, FIELD_PREP(IRDMA_UDA_CQPSQ_MG_HMC_FCN_ID, info->hmc_fcn_id)); diff --git a/sys/dev/irdma/irdma_uk.c b/sys/dev/irdma/irdma_uk.c --- a/sys/dev/irdma/irdma_uk.c +++ b/sys/dev/irdma/irdma_uk.c @@ -1,7 +1,7 @@ /*- * SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB * - * Copyright (c) 2015 - 2022 Intel Corporation + * Copyright (c) 2015 - 2023 Intel Corporation * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU @@ -275,7 +275,8 @@ if (qp->uk_attrs->hw_rev == IRDMA_GEN_1 && wqe_quanta == 1 && (IRDMA_RING_CURRENT_HEAD(qp->sq_ring) & 1)) { wqe_0 = qp->sq_base[IRDMA_RING_CURRENT_HEAD(qp->sq_ring)].elem; - wqe_0[3] = cpu_to_le64(FIELD_PREP(IRDMAQPSQ_VALID, !qp->swqe_polarity)); + wqe_0[3] = cpu_to_le64(FIELD_PREP(IRDMAQPSQ_VALID, + qp->swqe_polarity ? 0 : 1)); } qp->sq_wrtrk_array[*wqe_idx].wrid = info->wr_id; qp->sq_wrtrk_array[*wqe_idx].wr_len = total_size; @@ -596,22 +597,6 @@ return 0; } -/** - * irdma_set_mw_bind_wqe_gen_1 - set mw bind wqe - * @wqe: wqe for setting fragment - * @op_info: info for setting bind wqe values - */ -static void -irdma_set_mw_bind_wqe_gen_1(__le64 * wqe, - struct irdma_bind_window *op_info) -{ - set_64bit_val(wqe, IRDMA_BYTE_0, (uintptr_t)op_info->va); - set_64bit_val(wqe, IRDMA_BYTE_8, - FIELD_PREP(IRDMAQPSQ_PARENTMRSTAG, op_info->mw_stag) | - FIELD_PREP(IRDMAQPSQ_MWSTAG, op_info->mr_stag)); - set_64bit_val(wqe, IRDMA_BYTE_16, op_info->bind_len); -} - /** * irdma_copy_inline_data_gen_1 - Copy inline data to wqe * @wqe: pointer to wqe @@ -659,22 +644,6 @@ return data_size <= 16 ? IRDMA_QP_WQE_MIN_QUANTA : 2; } -/** - * irdma_set_mw_bind_wqe - set mw bind in wqe - * @wqe: wqe for setting mw bind - * @op_info: info for setting wqe values - */ -static void -irdma_set_mw_bind_wqe(__le64 * wqe, - struct irdma_bind_window *op_info) -{ - set_64bit_val(wqe, IRDMA_BYTE_0, (uintptr_t)op_info->va); - set_64bit_val(wqe, IRDMA_BYTE_8, - FIELD_PREP(IRDMAQPSQ_PARENTMRSTAG, op_info->mr_stag) | - FIELD_PREP(IRDMAQPSQ_MWSTAG, op_info->mw_stag)); - set_64bit_val(wqe, IRDMA_BYTE_16, op_info->bind_len); -} - /** * irdma_copy_inline_data - Copy inline data to wqe * @wqe: pointer to wqe @@ -1547,14 +1516,12 @@ .iw_copy_inline_data = irdma_copy_inline_data, .iw_inline_data_size_to_quanta = irdma_inline_data_size_to_quanta, .iw_set_fragment = irdma_set_fragment, - .iw_set_mw_bind_wqe = irdma_set_mw_bind_wqe, }; static const struct irdma_wqe_uk_ops iw_wqe_uk_ops_gen_1 = { .iw_copy_inline_data = irdma_copy_inline_data_gen_1, .iw_inline_data_size_to_quanta = irdma_inline_data_size_to_quanta_gen_1, .iw_set_fragment = irdma_set_fragment_gen_1, - .iw_set_mw_bind_wqe = irdma_set_mw_bind_wqe_gen_1, }; /** @@ -1760,6 +1727,9 @@ if (polarity != temp) break; + /* Ensure CQE contents are read after valid bit is checked */ + rmb(); + get_64bit_val(cqe, IRDMA_BYTE_8, &comp_ctx); if ((void *)(irdma_uintptr) comp_ctx == q) set_64bit_val(cqe, IRDMA_BYTE_8, 0); @@ -1771,48 +1741,6 @@ return 0; } -/** - * irdma_nop - post a nop - * @qp: hw qp ptr - * @wr_id: work request id - * @signaled: signaled for completion - * @post_sq: ring doorbell - */ -int -irdma_nop(struct irdma_qp_uk *qp, u64 wr_id, bool signaled, bool post_sq) -{ - __le64 *wqe; - u64 hdr; - u32 wqe_idx; - struct irdma_post_sq_info info = {0}; - u16 quanta = IRDMA_QP_WQE_MIN_QUANTA; - - info.push_wqe = qp->push_db ? true : false; - info.wr_id = wr_id; - wqe = irdma_qp_get_next_send_wqe(qp, &wqe_idx, &quanta, 0, &info); - if (!wqe) - return -ENOSPC; - - set_64bit_val(wqe, IRDMA_BYTE_0, 0); - set_64bit_val(wqe, IRDMA_BYTE_8, 0); - set_64bit_val(wqe, IRDMA_BYTE_16, 0); - - hdr = FIELD_PREP(IRDMAQPSQ_OPCODE, IRDMAQP_OP_NOP) | - FIELD_PREP(IRDMAQPSQ_SIGCOMPL, signaled) | - FIELD_PREP(IRDMAQPSQ_VALID, qp->swqe_polarity); - - irdma_wmb(); /* make sure WQE is populated before valid bit is set */ - - set_64bit_val(wqe, IRDMA_BYTE_24, hdr); - - if (info.push_wqe) - irdma_qp_push_wqe(qp, wqe, quanta, wqe_idx, post_sq); - else if (post_sq) - irdma_uk_qp_post_wr(qp); - - return 0; -} - /** * irdma_fragcnt_to_quanta_sq - calculate quanta based on fragment count for SQ * @frag_cnt: number of fragments diff --git a/sys/dev/irdma/irdma_user.h b/sys/dev/irdma/irdma_user.h --- a/sys/dev/irdma/irdma_user.h +++ b/sys/dev/irdma/irdma_user.h @@ -1,7 +1,7 @@ /*- * SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB * - * Copyright (c) 2015 - 2022 Intel Corporation + * Copyright (c) 2015 - 2023 Intel Corporation * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU @@ -568,7 +568,6 @@ u32 inline_data, u8 *shift); int irdma_get_sqdepth(struct irdma_uk_attrs *uk_attrs, u32 sq_size, u8 shift, u32 *sqdepth); int irdma_get_rqdepth(struct irdma_uk_attrs *uk_attrs, u32 rq_size, u8 shift, u32 *rqdepth); -int irdma_get_srqdepth(struct irdma_uk_attrs *uk_attrs, u32 srq_size, u8 shift, u32 *srqdepth); void irdma_qp_push_wqe(struct irdma_qp_uk *qp, __le64 *wqe, u16 quanta, u32 wqe_idx, bool post_sq); void irdma_clr_wqes(struct irdma_qp_uk *qp, u32 qp_wqe_idx); diff --git a/sys/dev/irdma/irdma_utils.c b/sys/dev/irdma/irdma_utils.c --- a/sys/dev/irdma/irdma_utils.c +++ b/sys/dev/irdma/irdma_utils.c @@ -1,7 +1,7 @@ /*- * SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB * - * Copyright (c) 2015 - 2022 Intel Corporation + * Copyright (c) 2015 - 2023 Intel Corporation * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU @@ -38,6 +38,169 @@ LIST_HEAD(irdma_handlers); DEFINE_SPINLOCK(irdma_handler_lock); +static const struct ae_desc ae_desc_list[] = { + {IRDMA_AE_AMP_UNALLOCATED_STAG, "Unallocated memory key (L-Key/R-Key)"}, + {IRDMA_AE_AMP_INVALID_STAG, "Invalid memory key (L-Key/R-Key)"}, + {IRDMA_AE_AMP_BAD_QP, + "Memory protection error: Accessing Memory Window (MW) which belongs to a different QP"}, + {IRDMA_AE_AMP_BAD_PD, + "Memory protection error: Accessing Memory Window (MW)/Memory Region (MR) which belongs to a different PD"}, + {IRDMA_AE_AMP_BAD_STAG_KEY, "Bad memory key (L-Key/R-Key)"}, + {IRDMA_AE_AMP_BAD_STAG_INDEX, "Bad memory key (L-Key/R-Key): Too large memory key index"}, + {IRDMA_AE_AMP_BOUNDS_VIOLATION, "Memory Window (MW)/Memory Region (MR) bounds violation"}, + {IRDMA_AE_AMP_RIGHTS_VIOLATION, "Memory Window (MW)/Memory Region (MR) rights violation"}, + {IRDMA_AE_AMP_TO_WRAP, + "Memory protection error: The address within Memory Window (MW)/Memory Region (MR) wraps"}, + {IRDMA_AE_AMP_FASTREG_VALID_STAG, + "Fastreg error: Registration to a valid MR"}, + {IRDMA_AE_AMP_FASTREG_MW_STAG, + "Fastreg error: Registration to a valid Memory Window (MW)"}, + {IRDMA_AE_AMP_FASTREG_INVALID_RIGHTS, "Fastreg error: Invalid rights"}, + {IRDMA_AE_AMP_FASTREG_INVALID_LENGTH, "Fastreg error: Invalid length"}, + {IRDMA_AE_AMP_INVALIDATE_SHARED, "Attempt to invalidate a shared MR"}, + {IRDMA_AE_AMP_INVALIDATE_NO_REMOTE_ACCESS_RIGHTS, + "Attempt to remotely invalidate Memory Window (MW)/Memory Region (MR) without rights"}, + {IRDMA_AE_AMP_INVALIDATE_MR_WITH_BOUND_WINDOWS, + "Attempt to invalidate MR with a bound Memory Window (MW)"}, + {IRDMA_AE_AMP_MWBIND_VALID_STAG, + "Attempt to bind an Memory Window (MW) with a valid MW memory key (L-Key/R-Key)"}, + {IRDMA_AE_AMP_MWBIND_OF_MR_STAG, + "Attempt to bind an Memory Window (MW) with an MR memory key (L-Key/R-Key)"}, + {IRDMA_AE_AMP_MWBIND_TO_ZERO_BASED_STAG, + "Attempt to bind an Memory Window (MW) to a zero based MR"}, + {IRDMA_AE_AMP_MWBIND_TO_MW_STAG, + "Attempt to bind an Memory Window (MW) using MW memory key (L-Key/R-Key) instead of MR memory key (L-Key/R-Key)"}, + {IRDMA_AE_AMP_MWBIND_INVALID_RIGHTS, "Memory Window (MW) bind error: Invalid rights"}, + {IRDMA_AE_AMP_MWBIND_INVALID_BOUNDS, "Memory Window (MW) bind error: Invalid bounds"}, + {IRDMA_AE_AMP_MWBIND_TO_INVALID_PARENT, + "Memory Window (MW) bind error: Invalid parent MR"}, + {IRDMA_AE_AMP_MWBIND_BIND_DISABLED, + "Memory Window (MW) bind error: Disabled bind support"}, + {IRDMA_AE_PRIV_OPERATION_DENIED, + "Denying a privileged operation on a non-privileged QP"}, + {IRDMA_AE_AMP_INVALIDATE_TYPE1_MW, "Memory Window (MW) error: Invalidate type 1 MW"}, + {IRDMA_AE_AMP_MWBIND_ZERO_BASED_TYPE1_MW, + "Memory Window (MW) bind error: Zero-based addressing for type 1 MW"}, + {IRDMA_AE_AMP_FASTREG_INVALID_PBL_HPS_CFG, + "Fastreg error: Invalid host page size config"}, + {IRDMA_AE_AMP_MWBIND_WRONG_TYPE, "MB bind error: Wrong Memory Window (MW) type"}, + {IRDMA_AE_AMP_FASTREG_PBLE_MISMATCH, + "Fastreg error: Invalid request to change physical MR to virtual or vice versa"}, + {IRDMA_AE_UDA_XMIT_DGRAM_TOO_LONG, + "Userspace Direct Access (UDA) QP xmit error: Packet length exceeds the QP MTU"}, + {IRDMA_AE_UDA_XMIT_BAD_PD, + "Userspace Direct Access (UDA) QP xmit error: Attempt to access a different PD"}, + {IRDMA_AE_UDA_XMIT_DGRAM_TOO_SHORT, + "Userspace Direct Access (UDA) QP xmit error: Too short packet length"}, + {IRDMA_AE_UDA_L4LEN_INVALID, + "Userspace Direct Access (UDA) error: Invalid packet length field"}, + {IRDMA_AE_BAD_CLOSE, + "iWARP error: Data is received when QP state is closing"}, + {IRDMA_AE_RDMAP_ROE_BAD_LLP_CLOSE, + "iWARP error: FIN is received when xmit data is pending"}, + {IRDMA_AE_CQ_OPERATION_ERROR, "CQ overflow"}, + {IRDMA_AE_RDMA_READ_WHILE_ORD_ZERO, + "QP error: Attempted RDMA Read when the outbound RDMA Read queue depth is zero"}, + {IRDMA_AE_STAG_ZERO_INVALID, + "Zero invalid memory key (L-Key/R-Key) on inbound RDMA R/W"}, + {IRDMA_AE_IB_RREQ_AND_Q1_FULL, + "QP error: Received RDMA Read request when the inbound RDMA Read queue is full"}, + {IRDMA_AE_IB_INVALID_REQUEST, + "QP error: Invalid operation detected by the remote peer"}, + {IRDMA_AE_WQE_UNEXPECTED_OPCODE, + "QP error: Invalid opcode in SQ WQE"}, + {IRDMA_AE_WQE_INVALID_PARAMETER, + "QP error: Invalid parameter in a WQE"}, + {IRDMA_AE_WQE_INVALID_FRAG_DATA, + "QP error: Invalid fragment in a WQE"}, + {IRDMA_AE_IB_REMOTE_ACCESS_ERROR, + "RoCEv2 error: Remote access error"}, + {IRDMA_AE_IB_REMOTE_OP_ERROR, + "RoCEv2 error: Remote operation error"}, + {IRDMA_AE_WQE_LSMM_TOO_LONG, "iWARP error: Connection error"}, + {IRDMA_AE_DDP_INVALID_MSN_GAP_IN_MSN, + "iWARP error: Invalid message sequence number"}, + {IRDMA_AE_DDP_UBE_DDP_MESSAGE_TOO_LONG_FOR_AVAILABLE_BUFFER, + "iWARP error: Inbound message is too long for the available buffer"}, + {IRDMA_AE_DDP_UBE_INVALID_DDP_VERSION, "iWARP error: Invalid DDP protocol version"}, + {IRDMA_AE_DDP_UBE_INVALID_MO, "Received message with too large offset"}, + {IRDMA_AE_DDP_UBE_INVALID_MSN_NO_BUFFER_AVAILABLE, + "iWARP error: Inbound Send message when no receive buffer is available"}, + {IRDMA_AE_DDP_UBE_INVALID_QN, "iWARP error: Invalid QP number in inbound packet"}, + {IRDMA_AE_DDP_NO_L_BIT, + "iWARP error: Last bit not set in an inbound packet which completes RDMA Read"}, + {IRDMA_AE_RDMAP_ROE_INVALID_RDMAP_VERSION, "iWARP error: Invalid RDMAP protocol version"}, + {IRDMA_AE_RDMAP_ROE_UNEXPECTED_OPCODE, "QP error: Invalid opcode"}, + {IRDMA_AE_ROE_INVALID_RDMA_READ_REQUEST, "Inbound Read request when QP isn't enabled for RDMA Read"}, + {IRDMA_AE_ROE_INVALID_RDMA_WRITE_OR_READ_RESP, + "Inbound RDMA Read response or RDMA Write when QP isn't enabled for RDMA R/W"}, + {IRDMA_AE_ROCE_RSP_LENGTH_ERROR, "RoCEv2 error: Received packet with incorrect length field"}, + {IRDMA_AE_ROCE_EMPTY_MCG, "RoCEv2 error: Multicast group has no valid members"}, + {IRDMA_AE_ROCE_BAD_MC_IP_ADDR, "RoCEv2 error: Multicast IP address doesn't match"}, + {IRDMA_AE_ROCE_BAD_MC_QPID, "RoCEv2 error: Multicast packet QP number isn't 0xffffff"}, + {IRDMA_AE_MCG_QP_PROTOCOL_MISMATCH, "RoCEv2 error: Multicast packet protocol mismatch"}, + {IRDMA_AE_INVALID_ARP_ENTRY, "Invalid ARP entry"}, + {IRDMA_AE_INVALID_TCP_OPTION_RCVD, "iWARP error: Invalid TCP option"}, + {IRDMA_AE_STALE_ARP_ENTRY, "Stale ARP entry"}, + {IRDMA_AE_INVALID_AH_ENTRY, "Invalid AH entry"}, + {IRDMA_AE_LLP_CLOSE_COMPLETE, + "iWARP event: Graceful close complete"}, + {IRDMA_AE_LLP_CONNECTION_RESET, + "iWARP event: Received a TCP packet with a RST bit set"}, + {IRDMA_AE_LLP_FIN_RECEIVED, + "iWARP event: Received a TCP packet with a FIN bit set"}, + {IRDMA_AE_LLP_RECEIVED_MARKER_AND_LENGTH_FIELDS_DONT_MATCH, + "iWARP error: Unable to close a gap in the TCP sequence"}, + {IRDMA_AE_LLP_RECEIVED_MPA_CRC_ERROR, "Received an ICRC error"}, + {IRDMA_AE_LLP_SEGMENT_TOO_SMALL, + "iWARP error: Received a packet with insufficient space for protocol headers"}, + {IRDMA_AE_LLP_SYN_RECEIVED, + "iWARP event: Received a TCP packet with a SYN bit set"}, + {IRDMA_AE_LLP_TERMINATE_RECEIVED, + "iWARP error: Received a terminate message"}, + {IRDMA_AE_LLP_TOO_MANY_RETRIES, "Connection error: The max number of retries has been reached"}, + {IRDMA_AE_LLP_TOO_MANY_KEEPALIVE_RETRIES, + "Connection error: The max number of keepalive retries has been reached"}, + {IRDMA_AE_LLP_DOUBT_REACHABILITY, + "Connection error: Doubt reachability (usually occurs after the max number of retries has been reached)"}, + {IRDMA_AE_LLP_CONNECTION_ESTABLISHED, + "iWARP event: Connection established"}, + {IRDMA_AE_RESOURCE_EXHAUSTION, + "QP error: Resource exhaustion"}, + {IRDMA_AE_RESET_SENT, + "Reset sent (as requested via Modify QP)"}, + {IRDMA_AE_TERMINATE_SENT, + "Terminate sent (as requested via Modify QP)"}, + {IRDMA_AE_RESET_NOT_SENT, + "Reset not sent (but requested via Modify QP)"}, + {IRDMA_AE_LCE_QP_CATASTROPHIC, + "QP error: HW transaction resulted in catastrophic error"}, + {IRDMA_AE_LCE_FUNCTION_CATASTROPHIC, + "PCIe function error: HW transaction resulted in catastrophic error"}, + {IRDMA_AE_LCE_CQ_CATASTROPHIC, + "CQ error: HW transaction resulted in catastrophic error"}, + {IRDMA_AE_QP_SUSPEND_COMPLETE, "QP event: Suspend complete"}, +}; + +/** + * irdma_get_ae_desc - returns AE description + * @ae_id: the AE number + */ +const char * +irdma_get_ae_desc(u16 ae_id) +{ + const char *desc = ""; + int i; + + for (i = 0; i < ARRAY_SIZE(ae_desc_list); i++) { + if (ae_desc_list[i].id == ae_id) { + desc = ae_desc_list[i].desc; + break; + } + } + return desc; +} + /** * irdma_arp_table -manage arp table * @rf: RDMA PCI function @@ -242,7 +405,7 @@ if (cqp_request->dynamic) { kfree(cqp_request); } else { - cqp_request->request_done = false; + WRITE_ONCE(cqp_request->request_done, false); cqp_request->callback_fcn = NULL; cqp_request->waiting = false; @@ -277,7 +440,7 @@ { if (cqp_request->waiting) { cqp_request->compl_info.error = true; - cqp_request->request_done = true; + WRITE_ONCE(cqp_request->request_done, true); wake_up(&cqp_request->waitq); } wait_event_timeout(cqp->remove_wq, @@ -328,23 +491,22 @@ struct irdma_cqp_request *cqp_request) { struct irdma_cqp_timeout cqp_timeout = {0}; - int timeout_threshold = CQP_TIMEOUT_THRESHOLD; bool cqp_error = false; int err_code = 0; - cqp_timeout.compl_cqp_cmds = rf->sc_dev.cqp_cmd_stats[IRDMA_OP_CMPL_CMDS]; + cqp_timeout.compl_cqp_cmds = atomic64_read(&rf->sc_dev.cqp->completed_ops); do { int wait_time_ms = rf->sc_dev.hw_attrs.max_cqp_compl_wait_time_ms; irdma_cqp_ce_handler(rf, &rf->ccq.sc_cq); if (wait_event_timeout(cqp_request->waitq, - cqp_request->request_done, + READ_ONCE(cqp_request->request_done), msecs_to_jiffies(wait_time_ms))) break; irdma_check_cqp_progress(&cqp_timeout, &rf->sc_dev); - if (cqp_timeout.count < timeout_threshold) + if (cqp_timeout.count < CQP_TIMEOUT_THRESHOLD) continue; if (!rf->reset) { @@ -397,7 +559,8 @@ [IRDMA_OP_MANAGE_HMC_PM_FUNC_TABLE] = "Manage HMC PM Function Table Cmd", [IRDMA_OP_SUSPEND] = "Suspend QP Cmd", [IRDMA_OP_RESUME] = "Resume QP Cmd", - [IRDMA_OP_MANAGE_VF_PBLE_BP] = "Manage VF PBLE Backing Pages Cmd", + [IRDMA_OP_MANAGE_VCHNL_REQ_PBLE_BP] = + "Manage Virtual Channel Requester Function PBLE Backing Pages Cmd", [IRDMA_OP_QUERY_FPM_VAL] = "Query FPM Values Cmd", [IRDMA_OP_COMMIT_FPM_VAL] = "Commit FPM Values Cmd", [IRDMA_OP_AH_CREATE] = "Create Address Handle Cmd", @@ -1620,41 +1783,6 @@ return status; } -/** - * irdma_cqp_up_map_cmd - Set the up-up mapping - * @dev: pointer to device structure - * @cmd: map command - * @map_info: pointer to up map info - */ -int -irdma_cqp_up_map_cmd(struct irdma_sc_dev *dev, u8 cmd, - struct irdma_up_info *map_info) -{ - struct irdma_pci_f *rf = dev_to_rf(dev); - struct irdma_cqp *iwcqp = &rf->cqp; - struct irdma_sc_cqp *cqp = &iwcqp->sc_cqp; - struct irdma_cqp_request *cqp_request; - struct cqp_cmds_info *cqp_info; - int status; - - cqp_request = irdma_alloc_and_get_cqp_request(iwcqp, false); - if (!cqp_request) - return -ENOMEM; - - cqp_info = &cqp_request->info; - memset(cqp_info, 0, sizeof(*cqp_info)); - cqp_info->cqp_cmd = cmd; - cqp_info->post_sq = 1; - cqp_info->in.u.up_map.info = *map_info; - cqp_info->in.u.up_map.cqp = cqp; - cqp_info->in.u.up_map.scratch = (uintptr_t)cqp_request; - - status = irdma_handle_cqp_op(rf, cqp_request); - irdma_put_cqp_request(&rf->cqp, cqp_request); - - return status; -} - /** * irdma_ah_cqp_op - perform an AH cqp operation * @rf: RDMA PCI function @@ -1707,7 +1835,7 @@ return -ENOMEM; if (wait) - sc_ah->ah_info.ah_valid = (cmd == IRDMA_OP_AH_CREATE); + sc_ah->ah_info.ah_valid = (cmd != IRDMA_OP_AH_DESTROY); return 0; } @@ -1852,15 +1980,10 @@ sizeofbitmap = (u64)pchunk->size >> pprm->pble_shift; - pchunk->bitmapmem.size = sizeofbitmap >> 3; - pchunk->bitmapmem.va = kzalloc(pchunk->bitmapmem.size, GFP_KERNEL); - - if (!pchunk->bitmapmem.va) + pchunk->bitmapbuf = bitmap_zalloc(sizeofbitmap, GFP_KERNEL); + if (!pchunk->bitmapbuf) return -ENOMEM; - pchunk->bitmapbuf = pchunk->bitmapmem.va; - bitmap_zero(pchunk->bitmapbuf, sizeofbitmap); - pchunk->sizeofbitmap = sizeofbitmap; /* each pble is 8 bytes hence shift by 3 */ pprm->total_pble_alloc += pchunk->size >> 3; @@ -2198,9 +2321,8 @@ clear_qp_ctx_addr(dma_mem.va); for (i = 0, j = 0; i < 32; i++, j += 4) irdma_debug(dev, IRDMA_DEBUG_QP, - "%d:\t [%08X %08x %08X %08X]\n", - (j * 4), ctx[j], ctx[j + 1], ctx[j + 2], - ctx[j + 3]); + "%d:\t [%08X %08x %08X %08X]\n", (j * 4), + ctx[j], ctx[j + 1], ctx[j + 2], ctx[j + 3]); } error: irdma_put_cqp_request(iwcqp, cqp_request); diff --git a/sys/dev/irdma/irdma_verbs.h b/sys/dev/irdma/irdma_verbs.h --- a/sys/dev/irdma/irdma_verbs.h +++ b/sys/dev/irdma/irdma_verbs.h @@ -1,7 +1,7 @@ /*- * SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB * - * Copyright (c) 2015 - 2022 Intel Corporation + * Copyright (c) 2015 - 2023 Intel Corporation * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU @@ -42,6 +42,8 @@ #define IRDMA_PKEY_TBL_SZ 1 #define IRDMA_DEFAULT_PKEY 0xFFFF +#define IRDMA_SHADOW_PGCNT 1 + #define iwdev_to_idev(iwdev) (&(iwdev)->rf->sc_dev) struct irdma_ucontext { @@ -73,14 +75,16 @@ spinlock_t udqp_list_lock; }; +union irdma_sockaddr { + struct sockaddr_in saddr_in; + struct sockaddr_in6 saddr_in6; +}; + struct irdma_av { u8 macaddr[16]; struct ib_ah_attr attrs; - union { - struct sockaddr saddr; - struct sockaddr_in saddr_in; - struct sockaddr_in6 saddr_in6; - } sgid_addr, dgid_addr; + union irdma_sockaddr sgid_addr; + union irdma_sockaddr dgid_addr; u8 net_type; }; @@ -246,6 +250,7 @@ int max_recv_wr; atomic_t close_timer_started; spinlock_t lock; /* serialize posting WRs to SQ/RQ */ + spinlock_t dwork_flush_lock; /* protect mod_delayed_work */ struct irdma_qp_context *iwqp_context; void *pbl_vbase; dma_addr_t pbl_pbase; @@ -266,12 +271,13 @@ wait_queue_head_t waitq; wait_queue_head_t mod_qp_waitq; u8 rts_ae_rcvd; - u8 active_conn : 1; - u8 user_mode : 1; - u8 hte_added : 1; - u8 flush_issued : 1; - u8 sig_all : 1; - u8 pau_mode : 1; + bool active_conn:1; + bool user_mode:1; + bool hte_added:1; + bool flush_issued:1; + bool sig_all:1; + bool pau_mode:1; + bool suspend_pending:1; }; struct irdma_udqs_work { @@ -308,6 +314,63 @@ return (u16)FIELD_GET(IRDMA_FW_VER_MINOR, dev->feature_info[IRDMA_FEATURE_FW_INFO]); } +static inline void set_ib_wc_op_sq(struct irdma_cq_poll_info *cq_poll_info, + struct ib_wc *entry) +{ + struct irdma_sc_qp *qp; + + switch (cq_poll_info->op_type) { + case IRDMA_OP_TYPE_RDMA_WRITE: + case IRDMA_OP_TYPE_RDMA_WRITE_SOL: + entry->opcode = IB_WC_RDMA_WRITE; + break; + case IRDMA_OP_TYPE_RDMA_READ_INV_STAG: + case IRDMA_OP_TYPE_RDMA_READ: + entry->opcode = IB_WC_RDMA_READ; + break; + case IRDMA_OP_TYPE_SEND_SOL: + case IRDMA_OP_TYPE_SEND_SOL_INV: + case IRDMA_OP_TYPE_SEND_INV: + case IRDMA_OP_TYPE_SEND: + entry->opcode = IB_WC_SEND; + break; + case IRDMA_OP_TYPE_FAST_REG_NSMR: + entry->opcode = IB_WC_REG_MR; + break; + case IRDMA_OP_TYPE_INV_STAG: + entry->opcode = IB_WC_LOCAL_INV; + break; + default: + qp = cq_poll_info->qp_handle; + irdma_dev_err(to_ibdev(qp->dev), "Invalid opcode = %d in CQE\n", + cq_poll_info->op_type); + entry->status = IB_WC_GENERAL_ERR; + } +} + +static inline void set_ib_wc_op_rq(struct irdma_cq_poll_info *cq_poll_info, + struct ib_wc *entry, bool send_imm_support) +{ + /** + * iWARP does not support sendImm, so the presence of Imm data + * must be WriteImm. + */ + if (!send_imm_support) { + entry->opcode = cq_poll_info->imm_valid ? + IB_WC_RECV_RDMA_WITH_IMM : + IB_WC_RECV; + return; + } + switch (cq_poll_info->op_type) { + case IB_OPCODE_RDMA_WRITE_ONLY_WITH_IMMEDIATE: + case IB_OPCODE_RDMA_WRITE_LAST_WITH_IMMEDIATE: + entry->opcode = IB_WC_RECV_RDMA_WITH_IMM; + break; + default: + entry->opcode = IB_WC_RECV; + } +} + /** * irdma_mcast_mac_v4 - Get the multicast MAC for an IP address * @ip_addr: IPv4 address @@ -317,7 +380,7 @@ static inline void irdma_mcast_mac_v4(u32 *ip_addr, u8 *mac) { u8 *ip = (u8 *)ip_addr; - unsigned char mac4[ETH_ALEN] = {0x01, 0x00, 0x5E, ip[2] & 0x7F, ip[1], + unsigned char mac4[ETHER_ADDR_LEN] = {0x01, 0x00, 0x5E, ip[2] & 0x7F, ip[1], ip[0]}; ether_addr_copy(mac, mac4); @@ -332,7 +395,7 @@ static inline void irdma_mcast_mac_v6(u32 *ip_addr, u8 *mac) { u8 *ip = (u8 *)ip_addr; - unsigned char mac6[ETH_ALEN] = {0x33, 0x33, ip[3], ip[2], ip[1], ip[0]}; + unsigned char mac6[ETHER_ADDR_LEN] = {0x33, 0x33, ip[3], ip[2], ip[1], ip[0]}; ether_addr_copy(mac, mac6); } diff --git a/sys/dev/irdma/irdma_verbs.c b/sys/dev/irdma/irdma_verbs.c --- a/sys/dev/irdma/irdma_verbs.c +++ b/sys/dev/irdma/irdma_verbs.c @@ -1,7 +1,7 @@ /*- * SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB * - * Copyright (c) 2015 - 2022 Intel Corporation + * Copyright (c) 2015 - 2023 Intel Corporation * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU @@ -289,9 +289,9 @@ return -EINVAL; } #endif - irdma_debug(&ucontext->iwdev->rf->sc_dev, - IRDMA_DEBUG_VERBS, "bar_offset [0x%lx] mmap_flag [%d]\n", - entry->bar_offset, entry->mmap_flag); + irdma_debug(&ucontext->iwdev->rf->sc_dev, IRDMA_DEBUG_VERBS, + "bar_offset [0x%lx] mmap_flag [%d]\n", entry->bar_offset, + entry->mmap_flag); pfn = (entry->bar_offset + pci_resource_start(ucontext->iwdev->rf->pcidev, 0)) >> PAGE_SHIFT; @@ -823,12 +823,16 @@ void irdma_sched_qp_flush_work(struct irdma_qp *iwqp) { + unsigned long flags; + if (iwqp->sc_qp.qp_uk.destroy_pending) return; irdma_qp_add_ref(&iwqp->ibqp); + spin_lock_irqsave(&iwqp->dwork_flush_lock, flags); if (mod_delayed_work(iwqp->iwdev->cleanup_wq, &iwqp->dwork_flush, msecs_to_jiffies(IRDMA_FLUSH_DELAY_MS))) irdma_qp_rem_ref(&iwqp->ibqp); + spin_unlock_irqrestore(&iwqp->dwork_flush_lock, flags); } void @@ -917,6 +921,22 @@ return 0; } +static int +irdma_wait_for_suspend(struct irdma_qp *iwqp) +{ + if (!wait_event_timeout(iwqp->iwdev->suspend_wq, + !iwqp->suspend_pending, + msecs_to_jiffies(IRDMA_EVENT_TIMEOUT_MS))) { + iwqp->suspend_pending = false; + irdma_dev_warn(&iwqp->iwdev->ibdev, + "modify_qp timed out waiting for suspend. qp_id = %d, last_ae = 0x%x\n", + iwqp->ibqp.qp_num, iwqp->last_aeq); + return -EBUSY; + } + + return 0; +} + /** * irdma_modify_qp_roce - modify qp request * @ibqp: qp's pointer for modify @@ -1112,9 +1132,8 @@ spin_lock_irqsave(&iwqp->lock, flags); if (attr_mask & IB_QP_STATE) { - if (!kc_ib_modify_qp_is_ok(iwqp->ibqp_state, attr->qp_state, - iwqp->ibqp.qp_type, attr_mask, - IB_LINK_LAYER_ETHERNET)) { + if (!ib_modify_qp_is_ok(iwqp->ibqp_state, attr->qp_state, + iwqp->ibqp.qp_type, attr_mask)) { irdma_dev_warn(&iwdev->ibdev, "modify_qp invalid for qp_id=%d, old_state=0x%x, new_state=0x%x\n", iwqp->ibqp.qp_num, iwqp->ibqp_state, @@ -1181,19 +1200,11 @@ info.next_iwarp_state = IRDMA_QP_STATE_SQD; issue_modify_qp = 1; + iwqp->suspend_pending = true; break; case IB_QPS_SQE: case IB_QPS_ERR: case IB_QPS_RESET: - if (iwqp->iwarp_state == IRDMA_QP_STATE_RTS) { - if (dev->hw_attrs.uk_attrs.hw_rev <= IRDMA_GEN_2) - irdma_cqp_qp_suspend_resume(&iwqp->sc_qp, IRDMA_OP_SUSPEND); - spin_unlock_irqrestore(&iwqp->lock, flags); - info.next_iwarp_state = IRDMA_QP_STATE_SQD; - irdma_hw_modify_qp(iwdev, iwqp, &info, true); - spin_lock_irqsave(&iwqp->lock, flags); - } - if (iwqp->iwarp_state == IRDMA_QP_STATE_ERROR) { spin_unlock_irqrestore(&iwqp->lock, flags); if (udata && udata->inlen) { @@ -1230,6 +1241,11 @@ ctx_info->rem_endpoint_idx = udp_info->arp_idx; if (irdma_hw_modify_qp(iwdev, iwqp, &info, true)) return -EINVAL; + if (info.next_iwarp_state == IRDMA_QP_STATE_SQD) { + ret = irdma_wait_for_suspend(iwqp); + if (ret) + return ret; + } spin_lock_irqsave(&iwqp->lock, flags); if (iwqp->iwarp_state == info.curr_iwarp_state) { iwqp->iwarp_state = info.next_iwarp_state; @@ -1520,8 +1536,8 @@ udata->outlen)); if (err) { irdma_remove_push_mmap_entries(iwqp); - irdma_debug(&iwdev->rf->sc_dev, - IRDMA_DEBUG_VERBS, "copy_to_udata failed\n"); + irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_VERBS, + "copy_to_udata failed\n"); return err; } } @@ -2217,6 +2233,177 @@ return ret; } +/* + * irdma_alloc_iwmr - Allocate iwmr @region - memory region @pd - protection domain @virt - virtual address @reg_type - + * registration type + */ +static struct irdma_mr * +irdma_alloc_iwmr(struct ib_umem *region, + struct ib_pd *pd, u64 virt, + enum irdma_memreg_type reg_type) +{ + struct irdma_pbl *iwpbl; + struct irdma_mr *iwmr; + + iwmr = kzalloc(sizeof(*iwmr), GFP_KERNEL); + if (!iwmr) + return ERR_PTR(-ENOMEM); + + iwpbl = &iwmr->iwpbl; + iwpbl->iwmr = iwmr; + iwmr->region = region; + iwmr->ibmr.pd = pd; + iwmr->ibmr.device = pd->device; + iwmr->ibmr.iova = virt; + iwmr->type = reg_type; + + /* Some OOT versions of irdma_copy_user_pg_addr require the pg mask */ + iwmr->page_msk = ~(IRDMA_HW_PAGE_SIZE - 1); + iwmr->page_size = IRDMA_HW_PAGE_SIZE; + iwmr->len = region->length; + iwpbl->user_base = virt; + iwmr->page_cnt = irdma_ib_umem_num_dma_blocks(region, iwmr->page_size, virt); + + return iwmr; +} + +static void +irdma_free_iwmr(struct irdma_mr *iwmr) +{ + kfree(iwmr); +} + +/* + * irdma_reg_user_mr_type_mem - Handle memory registration @iwmr - irdma mr @access - access rights + */ +static int +irdma_reg_user_mr_type_mem(struct irdma_mr *iwmr, int access) +{ + struct irdma_device *iwdev = to_iwdev(iwmr->ibmr.device); + struct irdma_pbl *iwpbl = &iwmr->iwpbl; + u32 stag; + int err; + u8 lvl; + + lvl = iwmr->page_cnt != 1 ? PBLE_LEVEL_1 | PBLE_LEVEL_2 : PBLE_LEVEL_0; + + err = irdma_setup_pbles(iwdev->rf, iwmr, lvl); + if (err) + return err; + + if (lvl) { + err = irdma_check_mr_contiguous(&iwpbl->pble_alloc, + iwmr->page_size); + if (err) { + irdma_free_pble(iwdev->rf->pble_rsrc, &iwpbl->pble_alloc); + iwpbl->pbl_allocated = false; + } + } + + stag = irdma_create_stag(iwdev); + if (!stag) { + err = -ENOMEM; + goto free_pble; + } + + iwmr->stag = stag; + iwmr->ibmr.rkey = stag; + iwmr->ibmr.lkey = stag; + iwmr->access = access; + err = irdma_hwreg_mr(iwdev, iwmr, access); + if (err) + goto err_hwreg; + + return 0; + +err_hwreg: + irdma_free_stag(iwdev, stag); + +free_pble: + if (iwpbl->pble_alloc.level != PBLE_LEVEL_0 && iwpbl->pbl_allocated) + irdma_free_pble(iwdev->rf->pble_rsrc, &iwpbl->pble_alloc); + + return err; +} + +/* + * irdma_reg_user_mr_type_qp - Handle QP memory registration @req - memory reg req @udata - user info @iwmr - irdma mr + */ +static int +irdma_reg_user_mr_type_qp(struct irdma_mem_reg_req req, + struct ib_udata *udata, + struct irdma_mr *iwmr) +{ + struct irdma_device *iwdev = to_iwdev(iwmr->ibmr.device); + struct irdma_pbl *iwpbl = &iwmr->iwpbl; + struct irdma_ucontext *ucontext; + unsigned long flags; + u32 total; + int err; + u8 lvl; + + total = req.sq_pages + req.rq_pages + IRDMA_SHADOW_PGCNT; + if (total > iwmr->page_cnt) + return -EINVAL; + + total = req.sq_pages + req.rq_pages; + lvl = total > 2 ? PBLE_LEVEL_1 : PBLE_LEVEL_0; + err = irdma_handle_q_mem(iwdev, &req, iwpbl, lvl); + if (err) + return err; + +#if __FreeBSD_version >= 1400026 + ucontext = rdma_udata_to_drv_context(udata, struct irdma_ucontext, ibucontext); +#else + ucontext = to_ucontext(iwmr->ibpd.pd->uobject->context); +#endif + spin_lock_irqsave(&ucontext->qp_reg_mem_list_lock, flags); + list_add_tail(&iwpbl->list, &ucontext->qp_reg_mem_list); + iwpbl->on_list = true; + spin_unlock_irqrestore(&ucontext->qp_reg_mem_list_lock, flags); + + return 0; +} + +/* + * irdma_reg_user_mr_type_cq - Handle CQ memory registration @req - memory reg req @udata - user info @iwmr - irdma mr + */ +static int +irdma_reg_user_mr_type_cq(struct irdma_mem_reg_req req, + struct ib_udata *udata, + struct irdma_mr *iwmr) +{ + struct irdma_device *iwdev = to_iwdev(iwmr->ibmr.device); + struct irdma_pbl *iwpbl = &iwmr->iwpbl; + struct irdma_ucontext *ucontext; + unsigned long flags; + u32 total; + int err; + u8 lvl; + + total = req.cq_pages + + ((iwdev->rf->sc_dev.hw_attrs.uk_attrs.feature_flags & IRDMA_FEATURE_CQ_RESIZE) ? 0 : IRDMA_SHADOW_PGCNT); + if (total > iwmr->page_cnt) + return -EINVAL; + + lvl = req.cq_pages > 1 ? PBLE_LEVEL_1 : PBLE_LEVEL_0; + err = irdma_handle_q_mem(iwdev, &req, iwpbl, lvl); + if (err) + return err; + +#if __FreeBSD_version >= 1400026 + ucontext = rdma_udata_to_drv_context(udata, struct irdma_ucontext, ibucontext); +#else + ucontext = to_ucontext(iwmr->ibmr.pd->uobject->context); +#endif + spin_lock_irqsave(&ucontext->cq_reg_mem_list_lock, flags); + list_add_tail(&iwpbl->list, &ucontext->cq_reg_mem_list); + iwpbl->on_list = true; + spin_unlock_irqrestore(&ucontext->cq_reg_mem_list_lock, flags); + + return 0; +} + /** * irdma_reg_user_mr - Register a user memory region * @pd: ptr of pd @@ -2233,18 +2420,10 @@ { #define IRDMA_MEM_REG_MIN_REQ_LEN offsetofend(struct irdma_mem_reg_req, sq_pages) struct irdma_device *iwdev = to_iwdev(pd->device); - struct irdma_ucontext *ucontext; - struct irdma_pble_alloc *palloc; - struct irdma_pbl *iwpbl; - struct irdma_mr *iwmr; - struct ib_umem *region; struct irdma_mem_reg_req req = {}; - u32 total, stag = 0; - u8 shadow_pgcnt = 1; - unsigned long flags; - int err = -EINVAL; - u8 lvl; - int ret; + struct ib_umem *region; + struct irdma_mr *iwmr; + int err; if (len > iwdev->rf->sc_dev.hw_attrs.max_mr_size) return ERR_PTR(-EINVAL); @@ -2265,119 +2444,41 @@ return ERR_PTR(-EFAULT); } - iwmr = kzalloc(sizeof(*iwmr), GFP_KERNEL); - if (!iwmr) { + iwmr = irdma_alloc_iwmr(region, pd, virt, req.reg_type); + if (IS_ERR(iwmr)) { ib_umem_release(region); - return ERR_PTR(-ENOMEM); + return (struct ib_mr *)iwmr; } - iwpbl = &iwmr->iwpbl; - iwpbl->iwmr = iwmr; - iwmr->region = region; - iwmr->ibmr.pd = pd; - iwmr->ibmr.device = pd->device; - iwmr->ibmr.iova = virt; - iwmr->page_size = IRDMA_HW_PAGE_SIZE; - iwmr->page_msk = ~(IRDMA_HW_PAGE_SIZE - 1); - - iwmr->len = region->length; - iwpbl->user_base = virt; - palloc = &iwpbl->pble_alloc; - iwmr->type = req.reg_type; - iwmr->page_cnt = irdma_ib_umem_num_dma_blocks(region, iwmr->page_size, virt); - switch (req.reg_type) { case IRDMA_MEMREG_TYPE_QP: - total = req.sq_pages + req.rq_pages + shadow_pgcnt; - if (total > iwmr->page_cnt) { - err = -EINVAL; - goto error; - } - total = req.sq_pages + req.rq_pages; - lvl = total > 2 ? PBLE_LEVEL_1 : PBLE_LEVEL_0; - err = irdma_handle_q_mem(iwdev, &req, iwpbl, lvl); + err = irdma_reg_user_mr_type_qp(req, udata, iwmr); if (err) goto error; -#if __FreeBSD_version >= 1400026 - ucontext = rdma_udata_to_drv_context(udata, struct irdma_ucontext, ibucontext); -#else - ucontext = to_ucontext(pd->uobject->context); -#endif - spin_lock_irqsave(&ucontext->qp_reg_mem_list_lock, flags); - list_add_tail(&iwpbl->list, &ucontext->qp_reg_mem_list); - iwpbl->on_list = true; - spin_unlock_irqrestore(&ucontext->qp_reg_mem_list_lock, flags); break; case IRDMA_MEMREG_TYPE_CQ: - if (iwdev->rf->sc_dev.hw_attrs.uk_attrs.feature_flags & IRDMA_FEATURE_CQ_RESIZE) - shadow_pgcnt = 0; - total = req.cq_pages + shadow_pgcnt; - if (total > iwmr->page_cnt) { - err = -EINVAL; - goto error; - } - - lvl = req.cq_pages > 1 ? PBLE_LEVEL_1 : PBLE_LEVEL_0; - err = irdma_handle_q_mem(iwdev, &req, iwpbl, lvl); + err = irdma_reg_user_mr_type_cq(req, udata, iwmr); if (err) goto error; -#if __FreeBSD_version >= 1400026 - ucontext = rdma_udata_to_drv_context(udata, struct irdma_ucontext, ibucontext); -#else - ucontext = to_ucontext(pd->uobject->context); -#endif - spin_lock_irqsave(&ucontext->cq_reg_mem_list_lock, flags); - list_add_tail(&iwpbl->list, &ucontext->cq_reg_mem_list); - iwpbl->on_list = true; - spin_unlock_irqrestore(&ucontext->cq_reg_mem_list_lock, flags); break; case IRDMA_MEMREG_TYPE_MEM: - lvl = iwmr->page_cnt != 1 ? PBLE_LEVEL_1 | PBLE_LEVEL_2 : PBLE_LEVEL_0; - err = irdma_setup_pbles(iwdev->rf, iwmr, lvl); + err = irdma_reg_user_mr_type_mem(iwmr, access); if (err) goto error; - if (lvl) { - ret = irdma_check_mr_contiguous(palloc, - iwmr->page_size); - if (ret) { - irdma_free_pble(iwdev->rf->pble_rsrc, palloc); - iwpbl->pbl_allocated = false; - } - } - - stag = irdma_create_stag(iwdev); - if (!stag) { - err = -ENOMEM; - goto error; - } - - iwmr->stag = stag; - iwmr->ibmr.rkey = stag; - iwmr->ibmr.lkey = stag; - iwmr->access = access; - err = irdma_hwreg_mr(iwdev, iwmr, access); - if (err) { - irdma_free_stag(iwdev, stag); - goto error; - } - break; default: + err = -EINVAL; goto error; } - iwmr->type = req.reg_type; - return &iwmr->ibmr; error: - if (palloc->level != PBLE_LEVEL_0 && iwpbl->pbl_allocated) - irdma_free_pble(iwdev->rf->pble_rsrc, palloc); ib_umem_release(region); - kfree(iwmr); + irdma_free_iwmr(iwmr); return ERR_PTR(err); } @@ -2825,8 +2926,7 @@ err = irdma_uk_post_receive(ukqp, &post_recv); if (err) { irdma_debug(&iwqp->iwdev->rf->sc_dev, IRDMA_DEBUG_VERBS, - "post_recv err %d\n", - err); + "post_recv err %d\n", err); goto out; } @@ -2876,64 +2976,6 @@ } } -static inline void -set_ib_wc_op_sq(struct irdma_cq_poll_info *cq_poll_info, - struct ib_wc *entry) -{ - struct irdma_sc_qp *qp; - - switch (cq_poll_info->op_type) { - case IRDMA_OP_TYPE_RDMA_WRITE: - case IRDMA_OP_TYPE_RDMA_WRITE_SOL: - entry->opcode = IB_WC_RDMA_WRITE; - break; - case IRDMA_OP_TYPE_RDMA_READ_INV_STAG: - case IRDMA_OP_TYPE_RDMA_READ: - entry->opcode = IB_WC_RDMA_READ; - break; - case IRDMA_OP_TYPE_SEND_SOL: - case IRDMA_OP_TYPE_SEND_SOL_INV: - case IRDMA_OP_TYPE_SEND_INV: - case IRDMA_OP_TYPE_SEND: - entry->opcode = IB_WC_SEND; - break; - case IRDMA_OP_TYPE_FAST_REG_NSMR: - entry->opcode = IB_WC_REG_MR; - break; - case IRDMA_OP_TYPE_INV_STAG: - entry->opcode = IB_WC_LOCAL_INV; - break; - default: - qp = cq_poll_info->qp_handle; - irdma_dev_err(to_ibdev(qp->dev), "Invalid opcode = %d in CQE\n", - cq_poll_info->op_type); - entry->status = IB_WC_GENERAL_ERR; - } -} - -static inline void -set_ib_wc_op_rq(struct irdma_cq_poll_info *cq_poll_info, - struct ib_wc *entry, bool send_imm_support) -{ - /** - * iWARP does not support sendImm, so the presence of Imm data - * must be WriteImm. - */ - if (!send_imm_support) { - entry->opcode = cq_poll_info->imm_valid ? - IB_WC_RECV_RDMA_WITH_IMM : IB_WC_RECV; - return; - } - switch (cq_poll_info->op_type) { - case IB_OPCODE_RDMA_WRITE_ONLY_WITH_IMMEDIATE: - case IB_OPCODE_RDMA_WRITE_LAST_WITH_IMMEDIATE: - entry->opcode = IB_WC_RECV_RDMA_WITH_IMM; - break; - default: - entry->opcode = IB_WC_RECV; - } -} - /** * irdma_process_cqe - process cqe info * @entry: processed cqe @@ -3119,8 +3161,7 @@ return npolled; error: irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_VERBS, - "%s: Error polling CQ, irdma_err: %d\n", - __func__, ret); + "%s: Error polling CQ, irdma_err: %d\n", __func__, ret); return ret; } @@ -3288,12 +3329,8 @@ int ret = 0; bool ipv4; u16 vlan_id; - union { - struct sockaddr saddr; - struct sockaddr_in saddr_in; - struct sockaddr_in6 saddr_in6; - } sgid_addr; - unsigned char dmac[ETH_ALEN]; + union irdma_sockaddr sgid_addr; + unsigned char dmac[ETHER_ADDR_LEN]; rdma_gid2ip((struct sockaddr *)&sgid_addr, ibgid); @@ -3303,9 +3340,8 @@ irdma_netdev_vlan_ipv6(ip_addr, &vlan_id, NULL); ipv4 = false; irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_VERBS, - "qp_id=%d, IP6address=%pI6\n", - ibqp->qp_num, - ip_addr); + "qp_id=%d, IP6address=%x:%x:%x:%x\n", ibqp->qp_num, + IRDMA_PRINT_IP6(ip_addr)); irdma_mcast_mac_v6(ip_addr, dmac); } else { ip_addr[0] = ntohl(sgid_addr.saddr_in.sin_addr.s_addr); @@ -3313,8 +3349,9 @@ vlan_id = irdma_get_vlan_ipv4(ip_addr); irdma_mcast_mac_v4(ip_addr, dmac); irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_VERBS, - "qp_id=%d, IP4address=%pI4, MAC=%pM\n", - ibqp->qp_num, ip_addr, dmac); + "qp_id=%d, IP4address=%x, MAC=%x:%x:%x:%x:%x:%x\n", + ibqp->qp_num, ip_addr[0], dmac[0], dmac[1], dmac[2], + dmac[3], dmac[4], dmac[5]); } spin_lock_irqsave(&rf->qh_list_lock, flags); @@ -3425,11 +3462,7 @@ struct irdma_mcast_grp_ctx_entry_info mcg_info = {0}; int ret; unsigned long flags; - union { - struct sockaddr saddr; - struct sockaddr_in saddr_in; - struct sockaddr_in6 saddr_in6; - } sgid_addr; + union irdma_sockaddr sgid_addr; rdma_gid2ip((struct sockaddr *)&sgid_addr, ibgid); if (!ipv6_addr_v4mapped((struct in6_addr *)ibgid)) @@ -3442,8 +3475,8 @@ mc_qht_elem = mcast_list_lookup_ip(rf, ip_addr); if (!mc_qht_elem) { spin_unlock_irqrestore(&rf->qh_list_lock, flags); - irdma_debug(&iwdev->rf->sc_dev, - IRDMA_DEBUG_VERBS, "address not found MCG\n"); + irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_VERBS, + "address not found MCG\n"); return 0; } @@ -3455,8 +3488,8 @@ ret = irdma_mcast_cqp_op(iwdev, &mc_qht_elem->mc_grp_ctx, IRDMA_OP_MC_DESTROY); if (ret) { - irdma_debug(&iwdev->rf->sc_dev, - IRDMA_DEBUG_VERBS, "failed MC_DESTROY MCG\n"); + irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_VERBS, + "failed MC_DESTROY MCG\n"); spin_lock_irqsave(&rf->qh_list_lock, flags); mcast_list_add(rf, mc_qht_elem); spin_unlock_irqrestore(&rf->qh_list_lock, flags); @@ -3473,8 +3506,8 @@ ret = irdma_mcast_cqp_op(iwdev, &mc_qht_elem->mc_grp_ctx, IRDMA_OP_MC_MODIFY); if (ret) { - irdma_debug(&iwdev->rf->sc_dev, - IRDMA_DEBUG_VERBS, "failed Modify MCG\n"); + irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_VERBS, + "failed Modify MCG\n"); return ret; } } diff --git a/sys/dev/irdma/irdma_ws.c b/sys/dev/irdma/irdma_ws.c --- a/sys/dev/irdma/irdma_ws.c +++ b/sys/dev/irdma/irdma_ws.c @@ -1,7 +1,7 @@ /*- * SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB * - * Copyright (c) 2017 - 2022 Intel Corporation + * Copyright (c) 2017 - 2023 Intel Corporation * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU @@ -293,13 +293,13 @@ ws_tree_root = vsi->dev->ws_tree_root; if (!ws_tree_root) { - irdma_debug(vsi->dev, IRDMA_DEBUG_WS, "Creating root node\n"); ws_tree_root = irdma_alloc_node(vsi, user_pri, WS_NODE_TYPE_PARENT, NULL); if (!ws_tree_root) { ret = -ENOMEM; goto exit; } + irdma_debug(vsi->dev, IRDMA_DEBUG_WS, "Creating root node = %d\n", ws_tree_root->index); ret = irdma_ws_cqp_cmd(vsi, ws_tree_root, IRDMA_OP_WS_ADD_NODE); if (ret) { diff --git a/sys/dev/irdma/osdep.h b/sys/dev/irdma/osdep.h --- a/sys/dev/irdma/osdep.h +++ b/sys/dev/irdma/osdep.h @@ -1,7 +1,7 @@ /*- * SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB * - * Copyright (c) 2021 - 2022 Intel Corporation + * Copyright (c) 2021 - 2023 Intel Corporation * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU @@ -51,11 +51,11 @@ #include #include #include +#include #include #include -#define ATOMIC atomic_t #define IOMEM #define IRDMA_NTOHS(a) ntohs(a) #define MAKEMASK(m, s) ((m) << (s)) @@ -96,6 +96,9 @@ #define irdma_mb() mb() #define irdma_wmb() wmb() +#ifndef smp_mb +#define smp_mb() mb() +#endif #define irdma_get_virt_to_phy vtophys #define __aligned_u64 uint64_t __aligned(8) @@ -111,7 +114,7 @@ #define irdma_print(S, ...) printf("%s:%d "S, __FUNCTION__, __LINE__, ##__VA_ARGS__) #define irdma_debug_buf(dev, mask, desc, buf, size) \ do { \ - u32 i; \ + u32 i; \ if (!((mask) & (dev)->debug_mask)) { \ break; \ } \ @@ -121,14 +124,14 @@ irdma_debug(dev, mask, "index %03d val: %016lx\n", i, ((unsigned long *)(buf))[i / 8]); \ } while(0) -#define irdma_debug(h, m, s, ...) \ -do { \ - if (!(h)) { \ - if ((m) == IRDMA_DEBUG_INIT) \ +#define irdma_debug(h, m, s, ...) \ +do { \ + if (!(h)) { \ + if ((m) == IRDMA_DEBUG_INIT) \ printf("irdma INIT " s, ##__VA_ARGS__); \ - } else if (((m) & (h)->debug_mask)) { \ - printf("irdma " s, ##__VA_ARGS__); \ - } \ + } else if (((m) & (h)->debug_mask)) { \ + printf("irdma " s, ##__VA_ARGS__); \ + } \ } while (0) #define irdma_dev_err(ibdev, fmt, ...) \ pr_err("%s:%s:%d ERR "fmt, (ibdev)->name, __func__, __LINE__, ##__VA_ARGS__) @@ -137,17 +140,8 @@ #define irdma_dev_info(a, b, ...) printf(b, ##__VA_ARGS__) #define irdma_pr_warn printf -#define dump_struct(s, sz, name) \ -do { \ - unsigned char *a; \ - printf("%s %u", (name), (unsigned int)(sz)); \ - for (a = (unsigned char*)(s); a < (unsigned char *)(s) + (sz) ; a ++) { \ - if ((u64)a % 8 == 0) \ - printf("\n%p ", a); \ - printf("%2x ", *a); \ - } \ - printf("\n"); \ -}while(0) +#define IRDMA_PRINT_IP6(ip6) \ + ((u32*)ip6)[0], ((u32*)ip6)[1], ((u32*)ip6)[2], ((u32*)ip6)[3] /* * debug definition end @@ -176,6 +170,7 @@ bus_size_t mem_bus_space_size; void *dev; struct irdma_task_arg task_arg; + atomic_t event_rfcnt; }; #define irdma_pr_info(fmt, args ...) printf("%s: WARN "fmt, __func__, ## args) @@ -187,38 +182,34 @@ #define rt_tos2priority(tos) (tos >> 5) #define ah_attr_to_dmac(attr) ((attr).dmac) -#define kc_ib_modify_qp_is_ok(cur_state, next_state, type, mask, ll) \ - ib_modify_qp_is_ok(cur_state, next_state, type, mask) -#define kc_rdma_gid_attr_network_type(sgid_attr, gid_type, gid) \ - ib_gid_to_network_type(gid_type, gid) #define irdma_del_timer_compat(tt) del_timer((tt)) #define IRDMA_TAILQ_FOREACH CK_STAILQ_FOREACH #define IRDMA_TAILQ_FOREACH_SAFE CK_STAILQ_FOREACH_SAFE #define between(a, b, c) (bool)(c-a >= b-a) -#define rd32(a, reg) irdma_rd32((a)->dev_context, (reg)) -#define wr32(a, reg, value) irdma_wr32((a)->dev_context, (reg), (value)) +#define rd32(a, reg) irdma_rd32((a)->dev_context, (reg)) +#define wr32(a, reg, value) irdma_wr32((a)->dev_context, (reg), (value)) -#define rd64(a, reg) irdma_rd64((a)->dev_context, (reg)) -#define wr64(a, reg, value) irdma_wr64((a)->dev_context, (reg), (value)) +#define rd64(a, reg) irdma_rd64((a)->dev_context, (reg)) +#define wr64(a, reg, value) irdma_wr64((a)->dev_context, (reg), (value)) #define db_wr32(value, a) writel((value), (a)) void *hw_to_dev(struct irdma_hw *hw); struct irdma_dma_mem { - void *va; - u64 pa; + void *va; + u64 pa; bus_dma_tag_t tag; bus_dmamap_t map; bus_dma_segment_t seg; bus_size_t size; - int nseg; - int flags; + int nseg; + int flags; }; struct irdma_virt_mem { - void *va; - u32 size; + void *va; + u32 size; }; struct irdma_dma_info {