diff --git a/contrib/ofed/libirdma/abi.h b/contrib/ofed/libirdma/abi.h --- a/contrib/ofed/libirdma/abi.h +++ b/contrib/ofed/libirdma/abi.h @@ -115,6 +115,7 @@ __u32 rsvd32; __u8 userspace_ver; __u8 rsvd8[3]; + __aligned_u64 comp_mask; }; struct irdma_get_context_resp { @@ -136,6 +137,7 @@ __u16 max_hw_sq_chunk; __u8 hw_rev; __u8 rsvd2; + __aligned_u64 comp_mask; }; struct irdma_ureg_mr { diff --git a/contrib/ofed/libirdma/i40iw_hw.h b/contrib/ofed/libirdma/i40iw_hw.h --- a/contrib/ofed/libirdma/i40iw_hw.h +++ b/contrib/ofed/libirdma/i40iw_hw.h @@ -41,8 +41,8 @@ I40IW_MAX_SGE_RD = 1, I40IW_MAX_PUSH_PAGE_COUNT = 0, I40IW_MAX_INLINE_DATA_SIZE = 48, - I40IW_MAX_IRD_SIZE = 63, - I40IW_MAX_ORD_SIZE = 127, + I40IW_MAX_IRD_SIZE = 64, + I40IW_MAX_ORD_SIZE = 64, I40IW_MAX_WQ_ENTRIES = 2048, I40IW_MAX_WQE_SIZE_RQ = 128, I40IW_MAX_PDS = 32768, diff --git a/contrib/ofed/libirdma/irdma-abi.h b/contrib/ofed/libirdma/irdma-abi.h --- a/contrib/ofed/libirdma/irdma-abi.h +++ b/contrib/ofed/libirdma/irdma-abi.h @@ -2,7 +2,7 @@ * SPDX-License-Identifier: (GPL-2.0 WITH Linux-syscall-note) OR Linux-OpenIB) * * - * Copyright (c) 2006 - 2021 Intel Corporation. All rights reserved. + * Copyright (c) 2006 - 2022 Intel Corporation. All rights reserved. * Copyright (c) 2005 Topspin Communications. All rights reserved. * Copyright (c) 2005 Cisco Systems. All rights reserved. * Copyright (c) 2005 Open Grid Computing, Inc. All rights reserved. @@ -53,10 +53,15 @@ IRDMA_MEMREG_TYPE_CQ = 2, }; +enum { + IRDMA_ALLOC_UCTX_USE_RAW_ATTR = 1 << 0, +}; + struct irdma_alloc_ucontext_req { __u32 rsvd32; __u8 userspace_ver; __u8 rsvd8[3]; + __aligned_u64 comp_mask; }; struct irdma_alloc_ucontext_resp { @@ -77,6 +82,7 @@ __u16 max_hw_sq_chunk; __u8 hw_rev; __u8 rsvd2; + __aligned_u64 comp_mask; }; struct irdma_alloc_pd_resp { diff --git a/contrib/ofed/libirdma/irdma_defs.h b/contrib/ofed/libirdma/irdma_defs.h --- a/contrib/ofed/libirdma/irdma_defs.h +++ b/contrib/ofed/libirdma/irdma_defs.h @@ -75,6 +75,7 @@ #define IRDMA_CQE_QTYPE_RQ 0 #define IRDMA_CQE_QTYPE_SQ 1 +#define IRDMA_QP_SW_MIN_WQSIZE 8 /* in WRs*/ #define IRDMA_QP_WQE_MIN_SIZE 32 #define IRDMA_QP_WQE_MAX_SIZE 256 #define IRDMA_QP_WQE_MIN_QUANTA 1 @@ -304,6 +305,17 @@ #define IRDMAPFINT_OICR_PE_PUSH_M BIT(27) #define IRDMAPFINT_OICR_PE_CRITERR_M BIT(28) +#define IRDMA_GET_RING_OFFSET(_ring, _i) \ + ( \ + ((_ring).head + (_i)) % (_ring).size \ + ) + +#define IRDMA_GET_CQ_ELEM_AT_OFFSET(_cq, _i, _cqe) \ + { \ + register __u32 offset; \ + offset = IRDMA_GET_RING_OFFSET((_cq)->cq_ring, _i); \ + (_cqe) = (_cq)->cq_base[offset].buf; \ + } #define IRDMA_GET_CURRENT_CQ_ELEM(_cq) \ ( \ (_cq)->cq_base[IRDMA_RING_CURRENT_HEAD((_cq)->cq_ring)].buf \ @@ -437,12 +449,6 @@ IRDMA_RING_MOVE_HEAD(_ring, _retcode); \ } -enum irdma_protocol_used { - IRDMA_ANY_PROTOCOL = 0, - IRDMA_IWARP_PROTOCOL_ONLY = 1, - IRDMA_ROCE_PROTOCOL_ONLY = 2, -}; - enum irdma_qp_wqe_size { IRDMA_WQE_SIZE_32 = 32, IRDMA_WQE_SIZE_64 = 64, diff --git a/contrib/ofed/libirdma/irdma_uk.c b/contrib/ofed/libirdma/irdma_uk.c --- a/contrib/ofed/libirdma/irdma_uk.c +++ b/contrib/ofed/libirdma/irdma_uk.c @@ -641,7 +641,7 @@ sge_len -= bytes_copied; if (!quanta_bytes_remaining) { - /* Remaining inline bytes reside after the hdr */ + /* Remaining inline bytes reside after hdr */ wqe += 16; quanta_bytes_remaining = 32; } @@ -710,7 +710,7 @@ if (!quanta_bytes_remaining) { quanta_bytes_remaining = 31; - /* Remaining inline bytes reside after the hdr */ + /* Remaining inline bytes reside after hdr */ if (first_quanta) { first_quanta = false; wqe += 16; @@ -1111,7 +1111,6 @@ u8 arm_next = 0; u8 arm_seq_num; - cq->armed = true; get_64bit_val(cq->shadow_area, IRDMA_BYTE_32, &temp_val); arm_seq_num = (u8)FIELD_GET(IRDMA_CQ_DBSA_ARM_SEQ_NUM, temp_val); arm_seq_num++; @@ -1338,6 +1337,8 @@ info->error = (bool)FIELD_GET(IRDMA_CQ_ERROR, qword3); info->push_dropped = (bool)FIELD_GET(IRDMACQ_PSHDROP, qword3); info->ipv4 = (bool)FIELD_GET(IRDMACQ_IPV4, qword3); + get_64bit_val(cqe, IRDMA_BYTE_8, &comp_ctx); + qp = (struct irdma_qp_uk *)(irdma_uintptr) comp_ctx; if (info->error) { info->major_err = FIELD_GET(IRDMA_CQ_MAJERR, qword3); info->minor_err = FIELD_GET(IRDMA_CQ_MINERR, qword3); @@ -1366,10 +1367,7 @@ info->qp_id = (u32)FIELD_GET(IRDMACQ_QPID, qword2); info->ud_src_qpn = (u32)FIELD_GET(IRDMACQ_UDSRCQPN, qword2); - get_64bit_val(cqe, IRDMA_BYTE_8, &comp_ctx); - info->solicited_event = (bool)FIELD_GET(IRDMACQ_SOEVENT, qword3); - qp = (struct irdma_qp_uk *)(irdma_uintptr) comp_ctx; if (!qp || qp->destroy_pending) { ret_code = EFAULT; goto exit; @@ -1493,7 +1491,8 @@ } } while (1); - if (info->op_type == IRDMA_OP_TYPE_BIND_MW && info->minor_err == FLUSH_PROT_ERR) + if (info->op_type == IRDMA_OP_TYPE_BIND_MW && + info->minor_err == FLUSH_PROT_ERR) info->minor_err = FLUSH_MW_BIND_ERR; qp->sq_flush_seen = true; if (!IRDMA_RING_MORE_WORK(qp->sq_ring)) diff --git a/contrib/ofed/libirdma/irdma_umain.h b/contrib/ofed/libirdma/irdma_umain.h --- a/contrib/ofed/libirdma/irdma_umain.h +++ b/contrib/ofed/libirdma/irdma_umain.h @@ -87,7 +87,8 @@ struct irdma_uk_attrs uk_attrs; void *db; int abi_ver; - bool legacy_mode; + bool legacy_mode:1; + bool use_raw_attrs:1; }; struct irdma_uqp; diff --git a/contrib/ofed/libirdma/irdma_umain.c b/contrib/ofed/libirdma/irdma_umain.c --- a/contrib/ofed/libirdma/irdma_umain.c +++ b/contrib/ofed/libirdma/irdma_umain.c @@ -49,7 +49,7 @@ /** * Driver version */ -char libirdma_version[] = "1.1.5-k"; +char libirdma_version[] = "1.1.11-k"; unsigned int irdma_dbg; @@ -170,7 +170,7 @@ iwvctx->uk_attrs.max_hw_sq_chunk = resp.max_hw_sq_chunk; iwvctx->uk_attrs.max_hw_cq_size = resp.max_hw_cq_size; iwvctx->uk_attrs.min_hw_cq_size = resp.min_hw_cq_size; - iwvctx->uk_attrs.min_hw_wq_size = IRDMA_MIN_WQ_SIZE_GEN2; + iwvctx->uk_attrs.min_hw_wq_size = IRDMA_QP_SW_MIN_WQSIZE; iwvctx->abi_ver = IRDMA_ABI_VER; mmap_key = resp.db_mmap_key; diff --git a/contrib/ofed/libirdma/irdma_user.h b/contrib/ofed/libirdma/irdma_user.h --- a/contrib/ofed/libirdma/irdma_user.h +++ b/contrib/ofed/libirdma/irdma_user.h @@ -211,7 +211,6 @@ IRDMA_Q2_BUF_SIZE = 256, IRDMA_QP_CTX_SIZE = 256, IRDMA_MAX_PDS = 262144, - IRDMA_MIN_WQ_SIZE_GEN2 = 8, }; enum irdma_addressing_type { @@ -521,7 +520,6 @@ u32 cq_size; struct irdma_ring cq_ring; u8 polarity; - bool armed:1; bool avoid_mem_cflct:1; }; diff --git a/contrib/ofed/libirdma/irdma_uverbs.c b/contrib/ofed/libirdma/irdma_uverbs.c --- a/contrib/ofed/libirdma/irdma_uverbs.c +++ b/contrib/ofed/libirdma/irdma_uverbs.c @@ -1566,11 +1566,10 @@ info.sq_size = info.sq_depth >> info.sq_shift; info.rq_size = info.rq_depth >> info.rq_shift; /** - * For older ABI version (less than 6) passes raw sq and rq - * quanta in cap.max_send_wr and cap.max_recv_wr. - * But then kernel had no way of calculating the actual qp size. + * Maintain backward compatibility with older ABI which pass sq + * and rq depth (in quanta) in cap.max_send_wr a cap.max_recv_wr */ - if (iwvctx->abi_ver <= 5) { + if (!iwvctx->use_raw_attrs) { attr->cap.max_send_wr = info.sq_size; attr->cap.max_recv_wr = info.rq_size; } diff --git a/contrib/ofed/libirdma/osdep.h b/contrib/ofed/libirdma/osdep.h --- a/contrib/ofed/libirdma/osdep.h +++ b/contrib/ofed/libirdma/osdep.h @@ -119,7 +119,7 @@ irdma_debug(dev, mask, "%s\n", desc); \ irdma_debug(dev, mask, "starting address virt=%p phy=%lxh\n", buf, irdma_get_virt_to_phy(buf)); \ for (i = 0; i < size ; i += 8) \ - irdma_debug(dev, mask, "index %03d val: %016lx\n", i, ((unsigned long *)buf)[i / 8]); \ + irdma_debug(dev, mask, "index %03d val: %016lx\n", i, ((unsigned long *)(buf))[i / 8]); \ } while(0) #define irdma_debug(h, m, s, ...) \ @@ -137,11 +137,12 @@ if (irdma_dbg) \ printf("libirdma-%s: " fmt, __func__, ##args); \ } while (0) -#define irdma_dev_err(a, b, ...) printf(b, ##__VA_ARGS__) -#define irdma_dev_warn(a, b, ...) printf(b, ##__VA_ARGS__) /*dev_warn(a, b)*/ +#define irdma_dev_err(ibdev, fmt, ...) \ + pr_err("%s:%s:%d ERR "fmt, (ibdev)->name, __func__, __LINE__, ##__VA_ARGS__) +#define irdma_dev_warn(ibdev, fmt, ...) \ + pr_warn("%s:%s:%d WARN "fmt, (ibdev)->name, __func__, __LINE__, ##__VA_ARGS__) #define irdma_dev_info(a, b, ...) printf(b, ##__VA_ARGS__) #define irdma_pr_warn printf -#define ibdev_err(ibdev, fmt, ...) printf("%s:"fmt, (ibdev)->name, ##__VA_ARGS__) #define dump_struct(s, sz, name) \ do { \ diff --git a/sys/dev/irdma/fbsd_kcompat.h b/sys/dev/irdma/fbsd_kcompat.h --- a/sys/dev/irdma/fbsd_kcompat.h +++ b/sys/dev/irdma/fbsd_kcompat.h @@ -40,10 +40,11 @@ #define TASKLET_DATA_TYPE unsigned long #define TASKLET_FUNC_TYPE void (*)(TASKLET_DATA_TYPE) +#ifndef tasklet_setup #define tasklet_setup(tasklet, callback) \ tasklet_init((tasklet), (TASKLET_FUNC_TYPE)(callback), \ (TASKLET_DATA_TYPE)(tasklet)) - +#endif #ifndef from_tasklet #define from_tasklet(var, callback_tasklet, tasklet_fieldname) \ container_of(callback_tasklet, typeof(*var), tasklet_fieldname) @@ -176,8 +177,7 @@ #else int irdma_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata); #endif -void irdma_get_eth_speed_and_width(u32 link_speed, u8 *active_speed, - u8 *active_width); +int ib_get_eth_speed(struct ib_device *dev, u32 port_num, u8 *speed, u8 *width); enum rdma_link_layer irdma_get_link_layer(struct ib_device *ibdev, u8 port_num); int irdma_roce_port_immutable(struct ib_device *ibdev, u8 port_num, @@ -197,6 +197,7 @@ struct rdma_hw_stats *stats, u8 port_num, int index); +void irdma_request_reset(struct irdma_pci_f *rf); int irdma_register_qset(struct irdma_sc_vsi *vsi, struct irdma_ws_node *tc_node); void irdma_unregister_qset(struct irdma_sc_vsi *vsi, @@ -337,4 +338,13 @@ ALIGN_DOWN(iova, pgsz))) / pgsz; } +static inline void addrconf_addr_eui48(u8 *deui, const char *const addr) +{ + memcpy(deui, addr, 3); + deui[3] = 0xFF; + deui[4] = 0xFE; + memcpy(deui + 5, addr + 3, 3); + deui[0] ^= 2; +} + #endif /* FBSD_KCOMPAT_H */ diff --git a/sys/dev/irdma/fbsd_kcompat.c b/sys/dev/irdma/fbsd_kcompat.c --- a/sys/dev/irdma/fbsd_kcompat.c +++ b/sys/dev/irdma/fbsd_kcompat.c @@ -93,6 +93,18 @@ } +void +irdma_request_reset(struct irdma_pci_f *rf) +{ + struct ice_rdma_peer *peer = rf->peer_info; + struct ice_rdma_request req = {0}; + + req.type = ICE_RDMA_EVENT_RESET; + + printf("%s:%d requesting pf-reset\n", __func__, __LINE__); + IRDMA_DI_REQ_HANDLER(peer, &req); +} + int irdma_register_qset(struct irdma_sc_vsi *vsi, struct irdma_ws_node *tc_node) { @@ -611,32 +623,38 @@ &rf->dcqcn_params.min_rate, 0, "set minimum rate limit value, in MBits per second, default=0"); + rf->dcqcn_params.dcqcn_f = 5; SYSCTL_ADD_U8(&rf->tun_info.irdma_sysctl_ctx, irdma_sysctl_oid_list, OID_AUTO, "dcqcn_F", CTLFLAG_RDTUN, &rf->dcqcn_params.dcqcn_f, 0, - "set number of times to stay in each stage of bandwidth recovery, default=0"); + "set number of times to stay in each stage of bandwidth recovery, default=5"); + rf->dcqcn_params.dcqcn_t = 0x37; SYSCTL_ADD_U16(&rf->tun_info.irdma_sysctl_ctx, irdma_sysctl_oid_list, OID_AUTO, "dcqcn_T", CTLFLAG_RDTUN, &rf->dcqcn_params.dcqcn_t, 0, - "set number of usecs that should elapse before increasing the CWND in DCQCN mode, default=0"); + "set number of usecs that should elapse before increasing the CWND in DCQCN mode, default=0x37"); + rf->dcqcn_params.dcqcn_b = 0x249f0; SYSCTL_ADD_U32(&rf->tun_info.irdma_sysctl_ctx, irdma_sysctl_oid_list, OID_AUTO, "dcqcn_B", CTLFLAG_RDTUN, &rf->dcqcn_params.dcqcn_b, 0, - "set number of MSS to add to the congestion window in additive increase mode, default=0"); + "set number of MSS to add to the congestion window in additive increase mode, default=0x249f0"); + rf->dcqcn_params.rai_factor = 1; SYSCTL_ADD_U16(&rf->tun_info.irdma_sysctl_ctx, irdma_sysctl_oid_list, OID_AUTO, "dcqcn_rai_factor", CTLFLAG_RDTUN, &rf->dcqcn_params.rai_factor, 0, - "set number of MSS to add to the congestion window in additive increase mode, default=0"); + "set number of MSS to add to the congestion window in additive increase mode, default=1"); + rf->dcqcn_params.hai_factor = 5; SYSCTL_ADD_U16(&rf->tun_info.irdma_sysctl_ctx, irdma_sysctl_oid_list, OID_AUTO, "dcqcn_hai_factor", CTLFLAG_RDTUN, &rf->dcqcn_params.hai_factor, 0, - "set number of MSS to add to the congestion window in hyperactive increase mode, default=0"); + "set number of MSS to add to the congestion window in hyperactive increase mode, default=5"); + rf->dcqcn_params.rreduce_mperiod = 50; SYSCTL_ADD_U32(&rf->tun_info.irdma_sysctl_ctx, irdma_sysctl_oid_list, OID_AUTO, "dcqcn_rreduce_mperiod", CTLFLAG_RDTUN, &rf->dcqcn_params.rreduce_mperiod, 0, - "set minimum time between 2 consecutive rate reductions for a single flow, default=0"); + "set minimum time between 2 consecutive rate reductions for a single flow, default=50"); } /** @@ -743,3 +761,31 @@ { kfree(chunk->bitmapmem.va); } + +void +irdma_cleanup_dead_qps(struct irdma_sc_vsi *vsi) +{ + struct irdma_sc_qp *qp = NULL; + struct irdma_qp *iwqp; + struct irdma_pci_f *rf; + u8 i; + + for (i = 0; i < IRDMA_MAX_USER_PRIORITY; i++) { + qp = irdma_get_qp_from_list(&vsi->qos[i].qplist, qp); + while (qp) { + if (qp->qp_uk.qp_type == IRDMA_QP_TYPE_UDA) { + qp = irdma_get_qp_from_list(&vsi->qos[i].qplist, qp); + continue; + } + iwqp = qp->qp_uk.back_qp; + rf = iwqp->iwdev->rf; + irdma_free_dma_mem(rf->sc_dev.hw, &iwqp->q2_ctx_mem); + irdma_free_dma_mem(rf->sc_dev.hw, &iwqp->kqp.dma_mem); + + kfree(iwqp->kqp.sq_wrid_mem); + kfree(iwqp->kqp.rq_wrid_mem); + qp = irdma_get_qp_from_list(&vsi->qos[i].qplist, qp); + kfree(iwqp); + } + } +} diff --git a/sys/dev/irdma/icrdma.c b/sys/dev/irdma/icrdma.c --- a/sys/dev/irdma/icrdma.c +++ b/sys/dev/irdma/icrdma.c @@ -53,7 +53,7 @@ /** * Driver version */ -char irdma_driver_version[] = "1.1.5-k"; +char irdma_driver_version[] = "1.1.11-k"; #define pf_if_d(peer) peer->ifp->if_dunit @@ -223,9 +223,13 @@ irdma_log_invalid_mtu(u16 mtu, struct irdma_sc_dev *dev) { if (mtu < IRDMA_MIN_MTU_IPV4) - irdma_dev_warn(dev, "MTU setting [%d] too low for RDMA traffic. Minimum MTU is 576 for IPv4\n", mtu); + irdma_dev_warn(to_ibdev(dev), + "MTU setting [%d] too low for RDMA traffic. Minimum MTU is 576 for IPv4\n", + mtu); else if (mtu < IRDMA_MIN_MTU_IPV6) - irdma_dev_warn(dev, "MTU setting [%d] too low for RDMA traffic. Minimum MTU is 1280 for IPv6\\n", mtu); + irdma_dev_warn(to_ibdev(dev), + "MTU setting [%d] too low for RDMA traffic. Minimum MTU is 1280 for IPv6\\n", + mtu); } /** @@ -336,22 +340,25 @@ int status = 0; if (iwdev->iw_status) { - irdma_debug(&rf->sc_dev, IRDMA_DEBUG_INIT, "Starting deferred closing %d (%d)\n", + irdma_debug(&rf->sc_dev, IRDMA_DEBUG_INIT, + "Starting deferred closing %d (%d)\n", rf->peer_info->pf_id, pf_if_d(peer)); irdma_dereg_ipaddr_event_cb(rf); irdma_ib_unregister_device(iwdev); req.type = ICE_RDMA_EVENT_VSI_FILTER_UPDATE; req.enable_filter = false; IRDMA_DI_REQ_HANDLER(peer, &req); + irdma_cleanup_dead_qps(&iwdev->vsi); irdma_rt_deinit_hw(iwdev); } else { - irdma_debug(&rf->sc_dev, IRDMA_DEBUG_INIT, "Starting deferred opening %d (%d)\n", + irdma_debug(&rf->sc_dev, IRDMA_DEBUG_INIT, + "Starting deferred opening %d (%d)\n", rf->peer_info->pf_id, pf_if_d(peer)); - l2params.mtu = peer->mtu; irdma_get_qos_info(&l2params, &peer->initial_qos_info); if (iwdev->rf->protocol_used != IRDMA_IWARP_PROTOCOL_ONLY) iwdev->dcb_vlan_mode = l2params.num_tc > 1 && !l2params.dscp_mode; + l2params.mtu = peer->mtu; status = irdma_rt_init_hw(iwdev, &l2params); if (status) { irdma_pr_err("RT init failed %d\n", status); @@ -368,7 +375,8 @@ req.enable_filter = true; IRDMA_DI_REQ_HANDLER(peer, &req); irdma_reg_ipaddr_event_cb(rf); - irdma_debug(&rf->sc_dev, IRDMA_DEBUG_INIT, "Deferred opening finished %d (%d)\n", + irdma_debug(&rf->sc_dev, IRDMA_DEBUG_INIT, + "Deferred opening finished %d (%d)\n", rf->peer_info->pf_id, pf_if_d(peer)); } } @@ -459,6 +467,7 @@ rf->rsrc_profile = IRDMA_HMC_PROFILE_DEFAULT; rf->rst_to = IRDMA_RST_TIMEOUT_HZ; rf->check_fc = irdma_check_fc_for_qp; + rf->gen_ops.request_reset = irdma_request_reset; irdma_set_rf_user_cfg_params(rf); rf->default_vsi.vsi_idx = peer->pf_vsi_num; @@ -483,6 +492,7 @@ iwdev->rcv_wscale = IRDMA_CM_DEFAULT_RCV_WND_SCALE; iwdev->roce_cwnd = IRDMA_ROCE_CWND_DEFAULT; iwdev->roce_ackcreds = IRDMA_ROCE_ACKCREDS_DEFAULT; + iwdev->roce_rtomin = 5; if (rf->protocol_used == IRDMA_ROCE_PROTOCOL_ONLY) { iwdev->roce_mode = true; @@ -583,7 +593,8 @@ struct irdma_handler *hdl; struct irdma_device *iwdev; - irdma_debug((struct irdma_sc_dev *)NULL, IRDMA_DEBUG_INIT, "removing %s\n", __FUNCTION__); + irdma_debug((struct irdma_sc_dev *)NULL, IRDMA_DEBUG_INIT, + "removing %s irdma%d\n", __func__, pf_if_d(peer)); hdl = irdma_find_handler(peer); if (!hdl) @@ -614,7 +625,8 @@ kfree(iwdev->hdl); kfree(iwdev->rf); ib_dealloc_device(&iwdev->ibdev); - irdma_pr_info("IRDMA hardware deinitialization complete\n"); + irdma_pr_info("IRDMA hardware deinitialization complete irdma%d\n", + pf_if_d(peer)); return 0; } diff --git a/sys/dev/irdma/icrdma_hw.h b/sys/dev/irdma/icrdma_hw.h --- a/sys/dev/irdma/icrdma_hw.h +++ b/sys/dev/irdma/icrdma_hw.h @@ -122,7 +122,7 @@ ICRDMA_MAX_STATS_COUNT = 128, ICRDMA_MAX_IRD_SIZE = 32, - ICRDMA_MAX_ORD_SIZE = 64, + ICRDMA_MAX_ORD_SIZE = 32, ICRDMA_MIN_WQ_SIZE = 8 /* WQEs */, }; diff --git a/sys/dev/irdma/irdma-abi.h b/sys/dev/irdma/irdma-abi.h --- a/sys/dev/irdma/irdma-abi.h +++ b/sys/dev/irdma/irdma-abi.h @@ -53,10 +53,15 @@ IRDMA_MEMREG_TYPE_CQ = 2, }; +enum { + IRDMA_ALLOC_UCTX_USE_RAW_ATTR = 1 << 0, +}; + struct irdma_alloc_ucontext_req { __u32 rsvd32; __u8 userspace_ver; __u8 rsvd8[3]; + __aligned_u64 comp_mask; }; struct irdma_alloc_ucontext_resp { @@ -77,6 +82,7 @@ __u16 max_hw_sq_chunk; __u8 hw_rev; __u8 rsvd2; + __aligned_u64 comp_mask; }; struct irdma_alloc_pd_resp { diff --git a/sys/dev/irdma/irdma_cm.h b/sys/dev/irdma/irdma_cm.h --- a/sys/dev/irdma/irdma_cm.h +++ b/sys/dev/irdma/irdma_cm.h @@ -428,9 +428,6 @@ int irdma_create_listen(struct iw_cm_id *cm_id, int backlog); int irdma_destroy_listen(struct iw_cm_id *cm_id); int irdma_add_arp(struct irdma_pci_f *rf, u32 *ip, const u8 *mac); -void irdma_cm_teardown_connections(struct irdma_device *iwdev, u32 *ipaddr, - struct irdma_cm_info *nfo, - bool disconnect_all); int irdma_cm_start(struct irdma_device *dev); int irdma_cm_stop(struct irdma_device *dev); bool irdma_ipv4_is_lpb(u32 loc_addr, u32 rem_addr); diff --git a/sys/dev/irdma/irdma_cm.c b/sys/dev/irdma/irdma_cm.c --- a/sys/dev/irdma/irdma_cm.c +++ b/sys/dev/irdma/irdma_cm.c @@ -206,9 +206,10 @@ event.event = type; event.status = status; - irdma_debug(iwdev_to_idev(cm_node->iwdev), IRDMA_DEBUG_CM, + irdma_debug(&cm_node->iwdev->rf->sc_dev, IRDMA_DEBUG_CM, "cm_node %p cm_id=%p state=%d accel=%d event_type=%d status=%d\n", - cm_node, cm_id, cm_node->accelerated, cm_node->state, type, status); + cm_node, cm_id, cm_node->accelerated, cm_node->state, type, + status); switch (type) { case IW_CM_EVENT_CONNECT_REQUEST: @@ -288,8 +289,9 @@ event->cm_info.rem_port = cm_node->rem_port; event->cm_info.loc_port = cm_node->loc_port; event->cm_info.cm_id = cm_node->cm_id; - irdma_debug(iwdev_to_idev(cm_node->iwdev), IRDMA_DEBUG_CM, - "node=%p event=%p type=%u dst=%pI4 src=%pI4\n", cm_node, + irdma_debug(&cm_node->iwdev->rf->sc_dev, IRDMA_DEBUG_CM, + "node=%p event=%p type=%u dst=%pI4 src=%pI4\n", + cm_node, event, type, event->cm_info.loc_addr, event->cm_info.rem_addr); irdma_cm_post_event(event); @@ -356,15 +358,13 @@ u32 hdr_len = 0; if (!cm_node->ah || !cm_node->ah->ah_info.ah_valid) { - irdma_debug(iwdev_to_idev(cm_node->iwdev), IRDMA_DEBUG_CM, - "AH invalid\n"); + irdma_debug(&cm_node->iwdev->rf->sc_dev, IRDMA_DEBUG_CM, "AH invalid\n"); return NULL; } sqbuf = irdma_puda_get_bufpool(vsi->ilq); if (!sqbuf) { - irdma_debug(iwdev_to_idev(cm_node->iwdev), IRDMA_DEBUG_CM, - "SQ buf NULL\n"); + irdma_debug(&cm_node->iwdev->rf->sc_dev, IRDMA_DEBUG_CM, "SQ buf NULL\n"); return NULL; } @@ -645,7 +645,7 @@ if (!sqbuf) return -ENOMEM; - irdma_debug(iwdev_to_idev(cm_node->iwdev), IRDMA_DEBUG_CM, + irdma_debug(&cm_node->iwdev->rf->sc_dev, IRDMA_DEBUG_CM, "caller: %pS cm_node %p cm_id=%p accel=%d state=%d rem_port=0x%04x, loc_port=0x%04x rem_addr=%pI4 loc_addr=%pI4\n", __builtin_return_address(0), cm_node, cm_node->cm_id, cm_node->accelerated, cm_node->state, cm_node->rem_port, @@ -666,8 +666,9 @@ irdma_cleanup_retrans_entry(cm_node); cm_node->cm_core->stats_connect_errs++; if (reset) { - irdma_debug(iwdev_to_idev(cm_node->iwdev), IRDMA_DEBUG_CM, - "cm_node=%p state=%d\n", cm_node, cm_node->state); + irdma_debug(&cm_node->iwdev->rf->sc_dev, + IRDMA_DEBUG_CM, "cm_node=%p state=%d\n", cm_node, + cm_node->state); atomic_inc(&cm_node->refcnt); irdma_send_reset(cm_node); } @@ -687,8 +688,9 @@ irdma_cleanup_retrans_entry(cm_node); cm_node->cm_core->stats_passive_errs++; cm_node->state = IRDMA_CM_STATE_CLOSED; - irdma_debug(iwdev_to_idev(cm_node->iwdev), IRDMA_DEBUG_CM, - "cm_node=%p state =%d\n", cm_node, cm_node->state); + irdma_debug(&cm_node->iwdev->rf->sc_dev, IRDMA_DEBUG_CM, + "cm_node=%p state =%d\n", + cm_node, cm_node->state); if (reset) irdma_send_reset(cm_node); else @@ -747,8 +749,7 @@ offset += 1; continue; case OPTION_NUM_MSS: - irdma_debug(iwdev_to_idev(cm_node->iwdev), - IRDMA_DEBUG_CM, + irdma_debug(&cm_node->iwdev->rf->sc_dev, IRDMA_DEBUG_CM, "MSS Length: %d Offset: %d Size: %d\n", all_options->mss.len, offset, optionsize); got_mss_option = 1; @@ -768,8 +769,7 @@ all_options->windowscale.shiftcount; break; default: - irdma_debug(iwdev_to_idev(cm_node->iwdev), - IRDMA_DEBUG_CM, + irdma_debug(&cm_node->iwdev->rf->sc_dev, IRDMA_DEBUG_CM, "Unsupported TCP Option: %x\n", all_options->base.optionnum); break; @@ -801,9 +801,9 @@ ret = irdma_process_options(cm_node, optionsloc, optionsize, (u32)tcph->th_flags & TH_SYN); if (ret) { - irdma_debug(iwdev_to_idev(cm_node->iwdev), - IRDMA_DEBUG_CM, - "Node %p, Sending Reset\n", cm_node); + irdma_debug(&cm_node->iwdev->rf->sc_dev, IRDMA_DEBUG_CM, + "Node %p, Sending Reset\n", + cm_node); if (passive) irdma_passive_open_err(cm_node, true); else @@ -950,8 +950,9 @@ &cm_node->mpa_hdr, MPA_KEY_REQUEST); if (!cm_node->mpa_hdr.size) { - irdma_debug(iwdev_to_idev(cm_node->iwdev), IRDMA_DEBUG_CM, - "mpa size = %d\n", cm_node->mpa_hdr.size); + irdma_debug(&cm_node->iwdev->rf->sc_dev, IRDMA_DEBUG_CM, + "mpa size = %d\n", + cm_node->mpa_hdr.size); return -EINVAL; } @@ -1061,9 +1062,9 @@ /* Not supported RDMA0 operation */ return -EOPNOTSUPP; - irdma_debug(iwdev_to_idev(cm_node->iwdev), IRDMA_DEBUG_CM, - "MPAV2 Negotiated ORD: %d, IRD: %d\n", cm_node->ord_size, - cm_node->ird_size); + irdma_debug(&cm_node->iwdev->rf->sc_dev, + IRDMA_DEBUG_CM, "MPAV2 Negotiated ORD: %d, IRD: %d\n", + cm_node->ord_size, cm_node->ird_size); return 0; } @@ -1084,8 +1085,8 @@ *type = IRDMA_MPA_REQUEST_ACCEPT; if (len < sizeof(struct ietf_mpa_v1)) { - irdma_debug(iwdev_to_idev(cm_node->iwdev), IRDMA_DEBUG_CM, - "ietf buffer small (%x)\n", len); + irdma_debug(&cm_node->iwdev->rf->sc_dev, + IRDMA_DEBUG_CM, "ietf buffer small (%x)\n", len); return -EINVAL; } @@ -1094,20 +1095,23 @@ priv_data_len = ntohs(mpa_frame->priv_data_len); if (priv_data_len > IETF_MAX_PRIV_DATA_LEN) { - irdma_debug(iwdev_to_idev(cm_node->iwdev), IRDMA_DEBUG_CM, - "private_data too big %d\n", priv_data_len); + irdma_debug(&cm_node->iwdev->rf->sc_dev, IRDMA_DEBUG_CM, + "private_data too big %d\n", + priv_data_len); return -EOVERFLOW; } if (mpa_frame->rev != IETF_MPA_V1 && mpa_frame->rev != IETF_MPA_V2) { - irdma_debug(iwdev_to_idev(cm_node->iwdev), IRDMA_DEBUG_CM, - "unsupported mpa rev = %d\n", mpa_frame->rev); + irdma_debug(&cm_node->iwdev->rf->sc_dev, IRDMA_DEBUG_CM, + "unsupported mpa rev = %d\n", + mpa_frame->rev); return -EINVAL; } if (mpa_frame->rev > cm_node->mpa_frame_rev) { - irdma_debug(iwdev_to_idev(cm_node->iwdev), IRDMA_DEBUG_CM, - "rev %d\n", mpa_frame->rev); + irdma_debug(&cm_node->iwdev->rf->sc_dev, IRDMA_DEBUG_CM, + "rev %d\n", + mpa_frame->rev); return -EINVAL; } @@ -1115,31 +1119,29 @@ if (cm_node->state != IRDMA_CM_STATE_MPAREQ_SENT) { if (memcmp(mpa_frame->key, IEFT_MPA_KEY_REQ, IETF_MPA_KEY_SIZE)) { - irdma_debug(iwdev_to_idev(cm_node->iwdev), - IRDMA_DEBUG_CM, - "Unexpected MPA Key received\n"); + irdma_debug(&cm_node->iwdev->rf->sc_dev, + IRDMA_DEBUG_CM, "Unexpected MPA Key received\n"); return -EINVAL; } } else { if (memcmp(mpa_frame->key, IEFT_MPA_KEY_REP, IETF_MPA_KEY_SIZE)) { - irdma_debug(iwdev_to_idev(cm_node->iwdev), - IRDMA_DEBUG_CM, - "Unexpected MPA Key received\n"); + irdma_debug(&cm_node->iwdev->rf->sc_dev, + IRDMA_DEBUG_CM, "Unexpected MPA Key received\n"); return -EINVAL; } } if (priv_data_len + mpa_hdr_len > len) { - irdma_debug(iwdev_to_idev(cm_node->iwdev), IRDMA_DEBUG_CM, - "ietf buffer len(%x + %x != %x)\n", priv_data_len, - mpa_hdr_len, len); + irdma_debug(&cm_node->iwdev->rf->sc_dev, + IRDMA_DEBUG_CM, "ietf buffer len(%x + %x != %x)\n", + priv_data_len, mpa_hdr_len, len); return -EOVERFLOW; } if (len > IRDMA_MAX_CM_BUF) { - irdma_debug(iwdev_to_idev(cm_node->iwdev), IRDMA_DEBUG_CM, - "ietf buffer large len = %d\n", len); + irdma_debug(&cm_node->iwdev->rf->sc_dev, + IRDMA_DEBUG_CM, "ietf buffer large len = %d\n", len); return -EOVERFLOW; } @@ -1211,7 +1213,7 @@ new_send->timetosend += (HZ / 10); if (cm_node->close_entry) { kfree(new_send); - irdma_debug(iwdev_to_idev(cm_node->iwdev), + irdma_debug(&cm_node->iwdev->rf->sc_dev, IRDMA_DEBUG_CM, "already close entry\n"); return -EINVAL; } @@ -1520,12 +1522,13 @@ * irdma_find_listener - find a cm node listening on this addr-port pair * @cm_core: cm's core * @dst_addr: listener ip addr + * @ipv4: flag indicating IPv4 when true * @dst_port: listener tcp port num * @vlan_id: virtual LAN ID * @listener_state: state to match with listen node's */ static struct irdma_cm_listener * -irdma_find_listener(struct irdma_cm_core *cm_core, u32 *dst_addr, u16 dst_port, +irdma_find_listener(struct irdma_cm_core *cm_core, u32 *dst_addr, bool ipv4, u16 dst_port, u16 vlan_id, enum irdma_cm_listener_state listener_state) { struct irdma_cm_listener *listen_node; @@ -1539,7 +1542,7 @@ list_for_each_entry(listen_node, &cm_core->listen_list, list) { memcpy(listen_addr, listen_node->loc_addr, sizeof(listen_addr)); listen_port = listen_node->loc_port; - if (listen_port != dst_port || + if (listen_node->ipv4 != ipv4 || listen_port != dst_port || !(listener_state & listen_node->listener_state)) continue; /* compare node pair, return node handle if a match */ @@ -1579,13 +1582,13 @@ child_listen_node = list_entry(pos, struct irdma_cm_listener, child_listen_list); if (child_listen_node->ipv4) - irdma_debug(iwdev_to_idev(iwdev), IRDMA_DEBUG_CM, + irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_CM, "removing child listen for IP=%pI4, port=%d, vlan=%d\n", child_listen_node->loc_addr, child_listen_node->loc_port, child_listen_node->vlan_id); else - irdma_debug(iwdev_to_idev(iwdev), IRDMA_DEBUG_CM, + irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_CM, "removing child listen for IP=%pI6, port=%d, vlan=%d\n", child_listen_node->loc_addr, child_listen_node->loc_port, @@ -1603,8 +1606,8 @@ } else { ret = 0; } - irdma_debug(iwdev_to_idev(iwdev), IRDMA_DEBUG_CM, - "Child listen node freed = %p\n", + irdma_debug(&iwdev->rf->sc_dev, + IRDMA_DEBUG_CM, "Child listen node freed = %p\n", child_listen_node); kfree(child_listen_node); cm_parent_listen_node->cm_core->stats_listen_nodes_destroyed++; @@ -1614,6 +1617,10 @@ return ret; } +static u8 irdma_get_egress_vlan_prio(u32 *loc_addr, u8 prio, bool ipv4){ + return prio; +} + /** * irdma_netdev_vlan_ipv6 - Gets the netdev and mac * @addr: local IPv6 address @@ -1702,20 +1709,18 @@ if_addr_rlock(ip_dev); IRDMA_TAILQ_FOREACH(ifp, &ip_dev->if_addrhead, ifa_link) { - irdma_debug(iwdev_to_idev(iwdev), IRDMA_DEBUG_CM, + irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_CM, "IP=%pI6, vlan_id=%d, MAC=%pM\n", &((struct sockaddr_in6 *)ifp->ifa_addr)->sin6_addr, rdma_vlan_dev_vlan_id(ip_dev), IF_LLADDR(ip_dev)); if (((struct sockaddr_in6 *)ifp->ifa_addr)->sin6_family != AF_INET6) continue; child_listen_node = kzalloc(sizeof(*child_listen_node), GFP_KERNEL); - irdma_debug(iwdev_to_idev(iwdev), IRDMA_DEBUG_CM, + irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_CM, "Allocating child listener %p\n", child_listen_node); if (!child_listen_node) { - irdma_debug(iwdev_to_idev(iwdev), - IRDMA_DEBUG_CM, - "listener memory allocation\n"); + irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_CM, "listener memory allocation\n"); ret = -ENOMEM; if_addr_runlock(ip_dev); goto exit; @@ -1729,6 +1734,11 @@ ((struct sockaddr_in6 *)ifp->ifa_addr)->sin6_addr.__u6_addr.__u6_addr32); memcpy(cm_info->loc_addr, child_listen_node->loc_addr, sizeof(cm_info->loc_addr)); + if (!iwdev->vsi.dscp_mode) + cm_info->user_pri = + irdma_get_egress_vlan_prio(child_listen_node->loc_addr, + cm_info->user_pri, + false); ret = irdma_manage_qhash(iwdev, cm_info, IRDMA_QHASH_TYPE_TCP_SYN, IRDMA_QHASH_MANAGE_TYPE_ADD, @@ -1785,20 +1795,19 @@ if_addr_rlock(ip_dev); IRDMA_TAILQ_FOREACH(ifa, &ip_dev->if_addrhead, ifa_link) { - irdma_debug(iwdev_to_idev(iwdev), IRDMA_DEBUG_CM, + irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_CM, "Allocating child CM Listener forIP=%pI4, vlan_id=%d, MAC=%pM\n", - &ifa->ifa_addr, - rdma_vlan_dev_vlan_id(ip_dev), IF_LLADDR(ip_dev)); + &ifa->ifa_addr, rdma_vlan_dev_vlan_id(ip_dev), + IF_LLADDR(ip_dev)); if (((struct sockaddr_in *)ifa->ifa_addr)->sin_family != AF_INET) continue; child_listen_node = kzalloc(sizeof(*child_listen_node), GFP_KERNEL); cm_parent_listen_node->cm_core->stats_listen_nodes_created++; - irdma_debug(iwdev_to_idev(iwdev), IRDMA_DEBUG_CM, + irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_CM, "Allocating child listener %p\n", child_listen_node); if (!child_listen_node) { - irdma_debug(iwdev_to_idev(iwdev), IRDMA_DEBUG_CM, - "listener memory allocation\n"); + irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_CM, "listener memory allocation\n"); if_addr_runlock(ip_dev); ret = -ENOMEM; goto exit; @@ -1812,6 +1821,11 @@ ntohl(((struct sockaddr_in *)ifa->ifa_addr)->sin_addr.s_addr); memcpy(cm_info->loc_addr, child_listen_node->loc_addr, sizeof(cm_info->loc_addr)); + if (!iwdev->vsi.dscp_mode) + cm_info->user_pri = + irdma_get_egress_vlan_prio(child_listen_node->loc_addr, + cm_info->user_pri, + true); ret = irdma_manage_qhash(iwdev, cm_info, IRDMA_QHASH_TYPE_TCP_SYN, IRDMA_QHASH_MANAGE_TYPE_ADD, @@ -1932,7 +1946,7 @@ err = irdma_send_reset(cm_node); if (err) { cm_node->state = IRDMA_CM_STATE_CLOSED; - irdma_debug(iwdev_to_idev(cm_node->iwdev), + irdma_debug(&cm_node->iwdev->rf->sc_dev, IRDMA_DEBUG_CM, "send reset failed\n"); } else { old_state = cm_node->state; @@ -1971,9 +1985,10 @@ cm_core->stats_listen_destroyed++; cm_core->stats_listen_nodes_destroyed++; - irdma_debug(iwdev_to_idev(listener->iwdev), IRDMA_DEBUG_CM, - "loc_port=0x%04x loc_addr=%pI4 cm_listen_node=%p cm_id=%p qhash_set=%d vlan_id=%d apbvt_del=%d\n", - listener->loc_port, listener->loc_addr, listener, listener->cm_id, listener->qhash_set, + irdma_debug(&listener->iwdev->rf->sc_dev, IRDMA_DEBUG_CM, + "loc_port=0x%04x loc_addr=%pI4 cm_listen_node=%p cm_id=%p qhash_set=%d vlan_id=%d apbvt_del=%d\n", + listener->loc_port, listener->loc_addr, listener, + listener->cm_id, listener->qhash_set, listener->vlan_id, apbvt_del); kfree(listener); listener = NULL; @@ -2105,12 +2120,10 @@ ah_info.ipv4_valid = true; ah_info.dest_ip_addr[0] = cm_node->rem_addr[0]; ah_info.src_ip_addr[0] = cm_node->loc_addr[0]; -#ifdef VIMAGE CURVNET_SET_QUIET(vnet); ah_info.do_lpbk = irdma_ipv4_is_lpb(ah_info.src_ip_addr[0], ah_info.dest_ip_addr[0]); CURVNET_RESTORE(); -#endif } else { memcpy(ah_info.dest_ip_addr, cm_node->rem_addr, sizeof(ah_info.dest_ip_addr)); @@ -2181,8 +2194,7 @@ cm_node->user_pri = cm_info->user_pri; if (listener) { if (listener->tos != cm_info->tos) - irdma_dev_warn( - &iwdev->rf->sc_dev, + irdma_dev_warn(&iwdev->ibdev, "application TOS[%d] and remote client TOS[%d] mismatch\n", listener->tos, cm_info->tos); if (iwdev->vsi.dscp_mode) { @@ -2190,9 +2202,14 @@ } else { cm_node->tos = max(listener->tos, cm_info->tos); cm_node->user_pri = rt_tos2priority(cm_node->tos); + cm_node->user_pri = + irdma_get_egress_vlan_prio(cm_info->loc_addr, + cm_node->user_pri, + cm_info->ipv4); } - irdma_debug(iwdev_to_idev(iwdev), IRDMA_DEBUG_DCB, - "listener: TOS:[%d] UP:[%d]\n", cm_node->tos, + irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_DCB, + "listener: TOS:[%d] UP:[%d]\n", + cm_node->tos, cm_node->user_pri); } memcpy(cm_node->loc_addr, cm_info->loc_addr, sizeof(cm_node->loc_addr)); @@ -2247,8 +2264,8 @@ /* if the node is destroyed before connection was accelerated */ if (!cm_node->accelerated && cm_node->accept_pend) { - irdma_debug(iwdev_to_idev(cm_node->iwdev), IRDMA_DEBUG_CM, - "node destroyed before established\n"); + irdma_debug(&cm_node->iwdev->rf->sc_dev, + IRDMA_DEBUG_CM, "node destroyed before established\n"); atomic_dec(&cm_node->listener->pend_accepts_cnt); } if (cm_node->close_entry) @@ -2371,8 +2388,9 @@ break; case IRDMA_CM_STATE_OFFLOADED: default: - irdma_debug(iwdev_to_idev(cm_node->iwdev), IRDMA_DEBUG_CM, - "bad state node state = %d\n", cm_node->state); + irdma_debug(&cm_node->iwdev->rf->sc_dev, IRDMA_DEBUG_CM, + "bad state node state = %d\n", + cm_node->state); break; } } @@ -2386,10 +2404,11 @@ irdma_handle_rst_pkt(struct irdma_cm_node *cm_node, struct irdma_puda_buf *rbuf) { - irdma_debug(iwdev_to_idev(cm_node->iwdev), IRDMA_DEBUG_CM, + irdma_debug(&cm_node->iwdev->rf->sc_dev, IRDMA_DEBUG_CM, "caller: %pS cm_node=%p state=%d rem_port=0x%04x loc_port=0x%04x rem_addr=%pI4 loc_addr=%pI4\n", __builtin_return_address(0), cm_node, cm_node->state, - cm_node->rem_port, cm_node->loc_port, cm_node->rem_addr, cm_node->loc_addr); + cm_node->rem_port, cm_node->loc_port, cm_node->rem_addr, + cm_node->loc_addr); irdma_cleanup_retrans_entry(cm_node); switch (cm_node->state) { @@ -2463,7 +2482,7 @@ switch (cm_node->state) { case IRDMA_CM_STATE_ESTABLISHED: if (res_type == IRDMA_MPA_REQUEST_REJECT) - irdma_debug(iwdev_to_idev(cm_node->iwdev), + irdma_debug(&cm_node->iwdev->rf->sc_dev, IRDMA_DEBUG_CM, "state for reject\n"); cm_node->state = IRDMA_CM_STATE_MPAREQ_RCVD; type = IRDMA_CM_EVENT_MPA_REQ; @@ -2483,8 +2502,9 @@ irdma_send_ack(cm_node); break; default: - irdma_debug(iwdev_to_idev(cm_node->iwdev), IRDMA_DEBUG_CM, - "wrong cm_node state =%d\n", cm_node->state); + irdma_debug(&cm_node->iwdev->rf->sc_dev, IRDMA_DEBUG_CM, + "wrong cm_node state =%d\n", + cm_node->state); break; } irdma_create_event(cm_node, type); @@ -2528,8 +2548,8 @@ !between(seq, rcv_nxt, (rcv_nxt + rcv_wnd))) err = -1; if (err) - irdma_debug(iwdev_to_idev(cm_node->iwdev), IRDMA_DEBUG_CM, - "seq number err\n"); + irdma_debug(&cm_node->iwdev->rf->sc_dev, + IRDMA_DEBUG_CM, "seq number err\n"); return err; } @@ -2636,7 +2656,7 @@ irdma_cleanup_retrans_entry(cm_node); /* active open */ if (irdma_check_syn(cm_node, tcph)) { - irdma_debug(iwdev_to_idev(cm_node->iwdev), + irdma_debug(&cm_node->iwdev->rf->sc_dev, IRDMA_DEBUG_CM, "check syn fail\n"); return; } @@ -2644,8 +2664,7 @@ /* setup options */ err = irdma_handle_tcp_options(cm_node, tcph, optionsize, 0); if (err) { - irdma_debug(iwdev_to_idev(cm_node->iwdev), - IRDMA_DEBUG_CM, + irdma_debug(&cm_node->iwdev->rf->sc_dev, IRDMA_DEBUG_CM, "cm_node=%p tcp_options failed\n", cm_node); break; @@ -2655,8 +2674,7 @@ irdma_send_ack(cm_node); /* ACK for the syn_ack */ err = irdma_send_mpa_request(cm_node); if (err) { - irdma_debug(iwdev_to_idev(cm_node->iwdev), - IRDMA_DEBUG_CM, + irdma_debug(&cm_node->iwdev->rf->sc_dev, IRDMA_DEBUG_CM, "cm_node=%p irdma_send_mpa_request failed\n", cm_node); break; @@ -2839,7 +2857,7 @@ unsigned long flags; /* cannot have multiple matching listeners */ - listener = irdma_find_listener(cm_core, cm_info->loc_addr, + listener = irdma_find_listener(cm_core, cm_info->loc_addr, cm_info->ipv4, cm_info->loc_port, cm_info->vlan_id, IRDMA_CM_LISTENER_EITHER_STATE); if (listener && @@ -2961,8 +2979,8 @@ cm_node->state = IRDMA_CM_STATE_CLOSED; if (irdma_send_reset(cm_node)) - irdma_debug(iwdev_to_idev(cm_node->iwdev), IRDMA_DEBUG_CM, - "send reset failed\n"); + irdma_debug(&cm_node->iwdev->rf->sc_dev, + IRDMA_DEBUG_CM, "send reset failed\n"); return ret; } @@ -3008,9 +3026,8 @@ break; case IRDMA_CM_STATE_OFFLOADED: if (cm_node->send_entry) - irdma_debug(iwdev_to_idev(cm_node->iwdev), - IRDMA_DEBUG_CM, - "CM send_entry in OFFLOADED state\n"); + irdma_debug(&cm_node->iwdev->rf->sc_dev, + IRDMA_DEBUG_CM, "CM send_entry in OFFLOADED state\n"); irdma_rem_ref_cm_node(cm_node); break; } @@ -3059,8 +3076,8 @@ cm_info.user_pri = (vtag & EVL_PRI_MASK) >> VLAN_PRIO_SHIFT; cm_info.vlan_id = vtag & EVL_VLID_MASK; - irdma_debug(iwdev_to_idev(cm_core->iwdev), - IRDMA_DEBUG_CM, "vlan_id=%d\n", + irdma_debug(&cm_core->iwdev->rf->sc_dev, IRDMA_DEBUG_CM, + "vlan_id=%d\n", cm_info.vlan_id); } else { cm_info.vlan_id = 0xFFFF; @@ -3096,12 +3113,13 @@ listener = irdma_find_listener(cm_core, cm_info.loc_addr, + cm_info.ipv4, cm_info.loc_port, cm_info.vlan_id, IRDMA_CM_LISTENER_ACTIVE_STATE); if (!listener) { cm_info.cm_id = NULL; - irdma_debug(iwdev_to_idev(cm_core->iwdev), + irdma_debug(&cm_core->iwdev->rf->sc_dev, IRDMA_DEBUG_CM, "no listener found\n"); return; } @@ -3110,7 +3128,7 @@ cm_node = irdma_make_cm_node(cm_core, iwdev, &cm_info, listener); if (!cm_node) { - irdma_debug(iwdev_to_idev(cm_core->iwdev), + irdma_debug(&cm_core->iwdev->rf->sc_dev, IRDMA_DEBUG_CM, "allocate node failed\n"); atomic_dec(&listener->refcnt); return; @@ -3195,15 +3213,10 @@ void irdma_cleanup_cm_core(struct irdma_cm_core *cm_core) { - unsigned long flags; - if (!cm_core) return; - spin_lock_irqsave(&cm_core->ht_lock, flags); - if (timer_pending(&cm_core->tcp_timer)) - del_timer_sync(&cm_core->tcp_timer); - spin_unlock_irqrestore(&cm_core->ht_lock, flags); + del_timer_sync(&cm_core->tcp_timer); destroy_workqueue(cm_core->event_wq); cm_core->dev->ws_reset(&cm_core->iwdev->vsi); @@ -3344,8 +3357,9 @@ spin_lock_irqsave(&iwdev->rf->qptable_lock, flags); if (!iwdev->rf->qp_table[iwqp->ibqp.qp_num]) { spin_unlock_irqrestore(&iwdev->rf->qptable_lock, flags); - irdma_debug(iwdev_to_idev(iwdev), IRDMA_DEBUG_CM, - "qp_id %d is already freed\n", iwqp->ibqp.qp_num); + irdma_debug(&iwdev->rf->sc_dev, + IRDMA_DEBUG_CM, "qp_id %d is already freed\n", + iwqp->ibqp.qp_num); kfree(work); return; } @@ -3368,8 +3382,7 @@ iwqp->active_conn = 0; /* close the CM node down if it is still active */ - irdma_debug(iwdev_to_idev(iwdev), IRDMA_DEBUG_CM, - "Call close API\n"); + irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_CM, "Call close API\n"); irdma_cm_close(iwqp->cm_node); } @@ -3473,7 +3486,7 @@ IW_CM_EVENT_DISCONNECT, disconn_status); if (err) - irdma_debug(iwdev_to_idev(iwdev), IRDMA_DEBUG_CM, + irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_CM, "disconnect event failed: - cm_id = %p\n", cm_id); } @@ -3482,7 +3495,7 @@ err = irdma_send_cm_event(iwqp->cm_node, cm_id, IW_CM_EVENT_CLOSE, 0); if (err) - irdma_debug(iwdev_to_idev(iwdev), IRDMA_DEBUG_CM, + irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_CM, "close event failed: - cm_id = %p\n", cm_id); irdma_qp_disconnect(iwqp); @@ -3565,8 +3578,8 @@ irdma_netdev_vlan_ipv6(cm_node->loc_addr, &cm_node->vlan_id, NULL); } - irdma_debug(iwdev_to_idev(iwdev), IRDMA_DEBUG_CM, - "Accept vlan_id=%d\n", cm_node->vlan_id); + irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_CM, "Accept vlan_id=%d\n", + cm_node->vlan_id); if (cm_node->state == IRDMA_CM_STATE_LISTENER_DESTROYED) { ret = -EINVAL; @@ -3646,7 +3659,7 @@ iwqp->rts_ae_rcvd, IRDMA_MAX_TIMEOUT); if (!wait_ret) { - irdma_debug(iwdev_to_idev(iwdev), IRDMA_DEBUG_CM, + irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_CM, "Slow Connection: cm_node=%p, loc_port=%d, rem_port=%d, cm_id=%p\n", cm_node, cm_node->loc_port, cm_node->rem_port, cm_node->cm_id); @@ -3664,7 +3677,7 @@ cm_node->accept_pend = 0; } - irdma_debug(iwdev_to_idev(iwdev), IRDMA_DEBUG_CM, + irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_CM, "rem_port=0x%04x, loc_port=0x%04x rem_addr=%pI4 loc_addr=%pI4 cm_node=%p cm_id=%p qp_id = %d\n\n", cm_node->rem_port, cm_node->loc_port, cm_node->rem_addr, cm_node->loc_addr, cm_node, cm_id, ibqp->qp_num); @@ -3770,17 +3783,20 @@ raddr6->sin6_addr.__u6_addr.__u6_addr32); cm_info.loc_port = ntohs(laddr6->sin6_port); cm_info.rem_port = ntohs(raddr6->sin6_port); - irdma_netdev_vlan_ipv6(cm_info.loc_addr, &cm_info.vlan_id, - NULL); + irdma_netdev_vlan_ipv6(cm_info.loc_addr, &cm_info.vlan_id, NULL); } cm_info.cm_id = cm_id; cm_info.qh_qpid = iwdev->vsi.ilq->qp_id; cm_info.tos = cm_id->tos; - if (iwdev->vsi.dscp_mode) + if (iwdev->vsi.dscp_mode) { cm_info.user_pri = iwqp->sc_qp.vsi->dscp_map[irdma_tos2dscp(cm_info.tos)]; - else + } else { cm_info.user_pri = rt_tos2priority(cm_id->tos); + cm_info.user_pri = irdma_get_egress_vlan_prio(cm_info.loc_addr, + cm_info.user_pri, + cm_info.ipv4); + } if (iwqp->sc_qp.dev->ws_add(iwqp->sc_qp.vsi, cm_info.user_pri)) return -ENOMEM; @@ -3788,8 +3804,9 @@ irdma_qp_add_qos(&iwqp->sc_qp); if (iwdev->rf->sc_dev.hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_2) iwdev->rf->check_fc(&iwdev->vsi, &iwqp->sc_qp); - irdma_debug(iwdev_to_idev(iwdev), IRDMA_DEBUG_DCB, - "TOS:[%d] UP:[%d]\n", cm_id->tos, cm_info.user_pri); + irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_DCB, + "TOS:[%d] UP:[%d]\n", cm_id->tos, + cm_info.user_pri); ret = irdma_create_cm_node(&iwdev->cm_core, iwdev, conn_param, &cm_info, &cm_node); @@ -3826,7 +3843,7 @@ goto err; } - irdma_debug(iwdev_to_idev(iwdev), IRDMA_DEBUG_CM, + irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_CM, "rem_port=0x%04x, loc_port=0x%04x rem_addr=%pI4 loc_addr=%pI4 cm_node=%p cm_id=%p qp_id = %d\n\n", cm_node->rem_port, cm_node->loc_port, cm_node->rem_addr, cm_node->loc_addr, cm_node, cm_id, ibqp->qp_num); @@ -3835,12 +3852,12 @@ err: if (cm_info.ipv4) - irdma_debug(iwdev_to_idev(iwdev), IRDMA_DEBUG_CM, - "connect() FAILED: dest addr=%pI4", + irdma_debug(&iwdev->rf->sc_dev, + IRDMA_DEBUG_CM, "connect() FAILED: dest addr=%pI4", cm_info.rem_addr); else - irdma_debug(iwdev_to_idev(iwdev), IRDMA_DEBUG_CM, - "connect() FAILED: dest addr=%pI6", + irdma_debug(&iwdev->rf->sc_dev, + IRDMA_DEBUG_CM, "connect() FAILED: dest addr=%pI6", cm_info.rem_addr); irdma_rem_ref_cm_node(cm_node); iwdev->cm_core.stats_connect_errs++; @@ -3911,8 +3928,8 @@ cm_listen_node = irdma_make_listen_node(&iwdev->cm_core, iwdev, &cm_info); if (!cm_listen_node) { - irdma_debug(iwdev_to_idev(iwdev), IRDMA_DEBUG_CM, - "cm_listen_node == NULL\n"); + irdma_debug(&iwdev->rf->sc_dev, + IRDMA_DEBUG_CM, "cm_listen_node == NULL\n"); return -ENOMEM; } @@ -3931,6 +3948,11 @@ if (err) goto error; } else { + if (!iwdev->vsi.dscp_mode) + cm_info.user_pri = cm_listen_node->user_pri = + irdma_get_egress_vlan_prio(cm_info.loc_addr, + cm_info.user_pri, + cm_info.ipv4); err = irdma_manage_qhash(iwdev, &cm_info, IRDMA_QHASH_TYPE_TCP_SYN, IRDMA_QHASH_MANAGE_TYPE_ADD, @@ -3948,9 +3970,10 @@ } cm_id->add_ref(cm_id); cm_listen_node->cm_core->stats_listen_created++; - irdma_debug(iwdev_to_idev(iwdev), IRDMA_DEBUG_CM, + irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_CM, "loc_port=0x%04x loc_addr=%pI4 cm_listen_node=%p cm_id=%p qhash_set=%d vlan_id=%d\n", - cm_listen_node->loc_port, cm_listen_node->loc_addr, cm_listen_node, cm_listen_node->cm_id, + cm_listen_node->loc_port, cm_listen_node->loc_addr, + cm_listen_node, cm_listen_node->cm_id, cm_listen_node->qhash_set, cm_listen_node->vlan_id); return 0; @@ -3976,8 +3999,8 @@ irdma_cm_del_listen(&iwdev->cm_core, cm_id->provider_data, true); else - irdma_debug(iwdev_to_idev(iwdev), IRDMA_DEBUG_CM, - "cm_id->provider_data was NULL\n"); + irdma_debug(&iwdev->rf->sc_dev, + IRDMA_DEBUG_CM, "cm_id->provider_data was NULL\n"); cm_id->rem_ref(cm_id); @@ -3985,7 +4008,8 @@ } /** - * irdma_teardown_list_prep - add conn nodes slated for tear down to list + * irdma_iw_teardown_list_prep - add conn nodes slated for tear + * down to list * @cm_core: cm's core * @teardown_list: a list to which cm_node will be selected * @ipaddr: pointer to ip address @@ -3993,11 +4017,11 @@ * @disconnect_all: flag indicating disconnect all QPs */ static void -irdma_teardown_list_prep(struct irdma_cm_core *cm_core, - struct list_head *teardown_list, - u32 *ipaddr, - struct irdma_cm_info *nfo, - bool disconnect_all) +irdma_iw_teardown_list_prep(struct irdma_cm_core *cm_core, + struct list_head *teardown_list, + u32 *ipaddr, + struct irdma_cm_info *nfo, + bool disconnect_all) { struct irdma_cm_node *cm_node; int bkt; @@ -4011,6 +4035,75 @@ } } +static inline bool +irdma_ip_vlan_match(u32 *ip1, u16 vlan_id1, + bool check_vlan, u32 *ip2, + u16 vlan_id2, bool ipv4) +{ + return (!check_vlan || vlan_id1 == vlan_id2) && + !memcmp(ip1, ip2, ipv4 ? 4 : 16); +} + +/** + * irdma_roce_teardown_list_prep - add conn nodes slated for + * tear down to list + * @iwdev: RDMA device + * @teardown_list: a list to which cm_node will be selected + * @ipaddr: pointer to ip address + * @nfo: pointer to cm_info structure instance + * @disconnect_all: flag indicating disconnect all QPs + */ +static void +irdma_roce_teardown_list_prep(struct irdma_device *iwdev, + struct list_head *teardown_list, + u32 *ipaddr, + struct irdma_cm_info *nfo, + bool disconnect_all) +{ + struct irdma_sc_vsi *vsi = &iwdev->vsi; + struct irdma_sc_qp *sc_qp; + struct list_head *list_node; + struct irdma_qp *qp; + unsigned long flags; + int i; + + for (i = 0; i < IRDMA_MAX_USER_PRIORITY; i++) { + mutex_lock(&vsi->qos[i].qos_mutex); + list_for_each(list_node, &vsi->qos[i].qplist) { + u32 qp_ip[4]; + + sc_qp = container_of(list_node, struct irdma_sc_qp, + list); + if (sc_qp->qp_uk.qp_type != IRDMA_QP_TYPE_ROCE_RC) + continue; + + qp = sc_qp->qp_uk.back_qp; + if (!disconnect_all) { + if (nfo->ipv4) + qp_ip[0] = qp->udp_info.local_ipaddr[3]; + else + memcpy(qp_ip, + &qp->udp_info.local_ipaddr[0], + sizeof(qp_ip)); + } + + if (disconnect_all || + irdma_ip_vlan_match(qp_ip, + qp->udp_info.vlan_tag & EVL_VLID_MASK, + qp->udp_info.insert_vlan_tag, + ipaddr, nfo->vlan_id, nfo->ipv4)) { + spin_lock_irqsave(&iwdev->rf->qptable_lock, flags); + if (iwdev->rf->qp_table[sc_qp->qp_uk.qp_id]) { + irdma_qp_add_ref(&qp->ibqp); + list_add(&qp->teardown_entry, teardown_list); + } + spin_unlock_irqrestore(&iwdev->rf->qptable_lock, flags); + } + } + mutex_unlock(&vsi->qos[i].qos_mutex); + } +} + /** * irdma_cm_event_connected - handle connected active node * @event: the info for cm_node of connection @@ -4054,7 +4147,7 @@ iwqp->rts_ae_rcvd, IRDMA_MAX_TIMEOUT); if (!wait_ret) - irdma_debug(iwdev_to_idev(iwdev), IRDMA_DEBUG_CM, + irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_CM, "Slow Connection: cm_node=%p, loc_port=%d, rem_port=%d, cm_id=%p\n", cm_node, cm_node->loc_port, cm_node->rem_port, cm_node->cm_id); @@ -4092,8 +4185,9 @@ if (!iwqp) return; - irdma_debug(iwdev_to_idev(cm_node->iwdev), IRDMA_DEBUG_CM, - "reset event %p - cm_id = %p\n", event->cm_node, cm_id); + irdma_debug(&cm_node->iwdev->rf->sc_dev, IRDMA_DEBUG_CM, + "reset event %p - cm_id = %p\n", + event->cm_node, cm_id); iwqp->cm_id = NULL; irdma_send_cm_event(cm_node, cm_node->cm_id, IW_CM_EVENT_DISCONNECT, @@ -4144,8 +4238,9 @@ irdma_event_connect_error(event); break; default: - irdma_debug(iwdev_to_idev(cm_node->iwdev), IRDMA_DEBUG_CM, - "bad event type = %d\n", event->type); + irdma_debug(&cm_node->iwdev->rf->sc_dev, IRDMA_DEBUG_CM, + "bad event type = %d\n", + event->type); break; } @@ -4174,8 +4269,9 @@ * * teardown QPs where source or destination addr matches ip addr */ -void -irdma_cm_teardown_connections(struct irdma_device *iwdev, u32 *ipaddr, +static void +irdma_cm_teardown_connections(struct irdma_device *iwdev, + u32 *ipaddr, struct irdma_cm_info *nfo, bool disconnect_all) { @@ -4185,20 +4281,34 @@ struct irdma_cm_node *cm_node; struct list_head teardown_list; struct ib_qp_attr attr; + struct irdma_qp *qp; INIT_LIST_HEAD(&teardown_list); rcu_read_lock(); - irdma_teardown_list_prep(cm_core, &teardown_list, ipaddr, nfo, disconnect_all); + irdma_iw_teardown_list_prep(cm_core, &teardown_list, ipaddr, nfo, disconnect_all); rcu_read_unlock(); + attr.qp_state = IB_QPS_ERR; list_for_each_safe(list_node, list_core_temp, &teardown_list) { cm_node = container_of(list_node, struct irdma_cm_node, teardown_entry); - attr.qp_state = IB_QPS_ERR; irdma_modify_qp(&cm_node->iwqp->ibqp, &attr, IB_QP_STATE, NULL); if (iwdev->rf->reset) irdma_cm_disconn(cm_node->iwqp); irdma_rem_ref_cm_node(cm_node); } + + if (!rdma_protocol_roce(&iwdev->ibdev, 1)) + return; + + INIT_LIST_HEAD(&teardown_list); + irdma_roce_teardown_list_prep(iwdev, &teardown_list, ipaddr, nfo, disconnect_all); + + list_for_each_safe(list_node, list_core_temp, &teardown_list) { + qp = container_of(list_node, struct irdma_qp, teardown_entry); + irdma_modify_qp_roce(&qp->ibqp, &attr, IB_QP_STATE, NULL); + irdma_ib_qp_event(qp, IRDMA_QP_EVENT_CATASTROPHIC); + irdma_qp_rem_ref(&qp->ibqp); + } } diff --git a/sys/dev/irdma/irdma_ctrl.c b/sys/dev/irdma/irdma_ctrl.c --- a/sys/dev/irdma/irdma_ctrl.c +++ b/sys/dev/irdma/irdma_ctrl.c @@ -2035,7 +2035,7 @@ * irdma_get_stats_idx - Return stats index * @vsi: pointer to the vsi */ -static u8 irdma_get_stats_idx(struct irdma_sc_vsi *vsi){ +static u16 irdma_get_stats_idx(struct irdma_sc_vsi *vsi){ struct irdma_stats_inst_info stats_info = {0}; struct irdma_sc_dev *dev = vsi->dev; @@ -2081,7 +2081,7 @@ /* when stat allocation is not required default to fcn_id. */ vsi->stats_idx = info->fcn_id; if (info->alloc_stats_inst) { - u8 stats_idx = irdma_get_stats_idx(vsi); + u16 stats_idx = irdma_get_stats_idx(vsi); if (stats_idx != IRDMA_INVALID_STATS_IDX) { vsi->stats_inst_alloc = true; @@ -2368,7 +2368,6 @@ info->ae_code | FIELD_PREP(IRDMA_CQPSQ_FWQE_AESOURCE, info->ae_src) : 0; set_64bit_val(wqe, IRDMA_BYTE_8, temp); - hdr = qp->qp_uk.qp_id | FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_FLUSH_WQES) | FIELD_PREP(IRDMA_CQPSQ_FWQE_GENERATE_AE, info->generate_ae) | @@ -2944,7 +2943,7 @@ * parses fpm commit info and copy base value * of hmc objects in hmc_info */ -static int +static void irdma_sc_parse_fpm_commit_buf(struct irdma_sc_dev *dev, __le64 * buf, struct irdma_hmc_obj_info *info, u32 *sd) @@ -3015,7 +3014,6 @@ else *sd = (u32)(size >> 21); - return 0; } /** @@ -3986,10 +3984,8 @@ ceq->polarity ^= 1; } while (cq_idx == IRDMA_INVALID_CQ_IDX); - if (cq) { - cq->cq_uk.armed = false; + if (cq) irdma_sc_cq_ack(cq); - } return cq; } @@ -4216,12 +4212,12 @@ case IRDMA_AE_LLP_RECEIVED_MPA_CRC_ERROR: case IRDMA_AE_LLP_SEGMENT_TOO_SMALL: case IRDMA_AE_LLP_TOO_MANY_RETRIES: + case IRDMA_AE_LCE_QP_CATASTROPHIC: case IRDMA_AE_LLP_DOUBT_REACHABILITY: case IRDMA_AE_LLP_CONNECTION_ESTABLISHED: case IRDMA_AE_RESET_SENT: case IRDMA_AE_TERMINATE_SENT: case IRDMA_AE_RESET_NOT_SENT: - case IRDMA_AE_LCE_QP_CATASTROPHIC: case IRDMA_AE_QP_SUSPEND_COMPLETE: case IRDMA_AE_UDA_L4LEN_INVALID: info->qp = true; @@ -4247,6 +4243,13 @@ info->qp = true; info->compl_ctx = compl_ctx; break; + case IRDMA_AE_RESOURCE_EXHAUSTION: + /* + * ae_src contains the exhausted resource with a unique decoding. Set RSVD here to prevent matching + * with a CQ or QP. + */ + ae_src = IRDMA_AE_SOURCE_RSVD; + break; default: break; } @@ -4257,6 +4260,7 @@ info->qp = true; info->rq = true; info->compl_ctx = compl_ctx; + info->err_rq_idx_valid = true; break; case IRDMA_AE_SOURCE_CQ: case IRDMA_AE_SOURCE_CQ_0110: @@ -4272,6 +4276,10 @@ info->compl_ctx = compl_ctx; break; case IRDMA_AE_SOURCE_IN_WR: + info->qp = true; + info->compl_ctx = compl_ctx; + info->in_rdrsp_wr = true; + break; case IRDMA_AE_SOURCE_IN_RR: info->qp = true; info->compl_ctx = compl_ctx; @@ -4300,12 +4308,11 @@ * @dev: sc device struct * @count: allocate count */ -int +void irdma_sc_repost_aeq_entries(struct irdma_sc_dev *dev, u32 count) { db_wr32(count, dev->aeq_alloc_db); - return 0; } /** @@ -4547,9 +4554,9 @@ ret_code = irdma_sc_commit_fpm_val(dev->cqp, 0, hmc_info->hmc_fn_id, &commit_fpm_mem, true, wait_type); if (!ret_code) - ret_code = irdma_sc_parse_fpm_commit_buf(dev, dev->fpm_commit_buf, - hmc_info->hmc_obj, - &hmc_info->sd_table.sd_cnt); + irdma_sc_parse_fpm_commit_buf(dev, dev->fpm_commit_buf, + hmc_info->hmc_obj, + &hmc_info->sd_table.sd_cnt); irdma_debug_buf(dev, IRDMA_DEBUG_HMC, "COMMIT FPM BUFFER", commit_fpm_mem.va, IRDMA_COMMIT_FPM_BUF_SIZE); @@ -4915,12 +4922,14 @@ hmc_info->hmc_obj[IRDMA_HMC_IW_RRFFL].cnt = hmc_info->hmc_obj[IRDMA_HMC_IW_RRF].cnt / hmc_fpm_misc->rrf_block_size; - if (hmc_info->hmc_obj[IRDMA_HMC_IW_OOISC].max_cnt) - hmc_info->hmc_obj[IRDMA_HMC_IW_OOISC].cnt = 32 * qpwanted; - if (hmc_info->hmc_obj[IRDMA_HMC_IW_OOISCFFL].max_cnt) - hmc_info->hmc_obj[IRDMA_HMC_IW_OOISCFFL].cnt = - hmc_info->hmc_obj[IRDMA_HMC_IW_OOISC].cnt / - hmc_fpm_misc->ooiscf_block_size; + if (dev->cqp->protocol_used == IRDMA_IWARP_PROTOCOL_ONLY) { + if (hmc_info->hmc_obj[IRDMA_HMC_IW_OOISC].max_cnt) + hmc_info->hmc_obj[IRDMA_HMC_IW_OOISC].cnt = 32 * qpwanted; + if (hmc_info->hmc_obj[IRDMA_HMC_IW_OOISCFFL].max_cnt) + hmc_info->hmc_obj[IRDMA_HMC_IW_OOISCFFL].cnt = + hmc_info->hmc_obj[IRDMA_HMC_IW_OOISC].cnt / + hmc_fpm_misc->ooiscf_block_size; + } } /** diff --git a/sys/dev/irdma/irdma_defs.h b/sys/dev/irdma/irdma_defs.h --- a/sys/dev/irdma/irdma_defs.h +++ b/sys/dev/irdma/irdma_defs.h @@ -138,6 +138,9 @@ #define IRDMA_AE_SOURCE_OUT_RR 0xd #define IRDMA_AE_SOURCE_OUT_RR_1111 0xf +#define IRDMA_AE_SOURCE_RSRC_EXHT_Q1 0x1 +#define IRDMA_AE_SOURCE_RSRC_EXHT_XT_RR 0x5 + #define IRDMA_TCP_STATE_NON_EXISTENT 0 #define IRDMA_TCP_STATE_CLOSED 1 #define IRDMA_TCP_STATE_LISTEN 2 @@ -193,6 +196,7 @@ #define IRDMA_CQE_QTYPE_RQ 0 #define IRDMA_CQE_QTYPE_SQ 1 +#define IRDMA_QP_SW_MIN_WQSIZE 8 /* in WRs*/ #define IRDMA_QP_WQE_MIN_SIZE 32 #define IRDMA_QP_WQE_MAX_SIZE 256 #define IRDMA_QP_WQE_MIN_QUANTA 1 @@ -1392,6 +1396,17 @@ ((_ring).tail + (_idx)) % (_ring).size \ ) +#define IRDMA_GET_RING_OFFSET(_ring, _i) \ + ( \ + ((_ring).head + (_i)) % (_ring).size \ + ) + +#define IRDMA_GET_CQ_ELEM_AT_OFFSET(_cq, _i, _cqe) \ + { \ + register __u32 offset; \ + offset = IRDMA_GET_RING_OFFSET((_cq)->cq_ring, _i); \ + (_cqe) = (_cq)->cq_base[offset].buf; \ + } #define IRDMA_GET_CURRENT_CQ_ELEM(_cq) \ ( \ (_cq)->cq_base[IRDMA_RING_CURRENT_HEAD((_cq)->cq_ring)].buf \ diff --git a/sys/dev/irdma/irdma_hw.c b/sys/dev/irdma/irdma_hw.c --- a/sys/dev/irdma/irdma_hw.c +++ b/sys/dev/irdma/irdma_hw.c @@ -119,13 +119,13 @@ if (status == -ENOENT) break; if (status) { - irdma_debug(dev, IRDMA_DEBUG_ERR, "puda status = %d\n", - status); + irdma_debug(dev, IRDMA_DEBUG_ERR, "puda status = %d\n", status); break; } if (compl_error) { irdma_debug(dev, IRDMA_DEBUG_ERR, - "puda compl_err =0x%x\n", compl_error); + "puda compl_err =0x%x\n", + compl_error); break; } } while (1); @@ -176,7 +176,6 @@ qp->sq_flush_code = info->sq; qp->rq_flush_code = info->rq; - qp_err = irdma_ae_to_qp_err_code(info->ae_id); qp->flush_code = qp_err.flush_code; @@ -220,6 +219,7 @@ struct irdma_qp *iwqp = NULL; struct irdma_cq *iwcq = NULL; struct irdma_sc_qp *qp = NULL; + struct irdma_device *iwdev = rf->iwdev; struct irdma_qp_host_ctx_info *ctx_info = NULL; unsigned long flags; @@ -235,9 +235,10 @@ break; aeqcnt++; - irdma_debug(dev, IRDMA_DEBUG_AEQ, + irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_AEQ, "ae_id = 0x%x bool qp=%d qp_id = %d tcp_state=%d iwarp_state=%d ae_src=%d\n", - info->ae_id, info->qp, info->qp_cq_id, info->tcp_state, info->iwarp_state, info->ae_src); + info->ae_id, info->qp, info->qp_cq_id, info->tcp_state, + info->iwarp_state, info->ae_src); if (info->qp) { spin_lock_irqsave(&rf->qptable_lock, flags); @@ -248,11 +249,14 @@ if (info->ae_id == IRDMA_AE_QP_SUSPEND_COMPLETE) { struct irdma_device *iwdev = rf->iwdev; + if (!iwdev->vsi.tc_change_pending) + continue; + atomic_dec(&iwdev->vsi.qp_suspend_reqs); wake_up(&iwdev->suspend_wq); continue; } - irdma_debug(dev, IRDMA_DEBUG_AEQ, + irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_AEQ, "qp_id %d is already freed\n", info->qp_cq_id); continue; @@ -329,8 +333,9 @@ case IRDMA_AE_LLP_TERMINATE_RECEIVED: irdma_terminate_received(qp, info); break; + case IRDMA_AE_LCE_CQ_CATASTROPHIC: case IRDMA_AE_CQ_OPERATION_ERROR: - irdma_dev_err(dev, + irdma_dev_err(&iwdev->ibdev, "Processing CQ[0x%x] op error, AE 0x%04X\n", info->qp_cq_id, info->ae_id); spin_lock_irqsave(&rf->cqtable_lock, flags); @@ -338,7 +343,7 @@ if (!iwcq) { spin_unlock_irqrestore(&rf->cqtable_lock, flags); - irdma_debug(dev, IRDMA_DEBUG_AEQ, + irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_AEQ, "cq_id %d is already freed\n", info->qp_cq_id); continue; @@ -358,7 +363,12 @@ break; case IRDMA_AE_RESET_NOT_SENT: case IRDMA_AE_LLP_DOUBT_REACHABILITY: + break; case IRDMA_AE_RESOURCE_EXHAUSTION: + irdma_dev_err(&iwdev->ibdev, + "Resource exhaustion reason: q1 = %d xmit or rreq = %d\n", + info->ae_src == IRDMA_AE_SOURCE_RSRC_EXHT_Q1, + info->ae_src == IRDMA_AE_SOURCE_RSRC_EXHT_XT_RR); break; case IRDMA_AE_PRIV_OPERATION_DENIED: case IRDMA_AE_RDMAP_ROE_BAD_LLP_CLOSE: @@ -381,13 +391,12 @@ case IRDMA_AE_LLP_TOO_MANY_RETRIES: case IRDMA_AE_LCE_QP_CATASTROPHIC: case IRDMA_AE_LCE_FUNCTION_CATASTROPHIC: - case IRDMA_AE_LCE_CQ_CATASTROPHIC: case IRDMA_AE_UDA_XMIT_DGRAM_TOO_LONG: default: - irdma_dev_err(dev, "abnormal ae_id = 0x%x bool qp=%d qp_id = %d, ae_source=%d\n", + irdma_dev_err(&iwdev->ibdev, "abnormal ae_id = 0x%x bool qp=%d qp_id = %d ae_source=%d\n", info->ae_id, info->qp, info->qp_cq_id, info->ae_src); if (rdma_protocol_roce(&iwqp->iwdev->ibdev, 1)) { - ctx_info->roce_info->err_rq_idx_valid = info->rq; + ctx_info->roce_info->err_rq_idx_valid = info->err_rq_idx_valid; if (info->rq) { ctx_info->roce_info->err_rq_idx = info->wqe_idx; irdma_sc_qp_setctx_roce(&iwqp->sc_qp, iwqp->host_ctx.va, @@ -397,7 +406,7 @@ irdma_cm_disconn(iwqp); break; } - ctx_info->iwarp_info->err_rq_idx_valid = info->rq; + ctx_info->iwarp_info->err_rq_idx_valid = info->err_rq_idx_valid; if (info->rq) { ctx_info->iwarp_info->err_rq_idx = info->wqe_idx; ctx_info->tcp_info_valid = false; @@ -438,9 +447,10 @@ * @t: tasklet_struct ptr */ static void -irdma_dpc(struct tasklet_struct *t) +irdma_dpc(unsigned long t) { - struct irdma_pci_f *rf = from_tasklet(rf, t, dpc_tasklet); + struct irdma_pci_f *rf = from_tasklet(rf, (struct tasklet_struct *)t, + dpc_tasklet); if (rf->msix_shared) irdma_process_ceq(rf, rf->ceqlist); @@ -453,9 +463,10 @@ * @t: tasklet_struct ptr */ static void -irdma_ceq_dpc(struct tasklet_struct *t) +irdma_ceq_dpc(unsigned long t) { - struct irdma_ceq *iwceq = from_tasklet(iwceq, t, dpc_tasklet); + struct irdma_ceq *iwceq = from_tasklet(iwceq, (struct tasklet_struct *)t, + dpc_tasklet); struct irdma_pci_f *rf = iwceq->rf; irdma_process_ceq(rf, iwceq); @@ -479,7 +490,7 @@ u32 size; if (!rf->msix_count) { - irdma_dev_err(&rf->sc_dev, "No MSI-X vectors reserved for RDMA.\n"); + irdma_dev_err(to_ibdev(&rf->sc_dev), "No MSI-X vectors reserved for RDMA.\n"); return -EINVAL; } @@ -605,8 +616,7 @@ destroy_workqueue(rf->cqp_cmpl_wq); status = irdma_sc_cqp_destroy(dev->cqp, free_hwcqp); if (status) - irdma_debug(dev, IRDMA_DEBUG_ERR, "Destroy CQP failed %d\n", - status); + irdma_debug(dev, IRDMA_DEBUG_ERR, "Destroy CQP failed %d\n", status); irdma_cleanup_pending_cqp_op(rf); irdma_free_dma_mem(dev->hw, &cqp->sq); @@ -653,8 +663,7 @@ aeq->sc_aeq.size = 0; status = irdma_cqp_aeq_cmd(dev, &aeq->sc_aeq, IRDMA_OP_AEQ_DESTROY); if (status) - irdma_debug(dev, IRDMA_DEBUG_ERR, "Destroy AEQ failed %d\n", - status); + irdma_debug(dev, IRDMA_DEBUG_ERR, "Destroy AEQ failed %d\n", status); exit: if (aeq->virtual_map) @@ -682,15 +691,15 @@ status = irdma_sc_ceq_destroy(&iwceq->sc_ceq, 0, 1); if (status) { - irdma_debug(dev, IRDMA_DEBUG_ERR, - "CEQ destroy command failed %d\n", status); + irdma_debug(dev, IRDMA_DEBUG_ERR, "CEQ destroy command failed %d\n", status); goto exit; } status = irdma_sc_cceq_destroy_done(&iwceq->sc_ceq); if (status) irdma_debug(dev, IRDMA_DEBUG_ERR, - "CEQ destroy completion failed %d\n", status); + "CEQ destroy completion failed %d\n", + status); exit: spin_lock_destroy(&iwceq->ce_lock); spin_lock_destroy(&iwceq->sc_ceq.req_cq_lock); @@ -776,8 +785,7 @@ if (!rf->reset) status = irdma_sc_ccq_destroy(dev->ccq, 0, true); if (status) - irdma_debug(dev, IRDMA_DEBUG_ERR, "CCQ destroy failed %d\n", - status); + irdma_debug(dev, IRDMA_DEBUG_ERR, "CCQ destroy failed %d\n", status); irdma_free_dma_mem(dev->hw, &ccq->mem_cq); } @@ -803,7 +811,8 @@ info.privileged = privileged; if (irdma_sc_del_hmc_obj(dev, &info, reset)) irdma_debug(dev, IRDMA_DEBUG_ERR, - "del HMC obj of type %d failed\n", obj_type); + "del HMC obj of type %d failed\n", + obj_type); } /** @@ -1005,8 +1014,7 @@ } status = irdma_sc_cqp_init(dev->cqp, &cqp_init_info); if (status) { - irdma_debug(dev, IRDMA_DEBUG_ERR, "cqp init status %d\n", - status); + irdma_debug(dev, IRDMA_DEBUG_ERR, "cqp init status %d\n", status); goto err_ctx; } @@ -1114,7 +1122,7 @@ &iwdev->mac_ip_table_idx); if (!status) { status = irdma_add_local_mac_entry(iwdev->rf, - (u8 *)IF_LLADDR(iwdev->netdev), + (const u8 *)IF_LLADDR(iwdev->netdev), (u8)iwdev->mac_ip_table_idx); if (status) irdma_del_local_mac_entry(iwdev->rf, @@ -1147,14 +1155,16 @@ msix_vec->res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE); if (!msix_vec->res) { irdma_debug(&rf->sc_dev, IRDMA_DEBUG_ERR, - "Unable to allocate bus resource int[%d]\n", rid); + "Unable to allocate bus resource int[%d]\n", + rid); return -EINVAL; } err = bus_setup_intr(dev, msix_vec->res, INTR_TYPE_NET | INTR_MPSAFE, NULL, handler, argument, &msix_vec->tag); if (err) { irdma_debug(&rf->sc_dev, IRDMA_DEBUG_ERR, - "Unable to register handler with %x status\n", err); + "Unable to register handler with %x status\n", + err); status = -EINVAL; goto fail_intr; } @@ -1185,20 +1195,24 @@ int status; if (rf->msix_shared && !ceq_id) { + snprintf(msix_vec->name, sizeof(msix_vec->name) - 1, + "irdma-%s-AEQCEQ-0", dev_name(&rf->pcidev->dev)); tasklet_setup(&rf->dpc_tasklet, irdma_dpc); status = irdma_irq_request(rf, msix_vec, irdma_irq_handler, rf); if (status) return status; - bus_describe_intr(rf->dev_ctx.dev, msix_vec->res, msix_vec->tag, "AEQCEQ"); + bus_describe_intr(rf->dev_ctx.dev, msix_vec->res, msix_vec->tag, "%s", msix_vec->name); } else { + snprintf(msix_vec->name, sizeof(msix_vec->name) - 1, + "irdma-%s-CEQ-%d", + dev_name(&rf->pcidev->dev), ceq_id); tasklet_setup(&iwceq->dpc_tasklet, irdma_ceq_dpc); status = irdma_irq_request(rf, msix_vec, irdma_ceq_handler, iwceq); if (status) return status; - bus_describe_intr(rf->dev_ctx.dev, msix_vec->res, msix_vec->tag, "CEQ"); + bus_describe_intr(rf->dev_ctx.dev, msix_vec->res, msix_vec->tag, "%s", msix_vec->name); } - msix_vec->ceq_id = ceq_id; rf->sc_dev.irq_ops->irdma_cfg_ceq(&rf->sc_dev, ceq_id, msix_vec->idx, true); @@ -1219,15 +1233,16 @@ u32 ret = 0; if (!rf->msix_shared) { + snprintf(msix_vec->name, sizeof(msix_vec->name) - 1, + "irdma-%s-AEQ", dev_name(&rf->pcidev->dev)); tasklet_setup(&rf->dpc_tasklet, irdma_dpc); ret = irdma_irq_request(rf, msix_vec, irdma_irq_handler, rf); if (ret) return ret; - bus_describe_intr(rf->dev_ctx.dev, msix_vec->res, msix_vec->tag, "irdma"); + bus_describe_intr(rf->dev_ctx.dev, msix_vec->res, msix_vec->tag, "%s", msix_vec->name); } if (ret) { - irdma_debug(&rf->sc_dev, IRDMA_DEBUG_ERR, - "aeq irq config fail\n"); + irdma_debug(&rf->sc_dev, IRDMA_DEBUG_ERR, "aeq irq config fail\n"); return -EINVAL; } @@ -1323,7 +1338,8 @@ status = irdma_create_ceq(rf, iwceq, 0, &rf->default_vsi); if (status) { irdma_debug(&rf->sc_dev, IRDMA_DEBUG_ERR, - "create ceq status = %d\n", status); + "create ceq status = %d\n", + status); goto exit; } @@ -1378,7 +1394,8 @@ status = irdma_create_ceq(rf, iwceq, ceq_id, vsi); if (status) { irdma_debug(&rf->sc_dev, IRDMA_DEBUG_ERR, - "create ceq status = %d\n", status); + "create ceq status = %d\n", + status); goto del_ceqs; } spin_lock_init(&iwceq->ce_lock); @@ -1555,8 +1572,7 @@ info.xmit_complete = irdma_free_sqbuf; status = irdma_puda_create_rsrc(&iwdev->vsi, &info); if (status) - irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_ERR, - "ilq create fail\n"); + irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_ERR, "ilq create fail\n"); return status; } @@ -1585,8 +1601,7 @@ info.tx_buf_cnt = 4096; status = irdma_puda_create_rsrc(&iwdev->vsi, &info); if (status) - irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_ERR, - "ieq create fail\n"); + irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_ERR, "ieq create fail\n"); return status; } @@ -1709,6 +1724,10 @@ info.bar0 = rf->hw.hw_addr; info.hmc_fn_id = rf->peer_info->pf_id; + /* + * the debug_mask is already assigned at this point through sysctl and so the value shouldn't be overwritten + */ + info.debug_mask = rf->sc_dev.debug_mask; info.hw = &rf->hw; status = irdma_sc_dev_init(&rf->sc_dev, &info); if (status) @@ -1733,8 +1752,7 @@ irdma_rt_deinit_hw(struct irdma_device *iwdev) { struct irdma_sc_qp qp = {{0}}; - irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_INIT, "state = %d\n", - iwdev->init_state); + irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_INIT, "state = %d\n", iwdev->init_state); switch (iwdev->init_state) { case IP_ADDR_REGISTERED: @@ -1765,8 +1783,7 @@ iwdev->rf->reset); break; default: - irdma_dev_warn(&iwdev->rf->sc_dev, "bad init_state = %d\n", - iwdev->init_state); + irdma_dev_warn(&iwdev->ibdev, "bad init_state = %d\n", iwdev->init_state); break; } @@ -1821,14 +1838,14 @@ static void irdma_get_used_rsrc(struct irdma_device *iwdev) { - iwdev->rf->used_pds = find_next_zero_bit(iwdev->rf->allocated_pds, - iwdev->rf->max_pd, 0); - iwdev->rf->used_qps = find_next_zero_bit(iwdev->rf->allocated_qps, - iwdev->rf->max_qp, 0); - iwdev->rf->used_cqs = find_next_zero_bit(iwdev->rf->allocated_cqs, - iwdev->rf->max_cq, 0); - iwdev->rf->used_mrs = find_next_zero_bit(iwdev->rf->allocated_mrs, - iwdev->rf->max_mr, 0); + iwdev->rf->used_pds = find_first_zero_bit(iwdev->rf->allocated_pds, + iwdev->rf->max_pd); + iwdev->rf->used_qps = find_first_zero_bit(iwdev->rf->allocated_qps, + iwdev->rf->max_qp); + iwdev->rf->used_cqs = find_first_zero_bit(iwdev->rf->allocated_cqs, + iwdev->rf->max_cq); + iwdev->rf->used_mrs = find_first_zero_bit(iwdev->rf->allocated_mrs, + iwdev->rf->max_mr); } void @@ -1864,7 +1881,7 @@ break; case INVALID_STATE: default: - irdma_pr_warn("bad init_state = %d\n", rf->init_state); + irdma_dev_warn(&rf->iwdev->ibdev, "bad init_state = %d\n", rf->init_state); break; } } @@ -1978,8 +1995,8 @@ return 0; } while (0); - irdma_dev_err(idev_to_dev(dev), "HW runtime init FAIL status = %d last cmpl = %d\n", - status, iwdev->init_state); + dev_err(&rf->pcidev->dev, "HW runtime init FAIL status = %d last cmpl = %d\n", + status, iwdev->init_state); irdma_rt_deinit_hw(iwdev); return status; @@ -2197,10 +2214,8 @@ if (info.error && irdma_cqp_crit_err(dev, cqp_request->info.cqp_cmd, info.maj_err_code, info.min_err_code)) - irdma_dev_err(dev, "cqp opcode = 0x%x maj_err_code = 0x%x min_err_code = 0x%x\n", - info.op_code, info.maj_err_code, - info.min_err_code); - + irdma_dev_err(&rf->iwdev->ibdev, "cqp opcode = 0x%x maj_err_code = 0x%x min_err_code = 0x%x\n", + info.op_code, info.maj_err_code, info.min_err_code); if (cqp_request) { cqp_request->compl_info.maj_err_code = info.maj_err_code; cqp_request->compl_info.min_err_code = info.min_err_code; @@ -2323,7 +2338,7 @@ * @idx: the index of the mac ip address to add */ int -irdma_add_local_mac_entry(struct irdma_pci_f *rf, u8 *mac_addr, u16 idx) +irdma_add_local_mac_entry(struct irdma_pci_f *rf, const u8 *mac_addr, u16 idx) { struct irdma_local_mac_entry_info *info; struct irdma_cqp *iwcqp = &rf->cqp; @@ -2414,7 +2429,7 @@ cqp_info->post_sq = 1; cqp_info->in.u.manage_apbvt_entry.cqp = &iwdev->rf->cqp.sc_cqp; cqp_info->in.u.manage_apbvt_entry.scratch = (uintptr_t)cqp_request; - irdma_debug(iwdev_to_idev(iwdev), IRDMA_DEBUG_DEV, "%s: port=0x%04x\n", + irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_DEV, "%s: port=0x%04x\n", (!add_port) ? "DELETE" : "ADD", accel_local_port); status = irdma_handle_cqp_op(iwdev->rf, cqp_request); @@ -2566,7 +2581,6 @@ bool wait) { struct irdma_qhash_table_info *info; - struct irdma_sc_dev *dev = &iwdev->rf->sc_dev; struct irdma_cqp *iwcqp = &iwdev->rf->cqp; struct irdma_cqp_request *cqp_request; struct cqp_cmds_info *cqp_info; @@ -2616,17 +2630,21 @@ atomic_inc(&cm_node->refcnt); } if (info->ipv4_valid) - irdma_debug(dev, IRDMA_DEBUG_CM, + irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_CM, "%s caller: %pS loc_port=0x%04x rem_port=0x%04x loc_addr=%pI4 rem_addr=%pI4 mac=%pM, vlan_id=%d cm_node=%p\n", - (!mtype) ? "DELETE" : "ADD", __builtin_return_address(0), - info->dest_port, info->src_port, info->dest_ip, info->src_ip, - info->mac_addr, cminfo->vlan_id, cmnode ? cmnode : NULL); + (!mtype) ? "DELETE" : "ADD", + __builtin_return_address(0), info->dest_port, + info->src_port, info->dest_ip, info->src_ip, + info->mac_addr, cminfo->vlan_id, + cmnode ? cmnode : NULL); else - irdma_debug(dev, IRDMA_DEBUG_CM, + irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_CM, "%s caller: %pS loc_port=0x%04x rem_port=0x%04x loc_addr=%pI6 rem_addr=%pI6 mac=%pM, vlan_id=%d cm_node=%p\n", - (!mtype) ? "DELETE" : "ADD", __builtin_return_address(0), - info->dest_port, info->src_port, info->dest_ip, info->src_ip, - info->mac_addr, cminfo->vlan_id, cmnode ? cmnode : NULL); + (!mtype) ? "DELETE" : "ADD", + __builtin_return_address(0), info->dest_port, + info->src_port, info->dest_ip, info->src_ip, + info->mac_addr, cminfo->vlan_id, + cmnode ? cmnode : NULL); cqp_info->in.u.manage_qhash_table_entry.cqp = &iwdev->rf->cqp.sc_cqp; cqp_info->in.u.manage_qhash_table_entry.scratch = (uintptr_t)cqp_request; @@ -2694,8 +2712,9 @@ qp->qp_uk.sq_flush_complete = true; } } + irdma_debug(&rf->sc_dev, IRDMA_DEBUG_VERBS, - "qp_id=%d qp_type=%d qpstate=%d ibqpstate=%d last_aeq=%d hw_iw_state=%d maj_err_code=%d min_err_code=%d\n", + "qp_id=%d qp_type=%d qpstate=%d ibqpstate=%d last_aeq=%d hw_iw_state=%d maj_err_code=%d min_err_code=%d\n", iwqp->ibqp.qp_num, rf->protocol_used, iwqp->iwarp_state, iwqp->ibqp_state, iwqp->last_aeq, iwqp->hw_iwarp_state, cqp_request->compl_info.maj_err_code, cqp_request->compl_info.min_err_code); @@ -2770,7 +2789,7 @@ info.rq_minor_code = flush_code; } if (irdma_upload_context && irdma_upload_qp_context(iwqp, 0, 1)) - irdma_print("failed to upload QP context\n"); + irdma_dev_warn(&iwqp->iwdev->ibdev, "failed to upload QP context\n"); if (!iwqp->user_mode) irdma_sched_qp_flush_work(iwqp); } diff --git a/sys/dev/irdma/irdma_kcompat.c b/sys/dev/irdma/irdma_kcompat.c --- a/sys/dev/irdma/irdma_kcompat.c +++ b/sys/dev/irdma/irdma_kcompat.c @@ -192,7 +192,7 @@ struct irdma_alloc_ucontext_req req = {0}; struct irdma_alloc_ucontext_resp uresp = {0}; struct irdma_ucontext *ucontext = to_ucontext(uctx); - struct irdma_uk_attrs *uk_attrs; + struct irdma_uk_attrs *uk_attrs = &iwdev->rf->sc_dev.hw_attrs.uk_attrs; if (udata->inlen < IRDMA_ALLOC_UCTX_MIN_REQ_LEN || udata->outlen < IRDMA_ALLOC_UCTX_MIN_RESP_LEN) @@ -207,7 +207,9 @@ ucontext->iwdev = iwdev; ucontext->abi_ver = req.userspace_ver; - uk_attrs = &iwdev->rf->sc_dev.hw_attrs.uk_attrs; + if (req.comp_mask & IRDMA_ALLOC_UCTX_USE_RAW_ATTR) + ucontext->use_raw_attrs = true; + /* GEN_1 support for libi40iw */ if (udata->outlen == IRDMA_ALLOC_UCTX_MIN_RESP_LEN) { if (uk_attrs->hw_rev != IRDMA_GEN_1) @@ -234,6 +236,7 @@ uresp.max_hw_cq_size = uk_attrs->max_hw_cq_size; uresp.min_hw_cq_size = uk_attrs->min_hw_cq_size; uresp.hw_rev = uk_attrs->hw_rev; + uresp.comp_mask |= IRDMA_ALLOC_UCTX_USE_RAW_ATTR; bar_off = (uintptr_t)iwdev->rf->sc_dev.hw_regs[IRDMA_DB_ADDR_OFFSET]; @@ -262,7 +265,7 @@ return 0; ver_error: - irdma_dev_err(&iwdev->rf->sc_dev, + irdma_dev_err(&iwdev->ibdev, "Invalid userspace driver version detected. Detected version %d, should be %d\n", req.userspace_ver, IRDMA_ABI_VER); return -EINVAL; @@ -285,7 +288,7 @@ struct irdma_alloc_ucontext_req req = {0}; struct irdma_alloc_ucontext_resp uresp = {0}; struct irdma_ucontext *ucontext; - struct irdma_uk_attrs *uk_attrs; + struct irdma_uk_attrs *uk_attrs = &iwdev->rf->sc_dev.hw_attrs.uk_attrs; if (udata->inlen < IRDMA_ALLOC_UCTX_MIN_REQ_LEN || udata->outlen < IRDMA_ALLOC_UCTX_MIN_RESP_LEN) @@ -304,7 +307,9 @@ ucontext->iwdev = iwdev; ucontext->abi_ver = req.userspace_ver; - uk_attrs = &iwdev->rf->sc_dev.hw_attrs.uk_attrs; + if (req.comp_mask & IRDMA_ALLOC_UCTX_USE_RAW_ATTR) + ucontext->use_raw_attrs = true; + /* GEN_1 legacy support with libi40iw */ if (udata->outlen == IRDMA_ALLOC_UCTX_MIN_RESP_LEN) { if (uk_attrs->hw_rev != IRDMA_GEN_1) { @@ -335,6 +340,7 @@ uresp.max_hw_cq_size = uk_attrs->max_hw_cq_size; uresp.min_hw_cq_size = uk_attrs->min_hw_cq_size; uresp.hw_rev = uk_attrs->hw_rev; + uresp.comp_mask |= IRDMA_ALLOC_UCTX_USE_RAW_ATTR; bar_off = (uintptr_t)iwdev->rf->sc_dev.hw_regs[IRDMA_DB_ADDR_OFFSET]; @@ -369,9 +375,9 @@ return &ucontext->ibucontext; ver_error: - ibdev_err(&iwdev->ibdev, - "Invalid userspace driver version detected. Detected version %d, should be %d\n", - req.userspace_ver, IRDMA_ABI_VER); + irdma_dev_err(&iwdev->ibdev, + "Invalid userspace driver version detected. Detected version %d, should be %d\n", + req.userspace_ver, IRDMA_ABI_VER); return ERR_PTR(-EINVAL); } #endif @@ -440,8 +446,7 @@ sc_pd = &iwpd->sc_pd; if (udata) { struct irdma_ucontext *ucontext = - rdma_udata_to_drv_context(udata, struct irdma_ucontext, - ibucontext); + rdma_udata_to_drv_context(udata, struct irdma_ucontext, ibucontext); irdma_sc_pd_init(dev, sc_pd, pd_id, ucontext->abi_ver); uresp.pd_id = pd_id; @@ -454,6 +459,9 @@ irdma_sc_pd_init(dev, sc_pd, pd_id, IRDMA_ABI_VER); } + spin_lock_init(&iwpd->udqp_list_lock); + INIT_LIST_HEAD(&iwpd->udqp_list); + return 0; error: @@ -509,6 +517,9 @@ irdma_sc_pd_init(dev, sc_pd, pd_id, IRDMA_ABI_VER); } + spin_lock_init(&iwpd->udqp_list_lock); + INIT_LIST_HEAD(&iwpd->udqp_list); + return &iwpd->ibpd; error: @@ -547,6 +558,54 @@ } #endif +/** + * irdma_find_qp_update_qs - update QS handle for UD QPs + * @rf: RDMA PCI function + * @pd: protection domain object + * @user_pri: selected user priority + */ +static void +irdma_find_qp_update_qs(struct irdma_pci_f *rf, + struct irdma_pd *pd, u8 user_pri) +{ + struct irdma_qp *iwqp; + struct list_head *tmp_node, *list_node; + struct irdma_udqs_work *work; + unsigned long flags; + bool qs_change; + + spin_lock_irqsave(&pd->udqp_list_lock, flags); + list_for_each_safe(list_node, tmp_node, &pd->udqp_list) { + qs_change = true; + iwqp = list_entry(list_node, struct irdma_qp, ud_list_elem); + irdma_qp_add_ref(&iwqp->ibqp); + /* check if qs_handle needs to be changed */ + if (iwqp->sc_qp.qs_handle == iwqp->sc_qp.vsi->qos[user_pri].qs_handle) { + if (iwqp->ctx_info.user_pri == user_pri) { + /* qs_handle and user_pri don't change */ + irdma_qp_rem_ref(&iwqp->ibqp); + continue; + } + qs_change = false; + } + /* perform qp qos change */ + work = kzalloc(sizeof(*work), GFP_ATOMIC); + if (!work) { + irdma_qp_rem_ref(&iwqp->ibqp); + spin_unlock_irqrestore(&pd->udqp_list_lock, flags); + return; + } + work->iwqp = iwqp; + work->user_prio = user_pri; + work->qs_change = qs_change; + INIT_WORK(&work->work, irdma_udqp_qs_worker); + if (qs_change) + irdma_cqp_qp_suspend_resume(&iwqp->sc_qp, IRDMA_OP_SUSPEND); + queue_work(rf->iwdev->cleanup_wq, &work->work); + } + spin_unlock_irqrestore(&pd->udqp_list_lock, flags); +} + static void irdma_fill_ah_info(struct vnet *vnet, struct irdma_ah_info *ah_info, const struct ib_gid_attr *sgid_attr, @@ -559,12 +618,10 @@ ntohl(((struct sockaddr_in *)dgid_addr)->sin_addr.s_addr); ah_info->src_ip_addr[0] = ntohl(((struct sockaddr_in *)sgid_addr)->sin_addr.s_addr); -#ifdef VIMAGE CURVNET_SET_QUIET(vnet); ah_info->do_lpbk = irdma_ipv4_is_lpb(ah_info->src_ip_addr[0], ah_info->dest_ip_addr[0]); CURVNET_RESTORE(); -#endif if (ipv4_is_multicast(((struct sockaddr_in *)dgid_addr)->sin_addr.s_addr)) { irdma_mcast_mac_v4(ah_info->dest_ip_addr, dmac); } @@ -581,12 +638,19 @@ } } +static inline u8 irdma_get_vlan_ndev_prio(struct ifnet *ndev, u8 prio){ + return prio; +} + static int irdma_create_ah_vlan_tag(struct irdma_device *iwdev, + struct irdma_pd *pd, struct irdma_ah_info *ah_info, const struct ib_gid_attr *sgid_attr, u8 *dmac) { + u16 vlan_prio; + if (sgid_attr->ndev && is_vlan_dev(sgid_attr->ndev)) ah_info->vlan_tag = vlan_dev_vlan_id(sgid_attr->ndev); else @@ -601,9 +665,12 @@ ah_info->vlan_tag = 0; if (ah_info->vlan_tag < VLAN_N_VID) { + struct ifnet *ndev = sgid_attr->ndev; + ah_info->insert_vlan_tag = true; - ah_info->vlan_tag |= - (u16)rt_tos2priority(ah_info->tc_tos) << VLAN_PRIO_SHIFT; + vlan_prio = (u16)irdma_get_vlan_ndev_prio(ndev, rt_tos2priority(ah_info->tc_tos)); + ah_info->vlan_tag |= vlan_prio << VLAN_PRIO_SHIFT; + irdma_find_qp_update_qs(iwdev->rf, pd, vlan_prio); } if (iwdev->roce_dcqcn_en) { ah_info->tc_tos &= ~ECN_CODE_PT_MASK; @@ -689,7 +756,7 @@ attr->grh.sgid_index, &sgid, &sgid_attr); rcu_read_unlock(); if (err) { - irdma_debug(iwdev_to_idev(iwdev), IRDMA_DEBUG_VERBS, + irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_VERBS, "GID lookup at idx=%d with port=%d failed\n", attr->grh.sgid_index, attr->port_num); err = -EINVAL; @@ -723,22 +790,20 @@ irdma_fill_ah_info(iwdev->netdev->if_vnet, ah_info, &sgid_attr, &sgid_addr.saddr, &dgid_addr.saddr, dmac, ah->av.net_type); - err = irdma_create_ah_vlan_tag(iwdev, ah_info, &sgid_attr, dmac); + err = irdma_create_ah_vlan_tag(iwdev, pd, ah_info, &sgid_attr, dmac); if (err) goto err_gid_l2; err = irdma_ah_cqp_op(iwdev->rf, sc_ah, IRDMA_OP_AH_CREATE, sleep, irdma_gsi_ud_qp_ah_cb, sc_ah); if (err) { - irdma_debug(iwdev_to_idev(iwdev), IRDMA_DEBUG_VERBS, - "CQP-OP Create AH fail"); + irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_DEV, "CQP-OP Create AH fail"); goto err_gid_l2; } err = irdma_create_ah_wait(rf, sc_ah, sleep); if (err) { - irdma_debug(iwdev_to_idev(iwdev), IRDMA_DEBUG_VERBS, - "CQP create AH timed out"); + irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_DEV, "CQP create AH timed out"); goto err_gid_l2; } @@ -859,7 +924,7 @@ attr->grh.sgid_index, &sgid, &sgid_attr); rcu_read_unlock(); if (err) { - irdma_debug(iwdev_to_idev(iwdev), IRDMA_DEBUG_VERBS, + irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_VERBS, "GID lookup at idx=%d with port=%d failed\n", attr->grh.sgid_index, attr->port_num); err = -EINVAL; @@ -895,22 +960,20 @@ irdma_fill_ah_info(iwdev->netdev->if_vnet, ah_info, &sgid_attr, &sgid_addr.saddr, &dgid_addr.saddr, dmac, ah->av.net_type); - err = irdma_create_ah_vlan_tag(iwdev, ah_info, &sgid_attr, dmac); + err = irdma_create_ah_vlan_tag(iwdev, pd, ah_info, &sgid_attr, dmac); if (err) goto err_gid_l2; err = irdma_ah_cqp_op(iwdev->rf, sc_ah, IRDMA_OP_AH_CREATE, sleep, irdma_gsi_ud_qp_ah_cb, sc_ah); if (err) { - irdma_debug(iwdev_to_idev(iwdev), IRDMA_DEBUG_VERBS, - "CQP-OP Create AH fail"); + irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_VERBS, "CQP-OP Create AH fail"); goto err_gid_l2; } err = irdma_create_ah_wait(rf, sc_ah, sleep); if (err) { - irdma_debug(iwdev_to_idev(iwdev), IRDMA_DEBUG_VERBS, - "CQP create AH timed out"); + irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_DEV, "CQP create AH timed out"); goto err_gid_l2; } @@ -990,6 +1053,7 @@ struct irdma_uk_attrs *uk_attrs = &dev->hw_attrs.uk_attrs; struct irdma_qp_init_info init_info = {{0}}; struct irdma_qp_host_ctx_info *ctx_info; + unsigned long flags; err_code = irdma_validate_qp_attrs(init_attr, iwdev); if (err_code) @@ -1074,8 +1138,7 @@ } if (err_code) { - irdma_debug(iwdev_to_idev(iwdev), IRDMA_DEBUG_VERBS, - "setup qp failed\n"); + irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_VERBS, "setup qp failed\n"); goto error; } @@ -1098,8 +1161,7 @@ ret = irdma_sc_qp_init(qp, &init_info); if (ret) { err_code = -EPROTO; - irdma_debug(iwdev_to_idev(iwdev), IRDMA_DEBUG_VERBS, - "qp_init fail\n"); + irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_VERBS, "qp_init fail\n"); goto error; } @@ -1130,6 +1192,10 @@ } irdma_qp_add_qos(&iwqp->sc_qp); + spin_lock_irqsave(&iwpd->udqp_list_lock, flags); + if (iwqp->sc_qp.qp_uk.qp_type == IRDMA_QP_TYPE_ROCE_UD) + list_add_tail(&iwqp->ud_list_elem, &iwpd->udqp_list); + spin_unlock_irqrestore(&iwpd->udqp_list_lock, flags); } if (udata) { @@ -1149,8 +1215,7 @@ err_code = ib_copy_to_udata(udata, &uresp, min(sizeof(uresp), udata->outlen)); if (err_code) { - irdma_debug(iwdev_to_idev(iwdev), IRDMA_DEBUG_VERBS, - "copy_to_udata failed\n"); + irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_VERBS, "copy_to_udata failed\n"); kc_irdma_destroy_qp(&iwqp->ibqp, udata); return ERR_PTR(err_code); } @@ -1180,19 +1245,25 @@ { struct irdma_qp *iwqp = to_iwqp(ibqp); struct irdma_device *iwdev = iwqp->iwdev; + unsigned long flags; if (iwqp->sc_qp.qp_uk.destroy_pending) goto free_rsrc; iwqp->sc_qp.qp_uk.destroy_pending = true; + + spin_lock_irqsave(&iwqp->iwpd->udqp_list_lock, flags); + if (iwqp->sc_qp.qp_uk.qp_type == IRDMA_QP_TYPE_ROCE_UD) + list_del(&iwqp->ud_list_elem); + spin_unlock_irqrestore(&iwqp->iwpd->udqp_list_lock, flags); + if (iwqp->iwarp_state == IRDMA_QP_STATE_RTS) irdma_modify_qp_to_err(&iwqp->sc_qp); irdma_qp_rem_ref(&iwqp->ibqp); wait_for_completion(&iwqp->free_qp); irdma_free_lsmm_rsrc(iwqp); - if (!iwdev->rf->reset && - irdma_cqp_qp_destroy_cmd(&iwdev->rf->sc_dev, &iwqp->sc_qp)) - return -ENOTRECOVERABLE; + if (!iwdev->rf->reset && irdma_cqp_qp_destroy_cmd(&iwdev->rf->sc_dev, &iwqp->sc_qp)) + return (iwdev->rf->rdma_ver <= IRDMA_GEN_2 && !iwqp->user_mode) ? 0 : -ENOTRECOVERABLE; free_rsrc: if (!iwqp->user_mode) { if (iwqp->iwscq) { @@ -1292,6 +1363,7 @@ cqe_64byte_ena = (dev->hw_attrs.uk_attrs.feature_flags & IRDMA_FEATURE_64_BYTE_CQE) ? true : false; ukinfo->avoid_mem_cflct = cqe_64byte_ena; iwcq->ibcq.cqe = info.cq_uk_init_info.cq_size; + atomic_set(&iwcq->armed, 0); if (attr->comp_vector < rf->ceqs_count) info.ceq_id = attr->comp_vector; info.ceq_id_valid = true; @@ -1403,8 +1475,7 @@ info.shadow_read_threshold = min(info.cq_uk_init_info.cq_size / 2, (u32)IRDMA_MAX_CQ_READ_THRESH); if (irdma_sc_cq_init(cq, &info)) { - irdma_debug(iwdev_to_idev(iwdev), IRDMA_DEBUG_VERBS, - "init cq fail\n"); + irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_VERBS, "init cq fail\n"); err_code = -EPROTO; goto cq_free_rsrc; } @@ -1434,8 +1505,7 @@ resp.cq_size = info.cq_uk_init_info.cq_size; if (ib_copy_to_udata(udata, &resp, min(sizeof(resp), udata->outlen))) { - irdma_debug(iwdev_to_idev(iwdev), IRDMA_DEBUG_VERBS, - "copy to user data\n"); + irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_VERBS, "copy to user data\n"); err_code = -EPROTO; goto cq_destroy; } @@ -1666,6 +1736,9 @@ ether_addr_copy(iwqp->ctx_info.roce_info->mac_addr, IF_LLADDR(sgid_attr.ndev)); } + av->net_type = kc_rdma_gid_attr_network_type(sgid_attr, + sgid_attr.gid_type, + &sgid); rdma_gid2ip((struct sockaddr *)&av->sgid_addr, &sgid); dev_put(sgid_attr.ndev); iwqp->sc_qp.user_pri = iwqp->ctx_info.user_pri; @@ -2047,37 +2120,6 @@ return 0; } -/** - * irdma_get_eth_speed_and_width - Get IB port speed and width from netdev speed - * @link_speed: netdev phy link speed - * @active_speed: IB port speed - * @active_width: IB port width - */ -void -irdma_get_eth_speed_and_width(u32 link_speed, u8 *active_speed, - u8 *active_width) -{ - if (link_speed <= SPEED_1000) { - *active_width = IB_WIDTH_1X; - *active_speed = IB_SPEED_SDR; - } else if (link_speed <= SPEED_10000) { - *active_width = IB_WIDTH_1X; - *active_speed = IB_SPEED_FDR10; - } else if (link_speed <= SPEED_20000) { - *active_width = IB_WIDTH_4X; - *active_speed = IB_SPEED_DDR; - } else if (link_speed <= SPEED_25000) { - *active_width = IB_WIDTH_1X; - *active_speed = IB_SPEED_EDR; - } else if (link_speed <= SPEED_40000) { - *active_width = IB_WIDTH_4X; - *active_speed = IB_SPEED_FDR10; - } else { - *active_width = IB_WIDTH_4X; - *active_speed = IB_SPEED_EDR; - } -} - /** * irdma_query_port - get port attributes * @ibdev: device pointer from stack @@ -2106,8 +2148,7 @@ props->state = IB_PORT_DOWN; props->phys_state = IB_PORT_PHYS_STATE_DISABLED; } - irdma_get_eth_speed_and_width(SPEED_100000, &props->active_speed, - &props->active_width); + ib_get_eth_speed(ibdev, port, &props->active_speed, &props->active_width); if (rdma_protocol_roce(ibdev, 1)) { props->gid_tbl_len = 32; @@ -2304,3 +2345,37 @@ if (iwdev->rf->rdma_ver >= IRDMA_GEN_2) iwdev->ibdev.uverbs_ex_cmd_mask |= BIT_ULL(IB_USER_VERBS_EX_CMD_CREATE_CQ); } + +int +ib_get_eth_speed(struct ib_device *ibdev, u32 port_num, u8 *speed, u8 *width) +{ + struct ifnet *netdev = ibdev->get_netdev(ibdev, port_num); + u32 netdev_speed; + + if (!netdev) + return -ENODEV; + + netdev_speed = netdev->if_baudrate; + dev_put(netdev); + if (netdev_speed <= SPEED_1000) { + *width = IB_WIDTH_1X; + *speed = IB_SPEED_SDR; + } else if (netdev_speed <= SPEED_10000) { + *width = IB_WIDTH_1X; + *speed = IB_SPEED_FDR10; + } else if (netdev_speed <= SPEED_20000) { + *width = IB_WIDTH_4X; + *speed = IB_SPEED_DDR; + } else if (netdev_speed <= SPEED_25000) { + *width = IB_WIDTH_1X; + *speed = IB_SPEED_EDR; + } else if (netdev_speed <= SPEED_40000) { + *width = IB_WIDTH_4X; + *speed = IB_SPEED_FDR10; + } else { + *width = IB_WIDTH_4X; + *speed = IB_SPEED_EDR; + } + + return 0; +} diff --git a/sys/dev/irdma/irdma_main.h b/sys/dev/irdma/irdma_main.h --- a/sys/dev/irdma/irdma_main.h +++ b/sys/dev/irdma/irdma_main.h @@ -133,6 +133,8 @@ #define IRDMA_REFLUSH BIT(2) #define IRDMA_FLUSH_WAIT BIT(3) +#define IRDMA_IRQ_NAME_STR_LEN 64 + enum init_completion_state { INVALID_STATE = 0, INITIAL_STATE, @@ -230,6 +232,7 @@ u32 irq; u32 cpu_affinity; u32 ceq_id; + char name[IRDMA_IRQ_NAME_STR_LEN]; struct resource *res; void *tag; }; @@ -376,6 +379,7 @@ u16 vsi_num; u8 rcv_wscale; u8 iw_status; + u8 roce_rtomin; u8 rd_fence_rate; bool override_rcv_wnd:1; bool override_cwnd:1; @@ -529,7 +533,7 @@ void irdma_put_cqp_request(struct irdma_cqp *cqp, struct irdma_cqp_request *cqp_request); int irdma_alloc_local_mac_entry(struct irdma_pci_f *rf, u16 *mac_tbl_idx); -int irdma_add_local_mac_entry(struct irdma_pci_f *rf, u8 *mac_addr, u16 idx); +int irdma_add_local_mac_entry(struct irdma_pci_f *rf, const u8 *mac_addr, u16 idx); void irdma_del_local_mac_entry(struct irdma_pci_f *rf, u16 idx); u32 irdma_initialize_hw_rsrc(struct irdma_pci_f *rf); @@ -589,6 +593,7 @@ void (*callback_fcn)(struct irdma_cqp_request *cqp_request), void *cb_param); void irdma_gsi_ud_qp_ah_cb(struct irdma_cqp_request *cqp_request); +void irdma_udqp_qs_worker(struct work_struct *work); bool irdma_cq_empty(struct irdma_cq *iwcq); int irdma_netdevice_event(struct notifier_block *notifier, unsigned long event, void *ptr); @@ -599,4 +604,5 @@ void irdma_add_handler(struct irdma_handler *hdl); void irdma_del_handler(struct irdma_handler *hdl); void cqp_compl_worker(struct work_struct *work); +void irdma_cleanup_dead_qps(struct irdma_sc_vsi *vsi); #endif /* IRDMA_MAIN_H */ diff --git a/sys/dev/irdma/irdma_pble.h b/sys/dev/irdma/irdma_pble.h --- a/sys/dev/irdma/irdma_pble.h +++ b/sys/dev/irdma/irdma_pble.h @@ -1,7 +1,7 @@ /*- * SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB * - * Copyright (c) 2015 - 2021 Intel Corporation + * Copyright (c) 2015 - 2022 Intel Corporation * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU @@ -148,7 +148,7 @@ struct irdma_pble_alloc *palloc); int irdma_get_pble(struct irdma_hmc_pble_rsrc *pble_rsrc, struct irdma_pble_alloc *palloc, u32 pble_cnt, - bool level1_only); + u8 lvl); int irdma_prm_add_pble_mem(struct irdma_pble_prm *pprm, struct irdma_chunk *pchunk); int irdma_prm_get_pbles(struct irdma_pble_prm *pprm, diff --git a/sys/dev/irdma/irdma_pble.c b/sys/dev/irdma/irdma_pble.c --- a/sys/dev/irdma/irdma_pble.c +++ b/sys/dev/irdma/irdma_pble.c @@ -1,7 +1,7 @@ /*- * SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB * - * Copyright (c) 2015 - 2021 Intel Corporation + * Copyright (c) 2015 - 2022 Intel Corporation * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU @@ -468,16 +468,16 @@ * get_lvl1_lvl2_pble - calls get_lvl1 and get_lvl2 pble routine * @pble_rsrc: pble resources * @palloc: contains all inforamtion regarding pble (idx + pble addr) - * @level1_only: flag for a level 1 PBLE + * @lvl: Bitmask for requested pble level */ static int get_lvl1_lvl2_pble(struct irdma_hmc_pble_rsrc *pble_rsrc, - struct irdma_pble_alloc *palloc, bool level1_only) + struct irdma_pble_alloc *palloc, u8 lvl) { int status = 0; status = get_lvl1_pble(pble_rsrc, palloc); - if (!status || level1_only || palloc->total_cnt <= PBLE_PER_PAGE) + if (!status || lvl == PBLE_LEVEL_1 || palloc->total_cnt <= PBLE_PER_PAGE) return status; status = get_lvl2_pble(pble_rsrc, palloc); @@ -490,12 +490,12 @@ * @pble_rsrc: pble resources * @palloc: contains all inforamtion regarding pble (idx + pble addr) * @pble_cnt: #of pbles requested - * @level1_only: true if only pble level 1 to acquire + * @lvl: requested pble level mask */ int irdma_get_pble(struct irdma_hmc_pble_rsrc *pble_rsrc, struct irdma_pble_alloc *palloc, u32 pble_cnt, - bool level1_only) + u8 lvl) { int status = 0; int max_sds = 0; @@ -509,7 +509,7 @@ /* * check first to see if we can get pble's without acquiring additional sd's */ - status = get_lvl1_lvl2_pble(pble_rsrc, palloc, level1_only); + status = get_lvl1_lvl2_pble(pble_rsrc, palloc, lvl); if (!status) goto exit; @@ -519,9 +519,9 @@ if (status) break; - status = get_lvl1_lvl2_pble(pble_rsrc, palloc, level1_only); + status = get_lvl1_lvl2_pble(pble_rsrc, palloc, lvl); /* if level1_only, only go through it once */ - if (!status || level1_only) + if (!status || lvl == PBLE_LEVEL_1) break; } diff --git a/sys/dev/irdma/irdma_protos.h b/sys/dev/irdma/irdma_protos.h --- a/sys/dev/irdma/irdma_protos.h +++ b/sys/dev/irdma/irdma_protos.h @@ -69,8 +69,6 @@ u8 op); int irdma_cqp_stats_inst_cmd(struct irdma_sc_vsi *vsi, u8 cmd, struct irdma_stats_inst_info *stats_info); -u16 irdma_alloc_ws_node_id(struct irdma_sc_dev *dev); -void irdma_free_ws_node_id(struct irdma_sc_dev *dev, u16 node_id); void irdma_update_stats(struct irdma_dev_hw_stats *hw_stats, struct irdma_gather_stats *gather_stats, struct irdma_gather_stats *last_gather_stats, diff --git a/sys/dev/irdma/irdma_puda.h b/sys/dev/irdma/irdma_puda.h --- a/sys/dev/irdma/irdma_puda.h +++ b/sys/dev/irdma/irdma_puda.h @@ -1,7 +1,7 @@ /*- * SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB * - * Copyright (c) 2015 - 2021 Intel Corporation + * Copyright (c) 2015 - 2022 Intel Corporation * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU @@ -126,7 +126,7 @@ u32 tx_buf_cnt; /* total bufs allocated will be rq_size + tx_buf_cnt */ u16 buf_size; u16 mss; /* FIXME: Windows driver still using this */ - u8 stats_idx; + u16 stats_idx; bool stats_idx_valid:1; int abi_ver; }; @@ -177,7 +177,7 @@ u64 pmode_count; u64 partials_handled; u16 mss; /* FIXME: Windows driver still using this */ - u8 stats_idx; + u16 stats_idx; bool check_crc:1; bool stats_idx_valid:1; }; diff --git a/sys/dev/irdma/irdma_puda.c b/sys/dev/irdma/irdma_puda.c --- a/sys/dev/irdma/irdma_puda.c +++ b/sys/dev/irdma/irdma_puda.c @@ -230,7 +230,6 @@ */ static __le64 * irdma_puda_get_next_send_wqe(struct irdma_qp_uk *qp, u32 *wqe_idx){ - __le64 *wqe = NULL; int ret_code = 0; *wqe_idx = IRDMA_RING_CURRENT_HEAD(qp->sq_ring); @@ -238,11 +237,9 @@ qp->swqe_polarity = !qp->swqe_polarity; IRDMA_RING_MOVE_HEAD(qp->sq_ring, ret_code); if (ret_code) - return wqe; - - wqe = qp->sq_base[*wqe_idx].elem; + return NULL; - return wqe; + return qp->sq_base[*wqe_idx].elem; } /** @@ -1516,8 +1513,7 @@ error: while (!list_empty(&pbufl)) { buf = (struct irdma_puda_buf *)(&pbufl)->prev; - list_del(&buf->list); - list_add(&buf->list, rxlist); + list_move(&buf->list, rxlist); } if (txbuf) irdma_puda_ret_bufpool(ieq, txbuf); diff --git a/sys/dev/irdma/irdma_type.h b/sys/dev/irdma/irdma_type.h --- a/sys/dev/irdma/irdma_type.h +++ b/sys/dev/irdma/irdma_type.h @@ -564,7 +564,6 @@ struct irdma_qos { struct list_head qplist; struct mutex qos_mutex; /* protect QoS attributes per QoS level */ - u64 lan_qos_handle; u32 l2_sched_node_id; u16 qs_handle; u8 traffic_class; @@ -990,6 +989,11 @@ bool in_rdrsp_wr:1; bool out_rdrsp:1; bool aeqe_overflow:1; + /* This flag is used to determine if we should pass the rq tail + * in the QP context for FW/HW. It is set when ae_src is rq for GEN1/GEN2 + * And additionally set for inbound atomic, read and write for GEN3 + */ + bool err_rq_idx_valid:1; u8 q2_data_written; u8 ae_src; }; @@ -1217,7 +1221,7 @@ struct irdma_aeq_init_info *info); int irdma_sc_get_next_aeqe(struct irdma_sc_aeq *aeq, struct irdma_aeqe_info *info); -int irdma_sc_repost_aeq_entries(struct irdma_sc_dev *dev, u32 count); +void irdma_sc_repost_aeq_entries(struct irdma_sc_dev *dev, u32 count); void irdma_sc_pd_init(struct irdma_sc_dev *dev, struct irdma_sc_pd *pd, u32 pd_id, int abi_ver); diff --git a/sys/dev/irdma/irdma_uda.c b/sys/dev/irdma/irdma_uda.c --- a/sys/dev/irdma/irdma_uda.c +++ b/sys/dev/irdma/irdma_uda.c @@ -1,7 +1,7 @@ /*- * SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB * - * Copyright (c) 2016 - 2021 Intel Corporation + * Copyright (c) 2016 - 2022 Intel Corporation * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU @@ -122,7 +122,7 @@ * irdma_create_mg_ctx() - create a mcg context * @info: multicast group context info */ -static int +static void irdma_create_mg_ctx(struct irdma_mcast_grp_info *info) { struct irdma_mcast_grp_ctx_entry_info *entry_info = NULL; @@ -142,8 +142,6 @@ ctx_idx++; } } - - return 0; } /** @@ -159,7 +157,6 @@ u64 scratch) { __le64 *wqe; - int ret_code = 0; if (info->mg_id >= IRDMA_UDA_MAX_FSI_MGS) { irdma_debug(cqp->dev, IRDMA_DEBUG_WQE, "mg_id out of range\n"); @@ -172,9 +169,7 @@ return -ENOSPC; } - ret_code = irdma_create_mg_ctx(info); - if (ret_code) - return ret_code; + irdma_create_mg_ctx(info); set_64bit_val(wqe, IRDMA_BYTE_32, info->dma_mem_mc.pa); set_64bit_val(wqe, IRDMA_BYTE_16, diff --git a/sys/dev/irdma/irdma_uk.c b/sys/dev/irdma/irdma_uk.c --- a/sys/dev/irdma/irdma_uk.c +++ b/sys/dev/irdma/irdma_uk.c @@ -641,7 +641,7 @@ sge_len -= bytes_copied; if (!quanta_bytes_remaining) { - /* Remaining inline bytes reside after the hdr */ + /* Remaining inline bytes reside after hdr */ wqe += 16; quanta_bytes_remaining = 32; } @@ -683,8 +683,8 @@ * @polarity: polarity of wqe valid bit */ static void -irdma_copy_inline_data(u8 *wqe, struct irdma_sge *sge_list, u32 num_sges, - u8 polarity) +irdma_copy_inline_data(u8 *wqe, struct irdma_sge *sge_list, + u32 num_sges, u8 polarity) { u8 inline_valid = polarity << IRDMA_INLINE_VALID_S; u32 quanta_bytes_remaining = 8; @@ -710,7 +710,7 @@ if (!quanta_bytes_remaining) { quanta_bytes_remaining = 31; - /* Remaining inline bytes reside after the hdr */ + /* Remaining inline bytes reside after hdr */ if (first_quanta) { first_quanta = false; wqe += 16; @@ -946,58 +946,6 @@ return 0; } -/** - * irdma_uk_mw_bind - bind Memory Window - * @qp: hw qp ptr - * @info: post sq information - * @post_sq: flag to post sq - */ -int -irdma_uk_mw_bind(struct irdma_qp_uk *qp, struct irdma_post_sq_info *info, - bool post_sq) -{ - __le64 *wqe; - struct irdma_bind_window *op_info; - u64 hdr; - u32 wqe_idx; - bool local_fence; - u16 quanta = IRDMA_QP_WQE_MIN_QUANTA; - - info->push_wqe = qp->push_db ? true : false; - op_info = &info->op.bind_window; - local_fence = info->local_fence; - - wqe = irdma_qp_get_next_send_wqe(qp, &wqe_idx, &quanta, 0, info); - if (!wqe) - return -ENOSPC; - - qp->wqe_ops.iw_set_mw_bind_wqe(wqe, op_info); - - hdr = FIELD_PREP(IRDMAQPSQ_OPCODE, IRDMA_OP_TYPE_BIND_MW) | - FIELD_PREP(IRDMAQPSQ_STAGRIGHTS, - ((op_info->ena_reads << 2) | (op_info->ena_writes << 3))) | - FIELD_PREP(IRDMAQPSQ_VABASEDTO, - (op_info->addressing_type == IRDMA_ADDR_TYPE_VA_BASED ? 1 : 0)) | - FIELD_PREP(IRDMAQPSQ_MEMWINDOWTYPE, - (op_info->mem_window_type_1 ? 1 : 0)) | - FIELD_PREP(IRDMAQPSQ_PUSHWQE, info->push_wqe) | - FIELD_PREP(IRDMAQPSQ_READFENCE, info->read_fence) | - FIELD_PREP(IRDMAQPSQ_LOCALFENCE, local_fence) | - FIELD_PREP(IRDMAQPSQ_SIGCOMPL, info->signaled) | - FIELD_PREP(IRDMAQPSQ_VALID, qp->swqe_polarity); - - irdma_wmb(); /* make sure WQE is populated before valid bit is set */ - - set_64bit_val(wqe, IRDMA_BYTE_24, hdr); - - if (info->push_wqe) - irdma_qp_push_wqe(qp, wqe, quanta, wqe_idx, post_sq); - else if (post_sq) - irdma_uk_qp_post_wr(qp); - - return 0; -} - /** * irdma_uk_post_receive - post receive wqe * @qp: hw qp ptr @@ -1111,7 +1059,6 @@ u8 arm_next = 0; u8 arm_seq_num; - cq->armed = true; get_64bit_val(cq->shadow_area, IRDMA_BYTE_32, &temp_val); arm_seq_num = (u8)FIELD_GET(IRDMA_CQ_DBSA_ARM_SEQ_NUM, temp_val); arm_seq_num++; @@ -1173,6 +1120,58 @@ return 0; } +/** + * irdma_detect_unsignaled_cmpls - check if unsignaled cmpl is to be reported + * @cq: hw cq + * @qp: hw qp + * @info: cq poll information collected + * @wge_idx: index of the WR in SQ ring + */ +static int +irdma_detect_unsignaled_cmpls(struct irdma_cq_uk *cq, + struct irdma_qp_uk *qp, + struct irdma_cq_poll_info *info, + u32 wqe_idx) +{ + u64 qword0, qword1, qword2, qword3; + __le64 *cqe, *wqe; + int i; + u32 widx; + + if (qp->sq_wrtrk_array[wqe_idx].signaled == 0) { + cqe = IRDMA_GET_CURRENT_CQ_ELEM(cq); + irdma_pr_err("%p %d %d\n", cqe, cq->cq_ring.head, wqe_idx); + for (i = -10; i <= 10; i++) { + IRDMA_GET_CQ_ELEM_AT_OFFSET(cq, i + cq->cq_ring.size, cqe); + get_64bit_val(cqe, IRDMA_BYTE_0, &qword0); + get_64bit_val(cqe, IRDMA_BYTE_8, &qword1); + get_64bit_val(cqe, IRDMA_BYTE_16, &qword2); + get_64bit_val(cqe, IRDMA_BYTE_24, &qword3); + widx = (u32)FIELD_GET(IRDMA_CQ_WQEIDX, qword3); + irdma_pr_err("%d %04x %p %016lx %016lx %016lx %016lx ", + i, widx, cqe, qword0, qword1, qword2, qword3); + if ((u8)FIELD_GET(IRDMA_CQ_SQ, qword3)) { + irdma_pr_err("%lx %x %x %x ", + qp->sq_wrtrk_array[widx].wrid, qp->sq_wrtrk_array[widx].wr_len, + qp->sq_wrtrk_array[widx].quanta, qp->sq_wrtrk_array[widx].signaled); + wqe = qp->sq_base[widx].elem; + get_64bit_val(wqe, IRDMA_BYTE_0, &qword0); + get_64bit_val(wqe, IRDMA_BYTE_8, &qword1); + get_64bit_val(wqe, IRDMA_BYTE_16, &qword2); + get_64bit_val(wqe, IRDMA_BYTE_24, &qword3); + + irdma_pr_err("%016lx %016lx %016lx %016lx \n", + qword0, qword1, qword2, qword3); + } else { + irdma_pr_err("\n"); + } + } + return -ENOENT; + } + + return 0; +} + /** * irdma_uk_cq_poll_cmpl - get cq completion info * @cq: hw cq @@ -1261,6 +1260,8 @@ info->error = (bool)FIELD_GET(IRDMA_CQ_ERROR, qword3); info->push_dropped = (bool)FIELD_GET(IRDMACQ_PSHDROP, qword3); info->ipv4 = (bool)FIELD_GET(IRDMACQ_IPV4, qword3); + get_64bit_val(cqe, IRDMA_BYTE_8, &comp_ctx); + qp = (struct irdma_qp_uk *)(irdma_uintptr) comp_ctx; if (info->error) { info->major_err = FIELD_GET(IRDMA_CQ_MAJERR, qword3); info->minor_err = FIELD_GET(IRDMA_CQ_MINERR, qword3); @@ -1289,10 +1290,7 @@ info->qp_id = (u32)FIELD_GET(IRDMACQ_QPID, qword2); info->ud_src_qpn = (u32)FIELD_GET(IRDMACQ_UDSRCQPN, qword2); - get_64bit_val(cqe, IRDMA_BYTE_8, &comp_ctx); - info->solicited_event = (bool)FIELD_GET(IRDMACQ_SOEVENT, qword3); - qp = (struct irdma_qp_uk *)(irdma_uintptr) comp_ctx; if (!qp || qp->destroy_pending) { ret_code = -EFAULT; goto exit; @@ -1382,6 +1380,9 @@ info->signaled = qp->sq_wrtrk_array[wqe_idx].signaled; if (!info->comp_status) info->bytes_xfered = qp->sq_wrtrk_array[wqe_idx].wr_len; + ret_code = irdma_detect_unsignaled_cmpls(cq, qp, info, wqe_idx); + if (ret_code != 0) + goto exit; info->op_type = (u8)FIELD_GET(IRDMACQ_OP, qword3); IRDMA_RING_SET_TAIL(qp->sq_ring, wqe_idx + qp->sq_wrtrk_array[wqe_idx].quanta); @@ -1404,7 +1405,8 @@ sw_wqe = qp->sq_base[tail].elem; get_64bit_val(sw_wqe, IRDMA_BYTE_24, &wqe_qword); - info->op_type = (u8)FIELD_GET(IRDMAQPSQ_OPCODE, wqe_qword); + info->op_type = (u8)FIELD_GET(IRDMAQPSQ_OPCODE, + wqe_qword); IRDMA_RING_SET_TAIL(qp->sq_ring, tail + qp->sq_wrtrk_array[tail].quanta); if (info->op_type != IRDMAQP_OP_NOP) { @@ -1415,7 +1417,8 @@ } } while (1); - if (info->op_type == IRDMA_OP_TYPE_BIND_MW && info->minor_err == FLUSH_PROT_ERR) + if (info->op_type == IRDMA_OP_TYPE_BIND_MW && + info->minor_err == FLUSH_PROT_ERR) info->minor_err = FLUSH_MW_BIND_ERR; qp->sq_flush_seen = true; if (!IRDMA_RING_MORE_WORK(qp->sq_ring)) diff --git a/sys/dev/irdma/irdma_user.h b/sys/dev/irdma/irdma_user.h --- a/sys/dev/irdma/irdma_user.h +++ b/sys/dev/irdma/irdma_user.h @@ -209,7 +209,6 @@ IRDMA_Q2_BUF_SIZE = 256, IRDMA_QP_CTX_SIZE = 256, IRDMA_MAX_PDS = 262144, - IRDMA_MIN_WQ_SIZE_GEN2 = 8, }; enum irdma_addressing_type { @@ -408,8 +407,6 @@ struct irdma_post_sq_info *info, bool post_sq); int irdma_uk_inline_send(struct irdma_qp_uk *qp, struct irdma_post_sq_info *info, bool post_sq); -int irdma_uk_mw_bind(struct irdma_qp_uk *qp, struct irdma_post_sq_info *info, - bool post_sq); int irdma_uk_post_nop(struct irdma_qp_uk *qp, u64 wr_id, bool signaled, bool post_sq); int irdma_uk_post_receive(struct irdma_qp_uk *qp, @@ -519,7 +516,6 @@ u32 cq_size; struct irdma_ring cq_ring; u8 polarity; - bool armed:1; bool avoid_mem_cflct:1; }; diff --git a/sys/dev/irdma/irdma_utils.c b/sys/dev/irdma/irdma_utils.c --- a/sys/dev/irdma/irdma_utils.c +++ b/sys/dev/irdma/irdma_utils.c @@ -174,7 +174,7 @@ iwdev->nb_netdevice_event.notifier_call = irdma_netdevice_event; ret = register_netdevice_notifier(&iwdev->nb_netdevice_event); if (ret) { - ibdev_err(&iwdev->ibdev, "register_netdevice_notifier failed\n"); + irdma_dev_err(&iwdev->ibdev, "register_netdevice_notifier failed\n"); return ret; } return ret; @@ -207,8 +207,7 @@ } } if (!cqp_request) { - irdma_debug(cqp->sc_cqp.dev, IRDMA_DEBUG_ERR, - "CQP Request Fail: No Memory"); + irdma_debug(cqp->sc_cqp.dev, IRDMA_DEBUG_ERR, "CQP Request Fail: No Memory"); return NULL; } @@ -413,6 +412,8 @@ [IRDMA_OP_WS_ADD_NODE] = "Add Work Scheduler Node Cmd", [IRDMA_OP_WS_MODIFY_NODE] = "Modify Work Scheduler Node Cmd", [IRDMA_OP_WS_DELETE_NODE] = "Delete Work Scheduler Node Cmd", + [IRDMA_OP_WS_FAILOVER_START] = "Failover Start Cmd", + [IRDMA_OP_WS_FAILOVER_COMPLETE] = "Failover Complete Cmd", [IRDMA_OP_SET_UP_MAP] = "Set UP-UP Mapping Cmd", [IRDMA_OP_GEN_AE] = "Generate AE Cmd", [IRDMA_OP_QUERY_RDMA_FEATURES] = "RDMA Get Features Cmd", @@ -450,8 +451,7 @@ irdma_debug(dev, IRDMA_DEBUG_CQP, "[%s Error][%s] maj=0x%x min=0x%x\n", irdma_noncrit_err_list[i].desc, - irdma_cqp_cmd_names[cqp_cmd], - maj_err_code, + irdma_cqp_cmd_names[cqp_cmd], maj_err_code, min_err_code); return false; } @@ -474,7 +474,7 @@ bool put_cqp_request = true; if (rf->reset) - return -EBUSY; + return 0; irdma_get_cqp_request(cqp_request); status = irdma_process_cqp_cmd(dev, info); @@ -494,10 +494,11 @@ if (irdma_cqp_crit_err(dev, info->cqp_cmd, cqp_request->compl_info.maj_err_code, cqp_request->compl_info.min_err_code)) - irdma_dev_err(dev, + irdma_dev_err(&rf->iwdev->ibdev, "[%s Error][op_code=%d] status=%d waiting=%d completion_err=%d maj=0x%x min=0x%x\n", - irdma_cqp_cmd_names[info->cqp_cmd], info->cqp_cmd, status, cqp_request->waiting, - cqp_request->compl_info.error, cqp_request->compl_info.maj_err_code, + irdma_cqp_cmd_names[info->cqp_cmd], info->cqp_cmd, status, + cqp_request->waiting, cqp_request->compl_info.error, + cqp_request->compl_info.maj_err_code, cqp_request->compl_info.min_err_code); if (put_cqp_request) @@ -559,7 +560,7 @@ } struct ib_device * -irdma_get_ibdev(struct irdma_sc_dev *dev) +to_ibdev(struct irdma_sc_dev *dev) { return &(container_of(dev, struct irdma_pci_f, sc_dev))->iwdev->ibdev; } @@ -1102,7 +1103,7 @@ struct irdma_gen_ae_info info = {0}; struct irdma_pci_f *rf = dev_to_rf(dev); - irdma_debug(dev, IRDMA_DEBUG_AEQ, "Generate MPA CRC AE\n"); + irdma_debug(&rf->sc_dev, IRDMA_DEBUG_AEQ, "Generate MPA CRC AE\n"); info.ae_code = IRDMA_AE_LLP_RECEIVED_MPA_CRC_ERROR; info.ae_src = IRDMA_AE_SOURCE_RQ; irdma_gen_ae(rf, qp, &info, false); @@ -1606,7 +1607,7 @@ status = irdma_sc_poll_for_cqp_op_done(cqp, IRDMA_CQP_OP_WORK_SCHED_NODE, &compl_info); node_info->qs_handle = compl_info.op_ret_val; - irdma_debug(cqp->dev, IRDMA_DEBUG_DCB, + irdma_debug(&rf->sc_dev, IRDMA_DEBUG_DCB, "opcode=%d, compl_info.retval=%d\n", compl_info.op_code, compl_info.op_ret_val); } else { @@ -2190,7 +2191,6 @@ ret = irdma_handle_cqp_op(rf, cqp_request); if (ret) goto error; - irdma_debug(dev, IRDMA_DEBUG_QP, "PRINT CONTXT QP [%d]\n", info->qp_id); { u32 i, j; @@ -2252,7 +2252,8 @@ irdma_debug(iwcq->sc_cq.dev, IRDMA_DEBUG_VERBS, "%s: Poll artificially generated completion for QP 0x%X, op %u, wr_id=0x%lx\n", - __func__, cq_poll_info->qp_id, cq_poll_info->op_type, cq_poll_info->wr_id); + __func__, cq_poll_info->qp_id, cq_poll_info->op_type, + cq_poll_info->wr_id); return 0; } @@ -2299,14 +2300,10 @@ __le64 *sw_wqe; u64 wqe_qword; u32 wqe_idx; - u8 compl_generated = 0; - unsigned long flags; - bool reschedule = false; + bool compl_generated = false; + unsigned long flags1; -#define SQ_COMPL_GENERATED (0x01) -#define RQ_COMPL_GENERATED (0x02) - - spin_lock_irqsave(&iwqp->iwscq->lock, flags); + spin_lock_irqsave(&iwqp->iwscq->lock, flags1); if (irdma_cq_empty(iwqp->iwscq)) { unsigned long flags2; @@ -2315,7 +2312,7 @@ cmpl = kzalloc(sizeof(*cmpl), GFP_ATOMIC); if (!cmpl) { spin_unlock_irqrestore(&iwqp->lock, flags2); - spin_unlock_irqrestore(&iwqp->iwscq->lock, flags); + spin_unlock_irqrestore(&iwqp->iwscq->lock, flags1); return; } @@ -2329,21 +2326,31 @@ cmpl->cpi.op_type = (u8)FIELD_GET(IRDMAQPSQ_OPCODE, wqe_qword); cmpl->cpi.q_type = IRDMA_CQE_QTYPE_SQ; /* remove the SQ WR by moving SQ tail */ - IRDMA_RING_SET_TAIL(*sq_ring, sq_ring->tail + qp->sq_wrtrk_array[sq_ring->tail].quanta); + IRDMA_RING_SET_TAIL(*sq_ring, + sq_ring->tail + qp->sq_wrtrk_array[sq_ring->tail].quanta); + if (cmpl->cpi.op_type == IRDMAQP_OP_NOP) { + kfree(cmpl); + continue; + } irdma_debug(iwqp->sc_qp.dev, IRDMA_DEBUG_DEV, - "%s: adding wr_id = 0x%lx SQ Completion to list qp_id=%d\n", __func__, cmpl->cpi.wr_id, qp->qp_id); + "%s: adding wr_id = 0x%lx SQ Completion to list qp_id=%d\n", + __func__, cmpl->cpi.wr_id, qp->qp_id); list_add_tail(&cmpl->list, &iwqp->iwscq->cmpl_generated); - compl_generated |= SQ_COMPL_GENERATED; + compl_generated = true; } spin_unlock_irqrestore(&iwqp->lock, flags2); - spin_unlock_irqrestore(&iwqp->iwscq->lock, flags); + spin_unlock_irqrestore(&iwqp->iwscq->lock, flags1); + if (compl_generated) { + irdma_comp_handler(iwqp->iwscq); + compl_generated = false; + } } else { - spin_unlock_irqrestore(&iwqp->iwscq->lock, flags); - reschedule = true; + spin_unlock_irqrestore(&iwqp->iwscq->lock, flags1); + irdma_sched_qp_flush_work(iwqp); } - spin_lock_irqsave(&iwqp->iwrcq->lock, flags); + spin_lock_irqsave(&iwqp->iwrcq->lock, flags1); if (irdma_cq_empty(iwqp->iwrcq)) { unsigned long flags2; @@ -2352,7 +2359,7 @@ cmpl = kzalloc(sizeof(*cmpl), GFP_ATOMIC); if (!cmpl) { spin_unlock_irqrestore(&iwqp->lock, flags2); - spin_unlock_irqrestore(&iwqp->iwrcq->lock, flags); + spin_unlock_irqrestore(&iwqp->iwrcq->lock, flags1); return; } @@ -2367,31 +2374,55 @@ IRDMA_RING_SET_TAIL(*rq_ring, rq_ring->tail + 1); irdma_debug(iwqp->sc_qp.dev, IRDMA_DEBUG_DEV, "%s: adding wr_id = 0x%lx RQ Completion to list qp_id=%d, wqe_idx=%d\n", - __func__, cmpl->cpi.wr_id, qp->qp_id, wqe_idx); + __func__, cmpl->cpi.wr_id, qp->qp_id, + wqe_idx); + list_add_tail(&cmpl->list, &iwqp->iwrcq->cmpl_generated); - compl_generated |= RQ_COMPL_GENERATED; + compl_generated = true; } spin_unlock_irqrestore(&iwqp->lock, flags2); - spin_unlock_irqrestore(&iwqp->iwrcq->lock, flags); + spin_unlock_irqrestore(&iwqp->iwrcq->lock, flags1); + if (compl_generated) + irdma_comp_handler(iwqp->iwrcq); } else { - spin_unlock_irqrestore(&iwqp->iwrcq->lock, flags); - reschedule = true; - } - - if (reschedule) + spin_unlock_irqrestore(&iwqp->iwrcq->lock, flags1); irdma_sched_qp_flush_work(iwqp); - if (compl_generated) { - if (iwqp->iwscq == iwqp->iwrcq) { - irdma_comp_handler(iwqp->iwscq); - } else { - if (compl_generated & SQ_COMPL_GENERATED) - irdma_comp_handler(iwqp->iwscq); - if (compl_generated & RQ_COMPL_GENERATED) - irdma_comp_handler(iwqp->iwrcq); - } - irdma_debug(iwqp->sc_qp.dev, IRDMA_DEBUG_VERBS, - "0x%X (SQ 0x1, RQ 0x2, both 0x3) completions generated for QP %d\n", - compl_generated, iwqp->ibqp.qp_num); } } + +/** + * irdma_udqp_qs_change - change qs for UD QP in a worker thread + * @iwqp: QP pointer + * @user_prio: new user priority value + * @qs_change: when false, only user priority changes, QS handle do not need to change + */ +static void +irdma_udqp_qs_change(struct irdma_qp *iwqp, u8 user_prio, bool qs_change) +{ + irdma_qp_rem_qos(&iwqp->sc_qp); + if (qs_change) + iwqp->sc_qp.dev->ws_remove(iwqp->sc_qp.vsi, iwqp->ctx_info.user_pri); + + iwqp->ctx_info.user_pri = user_prio; + iwqp->sc_qp.user_pri = user_prio; + + if (qs_change) + if (iwqp->sc_qp.dev->ws_add(iwqp->sc_qp.vsi, user_prio)) + irdma_dev_warn(&iwqp->iwdev->ibdev, + "WS add failed during %s, qp_id: %x user_pri: %x", + __func__, iwqp->ibqp.qp_num, user_prio); + irdma_qp_add_qos(&iwqp->sc_qp); +} + +void +irdma_udqp_qs_worker(struct work_struct *work) +{ + struct irdma_udqs_work *udqs_work = container_of(work, struct irdma_udqs_work, work); + + irdma_udqp_qs_change(udqs_work->iwqp, udqs_work->user_prio, udqs_work->qs_change); + if (udqs_work->qs_change) + irdma_cqp_qp_suspend_resume(&udqs_work->iwqp->sc_qp, IRDMA_OP_RESUME); + irdma_qp_rem_ref(&udqs_work->iwqp->ibqp); + kfree(udqs_work); +} diff --git a/sys/dev/irdma/irdma_verbs.h b/sys/dev/irdma/irdma_verbs.h --- a/sys/dev/irdma/irdma_verbs.h +++ b/sys/dev/irdma/irdma_verbs.h @@ -62,12 +62,15 @@ struct list_head vma_list; struct mutex vma_list_mutex; /* protect the vma_list */ int abi_ver; - bool legacy_mode; + bool legacy_mode:1; + bool use_raw_attrs:1; }; struct irdma_pd { struct ib_pd ibpd; struct irdma_sc_pd sc_pd; + struct list_head udqp_list; + spinlock_t udqp_list_lock; }; struct irdma_av { @@ -184,6 +187,15 @@ struct irdma_qp *iwqp; }; +struct if_notify_work { + struct work_struct work; + struct irdma_device *iwdev; + u32 ipaddr[4]; + u16 vlan_id; + bool ipv4:1; + bool ifup:1; +}; + struct iw_cm_id; struct irdma_qp_kmode { @@ -220,6 +232,7 @@ struct irdma_ah roce_ah; struct list_head teardown_entry; + struct list_head ud_list_elem; atomic_t refcnt; struct iw_cm_id *cm_id; struct irdma_cm_node *cm_node; @@ -261,6 +274,13 @@ u8 pau_mode : 1; }; +struct irdma_udqs_work { + struct work_struct work; + struct irdma_qp *iwqp; + u8 user_prio; + bool qs_change:1; +}; + enum irdma_mmap_flag { IRDMA_MMAP_IO_NC, IRDMA_MMAP_IO_WC, diff --git a/sys/dev/irdma/irdma_verbs.c b/sys/dev/irdma/irdma_verbs.c --- a/sys/dev/irdma/irdma_verbs.c +++ b/sys/dev/irdma/irdma_verbs.c @@ -55,7 +55,8 @@ return -EINVAL; memset(props, 0, sizeof(*props)); - ether_addr_copy((u8 *)&props->sys_image_guid, IF_LLADDR(iwdev->netdev)); + addrconf_addr_eui48((u8 *)&props->sys_image_guid, + IF_LLADDR(iwdev->netdev)); props->fw_ver = (u64)irdma_fw_major_ver(&rf->sc_dev) << 32 | irdma_fw_minor_ver(&rf->sc_dev); props->device_cap_flags = IB_DEVICE_MEM_WINDOW | @@ -193,7 +194,8 @@ spin_unlock_irqrestore(&ucontext->mmap_tbl_lock, flags); } while (retry_cnt++ < 10); - irdma_debug(iwdev_to_idev(ucontext->iwdev), IRDMA_DEBUG_VERBS, "mmap table add failed: Cannot find a unique key\n"); + irdma_debug(&ucontext->iwdev->rf->sc_dev, IRDMA_DEBUG_VERBS, + "mmap table add failed: Cannot find a unique key\n"); kfree(entry); return NULL; @@ -271,7 +273,7 @@ #if __FreeBSD_version >= 1400026 rdma_entry = rdma_user_mmap_entry_get(&ucontext->ibucontext, vma); if (!rdma_entry) { - irdma_debug(iwdev_to_idev(ucontext->iwdev), IRDMA_DEBUG_VERBS, + irdma_debug(&ucontext->iwdev->rf->sc_dev, IRDMA_DEBUG_VERBS, "pgoff[0x%lx] does not have valid entry\n", vma->vm_pgoff); return -EINVAL; @@ -281,15 +283,15 @@ #else entry = irdma_find_user_mmap_entry(ucontext, vma); if (!entry) { - irdma_debug(iwdev_to_idev(ucontext->iwdev), IRDMA_DEBUG_VERBS, + irdma_debug(&ucontext->iwdev->rf->sc_dev, IRDMA_DEBUG_VERBS, "pgoff[0x%lx] does not have valid entry\n", vma->vm_pgoff); return -EINVAL; } #endif - irdma_debug(iwdev_to_idev(ucontext->iwdev), IRDMA_DEBUG_VERBS, - "bar_offset [0x%lx] mmap_flag [%d]\n", entry->bar_offset, - entry->mmap_flag); + irdma_debug(&ucontext->iwdev->rf->sc_dev, + IRDMA_DEBUG_VERBS, "bar_offset [0x%lx] mmap_flag [%d]\n", + entry->bar_offset, entry->mmap_flag); pfn = (entry->bar_offset + pci_resource_start(ucontext->iwdev->rf->pcidev, 0)) >> PAGE_SHIFT; @@ -320,7 +322,7 @@ } if (ret) - irdma_debug(iwdev_to_idev(ucontext->iwdev), IRDMA_DEBUG_VERBS, + irdma_debug(&ucontext->iwdev->rf->sc_dev, IRDMA_DEBUG_VERBS, "bar_offset [0x%lx] mmap_flag[%d] err[%d]\n", entry->bar_offset, entry->mmap_flag, ret); #if __FreeBSD_version >= 1400026 @@ -463,7 +465,8 @@ push_wqe_mmap_key); #else iwqp->push_wqe_mmap_entry = irdma_user_mmap_entry_add_hash(ucontext, bar_off, - IRDMA_MMAP_IO_WC, push_wqe_mmap_key); + IRDMA_MMAP_IO_WC, + push_wqe_mmap_key); #endif if (!iwqp->push_wqe_mmap_entry) return -ENOMEM; @@ -477,7 +480,8 @@ #else iwqp->push_db_mmap_entry = irdma_user_mmap_entry_add_hash(ucontext, bar_off, - IRDMA_MMAP_IO_NC, push_db_mmap_key); + IRDMA_MMAP_IO_NC, + push_db_mmap_key); #endif if (!iwqp->push_db_mmap_entry) { #if __FreeBSD_version >= 1400026 @@ -532,6 +536,11 @@ struct irdma_qp_init_info *info, struct ib_qp_init_attr *init_attr) { +#if __FreeBSD_version >= 1400026 + struct irdma_ucontext *ucontext = rdma_udata_to_drv_context(udata, struct irdma_ucontext, ibucontext); +#else + struct irdma_ucontext *ucontext = to_ucontext(iwqp->iwpd->ibpd.uobject->context); +#endif struct irdma_qp_uk_init_info *ukinfo = &info->qp_uk_init_info; struct irdma_create_qp_req req = {0}; unsigned long flags; @@ -540,7 +549,7 @@ ret = ib_copy_from_udata(&req, udata, min(sizeof(req), udata->inlen)); if (ret) { - irdma_debug(iwdev_to_idev(iwdev), IRDMA_DEBUG_VERBS, + irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_VERBS, "ib_copy_from_data fail\n"); return ret; } @@ -548,11 +557,6 @@ iwqp->ctx_info.qp_compl_ctx = req.user_compl_ctx; iwqp->user_mode = 1; if (req.user_wqe_bufs) { -#if __FreeBSD_version >= 1400026 - struct irdma_ucontext *ucontext = rdma_udata_to_drv_context(udata, struct irdma_ucontext, ibucontext); -#else - struct irdma_ucontext *ucontext = to_ucontext(iwqp->iwpd->ibpd.uobject->context); -#endif info->qp_uk_init_info.legacy_mode = ucontext->legacy_mode; spin_lock_irqsave(&ucontext->qp_reg_mem_list_lock, flags); iwqp->iwpbl = irdma_get_pbl((unsigned long)req.user_wqe_bufs, @@ -561,16 +565,18 @@ if (!iwqp->iwpbl) { ret = -ENODATA; - irdma_debug(iwdev_to_idev(iwdev), IRDMA_DEBUG_VERBS, + irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_VERBS, "no pbl info\n"); return ret; } } - if (ukinfo->abi_ver <= 5) { + if (!ucontext->use_raw_attrs) { /** - * For ABI version less than 6 passes raw sq and rq - * quanta in cap.max_send_wr and cap.max_recv_wr. + * Maintain backward compat with older ABI which passes sq and + * rq depth in quanta in cap.max_send_wr and cap.max_recv_wr. + * There is no way to compute the correct value of + * iwqp->max_send_wr/max_recv_wr in the kernel. */ iwqp->max_send_wr = init_attr->cap.max_send_wr; iwqp->max_recv_wr = init_attr->cap.max_recv_wr; @@ -737,7 +743,7 @@ roce_info->wr_rdresp_en = true; roce_info->bind_en = true; roce_info->dcqcn_en = false; - roce_info->rtomin = 5; + roce_info->rtomin = iwdev->roce_rtomin; roce_info->ack_credits = iwdev->roce_ackcreds; roce_info->ird_size = dev->hw_attrs.max_hw_ird; @@ -817,6 +823,8 @@ void irdma_sched_qp_flush_work(struct irdma_qp *iwqp) { + if (iwqp->sc_qp.qp_uk.destroy_pending) + return; irdma_qp_add_ref(&iwqp->ibqp); if (mod_delayed_work(iwqp->iwdev->cleanup_wq, &iwqp->dwork_flush, msecs_to_jiffies(IRDMA_FLUSH_DELAY_MS))) @@ -1026,7 +1034,7 @@ av->attrs = attr->ah_attr; rdma_gid2ip((struct sockaddr *)&av->dgid_addr, &attr->ah_attr.grh.dgid); - if (av->sgid_addr.saddr.sa_family == AF_INET6) { + if (av->net_type == RDMA_NETWORK_IPV6) { __be32 *daddr = av->dgid_addr.saddr_in6.sin6_addr.__u6_addr.__u6_addr32; __be32 *saddr = @@ -1037,10 +1045,7 @@ udp_info->ipv4 = false; irdma_copy_ip_ntohl(local_ip, daddr); - - udp_info->arp_idx = irdma_arp_table(iwdev->rf, local_ip, - NULL, IRDMA_ARP_RESOLVE); - } else { + } else if (av->net_type == RDMA_NETWORK_IPV4) { __be32 saddr = av->sgid_addr.saddr_in.sin_addr.s_addr; __be32 daddr = av->dgid_addr.saddr_in.sin_addr.s_addr; @@ -1056,6 +1061,8 @@ udp_info->local_ipaddr[1] = 0; udp_info->local_ipaddr[2] = 0; udp_info->local_ipaddr[3] = ntohl(saddr); + } else { + return -EINVAL; } udp_info->arp_idx = irdma_add_arp(iwdev->rf, local_ip, @@ -1064,10 +1071,10 @@ if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC) { if (attr->max_rd_atomic > dev->hw_attrs.max_hw_ord) { - ibdev_err(&iwdev->ibdev, - "rd_atomic = %d, above max_hw_ord=%d\n", - attr->max_rd_atomic, - dev->hw_attrs.max_hw_ord); + irdma_dev_err(&iwdev->ibdev, + "rd_atomic = %d, above max_hw_ord=%d\n", + attr->max_rd_atomic, + dev->hw_attrs.max_hw_ord); return -EINVAL; } if (attr->max_rd_atomic) @@ -1077,10 +1084,10 @@ if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) { if (attr->max_dest_rd_atomic > dev->hw_attrs.max_hw_ird) { - ibdev_err(&iwdev->ibdev, - "rd_atomic = %d, above max_hw_ird=%d\n", - attr->max_rd_atomic, - dev->hw_attrs.max_hw_ird); + irdma_dev_err(&iwdev->ibdev, + "rd_atomic = %d, above max_hw_ird=%d\n", + attr->max_rd_atomic, + dev->hw_attrs.max_hw_ird); return -EINVAL; } if (attr->max_dest_rd_atomic) @@ -1098,19 +1105,20 @@ wait_event(iwqp->mod_qp_waitq, !atomic_read(&iwqp->hw_mod_qp_pend)); - irdma_debug(dev, IRDMA_DEBUG_VERBS, + irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_VERBS, "caller: %pS qp_id=%d to_ibqpstate=%d ibqpstate=%d irdma_qpstate=%d attr_mask=0x%x\n", - __builtin_return_address(0), ibqp->qp_num, attr->qp_state, iwqp->ibqp_state, - iwqp->iwarp_state, attr_mask); + __builtin_return_address(0), ibqp->qp_num, attr->qp_state, + iwqp->ibqp_state, iwqp->iwarp_state, attr_mask); spin_lock_irqsave(&iwqp->lock, flags); if (attr_mask & IB_QP_STATE) { if (!kc_ib_modify_qp_is_ok(iwqp->ibqp_state, attr->qp_state, iwqp->ibqp.qp_type, attr_mask, IB_LINK_LAYER_ETHERNET)) { - irdma_print("modify_qp invalid for qp_id=%d, old_state=0x%x, new_state=0x%x\n", - iwqp->ibqp.qp_num, iwqp->ibqp_state, - attr->qp_state); + irdma_dev_warn(&iwdev->ibdev, + "modify_qp invalid for qp_id=%d, old_state=0x%x, new_state=0x%x\n", + iwqp->ibqp.qp_num, iwqp->ibqp_state, + attr->qp_state); ret = -EINVAL; goto exit; } @@ -1178,6 +1186,8 @@ case IB_QPS_ERR: case IB_QPS_RESET: if (iwqp->iwarp_state == IRDMA_QP_STATE_RTS) { + if (dev->hw_attrs.uk_attrs.hw_rev <= IRDMA_GEN_2) + irdma_cqp_qp_suspend_resume(&iwqp->sc_qp, IRDMA_OP_SUSPEND); spin_unlock_irqrestore(&iwqp->lock, flags); info.next_iwarp_state = IRDMA_QP_STATE_SQD; irdma_hw_modify_qp(iwdev, iwqp, &info, true); @@ -1259,8 +1269,7 @@ udata->outlen)); if (ret) { irdma_remove_push_mmap_entries(iwqp); - irdma_debug(iwdev_to_idev(iwdev), - IRDMA_DEBUG_VERBS, + irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_VERBS, "copy_to_udata failed\n"); return ret; } @@ -1314,10 +1323,11 @@ offload_info = &iwqp->iwarp_info; tcp_info = &iwqp->tcp_info; wait_event(iwqp->mod_qp_waitq, !atomic_read(&iwqp->hw_mod_qp_pend)); - irdma_debug(dev, IRDMA_DEBUG_VERBS, + irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_VERBS, "caller: %pS qp_id=%d to_ibqpstate=%d ibqpstate=%d irdma_qpstate=%d last_aeq=%d hw_tcp_state=%d hw_iwarp_state=%d attr_mask=0x%x\n", - __builtin_return_address(0), ibqp->qp_num, attr->qp_state, iwqp->ibqp_state, iwqp->iwarp_state, - iwqp->last_aeq, iwqp->hw_tcp_state, iwqp->hw_iwarp_state, attr_mask); + __builtin_return_address(0), ibqp->qp_num, attr->qp_state, + iwqp->ibqp_state, iwqp->iwarp_state, iwqp->last_aeq, + iwqp->hw_tcp_state, iwqp->hw_iwarp_state, attr_mask); spin_lock_irqsave(&iwqp->lock, flags); if (attr_mask & IB_QP_STATE) { @@ -1510,8 +1520,8 @@ udata->outlen)); if (err) { irdma_remove_push_mmap_entries(iwqp); - irdma_debug(iwdev_to_idev(iwdev), IRDMA_DEBUG_VERBS, - "copy_to_udata failed\n"); + irdma_debug(&iwdev->rf->sc_dev, + IRDMA_DEBUG_VERBS, "copy_to_udata failed\n"); return err; } } @@ -1861,12 +1871,11 @@ * irdma_setup_pbles - copy user pg address to pble's * @rf: RDMA PCI function * @iwmr: mr pointer for this memory registration - * @use_pbles: flag if to use pble's - * @lvl_1_only: request only level 1 pble if true + * @lvl: requested pble levels */ static int irdma_setup_pbles(struct irdma_pci_f *rf, struct irdma_mr *iwmr, - bool use_pbles, bool lvl_1_only) + u8 lvl) { struct irdma_pbl *iwpbl = &iwmr->iwpbl; struct irdma_pble_alloc *palloc = &iwpbl->pble_alloc; @@ -1875,9 +1884,9 @@ int status; enum irdma_pble_level level = PBLE_LEVEL_1; - if (use_pbles) { + if (lvl) { status = irdma_get_pble(rf->pble_rsrc, palloc, iwmr->page_cnt, - lvl_1_only); + lvl); if (status) return status; @@ -1892,7 +1901,7 @@ irdma_copy_user_pgaddrs(iwmr, pbl, level); - if (use_pbles) + if (lvl) iwmr->pgaddrmem[0] = *pbl; return 0; @@ -1903,12 +1912,12 @@ * @iwdev: irdma device * @req: information for q memory management * @iwpbl: pble struct - * @use_pbles: flag to use pble + * @lvl: pble level mask */ static int irdma_handle_q_mem(struct irdma_device *iwdev, struct irdma_mem_reg_req *req, - struct irdma_pbl *iwpbl, bool use_pbles) + struct irdma_pbl *iwpbl, u8 lvl) { struct irdma_pble_alloc *palloc = &iwpbl->pble_alloc; struct irdma_mr *iwmr = iwpbl->iwmr; @@ -1921,11 +1930,11 @@ bool ret = true; pg_size = iwmr->page_size; - err = irdma_setup_pbles(iwdev->rf, iwmr, use_pbles, true); + err = irdma_setup_pbles(iwdev->rf, iwmr, lvl); if (err) return err; - if (use_pbles) + if (lvl) arr = palloc->level1.addr; switch (iwmr->type) { @@ -1933,7 +1942,7 @@ total = req->sq_pages + req->rq_pages; hmc_p = &qpmr->sq_pbl; qpmr->shadow = (dma_addr_t) arr[total]; - if (use_pbles) { + if (lvl) { ret = irdma_check_mem_contiguous(arr, req->sq_pages, pg_size); if (ret) @@ -1958,7 +1967,7 @@ if (!cqmr->split) cqmr->shadow = (dma_addr_t) arr[req->cq_pages]; - if (use_pbles) + if (lvl) ret = irdma_check_mem_contiguous(arr, req->cq_pages, pg_size); @@ -1968,12 +1977,11 @@ hmc_p->addr = arr[0]; break; default: - irdma_debug(iwdev_to_idev(iwdev), IRDMA_DEBUG_VERBS, - "MR type error\n"); + irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_VERBS, "MR type error\n"); err = -EINVAL; } - if (use_pbles && ret) { + if (lvl && ret) { irdma_free_pble(iwdev->rf->pble_rsrc, palloc); iwpbl->pbl_allocated = false; } @@ -2233,9 +2241,9 @@ struct irdma_mem_reg_req req = {}; u32 total, stag = 0; u8 shadow_pgcnt = 1; - bool use_pbles = false; unsigned long flags; int err = -EINVAL; + u8 lvl; int ret; if (len > iwdev->rf->sc_dev.hw_attrs.max_mr_size) @@ -2247,7 +2255,7 @@ region = ib_umem_get(pd->uobject->context, start, len, access, 0); if (IS_ERR(region)) { - irdma_debug(iwdev_to_idev(iwdev), IRDMA_DEBUG_VERBS, + irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_VERBS, "Failed to create ib_umem region\n"); return (struct ib_mr *)region; } @@ -2286,8 +2294,8 @@ goto error; } total = req.sq_pages + req.rq_pages; - use_pbles = (total > 2); - err = irdma_handle_q_mem(iwdev, &req, iwpbl, use_pbles); + lvl = total > 2 ? PBLE_LEVEL_1 : PBLE_LEVEL_0; + err = irdma_handle_q_mem(iwdev, &req, iwpbl, lvl); if (err) goto error; @@ -2310,8 +2318,8 @@ goto error; } - use_pbles = (req.cq_pages > 1); - err = irdma_handle_q_mem(iwdev, &req, iwpbl, use_pbles); + lvl = req.cq_pages > 1 ? PBLE_LEVEL_1 : PBLE_LEVEL_0; + err = irdma_handle_q_mem(iwdev, &req, iwpbl, lvl); if (err) goto error; @@ -2326,13 +2334,12 @@ spin_unlock_irqrestore(&ucontext->cq_reg_mem_list_lock, flags); break; case IRDMA_MEMREG_TYPE_MEM: - use_pbles = (iwmr->page_cnt != 1); - - err = irdma_setup_pbles(iwdev->rf, iwmr, use_pbles, false); + lvl = iwmr->page_cnt != 1 ? PBLE_LEVEL_1 | PBLE_LEVEL_2 : PBLE_LEVEL_0; + err = irdma_setup_pbles(iwdev->rf, iwmr, lvl); if (err) goto error; - if (use_pbles) { + if (lvl) { ret = irdma_check_mr_contiguous(palloc, iwmr->page_size); if (ret) { @@ -2436,13 +2443,13 @@ struct irdma_pble_alloc *palloc = &iwpbl->pble_alloc; struct ib_pd *pd = iwmr->ibmr.pd; struct ib_umem *region; - bool use_pbles; + u8 lvl; int err; region = ib_umem_get(pd->uobject->context, start, len, iwmr->access, 0); if (IS_ERR(region)) { - irdma_debug(iwdev_to_idev(iwdev), IRDMA_DEBUG_VERBS, + irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_VERBS, "Failed to create ib_umem region\n"); return (struct ib_mr *)region; } @@ -2457,13 +2464,13 @@ iwmr->page_cnt = irdma_ib_umem_num_dma_blocks(region, iwmr->page_size, virt); - use_pbles = (iwmr->page_cnt != 1); + lvl = iwmr->page_cnt != 1 ? PBLE_LEVEL_1 | PBLE_LEVEL_2 : PBLE_LEVEL_0; - err = irdma_setup_pbles(iwdev->rf, iwmr, use_pbles, false); + err = irdma_setup_pbles(iwdev->rf, iwmr, lvl); if (err) goto error; - if (use_pbles) { + if (lvl) { err = irdma_check_mr_contiguous(palloc, iwmr->page_size); if (err) { @@ -2727,6 +2734,7 @@ break; case IB_WR_LOCAL_INV: info.op_type = IRDMA_OP_TYPE_INV_STAG; + info.local_fence = info.read_fence; info.op.inv_local_stag.target_stag = ib_wr->ex.invalidate_rkey; err = irdma_uk_stag_local_invalidate(ukqp, &info, true); break; @@ -2759,8 +2767,7 @@ } default: err = -EINVAL; - irdma_debug(iwdev_to_idev(iwqp->iwdev), - IRDMA_DEBUG_VERBS, + irdma_debug(&iwqp->iwdev->rf->sc_dev, IRDMA_DEBUG_VERBS, "upost_send bad opcode = 0x%x\n", ib_wr->opcode); break; @@ -2779,6 +2786,7 @@ spin_unlock_irqrestore(&iwqp->lock, flags); irdma_sched_qp_flush_work(iwqp); } + if (err) *bad_wr = ib_wr; @@ -2816,8 +2824,8 @@ post_recv.sg_list = sg_list; err = irdma_uk_post_receive(ukqp, &post_recv); if (err) { - irdma_debug(iwdev_to_idev(iwqp->iwdev), - IRDMA_DEBUG_VERBS, "post_recv err %d\n", + irdma_debug(&iwqp->iwdev->rf->sc_dev, IRDMA_DEBUG_VERBS, + "post_recv err %d\n", err); goto out; } @@ -2829,6 +2837,7 @@ spin_unlock_irqrestore(&iwqp->lock, flags); if (iwqp->flush_issued) irdma_sched_qp_flush_work(iwqp); + if (err) *bad_wr = ib_wr; @@ -2896,8 +2905,8 @@ break; default: qp = cq_poll_info->qp_handle; - ibdev_err(irdma_get_ibdev(qp->dev), "Invalid opcode = %d in CQE\n", - cq_poll_info->op_type); + irdma_dev_err(to_ibdev(qp->dev), "Invalid opcode = %d in CQE\n", + cq_poll_info->op_type); entry->status = IB_WC_GENERAL_ERR; } } @@ -3109,8 +3118,9 @@ return npolled; error: - irdma_debug(iwdev_to_idev(iwdev), IRDMA_DEBUG_VERBS, - "%s: Error polling CQ, irdma_err: %d\n", __func__, ret); + irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_VERBS, + "%s: Error polling CQ, irdma_err: %d\n", + __func__, ret); return ret; } @@ -3292,8 +3302,9 @@ sgid_addr.saddr_in6.sin6_addr.__u6_addr.__u6_addr32); irdma_netdev_vlan_ipv6(ip_addr, &vlan_id, NULL); ipv4 = false; - irdma_debug(iwdev_to_idev(iwdev), IRDMA_DEBUG_VERBS, - "qp_id=%d, IP6address=%pI6\n", ibqp->qp_num, + irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_VERBS, + "qp_id=%d, IP6address=%pI6\n", + ibqp->qp_num, ip_addr); irdma_mcast_mac_v6(ip_addr, dmac); } else { @@ -3301,7 +3312,7 @@ ipv4 = true; vlan_id = irdma_get_vlan_ipv4(ip_addr); irdma_mcast_mac_v4(ip_addr, dmac); - irdma_debug(iwdev_to_idev(iwdev), IRDMA_DEBUG_VERBS, + irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_VERBS, "qp_id=%d, IP4address=%pI4, MAC=%pM\n", ibqp->qp_num, ip_addr, dmac); } @@ -3431,8 +3442,8 @@ mc_qht_elem = mcast_list_lookup_ip(rf, ip_addr); if (!mc_qht_elem) { spin_unlock_irqrestore(&rf->qh_list_lock, flags); - irdma_debug(iwdev_to_idev(iwdev), IRDMA_DEBUG_VERBS, - "address not found MCG\n"); + irdma_debug(&iwdev->rf->sc_dev, + IRDMA_DEBUG_VERBS, "address not found MCG\n"); return 0; } @@ -3444,8 +3455,8 @@ ret = irdma_mcast_cqp_op(iwdev, &mc_qht_elem->mc_grp_ctx, IRDMA_OP_MC_DESTROY); if (ret) { - irdma_debug(iwdev_to_idev(iwdev), IRDMA_DEBUG_VERBS, - "failed MC_DESTROY MCG\n"); + irdma_debug(&iwdev->rf->sc_dev, + IRDMA_DEBUG_VERBS, "failed MC_DESTROY MCG\n"); spin_lock_irqsave(&rf->qh_list_lock, flags); mcast_list_add(rf, mc_qht_elem); spin_unlock_irqrestore(&rf->qh_list_lock, flags); @@ -3462,8 +3473,8 @@ ret = irdma_mcast_cqp_op(iwdev, &mc_qht_elem->mc_grp_ctx, IRDMA_OP_MC_MODIFY); if (ret) { - irdma_debug(iwdev_to_idev(iwdev), IRDMA_DEBUG_VERBS, - "failed Modify MCG\n"); + irdma_debug(&iwdev->rf->sc_dev, + IRDMA_DEBUG_VERBS, "failed Modify MCG\n"); return ret; } } @@ -3496,23 +3507,6 @@ return 0; } -static __be64 irdma_mac_to_guid(struct ifnet *ndev){ - const unsigned char *mac = IF_LLADDR(ndev); - __be64 guid; - unsigned char *dst = (unsigned char *)&guid; - - dst[0] = mac[0] ^ 2; - dst[1] = mac[1]; - dst[2] = mac[2]; - dst[3] = 0xff; - dst[4] = 0xfe; - dst[5] = mac[3]; - dst[6] = mac[4]; - dst[7] = mac[5]; - - return guid; -} - static struct ifnet * irdma_get_netdev(struct ib_device *ibdev, u8 port_num) { @@ -3632,7 +3626,8 @@ { kc_set_roce_uverbs_cmd_mask(iwdev); iwdev->ibdev.node_type = RDMA_NODE_IB_CA; - iwdev->ibdev.node_guid = irdma_mac_to_guid(iwdev->netdev); + addrconf_addr_eui48((u8 *)&iwdev->ibdev.node_guid, + IF_LLADDR(iwdev->netdev)); irdma_set_device_roce_ops(&iwdev->ibdev); if (iwdev->rf->rdma_ver == IRDMA_GEN_2) irdma_set_device_mcast_ops(&iwdev->ibdev); @@ -3648,7 +3643,8 @@ struct ifnet *netdev = iwdev->netdev; iwdev->ibdev.node_type = RDMA_NODE_RNIC; - ether_addr_copy((u8 *)&iwdev->ibdev.node_guid, IF_LLADDR(netdev)); + addrconf_addr_eui48((u8 *)&iwdev->ibdev.node_guid, + IF_LLADDR(netdev)); iwdev->ibdev.iwcm = kzalloc(sizeof(*iwdev->ibdev.iwcm), GFP_KERNEL); if (!iwdev->ibdev.iwcm) return -ENOMEM; @@ -3675,7 +3671,6 @@ static int irdma_init_rdma_device(struct irdma_device *iwdev) { - struct pci_dev *pcidev = iwdev->rf->pcidev; int ret; iwdev->ibdev.owner = THIS_MODULE; @@ -3693,7 +3688,7 @@ iwdev->ibdev.phys_port_cnt = 1; iwdev->ibdev.num_comp_vectors = iwdev->rf->ceqs_count; iwdev->ibdev.dev.parent = iwdev->rf->dev_ctx.dev; - set_ibdev_dma_device(iwdev->ibdev, &pcidev->dev); + set_ibdev_dma_device(iwdev->ibdev, &iwdev->rf->pcidev->dev); irdma_set_device_ops(&iwdev->ibdev); if (iwdev->rf->rdma_ver == IRDMA_GEN_1) irdma_set_device_gen1_ops(&iwdev->ibdev); diff --git a/sys/dev/irdma/irdma_ws.h b/sys/dev/irdma/irdma_ws.h --- a/sys/dev/irdma/irdma_ws.h +++ b/sys/dev/irdma/irdma_ws.h @@ -1,7 +1,7 @@ /*- * SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB * - * Copyright (c) 2015 - 2021 Intel Corporation + * Copyright (c) 2015 - 2022 Intel Corporation * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU @@ -52,7 +52,6 @@ struct list_head siblings; struct list_head child_list_head; struct irdma_ws_node *parent; - u64 lan_qs_handle; /* opaque handle used by LAN */ u32 l2_sched_node_id; u16 index; u16 qs_handle; diff --git a/sys/dev/irdma/irdma_ws.c b/sys/dev/irdma/irdma_ws.c --- a/sys/dev/irdma/irdma_ws.c +++ b/sys/dev/irdma/irdma_ws.c @@ -83,7 +83,6 @@ if (!node->rel_bw) node->rel_bw = 1; - node->lan_qs_handle = vsi->qos[user_pri].lan_qos_handle; node->prio_type = IRDMA_PRIO_WEIGHTED_RR; } else { node->rel_bw = 1; @@ -383,7 +382,6 @@ for (i = 0; i < IRDMA_MAX_USER_PRIORITY; i++) { if (vsi->qos[i].traffic_class == traffic_class) { vsi->qos[i].qs_handle = tc_node->qs_handle; - vsi->qos[i].lan_qos_handle = tc_node->lan_qs_handle; vsi->qos[i].l2_sched_node_id = tc_node->l2_sched_node_id; vsi->qos[i].valid = true; } diff --git a/sys/dev/irdma/osdep.h b/sys/dev/irdma/osdep.h --- a/sys/dev/irdma/osdep.h +++ b/sys/dev/irdma/osdep.h @@ -118,7 +118,7 @@ irdma_debug(dev, mask, "%s\n", desc); \ irdma_debug(dev, mask, "starting address virt=%p phy=%lxh\n", buf, irdma_get_virt_to_phy(buf)); \ for (i = 0; i < size ; i += 8) \ - irdma_debug(dev, mask, "index %03d val: %016lx\n", i, ((unsigned long *)buf)[i / 8]); \ + irdma_debug(dev, mask, "index %03d val: %016lx\n", i, ((unsigned long *)(buf))[i / 8]); \ } while(0) #define irdma_debug(h, m, s, ...) \ @@ -130,11 +130,12 @@ printf("irdma " s, ##__VA_ARGS__); \ } \ } while (0) -#define irdma_dev_err(a, b, ...) printf(b, ##__VA_ARGS__) -#define irdma_dev_warn(a, b, ...) printf(b, ##__VA_ARGS__) /*dev_warn(a, b)*/ +#define irdma_dev_err(ibdev, fmt, ...) \ + pr_err("%s:%s:%d ERR "fmt, (ibdev)->name, __func__, __LINE__, ##__VA_ARGS__) +#define irdma_dev_warn(ibdev, fmt, ...) \ + pr_warn("%s:%s:%d WARN "fmt, (ibdev)->name, __func__, __LINE__, ##__VA_ARGS__) #define irdma_dev_info(a, b, ...) printf(b, ##__VA_ARGS__) #define irdma_pr_warn printf -#define ibdev_err(ibdev, fmt, ...) printf("%s:"fmt, (ibdev)->name, ##__VA_ARGS__) #define dump_struct(s, sz, name) \ do { \ @@ -245,6 +246,6 @@ int irdma_map_vm_page_list(struct irdma_hw *hw, void *va, u64 *pg_arr, u32 pg_cnt); -struct ib_device *irdma_get_ibdev(struct irdma_sc_dev *dev); +struct ib_device *to_ibdev(struct irdma_sc_dev *dev); #endif /* _ICRDMA_OSDEP_H_ */