Page MenuHomeFreeBSD
Files F161658671

D41425.diff
No OneTemporary
Actions

D41425.diff
View Options

diff --git a/contrib/ofed/libirdma/irdma_defs.h b/contrib/ofed/libirdma/irdma_defs.h
--- a/contrib/ofed/libirdma/irdma_defs.h
+++ b/contrib/ofed/libirdma/irdma_defs.h
@@ -1,7 +1,7 @@
/*-
* SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB
*
- * Copyright (c) 2015 - 2022 Intel Corporation
+ * Copyright (c) 2015 - 2023 Intel Corporation
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@@ -312,7 +312,7 @@
#define IRDMA_GET_CQ_ELEM_AT_OFFSET(_cq, _i, _cqe) \
{ \
- register __u32 offset; \
+ __u32 offset; \
offset = IRDMA_GET_RING_OFFSET((_cq)->cq_ring, _i); \
(_cqe) = (_cq)->cq_base[offset].buf; \
}
@@ -338,7 +338,7 @@
#define IRDMA_RING_MOVE_HEAD(_ring, _retcode) \
{ \
- register u32 size; \
+ u32 size; \
size = (_ring).size; \
if (!IRDMA_RING_FULL_ERR(_ring)) { \
(_ring).head = ((_ring).head + 1) % size; \
@@ -349,7 +349,7 @@
}
#define IRDMA_RING_MOVE_HEAD_BY_COUNT(_ring, _count, _retcode) \
{ \
- register u32 size; \
+ u32 size; \
size = (_ring).size; \
if ((IRDMA_RING_USED_QUANTA(_ring) + (_count)) < size) { \
(_ring).head = ((_ring).head + (_count)) % size; \
@@ -360,7 +360,7 @@
}
#define IRDMA_SQ_RING_MOVE_HEAD(_ring, _retcode) \
{ \
- register u32 size; \
+ u32 size; \
size = (_ring).size; \
if (!IRDMA_SQ_RING_FULL_ERR(_ring)) { \
(_ring).head = ((_ring).head + 1) % size; \
@@ -371,7 +371,7 @@
}
#define IRDMA_SQ_RING_MOVE_HEAD_BY_COUNT(_ring, _count, _retcode) \
{ \
- register u32 size; \
+ u32 size; \
size = (_ring).size; \
if ((IRDMA_RING_USED_QUANTA(_ring) + (_count)) < (size - 256)) { \
(_ring).head = ((_ring).head + (_count)) % size; \
@@ -457,6 +457,19 @@
IRDMA_WQE_SIZE_256 = 256,
};
+enum irdma_ws_op_type {
+ IRDMA_WS_OP_TYPE_NODE = 0,
+ IRDMA_WS_OP_TYPE_LEAF_NODE_GROUP,
+};
+
+enum irdma_ws_rate_limit_flags {
+ IRDMA_WS_RATE_LIMIT_FLAGS_VALID = 0x1,
+ IRDMA_WS_NO_RDMA_RATE_LIMIT = 0x2,
+ IRDMA_WS_LEAF_NODE_IS_PART_GROUP = 0x4,
+ IRDMA_WS_TREE_RATE_LIMITING = 0x8,
+ IRDMA_WS_PACING_CONTROL = 0x10,
+};
+
/**
* set_64bit_val - set 64 bit value to hw wqe
* @wqe_words: wqe addr to write
diff --git a/contrib/ofed/libirdma/irdma_uk.c b/contrib/ofed/libirdma/irdma_uk.c
--- a/contrib/ofed/libirdma/irdma_uk.c
+++ b/contrib/ofed/libirdma/irdma_uk.c
@@ -1,7 +1,7 @@
/*-
* SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB
*
- * Copyright (c) 2015 - 2022 Intel Corporation
+ * Copyright (c) 2015 - 2023 Intel Corporation
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@@ -275,7 +275,8 @@
if (qp->uk_attrs->hw_rev == IRDMA_GEN_1 && wqe_quanta == 1 &&
(IRDMA_RING_CURRENT_HEAD(qp->sq_ring) & 1)) {
wqe_0 = qp->sq_base[IRDMA_RING_CURRENT_HEAD(qp->sq_ring)].elem;
- wqe_0[3] = htole64(FIELD_PREP(IRDMAQPSQ_VALID, !qp->swqe_polarity));
+ wqe_0[3] = htole64(FIELD_PREP(IRDMAQPSQ_VALID,
+ qp->swqe_polarity ? 0 : 1));
}
qp->sq_wrtrk_array[*wqe_idx].wrid = info->wr_id;
qp->sq_wrtrk_array[*wqe_idx].wr_len = total_size;
@@ -683,8 +684,8 @@
* @polarity: polarity of wqe valid bit
*/
static void
-irdma_copy_inline_data(u8 *wqe, struct irdma_sge *sge_list, u32 num_sges,
- u8 polarity)
+irdma_copy_inline_data(u8 *wqe, struct irdma_sge *sge_list,
+ u32 num_sges, u8 polarity)
{
u8 inline_valid = polarity << IRDMA_INLINE_VALID_S;
u32 quanta_bytes_remaining = 8;
@@ -1173,7 +1174,7 @@
u32 end_idx)
{
__le64 *dst_wqe, *src_wqe;
- u32 wqe_idx;
+ u32 wqe_idx = 0;
u8 wqe_quanta = qp->rq_wqe_size_multiplier;
bool flip_polarity;
u64 val;
@@ -1480,7 +1481,8 @@
sw_wqe = qp->sq_base[tail].elem;
get_64bit_val(sw_wqe, IRDMA_BYTE_24,
&wqe_qword);
- info->op_type = (u8)FIELD_GET(IRDMAQPSQ_OPCODE, wqe_qword);
+ info->op_type = (u8)FIELD_GET(IRDMAQPSQ_OPCODE,
+ wqe_qword);
IRDMA_RING_SET_TAIL(qp->sq_ring,
tail + qp->sq_wrtrk_array[tail].quanta);
if (info->op_type != IRDMAQP_OP_NOP) {
@@ -1834,6 +1836,9 @@
if (polarity != temp)
break;
+ /* Ensure CQE contents are read after valid bit is checked */
+ udma_from_device_barrier();
+
get_64bit_val(cqe, IRDMA_BYTE_8, &comp_ctx);
if ((void *)(irdma_uintptr) comp_ctx == q)
set_64bit_val(cqe, IRDMA_BYTE_8, 0);
@@ -1845,48 +1850,6 @@
return 0;
}
-/**
- * irdma_nop - post a nop
- * @qp: hw qp ptr
- * @wr_id: work request id
- * @signaled: signaled for completion
- * @post_sq: ring doorbell
- */
-int
-irdma_nop(struct irdma_qp_uk *qp, u64 wr_id, bool signaled, bool post_sq)
-{
- __le64 *wqe;
- u64 hdr;
- u32 wqe_idx;
- struct irdma_post_sq_info info = {0};
- u16 quanta = IRDMA_QP_WQE_MIN_QUANTA;
-
- info.push_wqe = qp->push_db ? true : false;
- info.wr_id = wr_id;
- wqe = irdma_qp_get_next_send_wqe(qp, &wqe_idx, &quanta, 0, &info);
- if (!wqe)
- return ENOSPC;
-
- set_64bit_val(wqe, IRDMA_BYTE_0, 0);
- set_64bit_val(wqe, IRDMA_BYTE_8, 0);
- set_64bit_val(wqe, IRDMA_BYTE_16, 0);
-
- hdr = FIELD_PREP(IRDMAQPSQ_OPCODE, IRDMAQP_OP_NOP) |
- FIELD_PREP(IRDMAQPSQ_SIGCOMPL, signaled) |
- FIELD_PREP(IRDMAQPSQ_VALID, qp->swqe_polarity);
-
- udma_to_device_barrier(); /* make sure WQE is populated before valid bit is set */
-
- set_64bit_val(wqe, IRDMA_BYTE_24, hdr);
-
- if (info.push_wqe)
- irdma_qp_push_wqe(qp, wqe, quanta, wqe_idx, post_sq);
- else if (post_sq)
- irdma_uk_qp_post_wr(qp);
-
- return 0;
-}
-
/**
* irdma_fragcnt_to_quanta_sq - calculate quanta based on fragment count for SQ
* @frag_cnt: number of fragments
diff --git a/contrib/ofed/libirdma/irdma_umain.c b/contrib/ofed/libirdma/irdma_umain.c
--- a/contrib/ofed/libirdma/irdma_umain.c
+++ b/contrib/ofed/libirdma/irdma_umain.c
@@ -1,7 +1,7 @@
/*-
* SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB
*
- * Copyright (c) 2021 - 2022 Intel Corporation
+ * Copyright (c) 2021 - 2023 Intel Corporation
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@@ -49,7 +49,7 @@
/**
* Driver version
*/
-char libirdma_version[] = "1.1.11-k";
+char libirdma_version[] = "1.2.17-k";
unsigned int irdma_dbg;
diff --git a/contrib/ofed/libirdma/irdma_user.h b/contrib/ofed/libirdma/irdma_user.h
--- a/contrib/ofed/libirdma/irdma_user.h
+++ b/contrib/ofed/libirdma/irdma_user.h
@@ -1,7 +1,7 @@
/*-
* SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB
*
- * Copyright (c) 2015 - 2022 Intel Corporation
+ * Copyright (c) 2015 - 2023 Intel Corporation
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@@ -572,7 +572,6 @@
u32 inline_data, u8 *shift);
int irdma_get_sqdepth(struct irdma_uk_attrs *uk_attrs, u32 sq_size, u8 shift, u32 *sqdepth);
int irdma_get_rqdepth(struct irdma_uk_attrs *uk_attrs, u32 rq_size, u8 shift, u32 *rqdepth);
-int irdma_get_srqdepth(struct irdma_uk_attrs *uk_attrs, u32 srq_size, u8 shift, u32 *srqdepth);
void irdma_qp_push_wqe(struct irdma_qp_uk *qp, __le64 *wqe, u16 quanta,
u32 wqe_idx, bool post_sq);
void irdma_clr_wqes(struct irdma_qp_uk *qp, u32 qp_wqe_idx);
diff --git a/contrib/ofed/libirdma/irdma_uverbs.c b/contrib/ofed/libirdma/irdma_uverbs.c
--- a/contrib/ofed/libirdma/irdma_uverbs.c
+++ b/contrib/ofed/libirdma/irdma_uverbs.c
@@ -1,7 +1,7 @@
/*-
* SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB
*
- * Copyright (C) 2019 - 2022 Intel Corporation
+ * Copyright (C) 2019 - 2023 Intel Corporation
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@@ -266,11 +266,13 @@
if (!mw)
return NULL;
- if (ibv_cmd_alloc_mw(pd, type, mw, &cmd, sizeof(cmd), &resp,
- sizeof(resp))) {
+ err = ibv_cmd_alloc_mw(pd, type, mw, &cmd, sizeof(cmd), &resp,
+ sizeof(resp));
+ if (err) {
printf("%s: Failed to alloc memory window\n",
__func__);
free(mw);
+ errno = err;
return NULL;
}
@@ -446,8 +448,10 @@
if (!iwucq)
return NULL;
- if (pthread_spin_init(&iwucq->lock, PTHREAD_PROCESS_PRIVATE)) {
+ ret = pthread_spin_init(&iwucq->lock, PTHREAD_PROCESS_PRIVATE);
+ if (ret) {
free(iwucq);
+ errno = ret;
return NULL;
}
@@ -464,8 +468,10 @@
iwucq->buf_size = total_size;
info.cq_base = irdma_alloc_hw_buf(total_size);
- if (!info.cq_base)
+ if (!info.cq_base) {
+ ret = ENOMEM;
goto err_cq_base;
+ }
memset(info.cq_base, 0, total_size);
reg_mr_cmd.reg_type = IRDMA_MEMREG_TYPE_CQ;
@@ -476,17 +482,17 @@
IBV_ACCESS_LOCAL_WRITE, &iwucq->vmr.ibv_mr,
&reg_mr_cmd.ibv_cmd, sizeof(reg_mr_cmd),
&reg_mr_resp, sizeof(reg_mr_resp));
- if (ret) {
- errno = ret;
+ if (ret)
goto err_dereg_mr;
- }
iwucq->vmr.ibv_mr.pd = &iwvctx->iwupd->ibv_pd;
if (uk_attrs->feature_flags & IRDMA_FEATURE_CQ_RESIZE) {
info.shadow_area = irdma_alloc_hw_buf(IRDMA_DB_SHADOW_AREA_SIZE);
- if (!info.shadow_area)
+ if (!info.shadow_area) {
+ ret = ENOMEM;
goto err_alloc_shadow;
+ }
memset(info.shadow_area, 0, IRDMA_DB_SHADOW_AREA_SIZE);
reg_mr_shadow_cmd.reg_type = IRDMA_MEMREG_TYPE_CQ;
@@ -499,7 +505,6 @@
&reg_mr_shadow_resp, sizeof(reg_mr_shadow_resp));
if (ret) {
irdma_free_hw_buf(info.shadow_area, IRDMA_DB_SHADOW_AREA_SIZE);
- errno = ret;
goto err_alloc_shadow;
}
@@ -517,10 +522,8 @@
&cmd.ibv_cmd, sizeof(cmd.ibv_cmd), sizeof(cmd), &resp.ibv_resp,
sizeof(resp.ibv_resp), sizeof(resp));
attr_ex->cqe = ncqe;
- if (ret) {
- errno = ret;
+ if (ret)
goto err_create_cq;
- }
if (ext_cq)
irdma_ibvcq_ex_fill_priv_funcs(iwucq, attr_ex);
@@ -548,6 +551,7 @@
free(iwucq);
+ errno = ret;
return NULL;
}
@@ -1560,7 +1564,8 @@
memset(iwuqp, 0, sizeof(*iwuqp));
- if (pthread_spin_init(&iwuqp->lock, PTHREAD_PROCESS_PRIVATE))
+ status = pthread_spin_init(&iwuqp->lock, PTHREAD_PROCESS_PRIVATE);
+ if (status)
goto err_free_qp;
info.sq_size = info.sq_depth >> info.sq_shift;
@@ -1575,35 +1580,37 @@
}
iwuqp->recv_sges = calloc(attr->cap.max_recv_sge, sizeof(*iwuqp->recv_sges));
- if (!iwuqp->recv_sges)
+ if (!iwuqp->recv_sges) {
+ status = errno; /* preserve errno */
goto err_destroy_lock;
+ }
info.wqe_alloc_db = (u32 *)iwvctx->db;
info.legacy_mode = iwvctx->legacy_mode;
info.sq_wrtrk_array = calloc(info.sq_depth, sizeof(*info.sq_wrtrk_array));
- if (!info.sq_wrtrk_array)
+ if (!info.sq_wrtrk_array) {
+ status = errno; /* preserve errno */
goto err_free_rsges;
+ }
info.rq_wrid_array = calloc(info.rq_depth, sizeof(*info.rq_wrid_array));
- if (!info.rq_wrid_array)
+ if (!info.rq_wrid_array) {
+ status = errno; /* preserve errno */
goto err_free_sq_wrtrk;
+ }
iwuqp->sq_sig_all = attr->sq_sig_all;
iwuqp->qp_type = attr->qp_type;
status = irdma_vmapped_qp(iwuqp, pd, attr, &info, iwvctx->legacy_mode);
- if (status) {
- errno = status;
+ if (status)
goto err_free_rq_wrid;
- }
iwuqp->qp.back_qp = iwuqp;
iwuqp->qp.lock = &iwuqp->lock;
status = irdma_uk_qp_init(&iwuqp->qp, &info);
- if (status) {
- errno = status;
+ if (status)
goto err_free_vmap_qp;
- }
attr->cap.max_send_wr = (info.sq_depth - IRDMA_SQ_RSVD) >> info.sq_shift;
attr->cap.max_recv_wr = (info.rq_depth - IRDMA_RQ_RSVD) >> info.rq_shift;
@@ -1625,6 +1632,7 @@
printf("%s: failed to create QP\n", __func__);
free(iwuqp);
+ errno = status;
return NULL;
}
@@ -2081,11 +2089,10 @@
struct irdma_ucreate_ah_resp resp = {};
int err;
- err = ibv_query_gid(ibpd->context, attr->port_num, attr->grh.sgid_index,
- &sgid);
- if (err) {
+ if (ibv_query_gid(ibpd->context, attr->port_num, attr->grh.sgid_index,
+ &sgid)) {
fprintf(stderr, "irdma: Error from ibv_query_gid.\n");
- errno = err;
+ errno = ENOENT;
return NULL;
}
diff --git a/contrib/ofed/libirdma/osdep.h b/contrib/ofed/libirdma/osdep.h
--- a/contrib/ofed/libirdma/osdep.h
+++ b/contrib/ofed/libirdma/osdep.h
@@ -1,7 +1,7 @@
/*-
* SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB
*
- * Copyright (c) 2021 - 2022 Intel Corporation
+ * Copyright (c) 2021 - 2023 Intel Corporation
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@@ -46,7 +46,6 @@
#include <sys/bus_dma.h>
#include <sys/endian.h>
-#define ATOMIC atomic_t
#define IOMEM
#define IRDMA_NTOHL(a) ntohl(a)
#define IRDMA_NTOHS(a) ntohs(a)
@@ -80,23 +79,13 @@
#define STATS_TIMER_DELAY 60000
/* a couple of linux size defines */
-#define SZ_128 128
-#define SZ_2K SZ_128 * 16
-#define SZ_1G (SZ_1K * SZ_1K * SZ_1K)
-#define SPEED_1000 1000
-#define SPEED_10000 10000
-#define SPEED_20000 20000
-#define SPEED_25000 25000
-#define SPEED_40000 40000
-#define SPEED_100000 100000
-
#define BIT_ULL(a) (1ULL << (a))
#define min(a, b) ((a) > (b) ? (b) : (a))
#ifndef likely
-#define likely(x) __builtin_expect((x), 1)
+#define likely(x) __builtin_expect((x), 1)
#endif
#ifndef unlikely
-#define unlikely(x) __builtin_expect((x), 0)
+#define unlikely(x) __builtin_expect((x), 0)
#endif
#define __aligned_u64 uint64_t __aligned(8)
@@ -112,7 +101,7 @@
#define irdma_print(S, ...) printf("%s:%d "S, __FUNCTION__, __LINE__, ##__VA_ARGS__)
#define irdma_debug_buf(dev, mask, desc, buf, size) \
do { \
- u32 i; \
+ u32 i; \
if (!((mask) & (dev)->debug_mask)) { \
break; \
} \
@@ -122,20 +111,20 @@
irdma_debug(dev, mask, "index %03d val: %016lx\n", i, ((unsigned long *)(buf))[i / 8]); \
} while(0)
-#define irdma_debug(h, m, s, ...) \
-do { \
- if (!(h)) { \
- if ((m) == IRDMA_DEBUG_INIT) \
+#define irdma_debug(h, m, s, ...) \
+do { \
+ if (!(h)) { \
+ if ((m) == IRDMA_DEBUG_INIT) \
printf("irdma INIT " s, ##__VA_ARGS__); \
- } else if (((m) & (h)->debug_mask)) { \
- printf("irdma " s, ##__VA_ARGS__); \
- } \
+ } else if (((m) & (h)->debug_mask)) { \
+ printf("irdma " s, ##__VA_ARGS__); \
+ } \
} while (0)
extern unsigned int irdma_dbg;
-#define libirdma_debug(fmt, args...) \
-do { \
- if (irdma_dbg) \
- printf("libirdma-%s: " fmt, __func__, ##args); \
+#define libirdma_debug(fmt, args...) \
+do { \
+ if (irdma_dbg) \
+ printf("libirdma-%s: " fmt, __func__, ##args); \
} while (0)
#define irdma_dev_err(ibdev, fmt, ...) \
pr_err("%s:%s:%d ERR "fmt, (ibdev)->name, __func__, __LINE__, ##__VA_ARGS__)
@@ -144,18 +133,6 @@
#define irdma_dev_info(a, b, ...) printf(b, ##__VA_ARGS__)
#define irdma_pr_warn printf
-#define dump_struct(s, sz, name) \
-do { \
- unsigned char *a; \
- printf("%s %u", (name), (unsigned int)(sz)); \
- for (a = (unsigned char*)(s); a < (unsigned char *)(s) + (sz) ; a ++) { \
- if ((u64)a % 8 == 0) \
- printf("\n%p ", a); \
- printf("%2x ", *a); \
- } \
- printf("\n"); \
-}while(0)
-
/*
* debug definition end
*/
@@ -179,13 +156,6 @@
#define rt_tos2priority(tos) (tos >> 5)
#define ah_attr_to_dmac(attr) ((attr).dmac)
-#define kc_ib_modify_qp_is_ok(cur_state, next_state, type, mask, ll) \
- ib_modify_qp_is_ok(cur_state, next_state, type, mask)
-#define kc_typeq_ib_wr const
-#define kc_ifp_find ip_ifp_find
-#define kc_ifp6_find ip6_ifp_find
-#define kc_rdma_gid_attr_network_type(sgid_attr, gid_type, gid) \
- ib_gid_to_network_type(gid_type, gid)
#define irdma_del_timer_compat(tt) del_timer((tt))
#define IRDMA_TAILQ_FOREACH CK_STAILQ_FOREACH
#define IRDMA_TAILQ_FOREACH_SAFE CK_STAILQ_FOREACH_SAFE
@@ -193,36 +163,36 @@
static inline void db_wr32(__u32 val, __u32 *wqe_word)
{
- *wqe_word = val;
+ *wqe_word = val;
}
void *hw_to_dev(struct irdma_hw *hw);
struct irdma_dma_mem {
- void *va;
- u64 pa;
+ void *va;
+ u64 pa;
bus_dma_tag_t tag;
bus_dmamap_t map;
bus_dma_segment_t seg;
bus_size_t size;
- int nseg;
- int flags;
+ int nseg;
+ int flags;
};
struct irdma_virt_mem {
- void *va;
- u32 size;
+ void *va;
+ u32 size;
};
#ifndef verbs_mr
enum ibv_mr_type {
- IBV_MR_TYPE_MR,
- IBV_MR_TYPE_NULL_MR,
+ IBV_MR_TYPE_MR,
+ IBV_MR_TYPE_NULL_MR,
};
struct verbs_mr {
- struct ibv_mr ibv_mr;
- enum ibv_mr_type mr_type;
+ struct ibv_mr ibv_mr;
+ enum ibv_mr_type mr_type;
int access;
};
#define verbs_get_mr(mr) container_of((mr), struct verbs_mr, ibv_mr)
diff --git a/sys/dev/ice/ice_rdma.h b/sys/dev/ice/ice_rdma.h
--- a/sys/dev/ice/ice_rdma.h
+++ b/sys/dev/ice/ice_rdma.h
@@ -65,7 +65,7 @@
* considered stable.
*/
#define ICE_RDMA_MAJOR_VERSION 1
-#define ICE_RDMA_MINOR_VERSION 0
+#define ICE_RDMA_MINOR_VERSION 1
#define ICE_RDMA_PATCH_VERSION 0
/**
@@ -274,18 +274,19 @@
uint64_t baudrate;
};
/* MTU change event */
- struct {
- int mtu;
- };
+ int mtu;
/*
* TC/QoS/DCB change event
- * RESET event use prep variable only
* prep: if true, this is a pre-event, post-event otherwise
*/
struct {
struct ice_qos_params port_qos;
bool prep;
};
+ /*
+ * CRIT_ERR event
+ */
+ uint32_t oicr_reg;
};
};
diff --git a/sys/dev/ice/ice_rdma.c b/sys/dev/ice/ice_rdma.c
--- a/sys/dev/ice/ice_rdma.c
+++ b/sys/dev/ice/ice_rdma.c
@@ -122,11 +122,10 @@
{
struct ice_softc *sc = ice_rdma_peer_to_sc(peer);
- /*
- * Request that the driver re-initialize by bringing the interface
- * down and up.
- */
- ice_request_stack_reinit(sc);
+ /* Tell the base driver that RDMA is requesting a PFR */
+ ice_set_state(&sc->state, ICE_STATE_RESET_PFR_REQ);
+
+ /* XXX: Base driver will notify RDMA when it's done */
return (0);
}
@@ -332,6 +331,7 @@
switch(req->type) {
case ICE_RDMA_EVENT_RESET:
+ ice_rdma_pf_reset(peer);
break;
case ICE_RDMA_EVENT_QSET_REGISTER:
ice_rdma_qset_register_request(peer, &req->res);
@@ -866,3 +866,52 @@
IRDMA_EVENT_HANDLER(peer, &event);
sx_xunlock(&ice_rdma.mtx);
}
+
+/**
+ * ice_rdma_notify_pe_intr - notify irdma on incoming interrupts regarding PE
+ * @sc: the ice driver softc
+ * @oicr: interrupt cause
+ *
+ * Pass the information about received interrupt to RDMA driver if it was
+ * relating to PE. Specifically PE_CRITERR and HMC_ERR.
+ * The irdma driver shall decide what should be done upon these interrupts.
+ */
+void
+ice_rdma_notify_pe_intr(struct ice_softc *sc, uint32_t oicr)
+{
+ struct ice_rdma_peer *peer = &sc->rdma_entry.peer;
+ struct ice_rdma_event event;
+
+ memset(&event, 0, sizeof(struct ice_rdma_event));
+ event.type = ICE_RDMA_EVENT_CRIT_ERR;
+ event.oicr_reg = oicr;
+
+ sx_xlock(&ice_rdma.mtx);
+ if (sc->rdma_entry.attached && ice_rdma.registered)
+ IRDMA_EVENT_HANDLER(peer, &event);
+ sx_xunlock(&ice_rdma.mtx);
+}
+
+/**
+ * ice_rdma_notify_reset - notify irdma on incoming pf-reset
+ * @sc: the ice driver softc
+ *
+ * Inform irdma driver of an incoming PF reset.
+ * The irdma driver shall set its state to reset, and avoid using CQP
+ * anymore. Next step should be to call ice_rdma_pf_stop in order to
+ * remove resources.
+ */
+void
+ice_rdma_notify_reset(struct ice_softc *sc)
+{
+ struct ice_rdma_peer *peer = &sc->rdma_entry.peer;
+ struct ice_rdma_event event;
+
+ memset(&event, 0, sizeof(struct ice_rdma_event));
+ event.type = ICE_RDMA_EVENT_RESET;
+
+ sx_xlock(&ice_rdma.mtx);
+ if (sc->rdma_entry.attached && ice_rdma.registered)
+ IRDMA_EVENT_HANDLER(peer, &event);
+ sx_xunlock(&ice_rdma.mtx);
+}
diff --git a/sys/dev/ice/ice_rdma_internal.h b/sys/dev/ice/ice_rdma_internal.h
--- a/sys/dev/ice/ice_rdma_internal.h
+++ b/sys/dev/ice/ice_rdma_internal.h
@@ -99,4 +99,6 @@
void ice_rdma_link_change(struct ice_softc *sc, int linkstate, uint64_t baudrate);
void ice_rdma_notify_dcb_qos_change(struct ice_softc *sc);
void ice_rdma_dcb_qos_update(struct ice_softc *sc, struct ice_port_info *pi);
+void ice_rdma_notify_pe_intr(struct ice_softc *sc, uint32_t oicr);
+void ice_rdma_notify_reset(struct ice_softc *sc);
#endif
diff --git a/sys/dev/ice/if_ice_iflib.c b/sys/dev/ice/if_ice_iflib.c
--- a/sys/dev/ice/if_ice_iflib.c
+++ b/sys/dev/ice/if_ice_iflib.c
@@ -1279,9 +1279,11 @@
ice_set_state(&sc->state, ICE_STATE_RESET_PFR_REQ);
}
- if (oicr & PFINT_OICR_PE_CRITERR_M) {
- device_printf(dev, "Critical Protocol Engine Error detected!\n");
- ice_set_state(&sc->state, ICE_STATE_RESET_PFR_REQ);
+ if (oicr & (PFINT_OICR_PE_CRITERR_M | PFINT_OICR_HMC_ERR_M)) {
+ if (oicr & PFINT_OICR_HMC_ERR_M)
+ /* Log the HMC errors */
+ ice_log_hmc_error(hw, dev);
+ ice_rdma_notify_pe_intr(sc, oicr);
}
if (oicr & PFINT_OICR_PCI_EXCEPTION_M) {
@@ -1289,11 +1291,6 @@
ice_set_state(&sc->state, ICE_STATE_RESET_PFR_REQ);
}
- if (oicr & PFINT_OICR_HMC_ERR_M) {
- /* Log the HMC errors, but don't disable the interrupt cause */
- ice_log_hmc_error(hw, dev);
- }
-
return (FILTER_SCHEDULE_THREAD);
}
@@ -2299,6 +2296,8 @@
if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE))
return;
+ /* inform the RDMA client */
+ ice_rdma_notify_reset(sc);
/* stop the RDMA client */
ice_rdma_pf_stop(sc);
diff --git a/sys/dev/irdma/fbsd_kcompat.h b/sys/dev/irdma/fbsd_kcompat.h
--- a/sys/dev/irdma/fbsd_kcompat.h
+++ b/sys/dev/irdma/fbsd_kcompat.h
@@ -1,7 +1,7 @@
/*-
* SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB
*
- * Copyright (c) 2021 - 2022 Intel Corporation
+ * Copyright (c) 2021 - 2023 Intel Corporation
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@@ -63,21 +63,12 @@
ibdev.dma_device = (dev)
#define set_max_sge(props, rf) \
((props)->max_sge = (rf)->sc_dev.hw_attrs.uk_attrs.max_hw_wq_frags)
-#define kc_set_props_ip_gid_caps(props) \
- ((props)->port_cap_flags |= IB_PORT_IP_BASED_GIDS)
#define rdma_query_gid(ibdev, port, index, gid) \
ib_get_cached_gid(ibdev, port, index, gid, NULL)
#define kmap(pg) page_address(pg)
#define kmap_local_page(pg) page_address(pg)
#define kunmap(pg)
#define kunmap_local(pg)
-#if __FreeBSD_version >= 1400026
-#define kc_free_lsmm_dereg_mr(iwdev, iwqp) \
- ((iwdev)->ibdev.dereg_mr((iwqp)->lsmm_mr, NULL))
-#else
-#define kc_free_lsmm_dereg_mr(iwdev, iwqp) \
- ((iwdev)->ibdev.dereg_mr((iwqp)->lsmm_mr))
-#endif
#define IB_UVERBS_CQ_FLAGS_TIMESTAMP_COMPLETION IB_CQ_FLAGS_TIMESTAMP_COMPLETION
#if __FreeBSD_version < 1400026
@@ -100,6 +91,7 @@
struct irdma_tunable_info {
struct sysctl_ctx_list irdma_sysctl_ctx;
struct sysctl_oid *irdma_sysctl_tree;
+ struct sysctl_oid *sws_sysctl_tree;
char drv_ver[IRDMA_VER_LEN];
u8 roce_ena;
};
@@ -211,8 +203,6 @@
int kc_irdma_set_roce_cm_info(struct irdma_qp *iwqp,
struct ib_qp_attr *attr,
u16 *vlan_id);
-struct irdma_device *kc_irdma_get_device(struct ifnet *netdev);
-void kc_irdma_put_device(struct irdma_device *iwdev);
void kc_set_loc_seq_num_mss(struct irdma_cm_node *cm_node);
u16 kc_rdma_get_udp_sport(u32 fl, u32 lqpn, u32 rqpn);
@@ -230,6 +220,8 @@
int irdma_addr_resolve_neigh_ipv6(struct irdma_cm_node *cm_node, u32 *dest,
int arpindex);
void irdma_dcqcn_tunables_init(struct irdma_pci_f *rf);
+void irdma_sysctl_settings(struct irdma_pci_f *rf);
+void irdma_sw_stats_tunables_init(struct irdma_pci_f *rf);
u32 irdma_create_stag(struct irdma_device *iwdev);
void irdma_free_stag(struct irdma_device *iwdev, u32 stag);
diff --git a/sys/dev/irdma/fbsd_kcompat.c b/sys/dev/irdma/fbsd_kcompat.c
--- a/sys/dev/irdma/fbsd_kcompat.c
+++ b/sys/dev/irdma/fbsd_kcompat.c
@@ -1,7 +1,7 @@
/*-
* SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB
*
- * Copyright (c) 2021 - 2022 Intel Corporation
+ * Copyright (c) 2021 - 2023 Intel Corporation
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@@ -274,12 +274,15 @@
{
struct ifnet *ifp = iwdev->netdev;
struct ifnet *ifv;
+ struct epoch_tracker et;
int i;
irdma_add_ipv4_addr(iwdev, ifp);
irdma_add_ipv6_addr(iwdev, ifp);
for (i = 0; if_getvlantrunk(ifp) != NULL && i < VLAN_N_VID; ++i) {
+ NET_EPOCH_ENTER(et);
ifv = VLAN_DEVAT(ifp, i);
+ NET_EPOCH_EXIT(et);
if (!ifv)
continue;
irdma_add_ipv4_addr(iwdev, ifv);
@@ -362,7 +365,8 @@
struct nhop_object *nh;
if (dst_sin->sa_family == AF_INET6)
- nh = fib6_lookup(RT_DEFAULT_FIB, &((struct sockaddr_in6 *)dst_sin)->sin6_addr, 0, NHR_NONE, 0);
+ nh = fib6_lookup(RT_DEFAULT_FIB, &((struct sockaddr_in6 *)dst_sin)->sin6_addr,
+ ((struct sockaddr_in6 *)dst_sin)->sin6_scope_id, NHR_NONE, 0);
else
nh = fib4_lookup(RT_DEFAULT_FIB, ((struct sockaddr_in *)dst_sin)->sin_addr, 0, NHR_NONE, 0);
if (!nh || (nh->nh_ifp != netdev &&
@@ -592,6 +596,188 @@
return 0;
}
+enum irdma_cqp_stats_info {
+ IRDMA_CQP_REQ_CMDS = 28,
+ IRDMA_CQP_CMPL_CMDS = 29
+};
+
+static int
+irdma_sysctl_cqp_stats(SYSCTL_HANDLER_ARGS)
+{
+ struct irdma_sc_cqp *cqp = (struct irdma_sc_cqp *)arg1;
+ char rslt[192] = "no cqp available yet";
+ int rslt_size = sizeof(rslt) - 1;
+ int option = (int)arg2;
+
+ if (!cqp) {
+ return sysctl_handle_string(oidp, rslt, sizeof(rslt), req);
+ }
+
+ snprintf(rslt, sizeof(rslt), "");
+ switch (option) {
+ case IRDMA_CQP_REQ_CMDS:
+ snprintf(rslt, rslt_size, "%lu", cqp->requested_ops);
+ break;
+ case IRDMA_CQP_CMPL_CMDS:
+ snprintf(rslt, rslt_size, "%lu", atomic64_read(&cqp->completed_ops));
+ break;
+ }
+
+ return sysctl_handle_string(oidp, rslt, sizeof(rslt), req);
+}
+
+struct irdma_sw_stats_tunable_info {
+ u8 op_type;
+ const char name[32];
+ const char desc[32];
+ uintptr_t value;
+};
+
+static const struct irdma_sw_stats_tunable_info irdma_sws_list[] = {
+ {IRDMA_OP_CEQ_DESTROY, "ceq_destroy", "ceq_destroy", 0},
+ {IRDMA_OP_AEQ_DESTROY, "aeq_destroy", "aeq_destroy", 0},
+ {IRDMA_OP_DELETE_ARP_CACHE_ENTRY, "delete_arp_cache_entry",
+ "delete_arp_cache_entry", 0},
+ {IRDMA_OP_MANAGE_APBVT_ENTRY, "manage_apbvt_entry",
+ "manage_apbvt_entry", 0},
+ {IRDMA_OP_CEQ_CREATE, "ceq_create", "ceq_create", 0},
+ {IRDMA_OP_AEQ_CREATE, "aeq_create", "aeq_create", 0},
+ {IRDMA_OP_MANAGE_QHASH_TABLE_ENTRY, "manage_qhash_table_entry",
+ "manage_qhash_table_entry", 0},
+ {IRDMA_OP_QP_MODIFY, "qp_modify", "qp_modify", 0},
+ {IRDMA_OP_QP_UPLOAD_CONTEXT, "qp_upload_context", "qp_upload_context",
+ 0},
+ {IRDMA_OP_CQ_CREATE, "cq_create", "cq_create", 0},
+ {IRDMA_OP_CQ_DESTROY, "cq_destroy", "cq_destroy", 0},
+ {IRDMA_OP_QP_CREATE, "qp_create", "qp_create", 0},
+ {IRDMA_OP_QP_DESTROY, "qp_destroy", "qp_destroy", 0},
+ {IRDMA_OP_ALLOC_STAG, "alloc_stag", "alloc_stag", 0},
+ {IRDMA_OP_MR_REG_NON_SHARED, "mr_reg_non_shared", "mr_reg_non_shared",
+ 0},
+ {IRDMA_OP_DEALLOC_STAG, "dealloc_stag", "dealloc_stag", 0},
+ {IRDMA_OP_MW_ALLOC, "mw_alloc", "mw_alloc", 0},
+ {IRDMA_OP_QP_FLUSH_WQES, "qp_flush_wqes", "qp_flush_wqes", 0},
+ {IRDMA_OP_ADD_ARP_CACHE_ENTRY, "add_arp_cache_entry",
+ "add_arp_cache_entry", 0},
+ {IRDMA_OP_MANAGE_PUSH_PAGE, "manage_push_page", "manage_push_page", 0},
+ {IRDMA_OP_UPDATE_PE_SDS, "update_pe_sds", "update_pe_sds", 0},
+ {IRDMA_OP_MANAGE_HMC_PM_FUNC_TABLE, "manage_hmc_pm_func_table",
+ "manage_hmc_pm_func_table", 0},
+ {IRDMA_OP_SUSPEND, "suspend", "suspend", 0},
+ {IRDMA_OP_RESUME, "resume", "resume", 0},
+ {IRDMA_OP_MANAGE_VCHNL_REQ_PBLE_BP, "manage_vchnl_req_pble_bp",
+ "manage_vchnl_req_pble_bp", 0},
+ {IRDMA_OP_QUERY_FPM_VAL, "query_fpm_val", "query_fpm_val", 0},
+ {IRDMA_OP_COMMIT_FPM_VAL, "commit_fpm_val", "commit_fpm_val", 0},
+ {IRDMA_OP_AH_CREATE, "ah_create", "ah_create", 0},
+ {IRDMA_OP_AH_MODIFY, "ah_modify", "ah_modify", 0},
+ {IRDMA_OP_AH_DESTROY, "ah_destroy", "ah_destroy", 0},
+ {IRDMA_OP_MC_CREATE, "mc_create", "mc_create", 0},
+ {IRDMA_OP_MC_DESTROY, "mc_destroy", "mc_destroy", 0},
+ {IRDMA_OP_MC_MODIFY, "mc_modify", "mc_modify", 0},
+ {IRDMA_OP_STATS_ALLOCATE, "stats_allocate", "stats_allocate", 0},
+ {IRDMA_OP_STATS_FREE, "stats_free", "stats_free", 0},
+ {IRDMA_OP_STATS_GATHER, "stats_gather", "stats_gather", 0},
+ {IRDMA_OP_WS_ADD_NODE, "ws_add_node", "ws_add_node", 0},
+ {IRDMA_OP_WS_MODIFY_NODE, "ws_modify_node", "ws_modify_node", 0},
+ {IRDMA_OP_WS_DELETE_NODE, "ws_delete_node", "ws_delete_node", 0},
+ {IRDMA_OP_WS_FAILOVER_START, "ws_failover_start", "ws_failover_start",
+ 0},
+ {IRDMA_OP_WS_FAILOVER_COMPLETE, "ws_failover_complete",
+ "ws_failover_complete", 0},
+ {IRDMA_OP_SET_UP_MAP, "set_up_map", "set_up_map", 0},
+ {IRDMA_OP_GEN_AE, "gen_ae", "gen_ae", 0},
+ {IRDMA_OP_QUERY_RDMA_FEATURES, "query_rdma_features",
+ "query_rdma_features", 0},
+ {IRDMA_OP_ALLOC_LOCAL_MAC_ENTRY, "alloc_local_mac_entry",
+ "alloc_local_mac_entry", 0},
+ {IRDMA_OP_ADD_LOCAL_MAC_ENTRY, "add_local_mac_entry",
+ "add_local_mac_entry", 0},
+ {IRDMA_OP_DELETE_LOCAL_MAC_ENTRY, "delete_local_mac_entry",
+ "delete_local_mac_entry", 0},
+ {IRDMA_OP_CQ_MODIFY, "cq_modify", "cq_modify", 0}
+};
+
+static const struct irdma_sw_stats_tunable_info irdma_cmcs_list[] = {
+ {0, "cm_nodes_created", "cm_nodes_created",
+ offsetof(struct irdma_cm_core, stats_nodes_created)},
+ {0, "cm_nodes_destroyed", "cm_nodes_destroyed",
+ offsetof(struct irdma_cm_core, stats_nodes_destroyed)},
+ {0, "cm_listen_created", "cm_listen_created",
+ offsetof(struct irdma_cm_core, stats_listen_created)},
+ {0, "cm_listen_destroyed", "cm_listen_destroyed",
+ offsetof(struct irdma_cm_core, stats_listen_destroyed)},
+ {0, "cm_listen_nodes_created", "cm_listen_nodes_created",
+ offsetof(struct irdma_cm_core, stats_listen_nodes_created)},
+ {0, "cm_listen_nodes_destroyed", "cm_listen_nodes_destroyed",
+ offsetof(struct irdma_cm_core, stats_listen_nodes_destroyed)},
+ {0, "cm_lpbs", "cm_lpbs", offsetof(struct irdma_cm_core, stats_lpbs)},
+ {0, "cm_accepts", "cm_accepts", offsetof(struct irdma_cm_core,
+ stats_accepts)},
+ {0, "cm_rejects", "cm_rejects", offsetof(struct irdma_cm_core,
+ stats_rejects)},
+ {0, "cm_connect_errs", "cm_connect_errs",
+ offsetof(struct irdma_cm_core, stats_connect_errs)},
+ {0, "cm_passive_errs", "cm_passive_errs",
+ offsetof(struct irdma_cm_core, stats_passive_errs)},
+ {0, "cm_pkt_retrans", "cm_pkt_retrans", offsetof(struct irdma_cm_core,
+ stats_pkt_retrans)},
+ {0, "cm_backlog_drops", "cm_backlog_drops",
+ offsetof(struct irdma_cm_core, stats_backlog_drops)},
+};
+
+static const struct irdma_sw_stats_tunable_info irdma_ilqs32_list[] = {
+ {0, "ilq_avail_buf_count", "ilq_avail_buf_count",
+ offsetof(struct irdma_puda_rsrc, avail_buf_count)},
+ {0, "ilq_alloc_buf_count", "ilq_alloc_buf_count",
+ offsetof(struct irdma_puda_rsrc, alloc_buf_count)}
+};
+
+static const struct irdma_sw_stats_tunable_info irdma_ilqs_list[] = {
+ {0, "ilq_stats_buf_alloc_fail", "ilq_stats_buf_alloc_fail",
+ offsetof(struct irdma_puda_rsrc, stats_buf_alloc_fail)},
+ {0, "ilq_stats_pkt_rcvd", "ilq_stats_pkt_rcvd",
+ offsetof(struct irdma_puda_rsrc, stats_pkt_rcvd)},
+ {0, "ilq_stats_pkt_sent", "ilq_stats_pkt_sent",
+ offsetof(struct irdma_puda_rsrc, stats_pkt_sent)},
+ {0, "ilq_stats_rcvd_pkt_err", "ilq_stats_rcvd_pkt_err",
+ offsetof(struct irdma_puda_rsrc, stats_rcvd_pkt_err)},
+ {0, "ilq_stats_sent_pkt_q", "ilq_stats_sent_pkt_q",
+ offsetof(struct irdma_puda_rsrc, stats_sent_pkt_q)}
+};
+
+static const struct irdma_sw_stats_tunable_info irdma_ieqs32_list[] = {
+ {0, "ieq_avail_buf_count", "ieq_avail_buf_count",
+ offsetof(struct irdma_puda_rsrc, avail_buf_count)},
+ {0, "ieq_alloc_buf_count", "ieq_alloc_buf_count",
+ offsetof(struct irdma_puda_rsrc, alloc_buf_count)}
+};
+
+static const struct irdma_sw_stats_tunable_info irdma_ieqs_list[] = {
+ {0, "ieq_stats_buf_alloc_fail", "ieq_stats_buf_alloc_fail",
+ offsetof(struct irdma_puda_rsrc, stats_buf_alloc_fail)},
+ {0, "ieq_stats_pkt_rcvd", "ieq_stats_pkt_rcvd",
+ offsetof(struct irdma_puda_rsrc, stats_pkt_rcvd)},
+ {0, "ieq_stats_pkt_sent", "ieq_stats_pkt_sent",
+ offsetof(struct irdma_puda_rsrc, stats_pkt_sent)},
+ {0, "ieq_stats_rcvd_pkt_err", "ieq_stats_rcvd_pkt_err",
+ offsetof(struct irdma_puda_rsrc, stats_rcvd_pkt_err)},
+ {0, "ieq_stats_sent_pkt_q", "ieq_stats_sent_pkt_q",
+ offsetof(struct irdma_puda_rsrc, stats_sent_pkt_q)},
+ {0, "ieq_stats_bad_qp_id", "ieq_stats_bad_qp_id",
+ offsetof(struct irdma_puda_rsrc, stats_bad_qp_id)},
+ {0, "ieq_fpdu_processed", "ieq_fpdu_processed",
+ offsetof(struct irdma_puda_rsrc, fpdu_processed)},
+ {0, "ieq_bad_seq_num", "ieq_bad_seq_num",
+ offsetof(struct irdma_puda_rsrc, bad_seq_num)},
+ {0, "ieq_crc_err", "ieq_crc_err", offsetof(struct irdma_puda_rsrc,
+ crc_err)},
+ {0, "ieq_pmode_count", "ieq_pmode_count",
+ offsetof(struct irdma_puda_rsrc, pmode_count)},
+ {0, "ieq_partials_handled", "ieq_partials_handled",
+ offsetof(struct irdma_puda_rsrc, partials_handled)},
+};
+
/**
* irdma_dcqcn_tunables_init - create tunables for dcqcn settings
* @rf: RDMA PCI function
@@ -638,7 +824,7 @@
rf->dcqcn_params.dcqcn_t = 0x37;
SYSCTL_ADD_U16(&rf->tun_info.irdma_sysctl_ctx, irdma_sysctl_oid_list,
OID_AUTO, "dcqcn_T", CTLFLAG_RDTUN, &rf->dcqcn_params.dcqcn_t, 0,
- "set number of usecs that should elapse before increasing the CWND in DCQCN mode, default=0x37");
+ "number of us to elapse before increasing the CWND in DCQCN mode, default=0x37");
rf->dcqcn_params.dcqcn_b = 0x249f0;
SYSCTL_ADD_U32(&rf->tun_info.irdma_sysctl_ctx, irdma_sysctl_oid_list,
@@ -664,6 +850,90 @@
"set minimum time between 2 consecutive rate reductions for a single flow, default=50");
}
+/**
+ * irdma_sysctl_settings - sysctl runtime settings init
+ * @rf: RDMA PCI function
+ */
+void
+irdma_sysctl_settings(struct irdma_pci_f *rf)
+{
+ struct sysctl_oid_list *irdma_sysctl_oid_list;
+
+ irdma_sysctl_oid_list = SYSCTL_CHILDREN(rf->tun_info.irdma_sysctl_tree);
+
+ SYSCTL_ADD_BOOL(&rf->tun_info.irdma_sysctl_ctx, irdma_sysctl_oid_list,
+ OID_AUTO, "upload_context", CTLFLAG_RWTUN,
+ &irdma_upload_context, 0,
+ "allow for generating QP's upload context, default=0");
+}
+
+void
+irdma_sw_stats_tunables_init(struct irdma_pci_f *rf)
+{
+ struct sysctl_oid_list *sws_oid_list;
+ struct sysctl_ctx_list *irdma_ctx = &rf->tun_info.irdma_sysctl_ctx;
+ struct irdma_sc_dev *dev = &rf->sc_dev;
+ struct irdma_cm_core *cm_core = &rf->iwdev->cm_core;
+ struct irdma_puda_rsrc *ilq = rf->iwdev->vsi.ilq;
+ struct irdma_puda_rsrc *ieq = rf->iwdev->vsi.ieq;
+ u64 *ll_ptr;
+ u32 *l_ptr;
+ int cqp_stat_cnt = sizeof(irdma_sws_list) / sizeof(struct irdma_sw_stats_tunable_info);
+ int cmcore_stat_cnt = sizeof(irdma_cmcs_list) / sizeof(struct irdma_sw_stats_tunable_info);
+ int ilqs_stat_cnt = sizeof(irdma_ilqs_list) / sizeof(struct irdma_sw_stats_tunable_info);
+ int ilqs32_stat_cnt = sizeof(irdma_ilqs32_list) / sizeof(struct irdma_sw_stats_tunable_info);
+ int ieqs_stat_cnt = sizeof(irdma_ieqs_list) / sizeof(struct irdma_sw_stats_tunable_info);
+ int ieqs32_stat_cnt = sizeof(irdma_ieqs32_list) / sizeof(struct irdma_sw_stats_tunable_info);
+ int i;
+
+ sws_oid_list = SYSCTL_CHILDREN(rf->tun_info.sws_sysctl_tree);
+
+ for (i = 0; i < cqp_stat_cnt; ++i) {
+ SYSCTL_ADD_U64(irdma_ctx, sws_oid_list, OID_AUTO,
+ irdma_sws_list[i].name, CTLFLAG_RD,
+ &dev->cqp_cmd_stats[irdma_sws_list[i].op_type],
+ 0, irdma_sws_list[i].desc);
+ }
+ SYSCTL_ADD_PROC(irdma_ctx, sws_oid_list, OID_AUTO,
+ "req_cmds", CTLFLAG_RD | CTLTYPE_STRING,
+ dev->cqp, IRDMA_CQP_REQ_CMDS, irdma_sysctl_cqp_stats, "A",
+ "req_cmds");
+ SYSCTL_ADD_PROC(irdma_ctx, sws_oid_list, OID_AUTO,
+ "cmpl_cmds", CTLFLAG_RD | CTLTYPE_STRING,
+ dev->cqp, IRDMA_CQP_CMPL_CMDS, irdma_sysctl_cqp_stats, "A",
+ "cmpl_cmds");
+ for (i = 0; i < cmcore_stat_cnt; ++i) {
+ ll_ptr = (u64 *)((uintptr_t)cm_core + irdma_cmcs_list[i].value);
+ SYSCTL_ADD_U64(irdma_ctx, sws_oid_list, OID_AUTO,
+ irdma_cmcs_list[i].name, CTLFLAG_RD, ll_ptr,
+ 0, irdma_cmcs_list[i].desc);
+ }
+ for (i = 0; ilq && i < ilqs_stat_cnt; ++i) {
+ ll_ptr = (u64 *)((uintptr_t)ilq + irdma_ilqs_list[i].value);
+ SYSCTL_ADD_U64(irdma_ctx, sws_oid_list, OID_AUTO,
+ irdma_ilqs_list[i].name, CTLFLAG_RD, ll_ptr,
+ 0, irdma_ilqs_list[i].desc);
+ }
+ for (i = 0; ilq && i < ilqs32_stat_cnt; ++i) {
+ l_ptr = (u32 *)((uintptr_t)ilq + irdma_ilqs32_list[i].value);
+ SYSCTL_ADD_U32(irdma_ctx, sws_oid_list, OID_AUTO,
+ irdma_ilqs32_list[i].name, CTLFLAG_RD, l_ptr,
+ 0, irdma_ilqs32_list[i].desc);
+ }
+ for (i = 0; ieq && i < ieqs_stat_cnt; ++i) {
+ ll_ptr = (u64 *)((uintptr_t)ieq + irdma_ieqs_list[i].value);
+ SYSCTL_ADD_U64(irdma_ctx, sws_oid_list, OID_AUTO,
+ irdma_ieqs_list[i].name, CTLFLAG_RD, ll_ptr,
+ 0, irdma_ieqs_list[i].desc);
+ }
+ for (i = 0; ieq && i < ieqs32_stat_cnt; ++i) {
+ l_ptr = (u32 *)((uintptr_t)ieq + irdma_ieqs32_list[i].value);
+ SYSCTL_ADD_U32(irdma_ctx, sws_oid_list, OID_AUTO,
+ irdma_ieqs32_list[i].name, CTLFLAG_RD, l_ptr,
+ 0, irdma_ieqs32_list[i].desc);
+ }
+}
+
/**
* irdma_dmamap_cb - callback for bus_dmamap_load
*/
@@ -763,12 +1033,6 @@
return 0;
}
-inline void
-irdma_prm_rem_bitmapmem(struct irdma_hw *hw, struct irdma_chunk *chunk)
-{
- kfree(chunk->bitmapmem.va);
-}
-
void
irdma_cleanup_dead_qps(struct irdma_sc_vsi *vsi)
{
diff --git a/sys/dev/irdma/icrdma.c b/sys/dev/irdma/icrdma.c
--- a/sys/dev/irdma/icrdma.c
+++ b/sys/dev/irdma/icrdma.c
@@ -1,7 +1,7 @@
/*-
* SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB
*
- * Copyright (c) 2021 - 2022 Intel Corporation
+ * Copyright (c) 2021 - 2023 Intel Corporation
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@@ -53,7 +53,7 @@
/**
* Driver version
*/
-char irdma_driver_version[] = "1.1.11-k";
+char irdma_driver_version[] = "1.2.17-k";
/**
* irdma_init_tunable - prepare tunables
@@ -63,50 +63,56 @@
static void
irdma_init_tunable(struct irdma_pci_f *rf, uint8_t pf_id)
{
- struct sysctl_oid_list *irdma_sysctl_oid_list;
+ struct sysctl_oid_list *irdma_oid_list;
+ struct irdma_tunable_info *t_info = &rf->tun_info;
char pf_name[16];
snprintf(pf_name, 15, "irdma%d", pf_id);
- sysctl_ctx_init(&rf->tun_info.irdma_sysctl_ctx);
+ sysctl_ctx_init(&t_info->irdma_sysctl_ctx);
- rf->tun_info.irdma_sysctl_tree = SYSCTL_ADD_NODE(&rf->tun_info.irdma_sysctl_ctx,
- SYSCTL_STATIC_CHILDREN(_dev),
- OID_AUTO, pf_name, CTLFLAG_RD,
- NULL, "");
+ t_info->irdma_sysctl_tree = SYSCTL_ADD_NODE(&t_info->irdma_sysctl_ctx,
+ SYSCTL_STATIC_CHILDREN(_dev),
+ OID_AUTO, pf_name,
+ CTLFLAG_RD, NULL, "");
- irdma_sysctl_oid_list = SYSCTL_CHILDREN(rf->tun_info.irdma_sysctl_tree);
+ irdma_oid_list = SYSCTL_CHILDREN(t_info->irdma_sysctl_tree);
+ t_info->sws_sysctl_tree = SYSCTL_ADD_NODE(&t_info->irdma_sysctl_ctx,
+ irdma_oid_list, OID_AUTO,
+ "sw_stats", CTLFLAG_RD,
+ NULL, "");
/*
* debug mask setting
*/
- SYSCTL_ADD_S32(&rf->tun_info.irdma_sysctl_ctx, irdma_sysctl_oid_list,
+ SYSCTL_ADD_S32(&t_info->irdma_sysctl_ctx, irdma_oid_list,
OID_AUTO, "debug", CTLFLAG_RWTUN, &rf->sc_dev.debug_mask,
0, "irdma debug");
/*
* RoCEv2/iWARP setting RoCEv2 the default mode
*/
- rf->tun_info.roce_ena = 1;
- SYSCTL_ADD_U8(&rf->tun_info.irdma_sysctl_ctx, irdma_sysctl_oid_list, OID_AUTO,
- "roce_enable", CTLFLAG_RDTUN, &rf->tun_info.roce_ena, 0,
+ t_info->roce_ena = 1;
+ SYSCTL_ADD_U8(&t_info->irdma_sysctl_ctx, irdma_oid_list, OID_AUTO,
+ "roce_enable", CTLFLAG_RDTUN, &t_info->roce_ena, 0,
"RoCEv2 mode enable");
rf->protocol_used = IRDMA_IWARP_PROTOCOL_ONLY;
- if (rf->tun_info.roce_ena == 1)
+ if (t_info->roce_ena == 1)
rf->protocol_used = IRDMA_ROCE_PROTOCOL_ONLY;
- else if (rf->tun_info.roce_ena != 0)
+ else if (t_info->roce_ena != 0)
printf("%s:%d wrong roce_enable value (%d), using iWARP\n",
- __func__, __LINE__, rf->tun_info.roce_ena);
+ __func__, __LINE__, t_info->roce_ena);
printf("%s:%d protocol: %s, roce_enable value: %d\n", __func__, __LINE__,
(rf->protocol_used == IRDMA_IWARP_PROTOCOL_ONLY) ? "iWARP" : "RoCEv2",
- rf->tun_info.roce_ena);
+ t_info->roce_ena);
- snprintf(rf->tun_info.drv_ver, IRDMA_VER_LEN, "%s", irdma_driver_version);
- SYSCTL_ADD_STRING(&rf->tun_info.irdma_sysctl_ctx, irdma_sysctl_oid_list,
- OID_AUTO, "drv_ver", CTLFLAG_RDTUN, rf->tun_info.drv_ver,
+ snprintf(t_info->drv_ver, IRDMA_VER_LEN, "%s", irdma_driver_version);
+ SYSCTL_ADD_STRING(&t_info->irdma_sysctl_ctx, irdma_oid_list,
+ OID_AUTO, "drv_ver", CTLFLAG_RDTUN, t_info->drv_ver,
IRDMA_VER_LEN, "driver version");
irdma_dcqcn_tunables_init(rf);
+ irdma_sysctl_settings(rf);
}
/**
@@ -121,8 +127,6 @@
spin_lock_irqsave(&irdma_handler_lock, flags);
list_for_each_entry(hdl, &irdma_handlers, list) {
- if (!hdl)
- continue;
if (!hdl->iwdev->rf->peer_info)
continue;
if (hdl->iwdev->rf->peer_info->dev == p_dev->dev) {
@@ -159,9 +163,12 @@
* @qos_info: source, DCB settings structure
*/
static void
-irdma_get_qos_info(struct irdma_l2params *l2params, struct ice_qos_params *qos_info)
+irdma_get_qos_info(struct irdma_pci_f *rf, struct irdma_l2params *l2params,
+ struct ice_qos_params *qos_info)
{
int i;
+ char txt[7][128] = {"", "", "", "", "", "", ""};
+ u8 len;
l2params->num_tc = qos_info->num_tc;
l2params->num_apps = qos_info->num_apps;
@@ -183,33 +190,46 @@
l2params->dscp_mode = true;
memcpy(l2params->dscp_map, qos_info->dscp_map, sizeof(l2params->dscp_map));
}
- printf("%s:%d: l2params settings:\n num_tc %d,\n num_apps %d,\n",
- __func__, __LINE__, l2params->num_tc, l2params->num_apps);
- printf(" vsi_prio_type %d,\n vsi_rel_bw %d,\n egress_virt_up:",
- l2params->vsi_prio_type, l2params->vsi_rel_bw);
- for (i = 0; i < l2params->num_tc; i++)
- printf(" %d", l2params->tc_info[i].egress_virt_up);
- printf("\n ingress_virt_up:");
- for (i = 0; i < l2params->num_tc; i++)
- printf(" %d", l2params->tc_info[i].ingress_virt_up);
- printf("\n prio_type:");
- for (i = 0; i < l2params->num_tc; i++)
- printf(" %d", l2params->tc_info[i].prio_type);
- printf("\n rel_bw:");
- for (i = 0; i < l2params->num_tc; i++)
- printf(" %d", l2params->tc_info[i].rel_bw);
- printf("\n tc_ctx:");
- for (i = 0; i < l2params->num_tc; i++)
- printf(" %lu", l2params->tc_info[i].tc_ctx);
- printf("\n up2tc:");
+ if (!(rf->sc_dev.debug_mask & IRDMA_DEBUG_DCB))
+ return;
+ for (i = 0; i < l2params->num_tc; i++) {
+ len = strlen(txt[0]);
+ snprintf(txt[0] + len, sizeof(txt[0]) - 5, " %d",
+ l2params->tc_info[i].egress_virt_up);
+ len = strlen(txt[1]);
+ snprintf(txt[1] + len, sizeof(txt[1]) - 5, " %d",
+ l2params->tc_info[i].ingress_virt_up);
+ len = strlen(txt[2]);
+ snprintf(txt[2] + len, sizeof(txt[2]) - 5, " %d",
+ l2params->tc_info[i].prio_type);
+ len = strlen(txt[3]);
+ snprintf(txt[3] + len, sizeof(txt[3]) - 5, " %d",
+ l2params->tc_info[i].rel_bw);
+ len = strlen(txt[4]);
+ snprintf(txt[4] + len, sizeof(txt[4]) - 5, " %lu",
+ l2params->tc_info[i].tc_ctx);
+ }
+ len = strlen(txt[5]);
for (i = 0; i < IRDMA_MAX_USER_PRIORITY; i++)
- printf(" %d", l2params->up2tc[i]);
- printf(" dscp_mode: %d,\n", l2params->dscp_mode);
+ len += snprintf(txt[5] + len, sizeof(txt[5]) - 5, " %d",
+ l2params->up2tc[i]);
+ len = strlen(txt[6]);
for (i = 0; i < IRDMA_DSCP_NUM_VAL; i++)
- printf(" %d", l2params->dscp_map[i]);
- printf("\n");
-
- dump_struct(l2params, sizeof(*l2params), "l2params");
+ len += snprintf(txt[6] + len, sizeof(txt[6]) - 5, " %d",
+ l2params->dscp_map[i]);
+ irdma_debug(&rf->sc_dev, IRDMA_DEBUG_DCB, "num_tc: %d\n", l2params->num_tc);
+ irdma_debug(&rf->sc_dev, IRDMA_DEBUG_DCB, "num_apps: %d\n", l2params->num_apps);
+ irdma_debug(&rf->sc_dev, IRDMA_DEBUG_DCB, "vsi_prio_type: %d\n", l2params->vsi_prio_type);
+ irdma_debug(&rf->sc_dev, IRDMA_DEBUG_DCB, "vsi_rel_bw: %d\n", l2params->vsi_rel_bw);
+ irdma_debug(&rf->sc_dev, IRDMA_DEBUG_DCB, "egress_virt_up: %s\n", txt[0]);
+ irdma_debug(&rf->sc_dev, IRDMA_DEBUG_DCB, "ingress_virt_up:%s\n", txt[1]);
+ irdma_debug(&rf->sc_dev, IRDMA_DEBUG_DCB, "prio_type: %s\n", txt[2]);
+ irdma_debug(&rf->sc_dev, IRDMA_DEBUG_DCB, "rel_bw: %s\n", txt[3]);
+ irdma_debug(&rf->sc_dev, IRDMA_DEBUG_DCB, "tc_ctx: %s\n", txt[4]);
+ irdma_debug(&rf->sc_dev, IRDMA_DEBUG_DCB, "up2tc: %s\n", txt[5]);
+ irdma_debug(&rf->sc_dev, IRDMA_DEBUG_DCB, "dscp_mode: %s\n", txt[6]);
+
+ irdma_debug_buf(&rf->sc_dev, IRDMA_DEBUG_DCB, "l2params", l2params, sizeof(*l2params));
}
/**
@@ -230,6 +250,35 @@
mtu);
}
+/**
+ * irdma_get_event_name - convert type enum to string
+ * @type: event type enum
+ */
+static const char *
+irdma_get_event_name(enum ice_rdma_event_type type)
+{
+ switch (type) {
+ case ICE_RDMA_EVENT_LINK_CHANGE:
+ return "LINK CHANGE";
+ case ICE_RDMA_EVENT_MTU_CHANGE:
+ return "MTU CHANGE";
+ case ICE_RDMA_EVENT_TC_CHANGE:
+ return "TC CHANGE";
+ case ICE_RDMA_EVENT_API_CHANGE:
+ return "API CHANGE";
+ case ICE_RDMA_EVENT_CRIT_ERR:
+ return "CRITICAL ERROR";
+ case ICE_RDMA_EVENT_RESET:
+ return "RESET";
+ case ICE_RDMA_EVENT_QSET_REGISTER:
+ return "QSET REGISTER";
+ case ICE_RDMA_EVENT_VSI_FILTER_UPDATE:
+ return "VSI FILTER UPDATE";
+ default:
+ return "UNKNOWN";
+ }
+}
+
/**
* irdma_event_handler - handling events from lan driver
* @peer: the peer interface structure
@@ -242,9 +291,7 @@
struct irdma_l2params l2params = {};
printf("%s:%d event_handler %s (%x) on pf %d (%d)\n", __func__, __LINE__,
- (event->type == 1) ? "LINK CHANGE" :
- (event->type == 2) ? "MTU CHANGE" :
- (event->type == 3) ? "TC CHANGE" : "UNKNOWN",
+ irdma_get_event_name(event->type),
event->type, peer->pf_id, if_getdunit(peer->ifp));
iwdev = peer_to_iwdev(peer);
if (!iwdev) {
@@ -277,6 +324,12 @@
event->prep ? " " : "not ");
goto done;
}
+ if (!atomic_inc_not_zero(&iwdev->rf->dev_ctx.event_rfcnt)) {
+ printf("%s:%d (%d) EVENT_TC_CHANGE received, but not processed %d\n",
+ __func__, __LINE__, if_getdunit(peer->ifp),
+ atomic_read(&iwdev->rf->dev_ctx.event_rfcnt));
+ break;
+ }
if (event->prep) {
iwdev->vsi.tc_change_pending = true;
irdma_sc_suspend_resume_qps(&iwdev->vsi, IRDMA_OP_SUSPEND);
@@ -287,7 +340,7 @@
printf("%s:%d TC change preparation done\n", __func__, __LINE__);
} else {
l2params.tc_changed = true;
- irdma_get_qos_info(&l2params, &event->port_qos);
+ irdma_get_qos_info(iwdev->rf, &l2params, &event->port_qos);
if (iwdev->rf->protocol_used != IRDMA_IWARP_PROTOCOL_ONLY)
iwdev->dcb_vlan_mode = l2params.num_tc > 1 && !l2params.dscp_mode;
@@ -295,9 +348,32 @@
irdma_change_l2params(&iwdev->vsi, &l2params);
printf("%s:%d TC change done\n", __func__, __LINE__);
}
+ atomic_dec(&iwdev->rf->dev_ctx.event_rfcnt);
break;
case ICE_RDMA_EVENT_CRIT_ERR:
- printf("%s:%d event type received: %d\n", __func__, __LINE__, event->type);
+ if (event->oicr_reg & IRDMAPFINT_OICR_PE_CRITERR_M) {
+ u32 pe_criterr;
+
+#define IRDMA_Q1_RESOURCE_ERR 0x0001024d
+ pe_criterr = readl(iwdev->rf->sc_dev.hw_regs[IRDMA_GLPE_CRITERR]);
+ if (pe_criterr != IRDMA_Q1_RESOURCE_ERR) {
+ irdma_pr_err("critical PE Error, GLPE_CRITERR=0x%08x\n",
+ pe_criterr);
+ iwdev->rf->reset = true;
+ } else {
+ irdma_dev_warn(to_ibdev(&iwdev->rf->sc_dev),
+ "Q1 Resource Check\n");
+ }
+ }
+ if (event->oicr_reg & IRDMAPFINT_OICR_HMC_ERR_M) {
+ irdma_pr_err("HMC Error\n");
+ iwdev->rf->reset = true;
+ }
+ if (iwdev->rf->reset)
+ iwdev->rf->gen_ops.request_reset(iwdev->rf);
+ break;
+ case ICE_RDMA_EVENT_RESET:
+ iwdev->rf->reset = true;
break;
default:
printf("%s:%d event type unsupported: %d\n", __func__, __LINE__, event->type);
@@ -342,6 +418,15 @@
irdma_debug(&rf->sc_dev, IRDMA_DEBUG_INIT,
"Starting deferred closing %d (%d)\n",
rf->peer_info->pf_id, if_getdunit(peer->ifp));
+ atomic_dec(&rf->dev_ctx.event_rfcnt);
+ wait_event_timeout(iwdev->suspend_wq,
+ !atomic_read(&rf->dev_ctx.event_rfcnt),
+ IRDMA_MAX_TIMEOUT);
+ if (atomic_read(&rf->dev_ctx.event_rfcnt) != 0) {
+ printf("%s:%d (%d) waiting for event_rfcnt (%d) timeout, proceed with unload\n",
+ __func__, __LINE__, if_getdunit(peer->ifp),
+ atomic_read(&rf->dev_ctx.event_rfcnt));
+ }
irdma_dereg_ipaddr_event_cb(rf);
irdma_ib_unregister_device(iwdev);
req.type = ICE_RDMA_EVENT_VSI_FILTER_UPDATE;
@@ -353,7 +438,7 @@
irdma_debug(&rf->sc_dev, IRDMA_DEBUG_INIT,
"Starting deferred opening %d (%d)\n",
rf->peer_info->pf_id, if_getdunit(peer->ifp));
- irdma_get_qos_info(&l2params, &peer->initial_qos_info);
+ irdma_get_qos_info(iwdev->rf, &l2params, &peer->initial_qos_info);
if (iwdev->rf->protocol_used != IRDMA_IWARP_PROTOCOL_ONLY)
iwdev->dcb_vlan_mode = l2params.num_tc > 1 && !l2params.dscp_mode;
@@ -370,53 +455,18 @@
irdma_rt_deinit_hw(iwdev);
ib_dealloc_device(&iwdev->ibdev);
}
+ irdma_sw_stats_tunables_init(rf);
req.type = ICE_RDMA_EVENT_VSI_FILTER_UPDATE;
req.enable_filter = true;
IRDMA_DI_REQ_HANDLER(peer, &req);
irdma_reg_ipaddr_event_cb(rf);
+ atomic_inc(&rf->dev_ctx.event_rfcnt);
irdma_debug(&rf->sc_dev, IRDMA_DEBUG_INIT,
"Deferred opening finished %d (%d)\n",
rf->peer_info->pf_id, if_getdunit(peer->ifp));
}
}
-/**
- * irdma_open - Callback for operation open for RDMA device
- * @peer: the new peer interface structure
- *
- * Callback implementing the RDMA_OPEN function. Called by the ice driver to
- * notify the RDMA client driver that a new device has been initialized.
- */
-static int
-irdma_open(struct ice_rdma_peer *peer)
-{
- struct ice_rdma_event event = {0};
-
- event.type = ICE_RDMA_EVENT_MTU_CHANGE;
- event.mtu = peer->mtu;
-
- irdma_event_handler(peer, &event);
-
- return 0;
-}
-
-/**
- * irdma_close - Callback to notify that a peer device is down
- * @peer: the RDMA peer device being stopped
- *
- * Callback implementing the RDMA_CLOSE function. Called by the ice driver to
- * notify the RDMA client driver that a peer device is being stopped.
- */
-static int
-irdma_close(struct ice_rdma_peer *peer)
-{
- /*
- * This is called when ifconfig down. Keeping it for compatibility with ice. This event might be usefull for
- * future.
- */
- return 0;
-}
-
/**
* irdma_alloc_pcidev - allocate memory for pcidev and populate data
* @peer: the new peer interface structure
@@ -615,6 +665,7 @@
sysctl_ctx_free(&iwdev->rf->tun_info.irdma_sysctl_ctx);
hdl->iwdev->rf->tun_info.irdma_sysctl_tree = NULL;
+ hdl->iwdev->rf->tun_info.sws_sysctl_tree = NULL;
irdma_ctrl_deinit_hw(iwdev->rf);
@@ -630,6 +681,54 @@
return 0;
}
+/**
+ * irdma_open - Callback for operation open for RDMA device
+ * @peer: the new peer interface structure
+ *
+ * Callback implementing the RDMA_OPEN function. Called by the ice driver to
+ * notify the RDMA client driver that a new device has been initialized.
+ */
+static int
+irdma_open(struct ice_rdma_peer *peer)
+{
+ struct irdma_device *iwdev;
+ struct ice_rdma_event event = {0};
+
+ iwdev = peer_to_iwdev(peer);
+ if (iwdev) {
+ event.type = ICE_RDMA_EVENT_MTU_CHANGE;
+ event.mtu = peer->mtu;
+
+ irdma_event_handler(peer, &event);
+ } else {
+ irdma_probe(peer);
+ }
+
+ return 0;
+}
+
+/**
+ * irdma_close - Callback to notify that a peer device is down
+ * @peer: the RDMA peer device being stopped
+ *
+ * Callback implementing the RDMA_CLOSE function. Called by the ice driver to
+ * notify the RDMA client driver that a peer device is being stopped.
+ */
+static int
+irdma_close(struct ice_rdma_peer *peer)
+{
+ /*
+ * This is called when ifconfig down or pf-reset is about to happen.
+ */
+ struct irdma_device *iwdev;
+
+ iwdev = peer_to_iwdev(peer);
+ if (iwdev && iwdev->rf->reset)
+ irdma_remove(peer);
+
+ return 0;
+}
+
/**
* irdma_prep_for_unregister - ensure the driver is ready to unregister
*/
@@ -644,8 +743,6 @@
hdl_valid = false;
spin_lock_irqsave(&irdma_handler_lock, flags);
list_for_each_entry(hdl, &irdma_handlers, list) {
- if (!hdl)
- continue;
if (!hdl->iwdev->rf->peer_info)
continue;
hdl_valid = true;
diff --git a/sys/dev/irdma/irdma.h b/sys/dev/irdma/irdma.h
--- a/sys/dev/irdma/irdma.h
+++ b/sys/dev/irdma/irdma.h
@@ -1,7 +1,7 @@
/*-
* SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB
*
- * Copyright (c) 2017 - 2022 Intel Corporation
+ * Copyright (c) 2017 - 2023 Intel Corporation
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@@ -167,7 +167,7 @@
};
struct irdma_mcast_grp_info {
- u8 dest_mac_addr[ETH_ALEN];
+ u8 dest_mac_addr[ETHER_ADDR_LEN];
u16 vlan_id;
u16 hmc_fcn_id;
bool ipv4_valid:1;
diff --git a/sys/dev/irdma/irdma_cm.h b/sys/dev/irdma/irdma_cm.h
--- a/sys/dev/irdma/irdma_cm.h
+++ b/sys/dev/irdma/irdma_cm.h
@@ -1,7 +1,7 @@
/*-
* SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB
*
- * Copyright (c) 2015 - 2022 Intel Corporation
+ * Copyright (c) 2015 - 2023 Intel Corporation
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@@ -288,7 +288,7 @@
int backlog;
u16 loc_port;
u16 vlan_id;
- u8 loc_mac[ETH_ALEN];
+ u8 loc_mac[ETHER_ADDR_LEN];
u8 user_pri;
u8 tos;
bool qhash_set:1;
@@ -341,8 +341,8 @@
u16 mpav2_ird_ord;
u16 lsmm_size;
u8 pdata_buf[IETF_MAX_PRIV_DATA_LEN];
- u8 loc_mac[ETH_ALEN];
- u8 rem_mac[ETH_ALEN];
+ u8 loc_mac[ETHER_ADDR_LEN];
+ u8 rem_mac[ETHER_ADDR_LEN];
u8 user_pri;
u8 tos;
bool ack_rcvd:1;
@@ -358,6 +358,7 @@
/* Used by internal CM APIs to pass CM information*/
struct irdma_cm_info {
struct iw_cm_id *cm_id;
+ struct irdma_cqp_request *cqp_request;
u16 loc_port;
u16 rem_port;
u32 loc_addr[4];
@@ -410,6 +411,12 @@
void (*cm_free_ah)(struct irdma_cm_node *cm_node);
};
+struct irdma_add_mqh_cbs {
+ struct irdma_device *iwdev;
+ struct irdma_cm_info *cm_info;
+ struct irdma_cm_listener *cm_listen_node;
+};
+
int irdma_schedule_cm_timer(struct irdma_cm_node *cm_node,
struct irdma_puda_buf *sqbuf,
enum irdma_timer_type type, int send_retrans,
diff --git a/sys/dev/irdma/irdma_cm.c b/sys/dev/irdma/irdma_cm.c
--- a/sys/dev/irdma/irdma_cm.c
+++ b/sys/dev/irdma/irdma_cm.c
@@ -1,7 +1,7 @@
/*-
* SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB
*
- * Copyright (c) 2015 - 2022 Intel Corporation
+ * Copyright (c) 2015 - 2023 Intel Corporation
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@@ -290,10 +290,9 @@
event->cm_info.loc_port = cm_node->loc_port;
event->cm_info.cm_id = cm_node->cm_id;
irdma_debug(&cm_node->iwdev->rf->sc_dev, IRDMA_DEBUG_CM,
- "node=%p event=%p type=%u dst=%pI4 src=%pI4\n",
- cm_node,
- event, type, event->cm_info.loc_addr,
- event->cm_info.rem_addr);
+ "node=%p event=%p type=%u dst=%x src=%x\n", cm_node, event,
+ type, event->cm_info.loc_addr[0],
+ event->cm_info.rem_addr[0]);
irdma_cm_post_event(event);
return event;
@@ -646,10 +645,11 @@
return -ENOMEM;
irdma_debug(&cm_node->iwdev->rf->sc_dev, IRDMA_DEBUG_CM,
- "caller: %pS cm_node %p cm_id=%p accel=%d state=%d rem_port=0x%04x, loc_port=0x%04x rem_addr=%pI4 loc_addr=%pI4\n",
+ "caller: %pS cm_node %p cm_id=%p accel=%d state=%d rem_port=0x%04x, loc_port=0x%04x rem_addr=%x loc_addr=%x\n",
__builtin_return_address(0), cm_node, cm_node->cm_id,
cm_node->accelerated, cm_node->state, cm_node->rem_port,
- cm_node->loc_port, cm_node->rem_addr, cm_node->loc_addr);
+ cm_node->loc_port, cm_node->rem_addr[0],
+ cm_node->loc_addr[0]);
return irdma_schedule_cm_timer(cm_node, sqbuf, IRDMA_TIMER_TYPE_SEND, 0,
1);
@@ -666,9 +666,8 @@
irdma_cleanup_retrans_entry(cm_node);
cm_node->cm_core->stats_connect_errs++;
if (reset) {
- irdma_debug(&cm_node->iwdev->rf->sc_dev,
- IRDMA_DEBUG_CM, "cm_node=%p state=%d\n", cm_node,
- cm_node->state);
+ irdma_debug(&cm_node->iwdev->rf->sc_dev, IRDMA_DEBUG_CM,
+ "cm_node=%p state=%d\n", cm_node, cm_node->state);
atomic_inc(&cm_node->refcnt);
irdma_send_reset(cm_node);
}
@@ -689,8 +688,7 @@
cm_node->cm_core->stats_passive_errs++;
cm_node->state = IRDMA_CM_STATE_CLOSED;
irdma_debug(&cm_node->iwdev->rf->sc_dev, IRDMA_DEBUG_CM,
- "cm_node=%p state =%d\n",
- cm_node, cm_node->state);
+ "cm_node=%p state=%d\n", cm_node, cm_node->state);
if (reset)
irdma_send_reset(cm_node);
else
@@ -802,8 +800,7 @@
(u32)tcph->th_flags & TH_SYN);
if (ret) {
irdma_debug(&cm_node->iwdev->rf->sc_dev, IRDMA_DEBUG_CM,
- "Node %p, Sending Reset\n",
- cm_node);
+ "Node %p, Sending Reset\n", cm_node);
if (passive)
irdma_passive_open_err(cm_node, true);
else
@@ -951,8 +948,7 @@
MPA_KEY_REQUEST);
if (!cm_node->mpa_hdr.size) {
irdma_debug(&cm_node->iwdev->rf->sc_dev, IRDMA_DEBUG_CM,
- "mpa size = %d\n",
- cm_node->mpa_hdr.size);
+ "mpa size = %d\n", cm_node->mpa_hdr.size);
return -EINVAL;
}
@@ -1062,9 +1058,9 @@
/* Not supported RDMA0 operation */
return -EOPNOTSUPP;
- irdma_debug(&cm_node->iwdev->rf->sc_dev,
- IRDMA_DEBUG_CM, "MPAV2 Negotiated ORD: %d, IRD: %d\n",
- cm_node->ord_size, cm_node->ird_size);
+ irdma_debug(&cm_node->iwdev->rf->sc_dev, IRDMA_DEBUG_CM,
+ "MPAV2 Negotiated ORD: %d, IRD: %d\n", cm_node->ord_size,
+ cm_node->ird_size);
return 0;
}
@@ -1085,8 +1081,8 @@
*type = IRDMA_MPA_REQUEST_ACCEPT;
if (len < sizeof(struct ietf_mpa_v1)) {
- irdma_debug(&cm_node->iwdev->rf->sc_dev,
- IRDMA_DEBUG_CM, "ietf buffer small (%x)\n", len);
+ irdma_debug(&cm_node->iwdev->rf->sc_dev, IRDMA_DEBUG_CM,
+ "ietf buffer small (%x)\n", len);
return -EINVAL;
}
@@ -1096,22 +1092,19 @@
if (priv_data_len > IETF_MAX_PRIV_DATA_LEN) {
irdma_debug(&cm_node->iwdev->rf->sc_dev, IRDMA_DEBUG_CM,
- "private_data too big %d\n",
- priv_data_len);
+ "private_data too big %d\n", priv_data_len);
return -EOVERFLOW;
}
if (mpa_frame->rev != IETF_MPA_V1 && mpa_frame->rev != IETF_MPA_V2) {
irdma_debug(&cm_node->iwdev->rf->sc_dev, IRDMA_DEBUG_CM,
- "unsupported mpa rev = %d\n",
- mpa_frame->rev);
+ "unsupported mpa rev = %d\n", mpa_frame->rev);
return -EINVAL;
}
if (mpa_frame->rev > cm_node->mpa_frame_rev) {
irdma_debug(&cm_node->iwdev->rf->sc_dev, IRDMA_DEBUG_CM,
- "rev %d\n",
- mpa_frame->rev);
+ "rev %d\n", mpa_frame->rev);
return -EINVAL;
}
@@ -1119,29 +1112,29 @@
if (cm_node->state != IRDMA_CM_STATE_MPAREQ_SENT) {
if (memcmp(mpa_frame->key, IEFT_MPA_KEY_REQ,
IETF_MPA_KEY_SIZE)) {
- irdma_debug(&cm_node->iwdev->rf->sc_dev,
- IRDMA_DEBUG_CM, "Unexpected MPA Key received\n");
+ irdma_debug(&cm_node->iwdev->rf->sc_dev, IRDMA_DEBUG_CM,
+ "Unexpected MPA Key received\n");
return -EINVAL;
}
} else {
if (memcmp(mpa_frame->key, IEFT_MPA_KEY_REP,
IETF_MPA_KEY_SIZE)) {
- irdma_debug(&cm_node->iwdev->rf->sc_dev,
- IRDMA_DEBUG_CM, "Unexpected MPA Key received\n");
+ irdma_debug(&cm_node->iwdev->rf->sc_dev, IRDMA_DEBUG_CM,
+ "Unexpected MPA Key received\n");
return -EINVAL;
}
}
if (priv_data_len + mpa_hdr_len > len) {
- irdma_debug(&cm_node->iwdev->rf->sc_dev,
- IRDMA_DEBUG_CM, "ietf buffer len(%x + %x != %x)\n",
- priv_data_len, mpa_hdr_len, len);
+ irdma_debug(&cm_node->iwdev->rf->sc_dev, IRDMA_DEBUG_CM,
+ "ietf buffer len(%x + %x != %x)\n", priv_data_len,
+ mpa_hdr_len, len);
return -EOVERFLOW;
}
if (len > IRDMA_MAX_CM_BUF) {
- irdma_debug(&cm_node->iwdev->rf->sc_dev,
- IRDMA_DEBUG_CM, "ietf buffer large len = %d\n", len);
+ irdma_debug(&cm_node->iwdev->rf->sc_dev, IRDMA_DEBUG_CM,
+ "ietf buffer large len = %d\n", len);
return -EOVERFLOW;
}
@@ -1213,8 +1206,8 @@
new_send->timetosend += (HZ / 10);
if (cm_node->close_entry) {
kfree(new_send);
- irdma_debug(&cm_node->iwdev->rf->sc_dev,
- IRDMA_DEBUG_CM, "already close entry\n");
+ irdma_debug(&cm_node->iwdev->rf->sc_dev, IRDMA_DEBUG_CM,
+ "already close entry\n");
return -EINVAL;
}
@@ -1583,14 +1576,14 @@
child_listen_list);
if (child_listen_node->ipv4)
irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_CM,
- "removing child listen for IP=%pI4, port=%d, vlan=%d\n",
- child_listen_node->loc_addr,
+ "removing child listen for IP=%x, port=%d, vlan=%d\n",
+ child_listen_node->loc_addr[0],
child_listen_node->loc_port,
child_listen_node->vlan_id);
else
irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_CM,
- "removing child listen for IP=%pI6, port=%d, vlan=%d\n",
- child_listen_node->loc_addr,
+ "removing child listen for IP=%x:%x:%x:%x, port=%d, vlan=%d\n",
+ IRDMA_PRINT_IP6(child_listen_node->loc_addr),
child_listen_node->loc_port,
child_listen_node->vlan_id);
list_del(pos);
@@ -1606,8 +1599,8 @@
} else {
ret = 0;
}
- irdma_debug(&iwdev->rf->sc_dev,
- IRDMA_DEBUG_CM, "Child listen node freed = %p\n",
+ irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_CM,
+ "Child listen node freed = %p\n",
child_listen_node);
kfree(child_listen_node);
cm_parent_listen_node->cm_core->stats_listen_nodes_destroyed++;
@@ -1617,7 +1610,8 @@
return ret;
}
-static u8 irdma_get_egress_vlan_prio(u32 *loc_addr, u8 prio, bool ipv4){
+static u8 irdma_iw_get_vlan_prio(u32 *loc_addr, u8 prio, bool ipv4)
+{
return prio;
}
@@ -1679,14 +1673,33 @@
return vlan_id;
}
-struct irdma_add_mqh_cbs {
- struct irdma_device *iwdev;
- struct irdma_cm_info *cm_info;
- struct irdma_cm_listener *cm_listen_node;
-};
+static int
+irdma_manage_qhash_wait(struct irdma_pci_f *rf, struct irdma_cm_info *cm_info)
+{
+ struct irdma_cqp_request *cqp_request = cm_info->cqp_request;
+ int cnt = rf->sc_dev.hw_attrs.max_cqp_compl_wait_time_ms * CQP_TIMEOUT_THRESHOLD;
+ u32 ret_val;
+
+ if (!cqp_request)
+ return -ENOMEM;
+ do {
+ irdma_cqp_ce_handler(rf, &rf->ccq.sc_cq);
+ mdelay(1);
+ } while (!READ_ONCE(cqp_request->request_done) && --cnt);
+
+ ret_val = cqp_request->compl_info.op_ret_val;
+ irdma_put_cqp_request(&rf->cqp, cqp_request);
+ if (cnt) {
+ if (!ret_val)
+ return 0;
+ return -EINVAL;
+ }
+
+ return -ETIMEDOUT;
+}
/**
- * irdma_add_mqh_ifa_cb - Adds multiple qhashes for IPV4/IPv6
+ * irdma_add_mqh_ifa_cb - Adds multiple qhashes for IPv4/IPv6
* @arg: Calback argument structure from irdma_add_mqh
* @ifa: Current address to compute against
* @count: Current cumulative output of all callbacks in this iteration
@@ -1709,24 +1722,9 @@
if (count)
return 0;
- if (cm_info->ipv4)
- irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_CM,
- "Allocating child CM Listener forIP=%pI4, vlan_id=%d, MAC=%pM\n",
- &ifa->ifa_addr,
- rdma_vlan_dev_vlan_id(ip_dev), if_getlladdr(ip_dev));
- else
- irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_CM,
- "IP=%pI6, vlan_id=%d, MAC=%pM\n",
- &((struct sockaddr_in6 *)ifa->ifa_addr)->sin6_addr,
- rdma_vlan_dev_vlan_id(ip_dev),
- if_getlladdr(ip_dev));
- child_listen_node = kzalloc(sizeof(*child_listen_node), GFP_KERNEL);
- irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_CM,
- "Allocating child listener %p\n",
- child_listen_node);
+ child_listen_node = kzalloc(sizeof(*child_listen_node), GFP_ATOMIC);
if (!child_listen_node) {
- irdma_debug(&iwdev->rf->sc_dev,
- IRDMA_DEBUG_CM,
+ irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_CM,
"listener memory allocation\n");
return -ENOMEM;
}
@@ -1735,23 +1733,44 @@
sizeof(*child_listen_node));
cm_info->vlan_id = rdma_vlan_dev_vlan_id(ip_dev);
child_listen_node->vlan_id = cm_info->vlan_id;
- if (cm_info->ipv4)
+ if (cm_info->ipv4) {
+ irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_CM,
+ "Allocating child CM Listener forIP=%x, vlan_id=%d, MAC=%x:%x:%x:%x:%x:%x\n",
+ ((struct sockaddr_in *)&ifa->ifa_addr)->sin_addr.s_addr,
+ rdma_vlan_dev_vlan_id(ip_dev),
+ if_getlladdr(ip_dev)[0], if_getlladdr(ip_dev)[1],
+ if_getlladdr(ip_dev)[2], if_getlladdr(ip_dev)[3],
+ if_getlladdr(ip_dev)[4], if_getlladdr(ip_dev)[5]);
child_listen_node->loc_addr[0] =
ntohl(((struct sockaddr_in *)ifa->ifa_addr)->sin_addr.s_addr);
- else
+ } else {
+ irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_CM,
+ "IP=%x:%x:%x:%x, vlan_id=%d, MAC=%x:%x:%x:%x:%x:%x\n",
+ IRDMA_PRINT_IP6(&((struct sockaddr_in6 *)ifa->ifa_addr)->sin6_addr),
+ rdma_vlan_dev_vlan_id(ip_dev),
+ if_getlladdr(ip_dev)[0], if_getlladdr(ip_dev)[1],
+ if_getlladdr(ip_dev)[2], if_getlladdr(ip_dev)[3],
+ if_getlladdr(ip_dev)[4], if_getlladdr(ip_dev)[5]);
irdma_copy_ip_ntohl(child_listen_node->loc_addr,
((struct sockaddr_in6 *)ifa->ifa_addr)->sin6_addr.__u6_addr.__u6_addr32);
+ }
memcpy(cm_info->loc_addr, child_listen_node->loc_addr,
sizeof(cm_info->loc_addr));
if (!iwdev->vsi.dscp_mode)
cm_info->user_pri =
- irdma_get_egress_vlan_prio(child_listen_node->loc_addr,
- cm_info->user_pri,
- false);
+ irdma_iw_get_vlan_prio(child_listen_node->loc_addr,
+ cm_info->user_pri,
+ cm_info->ipv4);
ret = irdma_manage_qhash(iwdev, cm_info,
IRDMA_QHASH_TYPE_TCP_SYN,
IRDMA_QHASH_MANAGE_TYPE_ADD,
- NULL, true);
+ NULL, false);
+ if (ret) {
+ kfree(child_listen_node);
+ return ret;
+ }
+ /* wait for qhash finish */
+ ret = irdma_manage_qhash_wait(iwdev->rf, cm_info);
if (ret) {
kfree(child_listen_node);
return ret;
@@ -1782,7 +1801,7 @@
struct irdma_add_mqh_cbs cbs;
struct if_iter iter;
if_t ifp;
- int err;
+ int err = -ENOENT;
cbs.iwdev = iwdev;
cbs.cm_info = cm_info;
@@ -1882,8 +1901,8 @@
err = irdma_send_reset(cm_node);
if (err) {
cm_node->state = IRDMA_CM_STATE_CLOSED;
- irdma_debug(&cm_node->iwdev->rf->sc_dev,
- IRDMA_DEBUG_CM, "send reset failed\n");
+ irdma_debug(&cm_node->iwdev->rf->sc_dev, IRDMA_DEBUG_CM,
+ "send reset failed\n");
} else {
old_state = cm_node->state;
cm_node->state = IRDMA_CM_STATE_LISTENER_DESTROYED;
@@ -1922,8 +1941,8 @@
cm_core->stats_listen_destroyed++;
cm_core->stats_listen_nodes_destroyed++;
irdma_debug(&listener->iwdev->rf->sc_dev, IRDMA_DEBUG_CM,
- "loc_port=0x%04x loc_addr=%pI4 cm_listen_node=%p cm_id=%p qhash_set=%d vlan_id=%d apbvt_del=%d\n",
- listener->loc_port, listener->loc_addr, listener,
+ "loc_port=0x%04x loc_addr=%x cm_listen_node=%p cm_id=%p qhash_set=%d vlan_id=%d apbvt_del=%d\n",
+ listener->loc_port, listener->loc_addr[0], listener,
listener->cm_id, listener->qhash_set,
listener->vlan_id, apbvt_del);
kfree(listener);
@@ -2139,13 +2158,12 @@
cm_node->tos = max(listener->tos, cm_info->tos);
cm_node->user_pri = rt_tos2priority(cm_node->tos);
cm_node->user_pri =
- irdma_get_egress_vlan_prio(cm_info->loc_addr,
- cm_node->user_pri,
- cm_info->ipv4);
+ irdma_iw_get_vlan_prio(cm_info->loc_addr,
+ cm_node->user_pri,
+ cm_info->ipv4);
}
irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_DCB,
- "listener: TOS:[%d] UP:[%d]\n",
- cm_node->tos,
+ "listener: TOS:[%d] UP:[%d]\n", cm_node->tos,
cm_node->user_pri);
}
memcpy(cm_node->loc_addr, cm_info->loc_addr, sizeof(cm_node->loc_addr));
@@ -2200,8 +2218,8 @@
/* if the node is destroyed before connection was accelerated */
if (!cm_node->accelerated && cm_node->accept_pend) {
- irdma_debug(&cm_node->iwdev->rf->sc_dev,
- IRDMA_DEBUG_CM, "node destroyed before established\n");
+ irdma_debug(&cm_node->iwdev->rf->sc_dev, IRDMA_DEBUG_CM,
+ "node destroyed before established\n");
atomic_dec(&cm_node->listener->pend_accepts_cnt);
}
if (cm_node->close_entry)
@@ -2325,8 +2343,7 @@
case IRDMA_CM_STATE_OFFLOADED:
default:
irdma_debug(&cm_node->iwdev->rf->sc_dev, IRDMA_DEBUG_CM,
- "bad state node state = %d\n",
- cm_node->state);
+ "bad state node state = %d\n", cm_node->state);
break;
}
}
@@ -2341,10 +2358,10 @@
struct irdma_puda_buf *rbuf)
{
irdma_debug(&cm_node->iwdev->rf->sc_dev, IRDMA_DEBUG_CM,
- "caller: %pS cm_node=%p state=%d rem_port=0x%04x loc_port=0x%04x rem_addr=%pI4 loc_addr=%pI4\n",
+ "caller: %pS cm_node=%p state=%d rem_port=0x%04x loc_port=0x%04x rem_addr=%x loc_addr=%x\n",
__builtin_return_address(0), cm_node, cm_node->state,
- cm_node->rem_port, cm_node->loc_port, cm_node->rem_addr,
- cm_node->loc_addr);
+ cm_node->rem_port, cm_node->loc_port, cm_node->rem_addr[0],
+ cm_node->loc_addr[0]);
irdma_cleanup_retrans_entry(cm_node);
switch (cm_node->state) {
@@ -2418,8 +2435,8 @@
switch (cm_node->state) {
case IRDMA_CM_STATE_ESTABLISHED:
if (res_type == IRDMA_MPA_REQUEST_REJECT)
- irdma_debug(&cm_node->iwdev->rf->sc_dev,
- IRDMA_DEBUG_CM, "state for reject\n");
+ irdma_debug(&cm_node->iwdev->rf->sc_dev, IRDMA_DEBUG_CM,
+ "state for reject\n");
cm_node->state = IRDMA_CM_STATE_MPAREQ_RCVD;
type = IRDMA_CM_EVENT_MPA_REQ;
irdma_send_ack(cm_node); /* ACK received MPA request */
@@ -2439,8 +2456,7 @@
break;
default:
irdma_debug(&cm_node->iwdev->rf->sc_dev, IRDMA_DEBUG_CM,
- "wrong cm_node state =%d\n",
- cm_node->state);
+ "wrong cm_node state=%d\n", cm_node->state);
break;
}
irdma_create_event(cm_node, type);
@@ -2484,8 +2500,8 @@
!between(seq, rcv_nxt, (rcv_nxt + rcv_wnd)))
err = -1;
if (err)
- irdma_debug(&cm_node->iwdev->rf->sc_dev,
- IRDMA_DEBUG_CM, "seq number err\n");
+ irdma_debug(&cm_node->iwdev->rf->sc_dev, IRDMA_DEBUG_CM,
+ "seq number err\n");
return err;
}
@@ -2592,8 +2608,8 @@
irdma_cleanup_retrans_entry(cm_node);
/* active open */
if (irdma_check_syn(cm_node, tcph)) {
- irdma_debug(&cm_node->iwdev->rf->sc_dev,
- IRDMA_DEBUG_CM, "check syn fail\n");
+ irdma_debug(&cm_node->iwdev->rf->sc_dev, IRDMA_DEBUG_CM,
+ "check syn fail\n");
return;
}
cm_node->tcp_cntxt.rem_ack_num = ntohl(tcph->th_ack);
@@ -2601,8 +2617,7 @@
err = irdma_handle_tcp_options(cm_node, tcph, optionsize, 0);
if (err) {
irdma_debug(&cm_node->iwdev->rf->sc_dev, IRDMA_DEBUG_CM,
- "cm_node=%p tcp_options failed\n",
- cm_node);
+ "cm_node=%p tcp_options failed\n", cm_node);
break;
}
irdma_cleanup_retrans_entry(cm_node);
@@ -2915,8 +2930,8 @@
cm_node->state = IRDMA_CM_STATE_CLOSED;
if (irdma_send_reset(cm_node))
- irdma_debug(&cm_node->iwdev->rf->sc_dev,
- IRDMA_DEBUG_CM, "send reset failed\n");
+ irdma_debug(&cm_node->iwdev->rf->sc_dev, IRDMA_DEBUG_CM,
+ "send reset failed\n");
return ret;
}
@@ -2962,8 +2977,8 @@
break;
case IRDMA_CM_STATE_OFFLOADED:
if (cm_node->send_entry)
- irdma_debug(&cm_node->iwdev->rf->sc_dev,
- IRDMA_DEBUG_CM, "CM send_entry in OFFLOADED state\n");
+ irdma_debug(&cm_node->iwdev->rf->sc_dev, IRDMA_DEBUG_CM,
+ "CM send_entry in OFFLOADED state\n");
irdma_rem_ref_cm_node(cm_node);
break;
}
@@ -3013,8 +3028,7 @@
VLAN_PRIO_SHIFT;
cm_info.vlan_id = vtag & EVL_VLID_MASK;
irdma_debug(&cm_core->iwdev->rf->sc_dev, IRDMA_DEBUG_CM,
- "vlan_id=%d\n",
- cm_info.vlan_id);
+ "vlan_id=%d\n", cm_info.vlan_id);
} else {
cm_info.vlan_id = 0xFFFF;
}
@@ -3055,8 +3069,8 @@
IRDMA_CM_LISTENER_ACTIVE_STATE);
if (!listener) {
cm_info.cm_id = NULL;
- irdma_debug(&cm_core->iwdev->rf->sc_dev,
- IRDMA_DEBUG_CM, "no listener found\n");
+ irdma_debug(&cm_core->iwdev->rf->sc_dev, IRDMA_DEBUG_CM,
+ "no listener found\n");
return;
}
@@ -3064,8 +3078,8 @@
cm_node = irdma_make_cm_node(cm_core, iwdev, &cm_info,
listener);
if (!cm_node) {
- irdma_debug(&cm_core->iwdev->rf->sc_dev,
- IRDMA_DEBUG_CM, "allocate node failed\n");
+ irdma_debug(&cm_core->iwdev->rf->sc_dev, IRDMA_DEBUG_CM,
+ "allocate node failed\n");
atomic_dec(&listener->refcnt);
return;
}
@@ -3293,9 +3307,8 @@
spin_lock_irqsave(&iwdev->rf->qptable_lock, flags);
if (!iwdev->rf->qp_table[iwqp->ibqp.qp_num]) {
spin_unlock_irqrestore(&iwdev->rf->qptable_lock, flags);
- irdma_debug(&iwdev->rf->sc_dev,
- IRDMA_DEBUG_CM, "qp_id %d is already freed\n",
- iwqp->ibqp.qp_num);
+ irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_CM,
+ "qp_id %d is already freed\n", iwqp->ibqp.qp_num);
kfree(work);
return;
}
@@ -3467,7 +3480,7 @@
if (iwqp->ietf_mem.va) {
if (iwqp->lsmm_mr)
- kc_free_lsmm_dereg_mr(iwdev, iwqp);
+ iwdev->ibdev.dereg_mr(iwqp->lsmm_mr, NULL);
irdma_free_dma_mem(iwdev->rf->sc_dev.hw,
&iwqp->ietf_mem);
iwqp->ietf_mem.va = NULL;
@@ -3614,9 +3627,9 @@
}
irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_CM,
- "rem_port=0x%04x, loc_port=0x%04x rem_addr=%pI4 loc_addr=%pI4 cm_node=%p cm_id=%p qp_id = %d\n\n",
- cm_node->rem_port, cm_node->loc_port, cm_node->rem_addr,
- cm_node->loc_addr, cm_node, cm_id, ibqp->qp_num);
+ "rem_port=0x%04x, loc_port=0x%04x rem_addr=%x loc_addr=%x cm_node=%p cm_id=%p qp_id=%d\n\n",
+ cm_node->rem_port, cm_node->loc_port, cm_node->rem_addr[0],
+ cm_node->loc_addr[0], cm_node, cm_id, ibqp->qp_num);
cm_node->cm_core->stats_accepts++;
return 0;
@@ -3729,9 +3742,10 @@
iwqp->sc_qp.vsi->dscp_map[irdma_tos2dscp(cm_info.tos)];
} else {
cm_info.user_pri = rt_tos2priority(cm_id->tos);
- cm_info.user_pri = irdma_get_egress_vlan_prio(cm_info.loc_addr,
- cm_info.user_pri,
- cm_info.ipv4);
+ cm_info.user_pri =
+ irdma_iw_get_vlan_prio(cm_info.loc_addr,
+ cm_info.user_pri,
+ cm_info.ipv4);
}
if (iwqp->sc_qp.dev->ws_add(iwqp->sc_qp.vsi, cm_info.user_pri))
@@ -3740,9 +3754,8 @@
irdma_qp_add_qos(&iwqp->sc_qp);
if (iwdev->rf->sc_dev.hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_2)
iwdev->rf->check_fc(&iwdev->vsi, &iwqp->sc_qp);
- irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_DCB,
- "TOS:[%d] UP:[%d]\n", cm_id->tos,
- cm_info.user_pri);
+ irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_DCB, "TOS:[%d] UP:[%d]\n",
+ cm_id->tos, cm_info.user_pri);
ret = irdma_create_cm_node(&iwdev->cm_core, iwdev, conn_param, &cm_info,
&cm_node);
@@ -3780,21 +3793,21 @@
}
irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_CM,
- "rem_port=0x%04x, loc_port=0x%04x rem_addr=%pI4 loc_addr=%pI4 cm_node=%p cm_id=%p qp_id = %d\n\n",
- cm_node->rem_port, cm_node->loc_port, cm_node->rem_addr,
- cm_node->loc_addr, cm_node, cm_id, ibqp->qp_num);
+ "rem_port=0x%04x, loc_port=0x%04x rem_addr=%x loc_addr=%x cm_node=%p cm_id=%p qp_id = %d\n\n",
+ cm_node->rem_port, cm_node->loc_port, cm_node->rem_addr[0],
+ cm_node->loc_addr[0], cm_node, cm_id, ibqp->qp_num);
return 0;
err:
if (cm_info.ipv4)
- irdma_debug(&iwdev->rf->sc_dev,
- IRDMA_DEBUG_CM, "connect() FAILED: dest addr=%pI4",
- cm_info.rem_addr);
+ irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_CM,
+ "connect() FAILED: dest addr=%x",
+ cm_info.rem_addr[0]);
else
- irdma_debug(&iwdev->rf->sc_dev,
- IRDMA_DEBUG_CM, "connect() FAILED: dest addr=%pI6",
- cm_info.rem_addr);
+ irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_CM,
+ "connect() FAILED: dest addr=%x:%x:%x:%x",
+ IRDMA_PRINT_IP6(cm_info.rem_addr));
irdma_rem_ref_cm_node(cm_node);
iwdev->cm_core.stats_connect_errs++;
@@ -3864,8 +3877,8 @@
cm_listen_node = irdma_make_listen_node(&iwdev->cm_core, iwdev,
&cm_info);
if (!cm_listen_node) {
- irdma_debug(&iwdev->rf->sc_dev,
- IRDMA_DEBUG_CM, "cm_listen_node == NULL\n");
+ irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_CM,
+ "cm_listen_node == NULL\n");
return -ENOMEM;
}
@@ -3886,9 +3899,9 @@
} else {
if (!iwdev->vsi.dscp_mode)
cm_info.user_pri = cm_listen_node->user_pri =
- irdma_get_egress_vlan_prio(cm_info.loc_addr,
- cm_info.user_pri,
- cm_info.ipv4);
+ irdma_iw_get_vlan_prio(cm_info.loc_addr,
+ cm_info.user_pri,
+ cm_info.ipv4);
err = irdma_manage_qhash(iwdev, &cm_info,
IRDMA_QHASH_TYPE_TCP_SYN,
IRDMA_QHASH_MANAGE_TYPE_ADD,
@@ -3907,8 +3920,8 @@
cm_id->add_ref(cm_id);
cm_listen_node->cm_core->stats_listen_created++;
irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_CM,
- "loc_port=0x%04x loc_addr=%pI4 cm_listen_node=%p cm_id=%p qhash_set=%d vlan_id=%d\n",
- cm_listen_node->loc_port, cm_listen_node->loc_addr,
+ "loc_port=0x%04x loc_addr=%x cm_listen_node=%p cm_id=%p qhash_set=%d vlan_id=%d\n",
+ cm_listen_node->loc_port, cm_listen_node->loc_addr[0],
cm_listen_node, cm_listen_node->cm_id,
cm_listen_node->qhash_set, cm_listen_node->vlan_id);
@@ -3935,8 +3948,8 @@
irdma_cm_del_listen(&iwdev->cm_core, cm_id->provider_data,
true);
else
- irdma_debug(&iwdev->rf->sc_dev,
- IRDMA_DEBUG_CM, "cm_id->provider_data was NULL\n");
+ irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_CM,
+ "cm_id->provider_data was NULL\n");
cm_id->rem_ref(cm_id);
@@ -4122,8 +4135,7 @@
return;
irdma_debug(&cm_node->iwdev->rf->sc_dev, IRDMA_DEBUG_CM,
- "reset event %p - cm_id = %p\n",
- event->cm_node, cm_id);
+ "reset event %p - cm_id = %p\n", event->cm_node, cm_id);
iwqp->cm_id = NULL;
irdma_send_cm_event(cm_node, cm_node->cm_id, IW_CM_EVENT_DISCONNECT,
@@ -4175,8 +4187,7 @@
break;
default:
irdma_debug(&cm_node->iwdev->rf->sc_dev, IRDMA_DEBUG_CM,
- "bad event type = %d\n",
- event->type);
+ "bad event type = %d\n", event->type);
break;
}
diff --git a/sys/dev/irdma/irdma_ctrl.c b/sys/dev/irdma/irdma_ctrl.c
--- a/sys/dev/irdma/irdma_ctrl.c
+++ b/sys/dev/irdma/irdma_ctrl.c
@@ -1,7 +1,7 @@
/*-
* SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB
*
- * Copyright (c) 2015 - 2022 Intel Corporation
+ * Copyright (c) 2015 - 2023 Intel Corporation
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@@ -179,7 +179,8 @@
irdma_debug(qp->dev, IRDMA_DEBUG_DCB,
"DCB: Remove qp[%d] UP[%d] qset[%d] on_qoslist[%d]\n",
- qp->qp_uk.qp_id, qp->user_pri, qp->qs_handle, qp->on_qoslist);
+ qp->qp_uk.qp_id, qp->user_pri, qp->qs_handle,
+ qp->on_qoslist);
mutex_lock(&vsi->qos[qp->user_pri].qos_mutex);
if (qp->on_qoslist) {
qp->on_qoslist = false;
@@ -199,7 +200,8 @@
irdma_debug(qp->dev, IRDMA_DEBUG_DCB,
"DCB: Add qp[%d] UP[%d] qset[%d] on_qoslist[%d]\n",
- qp->qp_uk.qp_id, qp->user_pri, qp->qs_handle, qp->on_qoslist);
+ qp->qp_uk.qp_id, qp->user_pri, qp->qs_handle,
+ qp->on_qoslist);
mutex_lock(&vsi->qos[qp->user_pri].qos_mutex);
if (!qp->on_qoslist) {
list_add(&qp->list, &vsi->qos[qp->user_pri].qplist);
@@ -238,17 +240,14 @@
u64 scratch, bool post_sq)
{
__le64 *wqe;
- u64 temp, hdr;
+ u64 hdr;
wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
if (!wqe)
return -ENOSPC;
set_64bit_val(wqe, IRDMA_BYTE_8, info->reach_max);
- temp = info->mac_addr[5] | LS_64_1(info->mac_addr[4], 8) |
- LS_64_1(info->mac_addr[3], 16) | LS_64_1(info->mac_addr[2], 24) |
- LS_64_1(info->mac_addr[1], 32) | LS_64_1(info->mac_addr[0], 40);
- set_64bit_val(wqe, IRDMA_BYTE_16, temp);
+ set_64bit_val(wqe, IRDMA_BYTE_16, irdma_mac_to_u64(info->mac_addr));
hdr = info->arp_index |
FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_MANAGE_ARP) |
@@ -369,10 +368,7 @@
wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
if (!wqe)
return -ENOSPC;
- temp = info->mac_addr[5] | LS_64_1(info->mac_addr[4], 8) |
- LS_64_1(info->mac_addr[3], 16) | LS_64_1(info->mac_addr[2], 24) |
- LS_64_1(info->mac_addr[1], 32) | LS_64_1(info->mac_addr[0], 40);
- set_64bit_val(wqe, IRDMA_BYTE_0, temp);
+ set_64bit_val(wqe, IRDMA_BYTE_0, irdma_mac_to_u64(info->mac_addr));
qw1 = FIELD_PREP(IRDMA_CQPSQ_QHASH_QPN, info->qp_num) |
FIELD_PREP(IRDMA_CQPSQ_QHASH_DEST_PORT, info->dest_port);
@@ -711,18 +707,10 @@
struct irdma_udp_offload_info *udp;
u8 push_mode_en;
u32 push_idx;
- u64 mac;
roce_info = info->roce_info;
udp = info->udp_info;
- mac = LS_64_1(roce_info->mac_addr[5], 16) |
- LS_64_1(roce_info->mac_addr[4], 24) |
- LS_64_1(roce_info->mac_addr[3], 32) |
- LS_64_1(roce_info->mac_addr[2], 40) |
- LS_64_1(roce_info->mac_addr[1], 48) |
- LS_64_1(roce_info->mac_addr[0], 56);
-
qp->user_pri = info->user_pri;
if (qp->push_idx == IRDMA_INVALID_PUSH_PAGE_INDEX) {
push_mode_en = 0;
@@ -797,7 +785,8 @@
FIELD_PREP(IRDMAQPC_RXCQNUM, info->rcv_cq_num));
set_64bit_val(qp_ctx, IRDMA_BYTE_144,
FIELD_PREP(IRDMAQPC_STAT_INDEX, info->stats_idx));
- set_64bit_val(qp_ctx, IRDMA_BYTE_152, mac);
+ set_64bit_val(qp_ctx, IRDMA_BYTE_152,
+ FIELD_PREP(IRDMAQPC_MACADDRESS, irdma_mac_to_u64(roce_info->mac_addr)));
set_64bit_val(qp_ctx, IRDMA_BYTE_160,
FIELD_PREP(IRDMAQPC_ORDSIZE, roce_info->ord_size) |
FIELD_PREP(IRDMAQPC_IRDSIZE, irdma_sc_get_encoded_ird_size(roce_info->ird_size)) |
@@ -878,16 +867,13 @@
u64 scratch, bool post_sq)
{
__le64 *wqe;
- u64 temp, header;
+ u64 header;
wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
if (!wqe)
return -ENOSPC;
- temp = info->mac_addr[5] | LS_64_1(info->mac_addr[4], 8) |
- LS_64_1(info->mac_addr[3], 16) | LS_64_1(info->mac_addr[2], 24) |
- LS_64_1(info->mac_addr[1], 32) | LS_64_1(info->mac_addr[0], 40);
- set_64bit_val(wqe, IRDMA_BYTE_32, temp);
+ set_64bit_val(wqe, IRDMA_BYTE_32, irdma_mac_to_u64(info->mac_addr));
header = FIELD_PREP(IRDMA_CQPSQ_MLM_TABLEIDX, info->entry_idx) |
FIELD_PREP(IRDMA_CQPSQ_OPCODE,
@@ -1019,14 +1005,9 @@
FIELD_PREP(IRDMAQPC_Q2ADDR, qp->q2_pa >> 8) |
FIELD_PREP(IRDMAQPC_STAT_INDEX, info->stats_idx));
- if (dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2) {
- mac = LS_64_1(iw->mac_addr[5], 16) |
- LS_64_1(iw->mac_addr[4], 24) |
- LS_64_1(iw->mac_addr[3], 32) |
- LS_64_1(iw->mac_addr[2], 40) |
- LS_64_1(iw->mac_addr[1], 48) |
- LS_64_1(iw->mac_addr[0], 56);
- }
+ if (dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2)
+ mac = FIELD_PREP(IRDMAQPC_MACADDRESS,
+ irdma_mac_to_u64(iw->mac_addr));
set_64bit_val(qp_ctx, IRDMA_BYTE_152,
mac | FIELD_PREP(IRDMAQPC_LASTBYTESENT, iw->last_byte_sent));
@@ -1411,8 +1392,9 @@
qp->qp_uk.sq_wrtrk_array[wqe_idx].signaled = info->signaled;
irdma_debug(qp->dev, IRDMA_DEBUG_MR,
- "wr_id[%llxh] wqe_idx[%04d] location[%p]\n", (unsigned long long)info->wr_id,
- wqe_idx, &qp->qp_uk.sq_wrtrk_array[wqe_idx].wrid);
+ "wr_id[%llxh] wqe_idx[%04d] location[%p]\n",
+ (unsigned long long)info->wr_id, wqe_idx,
+ &qp->qp_uk.sq_wrtrk_array[wqe_idx].wrid);
temp = (info->addr_type == IRDMA_ADDR_TYPE_VA_BASED) ?
(uintptr_t)info->va : info->fbo;
@@ -1533,45 +1515,6 @@
irdma_sc_gen_rts_ae(qp);
}
-/**
- * irdma_sc_send_lsmm_nostag - for privilege qp
- * @qp: sc qp struct
- * @lsmm_buf: buffer with lsmm message
- * @size: size of lsmm buffer
- */
-void
-irdma_sc_send_lsmm_nostag(struct irdma_sc_qp *qp, void *lsmm_buf, u32 size)
-{
- __le64 *wqe;
- u64 hdr;
- struct irdma_qp_uk *qp_uk;
-
- qp_uk = &qp->qp_uk;
- wqe = qp_uk->sq_base->elem;
-
- set_64bit_val(wqe, IRDMA_BYTE_0, (uintptr_t)lsmm_buf);
-
- if (qp->qp_uk.uk_attrs->hw_rev == IRDMA_GEN_1)
- set_64bit_val(wqe, IRDMA_BYTE_8,
- FIELD_PREP(IRDMAQPSQ_GEN1_FRAG_LEN, size));
- else
- set_64bit_val(wqe, IRDMA_BYTE_8,
- FIELD_PREP(IRDMAQPSQ_FRAG_LEN, size) |
- FIELD_PREP(IRDMAQPSQ_VALID, qp->qp_uk.swqe_polarity));
- set_64bit_val(wqe, IRDMA_BYTE_16, 0);
-
- hdr = FIELD_PREP(IRDMAQPSQ_OPCODE, IRDMAQP_OP_RDMA_SEND) |
- FIELD_PREP(IRDMAQPSQ_STREAMMODE, 1) |
- FIELD_PREP(IRDMAQPSQ_WAITFORRCVPDU, 1) |
- FIELD_PREP(IRDMAQPSQ_VALID, qp->qp_uk.swqe_polarity);
- irdma_wmb(); /* make sure WQE is written before valid bit is set */
-
- set_64bit_val(wqe, IRDMA_BYTE_24, hdr);
-
- irdma_debug_buf(qp->dev, IRDMA_DEBUG_WQE, "SEND_LSMM_NOSTAG WQE", wqe,
- IRDMA_QP_WQE_MIN_SIZE);
-}
-
/**
* irdma_sc_send_rtt - send last read0 or write0
* @qp: sc qp struct
@@ -2239,16 +2182,15 @@
struct irdma_up_info *info, u64 scratch)
{
__le64 *wqe;
- u64 temp;
+ u64 temp = 0;
+ int i;
wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
if (!wqe)
return -ENOSPC;
- temp = info->map[0] | LS_64_1(info->map[1], 8) |
- LS_64_1(info->map[2], 16) | LS_64_1(info->map[3], 24) |
- LS_64_1(info->map[4], 32) | LS_64_1(info->map[5], 40) |
- LS_64_1(info->map[6], 48) | LS_64_1(info->map[7], 56);
+ for (i = 0; i < IRDMA_MAX_USER_PRIORITY; i++)
+ temp |= (u64)info->map[i] << (i * 8);
set_64bit_val(wqe, IRDMA_BYTE_0, temp);
set_64bit_val(wqe, IRDMA_BYTE_40,
@@ -2341,7 +2283,8 @@
if (!flush_sq && !flush_rq) {
irdma_debug(qp->dev, IRDMA_DEBUG_CQP,
- "Additional flush request ignored for qp %x\n", qp->qp_uk.qp_id);
+ "Additional flush request ignored for qp %x\n",
+ qp->qp_uk.qp_id);
return -EALREADY;
}
@@ -2649,11 +2592,11 @@
return -EINVAL;
ceq = cq->dev->ceq[cq->ceq_id];
- if (ceq && ceq->reg_cq)
+ if (ceq && ceq->reg_cq) {
ret_code = irdma_sc_add_cq_ctx(ceq, cq);
-
- if (ret_code)
- return ret_code;
+ if (ret_code)
+ return ret_code;
+ }
wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
if (!wqe) {
@@ -2837,11 +2780,12 @@
irdma_check_cqp_progress(struct irdma_cqp_timeout *timeout,
struct irdma_sc_dev *dev)
{
- if (timeout->compl_cqp_cmds != dev->cqp_cmd_stats[IRDMA_OP_CMPL_CMDS]) {
- timeout->compl_cqp_cmds = dev->cqp_cmd_stats[IRDMA_OP_CMPL_CMDS];
+ u64 completed_ops = atomic64_read(&dev->cqp->completed_ops);
+
+ if (timeout->compl_cqp_cmds != completed_ops) {
+ timeout->compl_cqp_cmds = completed_ops;
timeout->count = 0;
- } else if (timeout->compl_cqp_cmds !=
- dev->cqp_cmd_stats[IRDMA_OP_REQ_CMDS]) {
+ } else if (timeout->compl_cqp_cmds != dev->cqp->requested_ops) {
timeout->count++;
}
}
@@ -2886,7 +2830,7 @@
if (newtail != tail) {
/* SUCCESS */
IRDMA_RING_MOVE_TAIL(cqp->sq_ring);
- cqp->dev->cqp_cmd_stats[IRDMA_OP_CMPL_CMDS]++;
+ atomic64_inc(&cqp->completed_ops);
return 0;
}
irdma_usec_delay(cqp->dev->hw_attrs.max_sleep_count);
@@ -3252,8 +3196,8 @@
info->dev->cqp = cqp;
IRDMA_RING_INIT(cqp->sq_ring, cqp->sq_size);
- cqp->dev->cqp_cmd_stats[IRDMA_OP_REQ_CMDS] = 0;
- cqp->dev->cqp_cmd_stats[IRDMA_OP_CMPL_CMDS] = 0;
+ cqp->requested_ops = 0;
+ atomic64_set(&cqp->completed_ops, 0);
/* for the cqp commands backlog. */
INIT_LIST_HEAD(&cqp->dev->cqp_cmd_head);
@@ -3263,8 +3207,8 @@
irdma_debug(cqp->dev, IRDMA_DEBUG_WQE,
"sq_size[%04d] hw_sq_size[%04d] sq_base[%p] sq_pa[%llxh] cqp[%p] polarity[x%04x]\n",
- cqp->sq_size, cqp->hw_sq_size, cqp->sq_base, (unsigned long long)cqp->sq_pa, cqp,
- cqp->polarity);
+ cqp->sq_size, cqp->hw_sq_size, cqp->sq_base,
+ (unsigned long long)cqp->sq_pa, cqp, cqp->polarity);
return 0;
}
@@ -3409,7 +3353,7 @@
if (ret_code)
return NULL;
- cqp->dev->cqp_cmd_stats[IRDMA_OP_REQ_CMDS]++;
+ cqp->requested_ops++;
if (!*wqe_idx)
cqp->polarity = !cqp->polarity;
wqe = cqp->sq_base[*wqe_idx].elem;
@@ -3504,6 +3448,9 @@
if (polarity != ccq->cq_uk.polarity)
return -ENOENT;
+ /* Ensure CEQE contents are read after valid bit is checked */
+ rmb();
+
get_64bit_val(cqe, IRDMA_BYTE_8, &qp_ctx);
cqp = (struct irdma_sc_cqp *)(irdma_uintptr) qp_ctx;
info->error = (bool)FIELD_GET(IRDMA_CQ_ERROR, temp);
@@ -3539,7 +3486,7 @@
irdma_wmb(); /* make sure shadow area is updated before moving tail */
IRDMA_RING_MOVE_TAIL(cqp->sq_ring);
- ccq->dev->cqp_cmd_stats[IRDMA_OP_CMPL_CMDS]++;
+ atomic64_inc(&cqp->completed_ops);
return ret_code;
}
@@ -4159,13 +4106,17 @@
u8 polarity;
aeqe = IRDMA_GET_CURRENT_AEQ_ELEM(aeq);
- get_64bit_val(aeqe, IRDMA_BYTE_0, &compl_ctx);
get_64bit_val(aeqe, IRDMA_BYTE_8, &temp);
polarity = (u8)FIELD_GET(IRDMA_AEQE_VALID, temp);
if (aeq->polarity != polarity)
return -ENOENT;
+ /* Ensure AEQE contents are read after valid bit is checked */
+ rmb();
+
+ get_64bit_val(aeqe, IRDMA_BYTE_0, &compl_ctx);
+
irdma_debug_buf(aeq->dev, IRDMA_DEBUG_WQE, "AEQ_ENTRY WQE", aeqe, 16);
ae_src = (u8)FIELD_GET(IRDMA_AEQE_AESRC, temp);
@@ -4752,16 +4703,18 @@
*/
static u32 irdma_est_sd(struct irdma_sc_dev *dev,
struct irdma_hmc_info *hmc_info){
- int i;
+ struct irdma_hmc_obj_info *pble_info;
u64 size = 0;
u64 sd;
+ int i;
for (i = IRDMA_HMC_IW_QP; i < IRDMA_HMC_IW_MAX; i++)
if (i != IRDMA_HMC_IW_PBLE)
size += round_up(hmc_info->hmc_obj[i].cnt *
hmc_info->hmc_obj[i].size, 512);
- size += round_up(hmc_info->hmc_obj[IRDMA_HMC_IW_PBLE].cnt *
- hmc_info->hmc_obj[IRDMA_HMC_IW_PBLE].size, 512);
+
+ pble_info = &hmc_info->hmc_obj[IRDMA_HMC_IW_PBLE];
+ size += round_up(pble_info->cnt * pble_info->size, 512);
if (size & 0x1FFFFF)
sd = (size >> 21) + 1; /* add 1 for remainder */
else
@@ -4846,8 +4799,7 @@
goto exit;
} else if (feat_cnt > IRDMA_MAX_FEATURES) {
irdma_debug(dev, IRDMA_DEBUG_DEV,
- "feature buf size insufficient,"
- "retrying with larger buffer\n");
+ "feature buf size insufficient, retrying with larger buffer\n");
irdma_free_dma_mem(dev->hw, &feat_buf);
feat_buf.size = 8 * feat_cnt;
feat_buf.va = irdma_allocate_dma_mem(dev->hw, &feat_buf,
@@ -5062,9 +5014,9 @@
pblewanted > (512 * FPM_MULTIPLIER * sd_diff)) {
pblewanted -= 256 * FPM_MULTIPLIER * sd_diff;
continue;
- } else if (pblewanted > (100 * FPM_MULTIPLIER)) {
+ } else if (pblewanted > 100 * FPM_MULTIPLIER) {
pblewanted -= 10 * FPM_MULTIPLIER;
- } else if (pblewanted > FPM_MULTIPLIER) {
+ } else if (pblewanted > 16 * FPM_MULTIPLIER) {
pblewanted -= FPM_MULTIPLIER;
} else if (qpwanted <= 128) {
if (hmc_info->hmc_obj[IRDMA_HMC_IW_FSIMC].cnt > 256)
diff --git a/sys/dev/irdma/irdma_defs.h b/sys/dev/irdma/irdma_defs.h
--- a/sys/dev/irdma/irdma_defs.h
+++ b/sys/dev/irdma/irdma_defs.h
@@ -1,7 +1,7 @@
/*-
* SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB
*
- * Copyright (c) 2015 - 2022 Intel Corporation
+ * Copyright (c) 2015 - 2023 Intel Corporation
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@@ -250,40 +250,38 @@
IRDMA_OP_MANAGE_HMC_PM_FUNC_TABLE = 22,
IRDMA_OP_SUSPEND = 23,
IRDMA_OP_RESUME = 24,
- IRDMA_OP_MANAGE_VF_PBLE_BP = 25,
+ IRDMA_OP_MANAGE_VCHNL_REQ_PBLE_BP = 25,
IRDMA_OP_QUERY_FPM_VAL = 26,
IRDMA_OP_COMMIT_FPM_VAL = 27,
- IRDMA_OP_REQ_CMDS = 28,
- IRDMA_OP_CMPL_CMDS = 29,
- IRDMA_OP_AH_CREATE = 30,
- IRDMA_OP_AH_MODIFY = 31,
- IRDMA_OP_AH_DESTROY = 32,
- IRDMA_OP_MC_CREATE = 33,
- IRDMA_OP_MC_DESTROY = 34,
- IRDMA_OP_MC_MODIFY = 35,
- IRDMA_OP_STATS_ALLOCATE = 36,
- IRDMA_OP_STATS_FREE = 37,
- IRDMA_OP_STATS_GATHER = 38,
- IRDMA_OP_WS_ADD_NODE = 39,
- IRDMA_OP_WS_MODIFY_NODE = 40,
- IRDMA_OP_WS_DELETE_NODE = 41,
- IRDMA_OP_WS_FAILOVER_START = 42,
- IRDMA_OP_WS_FAILOVER_COMPLETE = 43,
- IRDMA_OP_SET_UP_MAP = 44,
- IRDMA_OP_GEN_AE = 45,
- IRDMA_OP_QUERY_RDMA_FEATURES = 46,
- IRDMA_OP_ALLOC_LOCAL_MAC_ENTRY = 47,
- IRDMA_OP_ADD_LOCAL_MAC_ENTRY = 48,
- IRDMA_OP_DELETE_LOCAL_MAC_ENTRY = 49,
- IRDMA_OP_CQ_MODIFY = 50,
+ IRDMA_OP_AH_CREATE = 28,
+ IRDMA_OP_AH_MODIFY = 29,
+ IRDMA_OP_AH_DESTROY = 30,
+ IRDMA_OP_MC_CREATE = 31,
+ IRDMA_OP_MC_DESTROY = 32,
+ IRDMA_OP_MC_MODIFY = 33,
+ IRDMA_OP_STATS_ALLOCATE = 34,
+ IRDMA_OP_STATS_FREE = 35,
+ IRDMA_OP_STATS_GATHER = 36,
+ IRDMA_OP_WS_ADD_NODE = 37,
+ IRDMA_OP_WS_MODIFY_NODE = 38,
+ IRDMA_OP_WS_DELETE_NODE = 39,
+ IRDMA_OP_WS_FAILOVER_START = 40,
+ IRDMA_OP_WS_FAILOVER_COMPLETE = 41,
+ IRDMA_OP_SET_UP_MAP = 42,
+ IRDMA_OP_GEN_AE = 43,
+ IRDMA_OP_QUERY_RDMA_FEATURES = 44,
+ IRDMA_OP_ALLOC_LOCAL_MAC_ENTRY = 45,
+ IRDMA_OP_ADD_LOCAL_MAC_ENTRY = 46,
+ IRDMA_OP_DELETE_LOCAL_MAC_ENTRY = 47,
+ IRDMA_OP_CQ_MODIFY = 48,
/* Must be last entry */
- IRDMA_MAX_CQP_OPS = 51,
+ IRDMA_MAX_CQP_OPS = 49,
};
/* CQP SQ WQES */
-#define IRDMA_CQP_OP_CREATE_QP 0
-#define IRDMA_CQP_OP_MODIFY_QP 0x1
+#define IRDMA_CQP_OP_CREATE_QP 0x00
+#define IRDMA_CQP_OP_MODIFY_QP 0x01
#define IRDMA_CQP_OP_DESTROY_QP 0x02
#define IRDMA_CQP_OP_CREATE_CQ 0x03
#define IRDMA_CQP_OP_MODIFY_CQ 0x04
@@ -295,12 +293,11 @@
#define IRDMA_CQP_OP_DEALLOC_STAG 0x0d
#define IRDMA_CQP_OP_MANAGE_LOC_MAC_TABLE 0x0e
#define IRDMA_CQP_OP_MANAGE_ARP 0x0f
-#define IRDMA_CQP_OP_MANAGE_VF_PBLE_BP 0x10
+#define IRDMA_CQP_OP_MANAGE_VCHNL_REQ_PBLE_BP 0x10
#define IRDMA_CQP_OP_MANAGE_PUSH_PAGES 0x11
#define IRDMA_CQP_OP_QUERY_RDMA_FEATURES 0x12
#define IRDMA_CQP_OP_UPLOAD_CONTEXT 0x13
#define IRDMA_CQP_OP_ALLOCATE_LOC_MAC_TABLE_ENTRY 0x14
-#define IRDMA_CQP_OP_UPLOAD_CONTEXT 0x13
#define IRDMA_CQP_OP_MANAGE_HMC_PM_FUNC_TABLE 0x15
#define IRDMA_CQP_OP_CREATE_CEQ 0x16
#define IRDMA_CQP_OP_DESTROY_CEQ 0x18
@@ -700,7 +697,6 @@
#define IRDMA_CQPSQ_QP_MACVALID BIT_ULL(51)
#define IRDMA_CQPSQ_QP_MSSCHANGE_S 52
#define IRDMA_CQPSQ_QP_MSSCHANGE BIT_ULL(52)
-
#define IRDMA_CQPSQ_QP_IGNOREMWBOUND_S 54
#define IRDMA_CQPSQ_QP_IGNOREMWBOUND BIT_ULL(54)
#define IRDMA_CQPSQ_QP_REMOVEHASHENTRY_S 55
@@ -1403,7 +1399,7 @@
#define IRDMA_GET_CQ_ELEM_AT_OFFSET(_cq, _i, _cqe) \
{ \
- register __u32 offset; \
+ __u32 offset; \
offset = IRDMA_GET_RING_OFFSET((_cq)->cq_ring, _i); \
(_cqe) = (_cq)->cq_base[offset].buf; \
}
@@ -1429,7 +1425,7 @@
#define IRDMA_RING_MOVE_HEAD(_ring, _retcode) \
{ \
- register u32 size; \
+ u32 size; \
size = (_ring).size; \
if (!IRDMA_RING_FULL_ERR(_ring)) { \
(_ring).head = ((_ring).head + 1) % size; \
@@ -1440,7 +1436,7 @@
}
#define IRDMA_RING_MOVE_HEAD_BY_COUNT(_ring, _count, _retcode) \
{ \
- register u32 size; \
+ u32 size; \
size = (_ring).size; \
if ((IRDMA_RING_USED_QUANTA(_ring) + (_count)) < size) { \
(_ring).head = ((_ring).head + (_count)) % size; \
@@ -1451,7 +1447,7 @@
}
#define IRDMA_SQ_RING_MOVE_HEAD(_ring, _retcode) \
{ \
- register u32 size; \
+ u32 size; \
size = (_ring).size; \
if (!IRDMA_SQ_RING_FULL_ERR(_ring)) { \
(_ring).head = ((_ring).head + 1) % size; \
@@ -1462,7 +1458,7 @@
}
#define IRDMA_SQ_RING_MOVE_HEAD_BY_COUNT(_ring, _count, _retcode) \
{ \
- register u32 size; \
+ u32 size; \
size = (_ring).size; \
if ((IRDMA_RING_USED_QUANTA(_ring) + (_count)) < (size - 256)) { \
(_ring).head = ((_ring).head + (_count)) % size; \
@@ -1554,6 +1550,19 @@
IRDMA_WQE_SIZE_256 = 256,
};
+enum irdma_ws_op_type {
+ IRDMA_WS_OP_TYPE_NODE = 0,
+ IRDMA_WS_OP_TYPE_LEAF_NODE_GROUP,
+};
+
+enum irdma_ws_rate_limit_flags {
+ IRDMA_WS_RATE_LIMIT_FLAGS_VALID = 0x1,
+ IRDMA_WS_NO_RDMA_RATE_LIMIT = 0x2,
+ IRDMA_WS_LEAF_NODE_IS_PART_GROUP = 0x4,
+ IRDMA_WS_TREE_RATE_LIMITING = 0x8,
+ IRDMA_WS_PACING_CONTROL = 0x10,
+};
+
enum irdma_ws_node_op {
IRDMA_ADD_NODE = 0,
IRDMA_MODIFY_NODE,
@@ -1580,12 +1589,6 @@
IRDMA_FEATURE_BUF_ALIGNMENT = 0x10,
};
-enum icrdma_protocol_used {
- ICRDMA_ANY_PROTOCOL = 0,
- ICRDMA_IWARP_PROTOCOL_ONLY = 1,
- ICRDMA_ROCE_PROTOCOL_ONLY = 2,
-};
-
/**
* set_64bit_val - set 64 bit value to hw wqe
* @wqe_words: wqe addr to write
diff --git a/sys/dev/irdma/irdma_hmc.h b/sys/dev/irdma/irdma_hmc.h
--- a/sys/dev/irdma/irdma_hmc.h
+++ b/sys/dev/irdma/irdma_hmc.h
@@ -1,7 +1,7 @@
/*-
* SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB
*
- * Copyright (c) 2015 - 2022 Intel Corporation
+ * Copyright (c) 2015 - 2023 Intel Corporation
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@@ -185,10 +185,6 @@
bool setsd);
int irdma_update_sds_noccq(struct irdma_sc_dev *dev,
struct irdma_update_sds_info *info);
-struct irdma_vfdev *irdma_vfdev_from_fpm(struct irdma_sc_dev *dev,
- u16 hmc_fn_id);
-struct irdma_hmc_info *irdma_vf_hmcinfo_from_fpm(struct irdma_sc_dev *dev,
- u16 hmc_fn_id);
int irdma_add_sd_table_entry(struct irdma_hw *hw,
struct irdma_hmc_info *hmc_info, u32 sd_index,
enum irdma_sd_entry_type type, u64 direct_mode_sz);
diff --git a/sys/dev/irdma/irdma_hmc.c b/sys/dev/irdma/irdma_hmc.c
--- a/sys/dev/irdma/irdma_hmc.c
+++ b/sys/dev/irdma/irdma_hmc.c
@@ -1,7 +1,7 @@
/*-
* SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB
*
- * Copyright (c) 2015 - 2022 Intel Corporation
+ * Copyright (c) 2015 - 2023 Intel Corporation
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@@ -422,7 +422,7 @@
if (info->start_idx >= info->hmc_info->hmc_obj[info->rsrc_type].cnt) {
irdma_debug(dev, IRDMA_DEBUG_HMC,
- "error start_idx[%04d] >= [type %04d].cnt[%04d]\n",
+ "error start_idx[%04d] >= [type %04d].cnt[%04d]\n",
info->start_idx, info->rsrc_type,
info->hmc_info->hmc_obj[info->rsrc_type].cnt);
return -EINVAL;
@@ -431,7 +431,7 @@
if ((info->start_idx + info->count) >
info->hmc_info->hmc_obj[info->rsrc_type].cnt) {
irdma_debug(dev, IRDMA_DEBUG_HMC,
- "error start_idx[%04d] + count %04d >= [type %04d].cnt[%04d]\n",
+ "error start_idx[%04d] + count %04d >= [type %04d].cnt[%04d]\n",
info->start_idx, info->count, info->rsrc_type,
info->hmc_info->hmc_obj[info->rsrc_type].cnt);
return -EINVAL;
diff --git a/sys/dev/irdma/irdma_hw.c b/sys/dev/irdma/irdma_hw.c
--- a/sys/dev/irdma/irdma_hw.c
+++ b/sys/dev/irdma/irdma_hw.c
@@ -1,7 +1,7 @@
/*-
* SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB
*
- * Copyright (c) 2015 - 2022 Intel Corporation
+ * Copyright (c) 2015 - 2023 Intel Corporation
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@@ -124,8 +124,7 @@
}
if (compl_error) {
irdma_debug(dev, IRDMA_DEBUG_ERR,
- "puda compl_err =0x%x\n",
- compl_error);
+ "puda compl_err = 0x%x\n", compl_error);
break;
}
} while (1);
@@ -194,12 +193,11 @@
irdma_complete_cqp_request(struct irdma_cqp *cqp,
struct irdma_cqp_request *cqp_request)
{
- if (cqp_request->waiting) {
- cqp_request->request_done = true;
+ WRITE_ONCE(cqp_request->request_done, true);
+ if (cqp_request->waiting)
wake_up(&cqp_request->waitq);
- } else if (cqp_request->callback_fcn) {
+ else if (cqp_request->callback_fcn)
cqp_request->callback_fcn(cqp_request);
- }
irdma_put_cqp_request(cqp, cqp_request);
}
@@ -236,8 +234,9 @@
aeqcnt++;
irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_AEQ,
- "ae_id = 0x%x bool qp=%d qp_id = %d tcp_state=%d iwarp_state=%d ae_src=%d\n",
- info->ae_id, info->qp, info->qp_cq_id, info->tcp_state,
+ "ae_id = 0x%x (%s), is_qp = %d, qp_id = %d, tcp_state = %d, iwarp_state = %d, ae_src = %d\n",
+ info->ae_id, irdma_get_ae_desc(info->ae_id),
+ info->qp, info->qp_cq_id, info->tcp_state,
info->iwarp_state, info->ae_src);
if (info->qp) {
@@ -323,7 +322,11 @@
break;
case IRDMA_AE_QP_SUSPEND_COMPLETE:
if (iwqp->iwdev->vsi.tc_change_pending) {
- atomic_dec(&iwqp->sc_qp.vsi->qp_suspend_reqs);
+ if (!atomic_dec_return(&iwqp->sc_qp.vsi->qp_suspend_reqs))
+ wake_up(&iwqp->iwdev->suspend_wq);
+ }
+ if (iwqp->suspend_pending) {
+ iwqp->suspend_pending = false;
wake_up(&iwqp->iwdev->suspend_wq);
}
break;
@@ -393,8 +396,10 @@
case IRDMA_AE_LCE_FUNCTION_CATASTROPHIC:
case IRDMA_AE_UDA_XMIT_DGRAM_TOO_LONG:
default:
- irdma_dev_err(&iwdev->ibdev, "abnormal ae_id = 0x%x bool qp=%d qp_id = %d ae_source=%d\n",
- info->ae_id, info->qp, info->qp_cq_id, info->ae_src);
+ irdma_dev_err(&iwdev->ibdev,
+ "AEQ: abnormal ae_id = 0x%x (%s), is_qp = %d, qp_id = %d, ae_source = %d\n",
+ info->ae_id, irdma_get_ae_desc(info->ae_id),
+ info->qp, info->qp_cq_id, info->ae_src);
if (rdma_protocol_roce(&iwqp->iwdev->ibdev, 1)) {
ctx_info->roce_info->err_rq_idx_valid = info->err_rq_idx_valid;
if (info->rq) {
@@ -698,8 +703,7 @@
status = irdma_sc_cceq_destroy_done(&iwceq->sc_ceq);
if (status)
irdma_debug(dev, IRDMA_DEBUG_ERR,
- "CEQ destroy completion failed %d\n",
- status);
+ "CEQ destroy completion failed %d\n", status);
exit:
spin_lock_destroy(&iwceq->ce_lock);
spin_lock_destroy(&iwceq->sc_ceq.req_cq_lock);
@@ -811,8 +815,7 @@
info.privileged = privileged;
if (irdma_sc_del_hmc_obj(dev, &info, reset))
irdma_debug(dev, IRDMA_DEBUG_ERR,
- "del HMC obj of type %d failed\n",
- obj_type);
+ "del HMC obj of type %d failed\n", obj_type);
}
/**
@@ -1026,7 +1029,7 @@
irdma_debug(dev, IRDMA_DEBUG_ERR,
"cqp create failed - status %d maj_err %d min_err %d\n",
status, maj_err, min_err);
- goto err_create;
+ goto err_ctx;
}
INIT_LIST_HEAD(&cqp->cqp_avail_reqs);
@@ -1040,7 +1043,6 @@
init_waitqueue_head(&cqp->remove_wq);
return 0;
-err_create:
err_ctx:
irdma_free_dma_mem(dev->hw, &cqp->sq);
err_sq:
@@ -1155,16 +1157,14 @@
msix_vec->res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE);
if (!msix_vec->res) {
irdma_debug(&rf->sc_dev, IRDMA_DEBUG_ERR,
- "Unable to allocate bus resource int[%d]\n",
- rid);
+ "Unable to allocate bus resource int[%d]\n", rid);
return -EINVAL;
}
err = bus_setup_intr(dev, msix_vec->res, INTR_TYPE_NET | INTR_MPSAFE,
NULL, handler, argument, &msix_vec->tag);
if (err) {
irdma_debug(&rf->sc_dev, IRDMA_DEBUG_ERR,
- "Unable to register handler with %x status\n",
- err);
+ "Unable to register handler with %x status\n", err);
status = -EINVAL;
goto fail_intr;
}
@@ -1230,20 +1230,21 @@
irdma_cfg_aeq_vector(struct irdma_pci_f *rf)
{
struct irdma_msix_vector *msix_vec = rf->iw_msixtbl;
- u32 ret = 0;
+ int status = 0;
if (!rf->msix_shared) {
snprintf(msix_vec->name, sizeof(msix_vec->name) - 1,
"irdma-%s-AEQ", dev_name(&rf->pcidev->dev));
tasklet_setup(&rf->dpc_tasklet, irdma_dpc);
- ret = irdma_irq_request(rf, msix_vec, irdma_irq_handler, rf);
- if (ret)
- return ret;
+ status = irdma_irq_request(rf, msix_vec, irdma_irq_handler, rf);
+ if (status)
+ return status;
bus_describe_intr(rf->dev_ctx.dev, msix_vec->res, msix_vec->tag, "%s", msix_vec->name);
}
- if (ret) {
+
+ if (status) {
irdma_debug(&rf->sc_dev, IRDMA_DEBUG_ERR, "aeq irq config fail\n");
- return -EINVAL;
+ return status;
}
rf->sc_dev.irq_ops->irdma_cfg_aeq(&rf->sc_dev, msix_vec->idx, true);
@@ -1338,8 +1339,7 @@
status = irdma_create_ceq(rf, iwceq, 0, &rf->default_vsi);
if (status) {
irdma_debug(&rf->sc_dev, IRDMA_DEBUG_ERR,
- "create ceq status = %d\n",
- status);
+ "create ceq status = %d\n", status);
goto exit;
}
@@ -1394,8 +1394,7 @@
status = irdma_create_ceq(rf, iwceq, ceq_id, vsi);
if (status) {
irdma_debug(&rf->sc_dev, IRDMA_DEBUG_ERR,
- "create ceq status = %d\n",
- status);
+ "create ceq status = %d\n", status);
goto del_ceqs;
}
spin_lock_init(&iwceq->ce_lock);
@@ -1634,8 +1633,8 @@
static int
irdma_hmc_setup(struct irdma_pci_f *rf)
{
- int status;
struct irdma_sc_dev *dev = &rf->sc_dev;
+ int status;
u32 qpcnt;
qpcnt = rsrc_limits_table[rf->limits_sel].qplimit;
@@ -2429,8 +2428,9 @@
cqp_info->post_sq = 1;
cqp_info->in.u.manage_apbvt_entry.cqp = &iwdev->rf->cqp.sc_cqp;
cqp_info->in.u.manage_apbvt_entry.scratch = (uintptr_t)cqp_request;
- irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_DEV, "%s: port=0x%04x\n",
- (!add_port) ? "DELETE" : "ADD", accel_local_port);
+ irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_DEV,
+ "%s: port=0x%04x\n", (!add_port) ? "DELETE" : "ADD",
+ accel_local_port);
status = irdma_handle_cqp_op(iwdev->rf, cqp_request);
irdma_put_cqp_request(&iwdev->rf->cqp, cqp_request);
@@ -2591,6 +2591,9 @@
if (!cqp_request)
return -ENOMEM;
+ cminfo->cqp_request = cqp_request;
+ if (!wait)
+ atomic_inc(&cqp_request->refcnt);
cqp_info = &cqp_request->info;
info = &cqp_info->in.u.manage_qhash_table_entry.info;
memset(info, 0, sizeof(*info));
@@ -2631,19 +2634,24 @@
}
if (info->ipv4_valid)
irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_CM,
- "%s caller: %pS loc_port=0x%04x rem_port=0x%04x loc_addr=%pI4 rem_addr=%pI4 mac=%pM, vlan_id=%d cm_node=%p\n",
+ "%s caller: %pS loc_port=0x%04x rem_port=0x%04x loc_addr=%x rem_addr=%x mac=%x:%x:%x:%x:%x:%x, vlan_id=%d cm_node=%p\n",
(!mtype) ? "DELETE" : "ADD",
- __builtin_return_address(0), info->dest_port,
- info->src_port, info->dest_ip, info->src_ip,
- info->mac_addr, cminfo->vlan_id,
- cmnode ? cmnode : NULL);
+ __builtin_return_address(0), info->src_port,
+ info->dest_port, info->src_ip[0], info->dest_ip[0],
+ info->mac_addr[0], info->mac_addr[1],
+ info->mac_addr[2], info->mac_addr[3],
+ info->mac_addr[4], info->mac_addr[5],
+ cminfo->vlan_id, cmnode ? cmnode : NULL);
else
irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_CM,
- "%s caller: %pS loc_port=0x%04x rem_port=0x%04x loc_addr=%pI6 rem_addr=%pI6 mac=%pM, vlan_id=%d cm_node=%p\n",
+ "%s caller: %pS loc_port=0x%04x rem_port=0x%04x loc_addr=%x:%x:%x:%x rem_addr=%x:%x:%x:%x mac=%x:%x:%x:%x:%x:%x, vlan_id=%d cm_node=%p\n",
(!mtype) ? "DELETE" : "ADD",
- __builtin_return_address(0), info->dest_port,
- info->src_port, info->dest_ip, info->src_ip,
- info->mac_addr, cminfo->vlan_id,
+ __builtin_return_address(0), info->src_port,
+ info->dest_port, IRDMA_PRINT_IP6(info->src_ip),
+ IRDMA_PRINT_IP6(info->dest_ip), info->mac_addr[0],
+ info->mac_addr[1], info->mac_addr[2],
+ info->mac_addr[3], info->mac_addr[4],
+ info->mac_addr[5], cminfo->vlan_id,
cmnode ? cmnode : NULL);
cqp_info->in.u.manage_qhash_table_entry.cqp = &iwdev->rf->cqp.sc_cqp;
@@ -2717,7 +2725,8 @@
"qp_id=%d qp_type=%d qpstate=%d ibqpstate=%d last_aeq=%d hw_iw_state=%d maj_err_code=%d min_err_code=%d\n",
iwqp->ibqp.qp_num, rf->protocol_used, iwqp->iwarp_state,
iwqp->ibqp_state, iwqp->last_aeq, iwqp->hw_iwarp_state,
- cqp_request->compl_info.maj_err_code, cqp_request->compl_info.min_err_code);
+ cqp_request->compl_info.maj_err_code,
+ cqp_request->compl_info.min_err_code);
put_cqp:
irdma_put_cqp_request(&rf->cqp, cqp_request);
diff --git a/sys/dev/irdma/irdma_kcompat.c b/sys/dev/irdma/irdma_kcompat.c
--- a/sys/dev/irdma/irdma_kcompat.c
+++ b/sys/dev/irdma/irdma_kcompat.c
@@ -1,7 +1,7 @@
/*-
* SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB
*
- * Copyright (c) 2018 - 2022 Intel Corporation
+ * Copyright (c) 2018 - 2023 Intel Corporation
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@@ -609,36 +609,37 @@
static void
irdma_fill_ah_info(struct vnet *vnet, struct irdma_ah_info *ah_info,
const struct ib_gid_attr *sgid_attr,
- struct sockaddr *sgid_addr, struct sockaddr *dgid_addr,
+ union irdma_sockaddr *sgid_addr,
+ union irdma_sockaddr *dgid_addr,
u8 *dmac, u8 net_type)
{
if (net_type == RDMA_NETWORK_IPV4) {
ah_info->ipv4_valid = true;
ah_info->dest_ip_addr[0] =
- ntohl(((struct sockaddr_in *)dgid_addr)->sin_addr.s_addr);
+ ntohl(dgid_addr->saddr_in.sin_addr.s_addr);
ah_info->src_ip_addr[0] =
- ntohl(((struct sockaddr_in *)sgid_addr)->sin_addr.s_addr);
+ ntohl(sgid_addr->saddr_in.sin_addr.s_addr);
CURVNET_SET_QUIET(vnet);
ah_info->do_lpbk = irdma_ipv4_is_lpb(ah_info->src_ip_addr[0],
ah_info->dest_ip_addr[0]);
CURVNET_RESTORE();
- if (ipv4_is_multicast(((struct sockaddr_in *)dgid_addr)->sin_addr.s_addr)) {
+ if (ipv4_is_multicast(dgid_addr->saddr_in.sin_addr.s_addr)) {
irdma_mcast_mac_v4(ah_info->dest_ip_addr, dmac);
}
} else {
irdma_copy_ip_ntohl(ah_info->dest_ip_addr,
- ((struct sockaddr_in6 *)dgid_addr)->sin6_addr.__u6_addr.__u6_addr32);
+ dgid_addr->saddr_in6.sin6_addr.__u6_addr.__u6_addr32);
irdma_copy_ip_ntohl(ah_info->src_ip_addr,
- ((struct sockaddr_in6 *)sgid_addr)->sin6_addr.__u6_addr.__u6_addr32);
+ sgid_addr->saddr_in6.sin6_addr.__u6_addr.__u6_addr32);
ah_info->do_lpbk = irdma_ipv6_is_lpb(ah_info->src_ip_addr,
ah_info->dest_ip_addr);
- if (rdma_is_multicast_addr(&((struct sockaddr_in6 *)dgid_addr)->sin6_addr)) {
+ if (rdma_is_multicast_addr(&dgid_addr->saddr_in6.sin6_addr)) {
irdma_mcast_mac_v6(ah_info->dest_ip_addr, dmac);
}
}
}
-static inline u8 irdma_get_vlan_ndev_prio(if_t ndev, u8 prio)
+static inline u8 irdma_roce_get_vlan_prio(if_t ndev, u8 prio)
{
return prio;
}
@@ -666,10 +667,9 @@
ah_info->vlan_tag = 0;
if (ah_info->vlan_tag < VLAN_N_VID) {
- if_t ndev = sgid_attr->ndev;
-
ah_info->insert_vlan_tag = true;
- vlan_prio = (u16)irdma_get_vlan_ndev_prio(ndev, rt_tos2priority(ah_info->tc_tos));
+ vlan_prio = (u16)irdma_roce_get_vlan_prio(sgid_attr->ndev,
+ rt_tos2priority(ah_info->tc_tos));
ah_info->vlan_tag |= vlan_prio << VLAN_PRIO_SHIFT;
irdma_find_qp_update_qs(iwdev->rf, pd, vlan_prio);
}
@@ -726,14 +726,10 @@
struct irdma_sc_ah *sc_ah;
u32 ah_id = 0;
struct irdma_ah_info *ah_info;
- struct irdma_create_ah_resp uresp;
- union {
- struct sockaddr saddr;
- struct sockaddr_in saddr_in;
- struct sockaddr_in6 saddr_in6;
- } sgid_addr, dgid_addr;
+ struct irdma_create_ah_resp uresp = {};
+ union irdma_sockaddr sgid_addr, dgid_addr;
int err;
- u8 dmac[ETH_ALEN];
+ u8 dmac[ETHER_ADDR_LEN];
bool sleep = (flags & RDMA_CREATE_AH_SLEEPABLE) != 0;
if (udata && udata->outlen < IRDMA_CREATE_AH_MIN_RESP_LEN)
@@ -766,15 +762,11 @@
rdma_gid2ip((struct sockaddr *)&sgid_addr, &sgid);
rdma_gid2ip((struct sockaddr *)&dgid_addr, &attr->grh.dgid);
ah->av.attrs = *attr;
- ah->av.net_type = kc_rdma_gid_attr_network_type(sgid_attr,
- sgid_attr.gid_type,
- &sgid);
+ ah->av.net_type = ib_gid_to_network_type(sgid_attr.gid_type, &sgid);
if (sgid_attr.ndev)
dev_put(sgid_attr.ndev);
- ah->av.sgid_addr.saddr = sgid_addr.saddr;
- ah->av.dgid_addr.saddr = dgid_addr.saddr;
ah_info = &sc_ah->ah_info;
ah_info->ah_idx = ah_id;
ah_info->pd_idx = pd->sc_pd.pd_id;
@@ -788,7 +780,7 @@
ether_addr_copy(dmac, attr->dmac);
- irdma_fill_ah_info(if_getvnet(iwdev->netdev), ah_info, &sgid_attr, &sgid_addr.saddr, &dgid_addr.saddr,
+ irdma_fill_ah_info(if_getvnet(iwdev->netdev), ah_info, &sgid_attr, &sgid_addr, &dgid_addr,
dmac, ah->av.net_type);
err = irdma_create_ah_vlan_tag(iwdev, pd, ah_info, &sgid_attr, dmac);
@@ -810,7 +802,7 @@
if (udata) {
uresp.ah_id = ah->sc_ah.ah_info.ah_idx;
- err = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
+ err = ib_copy_to_udata(udata, &uresp, min(sizeof(uresp), udata->outlen));
if (err) {
irdma_ah_cqp_op(iwdev->rf, &ah->sc_ah,
IRDMA_OP_AH_DESTROY, false, NULL, ah);
@@ -888,14 +880,10 @@
struct irdma_sc_ah *sc_ah;
u32 ah_id = 0;
struct irdma_ah_info *ah_info;
- struct irdma_create_ah_resp uresp;
- union {
- struct sockaddr saddr;
- struct sockaddr_in saddr_in;
- struct sockaddr_in6 saddr_in6;
- } sgid_addr, dgid_addr;
+ struct irdma_create_ah_resp uresp = {};
+ union irdma_sockaddr sgid_addr, dgid_addr;
int err;
- u8 dmac[ETH_ALEN];
+ u8 dmac[ETHER_ADDR_LEN];
bool sleep = udata ? true : false;
if (udata && udata->outlen < IRDMA_CREATE_AH_MIN_RESP_LEN)
@@ -934,15 +922,11 @@
rdma_gid2ip((struct sockaddr *)&sgid_addr, &sgid);
rdma_gid2ip((struct sockaddr *)&dgid_addr, &attr->grh.dgid);
ah->av.attrs = *attr;
- ah->av.net_type = kc_rdma_gid_attr_network_type(sgid_attr,
- sgid_attr.gid_type,
- &sgid);
+ ah->av.net_type = ib_gid_to_network_type(sgid_attr.gid_type, &sgid);
if (sgid_attr.ndev)
dev_put(sgid_attr.ndev);
- ah->av.sgid_addr.saddr = sgid_addr.saddr;
- ah->av.dgid_addr.saddr = dgid_addr.saddr;
ah_info = &sc_ah->ah_info;
ah_info->ah_idx = ah_id;
ah_info->pd_idx = pd->sc_pd.pd_id;
@@ -958,7 +942,7 @@
ib_resolve_eth_dmac(ibpd->device, attr);
irdma_ether_copy(dmac, attr);
- irdma_fill_ah_info(if_getvnet(iwdev->netdev), ah_info, &sgid_attr, &sgid_addr.saddr, &dgid_addr.saddr,
+ irdma_fill_ah_info(if_getvnet(iwdev->netdev), ah_info, &sgid_attr, &sgid_addr, &dgid_addr,
dmac, ah->av.net_type);
err = irdma_create_ah_vlan_tag(iwdev, pd, ah_info, &sgid_attr, dmac);
@@ -980,7 +964,7 @@
if (udata) {
uresp.ah_id = ah->sc_ah.ah_info.ah_idx;
- err = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
+ err = ib_copy_to_udata(udata, &uresp, min(sizeof(uresp), udata->outlen));
if (err) {
irdma_ah_cqp_op(iwdev->rf, &ah->sc_ah,
IRDMA_OP_AH_DESTROY, false, NULL, ah);
@@ -1129,6 +1113,8 @@
init_waitqueue_head(&iwqp->waitq);
init_waitqueue_head(&iwqp->mod_qp_waitq);
+ spin_lock_init(&iwqp->dwork_flush_lock);
+
if (udata) {
init_info.qp_uk_init_info.abi_ver = iwpd->sc_pd.abi_ver;
err_code = irdma_setup_umode_qp(udata, iwdev, iwqp, &init_info, init_attr);
@@ -1257,15 +1243,9 @@
list_del(&iwqp->ud_list_elem);
spin_unlock_irqrestore(&iwqp->iwpd->udqp_list_lock, flags);
- if (iwqp->iwarp_state == IRDMA_QP_STATE_RTS)
+ if (iwqp->iwarp_state >= IRDMA_QP_STATE_IDLE)
irdma_modify_qp_to_err(&iwqp->sc_qp);
- irdma_qp_rem_ref(&iwqp->ibqp);
- wait_for_completion(&iwqp->free_qp);
- irdma_free_lsmm_rsrc(iwqp);
- if (!iwdev->rf->reset && irdma_cqp_qp_destroy_cmd(&iwdev->rf->sc_dev, &iwqp->sc_qp))
- return (iwdev->rf->rdma_ver <= IRDMA_GEN_2 && !iwqp->user_mode) ? 0 : -ENOTRECOVERABLE;
-free_rsrc:
if (!iwqp->user_mode) {
if (iwqp->iwscq) {
irdma_clean_cqes(iwqp, iwqp->iwscq);
@@ -1273,6 +1253,12 @@
irdma_clean_cqes(iwqp, iwqp->iwrcq);
}
}
+ irdma_qp_rem_ref(&iwqp->ibqp);
+ wait_for_completion(&iwqp->free_qp);
+ irdma_free_lsmm_rsrc(iwqp);
+ if (!iwdev->rf->reset && irdma_cqp_qp_destroy_cmd(&iwdev->rf->sc_dev, &iwqp->sc_qp))
+ return (iwdev->rf->rdma_ver <= IRDMA_GEN_2 && !iwqp->user_mode) ? 0 : -ENOTRECOVERABLE;
+free_rsrc:
irdma_remove_push_mmap_entries(iwqp);
irdma_free_qp_rsrc(iwqp);
@@ -1464,7 +1450,7 @@
if (!iwcq->kmem_shadow.va) {
err_code = -ENOMEM;
- goto cq_free_rsrc;
+ goto cq_kmem_free;
}
info.shadow_area_pa = iwcq->kmem_shadow.pa;
ukinfo->shadow_area = iwcq->kmem_shadow.va;
@@ -1472,19 +1458,18 @@
info.cq_base_pa = iwcq->kmem.pa;
}
- if (dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2)
- info.shadow_read_threshold = min(info.cq_uk_init_info.cq_size / 2,
- (u32)IRDMA_MAX_CQ_READ_THRESH);
+ info.shadow_read_threshold = min(info.cq_uk_init_info.cq_size / 2,
+ (u32)IRDMA_MAX_CQ_READ_THRESH);
if (irdma_sc_cq_init(cq, &info)) {
irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_VERBS, "init cq fail\n");
err_code = -EPROTO;
- goto cq_free_rsrc;
+ goto cq_kmem_free;
}
cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true);
if (!cqp_request) {
err_code = -ENOMEM;
- goto cq_free_rsrc;
+ goto cq_kmem_free;
}
cqp_info = &cqp_request->info;
cqp_info->cqp_cmd = IRDMA_OP_CQ_CREATE;
@@ -1496,7 +1481,7 @@
irdma_put_cqp_request(&rf->cqp, cqp_request);
if (status) {
err_code = -ENOMEM;
- goto cq_free_rsrc;
+ goto cq_kmem_free;
}
if (udata) {
@@ -1522,8 +1507,13 @@
#endif
cq_destroy:
irdma_cq_wq_destroy(rf, cq);
+cq_kmem_free:
+ if (!iwcq->user_mode) {
+ irdma_free_dma_mem(dev->hw, &iwcq->kmem);
+ irdma_free_dma_mem(dev->hw, &iwcq->kmem_shadow);
+ }
cq_free_rsrc:
- irdma_cq_free_rsrc(rf, iwcq);
+ irdma_free_rsrc(rf, rf->allocated_cqs, cq_num);
#if __FreeBSD_version >= 1400026
return err_code;
#else
@@ -1737,9 +1727,7 @@
ether_addr_copy(iwqp->ctx_info.roce_info->mac_addr, if_getlladdr(sgid_attr.ndev));
}
- av->net_type = kc_rdma_gid_attr_network_type(sgid_attr,
- sgid_attr.gid_type,
- &sgid);
+ av->net_type = ib_gid_to_network_type(sgid_attr.gid_type, &sgid);
rdma_gid2ip((struct sockaddr *)&av->sgid_addr, &sgid);
dev_put(sgid_attr.ndev);
iwqp->sc_qp.user_pri = iwqp->ctx_info.user_pri;
@@ -1837,6 +1825,9 @@
int err_code;
u32 stag;
+ if (type != IB_MW_TYPE_1 && type != IB_MW_TYPE_2)
+ return ERR_PTR(-EINVAL);
+
iwmr = kzalloc(sizeof(*iwmr), GFP_KERNEL);
if (!iwmr)
return ERR_PTR(-ENOMEM);
@@ -2154,7 +2145,7 @@
if (rdma_protocol_roce(ibdev, 1)) {
props->gid_tbl_len = 32;
- kc_set_props_ip_gid_caps(props);
+ props->port_cap_flags |= IB_PORT_IP_BASED_GIDS;
props->pkey_tbl_len = IRDMA_PKEY_TBL_SZ;
} else {
props->gid_tbl_len = 1;
@@ -2381,3 +2372,15 @@
return 0;
}
+
+u64
+irdma_mac_to_u64(const u8 *eth_add)
+{
+ int idx;
+ u64 u64_eth_add;
+
+ for (idx = 0, u64_eth_add = 0; idx < ETHER_ADDR_LEN; idx++)
+ u64_eth_add = u64_eth_add << 8 | eth_add[idx];
+
+ return u64_eth_add;
+}
diff --git a/sys/dev/irdma/irdma_main.h b/sys/dev/irdma/irdma_main.h
--- a/sys/dev/irdma/irdma_main.h
+++ b/sys/dev/irdma/irdma_main.h
@@ -1,7 +1,7 @@
/*-
* SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB
*
- * Copyright (c) 2015 - 2022 Intel Corporation
+ * Copyright (c) 2015 - 2023 Intel Corporation
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@@ -152,6 +152,11 @@
IP_ADDR_REGISTERED, /* Last state of open */
};
+struct ae_desc {
+ u16 id;
+ const char *desc;
+};
+
struct irdma_rsrc_limits {
u32 qplimit;
u32 mrlimit;
@@ -180,8 +185,8 @@
void (*callback_fcn)(struct irdma_cqp_request *cqp_request);
void *param;
struct irdma_cqp_compl_info compl_info;
+ bool request_done; /* READ/WRITE_ONCE macros operate on it */
bool waiting:1;
- bool request_done:1;
bool dynamic:1;
};
@@ -224,7 +229,7 @@
struct irdma_arp_entry {
u32 ip_addr[4];
- u8 mac_addr[ETH_ALEN];
+ u8 mac_addr[ETHER_ADDR_LEN];
};
struct irdma_msix_vector {
@@ -535,6 +540,7 @@
int irdma_alloc_local_mac_entry(struct irdma_pci_f *rf, u16 *mac_tbl_idx);
int irdma_add_local_mac_entry(struct irdma_pci_f *rf, const u8 *mac_addr, u16 idx);
void irdma_del_local_mac_entry(struct irdma_pci_f *rf, u16 idx);
+const char *irdma_get_ae_desc(u16 ae_id);
u32 irdma_initialize_hw_rsrc(struct irdma_pci_f *rf);
void irdma_port_ibevent(struct irdma_device *iwdev);
diff --git a/sys/dev/irdma/irdma_pble.h b/sys/dev/irdma/irdma_pble.h
--- a/sys/dev/irdma/irdma_pble.h
+++ b/sys/dev/irdma/irdma_pble.h
@@ -1,7 +1,7 @@
/*-
* SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB
*
- * Copyright (c) 2015 - 2022 Intel Corporation
+ * Copyright (c) 2015 - 2023 Intel Corporation
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@@ -102,7 +102,7 @@
struct irdma_chunk {
struct list_head list;
struct irdma_dma_info dmainfo;
- void *bitmapbuf;
+ unsigned long *bitmapbuf;
u32 sizeofbitmap;
u64 size;
@@ -111,7 +111,6 @@
u32 pg_cnt;
enum irdma_alloc_type type;
struct irdma_sc_dev *dev;
- struct irdma_virt_mem bitmapmem;
struct irdma_virt_mem chunkmem;
};
@@ -162,5 +161,4 @@
unsigned long *flags);
void irdma_pble_free_paged_mem(struct irdma_chunk *chunk);
int irdma_pble_get_paged_mem(struct irdma_chunk *chunk, u32 pg_cnt);
-void irdma_prm_rem_bitmapmem(struct irdma_hw *hw, struct irdma_chunk *chunk);
#endif /* IRDMA_PBLE_H */
diff --git a/sys/dev/irdma/irdma_pble.c b/sys/dev/irdma/irdma_pble.c
--- a/sys/dev/irdma/irdma_pble.c
+++ b/sys/dev/irdma/irdma_pble.c
@@ -1,7 +1,7 @@
/*-
* SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB
*
- * Copyright (c) 2015 - 2022 Intel Corporation
+ * Copyright (c) 2015 - 2023 Intel Corporation
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@@ -57,8 +57,7 @@
list_del(&chunk->list);
if (chunk->type == PBLE_SD_PAGED)
irdma_pble_free_paged_mem(chunk);
- if (chunk->bitmapbuf)
- irdma_prm_rem_bitmapmem(pble_rsrc->dev->hw, chunk);
+ bitmap_free(chunk->bitmapbuf);
kfree(chunk->chunkmem.va);
}
spin_lock_destroy(&pinfo->prm_lock);
@@ -290,7 +289,8 @@
irdma_debug(dev, IRDMA_DEBUG_PBLE,
"pages = %d, unallocated_pble[%d] current_fpm_addr = %lx\n",
- pages, pble_rsrc->unallocated_pble, pble_rsrc->next_fpm_addr);
+ pages, pble_rsrc->unallocated_pble,
+ pble_rsrc->next_fpm_addr);
irdma_debug(dev, IRDMA_DEBUG_PBLE, "sd_entry_type = %d\n",
sd_entry_type);
if (sd_entry_type == IRDMA_SD_TYPE_DIRECT)
@@ -304,14 +304,14 @@
if (sd_entry_type == IRDMA_SD_TYPE_PAGED) {
ret_code = add_bp_pages(pble_rsrc, &info);
if (ret_code)
- goto error;
+ goto err_bp_pages;
else
pble_rsrc->stats_paged_sds++;
}
ret_code = irdma_prm_add_pble_mem(&pble_rsrc->pinfo, chunk);
if (ret_code)
- goto error;
+ goto err_bp_pages;
pble_rsrc->next_fpm_addr += chunk->size;
irdma_debug(dev, IRDMA_DEBUG_PBLE,
@@ -333,8 +333,8 @@
return 0;
error:
- if (chunk->bitmapbuf)
- irdma_prm_rem_bitmapmem(pble_rsrc->dev->hw, chunk);
+ bitmap_free(chunk->bitmapbuf);
+err_bp_pages:
kfree(chunk->chunkmem.va);
return ret_code;
diff --git a/sys/dev/irdma/irdma_protos.h b/sys/dev/irdma/irdma_protos.h
--- a/sys/dev/irdma/irdma_protos.h
+++ b/sys/dev/irdma/irdma_protos.h
@@ -1,7 +1,7 @@
/*-
* SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB
*
- * Copyright (c) 2016 - 2022 Intel Corporation
+ * Copyright (c) 2016 - 2023 Intel Corporation
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@@ -61,8 +61,6 @@
struct irdma_vsi_pestat *pestat, bool wait);
int irdma_cqp_ws_node_cmd(struct irdma_sc_dev *dev, u8 cmd,
struct irdma_ws_node_info *node_info);
-int irdma_cqp_up_map_cmd(struct irdma_sc_dev *dev, u8 cmd,
- struct irdma_up_info *map_info);
int irdma_cqp_ceq_cmd(struct irdma_sc_dev *dev, struct irdma_sc_ceq *sc_ceq,
u8 op);
int irdma_cqp_aeq_cmd(struct irdma_sc_dev *dev, struct irdma_sc_aeq *sc_aeq,
@@ -131,8 +129,6 @@
struct irdma_dma_mem *val_mem, u16 hmc_fn_id);
int irdma_alloc_query_fpm_buf(struct irdma_sc_dev *dev,
struct irdma_dma_mem *mem);
-int irdma_cqp_manage_hmc_fcn_cmd(struct irdma_sc_dev *dev,
- struct irdma_hmc_fcn_info *hmcfcninfo,
- u16 *pmf_idx);
void *irdma_remove_cqp_head(struct irdma_sc_dev *dev);
+u64 irdma_mac_to_u64(const u8 *eth_add);
#endif /* IRDMA_PROTOS_H */
diff --git a/sys/dev/irdma/irdma_puda.h b/sys/dev/irdma/irdma_puda.h
--- a/sys/dev/irdma/irdma_puda.h
+++ b/sys/dev/irdma/irdma_puda.h
@@ -1,7 +1,7 @@
/*-
* SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB
*
- * Copyright (c) 2015 - 2022 Intel Corporation
+ * Copyright (c) 2015 - 2023 Intel Corporation
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@@ -72,7 +72,7 @@
bool ipv4:1;
bool smac_valid:1;
bool vlan_valid:1;
- u8 smac[ETH_ALEN];
+ u8 smac[ETHER_ADDR_LEN];
};
struct irdma_puda_send_info {
@@ -109,7 +109,7 @@
bool smac_valid:1;
u32 seqnum;
u32 ah_id;
- u8 smac[ETH_ALEN];
+ u8 smac[ETHER_ADDR_LEN];
struct irdma_sc_vsi *vsi;
};
diff --git a/sys/dev/irdma/irdma_puda.c b/sys/dev/irdma/irdma_puda.c
--- a/sys/dev/irdma/irdma_puda.c
+++ b/sys/dev/irdma/irdma_puda.c
@@ -1,7 +1,7 @@
/*-
* SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB
*
- * Copyright (c) 2015 - 2022 Intel Corporation
+ * Copyright (c) 2015 - 2023 Intel Corporation
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@@ -270,6 +270,9 @@
if (valid_bit != cq_uk->polarity)
return -ENOENT;
+ /* Ensure CQE contents are read after valid bit is checked */
+ rmb();
+
if (cq->dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2)
ext_valid = (bool)FIELD_GET(IRDMA_CQ_EXTCQE, qword3);
@@ -283,6 +286,9 @@
if (polarity != cq_uk->polarity)
return -ENOENT;
+ /* Ensure ext CQE contents are read after ext valid bit is checked */
+ rmb();
+
IRDMA_RING_MOVE_HEAD_NOCHECK(cq_uk->cq_ring);
if (!IRDMA_RING_CURRENT_HEAD(cq_uk->cq_ring))
cq_uk->polarity = !cq_uk->polarity;
diff --git a/sys/dev/irdma/irdma_type.h b/sys/dev/irdma/irdma_type.h
--- a/sys/dev/irdma/irdma_type.h
+++ b/sys/dev/irdma/irdma_type.h
@@ -1,7 +1,7 @@
/*-
* SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB
*
- * Copyright (c) 2015 - 2022 Intel Corporation
+ * Copyright (c) 2015 - 2023 Intel Corporation
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@@ -398,6 +398,8 @@
struct irdma_dcqcn_cc_params dcqcn_params;
__le64 *host_ctx;
u64 *scratch_array;
+ u64 requested_ops;
+ atomic64_t completed_ops;
u32 cqp_id;
u32 sq_size;
u32 hw_sq_size;
@@ -532,7 +534,7 @@
bool use_cnp_up_override:1;
};
-#define IRDMA_MAX_WS_NODES 0x3FF
+#define IRDMA_MAX_WS_NODES 0x3FF
#define IRDMA_WS_NODE_INVALID 0xFFFF
struct irdma_ws_node_info {
@@ -598,7 +600,7 @@
bool tc_change_pending:1;
bool mtu_change_pending:1;
struct irdma_vsi_pestat *pestat;
- ATOMIC qp_suspend_reqs;
+ atomic_t qp_suspend_reqs;
int (*register_qset)(struct irdma_sc_vsi *vsi,
struct irdma_ws_node *tc_node);
void (*unregister_qset)(struct irdma_sc_vsi *vsi,
@@ -628,8 +630,6 @@
u32 IOMEM *aeq_alloc_db;
u32 IOMEM *cqp_db;
u32 IOMEM *cq_ack_db;
- u32 IOMEM *ceq_itr_mask_db;
- u32 IOMEM *aeq_itr_mask_db;
u32 IOMEM *hw_regs[IRDMA_MAX_REGS];
u32 ceq_itr; /* Interrupt throttle, usecs between interrupts: 0 disabled. 2 - 8160 */
u64 hw_masks[IRDMA_MAX_MASKS];
@@ -782,7 +782,6 @@
u32 first_pm_pbl_idx;
struct irdma_sc_vsi *vsi;
struct irdma_sc_cq **reg_cq;
- u32 reg_cq_idx;
};
struct irdma_aeq_init_info {
@@ -869,7 +868,7 @@
u16 t_high;
u16 t_low;
u8 last_byte_sent;
- u8 mac_addr[ETH_ALEN];
+ u8 mac_addr[ETHER_ADDR_LEN];
u8 rtomin;
};
@@ -901,7 +900,7 @@
u16 t_high;
u16 t_low;
u8 last_byte_sent;
- u8 mac_addr[ETH_ALEN];
+ u8 mac_addr[ETHER_ADDR_LEN];
u8 rtomin;
};
@@ -968,7 +967,6 @@
u32 rcv_cq_num;
u32 rem_endpoint_idx;
u16 stats_idx;
- bool srq_valid:1;
bool tcp_info_valid:1;
bool iwarp_info_valid:1;
bool stats_idx_valid:1;
@@ -1134,7 +1132,7 @@
};
struct irdma_add_arp_cache_entry_info {
- u8 mac_addr[ETH_ALEN];
+ u8 mac_addr[ETHER_ADDR_LEN];
u32 reach_max;
u16 arp_index;
bool permanent;
@@ -1151,7 +1149,7 @@
enum irdma_quad_entry_type entry_type;
bool vlan_valid:1;
bool ipv4_valid:1;
- u8 mac_addr[ETH_ALEN];
+ u8 mac_addr[ETHER_ADDR_LEN];
u16 vlan_id;
u8 user_pri;
u32 qp_num;
@@ -1249,7 +1247,6 @@
bool post_sq);
void irdma_sc_send_lsmm(struct irdma_sc_qp *qp, void *lsmm_buf, u32 size,
irdma_stag stag);
-void irdma_sc_send_lsmm_nostag(struct irdma_sc_qp *qp, void *lsmm_buf, u32 size);
void irdma_sc_send_rtt(struct irdma_sc_qp *qp, bool read);
void irdma_sc_qp_setctx(struct irdma_sc_qp *qp, __le64 *qp_ctx,
struct irdma_qp_host_ctx_info *info);
diff --git a/sys/dev/irdma/irdma_uda.h b/sys/dev/irdma/irdma_uda.h
--- a/sys/dev/irdma/irdma_uda.h
+++ b/sys/dev/irdma/irdma_uda.h
@@ -1,7 +1,7 @@
/*-
* SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB
*
- * Copyright (c) 2016 - 2021 Intel Corporation
+ * Copyright (c) 2016 - 2023 Intel Corporation
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@@ -54,7 +54,7 @@
u8 insert_vlan_tag;
u8 tc_tos;
u8 hop_ttl;
- u8 mac_addr[ETH_ALEN];
+ u8 mac_addr[ETHER_ADDR_LEN];
bool ah_valid:1;
bool ipv4_valid:1;
bool do_lpbk:1;
diff --git a/sys/dev/irdma/irdma_uda.c b/sys/dev/irdma/irdma_uda.c
--- a/sys/dev/irdma/irdma_uda.c
+++ b/sys/dev/irdma/irdma_uda.c
@@ -1,7 +1,7 @@
/*-
* SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB
*
- * Copyright (c) 2016 - 2022 Intel Corporation
+ * Copyright (c) 2016 - 2023 Intel Corporation
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@@ -59,12 +59,8 @@
if (!wqe)
return -ENOSPC;
- set_64bit_val(wqe, IRDMA_BYTE_0, LS_64_1(info->mac_addr[5], 16) |
- LS_64_1(info->mac_addr[4], 24) |
- LS_64_1(info->mac_addr[3], 32) |
- LS_64_1(info->mac_addr[2], 40) |
- LS_64_1(info->mac_addr[1], 48) |
- LS_64_1(info->mac_addr[0], 56));
+ set_64bit_val(wqe, IRDMA_BYTE_0,
+ FIELD_PREP(IRDMAQPC_MACADDRESS, irdma_mac_to_u64(info->mac_addr)));
qw1 = FIELD_PREP(IRDMA_UDA_CQPSQ_MAV_PDINDEXLO, info->pd_idx) |
FIELD_PREP(IRDMA_UDA_CQPSQ_MAV_TC, info->tc_tos) |
@@ -175,12 +171,7 @@
set_64bit_val(wqe, IRDMA_BYTE_16,
FIELD_PREP(IRDMA_UDA_CQPSQ_MG_VLANID, info->vlan_id) |
FIELD_PREP(IRDMA_UDA_CQPSQ_QS_HANDLE, info->qs_handle));
- set_64bit_val(wqe, IRDMA_BYTE_0, LS_64_1(info->dest_mac_addr[5], 0) |
- LS_64_1(info->dest_mac_addr[4], 8) |
- LS_64_1(info->dest_mac_addr[3], 16) |
- LS_64_1(info->dest_mac_addr[2], 24) |
- LS_64_1(info->dest_mac_addr[1], 32) |
- LS_64_1(info->dest_mac_addr[0], 40));
+ set_64bit_val(wqe, IRDMA_BYTE_0, irdma_mac_to_u64(info->dest_mac_addr));
set_64bit_val(wqe, IRDMA_BYTE_8,
FIELD_PREP(IRDMA_UDA_CQPSQ_MG_HMC_FCN_ID, info->hmc_fcn_id));
diff --git a/sys/dev/irdma/irdma_uk.c b/sys/dev/irdma/irdma_uk.c
--- a/sys/dev/irdma/irdma_uk.c
+++ b/sys/dev/irdma/irdma_uk.c
@@ -1,7 +1,7 @@
/*-
* SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB
*
- * Copyright (c) 2015 - 2022 Intel Corporation
+ * Copyright (c) 2015 - 2023 Intel Corporation
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@@ -275,7 +275,8 @@
if (qp->uk_attrs->hw_rev == IRDMA_GEN_1 && wqe_quanta == 1 &&
(IRDMA_RING_CURRENT_HEAD(qp->sq_ring) & 1)) {
wqe_0 = qp->sq_base[IRDMA_RING_CURRENT_HEAD(qp->sq_ring)].elem;
- wqe_0[3] = cpu_to_le64(FIELD_PREP(IRDMAQPSQ_VALID, !qp->swqe_polarity));
+ wqe_0[3] = cpu_to_le64(FIELD_PREP(IRDMAQPSQ_VALID,
+ qp->swqe_polarity ? 0 : 1));
}
qp->sq_wrtrk_array[*wqe_idx].wrid = info->wr_id;
qp->sq_wrtrk_array[*wqe_idx].wr_len = total_size;
@@ -596,22 +597,6 @@
return 0;
}
-/**
- * irdma_set_mw_bind_wqe_gen_1 - set mw bind wqe
- * @wqe: wqe for setting fragment
- * @op_info: info for setting bind wqe values
- */
-static void
-irdma_set_mw_bind_wqe_gen_1(__le64 * wqe,
- struct irdma_bind_window *op_info)
-{
- set_64bit_val(wqe, IRDMA_BYTE_0, (uintptr_t)op_info->va);
- set_64bit_val(wqe, IRDMA_BYTE_8,
- FIELD_PREP(IRDMAQPSQ_PARENTMRSTAG, op_info->mw_stag) |
- FIELD_PREP(IRDMAQPSQ_MWSTAG, op_info->mr_stag));
- set_64bit_val(wqe, IRDMA_BYTE_16, op_info->bind_len);
-}
-
/**
* irdma_copy_inline_data_gen_1 - Copy inline data to wqe
* @wqe: pointer to wqe
@@ -659,22 +644,6 @@
return data_size <= 16 ? IRDMA_QP_WQE_MIN_QUANTA : 2;
}
-/**
- * irdma_set_mw_bind_wqe - set mw bind in wqe
- * @wqe: wqe for setting mw bind
- * @op_info: info for setting wqe values
- */
-static void
-irdma_set_mw_bind_wqe(__le64 * wqe,
- struct irdma_bind_window *op_info)
-{
- set_64bit_val(wqe, IRDMA_BYTE_0, (uintptr_t)op_info->va);
- set_64bit_val(wqe, IRDMA_BYTE_8,
- FIELD_PREP(IRDMAQPSQ_PARENTMRSTAG, op_info->mr_stag) |
- FIELD_PREP(IRDMAQPSQ_MWSTAG, op_info->mw_stag));
- set_64bit_val(wqe, IRDMA_BYTE_16, op_info->bind_len);
-}
-
/**
* irdma_copy_inline_data - Copy inline data to wqe
* @wqe: pointer to wqe
@@ -1547,14 +1516,12 @@
.iw_copy_inline_data = irdma_copy_inline_data,
.iw_inline_data_size_to_quanta = irdma_inline_data_size_to_quanta,
.iw_set_fragment = irdma_set_fragment,
- .iw_set_mw_bind_wqe = irdma_set_mw_bind_wqe,
};
static const struct irdma_wqe_uk_ops iw_wqe_uk_ops_gen_1 = {
.iw_copy_inline_data = irdma_copy_inline_data_gen_1,
.iw_inline_data_size_to_quanta = irdma_inline_data_size_to_quanta_gen_1,
.iw_set_fragment = irdma_set_fragment_gen_1,
- .iw_set_mw_bind_wqe = irdma_set_mw_bind_wqe_gen_1,
};
/**
@@ -1760,6 +1727,9 @@
if (polarity != temp)
break;
+ /* Ensure CQE contents are read after valid bit is checked */
+ rmb();
+
get_64bit_val(cqe, IRDMA_BYTE_8, &comp_ctx);
if ((void *)(irdma_uintptr) comp_ctx == q)
set_64bit_val(cqe, IRDMA_BYTE_8, 0);
@@ -1771,48 +1741,6 @@
return 0;
}
-/**
- * irdma_nop - post a nop
- * @qp: hw qp ptr
- * @wr_id: work request id
- * @signaled: signaled for completion
- * @post_sq: ring doorbell
- */
-int
-irdma_nop(struct irdma_qp_uk *qp, u64 wr_id, bool signaled, bool post_sq)
-{
- __le64 *wqe;
- u64 hdr;
- u32 wqe_idx;
- struct irdma_post_sq_info info = {0};
- u16 quanta = IRDMA_QP_WQE_MIN_QUANTA;
-
- info.push_wqe = qp->push_db ? true : false;
- info.wr_id = wr_id;
- wqe = irdma_qp_get_next_send_wqe(qp, &wqe_idx, &quanta, 0, &info);
- if (!wqe)
- return -ENOSPC;
-
- set_64bit_val(wqe, IRDMA_BYTE_0, 0);
- set_64bit_val(wqe, IRDMA_BYTE_8, 0);
- set_64bit_val(wqe, IRDMA_BYTE_16, 0);
-
- hdr = FIELD_PREP(IRDMAQPSQ_OPCODE, IRDMAQP_OP_NOP) |
- FIELD_PREP(IRDMAQPSQ_SIGCOMPL, signaled) |
- FIELD_PREP(IRDMAQPSQ_VALID, qp->swqe_polarity);
-
- irdma_wmb(); /* make sure WQE is populated before valid bit is set */
-
- set_64bit_val(wqe, IRDMA_BYTE_24, hdr);
-
- if (info.push_wqe)
- irdma_qp_push_wqe(qp, wqe, quanta, wqe_idx, post_sq);
- else if (post_sq)
- irdma_uk_qp_post_wr(qp);
-
- return 0;
-}
-
/**
* irdma_fragcnt_to_quanta_sq - calculate quanta based on fragment count for SQ
* @frag_cnt: number of fragments
diff --git a/sys/dev/irdma/irdma_user.h b/sys/dev/irdma/irdma_user.h
--- a/sys/dev/irdma/irdma_user.h
+++ b/sys/dev/irdma/irdma_user.h
@@ -1,7 +1,7 @@
/*-
* SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB
*
- * Copyright (c) 2015 - 2022 Intel Corporation
+ * Copyright (c) 2015 - 2023 Intel Corporation
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@@ -568,7 +568,6 @@
u32 inline_data, u8 *shift);
int irdma_get_sqdepth(struct irdma_uk_attrs *uk_attrs, u32 sq_size, u8 shift, u32 *sqdepth);
int irdma_get_rqdepth(struct irdma_uk_attrs *uk_attrs, u32 rq_size, u8 shift, u32 *rqdepth);
-int irdma_get_srqdepth(struct irdma_uk_attrs *uk_attrs, u32 srq_size, u8 shift, u32 *srqdepth);
void irdma_qp_push_wqe(struct irdma_qp_uk *qp, __le64 *wqe, u16 quanta,
u32 wqe_idx, bool post_sq);
void irdma_clr_wqes(struct irdma_qp_uk *qp, u32 qp_wqe_idx);
diff --git a/sys/dev/irdma/irdma_utils.c b/sys/dev/irdma/irdma_utils.c
--- a/sys/dev/irdma/irdma_utils.c
+++ b/sys/dev/irdma/irdma_utils.c
@@ -1,7 +1,7 @@
/*-
* SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB
*
- * Copyright (c) 2015 - 2022 Intel Corporation
+ * Copyright (c) 2015 - 2023 Intel Corporation
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@@ -38,6 +38,169 @@
LIST_HEAD(irdma_handlers);
DEFINE_SPINLOCK(irdma_handler_lock);
+static const struct ae_desc ae_desc_list[] = {
+ {IRDMA_AE_AMP_UNALLOCATED_STAG, "Unallocated memory key (L-Key/R-Key)"},
+ {IRDMA_AE_AMP_INVALID_STAG, "Invalid memory key (L-Key/R-Key)"},
+ {IRDMA_AE_AMP_BAD_QP,
+ "Memory protection error: Accessing Memory Window (MW) which belongs to a different QP"},
+ {IRDMA_AE_AMP_BAD_PD,
+ "Memory protection error: Accessing Memory Window (MW)/Memory Region (MR) which belongs to a different PD"},
+ {IRDMA_AE_AMP_BAD_STAG_KEY, "Bad memory key (L-Key/R-Key)"},
+ {IRDMA_AE_AMP_BAD_STAG_INDEX, "Bad memory key (L-Key/R-Key): Too large memory key index"},
+ {IRDMA_AE_AMP_BOUNDS_VIOLATION, "Memory Window (MW)/Memory Region (MR) bounds violation"},
+ {IRDMA_AE_AMP_RIGHTS_VIOLATION, "Memory Window (MW)/Memory Region (MR) rights violation"},
+ {IRDMA_AE_AMP_TO_WRAP,
+ "Memory protection error: The address within Memory Window (MW)/Memory Region (MR) wraps"},
+ {IRDMA_AE_AMP_FASTREG_VALID_STAG,
+ "Fastreg error: Registration to a valid MR"},
+ {IRDMA_AE_AMP_FASTREG_MW_STAG,
+ "Fastreg error: Registration to a valid Memory Window (MW)"},
+ {IRDMA_AE_AMP_FASTREG_INVALID_RIGHTS, "Fastreg error: Invalid rights"},
+ {IRDMA_AE_AMP_FASTREG_INVALID_LENGTH, "Fastreg error: Invalid length"},
+ {IRDMA_AE_AMP_INVALIDATE_SHARED, "Attempt to invalidate a shared MR"},
+ {IRDMA_AE_AMP_INVALIDATE_NO_REMOTE_ACCESS_RIGHTS,
+ "Attempt to remotely invalidate Memory Window (MW)/Memory Region (MR) without rights"},
+ {IRDMA_AE_AMP_INVALIDATE_MR_WITH_BOUND_WINDOWS,
+ "Attempt to invalidate MR with a bound Memory Window (MW)"},
+ {IRDMA_AE_AMP_MWBIND_VALID_STAG,
+ "Attempt to bind an Memory Window (MW) with a valid MW memory key (L-Key/R-Key)"},
+ {IRDMA_AE_AMP_MWBIND_OF_MR_STAG,
+ "Attempt to bind an Memory Window (MW) with an MR memory key (L-Key/R-Key)"},
+ {IRDMA_AE_AMP_MWBIND_TO_ZERO_BASED_STAG,
+ "Attempt to bind an Memory Window (MW) to a zero based MR"},
+ {IRDMA_AE_AMP_MWBIND_TO_MW_STAG,
+ "Attempt to bind an Memory Window (MW) using MW memory key (L-Key/R-Key) instead of MR memory key (L-Key/R-Key)"},
+ {IRDMA_AE_AMP_MWBIND_INVALID_RIGHTS, "Memory Window (MW) bind error: Invalid rights"},
+ {IRDMA_AE_AMP_MWBIND_INVALID_BOUNDS, "Memory Window (MW) bind error: Invalid bounds"},
+ {IRDMA_AE_AMP_MWBIND_TO_INVALID_PARENT,
+ "Memory Window (MW) bind error: Invalid parent MR"},
+ {IRDMA_AE_AMP_MWBIND_BIND_DISABLED,
+ "Memory Window (MW) bind error: Disabled bind support"},
+ {IRDMA_AE_PRIV_OPERATION_DENIED,
+ "Denying a privileged operation on a non-privileged QP"},
+ {IRDMA_AE_AMP_INVALIDATE_TYPE1_MW, "Memory Window (MW) error: Invalidate type 1 MW"},
+ {IRDMA_AE_AMP_MWBIND_ZERO_BASED_TYPE1_MW,
+ "Memory Window (MW) bind error: Zero-based addressing for type 1 MW"},
+ {IRDMA_AE_AMP_FASTREG_INVALID_PBL_HPS_CFG,
+ "Fastreg error: Invalid host page size config"},
+ {IRDMA_AE_AMP_MWBIND_WRONG_TYPE, "MB bind error: Wrong Memory Window (MW) type"},
+ {IRDMA_AE_AMP_FASTREG_PBLE_MISMATCH,
+ "Fastreg error: Invalid request to change physical MR to virtual or vice versa"},
+ {IRDMA_AE_UDA_XMIT_DGRAM_TOO_LONG,
+ "Userspace Direct Access (UDA) QP xmit error: Packet length exceeds the QP MTU"},
+ {IRDMA_AE_UDA_XMIT_BAD_PD,
+ "Userspace Direct Access (UDA) QP xmit error: Attempt to access a different PD"},
+ {IRDMA_AE_UDA_XMIT_DGRAM_TOO_SHORT,
+ "Userspace Direct Access (UDA) QP xmit error: Too short packet length"},
+ {IRDMA_AE_UDA_L4LEN_INVALID,
+ "Userspace Direct Access (UDA) error: Invalid packet length field"},
+ {IRDMA_AE_BAD_CLOSE,
+ "iWARP error: Data is received when QP state is closing"},
+ {IRDMA_AE_RDMAP_ROE_BAD_LLP_CLOSE,
+ "iWARP error: FIN is received when xmit data is pending"},
+ {IRDMA_AE_CQ_OPERATION_ERROR, "CQ overflow"},
+ {IRDMA_AE_RDMA_READ_WHILE_ORD_ZERO,
+ "QP error: Attempted RDMA Read when the outbound RDMA Read queue depth is zero"},
+ {IRDMA_AE_STAG_ZERO_INVALID,
+ "Zero invalid memory key (L-Key/R-Key) on inbound RDMA R/W"},
+ {IRDMA_AE_IB_RREQ_AND_Q1_FULL,
+ "QP error: Received RDMA Read request when the inbound RDMA Read queue is full"},
+ {IRDMA_AE_IB_INVALID_REQUEST,
+ "QP error: Invalid operation detected by the remote peer"},
+ {IRDMA_AE_WQE_UNEXPECTED_OPCODE,
+ "QP error: Invalid opcode in SQ WQE"},
+ {IRDMA_AE_WQE_INVALID_PARAMETER,
+ "QP error: Invalid parameter in a WQE"},
+ {IRDMA_AE_WQE_INVALID_FRAG_DATA,
+ "QP error: Invalid fragment in a WQE"},
+ {IRDMA_AE_IB_REMOTE_ACCESS_ERROR,
+ "RoCEv2 error: Remote access error"},
+ {IRDMA_AE_IB_REMOTE_OP_ERROR,
+ "RoCEv2 error: Remote operation error"},
+ {IRDMA_AE_WQE_LSMM_TOO_LONG, "iWARP error: Connection error"},
+ {IRDMA_AE_DDP_INVALID_MSN_GAP_IN_MSN,
+ "iWARP error: Invalid message sequence number"},
+ {IRDMA_AE_DDP_UBE_DDP_MESSAGE_TOO_LONG_FOR_AVAILABLE_BUFFER,
+ "iWARP error: Inbound message is too long for the available buffer"},
+ {IRDMA_AE_DDP_UBE_INVALID_DDP_VERSION, "iWARP error: Invalid DDP protocol version"},
+ {IRDMA_AE_DDP_UBE_INVALID_MO, "Received message with too large offset"},
+ {IRDMA_AE_DDP_UBE_INVALID_MSN_NO_BUFFER_AVAILABLE,
+ "iWARP error: Inbound Send message when no receive buffer is available"},
+ {IRDMA_AE_DDP_UBE_INVALID_QN, "iWARP error: Invalid QP number in inbound packet"},
+ {IRDMA_AE_DDP_NO_L_BIT,
+ "iWARP error: Last bit not set in an inbound packet which completes RDMA Read"},
+ {IRDMA_AE_RDMAP_ROE_INVALID_RDMAP_VERSION, "iWARP error: Invalid RDMAP protocol version"},
+ {IRDMA_AE_RDMAP_ROE_UNEXPECTED_OPCODE, "QP error: Invalid opcode"},
+ {IRDMA_AE_ROE_INVALID_RDMA_READ_REQUEST, "Inbound Read request when QP isn't enabled for RDMA Read"},
+ {IRDMA_AE_ROE_INVALID_RDMA_WRITE_OR_READ_RESP,
+ "Inbound RDMA Read response or RDMA Write when QP isn't enabled for RDMA R/W"},
+ {IRDMA_AE_ROCE_RSP_LENGTH_ERROR, "RoCEv2 error: Received packet with incorrect length field"},
+ {IRDMA_AE_ROCE_EMPTY_MCG, "RoCEv2 error: Multicast group has no valid members"},
+ {IRDMA_AE_ROCE_BAD_MC_IP_ADDR, "RoCEv2 error: Multicast IP address doesn't match"},
+ {IRDMA_AE_ROCE_BAD_MC_QPID, "RoCEv2 error: Multicast packet QP number isn't 0xffffff"},
+ {IRDMA_AE_MCG_QP_PROTOCOL_MISMATCH, "RoCEv2 error: Multicast packet protocol mismatch"},
+ {IRDMA_AE_INVALID_ARP_ENTRY, "Invalid ARP entry"},
+ {IRDMA_AE_INVALID_TCP_OPTION_RCVD, "iWARP error: Invalid TCP option"},
+ {IRDMA_AE_STALE_ARP_ENTRY, "Stale ARP entry"},
+ {IRDMA_AE_INVALID_AH_ENTRY, "Invalid AH entry"},
+ {IRDMA_AE_LLP_CLOSE_COMPLETE,
+ "iWARP event: Graceful close complete"},
+ {IRDMA_AE_LLP_CONNECTION_RESET,
+ "iWARP event: Received a TCP packet with a RST bit set"},
+ {IRDMA_AE_LLP_FIN_RECEIVED,
+ "iWARP event: Received a TCP packet with a FIN bit set"},
+ {IRDMA_AE_LLP_RECEIVED_MARKER_AND_LENGTH_FIELDS_DONT_MATCH,
+ "iWARP error: Unable to close a gap in the TCP sequence"},
+ {IRDMA_AE_LLP_RECEIVED_MPA_CRC_ERROR, "Received an ICRC error"},
+ {IRDMA_AE_LLP_SEGMENT_TOO_SMALL,
+ "iWARP error: Received a packet with insufficient space for protocol headers"},
+ {IRDMA_AE_LLP_SYN_RECEIVED,
+ "iWARP event: Received a TCP packet with a SYN bit set"},
+ {IRDMA_AE_LLP_TERMINATE_RECEIVED,
+ "iWARP error: Received a terminate message"},
+ {IRDMA_AE_LLP_TOO_MANY_RETRIES, "Connection error: The max number of retries has been reached"},
+ {IRDMA_AE_LLP_TOO_MANY_KEEPALIVE_RETRIES,
+ "Connection error: The max number of keepalive retries has been reached"},
+ {IRDMA_AE_LLP_DOUBT_REACHABILITY,
+ "Connection error: Doubt reachability (usually occurs after the max number of retries has been reached)"},
+ {IRDMA_AE_LLP_CONNECTION_ESTABLISHED,
+ "iWARP event: Connection established"},
+ {IRDMA_AE_RESOURCE_EXHAUSTION,
+ "QP error: Resource exhaustion"},
+ {IRDMA_AE_RESET_SENT,
+ "Reset sent (as requested via Modify QP)"},
+ {IRDMA_AE_TERMINATE_SENT,
+ "Terminate sent (as requested via Modify QP)"},
+ {IRDMA_AE_RESET_NOT_SENT,
+ "Reset not sent (but requested via Modify QP)"},
+ {IRDMA_AE_LCE_QP_CATASTROPHIC,
+ "QP error: HW transaction resulted in catastrophic error"},
+ {IRDMA_AE_LCE_FUNCTION_CATASTROPHIC,
+ "PCIe function error: HW transaction resulted in catastrophic error"},
+ {IRDMA_AE_LCE_CQ_CATASTROPHIC,
+ "CQ error: HW transaction resulted in catastrophic error"},
+ {IRDMA_AE_QP_SUSPEND_COMPLETE, "QP event: Suspend complete"},
+};
+
+/**
+ * irdma_get_ae_desc - returns AE description
+ * @ae_id: the AE number
+ */
+const char *
+irdma_get_ae_desc(u16 ae_id)
+{
+ const char *desc = "";
+ int i;
+
+ for (i = 0; i < ARRAY_SIZE(ae_desc_list); i++) {
+ if (ae_desc_list[i].id == ae_id) {
+ desc = ae_desc_list[i].desc;
+ break;
+ }
+ }
+ return desc;
+}
+
/**
* irdma_arp_table -manage arp table
* @rf: RDMA PCI function
@@ -242,7 +405,7 @@
if (cqp_request->dynamic) {
kfree(cqp_request);
} else {
- cqp_request->request_done = false;
+ WRITE_ONCE(cqp_request->request_done, false);
cqp_request->callback_fcn = NULL;
cqp_request->waiting = false;
@@ -277,7 +440,7 @@
{
if (cqp_request->waiting) {
cqp_request->compl_info.error = true;
- cqp_request->request_done = true;
+ WRITE_ONCE(cqp_request->request_done, true);
wake_up(&cqp_request->waitq);
}
wait_event_timeout(cqp->remove_wq,
@@ -328,23 +491,22 @@
struct irdma_cqp_request *cqp_request)
{
struct irdma_cqp_timeout cqp_timeout = {0};
- int timeout_threshold = CQP_TIMEOUT_THRESHOLD;
bool cqp_error = false;
int err_code = 0;
- cqp_timeout.compl_cqp_cmds = rf->sc_dev.cqp_cmd_stats[IRDMA_OP_CMPL_CMDS];
+ cqp_timeout.compl_cqp_cmds = atomic64_read(&rf->sc_dev.cqp->completed_ops);
do {
int wait_time_ms = rf->sc_dev.hw_attrs.max_cqp_compl_wait_time_ms;
irdma_cqp_ce_handler(rf, &rf->ccq.sc_cq);
if (wait_event_timeout(cqp_request->waitq,
- cqp_request->request_done,
+ READ_ONCE(cqp_request->request_done),
msecs_to_jiffies(wait_time_ms)))
break;
irdma_check_cqp_progress(&cqp_timeout, &rf->sc_dev);
- if (cqp_timeout.count < timeout_threshold)
+ if (cqp_timeout.count < CQP_TIMEOUT_THRESHOLD)
continue;
if (!rf->reset) {
@@ -397,7 +559,8 @@
[IRDMA_OP_MANAGE_HMC_PM_FUNC_TABLE] = "Manage HMC PM Function Table Cmd",
[IRDMA_OP_SUSPEND] = "Suspend QP Cmd",
[IRDMA_OP_RESUME] = "Resume QP Cmd",
- [IRDMA_OP_MANAGE_VF_PBLE_BP] = "Manage VF PBLE Backing Pages Cmd",
+ [IRDMA_OP_MANAGE_VCHNL_REQ_PBLE_BP] =
+ "Manage Virtual Channel Requester Function PBLE Backing Pages Cmd",
[IRDMA_OP_QUERY_FPM_VAL] = "Query FPM Values Cmd",
[IRDMA_OP_COMMIT_FPM_VAL] = "Commit FPM Values Cmd",
[IRDMA_OP_AH_CREATE] = "Create Address Handle Cmd",
@@ -1620,41 +1783,6 @@
return status;
}
-/**
- * irdma_cqp_up_map_cmd - Set the up-up mapping
- * @dev: pointer to device structure
- * @cmd: map command
- * @map_info: pointer to up map info
- */
-int
-irdma_cqp_up_map_cmd(struct irdma_sc_dev *dev, u8 cmd,
- struct irdma_up_info *map_info)
-{
- struct irdma_pci_f *rf = dev_to_rf(dev);
- struct irdma_cqp *iwcqp = &rf->cqp;
- struct irdma_sc_cqp *cqp = &iwcqp->sc_cqp;
- struct irdma_cqp_request *cqp_request;
- struct cqp_cmds_info *cqp_info;
- int status;
-
- cqp_request = irdma_alloc_and_get_cqp_request(iwcqp, false);
- if (!cqp_request)
- return -ENOMEM;
-
- cqp_info = &cqp_request->info;
- memset(cqp_info, 0, sizeof(*cqp_info));
- cqp_info->cqp_cmd = cmd;
- cqp_info->post_sq = 1;
- cqp_info->in.u.up_map.info = *map_info;
- cqp_info->in.u.up_map.cqp = cqp;
- cqp_info->in.u.up_map.scratch = (uintptr_t)cqp_request;
-
- status = irdma_handle_cqp_op(rf, cqp_request);
- irdma_put_cqp_request(&rf->cqp, cqp_request);
-
- return status;
-}
-
/**
* irdma_ah_cqp_op - perform an AH cqp operation
* @rf: RDMA PCI function
@@ -1707,7 +1835,7 @@
return -ENOMEM;
if (wait)
- sc_ah->ah_info.ah_valid = (cmd == IRDMA_OP_AH_CREATE);
+ sc_ah->ah_info.ah_valid = (cmd != IRDMA_OP_AH_DESTROY);
return 0;
}
@@ -1852,15 +1980,10 @@
sizeofbitmap = (u64)pchunk->size >> pprm->pble_shift;
- pchunk->bitmapmem.size = sizeofbitmap >> 3;
- pchunk->bitmapmem.va = kzalloc(pchunk->bitmapmem.size, GFP_KERNEL);
-
- if (!pchunk->bitmapmem.va)
+ pchunk->bitmapbuf = bitmap_zalloc(sizeofbitmap, GFP_KERNEL);
+ if (!pchunk->bitmapbuf)
return -ENOMEM;
- pchunk->bitmapbuf = pchunk->bitmapmem.va;
- bitmap_zero(pchunk->bitmapbuf, sizeofbitmap);
-
pchunk->sizeofbitmap = sizeofbitmap;
/* each pble is 8 bytes hence shift by 3 */
pprm->total_pble_alloc += pchunk->size >> 3;
@@ -2198,9 +2321,8 @@
clear_qp_ctx_addr(dma_mem.va);
for (i = 0, j = 0; i < 32; i++, j += 4)
irdma_debug(dev, IRDMA_DEBUG_QP,
- "%d:\t [%08X %08x %08X %08X]\n",
- (j * 4), ctx[j], ctx[j + 1], ctx[j + 2],
- ctx[j + 3]);
+ "%d:\t [%08X %08x %08X %08X]\n", (j * 4),
+ ctx[j], ctx[j + 1], ctx[j + 2], ctx[j + 3]);
}
error:
irdma_put_cqp_request(iwcqp, cqp_request);
diff --git a/sys/dev/irdma/irdma_verbs.h b/sys/dev/irdma/irdma_verbs.h
--- a/sys/dev/irdma/irdma_verbs.h
+++ b/sys/dev/irdma/irdma_verbs.h
@@ -1,7 +1,7 @@
/*-
* SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB
*
- * Copyright (c) 2015 - 2022 Intel Corporation
+ * Copyright (c) 2015 - 2023 Intel Corporation
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@@ -42,6 +42,8 @@
#define IRDMA_PKEY_TBL_SZ 1
#define IRDMA_DEFAULT_PKEY 0xFFFF
+#define IRDMA_SHADOW_PGCNT 1
+
#define iwdev_to_idev(iwdev) (&(iwdev)->rf->sc_dev)
struct irdma_ucontext {
@@ -73,14 +75,16 @@
spinlock_t udqp_list_lock;
};
+union irdma_sockaddr {
+ struct sockaddr_in saddr_in;
+ struct sockaddr_in6 saddr_in6;
+};
+
struct irdma_av {
u8 macaddr[16];
struct ib_ah_attr attrs;
- union {
- struct sockaddr saddr;
- struct sockaddr_in saddr_in;
- struct sockaddr_in6 saddr_in6;
- } sgid_addr, dgid_addr;
+ union irdma_sockaddr sgid_addr;
+ union irdma_sockaddr dgid_addr;
u8 net_type;
};
@@ -246,6 +250,7 @@
int max_recv_wr;
atomic_t close_timer_started;
spinlock_t lock; /* serialize posting WRs to SQ/RQ */
+ spinlock_t dwork_flush_lock; /* protect mod_delayed_work */
struct irdma_qp_context *iwqp_context;
void *pbl_vbase;
dma_addr_t pbl_pbase;
@@ -266,12 +271,13 @@
wait_queue_head_t waitq;
wait_queue_head_t mod_qp_waitq;
u8 rts_ae_rcvd;
- u8 active_conn : 1;
- u8 user_mode : 1;
- u8 hte_added : 1;
- u8 flush_issued : 1;
- u8 sig_all : 1;
- u8 pau_mode : 1;
+ bool active_conn:1;
+ bool user_mode:1;
+ bool hte_added:1;
+ bool flush_issued:1;
+ bool sig_all:1;
+ bool pau_mode:1;
+ bool suspend_pending:1;
};
struct irdma_udqs_work {
@@ -308,6 +314,63 @@
return (u16)FIELD_GET(IRDMA_FW_VER_MINOR, dev->feature_info[IRDMA_FEATURE_FW_INFO]);
}
+static inline void set_ib_wc_op_sq(struct irdma_cq_poll_info *cq_poll_info,
+ struct ib_wc *entry)
+{
+ struct irdma_sc_qp *qp;
+
+ switch (cq_poll_info->op_type) {
+ case IRDMA_OP_TYPE_RDMA_WRITE:
+ case IRDMA_OP_TYPE_RDMA_WRITE_SOL:
+ entry->opcode = IB_WC_RDMA_WRITE;
+ break;
+ case IRDMA_OP_TYPE_RDMA_READ_INV_STAG:
+ case IRDMA_OP_TYPE_RDMA_READ:
+ entry->opcode = IB_WC_RDMA_READ;
+ break;
+ case IRDMA_OP_TYPE_SEND_SOL:
+ case IRDMA_OP_TYPE_SEND_SOL_INV:
+ case IRDMA_OP_TYPE_SEND_INV:
+ case IRDMA_OP_TYPE_SEND:
+ entry->opcode = IB_WC_SEND;
+ break;
+ case IRDMA_OP_TYPE_FAST_REG_NSMR:
+ entry->opcode = IB_WC_REG_MR;
+ break;
+ case IRDMA_OP_TYPE_INV_STAG:
+ entry->opcode = IB_WC_LOCAL_INV;
+ break;
+ default:
+ qp = cq_poll_info->qp_handle;
+ irdma_dev_err(to_ibdev(qp->dev), "Invalid opcode = %d in CQE\n",
+ cq_poll_info->op_type);
+ entry->status = IB_WC_GENERAL_ERR;
+ }
+}
+
+static inline void set_ib_wc_op_rq(struct irdma_cq_poll_info *cq_poll_info,
+ struct ib_wc *entry, bool send_imm_support)
+{
+ /**
+ * iWARP does not support sendImm, so the presence of Imm data
+ * must be WriteImm.
+ */
+ if (!send_imm_support) {
+ entry->opcode = cq_poll_info->imm_valid ?
+ IB_WC_RECV_RDMA_WITH_IMM :
+ IB_WC_RECV;
+ return;
+ }
+ switch (cq_poll_info->op_type) {
+ case IB_OPCODE_RDMA_WRITE_ONLY_WITH_IMMEDIATE:
+ case IB_OPCODE_RDMA_WRITE_LAST_WITH_IMMEDIATE:
+ entry->opcode = IB_WC_RECV_RDMA_WITH_IMM;
+ break;
+ default:
+ entry->opcode = IB_WC_RECV;
+ }
+}
+
/**
* irdma_mcast_mac_v4 - Get the multicast MAC for an IP address
* @ip_addr: IPv4 address
@@ -317,7 +380,7 @@
static inline void irdma_mcast_mac_v4(u32 *ip_addr, u8 *mac)
{
u8 *ip = (u8 *)ip_addr;
- unsigned char mac4[ETH_ALEN] = {0x01, 0x00, 0x5E, ip[2] & 0x7F, ip[1],
+ unsigned char mac4[ETHER_ADDR_LEN] = {0x01, 0x00, 0x5E, ip[2] & 0x7F, ip[1],
ip[0]};
ether_addr_copy(mac, mac4);
@@ -332,7 +395,7 @@
static inline void irdma_mcast_mac_v6(u32 *ip_addr, u8 *mac)
{
u8 *ip = (u8 *)ip_addr;
- unsigned char mac6[ETH_ALEN] = {0x33, 0x33, ip[3], ip[2], ip[1], ip[0]};
+ unsigned char mac6[ETHER_ADDR_LEN] = {0x33, 0x33, ip[3], ip[2], ip[1], ip[0]};
ether_addr_copy(mac, mac6);
}
diff --git a/sys/dev/irdma/irdma_verbs.c b/sys/dev/irdma/irdma_verbs.c
--- a/sys/dev/irdma/irdma_verbs.c
+++ b/sys/dev/irdma/irdma_verbs.c
@@ -1,7 +1,7 @@
/*-
* SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB
*
- * Copyright (c) 2015 - 2022 Intel Corporation
+ * Copyright (c) 2015 - 2023 Intel Corporation
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@@ -289,9 +289,9 @@
return -EINVAL;
}
#endif
- irdma_debug(&ucontext->iwdev->rf->sc_dev,
- IRDMA_DEBUG_VERBS, "bar_offset [0x%lx] mmap_flag [%d]\n",
- entry->bar_offset, entry->mmap_flag);
+ irdma_debug(&ucontext->iwdev->rf->sc_dev, IRDMA_DEBUG_VERBS,
+ "bar_offset [0x%lx] mmap_flag [%d]\n", entry->bar_offset,
+ entry->mmap_flag);
pfn = (entry->bar_offset +
pci_resource_start(ucontext->iwdev->rf->pcidev, 0)) >> PAGE_SHIFT;
@@ -823,12 +823,16 @@
void
irdma_sched_qp_flush_work(struct irdma_qp *iwqp)
{
+ unsigned long flags;
+
if (iwqp->sc_qp.qp_uk.destroy_pending)
return;
irdma_qp_add_ref(&iwqp->ibqp);
+ spin_lock_irqsave(&iwqp->dwork_flush_lock, flags);
if (mod_delayed_work(iwqp->iwdev->cleanup_wq, &iwqp->dwork_flush,
msecs_to_jiffies(IRDMA_FLUSH_DELAY_MS)))
irdma_qp_rem_ref(&iwqp->ibqp);
+ spin_unlock_irqrestore(&iwqp->dwork_flush_lock, flags);
}
void
@@ -917,6 +921,22 @@
return 0;
}
+static int
+irdma_wait_for_suspend(struct irdma_qp *iwqp)
+{
+ if (!wait_event_timeout(iwqp->iwdev->suspend_wq,
+ !iwqp->suspend_pending,
+ msecs_to_jiffies(IRDMA_EVENT_TIMEOUT_MS))) {
+ iwqp->suspend_pending = false;
+ irdma_dev_warn(&iwqp->iwdev->ibdev,
+ "modify_qp timed out waiting for suspend. qp_id = %d, last_ae = 0x%x\n",
+ iwqp->ibqp.qp_num, iwqp->last_aeq);
+ return -EBUSY;
+ }
+
+ return 0;
+}
+
/**
* irdma_modify_qp_roce - modify qp request
* @ibqp: qp's pointer for modify
@@ -1112,9 +1132,8 @@
spin_lock_irqsave(&iwqp->lock, flags);
if (attr_mask & IB_QP_STATE) {
- if (!kc_ib_modify_qp_is_ok(iwqp->ibqp_state, attr->qp_state,
- iwqp->ibqp.qp_type, attr_mask,
- IB_LINK_LAYER_ETHERNET)) {
+ if (!ib_modify_qp_is_ok(iwqp->ibqp_state, attr->qp_state,
+ iwqp->ibqp.qp_type, attr_mask)) {
irdma_dev_warn(&iwdev->ibdev,
"modify_qp invalid for qp_id=%d, old_state=0x%x, new_state=0x%x\n",
iwqp->ibqp.qp_num, iwqp->ibqp_state,
@@ -1181,19 +1200,11 @@
info.next_iwarp_state = IRDMA_QP_STATE_SQD;
issue_modify_qp = 1;
+ iwqp->suspend_pending = true;
break;
case IB_QPS_SQE:
case IB_QPS_ERR:
case IB_QPS_RESET:
- if (iwqp->iwarp_state == IRDMA_QP_STATE_RTS) {
- if (dev->hw_attrs.uk_attrs.hw_rev <= IRDMA_GEN_2)
- irdma_cqp_qp_suspend_resume(&iwqp->sc_qp, IRDMA_OP_SUSPEND);
- spin_unlock_irqrestore(&iwqp->lock, flags);
- info.next_iwarp_state = IRDMA_QP_STATE_SQD;
- irdma_hw_modify_qp(iwdev, iwqp, &info, true);
- spin_lock_irqsave(&iwqp->lock, flags);
- }
-
if (iwqp->iwarp_state == IRDMA_QP_STATE_ERROR) {
spin_unlock_irqrestore(&iwqp->lock, flags);
if (udata && udata->inlen) {
@@ -1230,6 +1241,11 @@
ctx_info->rem_endpoint_idx = udp_info->arp_idx;
if (irdma_hw_modify_qp(iwdev, iwqp, &info, true))
return -EINVAL;
+ if (info.next_iwarp_state == IRDMA_QP_STATE_SQD) {
+ ret = irdma_wait_for_suspend(iwqp);
+ if (ret)
+ return ret;
+ }
spin_lock_irqsave(&iwqp->lock, flags);
if (iwqp->iwarp_state == info.curr_iwarp_state) {
iwqp->iwarp_state = info.next_iwarp_state;
@@ -1520,8 +1536,8 @@
udata->outlen));
if (err) {
irdma_remove_push_mmap_entries(iwqp);
- irdma_debug(&iwdev->rf->sc_dev,
- IRDMA_DEBUG_VERBS, "copy_to_udata failed\n");
+ irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_VERBS,
+ "copy_to_udata failed\n");
return err;
}
}
@@ -2217,6 +2233,177 @@
return ret;
}
+/*
+ * irdma_alloc_iwmr - Allocate iwmr @region - memory region @pd - protection domain @virt - virtual address @reg_type -
+ * registration type
+ */
+static struct irdma_mr *
+irdma_alloc_iwmr(struct ib_umem *region,
+ struct ib_pd *pd, u64 virt,
+ enum irdma_memreg_type reg_type)
+{
+ struct irdma_pbl *iwpbl;
+ struct irdma_mr *iwmr;
+
+ iwmr = kzalloc(sizeof(*iwmr), GFP_KERNEL);
+ if (!iwmr)
+ return ERR_PTR(-ENOMEM);
+
+ iwpbl = &iwmr->iwpbl;
+ iwpbl->iwmr = iwmr;
+ iwmr->region = region;
+ iwmr->ibmr.pd = pd;
+ iwmr->ibmr.device = pd->device;
+ iwmr->ibmr.iova = virt;
+ iwmr->type = reg_type;
+
+ /* Some OOT versions of irdma_copy_user_pg_addr require the pg mask */
+ iwmr->page_msk = ~(IRDMA_HW_PAGE_SIZE - 1);
+ iwmr->page_size = IRDMA_HW_PAGE_SIZE;
+ iwmr->len = region->length;
+ iwpbl->user_base = virt;
+ iwmr->page_cnt = irdma_ib_umem_num_dma_blocks(region, iwmr->page_size, virt);
+
+ return iwmr;
+}
+
+static void
+irdma_free_iwmr(struct irdma_mr *iwmr)
+{
+ kfree(iwmr);
+}
+
+/*
+ * irdma_reg_user_mr_type_mem - Handle memory registration @iwmr - irdma mr @access - access rights
+ */
+static int
+irdma_reg_user_mr_type_mem(struct irdma_mr *iwmr, int access)
+{
+ struct irdma_device *iwdev = to_iwdev(iwmr->ibmr.device);
+ struct irdma_pbl *iwpbl = &iwmr->iwpbl;
+ u32 stag;
+ int err;
+ u8 lvl;
+
+ lvl = iwmr->page_cnt != 1 ? PBLE_LEVEL_1 | PBLE_LEVEL_2 : PBLE_LEVEL_0;
+
+ err = irdma_setup_pbles(iwdev->rf, iwmr, lvl);
+ if (err)
+ return err;
+
+ if (lvl) {
+ err = irdma_check_mr_contiguous(&iwpbl->pble_alloc,
+ iwmr->page_size);
+ if (err) {
+ irdma_free_pble(iwdev->rf->pble_rsrc, &iwpbl->pble_alloc);
+ iwpbl->pbl_allocated = false;
+ }
+ }
+
+ stag = irdma_create_stag(iwdev);
+ if (!stag) {
+ err = -ENOMEM;
+ goto free_pble;
+ }
+
+ iwmr->stag = stag;
+ iwmr->ibmr.rkey = stag;
+ iwmr->ibmr.lkey = stag;
+ iwmr->access = access;
+ err = irdma_hwreg_mr(iwdev, iwmr, access);
+ if (err)
+ goto err_hwreg;
+
+ return 0;
+
+err_hwreg:
+ irdma_free_stag(iwdev, stag);
+
+free_pble:
+ if (iwpbl->pble_alloc.level != PBLE_LEVEL_0 && iwpbl->pbl_allocated)
+ irdma_free_pble(iwdev->rf->pble_rsrc, &iwpbl->pble_alloc);
+
+ return err;
+}
+
+/*
+ * irdma_reg_user_mr_type_qp - Handle QP memory registration @req - memory reg req @udata - user info @iwmr - irdma mr
+ */
+static int
+irdma_reg_user_mr_type_qp(struct irdma_mem_reg_req req,
+ struct ib_udata *udata,
+ struct irdma_mr *iwmr)
+{
+ struct irdma_device *iwdev = to_iwdev(iwmr->ibmr.device);
+ struct irdma_pbl *iwpbl = &iwmr->iwpbl;
+ struct irdma_ucontext *ucontext;
+ unsigned long flags;
+ u32 total;
+ int err;
+ u8 lvl;
+
+ total = req.sq_pages + req.rq_pages + IRDMA_SHADOW_PGCNT;
+ if (total > iwmr->page_cnt)
+ return -EINVAL;
+
+ total = req.sq_pages + req.rq_pages;
+ lvl = total > 2 ? PBLE_LEVEL_1 : PBLE_LEVEL_0;
+ err = irdma_handle_q_mem(iwdev, &req, iwpbl, lvl);
+ if (err)
+ return err;
+
+#if __FreeBSD_version >= 1400026
+ ucontext = rdma_udata_to_drv_context(udata, struct irdma_ucontext, ibucontext);
+#else
+ ucontext = to_ucontext(iwmr->ibpd.pd->uobject->context);
+#endif
+ spin_lock_irqsave(&ucontext->qp_reg_mem_list_lock, flags);
+ list_add_tail(&iwpbl->list, &ucontext->qp_reg_mem_list);
+ iwpbl->on_list = true;
+ spin_unlock_irqrestore(&ucontext->qp_reg_mem_list_lock, flags);
+
+ return 0;
+}
+
+/*
+ * irdma_reg_user_mr_type_cq - Handle CQ memory registration @req - memory reg req @udata - user info @iwmr - irdma mr
+ */
+static int
+irdma_reg_user_mr_type_cq(struct irdma_mem_reg_req req,
+ struct ib_udata *udata,
+ struct irdma_mr *iwmr)
+{
+ struct irdma_device *iwdev = to_iwdev(iwmr->ibmr.device);
+ struct irdma_pbl *iwpbl = &iwmr->iwpbl;
+ struct irdma_ucontext *ucontext;
+ unsigned long flags;
+ u32 total;
+ int err;
+ u8 lvl;
+
+ total = req.cq_pages +
+ ((iwdev->rf->sc_dev.hw_attrs.uk_attrs.feature_flags & IRDMA_FEATURE_CQ_RESIZE) ? 0 : IRDMA_SHADOW_PGCNT);
+ if (total > iwmr->page_cnt)
+ return -EINVAL;
+
+ lvl = req.cq_pages > 1 ? PBLE_LEVEL_1 : PBLE_LEVEL_0;
+ err = irdma_handle_q_mem(iwdev, &req, iwpbl, lvl);
+ if (err)
+ return err;
+
+#if __FreeBSD_version >= 1400026
+ ucontext = rdma_udata_to_drv_context(udata, struct irdma_ucontext, ibucontext);
+#else
+ ucontext = to_ucontext(iwmr->ibmr.pd->uobject->context);
+#endif
+ spin_lock_irqsave(&ucontext->cq_reg_mem_list_lock, flags);
+ list_add_tail(&iwpbl->list, &ucontext->cq_reg_mem_list);
+ iwpbl->on_list = true;
+ spin_unlock_irqrestore(&ucontext->cq_reg_mem_list_lock, flags);
+
+ return 0;
+}
+
/**
* irdma_reg_user_mr - Register a user memory region
* @pd: ptr of pd
@@ -2233,18 +2420,10 @@
{
#define IRDMA_MEM_REG_MIN_REQ_LEN offsetofend(struct irdma_mem_reg_req, sq_pages)
struct irdma_device *iwdev = to_iwdev(pd->device);
- struct irdma_ucontext *ucontext;
- struct irdma_pble_alloc *palloc;
- struct irdma_pbl *iwpbl;
- struct irdma_mr *iwmr;
- struct ib_umem *region;
struct irdma_mem_reg_req req = {};
- u32 total, stag = 0;
- u8 shadow_pgcnt = 1;
- unsigned long flags;
- int err = -EINVAL;
- u8 lvl;
- int ret;
+ struct ib_umem *region;
+ struct irdma_mr *iwmr;
+ int err;
if (len > iwdev->rf->sc_dev.hw_attrs.max_mr_size)
return ERR_PTR(-EINVAL);
@@ -2265,119 +2444,41 @@
return ERR_PTR(-EFAULT);
}
- iwmr = kzalloc(sizeof(*iwmr), GFP_KERNEL);
- if (!iwmr) {
+ iwmr = irdma_alloc_iwmr(region, pd, virt, req.reg_type);
+ if (IS_ERR(iwmr)) {
ib_umem_release(region);
- return ERR_PTR(-ENOMEM);
+ return (struct ib_mr *)iwmr;
}
- iwpbl = &iwmr->iwpbl;
- iwpbl->iwmr = iwmr;
- iwmr->region = region;
- iwmr->ibmr.pd = pd;
- iwmr->ibmr.device = pd->device;
- iwmr->ibmr.iova = virt;
- iwmr->page_size = IRDMA_HW_PAGE_SIZE;
- iwmr->page_msk = ~(IRDMA_HW_PAGE_SIZE - 1);
-
- iwmr->len = region->length;
- iwpbl->user_base = virt;
- palloc = &iwpbl->pble_alloc;
- iwmr->type = req.reg_type;
- iwmr->page_cnt = irdma_ib_umem_num_dma_blocks(region, iwmr->page_size, virt);
-
switch (req.reg_type) {
case IRDMA_MEMREG_TYPE_QP:
- total = req.sq_pages + req.rq_pages + shadow_pgcnt;
- if (total > iwmr->page_cnt) {
- err = -EINVAL;
- goto error;
- }
- total = req.sq_pages + req.rq_pages;
- lvl = total > 2 ? PBLE_LEVEL_1 : PBLE_LEVEL_0;
- err = irdma_handle_q_mem(iwdev, &req, iwpbl, lvl);
+ err = irdma_reg_user_mr_type_qp(req, udata, iwmr);
if (err)
goto error;
-#if __FreeBSD_version >= 1400026
- ucontext = rdma_udata_to_drv_context(udata, struct irdma_ucontext, ibucontext);
-#else
- ucontext = to_ucontext(pd->uobject->context);
-#endif
- spin_lock_irqsave(&ucontext->qp_reg_mem_list_lock, flags);
- list_add_tail(&iwpbl->list, &ucontext->qp_reg_mem_list);
- iwpbl->on_list = true;
- spin_unlock_irqrestore(&ucontext->qp_reg_mem_list_lock, flags);
break;
case IRDMA_MEMREG_TYPE_CQ:
- if (iwdev->rf->sc_dev.hw_attrs.uk_attrs.feature_flags & IRDMA_FEATURE_CQ_RESIZE)
- shadow_pgcnt = 0;
- total = req.cq_pages + shadow_pgcnt;
- if (total > iwmr->page_cnt) {
- err = -EINVAL;
- goto error;
- }
-
- lvl = req.cq_pages > 1 ? PBLE_LEVEL_1 : PBLE_LEVEL_0;
- err = irdma_handle_q_mem(iwdev, &req, iwpbl, lvl);
+ err = irdma_reg_user_mr_type_cq(req, udata, iwmr);
if (err)
goto error;
-#if __FreeBSD_version >= 1400026
- ucontext = rdma_udata_to_drv_context(udata, struct irdma_ucontext, ibucontext);
-#else
- ucontext = to_ucontext(pd->uobject->context);
-#endif
- spin_lock_irqsave(&ucontext->cq_reg_mem_list_lock, flags);
- list_add_tail(&iwpbl->list, &ucontext->cq_reg_mem_list);
- iwpbl->on_list = true;
- spin_unlock_irqrestore(&ucontext->cq_reg_mem_list_lock, flags);
break;
case IRDMA_MEMREG_TYPE_MEM:
- lvl = iwmr->page_cnt != 1 ? PBLE_LEVEL_1 | PBLE_LEVEL_2 : PBLE_LEVEL_0;
- err = irdma_setup_pbles(iwdev->rf, iwmr, lvl);
+ err = irdma_reg_user_mr_type_mem(iwmr, access);
if (err)
goto error;
- if (lvl) {
- ret = irdma_check_mr_contiguous(palloc,
- iwmr->page_size);
- if (ret) {
- irdma_free_pble(iwdev->rf->pble_rsrc, palloc);
- iwpbl->pbl_allocated = false;
- }
- }
-
- stag = irdma_create_stag(iwdev);
- if (!stag) {
- err = -ENOMEM;
- goto error;
- }
-
- iwmr->stag = stag;
- iwmr->ibmr.rkey = stag;
- iwmr->ibmr.lkey = stag;
- iwmr->access = access;
- err = irdma_hwreg_mr(iwdev, iwmr, access);
- if (err) {
- irdma_free_stag(iwdev, stag);
- goto error;
- }
-
break;
default:
+ err = -EINVAL;
goto error;
}
- iwmr->type = req.reg_type;
-
return &iwmr->ibmr;
error:
- if (palloc->level != PBLE_LEVEL_0 && iwpbl->pbl_allocated)
- irdma_free_pble(iwdev->rf->pble_rsrc, palloc);
ib_umem_release(region);
- kfree(iwmr);
+ irdma_free_iwmr(iwmr);
return ERR_PTR(err);
}
@@ -2825,8 +2926,7 @@
err = irdma_uk_post_receive(ukqp, &post_recv);
if (err) {
irdma_debug(&iwqp->iwdev->rf->sc_dev, IRDMA_DEBUG_VERBS,
- "post_recv err %d\n",
- err);
+ "post_recv err %d\n", err);
goto out;
}
@@ -2876,64 +2976,6 @@
}
}
-static inline void
-set_ib_wc_op_sq(struct irdma_cq_poll_info *cq_poll_info,
- struct ib_wc *entry)
-{
- struct irdma_sc_qp *qp;
-
- switch (cq_poll_info->op_type) {
- case IRDMA_OP_TYPE_RDMA_WRITE:
- case IRDMA_OP_TYPE_RDMA_WRITE_SOL:
- entry->opcode = IB_WC_RDMA_WRITE;
- break;
- case IRDMA_OP_TYPE_RDMA_READ_INV_STAG:
- case IRDMA_OP_TYPE_RDMA_READ:
- entry->opcode = IB_WC_RDMA_READ;
- break;
- case IRDMA_OP_TYPE_SEND_SOL:
- case IRDMA_OP_TYPE_SEND_SOL_INV:
- case IRDMA_OP_TYPE_SEND_INV:
- case IRDMA_OP_TYPE_SEND:
- entry->opcode = IB_WC_SEND;
- break;
- case IRDMA_OP_TYPE_FAST_REG_NSMR:
- entry->opcode = IB_WC_REG_MR;
- break;
- case IRDMA_OP_TYPE_INV_STAG:
- entry->opcode = IB_WC_LOCAL_INV;
- break;
- default:
- qp = cq_poll_info->qp_handle;
- irdma_dev_err(to_ibdev(qp->dev), "Invalid opcode = %d in CQE\n",
- cq_poll_info->op_type);
- entry->status = IB_WC_GENERAL_ERR;
- }
-}
-
-static inline void
-set_ib_wc_op_rq(struct irdma_cq_poll_info *cq_poll_info,
- struct ib_wc *entry, bool send_imm_support)
-{
- /**
- * iWARP does not support sendImm, so the presence of Imm data
- * must be WriteImm.
- */
- if (!send_imm_support) {
- entry->opcode = cq_poll_info->imm_valid ?
- IB_WC_RECV_RDMA_WITH_IMM : IB_WC_RECV;
- return;
- }
- switch (cq_poll_info->op_type) {
- case IB_OPCODE_RDMA_WRITE_ONLY_WITH_IMMEDIATE:
- case IB_OPCODE_RDMA_WRITE_LAST_WITH_IMMEDIATE:
- entry->opcode = IB_WC_RECV_RDMA_WITH_IMM;
- break;
- default:
- entry->opcode = IB_WC_RECV;
- }
-}
-
/**
* irdma_process_cqe - process cqe info
* @entry: processed cqe
@@ -3119,8 +3161,7 @@
return npolled;
error:
irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_VERBS,
- "%s: Error polling CQ, irdma_err: %d\n",
- __func__, ret);
+ "%s: Error polling CQ, irdma_err: %d\n", __func__, ret);
return ret;
}
@@ -3288,12 +3329,8 @@
int ret = 0;
bool ipv4;
u16 vlan_id;
- union {
- struct sockaddr saddr;
- struct sockaddr_in saddr_in;
- struct sockaddr_in6 saddr_in6;
- } sgid_addr;
- unsigned char dmac[ETH_ALEN];
+ union irdma_sockaddr sgid_addr;
+ unsigned char dmac[ETHER_ADDR_LEN];
rdma_gid2ip((struct sockaddr *)&sgid_addr, ibgid);
@@ -3303,9 +3340,8 @@
irdma_netdev_vlan_ipv6(ip_addr, &vlan_id, NULL);
ipv4 = false;
irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_VERBS,
- "qp_id=%d, IP6address=%pI6\n",
- ibqp->qp_num,
- ip_addr);
+ "qp_id=%d, IP6address=%x:%x:%x:%x\n", ibqp->qp_num,
+ IRDMA_PRINT_IP6(ip_addr));
irdma_mcast_mac_v6(ip_addr, dmac);
} else {
ip_addr[0] = ntohl(sgid_addr.saddr_in.sin_addr.s_addr);
@@ -3313,8 +3349,9 @@
vlan_id = irdma_get_vlan_ipv4(ip_addr);
irdma_mcast_mac_v4(ip_addr, dmac);
irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_VERBS,
- "qp_id=%d, IP4address=%pI4, MAC=%pM\n",
- ibqp->qp_num, ip_addr, dmac);
+ "qp_id=%d, IP4address=%x, MAC=%x:%x:%x:%x:%x:%x\n",
+ ibqp->qp_num, ip_addr[0], dmac[0], dmac[1], dmac[2],
+ dmac[3], dmac[4], dmac[5]);
}
spin_lock_irqsave(&rf->qh_list_lock, flags);
@@ -3425,11 +3462,7 @@
struct irdma_mcast_grp_ctx_entry_info mcg_info = {0};
int ret;
unsigned long flags;
- union {
- struct sockaddr saddr;
- struct sockaddr_in saddr_in;
- struct sockaddr_in6 saddr_in6;
- } sgid_addr;
+ union irdma_sockaddr sgid_addr;
rdma_gid2ip((struct sockaddr *)&sgid_addr, ibgid);
if (!ipv6_addr_v4mapped((struct in6_addr *)ibgid))
@@ -3442,8 +3475,8 @@
mc_qht_elem = mcast_list_lookup_ip(rf, ip_addr);
if (!mc_qht_elem) {
spin_unlock_irqrestore(&rf->qh_list_lock, flags);
- irdma_debug(&iwdev->rf->sc_dev,
- IRDMA_DEBUG_VERBS, "address not found MCG\n");
+ irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_VERBS,
+ "address not found MCG\n");
return 0;
}
@@ -3455,8 +3488,8 @@
ret = irdma_mcast_cqp_op(iwdev, &mc_qht_elem->mc_grp_ctx,
IRDMA_OP_MC_DESTROY);
if (ret) {
- irdma_debug(&iwdev->rf->sc_dev,
- IRDMA_DEBUG_VERBS, "failed MC_DESTROY MCG\n");
+ irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_VERBS,
+ "failed MC_DESTROY MCG\n");
spin_lock_irqsave(&rf->qh_list_lock, flags);
mcast_list_add(rf, mc_qht_elem);
spin_unlock_irqrestore(&rf->qh_list_lock, flags);
@@ -3473,8 +3506,8 @@
ret = irdma_mcast_cqp_op(iwdev, &mc_qht_elem->mc_grp_ctx,
IRDMA_OP_MC_MODIFY);
if (ret) {
- irdma_debug(&iwdev->rf->sc_dev,
- IRDMA_DEBUG_VERBS, "failed Modify MCG\n");
+ irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_VERBS,
+ "failed Modify MCG\n");
return ret;
}
}
diff --git a/sys/dev/irdma/irdma_ws.c b/sys/dev/irdma/irdma_ws.c
--- a/sys/dev/irdma/irdma_ws.c
+++ b/sys/dev/irdma/irdma_ws.c
@@ -1,7 +1,7 @@
/*-
* SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB
*
- * Copyright (c) 2017 - 2022 Intel Corporation
+ * Copyright (c) 2017 - 2023 Intel Corporation
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@@ -293,13 +293,13 @@
ws_tree_root = vsi->dev->ws_tree_root;
if (!ws_tree_root) {
- irdma_debug(vsi->dev, IRDMA_DEBUG_WS, "Creating root node\n");
ws_tree_root = irdma_alloc_node(vsi, user_pri,
WS_NODE_TYPE_PARENT, NULL);
if (!ws_tree_root) {
ret = -ENOMEM;
goto exit;
}
+ irdma_debug(vsi->dev, IRDMA_DEBUG_WS, "Creating root node = %d\n", ws_tree_root->index);
ret = irdma_ws_cqp_cmd(vsi, ws_tree_root, IRDMA_OP_WS_ADD_NODE);
if (ret) {
diff --git a/sys/dev/irdma/osdep.h b/sys/dev/irdma/osdep.h
--- a/sys/dev/irdma/osdep.h
+++ b/sys/dev/irdma/osdep.h
@@ -1,7 +1,7 @@
/*-
* SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB
*
- * Copyright (c) 2021 - 2022 Intel Corporation
+ * Copyright (c) 2021 - 2023 Intel Corporation
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@@ -51,11 +51,11 @@
#include <linux/spinlock.h>
#include <linux/timer.h>
#include <linux/workqueue.h>
+#include <linux/atomic.h>
#include <sys/bus.h>
#include <machine/bus.h>
-#define ATOMIC atomic_t
#define IOMEM
#define IRDMA_NTOHS(a) ntohs(a)
#define MAKEMASK(m, s) ((m) << (s))
@@ -96,6 +96,9 @@
#define irdma_mb() mb()
#define irdma_wmb() wmb()
+#ifndef smp_mb
+#define smp_mb() mb()
+#endif
#define irdma_get_virt_to_phy vtophys
#define __aligned_u64 uint64_t __aligned(8)
@@ -111,7 +114,7 @@
#define irdma_print(S, ...) printf("%s:%d "S, __FUNCTION__, __LINE__, ##__VA_ARGS__)
#define irdma_debug_buf(dev, mask, desc, buf, size) \
do { \
- u32 i; \
+ u32 i; \
if (!((mask) & (dev)->debug_mask)) { \
break; \
} \
@@ -121,14 +124,14 @@
irdma_debug(dev, mask, "index %03d val: %016lx\n", i, ((unsigned long *)(buf))[i / 8]); \
} while(0)
-#define irdma_debug(h, m, s, ...) \
-do { \
- if (!(h)) { \
- if ((m) == IRDMA_DEBUG_INIT) \
+#define irdma_debug(h, m, s, ...) \
+do { \
+ if (!(h)) { \
+ if ((m) == IRDMA_DEBUG_INIT) \
printf("irdma INIT " s, ##__VA_ARGS__); \
- } else if (((m) & (h)->debug_mask)) { \
- printf("irdma " s, ##__VA_ARGS__); \
- } \
+ } else if (((m) & (h)->debug_mask)) { \
+ printf("irdma " s, ##__VA_ARGS__); \
+ } \
} while (0)
#define irdma_dev_err(ibdev, fmt, ...) \
pr_err("%s:%s:%d ERR "fmt, (ibdev)->name, __func__, __LINE__, ##__VA_ARGS__)
@@ -137,17 +140,8 @@
#define irdma_dev_info(a, b, ...) printf(b, ##__VA_ARGS__)
#define irdma_pr_warn printf
-#define dump_struct(s, sz, name) \
-do { \
- unsigned char *a; \
- printf("%s %u", (name), (unsigned int)(sz)); \
- for (a = (unsigned char*)(s); a < (unsigned char *)(s) + (sz) ; a ++) { \
- if ((u64)a % 8 == 0) \
- printf("\n%p ", a); \
- printf("%2x ", *a); \
- } \
- printf("\n"); \
-}while(0)
+#define IRDMA_PRINT_IP6(ip6) \
+ ((u32*)ip6)[0], ((u32*)ip6)[1], ((u32*)ip6)[2], ((u32*)ip6)[3]
/*
* debug definition end
@@ -176,6 +170,7 @@
bus_size_t mem_bus_space_size;
void *dev;
struct irdma_task_arg task_arg;
+ atomic_t event_rfcnt;
};
#define irdma_pr_info(fmt, args ...) printf("%s: WARN "fmt, __func__, ## args)
@@ -187,38 +182,34 @@
#define rt_tos2priority(tos) (tos >> 5)
#define ah_attr_to_dmac(attr) ((attr).dmac)
-#define kc_ib_modify_qp_is_ok(cur_state, next_state, type, mask, ll) \
- ib_modify_qp_is_ok(cur_state, next_state, type, mask)
-#define kc_rdma_gid_attr_network_type(sgid_attr, gid_type, gid) \
- ib_gid_to_network_type(gid_type, gid)
#define irdma_del_timer_compat(tt) del_timer((tt))
#define IRDMA_TAILQ_FOREACH CK_STAILQ_FOREACH
#define IRDMA_TAILQ_FOREACH_SAFE CK_STAILQ_FOREACH_SAFE
#define between(a, b, c) (bool)(c-a >= b-a)
-#define rd32(a, reg) irdma_rd32((a)->dev_context, (reg))
-#define wr32(a, reg, value) irdma_wr32((a)->dev_context, (reg), (value))
+#define rd32(a, reg) irdma_rd32((a)->dev_context, (reg))
+#define wr32(a, reg, value) irdma_wr32((a)->dev_context, (reg), (value))
-#define rd64(a, reg) irdma_rd64((a)->dev_context, (reg))
-#define wr64(a, reg, value) irdma_wr64((a)->dev_context, (reg), (value))
+#define rd64(a, reg) irdma_rd64((a)->dev_context, (reg))
+#define wr64(a, reg, value) irdma_wr64((a)->dev_context, (reg), (value))
#define db_wr32(value, a) writel((value), (a))
void *hw_to_dev(struct irdma_hw *hw);
struct irdma_dma_mem {
- void *va;
- u64 pa;
+ void *va;
+ u64 pa;
bus_dma_tag_t tag;
bus_dmamap_t map;
bus_dma_segment_t seg;
bus_size_t size;
- int nseg;
- int flags;
+ int nseg;
+ int flags;
};
struct irdma_virt_mem {
- void *va;
- u32 size;
+ void *va;
+ u32 size;
};
struct irdma_dma_info {

File Metadata

Mime Type
text/plain
Expires
Mon, Jul 6, 4:50 PM (4 h, 41 m)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
34762734
Default Alt Text
D41425.diff (172 KB)

Event Timeline