diff --git a/contrib/ofed/libirdma/abi.h b/contrib/ofed/libirdma/abi.h
--- a/contrib/ofed/libirdma/abi.h
+++ b/contrib/ofed/libirdma/abi.h
@@ -1,7 +1,7 @@
/*-
* SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB
*
- * Copyright (C) 2019 - 2022 Intel Corporation
+ * Copyright (C) 2019 - 2023 Intel Corporation
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@@ -31,7 +31,6 @@
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
-/*$FreeBSD$*/
#ifndef PROVIDER_IRDMA_ABI_H
#define PROVIDER_IRDMA_ABI_H
@@ -86,6 +85,7 @@
struct ibv_create_qp ibv_cmd;
__aligned_u64 user_wqe_bufs;
__aligned_u64 user_compl_ctx;
+ __aligned_u64 comp_mask;
};
struct irdma_ucreate_qp_resp {
@@ -98,6 +98,9 @@
__u8 lsmm;
__u8 rsvd;
__u32 qp_caps;
+ __aligned_u64 comp_mask;
+ __u8 start_wqe_idx;
+ __u8 rsvd2[7];
};
struct irdma_umodify_qp_resp {
@@ -138,6 +141,8 @@
__u8 hw_rev;
__u8 rsvd2;
__aligned_u64 comp_mask;
+ __u16 min_hw_wq_size;
+ __u8 rsvd3[6];
};
struct irdma_ureg_mr {
diff --git a/contrib/ofed/libirdma/i40e_devids.h b/contrib/ofed/libirdma/i40e_devids.h
--- a/contrib/ofed/libirdma/i40e_devids.h
+++ b/contrib/ofed/libirdma/i40e_devids.h
@@ -31,7 +31,6 @@
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
-/*$FreeBSD$*/
#ifndef I40E_DEVIDS_H
#define I40E_DEVIDS_H
diff --git a/contrib/ofed/libirdma/i40iw_hw.h b/contrib/ofed/libirdma/i40iw_hw.h
--- a/contrib/ofed/libirdma/i40iw_hw.h
+++ b/contrib/ofed/libirdma/i40iw_hw.h
@@ -1,7 +1,7 @@
/*-
* SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB
*
- * Copyright (c) 2015 - 2022 Intel Corporation
+ * Copyright (c) 2015 - 2023 Intel Corporation
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@@ -31,7 +31,6 @@
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
-/*$FreeBSD$*/
#ifndef I40IW_HW_H
#define I40IW_HW_H
diff --git a/contrib/ofed/libirdma/ice_devids.h b/contrib/ofed/libirdma/ice_devids.h
--- a/contrib/ofed/libirdma/ice_devids.h
+++ b/contrib/ofed/libirdma/ice_devids.h
@@ -31,7 +31,6 @@
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
-/*$FreeBSD$*/
#ifndef ICE_DEVIDS_H
#define ICE_DEVIDS_H
diff --git a/contrib/ofed/libirdma/irdma-abi.h b/contrib/ofed/libirdma/irdma-abi.h
--- a/contrib/ofed/libirdma/irdma-abi.h
+++ b/contrib/ofed/libirdma/irdma-abi.h
@@ -35,7 +35,6 @@
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
-/*$FreeBSD$*/
#ifndef IRDMA_ABI_H
#define IRDMA_ABI_H
@@ -55,6 +54,11 @@
enum {
IRDMA_ALLOC_UCTX_USE_RAW_ATTR = 1 << 0,
+ IRDMA_ALLOC_UCTX_MIN_HW_WQ_SIZE = 1 << 1,
+};
+
+enum {
+ IRDMA_CREATE_QP_USE_START_WQE_IDX = 1 << 0,
};
struct irdma_alloc_ucontext_req {
@@ -83,6 +87,8 @@
__u8 hw_rev;
__u8 rsvd2;
__aligned_u64 comp_mask;
+ __u16 min_hw_wq_size;
+ __u8 rsvd3[6];
};
struct irdma_alloc_pd_resp {
@@ -102,6 +108,7 @@
struct irdma_create_qp_req {
__aligned_u64 user_wqe_bufs;
__aligned_u64 user_compl_ctx;
+ __aligned_u64 comp_mask;
};
struct irdma_mem_reg_req {
@@ -131,6 +138,9 @@
__u8 lsmm;
__u8 rsvd;
__u32 qp_caps;
+ __aligned_u64 comp_mask;
+ __u8 start_wqe_idx;
+ __u8 rsvd2[7];
};
struct irdma_modify_qp_resp {
diff --git a/contrib/ofed/libirdma/irdma.h b/contrib/ofed/libirdma/irdma.h
--- a/contrib/ofed/libirdma/irdma.h
+++ b/contrib/ofed/libirdma/irdma.h
@@ -31,7 +31,6 @@
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
-/*$FreeBSD$*/
#ifndef IRDMA_H
#define IRDMA_H
@@ -47,7 +46,6 @@
IRDMA_GEN_RSVD = 0,
IRDMA_GEN_1 = 1,
IRDMA_GEN_2 = 2,
- IRDMA_GEN_MAX = 2,
};
struct irdma_uk_attrs {
diff --git a/contrib/ofed/libirdma/irdma_defs.h b/contrib/ofed/libirdma/irdma_defs.h
--- a/contrib/ofed/libirdma/irdma_defs.h
+++ b/contrib/ofed/libirdma/irdma_defs.h
@@ -31,7 +31,6 @@
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
-/*$FreeBSD$*/
#ifndef IRDMA_DEFS_H
#define IRDMA_DEFS_H
@@ -457,19 +456,6 @@
IRDMA_WQE_SIZE_256 = 256,
};
-enum irdma_ws_op_type {
- IRDMA_WS_OP_TYPE_NODE = 0,
- IRDMA_WS_OP_TYPE_LEAF_NODE_GROUP,
-};
-
-enum irdma_ws_rate_limit_flags {
- IRDMA_WS_RATE_LIMIT_FLAGS_VALID = 0x1,
- IRDMA_WS_NO_RDMA_RATE_LIMIT = 0x2,
- IRDMA_WS_LEAF_NODE_IS_PART_GROUP = 0x4,
- IRDMA_WS_TREE_RATE_LIMITING = 0x8,
- IRDMA_WS_PACING_CONTROL = 0x10,
-};
-
/**
* set_64bit_val - set 64 bit value to hw wqe
* @wqe_words: wqe addr to write
diff --git a/contrib/ofed/libirdma/irdma_uk.c b/contrib/ofed/libirdma/irdma_uk.c
--- a/contrib/ofed/libirdma/irdma_uk.c
+++ b/contrib/ofed/libirdma/irdma_uk.c
@@ -31,7 +31,6 @@
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
-/*$FreeBSD$*/
#include "osdep.h"
#include "irdma_defs.h"
@@ -46,16 +45,16 @@
* @valid: The wqe valid
*/
static void
-irdma_set_fragment(__le64 * wqe, u32 offset, struct irdma_sge *sge,
+irdma_set_fragment(__le64 * wqe, u32 offset, struct ibv_sge *sge,
u8 valid)
{
if (sge) {
set_64bit_val(wqe, offset,
- FIELD_PREP(IRDMAQPSQ_FRAG_TO, sge->tag_off));
+ FIELD_PREP(IRDMAQPSQ_FRAG_TO, sge->addr));
set_64bit_val(wqe, offset + IRDMA_BYTE_8,
FIELD_PREP(IRDMAQPSQ_VALID, valid) |
- FIELD_PREP(IRDMAQPSQ_FRAG_LEN, sge->len) |
- FIELD_PREP(IRDMAQPSQ_FRAG_STAG, sge->stag));
+ FIELD_PREP(IRDMAQPSQ_FRAG_LEN, sge->length) |
+ FIELD_PREP(IRDMAQPSQ_FRAG_STAG, sge->lkey));
} else {
set_64bit_val(wqe, offset, 0);
set_64bit_val(wqe, offset + IRDMA_BYTE_8,
@@ -72,14 +71,14 @@
*/
static void
irdma_set_fragment_gen_1(__le64 * wqe, u32 offset,
- struct irdma_sge *sge, u8 valid)
+ struct ibv_sge *sge, u8 valid)
{
if (sge) {
set_64bit_val(wqe, offset,
- FIELD_PREP(IRDMAQPSQ_FRAG_TO, sge->tag_off));
+ FIELD_PREP(IRDMAQPSQ_FRAG_TO, sge->addr));
set_64bit_val(wqe, offset + IRDMA_BYTE_8,
- FIELD_PREP(IRDMAQPSQ_GEN1_FRAG_LEN, sge->len) |
- FIELD_PREP(IRDMAQPSQ_GEN1_FRAG_STAG, sge->stag));
+ FIELD_PREP(IRDMAQPSQ_GEN1_FRAG_LEN, sge->length) |
+ FIELD_PREP(IRDMAQPSQ_GEN1_FRAG_STAG, sge->lkey));
} else {
set_64bit_val(wqe, offset, 0);
set_64bit_val(wqe, offset + IRDMA_BYTE_8, 0);
@@ -210,8 +209,7 @@
if (IRDMA_RING_CURRENT_HEAD(qp->initial_ring) !=
IRDMA_RING_CURRENT_TAIL(qp->sq_ring) &&
!qp->push_mode) {
- if (post_sq)
- irdma_uk_qp_post_wr(qp);
+ irdma_uk_qp_post_wr(qp);
} else {
push = (__le64 *) ((uintptr_t)qp->push_wqe +
(wqe_idx & 0x7) * 0x20);
@@ -339,7 +337,7 @@
return EINVAL;
for (i = 0; i < op_info->num_lo_sges; i++)
- total_size += op_info->lo_sg_list[i].len;
+ total_size += op_info->lo_sg_list[i].length;
read_fence |= info->read_fence;
@@ -358,7 +356,7 @@
qp->sq_wrtrk_array[wqe_idx].signaled = info->signaled;
set_64bit_val(wqe, IRDMA_BYTE_16,
- FIELD_PREP(IRDMAQPSQ_FRAG_TO, op_info->rem_addr.tag_off));
+ FIELD_PREP(IRDMAQPSQ_FRAG_TO, op_info->rem_addr.addr));
if (info->imm_data_valid) {
set_64bit_val(wqe, IRDMA_BYTE_0,
@@ -387,7 +385,7 @@
++addl_frag_cnt;
}
- hdr = FIELD_PREP(IRDMAQPSQ_REMSTAG, op_info->rem_addr.stag) |
+ hdr = FIELD_PREP(IRDMAQPSQ_REMSTAG, op_info->rem_addr.lkey) |
FIELD_PREP(IRDMAQPSQ_OPCODE, info->op_type) |
FIELD_PREP(IRDMAQPSQ_IMMDATAFLAG, info->imm_data_valid) |
FIELD_PREP(IRDMAQPSQ_REPORTRTT, info->report_rtt) |
@@ -438,7 +436,7 @@
return EINVAL;
for (i = 0; i < op_info->num_lo_sges; i++)
- total_size += op_info->lo_sg_list[i].len;
+ total_size += op_info->lo_sg_list[i].length;
ret_code = irdma_fragcnt_to_quanta_sq(op_info->num_lo_sges, &quanta);
if (ret_code)
@@ -476,8 +474,8 @@
++addl_frag_cnt;
}
set_64bit_val(wqe, IRDMA_BYTE_16,
- FIELD_PREP(IRDMAQPSQ_FRAG_TO, op_info->rem_addr.tag_off));
- hdr = FIELD_PREP(IRDMAQPSQ_REMSTAG, op_info->rem_addr.stag) |
+ FIELD_PREP(IRDMAQPSQ_FRAG_TO, op_info->rem_addr.addr));
+ hdr = FIELD_PREP(IRDMAQPSQ_REMSTAG, op_info->rem_addr.lkey) |
FIELD_PREP(IRDMAQPSQ_REPORTRTT, (info->report_rtt ? 1 : 0)) |
FIELD_PREP(IRDMAQPSQ_ADDFRAGCNT, addl_frag_cnt) |
FIELD_PREP(IRDMAQPSQ_OPCODE,
@@ -526,7 +524,7 @@
return EINVAL;
for (i = 0; i < op_info->num_sges; i++)
- total_size += op_info->sg_list[i].len;
+ total_size += op_info->sg_list[i].length;
if (info->imm_data_valid)
frag_cnt = op_info->num_sges + 1;
@@ -621,15 +619,15 @@
* @polarity: compatibility parameter
*/
static void
-irdma_copy_inline_data_gen_1(u8 *wqe, struct irdma_sge *sge_list,
+irdma_copy_inline_data_gen_1(u8 *wqe, struct ibv_sge *sge_list,
u32 num_sges, u8 polarity)
{
u32 quanta_bytes_remaining = 16;
u32 i;
for (i = 0; i < num_sges; i++) {
- u8 *cur_sge = (u8 *)(uintptr_t)sge_list[i].tag_off;
- u32 sge_len = sge_list[i].len;
+ u8 *cur_sge = (u8 *)(uintptr_t)sge_list[i].addr;
+ u32 sge_len = sge_list[i].length;
while (sge_len) {
u32 bytes_copied;
@@ -684,7 +682,7 @@
* @polarity: polarity of wqe valid bit
*/
static void
-irdma_copy_inline_data(u8 *wqe, struct irdma_sge *sge_list,
+irdma_copy_inline_data(u8 *wqe, struct ibv_sge *sge_list,
u32 num_sges, u8 polarity)
{
u8 inline_valid = polarity << IRDMA_INLINE_VALID_S;
@@ -695,8 +693,8 @@
wqe += 8;
for (i = 0; i < num_sges; i++) {
- u8 *cur_sge = (u8 *)(uintptr_t)sge_list[i].tag_off;
- u32 sge_len = sge_list[i].len;
+ u8 *cur_sge = (u8 *)(uintptr_t)sge_list[i].addr;
+ u32 sge_len = sge_list[i].length;
while (sge_len) {
u32 bytes_copied;
@@ -776,7 +774,7 @@
return EINVAL;
for (i = 0; i < op_info->num_lo_sges; i++)
- total_size += op_info->lo_sg_list[i].len;
+ total_size += op_info->lo_sg_list[i].length;
if (unlikely(total_size > qp->max_inline_data))
return EINVAL;
@@ -789,9 +787,9 @@
qp->sq_wrtrk_array[wqe_idx].signaled = info->signaled;
read_fence |= info->read_fence;
set_64bit_val(wqe, IRDMA_BYTE_16,
- FIELD_PREP(IRDMAQPSQ_FRAG_TO, op_info->rem_addr.tag_off));
+ FIELD_PREP(IRDMAQPSQ_FRAG_TO, op_info->rem_addr.addr));
- hdr = FIELD_PREP(IRDMAQPSQ_REMSTAG, op_info->rem_addr.stag) |
+ hdr = FIELD_PREP(IRDMAQPSQ_REMSTAG, op_info->rem_addr.lkey) |
FIELD_PREP(IRDMAQPSQ_OPCODE, info->op_type) |
FIELD_PREP(IRDMAQPSQ_INLINEDATALEN, total_size) |
FIELD_PREP(IRDMAQPSQ_REPORTRTT, info->report_rtt ? 1 : 0) |
@@ -847,7 +845,7 @@
return EINVAL;
for (i = 0; i < op_info->num_sges; i++)
- total_size += op_info->sg_list[i].len;
+ total_size += op_info->sg_list[i].length;
if (unlikely(total_size > qp->max_inline_data))
return EINVAL;
@@ -912,7 +910,7 @@
u64 hdr;
u32 wqe_idx;
bool local_fence = false;
- struct irdma_sge sge = {0};
+ struct ibv_sge sge = {0};
u16 quanta = IRDMA_QP_WQE_MIN_QUANTA;
info->push_wqe = qp->push_db ? true : false;
@@ -923,7 +921,7 @@
if (!wqe)
return ENOSPC;
- sge.stag = op_info->target_stag;
+ sge.lkey = op_info->target_stag;
qp->wqe_ops.iw_set_fragment(wqe, IRDMA_BYTE_0, &sge, 0);
set_64bit_val(wqe, IRDMA_BYTE_16, 0);
@@ -1437,8 +1435,7 @@
IRDMA_RING_MOVE_TAIL(cq->cq_ring);
set_64bit_val(cq->shadow_area, IRDMA_BYTE_0,
IRDMA_RING_CURRENT_HEAD(cq->cq_ring));
- memset(info, 0,
- sizeof(struct irdma_cq_poll_info));
+ memset(info, 0, sizeof(*info));
return irdma_uk_cq_poll_cmpl(cq, info);
}
}
@@ -1511,7 +1508,6 @@
if (pring && IRDMA_RING_MORE_WORK(*pring))
move_cq_head = false;
}
-
if (move_cq_head) {
IRDMA_RING_MOVE_HEAD_NOCHECK(cq->cq_ring);
if (!IRDMA_RING_CURRENT_HEAD(cq->cq_ring))
@@ -1592,10 +1588,12 @@
int
irdma_get_sqdepth(struct irdma_uk_attrs *uk_attrs, u32 sq_size, u8 shift, u32 *sqdepth)
{
+ u32 min_size = (u32)uk_attrs->min_hw_wq_size << shift;
+
*sqdepth = irdma_round_up_wq((sq_size << shift) + IRDMA_SQ_RSVD);
- if (*sqdepth < ((u32)uk_attrs->min_hw_wq_size << shift))
- *sqdepth = uk_attrs->min_hw_wq_size << shift;
+ if (*sqdepth < min_size)
+ *sqdepth = min_size;
else if (*sqdepth > uk_attrs->max_hw_wq_quanta)
return EINVAL;
@@ -1609,10 +1607,12 @@
int
irdma_get_rqdepth(struct irdma_uk_attrs *uk_attrs, u32 rq_size, u8 shift, u32 *rqdepth)
{
+ u32 min_size = (u32)uk_attrs->min_hw_wq_size << shift;
+
*rqdepth = irdma_round_up_wq((rq_size << shift) + IRDMA_RQ_RSVD);
- if (*rqdepth < ((u32)uk_attrs->min_hw_wq_size << shift))
- *rqdepth = uk_attrs->min_hw_wq_size << shift;
+ if (*rqdepth < min_size)
+ *rqdepth = min_size;
else if (*rqdepth > uk_attrs->max_hw_rq_quanta)
return EINVAL;
@@ -1645,41 +1645,16 @@
{
u16 move_cnt = 1;
- if (qp->uk_attrs->feature_flags & IRDMA_FEATURE_RTS_AE)
+ if (info->start_wqe_idx)
+ move_cnt = info->start_wqe_idx;
+ else if (qp->uk_attrs->feature_flags & IRDMA_FEATURE_RTS_AE)
move_cnt = 3;
-
qp->conn_wqes = move_cnt;
IRDMA_RING_MOVE_HEAD_BY_COUNT_NOCHECK(qp->sq_ring, move_cnt);
IRDMA_RING_MOVE_TAIL_BY_COUNT(qp->sq_ring, move_cnt);
IRDMA_RING_MOVE_HEAD_BY_COUNT_NOCHECK(qp->initial_ring, move_cnt);
}
-/**
- * irdma_uk_calc_shift_wq - calculate WQE shift for both SQ and RQ
- * @ukinfo: qp initialization info
- * @sq_shift: Returns shift of SQ
- * @rq_shift: Returns shift of RQ
- */
-void
-irdma_uk_calc_shift_wq(struct irdma_qp_uk_init_info *ukinfo, u8 *sq_shift,
- u8 *rq_shift)
-{
- bool imm_support = ukinfo->uk_attrs->hw_rev >= IRDMA_GEN_2 ? true : false;
-
- irdma_get_wqe_shift(ukinfo->uk_attrs,
- imm_support ? ukinfo->max_sq_frag_cnt + 1 :
- ukinfo->max_sq_frag_cnt,
- ukinfo->max_inline_data, sq_shift);
-
- irdma_get_wqe_shift(ukinfo->uk_attrs, ukinfo->max_rq_frag_cnt, 0,
- rq_shift);
-
- if (ukinfo->uk_attrs->hw_rev == IRDMA_GEN_1) {
- if (ukinfo->abi_ver > 4)
- *rq_shift = IRDMA_MAX_RQ_WQE_SHIFT_GEN1;
- }
-}
-
/**
* irdma_uk_calc_depth_shift_sq - calculate depth and shift for SQ size.
* @ukinfo: qp initialization info
@@ -1692,6 +1667,7 @@
{
bool imm_support = ukinfo->uk_attrs->hw_rev >= IRDMA_GEN_2 ? true : false;
int status;
+
irdma_get_wqe_shift(ukinfo->uk_attrs,
imm_support ? ukinfo->max_sq_frag_cnt + 1 :
ukinfo->max_sq_frag_cnt,
@@ -1786,6 +1762,8 @@
qp->wqe_ops = iw_wqe_uk_ops_gen_1;
else
qp->wqe_ops = iw_wqe_uk_ops;
+ qp->start_wqe_idx = info->start_wqe_idx;
+
return ret_code;
}
diff --git a/contrib/ofed/libirdma/irdma_umain.h b/contrib/ofed/libirdma/irdma_umain.h
--- a/contrib/ofed/libirdma/irdma_umain.h
+++ b/contrib/ofed/libirdma/irdma_umain.h
@@ -31,7 +31,6 @@
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
-/*$FreeBSD$*/
#ifndef IRDMA_UMAIN_H
#define IRDMA_UMAIN_H
@@ -97,7 +96,6 @@
LIST_ENTRY(irdma_cq_buf) list;
struct irdma_cq_uk cq;
struct verbs_mr vmr;
- size_t buf_size;
};
extern pthread_mutex_t sigusr1_wait_mutex;
@@ -143,7 +141,6 @@
struct ibv_recv_wr *pend_rx_wr;
struct irdma_qp_uk qp;
enum ibv_qp_type qp_type;
- struct irdma_sge *recv_sges;
};
/* irdma_uverbs.c */
diff --git a/contrib/ofed/libirdma/irdma_umain.c b/contrib/ofed/libirdma/irdma_umain.c
--- a/contrib/ofed/libirdma/irdma_umain.c
+++ b/contrib/ofed/libirdma/irdma_umain.c
@@ -1,7 +1,7 @@
/*-
* SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB
*
- * Copyright (c) 2021 - 2023 Intel Corporation
+ * Copyright (c) 2021 - 2022 Intel Corporation
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@@ -31,7 +31,6 @@
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
-/*$FreeBSD$*/
#include <sys/mman.h>
@@ -49,7 +48,7 @@
/**
* Driver version
*/
-char libirdma_version[] = "1.2.17-k";
+char libirdma_version[] = "1.2.36-k";
unsigned int irdma_dbg;
diff --git a/contrib/ofed/libirdma/irdma_uquery.h b/contrib/ofed/libirdma/irdma_uquery.h
--- a/contrib/ofed/libirdma/irdma_uquery.h
+++ b/contrib/ofed/libirdma/irdma_uquery.h
@@ -31,7 +31,6 @@
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
-/*$FreeBSD$*/
#ifndef IRDMA_UQUERY_H
diff --git a/contrib/ofed/libirdma/irdma_user.h b/contrib/ofed/libirdma/irdma_user.h
--- a/contrib/ofed/libirdma/irdma_user.h
+++ b/contrib/ofed/libirdma/irdma_user.h
@@ -31,7 +31,6 @@
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
-/*$FreeBSD$*/
#ifndef IRDMA_USER_H
#define IRDMA_USER_H
@@ -51,7 +50,7 @@
#define irdma_access_privileges u32
#define irdma_physical_fragment u64
#define irdma_address_list u64 *
-#define irdma_sgl struct irdma_sge *
+#define irdma_sgl struct ibv_sge *
#define IRDMA_MAX_MR_SIZE 0x200000000000ULL
@@ -81,96 +80,6 @@
#define IRDMA_OP_TYPE_REC_IMM 0x3f
#define IRDMA_FLUSH_MAJOR_ERR 1
-#define IRDMA_SRQFLUSH_RSVD_MAJOR_ERR 0xfffe
-
-/* Async Events codes */
-#define IRDMA_AE_AMP_UNALLOCATED_STAG 0x0102
-#define IRDMA_AE_AMP_INVALID_STAG 0x0103
-#define IRDMA_AE_AMP_BAD_QP 0x0104
-#define IRDMA_AE_AMP_BAD_PD 0x0105
-#define IRDMA_AE_AMP_BAD_STAG_KEY 0x0106
-#define IRDMA_AE_AMP_BAD_STAG_INDEX 0x0107
-#define IRDMA_AE_AMP_BOUNDS_VIOLATION 0x0108
-#define IRDMA_AE_AMP_RIGHTS_VIOLATION 0x0109
-#define IRDMA_AE_AMP_TO_WRAP 0x010a
-#define IRDMA_AE_AMP_FASTREG_VALID_STAG 0x010c
-#define IRDMA_AE_AMP_FASTREG_MW_STAG 0x010d
-#define IRDMA_AE_AMP_FASTREG_INVALID_RIGHTS 0x010e
-#define IRDMA_AE_AMP_FASTREG_INVALID_LENGTH 0x0110
-#define IRDMA_AE_AMP_INVALIDATE_SHARED 0x0111
-#define IRDMA_AE_AMP_INVALIDATE_NO_REMOTE_ACCESS_RIGHTS 0x0112
-#define IRDMA_AE_AMP_INVALIDATE_MR_WITH_BOUND_WINDOWS 0x0113
-#define IRDMA_AE_AMP_MWBIND_VALID_STAG 0x0114
-#define IRDMA_AE_AMP_MWBIND_OF_MR_STAG 0x0115
-#define IRDMA_AE_AMP_MWBIND_TO_ZERO_BASED_STAG 0x0116
-#define IRDMA_AE_AMP_MWBIND_TO_MW_STAG 0x0117
-#define IRDMA_AE_AMP_MWBIND_INVALID_RIGHTS 0x0118
-#define IRDMA_AE_AMP_MWBIND_INVALID_BOUNDS 0x0119
-#define IRDMA_AE_AMP_MWBIND_TO_INVALID_PARENT 0x011a
-#define IRDMA_AE_AMP_MWBIND_BIND_DISABLED 0x011b
-#define IRDMA_AE_PRIV_OPERATION_DENIED 0x011c
-#define IRDMA_AE_AMP_INVALIDATE_TYPE1_MW 0x011d
-#define IRDMA_AE_AMP_MWBIND_ZERO_BASED_TYPE1_MW 0x011e
-#define IRDMA_AE_AMP_FASTREG_INVALID_PBL_HPS_CFG 0x011f
-#define IRDMA_AE_AMP_MWBIND_WRONG_TYPE 0x0120
-#define IRDMA_AE_AMP_FASTREG_PBLE_MISMATCH 0x0121
-#define IRDMA_AE_UDA_XMIT_DGRAM_TOO_LONG 0x0132
-#define IRDMA_AE_UDA_XMIT_BAD_PD 0x0133
-#define IRDMA_AE_UDA_XMIT_DGRAM_TOO_SHORT 0x0134
-#define IRDMA_AE_UDA_L4LEN_INVALID 0x0135
-#define IRDMA_AE_BAD_CLOSE 0x0201
-#define IRDMA_AE_RDMAP_ROE_BAD_LLP_CLOSE 0x0202
-#define IRDMA_AE_CQ_OPERATION_ERROR 0x0203
-#define IRDMA_AE_RDMA_READ_WHILE_ORD_ZERO 0x0205
-#define IRDMA_AE_STAG_ZERO_INVALID 0x0206
-#define IRDMA_AE_IB_RREQ_AND_Q1_FULL 0x0207
-#define IRDMA_AE_IB_INVALID_REQUEST 0x0208
-#define IRDMA_AE_WQE_UNEXPECTED_OPCODE 0x020a
-#define IRDMA_AE_WQE_INVALID_PARAMETER 0x020b
-#define IRDMA_AE_WQE_INVALID_FRAG_DATA 0x020c
-#define IRDMA_AE_IB_REMOTE_ACCESS_ERROR 0x020d
-#define IRDMA_AE_IB_REMOTE_OP_ERROR 0x020e
-#define IRDMA_AE_WQE_LSMM_TOO_LONG 0x0220
-#define IRDMA_AE_DDP_INVALID_MSN_GAP_IN_MSN 0x0301
-#define IRDMA_AE_DDP_UBE_DDP_MESSAGE_TOO_LONG_FOR_AVAILABLE_BUFFER 0x0303
-#define IRDMA_AE_DDP_UBE_INVALID_DDP_VERSION 0x0304
-#define IRDMA_AE_DDP_UBE_INVALID_MO 0x0305
-#define IRDMA_AE_DDP_UBE_INVALID_MSN_NO_BUFFER_AVAILABLE 0x0306
-#define IRDMA_AE_DDP_UBE_INVALID_QN 0x0307
-#define IRDMA_AE_DDP_NO_L_BIT 0x0308
-#define IRDMA_AE_RDMAP_ROE_INVALID_RDMAP_VERSION 0x0311
-#define IRDMA_AE_RDMAP_ROE_UNEXPECTED_OPCODE 0x0312
-#define IRDMA_AE_ROE_INVALID_RDMA_READ_REQUEST 0x0313
-#define IRDMA_AE_ROE_INVALID_RDMA_WRITE_OR_READ_RESP 0x0314
-#define IRDMA_AE_ROCE_RSP_LENGTH_ERROR 0x0316
-#define IRDMA_AE_ROCE_EMPTY_MCG 0x0380
-#define IRDMA_AE_ROCE_BAD_MC_IP_ADDR 0x0381
-#define IRDMA_AE_ROCE_BAD_MC_QPID 0x0382
-#define IRDMA_AE_MCG_QP_PROTOCOL_MISMATCH 0x0383
-#define IRDMA_AE_INVALID_ARP_ENTRY 0x0401
-#define IRDMA_AE_INVALID_TCP_OPTION_RCVD 0x0402
-#define IRDMA_AE_STALE_ARP_ENTRY 0x0403
-#define IRDMA_AE_INVALID_AH_ENTRY 0x0406
-#define IRDMA_AE_LLP_CLOSE_COMPLETE 0x0501
-#define IRDMA_AE_LLP_CONNECTION_RESET 0x0502
-#define IRDMA_AE_LLP_FIN_RECEIVED 0x0503
-#define IRDMA_AE_LLP_RECEIVED_MARKER_AND_LENGTH_FIELDS_DONT_MATCH 0x0504
-#define IRDMA_AE_LLP_RECEIVED_MPA_CRC_ERROR 0x0505
-#define IRDMA_AE_LLP_SEGMENT_TOO_SMALL 0x0507
-#define IRDMA_AE_LLP_SYN_RECEIVED 0x0508
-#define IRDMA_AE_LLP_TERMINATE_RECEIVED 0x0509
-#define IRDMA_AE_LLP_TOO_MANY_RETRIES 0x050a
-#define IRDMA_AE_LLP_TOO_MANY_KEEPALIVE_RETRIES 0x050b
-#define IRDMA_AE_LLP_DOUBT_REACHABILITY 0x050c
-#define IRDMA_AE_LLP_CONNECTION_ESTABLISHED 0x050e
-#define IRDMA_AE_RESOURCE_EXHAUSTION 0x0520
-#define IRDMA_AE_RESET_SENT 0x0601
-#define IRDMA_AE_TERMINATE_SENT 0x0602
-#define IRDMA_AE_RESET_NOT_SENT 0x0603
-#define IRDMA_AE_LCE_QP_CATASTROPHIC 0x0700
-#define IRDMA_AE_LCE_FUNCTION_CATASTROPHIC 0x0701
-#define IRDMA_AE_LCE_CQ_CATASTROPHIC 0x0702
-#define IRDMA_AE_QP_SUSPEND_COMPLETE 0x0900
enum irdma_device_caps_const {
IRDMA_WQE_SIZE = 4,
@@ -202,8 +111,7 @@
IRDMA_MAX_OUTBOUND_MSG_SIZE = 65537,
/* 64K +1 */
IRDMA_MAX_INBOUND_MSG_SIZE = 65537,
- IRDMA_MAX_PUSH_PAGE_COUNT = 1024,
- IRDMA_MAX_PE_ENA_VF_COUNT = 32,
+ IRDMA_MAX_PE_ENA_VF_COUNT = 32,
IRDMA_MAX_VF_FPM_ID = 47,
IRDMA_MAX_SQ_PAYLOAD_SIZE = 2145386496,
IRDMA_MAX_INLINE_DATA_SIZE = 101,
@@ -230,12 +138,7 @@
FLUSH_RETRY_EXC_ERR,
FLUSH_MW_BIND_ERR,
FLUSH_REM_INV_REQ_ERR,
-};
-
-enum irdma_qp_event_type {
- IRDMA_QP_EVENT_CATASTROPHIC,
- IRDMA_QP_EVENT_ACCESS_ERR,
- IRDMA_QP_EVENT_REQ_ERR,
+ FLUSH_RNR_RETRY_EXC_ERR,
};
enum irdma_cmpl_status {
@@ -283,12 +186,6 @@
struct irdma_qp_uk_init_info;
struct irdma_cq_uk_init_info;
-struct irdma_sge {
- irdma_tagged_offset tag_off;
- u32 len;
- irdma_stag stag;
-};
-
struct irdma_ring {
volatile u32 head;
volatile u32 tail; /* effective tail */
@@ -320,13 +217,13 @@
struct irdma_rdma_write {
irdma_sgl lo_sg_list;
u32 num_lo_sges;
- struct irdma_sge rem_addr;
+ struct ibv_sge rem_addr;
};
struct irdma_rdma_read {
irdma_sgl lo_sg_list;
u32 num_lo_sges;
- struct irdma_sge rem_addr;
+ struct ibv_sge rem_addr;
};
struct irdma_bind_window {
@@ -400,11 +297,6 @@
} stat;
};
-struct qp_err_code {
- enum irdma_flush_opcode flush_code;
- enum irdma_qp_event_type event_type;
-};
-
int irdma_uk_inline_rdma_write(struct irdma_qp_uk *qp,
struct irdma_post_sq_info *info, bool post_sq);
int irdma_uk_inline_send(struct irdma_qp_uk *qp,
@@ -427,9 +319,9 @@
bool post_sq);
struct irdma_wqe_uk_ops {
- void (*iw_copy_inline_data)(u8 *dest, struct irdma_sge *sge_list, u32 num_sges, u8 polarity);
+ void (*iw_copy_inline_data)(u8 *dest, struct ibv_sge *sge_list, u32 num_sges, u8 polarity);
u16 (*iw_inline_data_size_to_quanta)(u32 data_size);
- void (*iw_set_fragment)(__le64 *wqe, u32 offset, struct irdma_sge *sge,
+ void (*iw_set_fragment)(__le64 *wqe, u32 offset, struct ibv_sge *sge,
u8 valid);
void (*iw_set_mw_bind_wqe)(__le64 *wqe,
struct irdma_bind_window *op_info);
@@ -445,8 +337,6 @@
struct irdma_cq_uk_init_info *info);
int irdma_uk_qp_init(struct irdma_qp_uk *qp,
struct irdma_qp_uk_init_info *info);
-void irdma_uk_calc_shift_wq(struct irdma_qp_uk_init_info *ukinfo, u8 *sq_shift,
- u8 *rq_shift);
int irdma_uk_calc_depth_shift_sq(struct irdma_qp_uk_init_info *ukinfo,
u32 *sq_depth, u8 *sq_shift);
int irdma_uk_calc_depth_shift_rq(struct irdma_qp_uk_init_info *ukinfo,
@@ -495,6 +385,7 @@
u8 rwqe_polarity;
u8 rq_wqe_size;
u8 rq_wqe_size_multiplier;
+ u8 start_wqe_idx;
bool deferred_flag:1;
bool push_mode:1; /* whether the last post wqe was pushed */
bool push_dropped:1;
@@ -542,6 +433,7 @@
u32 sq_depth;
u32 rq_depth;
u8 first_sq_wq;
+ u8 start_wqe_idx;
u8 type;
u8 sq_shift;
u8 rq_shift;
@@ -575,75 +467,4 @@
void irdma_qp_push_wqe(struct irdma_qp_uk *qp, __le64 *wqe, u16 quanta,
u32 wqe_idx, bool post_sq);
void irdma_clr_wqes(struct irdma_qp_uk *qp, u32 qp_wqe_idx);
-
-static inline struct qp_err_code irdma_ae_to_qp_err_code(u16 ae_id)
-{
- struct qp_err_code qp_err = { 0 };
-
- switch (ae_id) {
- case IRDMA_AE_AMP_BOUNDS_VIOLATION:
- case IRDMA_AE_AMP_INVALID_STAG:
- case IRDMA_AE_AMP_RIGHTS_VIOLATION:
- case IRDMA_AE_AMP_UNALLOCATED_STAG:
- case IRDMA_AE_AMP_BAD_PD:
- case IRDMA_AE_AMP_BAD_QP:
- case IRDMA_AE_AMP_BAD_STAG_KEY:
- case IRDMA_AE_AMP_BAD_STAG_INDEX:
- case IRDMA_AE_AMP_TO_WRAP:
- case IRDMA_AE_PRIV_OPERATION_DENIED:
- qp_err.flush_code = FLUSH_PROT_ERR;
- qp_err.event_type = IRDMA_QP_EVENT_ACCESS_ERR;
- break;
- case IRDMA_AE_UDA_XMIT_BAD_PD:
- case IRDMA_AE_WQE_UNEXPECTED_OPCODE:
- qp_err.flush_code = FLUSH_LOC_QP_OP_ERR;
- qp_err.event_type = IRDMA_QP_EVENT_CATASTROPHIC;
- break;
- case IRDMA_AE_UDA_XMIT_DGRAM_TOO_SHORT:
- case IRDMA_AE_UDA_XMIT_DGRAM_TOO_LONG:
- case IRDMA_AE_UDA_L4LEN_INVALID:
- case IRDMA_AE_DDP_UBE_INVALID_MO:
- case IRDMA_AE_DDP_UBE_DDP_MESSAGE_TOO_LONG_FOR_AVAILABLE_BUFFER:
- qp_err.flush_code = FLUSH_LOC_LEN_ERR;
- qp_err.event_type = IRDMA_QP_EVENT_CATASTROPHIC;
- break;
- case IRDMA_AE_AMP_INVALIDATE_NO_REMOTE_ACCESS_RIGHTS:
- case IRDMA_AE_IB_REMOTE_ACCESS_ERROR:
- qp_err.flush_code = FLUSH_REM_ACCESS_ERR;
- qp_err.event_type = IRDMA_QP_EVENT_ACCESS_ERR;
- break;
- case IRDMA_AE_AMP_MWBIND_INVALID_RIGHTS:
- case IRDMA_AE_AMP_MWBIND_BIND_DISABLED:
- case IRDMA_AE_AMP_MWBIND_INVALID_BOUNDS:
- case IRDMA_AE_AMP_MWBIND_VALID_STAG:
- qp_err.flush_code = FLUSH_MW_BIND_ERR;
- qp_err.event_type = IRDMA_QP_EVENT_ACCESS_ERR;
- break;
- case IRDMA_AE_LLP_TOO_MANY_RETRIES:
- qp_err.flush_code = FLUSH_RETRY_EXC_ERR;
- qp_err.event_type = IRDMA_QP_EVENT_CATASTROPHIC;
- break;
- case IRDMA_AE_IB_INVALID_REQUEST:
- qp_err.flush_code = FLUSH_REM_INV_REQ_ERR;
- qp_err.event_type = IRDMA_QP_EVENT_REQ_ERR;
- break;
- case IRDMA_AE_LLP_SEGMENT_TOO_SMALL:
- case IRDMA_AE_LLP_RECEIVED_MPA_CRC_ERROR:
- case IRDMA_AE_ROCE_RSP_LENGTH_ERROR:
- case IRDMA_AE_IB_REMOTE_OP_ERROR:
- qp_err.flush_code = FLUSH_REM_OP_ERR;
- qp_err.event_type = IRDMA_QP_EVENT_CATASTROPHIC;
- break;
- case IRDMA_AE_LCE_QP_CATASTROPHIC:
- qp_err.flush_code = FLUSH_FATAL_ERR;
- qp_err.event_type = IRDMA_QP_EVENT_CATASTROPHIC;
- break;
- default:
- qp_err.flush_code = FLUSH_GENERAL_ERR;
- qp_err.event_type = IRDMA_QP_EVENT_CATASTROPHIC;
- break;
- }
-
- return qp_err;
-}
#endif /* IRDMA_USER_H */
diff --git a/contrib/ofed/libirdma/irdma_uverbs.c b/contrib/ofed/libirdma/irdma_uverbs.c
--- a/contrib/ofed/libirdma/irdma_uverbs.c
+++ b/contrib/ofed/libirdma/irdma_uverbs.c
@@ -31,7 +31,6 @@
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
-/*$FreeBSD$*/
#include <config.h>
#include <stdlib.h>
@@ -153,6 +152,7 @@
err_free:
free(iwupd);
+
errno = err;
return NULL;
}
@@ -164,7 +164,6 @@
int
irdma_ufree_pd(struct ibv_pd *pd)
{
- struct irdma_uvcontext *iwvctx = container_of(pd->context, struct irdma_uvcontext, ibv_ctx);
struct irdma_upd *iwupd;
int ret;
@@ -375,12 +374,12 @@
* @cqe_64byte_ena: enable 64byte cqe
*/
static inline int
-get_cq_size(int ncqe, u8 hw_rev, bool cqe_64byte_ena)
+get_cq_size(int ncqe, u8 hw_rev)
{
ncqe++;
/* Completions with immediate require 1 extra entry */
- if (!cqe_64byte_ena && hw_rev > IRDMA_GEN_1)
+ if (hw_rev > IRDMA_GEN_1)
ncqe *= 2;
if (ncqe < IRDMA_U_MINCQ_SIZE)
@@ -389,11 +388,8 @@
return ncqe;
}
-static inline size_t get_cq_total_bytes(u32 cq_size, bool cqe_64byte_ena){
- if (cqe_64byte_ena)
- return roundup(cq_size * sizeof(struct irdma_extended_cqe), IRDMA_HW_PAGE_SIZE);
- else
- return roundup(cq_size * sizeof(struct irdma_cqe), IRDMA_HW_PAGE_SIZE);
+static inline size_t get_cq_total_bytes(u32 cq_size) {
+ return roundup(cq_size * sizeof(struct irdma_cqe), IRDMA_HW_PAGE_SIZE);
}
/**
@@ -421,7 +417,6 @@
u32 cq_pages;
int ret, ncqe;
u8 hw_rev;
- bool cqe_64byte_ena;
iwvctx = container_of(context, struct irdma_uvcontext, ibv_ctx);
uk_attrs = &iwvctx->uk_attrs;
@@ -455,11 +450,10 @@
return NULL;
}
- cqe_64byte_ena = uk_attrs->feature_flags & IRDMA_FEATURE_64_BYTE_CQE ? true : false;
- info.cq_size = get_cq_size(attr_ex->cqe, hw_rev, cqe_64byte_ena);
+ info.cq_size = get_cq_size(attr_ex->cqe, hw_rev);
+ total_size = get_cq_total_bytes(info.cq_size);
iwucq->comp_vector = attr_ex->comp_vector;
LIST_INIT(&iwucq->resize_list);
- total_size = get_cq_total_bytes(info.cq_size, cqe_64byte_ena);
cq_pages = total_size >> IRDMA_HW_PAGE_SHIFT;
if (!(uk_attrs->feature_flags & IRDMA_FEATURE_CQ_RESIZE))
@@ -529,8 +523,6 @@
info.cq_id = resp.cq_id;
/* Do not report the CQE's reserved for immediate and burned by HW */
iwucq->verbs_cq.cq.cqe = ncqe;
- if (cqe_64byte_ena)
- info.avoid_mem_cflct = true;
info.cqe_alloc_db = (u32 *)((u8 *)iwvctx->db + IRDMA_DB_CQ_OFFSET);
irdma_uk_cq_init(&iwucq->cq, &info);
return &iwucq->verbs_cq.cq_ex;
@@ -586,7 +578,7 @@
irdma_free_cq_buf(struct irdma_cq_buf *cq_buf)
{
ibv_cmd_dereg_mr(&cq_buf->vmr.ibv_mr);
- irdma_free_hw_buf(cq_buf->cq.cq_base, cq_buf->buf_size);
+ irdma_free_hw_buf(cq_buf->cq.cq_base, get_cq_total_bytes(cq_buf->cq.cq_size));
free(cq_buf);
}
@@ -1323,6 +1315,8 @@
cmd.user_wqe_bufs = (__u64) ((uintptr_t)info->sq);
cmd.user_compl_ctx = (__u64) (uintptr_t)&iwuqp->qp;
+ cmd.comp_mask |= IRDMA_CREATE_QP_USE_START_WQE_IDX;
+
ret = ibv_cmd_create_qp(pd, &iwuqp->ibv_qp, attr, &cmd.ibv_cmd,
sizeof(cmd), &resp.ibv_resp,
sizeof(struct irdma_ucreate_qp_resp));
@@ -1332,6 +1326,8 @@
info->sq_size = resp.actual_sq_size;
info->rq_size = resp.actual_rq_size;
info->first_sq_wq = legacy_mode ? 1 : resp.lsmm;
+ if (resp.comp_mask & IRDMA_CREATE_QP_USE_START_WQE_IDX)
+ info->start_wqe_idx = resp.start_wqe_idx;
info->qp_caps = resp.qp_caps;
info->qp_id = resp.qp_id;
iwuqp->irdma_drv_opt = resp.irdma_drv_opt;
@@ -1380,6 +1376,8 @@
if (attr->cap.max_send_sge > uk_attrs->max_hw_wq_frags ||
attr->cap.max_recv_sge > uk_attrs->max_hw_wq_frags ||
+ attr->cap.max_send_wr > uk_attrs->max_hw_wq_quanta ||
+ attr->cap.max_recv_wr > uk_attrs->max_hw_rq_quanta ||
attr->cap.max_inline_data > uk_attrs->max_hw_inline) {
errno = EINVAL;
return NULL;
@@ -1431,18 +1429,12 @@
attr->cap.max_recv_wr = info.rq_size;
}
- iwuqp->recv_sges = calloc(attr->cap.max_recv_sge, sizeof(*iwuqp->recv_sges));
- if (!iwuqp->recv_sges) {
- status = errno; /* preserve errno */
- goto err_destroy_lock;
- }
-
info.wqe_alloc_db = (u32 *)iwvctx->db;
info.legacy_mode = iwvctx->legacy_mode;
info.sq_wrtrk_array = calloc(info.sq_depth, sizeof(*info.sq_wrtrk_array));
if (!info.sq_wrtrk_array) {
status = errno; /* preserve errno */
- goto err_free_rsges;
+ goto err_destroy_lock;
}
info.rq_wrid_array = calloc(info.rq_depth, sizeof(*info.rq_wrid_array));
@@ -1476,8 +1468,6 @@
free(info.rq_wrid_array);
err_free_sq_wrtrk:
free(info.sq_wrtrk_array);
-err_free_rsges:
- free(iwuqp->recv_sges);
err_destroy_lock:
pthread_spin_destroy(&iwuqp->lock);
err_free_qp:
@@ -1636,7 +1626,6 @@
free(iwuqp->qp.rq_wrid_array);
irdma_free_hw_buf(iwuqp->qp.sq_base, iwuqp->buf_size);
- free(iwuqp->recv_sges);
free(iwuqp);
return 0;
@@ -1646,26 +1635,6 @@
return ret;
}
-/**
- * irdma_copy_sg_list - copy sg list for qp
- * @sg_list: copied into sg_list
- * @sgl: copy from sgl
- * @num_sges: count of sg entries
- * @max_sges: count of max supported sg entries
- */
-static void
-irdma_copy_sg_list(struct irdma_sge *sg_list, struct ibv_sge *sgl,
- int num_sges)
-{
- int i;
-
- for (i = 0; i < num_sges; i++) {
- sg_list[i].tag_off = sgl[i].addr;
- sg_list[i].len = sgl[i].length;
- sg_list[i].stag = sgl[i].lkey;
- }
-}
-
/**
* calc_type2_mw_stag - calculate type 2 MW stag
* @rkey: desired rkey of the MW
@@ -1744,7 +1713,7 @@
info.stag_to_inv = ib_wr->imm_data;
}
info.op.send.num_sges = ib_wr->num_sge;
- info.op.send.sg_list = (struct irdma_sge *)ib_wr->sg_list;
+ info.op.send.sg_list = (struct ibv_sge *)ib_wr->sg_list;
if (ib_qp->qp_type == IBV_QPT_UD) {
struct irdma_uah *ah = container_of(ib_wr->wr.ud.ah,
struct irdma_uah, ibv_ah);
@@ -1775,9 +1744,9 @@
info.op_type = IRDMA_OP_TYPE_RDMA_WRITE;
info.op.rdma_write.num_lo_sges = ib_wr->num_sge;
- info.op.rdma_write.lo_sg_list = (void *)ib_wr->sg_list;
- info.op.rdma_write.rem_addr.tag_off = ib_wr->wr.rdma.remote_addr;
- info.op.rdma_write.rem_addr.stag = ib_wr->wr.rdma.rkey;
+ info.op.rdma_write.lo_sg_list = ib_wr->sg_list;
+ info.op.rdma_write.rem_addr.addr = ib_wr->wr.rdma.remote_addr;
+ info.op.rdma_write.rem_addr.lkey = ib_wr->wr.rdma.rkey;
if (ib_wr->send_flags & IBV_SEND_INLINE)
err = irdma_uk_inline_rdma_write(&iwuqp->qp, &info, false);
else
@@ -1789,10 +1758,10 @@
break;
}
info.op_type = IRDMA_OP_TYPE_RDMA_READ;
- info.op.rdma_read.rem_addr.tag_off = ib_wr->wr.rdma.remote_addr;
- info.op.rdma_read.rem_addr.stag = ib_wr->wr.rdma.rkey;
+ info.op.rdma_read.rem_addr.addr = ib_wr->wr.rdma.remote_addr;
+ info.op.rdma_read.rem_addr.lkey = ib_wr->wr.rdma.rkey;
- info.op.rdma_read.lo_sg_list = (void *)ib_wr->sg_list;
+ info.op.rdma_read.lo_sg_list = ib_wr->sg_list;
info.op.rdma_read.num_lo_sges = ib_wr->num_sge;
err = irdma_uk_rdma_read(&iwuqp->qp, &info, false, false);
break;
@@ -1874,14 +1843,11 @@
struct ibv_recv_wr **bad_wr)
{
struct irdma_post_rq_info post_recv = {};
- struct irdma_sge *sg_list;
struct irdma_uqp *iwuqp;
bool reflush = false;
int err = 0;
iwuqp = container_of(ib_qp, struct irdma_uqp, ibv_qp);
- sg_list = iwuqp->recv_sges;
-
err = pthread_spin_lock(&iwuqp->lock);
if (err)
return err;
@@ -1898,8 +1864,7 @@
}
post_recv.num_sges = ib_wr->num_sge;
post_recv.wr_id = ib_wr->wr_id;
- irdma_copy_sg_list(sg_list, ib_wr->sg_list, ib_wr->num_sge);
- post_recv.sg_list = sg_list;
+ post_recv.sg_list = ib_wr->sg_list;
err = irdma_uk_post_receive(&iwuqp->qp, &post_recv);
if (err) {
*bad_wr = ib_wr;
@@ -2023,7 +1988,6 @@
u32 cq_pages;
int cqe_needed;
int ret = 0;
- bool cqe_64byte_ena;
iwucq = container_of(cq, struct irdma_ucq, verbs_cq.cq);
iwvctx = container_of(cq->context, struct irdma_uvcontext, ibv_ctx);
@@ -2035,14 +1999,11 @@
if (cqe < uk_attrs->min_hw_cq_size || cqe > uk_attrs->max_hw_cq_size - 1)
return EINVAL;
- cqe_64byte_ena = uk_attrs->feature_flags & IRDMA_FEATURE_64_BYTE_CQE ? true : false;
-
- cqe_needed = get_cq_size(cqe, uk_attrs->hw_rev, cqe_64byte_ena);
-
+ cqe_needed = get_cq_size(cqe, uk_attrs->hw_rev);
if (cqe_needed == iwucq->cq.cq_size)
return 0;
- cq_size = get_cq_total_bytes(cqe_needed, cqe_64byte_ena);
+ cq_size = get_cq_total_bytes(cqe_needed);
cq_pages = cq_size >> IRDMA_HW_PAGE_SHIFT;
cq_base = irdma_alloc_hw_buf(cq_size);
if (!cq_base)
@@ -2078,7 +2039,6 @@
goto err_resize;
memcpy(&cq_buf->cq, &iwucq->cq, sizeof(cq_buf->cq));
- cq_buf->buf_size = cq_size;
cq_buf->vmr = iwucq->vmr;
iwucq->vmr = new_mr;
irdma_uk_cq_resize(&iwucq->cq, cq_base, cqe_needed);
diff --git a/contrib/ofed/libirdma/osdep.h b/contrib/ofed/libirdma/osdep.h
--- a/contrib/ofed/libirdma/osdep.h
+++ b/contrib/ofed/libirdma/osdep.h
@@ -31,7 +31,6 @@
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
-/*$FreeBSD$*/
#ifndef _ICRDMA_OSDEP_H_
#define _ICRDMA_OSDEP_H_
@@ -78,7 +77,6 @@
/* constants */
#define STATS_TIMER_DELAY 60000
-/* a couple of linux size defines */
#define BIT_ULL(a) (1ULL << (a))
#define min(a, b) ((a) > (b) ? (b) : (a))
#ifndef likely
@@ -91,9 +89,6 @@
#define __aligned_u64 uint64_t __aligned(8)
#define VLAN_PRIO_SHIFT 13
-#if __FreeBSD_version < 1400000
-#define IB_USER_VERBS_EX_CMD_MODIFY_QP IB_USER_VERBS_CMD_MODIFY_QP
-#endif
/*
* debug definition section
diff --git a/sys/dev/irdma/fbsd_kcompat.h b/sys/dev/irdma/fbsd_kcompat.h
--- a/sys/dev/irdma/fbsd_kcompat.h
+++ b/sys/dev/irdma/fbsd_kcompat.h
@@ -48,33 +48,23 @@
#define from_tasklet(var, callback_tasklet, tasklet_fieldname) \
container_of(callback_tasklet, typeof(*var), tasklet_fieldname)
#endif
-
-#if __FreeBSD_version >= 1400000
#define IRDMA_SET_RDMA_OBJ_SIZE(ib_struct, drv_struct, member) \
(sizeof(struct drv_struct) + \
BUILD_BUG_ON_ZERO(offsetof(struct drv_struct, member)) + \
BUILD_BUG_ON_ZERO( \
!__same_type(((struct drv_struct *)NULL)->member, \
struct ib_struct)))
-#endif /* __FreeBSD_version > 1400000 */
#define set_ibdev_dma_device(ibdev, dev) \
ibdev.dma_device = (dev)
#define set_max_sge(props, rf) \
((props)->max_sge = (rf)->sc_dev.hw_attrs.uk_attrs.max_hw_wq_frags)
-#define rdma_query_gid(ibdev, port, index, gid) \
- ib_get_cached_gid(ibdev, port, index, gid, NULL)
#define kmap(pg) page_address(pg)
#define kmap_local_page(pg) page_address(pg)
#define kunmap(pg)
#define kunmap_local(pg)
#define IB_UVERBS_CQ_FLAGS_TIMESTAMP_COMPLETION IB_CQ_FLAGS_TIMESTAMP_COMPLETION
-#if __FreeBSD_version < 1400026
-#define kc_irdma_destroy_qp(ibqp, udata) irdma_destroy_qp(ibqp)
-#else
-#define kc_irdma_destroy_qp(ibqp, udata) irdma_destroy_qp(ibqp, udata)
-#endif
#ifndef IB_QP_ATTR_STANDARD_BITS
#define IB_QP_ATTR_STANDARD_BITS GENMASK(20, 0)
#endif
@@ -135,53 +125,23 @@
return rdma_id->route.addr.dev_addr.net;
}
-#if __FreeBSD_version < 1400026
-struct ib_cq *irdma_create_cq(struct ib_device *ibdev,
- const struct ib_cq_init_attr *attr,
- struct ib_ucontext *context,
- struct ib_udata *udata);
-#else
int irdma_create_cq(struct ib_cq *ibcq,
const struct ib_cq_init_attr *attr,
struct ib_udata *udata);
-#endif
struct ib_qp *irdma_create_qp(struct ib_pd *ibpd,
struct ib_qp_init_attr *init_attr,
struct ib_udata *udata);
-#if __FreeBSD_version >= 1400026
int irdma_create_ah(struct ib_ah *ib_ah,
struct ib_ah_attr *attr, u32 flags,
struct ib_udata *udata);
int irdma_create_ah_stub(struct ib_ah *ib_ah,
struct ib_ah_attr *attr, u32 flags,
struct ib_udata *udata);
-#else
-struct ib_ah *irdma_create_ah(struct ib_pd *ibpd,
- struct ib_ah_attr *attr,
- struct ib_udata *udata);
-struct ib_ah *irdma_create_ah_stub(struct ib_pd *ibpd,
- struct ib_ah_attr *attr,
- struct ib_udata *udata);
-#endif
void irdma_ether_copy(u8 *dmac, struct ib_ah_attr *attr);
-
-#if __FreeBSD_version >= 1400026
void irdma_destroy_ah(struct ib_ah *ibah, u32 flags);
void irdma_destroy_ah_stub(struct ib_ah *ibah, u32 flags);
-#else
-int irdma_destroy_ah(struct ib_ah *ibah);
-int irdma_destroy_ah_stub(struct ib_ah *ibah);
-#endif
-#if __FreeBSD_version < 1400026
-int irdma_destroy_qp(struct ib_qp *ibqp);
-#else
int irdma_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata);
-#endif
-#if __FreeBSD_version < 1400026
-int irdma_dereg_mr(struct ib_mr *ib_mr);
-#else
int irdma_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata);
-#endif
int ib_get_eth_speed(struct ib_device *dev, u32 port_num, u8 *speed, u8 *width);
enum rdma_link_layer irdma_get_link_layer(struct ib_device *ibdev,
u8 port_num);
@@ -208,10 +168,6 @@
void irdma_unregister_qset(struct irdma_sc_vsi *vsi,
struct irdma_ws_node *tc_node);
void ib_unregister_device(struct ib_device *ibdev);
-#if __FreeBSD_version < 1400026
-int rdma_user_mmap_io(struct ib_ucontext *ucontext, struct vm_area_struct *vma,
- unsigned long pfn, unsigned long size, pgprot_t prot);
-#endif
void irdma_disassociate_ucontext(struct ib_ucontext *context);
int kc_irdma_set_roce_cm_info(struct irdma_qp *iwqp,
struct ib_qp_attr *attr,
@@ -245,13 +201,8 @@
struct irdma_mr;
struct irdma_cq;
struct irdma_cq_buf;
-#if __FreeBSD_version < 1400026
-struct ib_mr *irdma_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type,
- u32 max_num_sg);
-#else
struct ib_mr *irdma_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type,
u32 max_num_sg, struct ib_udata *udata);
-#endif
int irdma_hwreg_mr(struct irdma_device *iwdev, struct irdma_mr *iwmr,
u16 access);
struct ib_mr *irdma_rereg_mr_trans(struct irdma_mr *iwmr, u64 start, u64 len,
@@ -264,8 +215,8 @@
int irdma_validate_qp_attrs(struct ib_qp_init_attr *init_attr,
struct irdma_device *iwdev);
void irdma_setup_virt_qp(struct irdma_device *iwdev,
- struct irdma_qp *iwqp,
- struct irdma_qp_init_info *init_info);
+ struct irdma_qp *iwqp,
+ struct irdma_qp_init_info *init_info);
int irdma_setup_kmode_qp(struct irdma_device *iwdev,
struct irdma_qp *iwqp,
struct irdma_qp_init_info *info,
@@ -281,35 +232,14 @@
struct irdma_qp_host_ctx_info *ctx_info);
int irdma_cqp_create_qp_cmd(struct irdma_qp *iwqp);
void irdma_dealloc_push_page(struct irdma_pci_f *rf,
- struct irdma_sc_qp *qp);
+ struct irdma_qp *iwqp);
int irdma_process_resize_list(struct irdma_cq *iwcq, struct irdma_device *iwdev,
struct irdma_cq_buf *lcqe_buf);
-#if __FreeBSD_version < 1400026
-int irdma_destroy_cq(struct ib_cq *ib_cq);
-#else
void irdma_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata);
-#endif
-#if __FreeBSD_version < 1400026
-struct ib_ucontext *irdma_alloc_ucontext(struct ib_device *, struct ib_udata *);
-#else
int irdma_alloc_ucontext(struct ib_ucontext *uctx, struct ib_udata *udata);
-#endif
-#if __FreeBSD_version < 1400026
-int irdma_dealloc_ucontext(struct ib_ucontext *);
-#else
void irdma_dealloc_ucontext(struct ib_ucontext *context);
-#endif
-#if __FreeBSD_version < 1400026
-struct ib_pd *irdma_alloc_pd(struct ib_device *, struct ib_ucontext *,
- struct ib_udata *);
-#else
int irdma_alloc_pd(struct ib_pd *pd, struct ib_udata *udata);
-#endif
-#if __FreeBSD_version < 1400026
-int irdma_dealloc_pd(struct ib_pd *);
-#else
void irdma_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata);
-#endif
int irdma_add_gid(struct ib_device *, u8, unsigned int, const union ib_gid *,
const struct ib_gid_attr *, void **);
int irdma_del_gid(struct ib_device *, u8, unsigned int, void **);
diff --git a/sys/dev/irdma/fbsd_kcompat.c b/sys/dev/irdma/fbsd_kcompat.c
--- a/sys/dev/irdma/fbsd_kcompat.c
+++ b/sys/dev/irdma/fbsd_kcompat.c
@@ -662,8 +662,7 @@
"manage_hmc_pm_func_table", 0},
{IRDMA_OP_SUSPEND, "suspend", "suspend", 0},
{IRDMA_OP_RESUME, "resume", "resume", 0},
- {IRDMA_OP_MANAGE_VCHNL_REQ_PBLE_BP, "manage_vchnl_req_pble_bp",
- "manage_vchnl_req_pble_bp", 0},
+ {25, "manage_vchnl_req_pble_bp", "manage_vchnl_req_pble_bp", 0},
{IRDMA_OP_QUERY_FPM_VAL, "query_fpm_val", "query_fpm_val", 0},
{IRDMA_OP_COMMIT_FPM_VAL, "commit_fpm_val", "commit_fpm_val", 0},
{IRDMA_OP_AH_CREATE, "ah_create", "ah_create", 0},
diff --git a/sys/dev/irdma/icrdma.c b/sys/dev/irdma/icrdma.c
--- a/sys/dev/irdma/icrdma.c
+++ b/sys/dev/irdma/icrdma.c
@@ -52,7 +52,7 @@
/**
* Driver version
*/
-char irdma_driver_version[] = "1.2.17-k";
+char irdma_driver_version[] = "1.2.36-k";
/**
* irdma_init_tunable - prepare tunables
@@ -245,7 +245,7 @@
mtu);
else if (mtu < IRDMA_MIN_MTU_IPV6)
irdma_dev_warn(to_ibdev(dev),
- "MTU setting [%d] too low for RDMA traffic. Minimum MTU is 1280 for IPv6\\n",
+ "MTU setting [%d] too low for RDMA traffic. Minimum MTU is 1280 for IPv6\n",
mtu);
}
diff --git a/sys/dev/irdma/icrdma_hw.h b/sys/dev/irdma/icrdma_hw.h
--- a/sys/dev/irdma/icrdma_hw.h
+++ b/sys/dev/irdma/icrdma_hw.h
@@ -1,7 +1,7 @@
/*-
* SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB
*
- * Copyright (c) 2017 - 2022 Intel Corporation
+ * Copyright (c) 2017 - 2023 Intel Corporation
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@@ -123,6 +123,7 @@
ICRDMA_MAX_IRD_SIZE = 32,
ICRDMA_MAX_ORD_SIZE = 32,
ICRDMA_MIN_WQ_SIZE = 8 /* WQEs */,
+ ICRDMA_MAX_PUSH_PAGE_COUNT = 256,
};
diff --git a/sys/dev/irdma/icrdma_hw.c b/sys/dev/irdma/icrdma_hw.c
--- a/sys/dev/irdma/icrdma_hw.c
+++ b/sys/dev/irdma/icrdma_hw.c
@@ -1,7 +1,7 @@
/*-
* SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB
*
- * Copyright (c) 2017 - 2022 Intel Corporation
+ * Copyright (c) 2017 - 2023 Intel Corporation
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@@ -76,7 +76,7 @@
ICRDMA_CQPSQ_CQ_CEQID,
ICRDMA_CQPSQ_CQ_CQID,
ICRDMA_COMMIT_FPM_CQCNT,
- ICRDMA_CQPSQ_UPESD_HMCFNID
+ ICRDMA_CQPSQ_UPESD_HMCFNID,
};
static u8 icrdma_shifts[IRDMA_MAX_SHIFTS] = {
@@ -86,7 +86,7 @@
ICRDMA_CQPSQ_CQ_CEQID_S,
ICRDMA_CQPSQ_CQ_CQID_S,
ICRDMA_COMMIT_FPM_CQCNT_S,
- ICRDMA_CQPSQ_UPESD_HMCFNID_S
+ ICRDMA_CQPSQ_UPESD_HMCFNID_S,
};
/**
@@ -210,8 +210,6 @@
dev->hw_regs[i] = (u32 IOMEM *) (hw_addr + icrdma_regs[i]);
}
- dev->hw_attrs.max_hw_vf_fpm_id = IRDMA_MAX_VF_FPM_ID;
- dev->hw_attrs.first_hw_vf_fpm_id = IRDMA_FIRST_VF_FPM_ID;
for (i = 0; i < IRDMA_MAX_SHIFTS; ++i)
dev->hw_shifts[i] = icrdma_shifts[i];
@@ -231,6 +229,7 @@
dev->hw_attrs.max_hw_ord = ICRDMA_MAX_ORD_SIZE;
dev->hw_attrs.max_stat_inst = ICRDMA_MAX_STATS_COUNT;
dev->hw_attrs.max_stat_idx = IRDMA_HW_STAT_INDEX_MAX_GEN_2;
+ dev->hw_attrs.max_hw_device_pages = ICRDMA_MAX_PUSH_PAGE_COUNT;
dev->hw_attrs.uk_attrs.max_hw_wq_frags = ICRDMA_MAX_WQ_FRAGMENT_COUNT;
dev->hw_attrs.uk_attrs.max_hw_read_sges = ICRDMA_MAX_SGE_RD;
diff --git a/sys/dev/irdma/irdma-abi.h b/sys/dev/irdma/irdma-abi.h
--- a/sys/dev/irdma/irdma-abi.h
+++ b/sys/dev/irdma/irdma-abi.h
@@ -54,6 +54,11 @@
enum {
IRDMA_ALLOC_UCTX_USE_RAW_ATTR = 1 << 0,
+ IRDMA_ALLOC_UCTX_MIN_HW_WQ_SIZE = 1 << 1,
+};
+
+enum {
+ IRDMA_CREATE_QP_USE_START_WQE_IDX = 1 << 0,
};
struct irdma_alloc_ucontext_req {
@@ -82,6 +87,8 @@
__u8 hw_rev;
__u8 rsvd2;
__aligned_u64 comp_mask;
+ __u16 min_hw_wq_size;
+ __u8 rsvd3[6];
};
struct irdma_alloc_pd_resp {
@@ -101,6 +108,7 @@
struct irdma_create_qp_req {
__aligned_u64 user_wqe_bufs;
__aligned_u64 user_compl_ctx;
+ __aligned_u64 comp_mask;
};
struct irdma_mem_reg_req {
@@ -130,6 +138,9 @@
__u8 lsmm;
__u8 rsvd;
__u32 qp_caps;
+ __aligned_u64 comp_mask;
+ __u8 start_wqe_idx;
+ __u8 rsvd2[7];
};
struct irdma_modify_qp_resp {
diff --git a/sys/dev/irdma/irdma.h b/sys/dev/irdma/irdma.h
--- a/sys/dev/irdma/irdma.h
+++ b/sys/dev/irdma/irdma.h
@@ -1,7 +1,7 @@
/*-
* SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB
*
- * Copyright (c) 2017 - 2023 Intel Corporation
+ * Copyright (c) 2017 - 2022 Intel Corporation
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@@ -183,7 +183,7 @@
IRDMA_GEN_RSVD = 0,
IRDMA_GEN_1 = 1,
IRDMA_GEN_2 = 2,
- IRDMA_GEN_MAX = 2,
+ IRDMA_GEN_MAX = IRDMA_GEN_2,
};
struct irdma_uk_attrs {
diff --git a/sys/dev/irdma/irdma_cm.c b/sys/dev/irdma/irdma_cm.c
--- a/sys/dev/irdma/irdma_cm.c
+++ b/sys/dev/irdma/irdma_cm.c
@@ -1079,14 +1079,14 @@
*type = IRDMA_MPA_REQUEST_ACCEPT;
- if (len < sizeof(struct ietf_mpa_v1)) {
+ if (len < sizeof(*mpa_frame)) {
irdma_debug(&cm_node->iwdev->rf->sc_dev, IRDMA_DEBUG_CM,
"ietf buffer small (%x)\n", len);
return -EINVAL;
}
mpa_frame = (struct ietf_mpa_v1 *)buf;
- mpa_hdr_len = sizeof(struct ietf_mpa_v1);
+ mpa_hdr_len = sizeof(*mpa_frame);
priv_data_len = ntohs(mpa_frame->priv_data_len);
if (priv_data_len > IETF_MAX_PRIV_DATA_LEN) {
@@ -1440,6 +1440,7 @@
sizeof(struct option_base) + TCP_OPTIONS_PADDING];
struct irdma_kmem_info opts;
int optionssize = 0;
+
/* Sending MSS option */
union all_known_options *options;
@@ -1615,7 +1616,8 @@
}
/**
- * irdma_netdev_vlan_ipv6 - Gets the netdev and mac
+ * irdma_get_vlan_mac_ipv6 - Get the vlan and mac for an IPv6
+ * address
* @addr: local IPv6 address
* @vlan_id: vlan id for the given IPv6 address
* @mac: mac address for the given IPv6 address
@@ -1623,14 +1625,12 @@
* Returns the net_device of the IPv6 address and also sets the
* vlan id and mac for that address.
*/
-if_t
-irdma_netdev_vlan_ipv6(struct iw_cm_id *cm_id, u32 *addr, u16 *vlan_id, u8 *mac)
+void
+irdma_get_vlan_mac_ipv6(struct iw_cm_id *cm_id, u32 *addr, u16 *vlan_id, u8 *mac)
{
if_t ip_dev = NULL;
struct in6_addr laddr6;
-#ifdef VIMAGE
- struct vnet *vnet = irdma_cmid_to_vnet(cm_id);
-#endif
+ struct vnet *vnet = &init_net;
struct ifaddr *ifa;
u16 scope_id = 0;
@@ -1645,10 +1645,9 @@
scope_id = ntohs(laddr6.__u6_addr.__u6_addr16[1]);
#ifdef VIMAGE
- ip_dev = ip6_ifp_find(vnet, laddr6, scope_id);
-#else
- ip_dev = ip6_ifp_find(&init_net, laddr6, scope_id);
+ vnet = irdma_cmid_to_vnet(cm_id);
#endif
+ ip_dev = ip6_ifp_find(vnet, laddr6, scope_id);
if (ip_dev) {
if (vlan_id)
*vlan_id = rdma_vlan_dev_vlan_id(ip_dev);
@@ -1656,8 +1655,6 @@
if (ifa && ifa->ifa_addr && mac)
ether_addr_copy(mac, if_getlladdr(ip_dev));
}
-
- return ip_dev;
}
/**
@@ -1668,16 +1665,13 @@
irdma_get_vlan_ipv4(struct iw_cm_id *cm_id, u32 *addr)
{
if_t netdev;
-#ifdef VIMAGE
- struct vnet *vnet = irdma_cmid_to_vnet(cm_id);
-#endif
+ struct vnet *vnet = &init_net;
u16 vlan_id = 0xFFFF;
#ifdef VIMAGE
- netdev = ip_ifp_find(vnet, htonl(addr[0]));
-#else
- netdev = ip_ifp_find(&init_net, htonl(addr[0]));
+ vnet = irdma_cmid_to_vnet(cm_id);
#endif
+ netdev = ip_ifp_find(vnet, htonl(addr[0]));
if (netdev) {
vlan_id = rdma_vlan_dev_vlan_id(netdev);
dev_put(netdev);
@@ -2546,7 +2540,7 @@
u32 inc_sequence;
int optionsize;
- optionsize = (tcph->th_off << 2) - sizeof(struct tcphdr);
+ optionsize = (tcph->th_off << 2) - sizeof(*tcph);
inc_sequence = ntohl(tcph->th_seq);
switch (cm_node->state) {
@@ -2613,7 +2607,7 @@
u32 inc_sequence;
int optionsize;
- optionsize = (tcph->th_off << 2) - sizeof(struct tcphdr);
+ optionsize = (tcph->th_off << 2) - sizeof(*tcph);
inc_sequence = ntohl(tcph->th_seq);
switch (cm_node->state) {
case IRDMA_CM_STATE_SYN_SENT:
@@ -2687,7 +2681,7 @@
int optionsize;
u32 datasize = rbuf->datalen;
- optionsize = (tcph->th_off << 2) - sizeof(struct tcphdr);
+ optionsize = (tcph->th_off << 2) - sizeof(*tcph);
if (irdma_check_seq(cm_node, tcph))
return -EINVAL;
@@ -3493,6 +3487,7 @@
if (iwqp->ietf_mem.va) {
if (iwqp->lsmm_mr)
iwdev->ibdev.dereg_mr(iwqp->lsmm_mr, NULL);
+
irdma_free_dma_mem(iwdev->rf->sc_dev.hw,
&iwqp->ietf_mem);
iwqp->ietf_mem.va = NULL;
@@ -3536,8 +3531,8 @@
cm_node->vlan_id = irdma_get_vlan_ipv4(cm_id, cm_node->loc_addr);
} else {
cm_node->ipv4 = false;
- irdma_netdev_vlan_ipv6(cm_id, cm_node->loc_addr,
- &cm_node->vlan_id, NULL);
+ irdma_get_vlan_mac_ipv6(cm_id, cm_node->loc_addr, &cm_node->vlan_id,
+ NULL);
}
irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_CM, "Accept vlan_id=%d\n",
cm_node->vlan_id);
@@ -3744,7 +3739,7 @@
raddr6->sin6_addr.__u6_addr.__u6_addr32);
cm_info.loc_port = ntohs(laddr6->sin6_port);
cm_info.rem_port = ntohs(raddr6->sin6_port);
- irdma_netdev_vlan_ipv6(cm_id, cm_info.loc_addr, &cm_info.vlan_id, NULL);
+ irdma_get_vlan_mac_ipv6(cm_id, cm_info.loc_addr, &cm_info.vlan_id, NULL);
}
cm_info.cm_id = cm_id;
cm_info.qh_qpid = iwdev->vsi.ilq->qp_id;
@@ -3873,8 +3868,8 @@
laddr6->sin6_addr.__u6_addr.__u6_addr32);
cm_info.loc_port = ntohs(laddr6->sin6_port);
if (!IN6_IS_ADDR_UNSPECIFIED(&laddr6->sin6_addr)) {
- irdma_netdev_vlan_ipv6(cm_id, cm_info.loc_addr,
- &cm_info.vlan_id, NULL);
+ irdma_get_vlan_mac_ipv6(cm_id, cm_info.loc_addr,
+ &cm_info.vlan_id, NULL);
} else {
cm_info.vlan_id = 0xFFFF;
wildcard = true;
diff --git a/sys/dev/irdma/irdma_ctrl.c b/sys/dev/irdma/irdma_ctrl.c
--- a/sys/dev/irdma/irdma_ctrl.c
+++ b/sys/dev/irdma/irdma_ctrl.c
@@ -785,14 +785,14 @@
set_64bit_val(qp_ctx, IRDMA_BYTE_144,
FIELD_PREP(IRDMAQPC_STAT_INDEX, info->stats_idx));
set_64bit_val(qp_ctx, IRDMA_BYTE_152,
- FIELD_PREP(IRDMAQPC_MACADDRESS, irdma_mac_to_u64(roce_info->mac_addr)));
+ FIELD_PREP(IRDMAQPC_MACADDRESS,
+ irdma_mac_to_u64(roce_info->mac_addr)));
set_64bit_val(qp_ctx, IRDMA_BYTE_160,
FIELD_PREP(IRDMAQPC_ORDSIZE, roce_info->ord_size) |
FIELD_PREP(IRDMAQPC_IRDSIZE, irdma_sc_get_encoded_ird_size(roce_info->ird_size)) |
FIELD_PREP(IRDMAQPC_WRRDRSPOK, roce_info->wr_rdresp_en) |
FIELD_PREP(IRDMAQPC_RDOK, roce_info->rd_en) |
FIELD_PREP(IRDMAQPC_USESTATSINSTANCE, info->stats_idx_valid) |
- FIELD_PREP(IRDMAQPC_BINDEN, roce_info->bind_en) |
FIELD_PREP(IRDMAQPC_FASTREGEN, roce_info->fast_reg_en) |
FIELD_PREP(IRDMAQPC_DCQCNENABLE, roce_info->dcqcn_en) |
FIELD_PREP(IRDMAQPC_RCVNOICRC, roce_info->rcv_no_icrc) |
@@ -1016,7 +1016,6 @@
FIELD_PREP(IRDMAQPC_WRRDRSPOK, iw->wr_rdresp_en) |
FIELD_PREP(IRDMAQPC_RDOK, iw->rd_en) |
FIELD_PREP(IRDMAQPC_SNDMARKERS, iw->snd_mark_en) |
- FIELD_PREP(IRDMAQPC_BINDEN, iw->bind_en) |
FIELD_PREP(IRDMAQPC_FASTREGEN, iw->fast_reg_en) |
FIELD_PREP(IRDMAQPC_PRIVEN, iw->priv_mode_en) |
FIELD_PREP(IRDMAQPC_USESTATSINSTANCE, info->stats_idx_valid) |
@@ -1466,6 +1465,15 @@
set_64bit_val(wqe, IRDMA_BYTE_24, hdr);
irdma_debug_buf(qp->dev, IRDMA_DEBUG_QP, "CONN EST WQE", wqe,
IRDMA_QP_WQE_MIN_SIZE);
+ if (qp->qp_uk.start_wqe_idx) {
+ wqe = qp_uk->sq_base[3].elem;
+ hdr = FIELD_PREP(IRDMAQPSQ_OPCODE, IRDMAQP_OP_NOP) |
+ FIELD_PREP(IRDMAQPSQ_LOCALFENCE, 1) |
+ FIELD_PREP(IRDMAQPSQ_VALID, qp->qp_uk.swqe_polarity);
+ irdma_wmb(); /* make sure WQE is written before valid bit is set */
+
+ set_64bit_val(wqe, IRDMA_BYTE_24, hdr);
+ }
}
/**
@@ -1831,7 +1839,7 @@
if (copy_len)
irdma_memcpy(termhdr + 1, pkt, copy_len);
- return sizeof(struct irdma_terminate_hdr) + copy_len;
+ return sizeof(*termhdr) + copy_len;
}
/**
@@ -2124,10 +2132,6 @@
IRDMA_CQP_WQE_SIZE * 8);
irdma_sc_cqp_post_sq(cqp);
- irdma_debug(cqp->dev, IRDMA_DEBUG_STATS,
- "CQP SQ head 0x%x tail 0x%x size 0x%x\n", cqp->sq_ring.head,
- cqp->sq_ring.tail, cqp->sq_ring.size);
-
return 0;
}
@@ -2607,17 +2611,18 @@
set_64bit_val(wqe, IRDMA_BYTE_0, cq->cq_uk.cq_size);
set_64bit_val(wqe, IRDMA_BYTE_8, RS_64_1(cq, 1));
set_64bit_val(wqe, IRDMA_BYTE_16,
- FIELD_PREP(IRDMA_CQPSQ_CQ_SHADOW_READ_THRESHOLD, cq->shadow_read_threshold));
- set_64bit_val(wqe, IRDMA_BYTE_32, (cq->virtual_map ? 0 : cq->cq_pa));
+ FIELD_PREP(IRDMA_CQPSQ_CQ_SHADOW_READ_THRESHOLD,
+ cq->shadow_read_threshold));
+ set_64bit_val(wqe, IRDMA_BYTE_32, cq->virtual_map ? 0 : cq->cq_pa);
set_64bit_val(wqe, IRDMA_BYTE_40, cq->shadow_area_pa);
set_64bit_val(wqe, IRDMA_BYTE_48,
- FIELD_PREP(IRDMA_CQPSQ_CQ_FIRSTPMPBLIDX, (cq->virtual_map ? cq->first_pm_pbl_idx : 0)));
+ FIELD_PREP(IRDMA_CQPSQ_CQ_FIRSTPMPBLIDX,
+ cq->virtual_map ? cq->first_pm_pbl_idx : 0));
set_64bit_val(wqe, IRDMA_BYTE_56,
FIELD_PREP(IRDMA_CQPSQ_TPHVAL, cq->tph_val) |
FIELD_PREP(IRDMA_CQPSQ_VSIIDX, cq->vsi->vsi_idx));
-
hdr = FLD_LS_64(cq->dev, cq->cq_uk.cq_id, IRDMA_CQPSQ_CQ_CQID) |
- FLD_LS_64(cq->dev, (cq->ceq_id_valid ? cq->ceq_id : 0),
+ FLD_LS_64(cq->dev, cq->ceq_id_valid ? cq->ceq_id : 0,
IRDMA_CQPSQ_CQ_CEQID) |
FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_CREATE_CQ) |
FIELD_PREP(IRDMA_CQPSQ_CQ_LPBLSIZE, cq->pbl_chunk_size) |
@@ -2935,10 +2940,12 @@
IRDMA_HMC_IW_HDR);
irdma_sc_decode_fpm_commit(dev, buf, IRDMA_BYTE_152, info,
IRDMA_HMC_IW_MD);
- irdma_sc_decode_fpm_commit(dev, buf, IRDMA_BYTE_160, info,
- IRDMA_HMC_IW_OOISC);
- irdma_sc_decode_fpm_commit(dev, buf, IRDMA_BYTE_168, info,
- IRDMA_HMC_IW_OOISCFFL);
+ if (dev->cqp->protocol_used == IRDMA_IWARP_PROTOCOL_ONLY) {
+ irdma_sc_decode_fpm_commit(dev, buf, IRDMA_BYTE_160, info,
+ IRDMA_HMC_IW_OOISC);
+ irdma_sc_decode_fpm_commit(dev, buf, IRDMA_BYTE_168, info,
+ IRDMA_HMC_IW_OOISCFFL);
+ }
}
/* searching for the last object in HMC to find the size of the HMC area. */
@@ -3071,15 +3078,18 @@
irdma_sc_decode_fpm_query(buf, 144, obj_info, IRDMA_HMC_IW_HDR);
irdma_sc_decode_fpm_query(buf, 152, obj_info, IRDMA_HMC_IW_MD);
- irdma_sc_decode_fpm_query(buf, 160, obj_info, IRDMA_HMC_IW_OOISC);
-
- get_64bit_val(buf, IRDMA_BYTE_168, &temp);
- obj_info[IRDMA_HMC_IW_OOISCFFL].max_cnt = (u32)temp;
- obj_info[IRDMA_HMC_IW_OOISCFFL].size = 4;
- hmc_fpm_misc->ooiscf_block_size = FIELD_GET(IRDMA_QUERY_FPM_OOISCFBLOCKSIZE, temp);
- if (!hmc_fpm_misc->ooiscf_block_size &&
- obj_info[IRDMA_HMC_IW_OOISCFFL].max_cnt)
- return -EINVAL;
+
+ if (dev->cqp->protocol_used == IRDMA_IWARP_PROTOCOL_ONLY) {
+ irdma_sc_decode_fpm_query(buf, 160, obj_info, IRDMA_HMC_IW_OOISC);
+
+ get_64bit_val(buf, IRDMA_BYTE_168, &temp);
+ obj_info[IRDMA_HMC_IW_OOISCFFL].max_cnt = (u32)temp;
+ obj_info[IRDMA_HMC_IW_OOISCFFL].size = 4;
+ hmc_fpm_misc->ooiscf_block_size = FIELD_GET(IRDMA_QUERY_FPM_OOISCFBLOCKSIZE, temp);
+ if (!hmc_fpm_misc->ooiscf_block_size &&
+ obj_info[IRDMA_HMC_IW_OOISCFFL].max_cnt)
+ return -EINVAL;
+ }
return 0;
}
@@ -3251,6 +3261,7 @@
temp = FIELD_PREP(IRDMA_CQPHC_ENABLED_VFS, cqp->ena_vf_count) |
FIELD_PREP(IRDMA_CQPHC_HMC_PROFILE, cqp->hmc_profile);
+
if (hw_rev >= IRDMA_GEN_2)
temp |= FIELD_PREP(IRDMA_CQPHC_EN_REM_ENDPOINT_TRK,
cqp->en_rem_endpoint_trk);
@@ -3399,11 +3410,13 @@
void
irdma_sc_ccq_arm(struct irdma_sc_cq *ccq)
{
+ unsigned long flags;
u64 temp_val;
u16 sw_cq_sel;
u8 arm_next_se;
u8 arm_seq_num;
+ spin_lock_irqsave(&ccq->dev->cqp_lock, flags);
get_64bit_val(ccq->cq_uk.shadow_area, IRDMA_BYTE_32, &temp_val);
sw_cq_sel = (u16)FIELD_GET(IRDMA_CQ_DBSA_SW_CQ_SELECT, temp_val);
arm_next_se = (u8)FIELD_GET(IRDMA_CQ_DBSA_ARM_NEXT_SE, temp_val);
@@ -3414,6 +3427,7 @@
FIELD_PREP(IRDMA_CQ_DBSA_ARM_NEXT_SE, arm_next_se) |
FIELD_PREP(IRDMA_CQ_DBSA_ARM_NEXT, 1);
set_64bit_val(ccq->cq_uk.shadow_area, IRDMA_BYTE_32, temp_val);
+ spin_unlock_irqrestore(&ccq->dev->cqp_lock, flags);
irdma_wmb(); /* make sure shadow area is updated before arming */
@@ -3436,6 +3450,7 @@
u32 error;
u8 polarity;
int ret_code = 0;
+ unsigned long flags;
if (ccq->cq_uk.avoid_mem_cflct)
cqe = IRDMA_GET_CURRENT_EXTENDED_CQ_ELEM(&ccq->cq_uk);
@@ -3484,7 +3499,9 @@
irdma_wmb(); /* make sure shadow area is updated before moving tail */
+ spin_lock_irqsave(&cqp->dev->cqp_lock, flags);
IRDMA_RING_MOVE_TAIL(cqp->sq_ring);
+ spin_unlock_irqrestore(&cqp->dev->cqp_lock, flags);
atomic64_inc(&cqp->completed_ops);
return ret_code;
@@ -3809,7 +3826,6 @@
if (ceq->reg_cq)
irdma_sc_remove_cq_ctx(ceq, ceq->dev->ccq);
-
cqp = ceq->dev->cqp;
cqp->process_cqp_sds = irdma_update_sds_noccq;
@@ -3834,7 +3850,6 @@
if (ret_code)
return ret_code;
}
-
ret_code = irdma_sc_ceq_create(ceq, scratch, true);
if (!ret_code)
return irdma_sc_cceq_create_done(ceq);
@@ -3924,7 +3939,6 @@
cq_idx = irdma_sc_find_reg_cq(ceq, cq);
spin_unlock_irqrestore(&ceq->req_cq_lock, flags);
}
-
IRDMA_RING_MOVE_TAIL(ceq->ceq_ring);
if (!IRDMA_RING_CURRENT_TAIL(ceq->ceq_ring))
ceq->polarity ^= 1;
@@ -4155,6 +4169,7 @@
case IRDMA_AE_ROE_INVALID_RDMA_READ_REQUEST:
case IRDMA_AE_ROE_INVALID_RDMA_WRITE_OR_READ_RESP:
case IRDMA_AE_ROCE_RSP_LENGTH_ERROR:
+ case IRDMA_AE_ROCE_REQ_LENGTH_ERROR:
case IRDMA_AE_INVALID_ARP_ENTRY:
case IRDMA_AE_INVALID_TCP_OPTION_RCVD:
case IRDMA_AE_STALE_ARP_ENTRY:
@@ -4707,10 +4722,11 @@
u64 sd;
int i;
- for (i = IRDMA_HMC_IW_QP; i < IRDMA_HMC_IW_MAX; i++)
+ for (i = IRDMA_HMC_IW_QP; i < IRDMA_HMC_IW_MAX; i++) {
if (i != IRDMA_HMC_IW_PBLE)
size += round_up(hmc_info->hmc_obj[i].cnt *
hmc_info->hmc_obj[i].size, 512);
+ }
pble_info = &hmc_info->hmc_obj[IRDMA_HMC_IW_PBLE];
size += round_up(pble_info->cnt * pble_info->size, 512);
@@ -4720,7 +4736,7 @@
sd = size >> 21;
if (sd > 0xFFFFFFFF) {
irdma_debug(dev, IRDMA_DEBUG_HMC, "sd overflow[%ld]\n", sd);
- sd = 0xFFFFFFFF - 1;
+ sd = 0xFFFFFFFE;
}
return (u32)sd;
@@ -4776,10 +4792,9 @@
int
irdma_get_rdma_features(struct irdma_sc_dev *dev)
{
- int ret_code;
+ int ret_code, byte_idx, feat_type, feat_cnt, feat_idx;
struct irdma_dma_mem feat_buf;
u64 temp;
- u16 byte_idx, feat_type, feat_cnt, feat_idx;
feat_buf.size = IRDMA_FEATURE_BUF_SIZE;
feat_buf.va = irdma_allocate_dma_mem(dev->hw, &feat_buf, feat_buf.size,
@@ -4822,7 +4837,7 @@
irdma_debug_buf(dev, IRDMA_DEBUG_WQE, "QUERY RDMA FEATURES", feat_buf.va,
feat_cnt * 8);
- for (byte_idx = 0, feat_idx = 0; feat_idx < min(feat_cnt, (u16)IRDMA_MAX_FEATURES);
+ for (byte_idx = 0, feat_idx = 0; feat_idx < min(feat_cnt, IRDMA_MAX_FEATURES);
feat_idx++, byte_idx += 8) {
get_64bit_val(feat_buf.va, byte_idx, &temp);
feat_type = FIELD_GET(IRDMA_FEATURE_TYPE, temp);
@@ -5409,6 +5424,7 @@
irdma_cfg_aeq(struct irdma_sc_dev *dev, u32 idx, bool enable)
{
u32 reg_val;
+
reg_val = FIELD_PREP(IRDMA_PFINT_AEQCTL_CAUSE_ENA, enable) |
FIELD_PREP(IRDMA_PFINT_AEQCTL_MSIX_INDX, idx) |
FIELD_PREP(IRDMA_PFINT_AEQCTL_ITR_INDX, IRDMA_IDX_NOITR);
@@ -5423,14 +5439,16 @@
void
sc_vsi_update_stats(struct irdma_sc_vsi *vsi)
{
- struct irdma_gather_stats *gather_stats;
- struct irdma_gather_stats *last_gather_stats;
-
- gather_stats = vsi->pestat->gather_info.gather_stats_va;
- last_gather_stats = vsi->pestat->gather_info.last_gather_stats_va;
- irdma_update_stats(&vsi->pestat->hw_stats, gather_stats,
- last_gather_stats, vsi->dev->hw_stats_map,
- vsi->dev->hw_attrs.max_stat_idx);
+ struct irdma_dev_hw_stats *hw_stats = &vsi->pestat->hw_stats;
+ struct irdma_gather_stats *gather_stats =
+ vsi->pestat->gather_info.gather_stats_va;
+ struct irdma_gather_stats *last_gather_stats =
+ vsi->pestat->gather_info.last_gather_stats_va;
+ const struct irdma_hw_stat_map *map = vsi->dev->hw_stats_map;
+ u16 max_stat_idx = vsi->dev->hw_attrs.max_stat_idx;
+
+ irdma_update_stats(hw_stats, gather_stats, last_gather_stats,
+ map, max_stat_idx);
}
/**
@@ -5500,7 +5518,6 @@
dev->hw_attrs.max_hw_outbound_msg_size = IRDMA_MAX_OUTBOUND_MSG_SIZE;
dev->hw_attrs.max_mr_size = IRDMA_MAX_MR_SIZE;
dev->hw_attrs.max_hw_inbound_msg_size = IRDMA_MAX_INBOUND_MSG_SIZE;
- dev->hw_attrs.max_hw_device_pages = IRDMA_MAX_PUSH_PAGE_COUNT;
dev->hw_attrs.uk_attrs.max_hw_inline = IRDMA_MAX_INLINE_DATA_SIZE;
dev->hw_attrs.max_hw_wqes = IRDMA_MAX_WQ_ENTRIES;
dev->hw_attrs.max_qp_wr = IRDMA_MAX_QP_WRS(IRDMA_MAX_QUANTA_PER_WR);
@@ -5528,7 +5545,6 @@
val, db_size);
return -ENODEV;
}
- dev->db_addr = dev->hw->hw_addr + (uintptr_t)dev->hw_regs[IRDMA_DB_ADDR_OFFSET];
return ret_code;
}
diff --git a/sys/dev/irdma/irdma_defs.h b/sys/dev/irdma/irdma_defs.h
--- a/sys/dev/irdma/irdma_defs.h
+++ b/sys/dev/irdma/irdma_defs.h
@@ -249,7 +249,6 @@
IRDMA_OP_MANAGE_HMC_PM_FUNC_TABLE = 22,
IRDMA_OP_SUSPEND = 23,
IRDMA_OP_RESUME = 24,
- IRDMA_OP_MANAGE_VCHNL_REQ_PBLE_BP = 25,
IRDMA_OP_QUERY_FPM_VAL = 26,
IRDMA_OP_COMMIT_FPM_VAL = 27,
IRDMA_OP_AH_CREATE = 28,
@@ -292,7 +291,6 @@
#define IRDMA_CQP_OP_DEALLOC_STAG 0x0d
#define IRDMA_CQP_OP_MANAGE_LOC_MAC_TABLE 0x0e
#define IRDMA_CQP_OP_MANAGE_ARP 0x0f
-#define IRDMA_CQP_OP_MANAGE_VCHNL_REQ_PBLE_BP 0x10
#define IRDMA_CQP_OP_MANAGE_PUSH_PAGES 0x11
#define IRDMA_CQP_OP_QUERY_RDMA_FEATURES 0x12
#define IRDMA_CQP_OP_UPLOAD_CONTEXT 0x13
@@ -849,7 +847,6 @@
#define IRDMA_CQPSQ_UCTX_RAWFORMAT BIT_ULL(61)
#define IRDMA_CQPSQ_UCTX_FREEZEQP_S 62
#define IRDMA_CQPSQ_UCTX_FREEZEQP BIT_ULL(62)
-
#define IRDMA_CQPSQ_MHMC_VFIDX_S 0
#define IRDMA_CQPSQ_MHMC_VFIDX GENMASK_ULL(15, 0)
#define IRDMA_CQPSQ_MHMC_FREEPMFN_S 62
diff --git a/sys/dev/irdma/irdma_hw.c b/sys/dev/irdma/irdma_hw.c
--- a/sys/dev/irdma/irdma_hw.c
+++ b/sys/dev/irdma/irdma_hw.c
@@ -276,6 +276,7 @@
switch (info->ae_id) {
struct irdma_cm_node *cm_node;
+
case IRDMA_AE_LLP_CONNECTION_ESTABLISHED:
cm_node = iwqp->cm_node;
if (cm_node->accept_pend) {
@@ -393,6 +394,7 @@
case IRDMA_AE_LLP_TOO_MANY_RETRIES:
case IRDMA_AE_LCE_QP_CATASTROPHIC:
case IRDMA_AE_LCE_FUNCTION_CATASTROPHIC:
+ case IRDMA_AE_LLP_TOO_MANY_RNRS:
case IRDMA_AE_UDA_XMIT_DGRAM_TOO_LONG:
default:
irdma_dev_err(&iwdev->ibdev,
@@ -489,7 +491,7 @@
{
struct irdma_qvlist_info *iw_qvlist;
struct irdma_qv_info *iw_qvinfo;
- u32 ceq_idx;
+ u16 ceq_idx;
u32 i;
u32 size;
@@ -499,8 +501,8 @@
}
size = sizeof(struct irdma_msix_vector) * rf->msix_count;
- size += sizeof(struct irdma_qvlist_info);
- size += sizeof(struct irdma_qv_info) * rf->msix_count - 1;
+ size += sizeof(*iw_qvlist);
+ size += sizeof(*iw_qvinfo) * rf->msix_count - 1;
rf->iw_msixtbl = kzalloc(size, GFP_KERNEL);
if (!rf->iw_msixtbl)
return -ENOMEM;
@@ -599,6 +601,13 @@
dev->irq_ops->irdma_dis_irq(dev, msix_vec->idx);
irdma_free_irq(rf, msix_vec);
+ if (rf == dev_id) {
+ tasklet_kill(&rf->dpc_tasklet);
+ } else {
+ struct irdma_ceq *iwceq = (struct irdma_ceq *)dev_id;
+
+ tasklet_kill(&iwceq->dpc_tasklet);
+ }
}
/**
@@ -963,13 +972,13 @@
u16 maj_err, min_err;
int i, status;
- cqp->cqp_requests = kcalloc(sqsize, sizeof(*cqp->cqp_requests), GFP_KERNEL);
- memset(cqp->cqp_requests, 0, sqsize * sizeof(*cqp->cqp_requests));
+ cqp->cqp_requests = kcalloc(sqsize, sizeof(*cqp->cqp_requests),
+ GFP_KERNEL);
if (!cqp->cqp_requests)
return -ENOMEM;
- cqp->scratch_array = kcalloc(sqsize, sizeof(*cqp->scratch_array), GFP_KERNEL);
- memset(cqp->scratch_array, 0, sqsize * sizeof(*cqp->scratch_array));
+ cqp->scratch_array = kcalloc(sqsize, sizeof(*cqp->scratch_array),
+ GFP_KERNEL);
if (!cqp->scratch_array) {
status = -ENOMEM;
goto err_scratch;
@@ -1189,7 +1198,7 @@
*/
static int
irdma_cfg_ceq_vector(struct irdma_pci_f *rf, struct irdma_ceq *iwceq,
- u32 ceq_id, struct irdma_msix_vector *msix_vec)
+ u16 ceq_id, struct irdma_msix_vector *msix_vec)
{
int status;
@@ -1263,7 +1272,7 @@
*/
static int
irdma_create_ceq(struct irdma_pci_f *rf, struct irdma_ceq *iwceq,
- u32 ceq_id, struct irdma_sc_vsi *vsi)
+ u16 ceq_id, struct irdma_sc_vsi *vsi)
{
int status;
struct irdma_ceq_init_info info = {0};
@@ -1380,7 +1389,7 @@
irdma_setup_ceqs(struct irdma_pci_f *rf, struct irdma_sc_vsi *vsi)
{
u32 i;
- u32 ceq_id;
+ u16 ceq_id;
struct irdma_ceq *iwceq;
struct irdma_msix_vector *msix_vec;
int status;
@@ -1750,6 +1759,7 @@
irdma_rt_deinit_hw(struct irdma_device *iwdev)
{
struct irdma_sc_qp qp = {{0}};
+
irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_INIT, "state = %d\n", iwdev->init_state);
switch (iwdev->init_state) {
@@ -2011,6 +2021,7 @@
{
struct irdma_sc_dev *dev = &rf->sc_dev;
int status;
+
do {
status = irdma_setup_init_state(rf);
if (status)
@@ -2209,14 +2220,19 @@
cqp_request = (struct irdma_cqp_request *)
(uintptr_t)info.scratch;
- if (info.error && irdma_cqp_crit_err(dev, cqp_request->info.cqp_cmd,
+ if (info.error && irdma_cqp_crit_err(dev,
+ cqp_request->info.cqp_cmd,
info.maj_err_code,
info.min_err_code))
- irdma_dev_err(&rf->iwdev->ibdev, "cqp opcode = 0x%x maj_err_code = 0x%x min_err_code = 0x%x\n",
- info.op_code, info.maj_err_code, info.min_err_code);
+ irdma_dev_err(&rf->iwdev->ibdev,
+ "cqp opcode = 0x%x maj_err_code = 0x%x min_err_code = 0x%x\n",
+ info.op_code, info.maj_err_code,
+ info.min_err_code);
if (cqp_request) {
- cqp_request->compl_info.maj_err_code = info.maj_err_code;
- cqp_request->compl_info.min_err_code = info.min_err_code;
+ cqp_request->compl_info.maj_err_code =
+ info.maj_err_code;
+ cqp_request->compl_info.min_err_code =
+ info.min_err_code;
cqp_request->compl_info.op_ret_val = info.op_ret_val;
cqp_request->compl_info.error = info.error;
irdma_complete_cqp_request(&rf->cqp, cqp_request);
diff --git a/sys/dev/irdma/irdma_kcompat.c b/sys/dev/irdma/irdma_kcompat.c
--- a/sys/dev/irdma/irdma_kcompat.c
+++ b/sys/dev/irdma/irdma_kcompat.c
@@ -96,7 +96,6 @@
return 0;
}
-#if __FreeBSD_version >= 1400026
/**
* irdma_alloc_mr - register stag for fast memory registration
* @pd: ibpd pointer
@@ -108,18 +107,6 @@
irdma_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type,
u32 max_num_sg, struct ib_udata *udata)
{
-#else
-/**
- * irdma_alloc_mr - register stag for fast memory registration
- * @pd: ibpd pointer
- * @mr_type: memory for stag registrion
- * @max_num_sg: man number of pages
- */
-struct ib_mr *
-irdma_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type,
- u32 max_num_sg)
-{
-#endif
struct irdma_device *iwdev = to_iwdev(pd->device);
struct irdma_pble_alloc *palloc;
struct irdma_pbl *iwpbl;
@@ -174,7 +161,6 @@
#define IRDMA_ALLOC_UCTX_MIN_REQ_LEN offsetofend(struct irdma_alloc_ucontext_req, rsvd8)
#define IRDMA_ALLOC_UCTX_MIN_RESP_LEN offsetofend(struct irdma_alloc_ucontext_resp, rsvd)
-#if __FreeBSD_version >= 1400026
/**
* irdma_alloc_ucontext - Allocate the user context data structure
* @uctx: context
@@ -236,6 +222,8 @@
uresp.min_hw_cq_size = uk_attrs->min_hw_cq_size;
uresp.hw_rev = uk_attrs->hw_rev;
uresp.comp_mask |= IRDMA_ALLOC_UCTX_USE_RAW_ATTR;
+ uresp.min_hw_wq_size = uk_attrs->min_hw_wq_size;
+ uresp.comp_mask |= IRDMA_ALLOC_UCTX_MIN_HW_WQ_SIZE;
bar_off =
(uintptr_t)iwdev->rf->sc_dev.hw_regs[IRDMA_DB_ADDR_OFFSET];
@@ -269,119 +257,8 @@
req.userspace_ver, IRDMA_ABI_VER);
return -EINVAL;
}
-#endif
-
-#if __FreeBSD_version < 1400026
-/**
- * irdma_alloc_ucontext - Allocate the user context data structure
- * @ibdev: ib device pointer
- * @udata: user data
- *
- * This keeps track of all objects associated with a particular
- * user-mode client.
- */
-struct ib_ucontext *
-irdma_alloc_ucontext(struct ib_device *ibdev, struct ib_udata *udata)
-{
- struct irdma_device *iwdev = to_iwdev(ibdev);
- struct irdma_alloc_ucontext_req req = {0};
- struct irdma_alloc_ucontext_resp uresp = {0};
- struct irdma_ucontext *ucontext;
- struct irdma_uk_attrs *uk_attrs = &iwdev->rf->sc_dev.hw_attrs.uk_attrs;
-
- if (udata->inlen < IRDMA_ALLOC_UCTX_MIN_REQ_LEN ||
- udata->outlen < IRDMA_ALLOC_UCTX_MIN_RESP_LEN)
- return ERR_PTR(-EINVAL);
-
- if (ib_copy_from_udata(&req, udata, min(sizeof(req), udata->inlen)))
- return ERR_PTR(-EINVAL);
-
- if (req.userspace_ver < 4 || req.userspace_ver > IRDMA_ABI_VER)
- goto ver_error;
-
- ucontext = kzalloc(sizeof(*ucontext), GFP_KERNEL);
- if (!ucontext)
- return ERR_PTR(-ENOMEM);
-
- ucontext->iwdev = iwdev;
- ucontext->abi_ver = req.userspace_ver;
-
- if (req.comp_mask & IRDMA_ALLOC_UCTX_USE_RAW_ATTR)
- ucontext->use_raw_attrs = true;
-
- /* GEN_1 legacy support with libi40iw */
- if (udata->outlen == IRDMA_ALLOC_UCTX_MIN_RESP_LEN) {
- if (uk_attrs->hw_rev != IRDMA_GEN_1) {
- kfree(ucontext);
- return ERR_PTR(-EOPNOTSUPP);
- }
-
- ucontext->legacy_mode = true;
- uresp.max_qps = iwdev->rf->max_qp;
- uresp.max_pds = iwdev->rf->sc_dev.hw_attrs.max_hw_pds;
- uresp.wq_size = iwdev->rf->sc_dev.hw_attrs.max_qp_wr * 2;
- uresp.kernel_ver = req.userspace_ver;
- if (ib_copy_to_udata(udata, &uresp, min(sizeof(uresp), udata->outlen))) {
- kfree(ucontext);
- return ERR_PTR(-EFAULT);
- }
- } else {
- u64 bar_off;
-
- uresp.kernel_ver = IRDMA_ABI_VER;
- uresp.feature_flags = uk_attrs->feature_flags;
- uresp.max_hw_wq_frags = uk_attrs->max_hw_wq_frags;
- uresp.max_hw_read_sges = uk_attrs->max_hw_read_sges;
- uresp.max_hw_inline = uk_attrs->max_hw_inline;
- uresp.max_hw_rq_quanta = uk_attrs->max_hw_rq_quanta;
- uresp.max_hw_wq_quanta = uk_attrs->max_hw_wq_quanta;
- uresp.max_hw_sq_chunk = uk_attrs->max_hw_sq_chunk;
- uresp.max_hw_cq_size = uk_attrs->max_hw_cq_size;
- uresp.min_hw_cq_size = uk_attrs->min_hw_cq_size;
- uresp.hw_rev = uk_attrs->hw_rev;
- uresp.comp_mask |= IRDMA_ALLOC_UCTX_USE_RAW_ATTR;
-
- bar_off =
- (uintptr_t)iwdev->rf->sc_dev.hw_regs[IRDMA_DB_ADDR_OFFSET];
- spin_lock_init(&ucontext->mmap_tbl_lock);
- ucontext->db_mmap_entry =
- irdma_user_mmap_entry_add_hash(ucontext, bar_off,
- IRDMA_MMAP_IO_NC,
- &uresp.db_mmap_key);
- if (!ucontext->db_mmap_entry) {
- spin_lock_destroy(&ucontext->mmap_tbl_lock);
- kfree(ucontext);
- return ERR_PTR(-ENOMEM);
- }
- if (ib_copy_to_udata(udata, &uresp,
- min(sizeof(uresp), udata->outlen))) {
- irdma_user_mmap_entry_del_hash(ucontext->db_mmap_entry);
- spin_lock_destroy(&ucontext->mmap_tbl_lock);
- kfree(ucontext);
- return ERR_PTR(-EFAULT);
- }
- }
-
- INIT_LIST_HEAD(&ucontext->cq_reg_mem_list);
- spin_lock_init(&ucontext->cq_reg_mem_list_lock);
- INIT_LIST_HEAD(&ucontext->qp_reg_mem_list);
- spin_lock_init(&ucontext->qp_reg_mem_list_lock);
- INIT_LIST_HEAD(&ucontext->vma_list);
- mutex_init(&ucontext->vma_list_mutex);
-
- return &ucontext->ibucontext;
-
-ver_error:
- irdma_dev_err(&iwdev->ibdev,
- "Invalid userspace driver version detected. Detected version %d, should be %d\n",
- req.userspace_ver, IRDMA_ABI_VER);
- return ERR_PTR(-EINVAL);
-}
-#endif
-
-#if __FreeBSD_version >= 1400026
/**
* irdma_dealloc_ucontext - deallocate the user context data structure
* @context: user context created during alloc
@@ -395,28 +272,9 @@
return;
}
-#endif
-#if __FreeBSD_version < 1400026
-/**
- * irdma_dealloc_ucontext - deallocate the user context data structure
- * @context: user context created during alloc
- */
-int
-irdma_dealloc_ucontext(struct ib_ucontext *context)
-{
- struct irdma_ucontext *ucontext = to_ucontext(context);
-
- irdma_user_mmap_entry_del_hash(ucontext->db_mmap_entry);
- spin_lock_destroy(&ucontext->mmap_tbl_lock);
- kfree(ucontext);
-
- return 0;
-}
-#endif
#define IRDMA_ALLOC_PD_MIN_RESP_LEN offsetofend(struct irdma_alloc_pd_resp, rsvd)
-#if __FreeBSD_version >= 1400026
/**
* irdma_alloc_pd - allocate protection domain
* @pd: protection domain
@@ -469,70 +327,8 @@
return err;
}
-#endif
-#if __FreeBSD_version < 1400026
-/**
- * irdma_alloc_pd - allocate protection domain
- * @ibdev: IB device
- * @context: user context
- * @udata: user data
- */
-struct ib_pd *
-irdma_alloc_pd(struct ib_device *ibdev, struct ib_ucontext *context, struct ib_udata *udata)
-{
- struct irdma_pd *iwpd;
- struct irdma_device *iwdev = to_iwdev(ibdev);
- struct irdma_sc_dev *dev = &iwdev->rf->sc_dev;
- struct irdma_pci_f *rf = iwdev->rf;
- struct irdma_alloc_pd_resp uresp = {0};
- struct irdma_sc_pd *sc_pd;
- u32 pd_id = 0;
- int err;
-
- err = irdma_alloc_rsrc(rf, rf->allocated_pds, rf->max_pd, &pd_id,
- &rf->next_pd);
- if (err)
- return ERR_PTR(err);
- iwpd = kzalloc(sizeof(*iwpd), GFP_KERNEL);
- if (!iwpd) {
- err = -ENOMEM;
- goto free_res;
- }
-
- sc_pd = &iwpd->sc_pd;
- if (udata) {
- struct irdma_ucontext *ucontext = to_ucontext(context);
-
- irdma_sc_pd_init(dev, sc_pd, pd_id, ucontext->abi_ver);
- uresp.pd_id = pd_id;
- if (ib_copy_to_udata(udata, &uresp,
- min(sizeof(uresp), udata->outlen))) {
- err = -EFAULT;
- goto error;
- }
- } else {
- irdma_sc_pd_init(dev, sc_pd, pd_id, IRDMA_ABI_VER);
- }
-
- spin_lock_init(&iwpd->udqp_list_lock);
- INIT_LIST_HEAD(&iwpd->udqp_list);
-
- return &iwpd->ibpd;
-
-error:
- kfree(iwpd);
-free_res:
-
- irdma_free_rsrc(rf, rf->allocated_pds, pd_id);
-
- return ERR_PTR(err);
-}
-
-#endif
-
-#if __FreeBSD_version >= 1400026
void
irdma_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
{
@@ -542,20 +338,7 @@
irdma_free_rsrc(iwdev->rf, iwdev->rf->allocated_pds, iwpd->sc_pd.pd_id);
}
-#endif
-
-#if __FreeBSD_version < 1400026
-int
-irdma_dealloc_pd(struct ib_pd *ibpd)
-{
- struct irdma_pd *iwpd = to_iwpd(ibpd);
- struct irdma_device *iwdev = to_iwdev(ibpd->device);
- irdma_free_rsrc(iwdev->rf, iwdev->rf->allocated_pds, iwpd->sc_pd.pd_id);
- kfree(iwpd);
- return 0;
-}
-#endif
/**
* irdma_find_qp_update_qs - update QS handle for UD QPs
@@ -684,24 +467,40 @@
irdma_create_ah_wait(struct irdma_pci_f *rf,
struct irdma_sc_ah *sc_ah, bool sleep)
{
+ int ret;
+
if (!sleep) {
int cnt = rf->sc_dev.hw_attrs.max_cqp_compl_wait_time_ms *
CQP_TIMEOUT_THRESHOLD;
+ struct irdma_cqp_request *cqp_request =
+ sc_ah->ah_info.cqp_request;
do {
irdma_cqp_ce_handler(rf, &rf->ccq.sc_cq);
mdelay(1);
- } while (!sc_ah->ah_info.ah_valid && --cnt);
+ } while (!READ_ONCE(cqp_request->request_done) && --cnt);
- if (!cnt)
- return -ETIMEDOUT;
+ if (cnt && !cqp_request->compl_info.op_ret_val) {
+ irdma_put_cqp_request(&rf->cqp, cqp_request);
+ sc_ah->ah_info.ah_valid = true;
+ } else {
+ ret = !cnt ? -ETIMEDOUT : -EINVAL;
+ irdma_dev_err(&rf->iwdev->ibdev, "CQP create AH error ret = %d opt_ret_val = %d",
+ ret, cqp_request->compl_info.op_ret_val);
+ irdma_put_cqp_request(&rf->cqp, cqp_request);
+ if (!cnt && !rf->reset) {
+ rf->reset = true;
+ rf->gen_ops.request_reset(rf);
+ }
+ return ret;
+ }
}
+
return 0;
}
#define IRDMA_CREATE_AH_MIN_RESP_LEN offsetofend(struct irdma_create_ah_resp, rsvd)
-#if __FreeBSD_version >= 1400026
/**
* irdma_create_ah - create address handle
* @ib_ah: ptr to AH
@@ -787,17 +586,15 @@
goto err_gid_l2;
err = irdma_ah_cqp_op(iwdev->rf, sc_ah, IRDMA_OP_AH_CREATE,
- sleep, irdma_gsi_ud_qp_ah_cb, sc_ah);
+ sleep, NULL, sc_ah);
if (err) {
irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_DEV, "CQP-OP Create AH fail");
goto err_gid_l2;
}
err = irdma_create_ah_wait(rf, sc_ah, sleep);
- if (err) {
- irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_DEV, "CQP create AH timed out");
+ if (err)
goto err_gid_l2;
- }
if (udata) {
uresp.ah_id = ah->sc_ah.ah_info.ah_idx;
@@ -815,7 +612,6 @@
return err;
}
-#endif
void
irdma_ether_copy(u8 *dmac, struct ib_ah_attr *attr)
@@ -823,162 +619,20 @@
ether_addr_copy(dmac, attr->dmac);
}
-#if __FreeBSD_version < 1400026
-struct ib_ah *
-irdma_create_ah_stub(struct ib_pd *ibpd,
- struct ib_ah_attr *attr,
- struct ib_udata *udata)
-#else
int
irdma_create_ah_stub(struct ib_ah *ib_ah,
struct ib_ah_attr *attr, u32 flags,
struct ib_udata *udata)
-#endif
{
-#if __FreeBSD_version >= 1400026
return -ENOSYS;
-#else
- return ERR_PTR(-ENOSYS);
-#endif
}
-#if __FreeBSD_version >= 1400026
void
irdma_destroy_ah_stub(struct ib_ah *ibah, u32 flags)
{
return;
}
-#else
-int
-irdma_destroy_ah_stub(struct ib_ah *ibah)
-{
- return -ENOSYS;
-}
-#endif
-#if __FreeBSD_version < 1400026
-/**
- * irdma_create_ah - create address handle
- * @ibpd: ptr to pd
- * @attr: address handle attributes
- * @udata: user data
- *
- * returns a pointer to an address handle
- */
-struct ib_ah *
-irdma_create_ah(struct ib_pd *ibpd,
- struct ib_ah_attr *attr,
- struct ib_udata *udata)
-{
- struct irdma_pd *pd = to_iwpd(ibpd);
- struct irdma_device *iwdev = to_iwdev(ibpd->device);
- struct irdma_ah *ah;
- union ib_gid sgid;
- struct ib_gid_attr sgid_attr;
- struct irdma_pci_f *rf = iwdev->rf;
- struct irdma_sc_ah *sc_ah;
- u32 ah_id = 0;
- struct irdma_ah_info *ah_info;
- struct irdma_create_ah_resp uresp = {};
- union irdma_sockaddr sgid_addr, dgid_addr;
- int err;
- u8 dmac[ETHER_ADDR_LEN];
- bool sleep = udata ? true : false;
-
- if (udata && udata->outlen < IRDMA_CREATE_AH_MIN_RESP_LEN)
- return ERR_PTR(-EINVAL);
-
- err = irdma_alloc_rsrc(rf, rf->allocated_ahs,
- rf->max_ah, &ah_id, &rf->next_ah);
-
- if (err)
- return ERR_PTR(err);
-
- ah = kzalloc(sizeof(*ah), GFP_ATOMIC);
- if (!ah) {
- irdma_free_rsrc(rf, rf->allocated_ahs, ah_id);
- return ERR_PTR(-ENOMEM);
- }
-
- ah->pd = pd;
- sc_ah = &ah->sc_ah;
- sc_ah->ah_info.ah_idx = ah_id;
- sc_ah->ah_info.vsi = &iwdev->vsi;
- irdma_sc_init_ah(&rf->sc_dev, sc_ah);
- ah->sgid_index = attr->grh.sgid_index;
- memcpy(&ah->dgid, &attr->grh.dgid, sizeof(ah->dgid));
- rcu_read_lock();
- err = ib_get_cached_gid(&iwdev->ibdev, attr->port_num,
- attr->grh.sgid_index, &sgid, &sgid_attr);
- rcu_read_unlock();
- if (err) {
- irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_VERBS,
- "GID lookup at idx=%d with port=%d failed\n",
- attr->grh.sgid_index, attr->port_num);
- err = -EINVAL;
- goto err_gid_l2;
- }
- rdma_gid2ip((struct sockaddr *)&sgid_addr, &sgid);
- rdma_gid2ip((struct sockaddr *)&dgid_addr, &attr->grh.dgid);
- ah->av.attrs = *attr;
- ah->av.net_type = ib_gid_to_network_type(sgid_attr.gid_type, &sgid);
-
- if (sgid_attr.ndev)
- dev_put(sgid_attr.ndev);
-
- ah_info = &sc_ah->ah_info;
- ah_info->ah_idx = ah_id;
- ah_info->pd_idx = pd->sc_pd.pd_id;
-
- ether_addr_copy(ah_info->mac_addr, if_getlladdr(iwdev->netdev));
- if (attr->ah_flags & IB_AH_GRH) {
- ah_info->flow_label = attr->grh.flow_label;
- ah_info->hop_ttl = attr->grh.hop_limit;
- ah_info->tc_tos = attr->grh.traffic_class;
- }
-
- if (udata)
- ib_resolve_eth_dmac(ibpd->device, attr);
- irdma_ether_copy(dmac, attr);
-
- irdma_fill_ah_info(if_getvnet(iwdev->netdev), ah_info, &sgid_attr, &sgid_addr, &dgid_addr,
- dmac, ah->av.net_type);
-
- err = irdma_create_ah_vlan_tag(iwdev, pd, ah_info, &sgid_attr, dmac);
- if (err)
- goto err_gid_l2;
-
- err = irdma_ah_cqp_op(iwdev->rf, sc_ah, IRDMA_OP_AH_CREATE,
- sleep, irdma_gsi_ud_qp_ah_cb, sc_ah);
- if (err) {
- irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_VERBS, "CQP-OP Create AH fail");
- goto err_gid_l2;
- }
-
- err = irdma_create_ah_wait(rf, sc_ah, sleep);
- if (err) {
- irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_DEV, "CQP create AH timed out");
- goto err_gid_l2;
- }
-
- if (udata) {
- uresp.ah_id = ah->sc_ah.ah_info.ah_idx;
- err = ib_copy_to_udata(udata, &uresp, min(sizeof(uresp), udata->outlen));
- if (err) {
- irdma_ah_cqp_op(iwdev->rf, &ah->sc_ah,
- IRDMA_OP_AH_DESTROY, false, NULL, ah);
- goto err_gid_l2;
- }
- }
-
- return &ah->ibah;
-err_gid_l2:
- kfree(ah);
- irdma_free_rsrc(iwdev->rf, iwdev->rf->allocated_ahs, ah_id);
-
- return ERR_PTR(err);
-}
-#endif
/**
* irdma_free_qp_rsrc - free up memory resources for qp
@@ -992,7 +646,7 @@
u32 qp_num = iwqp->ibqp.qp_num;
irdma_ieq_cleanup_qp(iwdev->vsi.ieq, &iwqp->sc_qp);
- irdma_dealloc_push_page(rf, &iwqp->sc_qp);
+ irdma_dealloc_push_page(rf, iwqp);
if (iwqp->sc_qp.vsi) {
irdma_qp_rem_qos(&iwqp->sc_qp);
iwqp->sc_qp.dev->ws_remove(iwqp->sc_qp.vsi,
@@ -1186,12 +840,17 @@
if (udata) {
/* GEN_1 legacy support with libi40iw does not have expanded uresp struct */
- if (udata->outlen < sizeof(uresp)) {
+ if (udata->outlen == IRDMA_CREATE_QP_MIN_RESP_LEN) {
uresp.lsmm = 1;
uresp.push_idx = IRDMA_INVALID_PUSH_PAGE_INDEX_GEN_1;
} else {
- if (rdma_protocol_iwarp(&iwdev->ibdev, 1))
+ if (rdma_protocol_iwarp(&iwdev->ibdev, 1)) {
uresp.lsmm = 1;
+ if (qp->qp_uk.start_wqe_idx) {
+ uresp.comp_mask |= IRDMA_CREATE_QP_USE_START_WQE_IDX;
+ uresp.start_wqe_idx = qp->qp_uk.start_wqe_idx;
+ }
+ }
}
uresp.actual_sq_size = init_info.qp_uk_init_info.sq_size;
uresp.actual_rq_size = init_info.qp_uk_init_info.rq_size;
@@ -1202,7 +861,7 @@
min(sizeof(uresp), udata->outlen));
if (err_code) {
irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_VERBS, "copy_to_udata failed\n");
- kc_irdma_destroy_qp(&iwqp->ibqp, udata);
+ irdma_destroy_qp(&iwqp->ibqp, udata);
return ERR_PTR(err_code);
}
}
@@ -1221,13 +880,8 @@
* @ibqp: qp's ib pointer also to get to device's qp address
* @udata: user data
*/
-#if __FreeBSD_version >= 1400026
int
irdma_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata)
-#else
-int
-irdma_destroy_qp(struct ib_qp *ibqp)
-#endif
{
struct irdma_qp *iwqp = to_iwqp(ibqp);
struct irdma_device *iwdev = iwqp->iwdev;
@@ -1270,31 +924,17 @@
* @attr: attributes for cq
* @udata: user data
*/
-#if __FreeBSD_version >= 1400026
int
irdma_create_cq(struct ib_cq *ibcq,
const struct ib_cq_init_attr *attr,
struct ib_udata *udata)
-#else
-struct ib_cq *
-irdma_create_cq(struct ib_device *ibdev,
- const struct ib_cq_init_attr *attr,
- struct ib_ucontext *context,
- struct ib_udata *udata)
-#endif
{
#define IRDMA_CREATE_CQ_MIN_REQ_LEN offsetofend(struct irdma_create_cq_req, user_cq_buf)
#define IRDMA_CREATE_CQ_MIN_RESP_LEN offsetofend(struct irdma_create_cq_resp, cq_size)
-#if __FreeBSD_version >= 1400026
struct ib_device *ibdev = ibcq->device;
-#endif
struct irdma_device *iwdev = to_iwdev(ibdev);
struct irdma_pci_f *rf = iwdev->rf;
-#if __FreeBSD_version >= 1400026
struct irdma_cq *iwcq = to_iwcq(ibcq);
-#else
- struct irdma_cq *iwcq;
-#endif
u32 cq_num = 0;
struct irdma_sc_cq *cq;
struct irdma_sc_dev *dev = &rf->sc_dev;
@@ -1308,7 +948,6 @@
int entries = attr->cqe;
bool cqe_64byte_ena;
-#if __FreeBSD_version >= 1400026
err_code = cq_validate_flags(attr->flags, dev->hw_attrs.uk_attrs.hw_rev);
if (err_code)
return err_code;
@@ -1316,27 +955,10 @@
if (udata && (udata->inlen < IRDMA_CREATE_CQ_MIN_REQ_LEN ||
udata->outlen < IRDMA_CREATE_CQ_MIN_RESP_LEN))
return -EINVAL;
-#else
- err_code = cq_validate_flags(attr->flags, dev->hw_attrs.uk_attrs.hw_rev);
- if (err_code)
- return ERR_PTR(err_code);
-
- if (udata && (udata->inlen < IRDMA_CREATE_CQ_MIN_REQ_LEN ||
- udata->outlen < IRDMA_CREATE_CQ_MIN_RESP_LEN))
- return ERR_PTR(-EINVAL);
-
- iwcq = kzalloc(sizeof(*iwcq), GFP_KERNEL);
- if (!iwcq)
- return ERR_PTR(-ENOMEM);
-#endif
err_code = irdma_alloc_rsrc(rf, rf->allocated_cqs, rf->max_cq, &cq_num,
&rf->next_cq);
if (err_code)
-#if __FreeBSD_version >= 1400026
return err_code;
-#else
- goto error;
-#endif
cq = &iwcq->sc_cq;
cq->back_cq = iwcq;
atomic_set(&iwcq->refcnt, 1);
@@ -1366,11 +988,7 @@
struct irdma_cq_mr *cqmr_shadow;
iwcq->user_mode = true;
-#if __FreeBSD_version >= 1400026
ucontext = rdma_udata_to_drv_context(udata, struct irdma_ucontext, ibucontext);
-#else
- ucontext = to_ucontext(context);
-#endif
if (ib_copy_from_udata(&req, udata,
min(sizeof(req), udata->inlen))) {
@@ -1499,11 +1117,7 @@
rf->cq_table[cq_num] = iwcq;
init_completion(&iwcq->free_cq);
-#if __FreeBSD_version >= 1400026
return 0;
-#else
- return &iwcq->ibcq;
-#endif
cq_destroy:
irdma_cq_wq_destroy(rf, cq);
cq_kmem_free:
@@ -1513,13 +1127,7 @@
}
cq_free_rsrc:
irdma_free_rsrc(rf, rf->allocated_cqs, cq_num);
-#if __FreeBSD_version >= 1400026
return err_code;
-#else
-error:
- kfree(iwcq);
- return ERR_PTR(err_code);
-#endif
}
/**
@@ -1569,7 +1177,6 @@
* @ah_flags: destroy flags
*/
-#if __FreeBSD_version >= 1400026
void
irdma_destroy_ah(struct ib_ah *ibah, u32 ah_flags)
{
@@ -1582,33 +1189,10 @@
irdma_free_rsrc(iwdev->rf, iwdev->rf->allocated_ahs,
ah->sc_ah.ah_info.ah_idx);
}
-#endif
-#if __FreeBSD_version < 1400026
-int
-irdma_destroy_ah(struct ib_ah *ibah)
-{
- struct irdma_device *iwdev = to_iwdev(ibah->device);
- struct irdma_ah *ah = to_iwah(ibah);
-
- irdma_ah_cqp_op(iwdev->rf, &ah->sc_ah, IRDMA_OP_AH_DESTROY,
- false, NULL, ah);
-
- irdma_free_rsrc(iwdev->rf, iwdev->rf->allocated_ahs,
- ah->sc_ah.ah_info.ah_idx);
- kfree(ah);
- return 0;
-}
-#endif
-
-#if __FreeBSD_version >= 1400026
int
irdma_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata)
-#else
-int
-irdma_dereg_mr(struct ib_mr *ib_mr)
-#endif
{
struct irdma_mr *iwmr = to_iwmr(ib_mr);
struct irdma_device *iwdev = to_iwdev(ib_mr->device);
@@ -1618,15 +1202,9 @@
if (iwmr->type != IRDMA_MEMREG_TYPE_MEM) {
if (iwmr->region) {
struct irdma_ucontext *ucontext;
-#if __FreeBSD_version >= 1400026
ucontext = rdma_udata_to_drv_context(udata, struct irdma_ucontext, ibucontext);
-#else
- struct ib_pd *ibpd = ib_mr->pd;
-
- ucontext = to_ucontext(ibpd->uobject->context);
-#endif
irdma_del_memlist(iwmr, ucontext);
}
goto done;
@@ -1734,7 +1312,6 @@
return 0;
}
-#if __FreeBSD_version >= 1400026
/**
* irdma_destroy_cq - destroy cq
* @ib_cq: cq pointer
@@ -1769,90 +1346,6 @@
irdma_cq_free_rsrc(iwdev->rf, iwcq);
}
-#endif
-#if __FreeBSD_version < 1400026
-/**
- * irdma_destroy_cq - destroy cq
- * @ib_cq: cq pointer
- */
-int
-irdma_destroy_cq(struct ib_cq *ib_cq)
-{
- struct irdma_device *iwdev = to_iwdev(ib_cq->device);
- struct irdma_cq *iwcq = to_iwcq(ib_cq);
- struct irdma_sc_cq *cq = &iwcq->sc_cq;
- struct irdma_sc_dev *dev = cq->dev;
- struct irdma_sc_ceq *ceq = dev->ceq[cq->ceq_id];
- struct irdma_ceq *iwceq = container_of(ceq, struct irdma_ceq, sc_ceq);
- unsigned long flags;
-
- spin_lock_irqsave(&iwcq->lock, flags);
- if (!list_empty(&iwcq->cmpl_generated))
- irdma_remove_cmpls_list(iwcq);
- if (!list_empty(&iwcq->resize_list))
- irdma_process_resize_list(iwcq, iwdev, NULL);
- spin_unlock_irqrestore(&iwcq->lock, flags);
-
- irdma_cq_rem_ref(ib_cq);
- wait_for_completion(&iwcq->free_cq);
-
- irdma_cq_wq_destroy(iwdev->rf, cq);
-
- spin_lock_irqsave(&iwceq->ce_lock, flags);
- irdma_sc_cleanup_ceqes(cq, ceq);
- spin_unlock_irqrestore(&iwceq->ce_lock, flags);
-
- irdma_cq_free_rsrc(iwdev->rf, iwcq);
- kfree(iwcq);
-
- return 0;
-}
-
-#endif
-/**
- * irdma_alloc_mw - Allocate memory window
- * @pd: Protection domain
- * @type: Window type
- * @udata: user data pointer
- */
-struct ib_mw *
-irdma_alloc_mw(struct ib_pd *pd, enum ib_mw_type type,
- struct ib_udata *udata)
-{
- struct irdma_device *iwdev = to_iwdev(pd->device);
- struct irdma_mr *iwmr;
- int err_code;
- u32 stag;
-
- if (type != IB_MW_TYPE_1 && type != IB_MW_TYPE_2)
- return ERR_PTR(-EINVAL);
-
- iwmr = kzalloc(sizeof(*iwmr), GFP_KERNEL);
- if (!iwmr)
- return ERR_PTR(-ENOMEM);
-
- stag = irdma_create_stag(iwdev);
- if (!stag) {
- kfree(iwmr);
- return ERR_PTR(-ENOMEM);
- }
-
- iwmr->stag = stag;
- iwmr->ibmw.rkey = stag;
- iwmr->ibmw.pd = pd;
- iwmr->ibmw.type = type;
- iwmr->ibmw.device = pd->device;
-
- err_code = irdma_hw_alloc_mw(iwdev, iwmr);
- if (err_code) {
- irdma_free_stag(iwdev, stag);
- kfree(iwmr);
- return ERR_PTR(err_code);
- }
-
- return &iwmr->ibmw;
-}
-
/**
* kc_set_loc_seq_num_mss - Set local seq number and mss
* @cm_node: cm node info
@@ -1875,113 +1368,6 @@
(cm_node->iwdev->vsi.mtu - IRDMA_MTU_TO_MSS_IPV6);
}
-#if __FreeBSD_version < 1400026
-struct irdma_vma_data {
- struct list_head list;
- struct vm_area_struct *vma;
- struct mutex *vma_list_mutex; /* protect the vma_list */
-};
-
-/**
- * irdma_vma_open -
- * @vma: User VMA
- */
-static void
-irdma_vma_open(struct vm_area_struct *vma)
-{
- vma->vm_ops = NULL;
-}
-
-/**
- * irdma_vma_close - Remove vma data from vma list
- * @vma: User VMA
- */
-static void
-irdma_vma_close(struct vm_area_struct *vma)
-{
- struct irdma_vma_data *vma_data;
-
- vma_data = vma->vm_private_data;
- vma->vm_private_data = NULL;
- vma_data->vma = NULL;
- mutex_lock(vma_data->vma_list_mutex);
- list_del(&vma_data->list);
- mutex_unlock(vma_data->vma_list_mutex);
- kfree(vma_data);
-}
-
-static const struct vm_operations_struct irdma_vm_ops = {
- .open = irdma_vma_open,
- .close = irdma_vma_close
-};
-
-/**
- * irdma_set_vma_data - Save vma data in context list
- * @vma: User VMA
- * @context: ib user context
- */
-static int
-irdma_set_vma_data(struct vm_area_struct *vma,
- struct irdma_ucontext *context)
-{
- struct list_head *vma_head = &context->vma_list;
- struct irdma_vma_data *vma_entry;
-
- vma_entry = kzalloc(sizeof(*vma_entry), GFP_KERNEL);
- if (!vma_entry)
- return -ENOMEM;
-
- vma->vm_private_data = vma_entry;
- vma->vm_ops = &irdma_vm_ops;
-
- vma_entry->vma = vma;
- vma_entry->vma_list_mutex = &context->vma_list_mutex;
-
- mutex_lock(&context->vma_list_mutex);
- list_add(&vma_entry->list, vma_head);
- mutex_unlock(&context->vma_list_mutex);
-
- return 0;
-}
-
-/**
- * irdma_disassociate_ucontext - Disassociate user context
- * @context: ib user context
- */
-void
-irdma_disassociate_ucontext(struct ib_ucontext *context)
-{
- struct irdma_ucontext *ucontext = to_ucontext(context);
-
- struct irdma_vma_data *vma_data, *n;
- struct vm_area_struct *vma;
-
- mutex_lock(&ucontext->vma_list_mutex);
- list_for_each_entry_safe(vma_data, n, &ucontext->vma_list, list) {
- vma = vma_data->vma;
- zap_vma_ptes(vma, vma->vm_start, PAGE_SIZE);
-
- vma->vm_ops = NULL;
- list_del(&vma_data->list);
- kfree(vma_data);
- }
- mutex_unlock(&ucontext->vma_list_mutex);
-}
-
-int
-rdma_user_mmap_io(struct ib_ucontext *context, struct vm_area_struct *vma,
- unsigned long pfn, unsigned long size, pgprot_t prot)
-{
- if (io_remap_pfn_range(vma,
- vma->vm_start,
- pfn,
- size,
- prot))
- return -EAGAIN;
-
- return irdma_set_vma_data(vma, to_ucontext(context));
-}
-#else
/**
* irdma_disassociate_ucontext - Disassociate user context
* @context: ib user context
@@ -1990,7 +1376,6 @@
irdma_disassociate_ucontext(struct ib_ucontext *context)
{
}
-#endif
struct ib_device *
ib_device_get_by_netdev(if_t netdev, int driver_id)
@@ -2032,7 +1417,7 @@
{
int ret;
- ret = rdma_query_gid(ibdev, port, index, gid);
+ ret = ib_get_cached_gid(ibdev, port, index, gid, NULL);
if (ret == -EAGAIN) {
memcpy(gid, &zgid, sizeof(*gid));
return 0;
@@ -2144,7 +1529,7 @@
if (rdma_protocol_roce(ibdev, 1)) {
props->gid_tbl_len = 32;
- props->port_cap_flags |= IB_PORT_IP_BASED_GIDS;
+ props->port_cap_flags |= IB_PORT_IP_BASED_GIDS;
props->pkey_tbl_len = IRDMA_PKEY_TBL_SZ;
} else {
props->gid_tbl_len = 1;
@@ -2325,9 +1710,6 @@
BIT_ULL(IB_USER_VERBS_CMD_QUERY_QP) |
BIT_ULL(IB_USER_VERBS_CMD_POLL_CQ) |
BIT_ULL(IB_USER_VERBS_CMD_DESTROY_QP) |
- BIT_ULL(IB_USER_VERBS_CMD_ALLOC_MW) |
- BIT_ULL(IB_USER_VERBS_CMD_BIND_MW) |
- BIT_ULL(IB_USER_VERBS_CMD_DEALLOC_MW) |
BIT_ULL(IB_USER_VERBS_CMD_POST_RECV) |
BIT_ULL(IB_USER_VERBS_CMD_POST_SEND);
iwdev->ibdev.uverbs_ex_cmd_mask =
diff --git a/sys/dev/irdma/irdma_main.h b/sys/dev/irdma/irdma_main.h
--- a/sys/dev/irdma/irdma_main.h
+++ b/sys/dev/irdma/irdma_main.h
@@ -44,9 +44,7 @@
#include <netinet/if_ether.h>
#include <linux/slab.h>
#include <linux/rculist.h>
-#if __FreeBSD_version >= 1400000
#include <rdma/uverbs_ioctl.h>
-#endif
#include <rdma/ib_smi.h>
#include <rdma/ib_verbs.h>
#include <rdma/ib_pack.h>
@@ -185,7 +183,7 @@
void (*callback_fcn)(struct irdma_cqp_request *cqp_request);
void *param;
struct irdma_cqp_compl_info compl_info;
- bool request_done; /* READ/WRITE_ONCE macros operate on it */
+ u8 request_done; /* READ/WRITE_ONCE macros operate on it */
bool waiting:1;
bool dynamic:1;
};
@@ -236,7 +234,7 @@
u32 idx;
u32 irq;
u32 cpu_affinity;
- u32 ceq_id;
+ u16 ceq_id;
char name[IRDMA_IRQ_NAME_STR_LEN];
struct resource *res;
void *tag;
@@ -378,7 +376,6 @@
u32 roce_ackcreds;
u32 vendor_id;
u32 vendor_part_id;
- u32 push_mode;
u32 rcv_wnd;
u16 mac_ip_table_idx;
u16 vsi_num;
@@ -392,6 +389,7 @@
bool override_ooo:1;
bool override_rd_fence_rate:1;
bool override_rtomin:1;
+ bool push_mode:1;
bool roce_mode:1;
bool roce_dcqcn_en:1;
bool dcb_vlan_mode:1;
@@ -419,7 +417,6 @@
return container_of(ibucontext, struct irdma_ucontext, ibucontext);
}
-#if __FreeBSD_version >= 1400026
static inline struct irdma_user_mmap_entry *
to_irdma_mmap_entry(struct rdma_user_mmap_entry *rdma_entry)
{
@@ -427,7 +424,6 @@
rdma_entry);
}
-#endif
static inline struct irdma_pd *to_iwpd(struct ib_pd *ibpd)
{
return container_of(ibpd, struct irdma_pd, ibpd);
@@ -586,8 +582,8 @@
void irdma_copy_ip_ntohl(u32 *dst, __be32 *src);
void irdma_copy_ip_htonl(__be32 *dst, u32 *src);
u16 irdma_get_vlan_ipv4(struct iw_cm_id *cm_id, u32 *addr);
-if_t irdma_netdev_vlan_ipv6(struct iw_cm_id *cm_id, u32 *addr, u16 *vlan_id,
- u8 *mac);
+void irdma_get_vlan_mac_ipv6(struct iw_cm_id *cm_id, u32 *addr, u16 *vlan_id,
+ u8 *mac);
struct ib_mr *irdma_reg_phys_mr(struct ib_pd *ib_pd, u64 addr, u64 size,
int acc, u64 *iova_start);
int irdma_upload_qp_context(struct irdma_qp *iwqp, bool freeze, bool raw);
@@ -599,7 +595,6 @@
bool wait,
void (*callback_fcn)(struct irdma_cqp_request *cqp_request),
void *cb_param);
-void irdma_gsi_ud_qp_ah_cb(struct irdma_cqp_request *cqp_request);
void irdma_udqp_qs_worker(struct work_struct *work);
bool irdma_cq_empty(struct irdma_cq *iwcq);
int irdma_netdevice_event(struct notifier_block *notifier, unsigned long event,
diff --git a/sys/dev/irdma/irdma_puda.h b/sys/dev/irdma/irdma_puda.h
--- a/sys/dev/irdma/irdma_puda.h
+++ b/sys/dev/irdma/irdma_puda.h
@@ -1,7 +1,7 @@
/*-
* SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB
*
- * Copyright (c) 2015 - 2023 Intel Corporation
+ * Copyright (c) 2015 - 2022 Intel Corporation
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
diff --git a/sys/dev/irdma/irdma_puda.c b/sys/dev/irdma/irdma_puda.c
--- a/sys/dev/irdma/irdma_puda.c
+++ b/sys/dev/irdma/irdma_puda.c
@@ -47,6 +47,7 @@
static void
irdma_ilq_putback_rcvbuf(struct irdma_sc_qp *qp,
struct irdma_puda_buf *buf, u32 wqe_idx);
+
/**
* irdma_puda_get_listbuf - get buffer from puda list
* @list: list to use for buffers (ILQ or IEQ)
@@ -181,7 +182,7 @@
struct irdma_puda_buf *buf;
struct irdma_virt_mem buf_mem;
- buf_mem.size = sizeof(struct irdma_puda_buf);
+ buf_mem.size = sizeof(*buf);
buf_mem.va = kzalloc(buf_mem.size, GFP_KERNEL);
if (!buf_mem.va)
return NULL;
@@ -935,6 +936,7 @@
struct irdma_sc_ceq *ceq;
ceq = vsi->dev->ceq[0];
+
switch (type) {
case IRDMA_PUDA_RSRC_TYPE_ILQ:
rsrc = vsi->ilq;
@@ -1008,7 +1010,7 @@
bool virtdma = false;
unsigned long flags;
- buf_mem.size = count * sizeof(struct irdma_puda_buf);
+ buf_mem.size = count * sizeof(*buf);
buf_mem.va = kzalloc(buf_mem.size, GFP_KERNEL);
if (!buf_mem.va) {
irdma_debug(rsrc->dev, IRDMA_DEBUG_PUDA,
@@ -1103,7 +1105,7 @@
struct irdma_virt_mem *vmem;
info->count = 1;
- pudasize = sizeof(struct irdma_puda_rsrc);
+ pudasize = sizeof(*rsrc);
sqwridsize = info->sq_size * sizeof(struct irdma_sq_uk_wr_trk_info);
rqwridsize = info->rq_size * 8;
switch (info->type) {
@@ -1703,6 +1705,7 @@
struct irdma_pfpdu *pfpdu = &qp->pfpdu;
u32 *hw_host_ctx = (u32 *)qp->hw_host_ctx;
u32 rcv_wnd = hw_host_ctx[23];
+
/* first partial seq # in q2 */
u32 fps = *(u32 *)(qp->q2_buf + Q2_FPSN_OFFSET);
struct list_head *rxlist = &pfpdu->rxlist;
diff --git a/sys/dev/irdma/irdma_type.h b/sys/dev/irdma/irdma_type.h
--- a/sys/dev/irdma/irdma_type.h
+++ b/sys/dev/irdma/irdma_type.h
@@ -42,6 +42,8 @@
#include "irdma_hmc.h"
#include "irdma_uda.h"
#include "irdma_ws.h"
+#include "irdma_pble.h"
+
enum irdma_debug_flag {
IRDMA_DEBUG_NONE = 0x00000000,
IRDMA_DEBUG_ERR = 0x00000001,
@@ -70,6 +72,8 @@
IRDMA_DEBUG_ALL = 0xFFFFFFFF,
};
+#define RSVD_OFFSET 0xFFFFFFFF
+
enum irdma_page_size {
IRDMA_PAGE_SIZE_4K = 0,
IRDMA_PAGE_SIZE_2M,
@@ -472,7 +476,7 @@
bool virtual_map:1;
bool check_overflow:1;
bool ceq_id_valid:1;
- bool tph_en;
+ bool tph_en:1;
};
struct irdma_sc_qp {
@@ -520,9 +524,9 @@
};
struct irdma_stats_inst_info {
- bool use_hmc_fcn_index;
u16 hmc_fn_id;
u16 stats_idx;
+ bool use_hmc_fcn_index:1;
};
struct irdma_up_info {
@@ -570,7 +574,7 @@
u8 traffic_class;
u8 rel_bw;
u8 prio_type;
- bool valid;
+ bool valid:1;
};
struct irdma_config_check {
@@ -623,7 +627,6 @@
__le64 *fpm_query_buf;
__le64 *fpm_commit_buf;
struct irdma_hw *hw;
- u8 IOMEM *db_addr;
u32 IOMEM *wqe_alloc_db;
u32 IOMEM *cq_arm_db;
u32 IOMEM *aeq_alloc_db;
@@ -649,8 +652,6 @@
u32 debug_mask;
u16 num_vfs;
u16 hmc_fn_id;
- u8 vf_id;
- bool vchnl_up:1;
bool ceq_valid:1;
u8 pci_rev;
int (*ws_add)(struct irdma_sc_vsi *vsi, u8 user_pri);
@@ -666,7 +667,7 @@
u8 pbl_chunk_size;
u32 first_pm_pbl_idx;
bool virtual_map:1;
- bool check_overflow;
+ bool check_overflow:1;
bool cq_resize:1;
};
@@ -676,7 +677,7 @@
bool cq_num_valid:1;
bool arp_cache_idx_valid:1;
bool mac_valid:1;
- bool force_lpb;
+ bool force_lpb:1;
u8 next_iwarp_state;
};
@@ -709,7 +710,7 @@
u16 maj_err_code;
u16 min_err_code;
u8 op_code;
- bool error;
+ bool error:1;
};
struct irdma_qos_tc_info {
@@ -751,7 +752,7 @@
struct irdma_vsi_stats_info {
struct irdma_vsi_pestat *pestat;
u8 fcn_id;
- bool alloc_stats_inst;
+ bool alloc_stats_inst:1;
};
struct irdma_device_init_info {
@@ -789,7 +790,7 @@
u32 *aeqe_base;
void *pbl_list;
u32 elem_cnt;
- bool virtual_map;
+ bool virtual_map:1;
u8 pbl_chunk_size;
u32 first_pm_pbl_idx;
u32 msix_idx;
@@ -856,7 +857,6 @@
bool dcqcn_en:1;
bool rcv_no_icrc:1;
bool wr_rdresp_en:1;
- bool bind_en:1;
bool fast_reg_en:1;
bool priv_mode_en:1;
bool rd_en:1;
@@ -888,7 +888,6 @@
bool snd_mark_en:1;
bool rcv_mark_en:1;
bool wr_rdresp_en:1;
- bool bind_en:1;
bool fast_reg_en:1;
bool priv_mode_en:1;
bool rd_en:1;
@@ -1134,12 +1133,12 @@
u8 mac_addr[ETHER_ADDR_LEN];
u32 reach_max;
u16 arp_index;
- bool permanent;
+ bool permanent:1;
};
struct irdma_apbvt_info {
u16 port;
- bool add;
+ bool add:1;
};
struct irdma_qhash_table_info {
diff --git a/sys/dev/irdma/irdma_uda.h b/sys/dev/irdma/irdma_uda.h
--- a/sys/dev/irdma/irdma_uda.h
+++ b/sys/dev/irdma/irdma_uda.h
@@ -43,6 +43,7 @@
struct irdma_ah_info {
struct irdma_sc_vsi *vsi;
+ struct irdma_cqp_request *cqp_request;
u32 pd_idx;
u32 dst_arpindex;
u32 dest_ip_addr[4];
diff --git a/sys/dev/irdma/irdma_uk.c b/sys/dev/irdma/irdma_uk.c
--- a/sys/dev/irdma/irdma_uk.c
+++ b/sys/dev/irdma/irdma_uk.c
@@ -45,16 +45,16 @@
* @valid: The wqe valid
*/
static void
-irdma_set_fragment(__le64 * wqe, u32 offset, struct irdma_sge *sge,
+irdma_set_fragment(__le64 * wqe, u32 offset, struct ib_sge *sge,
u8 valid)
{
if (sge) {
set_64bit_val(wqe, offset,
- FIELD_PREP(IRDMAQPSQ_FRAG_TO, sge->tag_off));
+ FIELD_PREP(IRDMAQPSQ_FRAG_TO, sge->addr));
set_64bit_val(wqe, offset + IRDMA_BYTE_8,
FIELD_PREP(IRDMAQPSQ_VALID, valid) |
- FIELD_PREP(IRDMAQPSQ_FRAG_LEN, sge->len) |
- FIELD_PREP(IRDMAQPSQ_FRAG_STAG, sge->stag));
+ FIELD_PREP(IRDMAQPSQ_FRAG_LEN, sge->length) |
+ FIELD_PREP(IRDMAQPSQ_FRAG_STAG, sge->lkey));
} else {
set_64bit_val(wqe, offset, 0);
set_64bit_val(wqe, offset + IRDMA_BYTE_8,
@@ -71,14 +71,14 @@
*/
static void
irdma_set_fragment_gen_1(__le64 * wqe, u32 offset,
- struct irdma_sge *sge, u8 valid)
+ struct ib_sge *sge, u8 valid)
{
if (sge) {
set_64bit_val(wqe, offset,
- FIELD_PREP(IRDMAQPSQ_FRAG_TO, sge->tag_off));
+ FIELD_PREP(IRDMAQPSQ_FRAG_TO, sge->addr));
set_64bit_val(wqe, offset + IRDMA_BYTE_8,
- FIELD_PREP(IRDMAQPSQ_GEN1_FRAG_LEN, sge->len) |
- FIELD_PREP(IRDMAQPSQ_GEN1_FRAG_STAG, sge->stag));
+ FIELD_PREP(IRDMAQPSQ_GEN1_FRAG_LEN, sge->length) |
+ FIELD_PREP(IRDMAQPSQ_GEN1_FRAG_STAG, sge->lkey));
} else {
set_64bit_val(wqe, offset, 0);
set_64bit_val(wqe, offset + IRDMA_BYTE_8, 0);
@@ -209,8 +209,7 @@
if (IRDMA_RING_CURRENT_HEAD(qp->initial_ring) !=
IRDMA_RING_CURRENT_TAIL(qp->sq_ring) &&
!qp->push_mode) {
- if (post_sq)
- irdma_uk_qp_post_wr(qp);
+ irdma_uk_qp_post_wr(qp);
} else {
push = (__le64 *) ((uintptr_t)qp->push_wqe +
(wqe_idx & 0x7) * 0x20);
@@ -338,7 +337,7 @@
return -EINVAL;
for (i = 0; i < op_info->num_lo_sges; i++)
- total_size += op_info->lo_sg_list[i].len;
+ total_size += op_info->lo_sg_list[i].length;
read_fence |= info->read_fence;
@@ -357,7 +356,7 @@
qp->sq_wrtrk_array[wqe_idx].signaled = info->signaled;
set_64bit_val(wqe, IRDMA_BYTE_16,
- FIELD_PREP(IRDMAQPSQ_FRAG_TO, op_info->rem_addr.tag_off));
+ FIELD_PREP(IRDMAQPSQ_FRAG_TO, op_info->rem_addr.addr));
if (info->imm_data_valid) {
set_64bit_val(wqe, IRDMA_BYTE_0,
@@ -386,7 +385,7 @@
++addl_frag_cnt;
}
- hdr = FIELD_PREP(IRDMAQPSQ_REMSTAG, op_info->rem_addr.stag) |
+ hdr = FIELD_PREP(IRDMAQPSQ_REMSTAG, op_info->rem_addr.lkey) |
FIELD_PREP(IRDMAQPSQ_OPCODE, info->op_type) |
FIELD_PREP(IRDMAQPSQ_IMMDATAFLAG, info->imm_data_valid) |
FIELD_PREP(IRDMAQPSQ_REPORTRTT, info->report_rtt) |
@@ -437,7 +436,7 @@
return -EINVAL;
for (i = 0; i < op_info->num_lo_sges; i++)
- total_size += op_info->lo_sg_list[i].len;
+ total_size += op_info->lo_sg_list[i].length;
ret_code = irdma_fragcnt_to_quanta_sq(op_info->num_lo_sges, &quanta);
if (ret_code)
@@ -475,8 +474,8 @@
++addl_frag_cnt;
}
set_64bit_val(wqe, IRDMA_BYTE_16,
- FIELD_PREP(IRDMAQPSQ_FRAG_TO, op_info->rem_addr.tag_off));
- hdr = FIELD_PREP(IRDMAQPSQ_REMSTAG, op_info->rem_addr.stag) |
+ FIELD_PREP(IRDMAQPSQ_FRAG_TO, op_info->rem_addr.addr));
+ hdr = FIELD_PREP(IRDMAQPSQ_REMSTAG, op_info->rem_addr.lkey) |
FIELD_PREP(IRDMAQPSQ_REPORTRTT, (info->report_rtt ? 1 : 0)) |
FIELD_PREP(IRDMAQPSQ_ADDFRAGCNT, addl_frag_cnt) |
FIELD_PREP(IRDMAQPSQ_OPCODE,
@@ -525,7 +524,7 @@
return -EINVAL;
for (i = 0; i < op_info->num_sges; i++)
- total_size += op_info->sg_list[i].len;
+ total_size += op_info->sg_list[i].length;
if (info->imm_data_valid)
frag_cnt = op_info->num_sges + 1;
@@ -604,15 +603,15 @@
* @polarity: compatibility parameter
*/
static void
-irdma_copy_inline_data_gen_1(u8 *wqe, struct irdma_sge *sge_list,
+irdma_copy_inline_data_gen_1(u8 *wqe, struct ib_sge *sge_list,
u32 num_sges, u8 polarity)
{
u32 quanta_bytes_remaining = 16;
u32 i;
for (i = 0; i < num_sges; i++) {
- u8 *cur_sge = (u8 *)(uintptr_t)sge_list[i].tag_off;
- u32 sge_len = sge_list[i].len;
+ u8 *cur_sge = (u8 *)(uintptr_t)sge_list[i].addr;
+ u32 sge_len = sge_list[i].length;
while (sge_len) {
u32 bytes_copied;
@@ -651,7 +650,7 @@
* @polarity: polarity of wqe valid bit
*/
static void
-irdma_copy_inline_data(u8 *wqe, struct irdma_sge *sge_list,
+irdma_copy_inline_data(u8 *wqe, struct ib_sge *sge_list,
u32 num_sges, u8 polarity)
{
u8 inline_valid = polarity << IRDMA_INLINE_VALID_S;
@@ -662,8 +661,8 @@
wqe += 8;
for (i = 0; i < num_sges; i++) {
- u8 *cur_sge = (u8 *)(uintptr_t)sge_list[i].tag_off;
- u32 sge_len = sge_list[i].len;
+ u8 *cur_sge = (u8 *)(uintptr_t)sge_list[i].addr;
+ u32 sge_len = sge_list[i].length;
while (sge_len) {
u32 bytes_copied;
@@ -743,7 +742,7 @@
return -EINVAL;
for (i = 0; i < op_info->num_lo_sges; i++)
- total_size += op_info->lo_sg_list[i].len;
+ total_size += op_info->lo_sg_list[i].length;
if (unlikely(total_size > qp->max_inline_data))
return -EINVAL;
@@ -756,9 +755,9 @@
qp->sq_wrtrk_array[wqe_idx].signaled = info->signaled;
read_fence |= info->read_fence;
set_64bit_val(wqe, IRDMA_BYTE_16,
- FIELD_PREP(IRDMAQPSQ_FRAG_TO, op_info->rem_addr.tag_off));
+ FIELD_PREP(IRDMAQPSQ_FRAG_TO, op_info->rem_addr.addr));
- hdr = FIELD_PREP(IRDMAQPSQ_REMSTAG, op_info->rem_addr.stag) |
+ hdr = FIELD_PREP(IRDMAQPSQ_REMSTAG, op_info->rem_addr.lkey) |
FIELD_PREP(IRDMAQPSQ_OPCODE, info->op_type) |
FIELD_PREP(IRDMAQPSQ_INLINEDATALEN, total_size) |
FIELD_PREP(IRDMAQPSQ_REPORTRTT, info->report_rtt ? 1 : 0) |
@@ -814,7 +813,7 @@
return -EINVAL;
for (i = 0; i < op_info->num_sges; i++)
- total_size += op_info->sg_list[i].len;
+ total_size += op_info->sg_list[i].length;
if (unlikely(total_size > qp->max_inline_data))
return -EINVAL;
@@ -879,7 +878,7 @@
u64 hdr;
u32 wqe_idx;
bool local_fence = false;
- struct irdma_sge sge = {0};
+ struct ib_sge sge = {0};
u16 quanta = IRDMA_QP_WQE_MIN_QUANTA;
info->push_wqe = qp->push_db ? true : false;
@@ -890,7 +889,7 @@
if (!wqe)
return -ENOSPC;
- sge.stag = op_info->target_stag;
+ sge.lkey = op_info->target_stag;
qp->wqe_ops.iw_set_fragment(wqe, IRDMA_BYTE_0, &sge, 0);
set_64bit_val(wqe, IRDMA_BYTE_16, 0);
@@ -1327,8 +1326,7 @@
IRDMA_RING_MOVE_TAIL(cq->cq_ring);
set_64bit_val(cq->shadow_area, IRDMA_BYTE_0,
IRDMA_RING_CURRENT_HEAD(cq->cq_ring));
- memset(info, 0,
- sizeof(struct irdma_cq_poll_info));
+ memset(info, 0, sizeof(*info));
return irdma_uk_cq_poll_cmpl(cq, info);
}
}
@@ -1484,10 +1482,12 @@
int
irdma_get_sqdepth(struct irdma_uk_attrs *uk_attrs, u32 sq_size, u8 shift, u32 *sqdepth)
{
+ u32 min_size = (u32)uk_attrs->min_hw_wq_size << shift;
+
*sqdepth = irdma_round_up_wq((sq_size << shift) + IRDMA_SQ_RSVD);
- if (*sqdepth < ((u32)uk_attrs->min_hw_wq_size << shift))
- *sqdepth = uk_attrs->min_hw_wq_size << shift;
+ if (*sqdepth < min_size)
+ *sqdepth = min_size;
else if (*sqdepth > uk_attrs->max_hw_wq_quanta)
return -EINVAL;
@@ -1501,10 +1501,12 @@
int
irdma_get_rqdepth(struct irdma_uk_attrs *uk_attrs, u32 rq_size, u8 shift, u32 *rqdepth)
{
+ u32 min_size = (u32)uk_attrs->min_hw_wq_size << shift;
+
*rqdepth = irdma_round_up_wq((rq_size << shift) + IRDMA_RQ_RSVD);
- if (*rqdepth < ((u32)uk_attrs->min_hw_wq_size << shift))
- *rqdepth = uk_attrs->min_hw_wq_size << shift;
+ if (*rqdepth < min_size)
+ *rqdepth = min_size;
else if (*rqdepth > uk_attrs->max_hw_rq_quanta)
return -EINVAL;
@@ -1535,9 +1537,10 @@
{
u16 move_cnt = 1;
- if (qp->uk_attrs->feature_flags & IRDMA_FEATURE_RTS_AE)
+ if (info->start_wqe_idx)
+ move_cnt = info->start_wqe_idx;
+ else if (qp->uk_attrs->feature_flags & IRDMA_FEATURE_RTS_AE)
move_cnt = 3;
-
qp->conn_wqes = move_cnt;
IRDMA_RING_MOVE_HEAD_BY_COUNT_NOCHECK(qp->sq_ring, move_cnt);
IRDMA_RING_MOVE_TAIL_BY_COUNT(qp->sq_ring, move_cnt);
@@ -1582,6 +1585,7 @@
{
bool imm_support = ukinfo->uk_attrs->hw_rev >= IRDMA_GEN_2 ? true : false;
int status;
+
irdma_get_wqe_shift(ukinfo->uk_attrs,
imm_support ? ukinfo->max_sq_frag_cnt + 1 :
ukinfo->max_sq_frag_cnt,
@@ -1676,6 +1680,8 @@
qp->wqe_ops = iw_wqe_uk_ops_gen_1;
else
qp->wqe_ops = iw_wqe_uk_ops;
+ qp->start_wqe_idx = info->start_wqe_idx;
+
return ret_code;
}
diff --git a/sys/dev/irdma/irdma_user.h b/sys/dev/irdma/irdma_user.h
--- a/sys/dev/irdma/irdma_user.h
+++ b/sys/dev/irdma/irdma_user.h
@@ -35,6 +35,8 @@
#ifndef IRDMA_USER_H
#define IRDMA_USER_H
+#include <rdma/ib_verbs.h>
+
#define irdma_handle void *
#define irdma_adapter_handle irdma_handle
#define irdma_qp_handle irdma_handle
@@ -48,7 +50,7 @@
#define irdma_access_privileges u32
#define irdma_physical_fragment u64
#define irdma_address_list u64 *
-#define irdma_sgl struct irdma_sge *
+#define irdma_sgl struct ib_sge *
#define IRDMA_MAX_MR_SIZE 0x200000000000ULL
@@ -78,8 +80,6 @@
#define IRDMA_OP_TYPE_REC_IMM 0x3f
#define IRDMA_FLUSH_MAJOR_ERR 1
-#define IRDMA_SRQFLUSH_RSVD_MAJOR_ERR 0xfffe
-
/* Async Events codes */
#define IRDMA_AE_AMP_UNALLOCATED_STAG 0x0102
#define IRDMA_AE_AMP_INVALID_STAG 0x0103
@@ -140,6 +140,7 @@
#define IRDMA_AE_ROE_INVALID_RDMA_READ_REQUEST 0x0313
#define IRDMA_AE_ROE_INVALID_RDMA_WRITE_OR_READ_RESP 0x0314
#define IRDMA_AE_ROCE_RSP_LENGTH_ERROR 0x0316
+#define IRDMA_AE_ROCE_REQ_LENGTH_ERROR 0x0318
#define IRDMA_AE_ROCE_EMPTY_MCG 0x0380
#define IRDMA_AE_ROCE_BAD_MC_IP_ADDR 0x0381
#define IRDMA_AE_ROCE_BAD_MC_QPID 0x0382
@@ -160,6 +161,7 @@
#define IRDMA_AE_LLP_TOO_MANY_KEEPALIVE_RETRIES 0x050b
#define IRDMA_AE_LLP_DOUBT_REACHABILITY 0x050c
#define IRDMA_AE_LLP_CONNECTION_ESTABLISHED 0x050e
+#define IRDMA_AE_LLP_TOO_MANY_RNRS 0x050f
#define IRDMA_AE_RESOURCE_EXHAUSTION 0x0520
#define IRDMA_AE_RESET_SENT 0x0601
#define IRDMA_AE_TERMINATE_SENT 0x0602
@@ -199,8 +201,7 @@
IRDMA_MAX_OUTBOUND_MSG_SIZE = 65537,
/* 64K +1 */
IRDMA_MAX_INBOUND_MSG_SIZE = 65537,
- IRDMA_MAX_PUSH_PAGE_COUNT = 1024,
- IRDMA_MAX_PE_ENA_VF_COUNT = 32,
+ IRDMA_MAX_PE_ENA_VF_COUNT = 32,
IRDMA_MAX_VF_FPM_ID = 47,
IRDMA_MAX_SQ_PAYLOAD_SIZE = 2145386496,
IRDMA_MAX_INLINE_DATA_SIZE = 101,
@@ -227,6 +228,7 @@
FLUSH_RETRY_EXC_ERR,
FLUSH_MW_BIND_ERR,
FLUSH_REM_INV_REQ_ERR,
+ FLUSH_RNR_RETRY_EXC_ERR,
};
enum irdma_qp_event_type {
@@ -280,12 +282,6 @@
struct irdma_qp_uk_init_info;
struct irdma_cq_uk_init_info;
-struct irdma_sge {
- irdma_tagged_offset tag_off;
- u32 len;
- irdma_stag stag;
-};
-
struct irdma_ring {
volatile u32 head;
volatile u32 tail; /* effective tail */
@@ -317,13 +313,13 @@
struct irdma_rdma_write {
irdma_sgl lo_sg_list;
u32 num_lo_sges;
- struct irdma_sge rem_addr;
+ struct ib_sge rem_addr;
};
struct irdma_rdma_read {
irdma_sgl lo_sg_list;
u32 num_lo_sges;
- struct irdma_sge rem_addr;
+ struct ib_sge rem_addr;
};
struct irdma_bind_window {
@@ -422,9 +418,9 @@
bool post_sq);
struct irdma_wqe_uk_ops {
- void (*iw_copy_inline_data)(u8 *dest, struct irdma_sge *sge_list, u32 num_sges, u8 polarity);
+ void (*iw_copy_inline_data)(u8 *dest, struct ib_sge *sge_list, u32 num_sges, u8 polarity);
u16 (*iw_inline_data_size_to_quanta)(u32 data_size);
- void (*iw_set_fragment)(__le64 *wqe, u32 offset, struct irdma_sge *sge,
+ void (*iw_set_fragment)(__le64 *wqe, u32 offset, struct ib_sge *sge,
u8 valid);
void (*iw_set_mw_bind_wqe)(__le64 *wqe,
struct irdma_bind_window *op_info);
@@ -490,6 +486,7 @@
u8 rwqe_polarity;
u8 rq_wqe_size;
u8 rq_wqe_size_multiplier;
+ u8 start_wqe_idx;
bool deferred_flag:1;
bool push_mode:1; /* whether the last post wqe was pushed */
bool push_dropped:1;
@@ -537,6 +534,7 @@
u32 sq_depth;
u32 rq_depth;
u8 first_sq_wq;
+ u8 start_wqe_idx;
u8 type;
u8 sq_shift;
u8 rq_shift;
@@ -625,10 +623,15 @@
case IRDMA_AE_LLP_SEGMENT_TOO_SMALL:
case IRDMA_AE_LLP_RECEIVED_MPA_CRC_ERROR:
case IRDMA_AE_ROCE_RSP_LENGTH_ERROR:
+ case IRDMA_AE_ROCE_REQ_LENGTH_ERROR:
case IRDMA_AE_IB_REMOTE_OP_ERROR:
qp_err.flush_code = FLUSH_REM_OP_ERR;
qp_err.event_type = IRDMA_QP_EVENT_CATASTROPHIC;
break;
+ case IRDMA_AE_LLP_TOO_MANY_RNRS:
+ qp_err.flush_code = FLUSH_RNR_RETRY_EXC_ERR;
+ qp_err.event_type = IRDMA_QP_EVENT_CATASTROPHIC;
+ break;
case IRDMA_AE_LCE_QP_CATASTROPHIC:
qp_err.flush_code = FLUSH_FATAL_ERR;
qp_err.event_type = IRDMA_QP_EVENT_CATASTROPHIC;
diff --git a/sys/dev/irdma/irdma_utils.c b/sys/dev/irdma/irdma_utils.c
--- a/sys/dev/irdma/irdma_utils.c
+++ b/sys/dev/irdma/irdma_utils.c
@@ -164,6 +164,7 @@
"Connection error: Doubt reachability (usually occurs after the max number of retries has been reached)"},
{IRDMA_AE_LLP_CONNECTION_ESTABLISHED,
"iWARP event: Connection established"},
+ {IRDMA_AE_LLP_TOO_MANY_RNRS, "RoCEv2: Too many RNR NACKs"},
{IRDMA_AE_RESOURCE_EXHAUSTION,
"QP error: Resource exhaustion"},
{IRDMA_AE_RESET_SENT,
@@ -437,11 +438,11 @@
irdma_free_pending_cqp_request(struct irdma_cqp *cqp,
struct irdma_cqp_request *cqp_request)
{
- if (cqp_request->waiting) {
- cqp_request->compl_info.error = true;
- WRITE_ONCE(cqp_request->request_done, true);
+ cqp_request->compl_info.error = true;
+ WRITE_ONCE(cqp_request->request_done, true);
+
+ if (cqp_request->waiting)
wake_up(&cqp_request->waitq);
- }
wait_event_timeout(cqp->remove_wq,
atomic_read(&cqp_request->refcnt) == 1, 1000);
irdma_put_cqp_request(cqp, cqp_request);
@@ -558,8 +559,6 @@
[IRDMA_OP_MANAGE_HMC_PM_FUNC_TABLE] = "Manage HMC PM Function Table Cmd",
[IRDMA_OP_SUSPEND] = "Suspend QP Cmd",
[IRDMA_OP_RESUME] = "Resume QP Cmd",
- [IRDMA_OP_MANAGE_VCHNL_REQ_PBLE_BP] =
- "Manage Virtual Channel Requester Function PBLE Backing Pages Cmd",
[IRDMA_OP_QUERY_FPM_VAL] = "Query FPM Values Cmd",
[IRDMA_OP_COMMIT_FPM_VAL] = "Commit FPM Values Cmd",
[IRDMA_OP_AH_CREATE] = "Create Address Handle Cmd",
@@ -591,7 +590,7 @@
{0xffff, 0x8007, "Modify QP Bad Close"},
{0xffff, 0x8009, "LLP Closed"},
{0xffff, 0x800a, "Reset Not Sent"},
- {0xffff, 0x200, "Failover Pending"}
+ {0xffff, 0x0200, "Failover Pending"},
};
/**
@@ -1055,15 +1054,16 @@
/**
* irdma_dealloc_push_page - free a push page for qp
* @rf: RDMA PCI function
- * @qp: hardware control qp
+ * @iwqp: QP pointer
*/
void
irdma_dealloc_push_page(struct irdma_pci_f *rf,
- struct irdma_sc_qp *qp)
+ struct irdma_qp *iwqp)
{
struct irdma_cqp_request *cqp_request;
struct cqp_cmds_info *cqp_info;
int status;
+ struct irdma_sc_qp *qp = &iwqp->sc_qp;
if (qp->push_idx == IRDMA_INVALID_PUSH_PAGE_INDEX)
return;
@@ -1564,16 +1564,6 @@
del_timer_sync(&devstat->stats_timer);
}
-/**
- * irdma_process_stats - Checking for wrap and update stats
- * @pestat: stats structure pointer
- */
-static inline void
-irdma_process_stats(struct irdma_vsi_pestat *pestat)
-{
- sc_vsi_update_stats(pestat->vsi);
-}
-
/**
* irdma_process_cqp_stats - Checking for wrap and update stats
* @cqp_request: cqp_request structure pointer
@@ -1583,7 +1573,7 @@
{
struct irdma_vsi_pestat *pestat = cqp_request->param;
- irdma_process_stats(pestat);
+ sc_vsi_update_stats(pestat->vsi);
}
/**
@@ -1619,7 +1609,7 @@
cqp_request->callback_fcn = irdma_process_cqp_stats;
status = irdma_handle_cqp_op(rf, cqp_request);
if (wait)
- irdma_process_stats(pestat);
+ sc_vsi_update_stats(pestat->vsi);
irdma_put_cqp_request(&rf->cqp, cqp_request);
return status;
@@ -1814,6 +1804,10 @@
cqp_info->cqp_cmd = cmd;
cqp_info->post_sq = 1;
if (cmd == IRDMA_OP_AH_CREATE) {
+ if (!wait)
+ irdma_get_cqp_request(cqp_request);
+ sc_ah->ah_info.cqp_request = cqp_request;
+
cqp_info->in.u.ah_create.info = sc_ah->ah_info;
cqp_info->in.u.ah_create.scratch = (uintptr_t)cqp_request;
cqp_info->in.u.ah_create.cqp = &rf->cqp.sc_cqp;
@@ -1948,21 +1942,6 @@
kfree(ah);
}
-/**
- * irdma_gsi_ud_qp_ah_cb - callback after creation of AH for GSI/ID QP
- * @cqp_request: pointer to cqp_request of create AH
- */
-void
-irdma_gsi_ud_qp_ah_cb(struct irdma_cqp_request *cqp_request)
-{
- struct irdma_sc_ah *sc_ah = cqp_request->param;
-
- if (!cqp_request->compl_info.op_ret_val)
- sc_ah->ah_info.ah_valid = true;
- else
- sc_ah->ah_info.ah_valid = false;
-}
-
/**
* irdma_prm_add_pble_mem - add moemory to pble resources
* @pprm: pble resource manager
@@ -2010,6 +1989,7 @@
struct list_head *chunk_entry = (&pprm->clist)->next;
u32 offset;
unsigned long flags;
+
*vaddr = NULL;
*fpm_addr = 0;
diff --git a/sys/dev/irdma/irdma_verbs.h b/sys/dev/irdma/irdma_verbs.h
--- a/sys/dev/irdma/irdma_verbs.h
+++ b/sys/dev/irdma/irdma_verbs.h
@@ -48,13 +48,7 @@
struct irdma_ucontext {
struct ib_ucontext ibucontext;
struct irdma_device *iwdev;
-#if __FreeBSD_version >= 1400026
struct rdma_user_mmap_entry *db_mmap_entry;
-#else
- struct irdma_user_mmap_entry *db_mmap_entry;
- DECLARE_HASHTABLE(mmap_hash_tbl, 6);
- spinlock_t mmap_tbl_lock; /* protect mmap hash table entries */
-#endif
struct list_head cq_reg_mem_list;
spinlock_t cq_reg_mem_list_lock; /* protect CQ memory list */
struct list_head qp_reg_mem_list;
@@ -215,13 +209,8 @@
struct irdma_cq *iwscq;
struct irdma_cq *iwrcq;
struct irdma_pd *iwpd;
-#if __FreeBSD_version >= 1400026
struct rdma_user_mmap_entry *push_wqe_mmap_entry;
struct rdma_user_mmap_entry *push_db_mmap_entry;
-#else
- struct irdma_user_mmap_entry *push_wqe_mmap_entry;
- struct irdma_user_mmap_entry *push_db_mmap_entry;
-#endif
struct irdma_qp_host_ctx_info ctx_info;
union {
struct irdma_iwarp_offload_info iwarp_info;
@@ -263,7 +252,7 @@
struct irdma_dma_mem host_ctx;
struct timer_list terminate_timer;
struct irdma_pbl *iwpbl;
- struct irdma_sge *sg_list;
+ struct ib_sge *sg_list;
struct irdma_dma_mem q2_ctx_mem;
struct irdma_dma_mem ietf_mem;
struct completion free_qp;
@@ -292,13 +281,7 @@
};
struct irdma_user_mmap_entry {
-#if __FreeBSD_version >= 1400026
struct rdma_user_mmap_entry rdma_entry;
-#else
- struct irdma_ucontext *ucontext;
- struct hlist_node hlist;
- u64 pgoff_key; /* Used to compute offset (in bytes) returned to user libc's mmap */
-#endif
u64 bar_offset;
u8 mmap_flag;
};
@@ -399,16 +382,9 @@
ether_addr_copy(mac, mac6);
}
-#if __FreeBSD_version >= 1400026
struct rdma_user_mmap_entry*
irdma_user_mmap_entry_insert(struct irdma_ucontext *ucontext, u64 bar_offset,
enum irdma_mmap_flag mmap_flag, u64 *mmap_offset);
-#else
-struct irdma_user_mmap_entry *
-irdma_user_mmap_entry_add_hash(struct irdma_ucontext *ucontext, u64 bar_offset,
- enum irdma_mmap_flag mmap_flag, u64 *mmap_offset);
-void irdma_user_mmap_entry_del_hash(struct irdma_user_mmap_entry *entry);
-#endif
int irdma_ib_register_device(struct irdma_device *iwdev);
void irdma_ib_unregister_device(struct irdma_device *iwdev);
void irdma_ib_qp_event(struct irdma_qp *iwqp, enum irdma_qp_event_type event);
diff --git a/sys/dev/irdma/irdma_verbs.c b/sys/dev/irdma/irdma_verbs.c
--- a/sys/dev/irdma/irdma_verbs.c
+++ b/sys/dev/irdma/irdma_verbs.c
@@ -72,7 +72,6 @@
props->max_cq = rf->max_cq - rf->used_cqs;
props->max_cqe = rf->max_cqe - 1;
props->max_mr = rf->max_mr - rf->used_mrs;
- props->max_mw = props->max_mr;
props->max_pd = rf->max_pd - rf->used_pds;
props->max_sge_rd = hw_attrs->uk_attrs.max_hw_read_sges;
props->max_qp_rd_atom = hw_attrs->max_hw_ird;
@@ -107,16 +106,10 @@
pfn = ((uintptr_t)ucontext->iwdev->rf->sc_dev.hw_regs[IRDMA_DB_ADDR_OFFSET] +
pci_resource_start(ucontext->iwdev->rf->pcidev, 0)) >> PAGE_SHIFT;
-#if __FreeBSD_version >= 1400026
return rdma_user_mmap_io(&ucontext->ibucontext, vma, pfn, PAGE_SIZE,
pgprot_noncached(vma->vm_page_prot), NULL);
-#else
- return rdma_user_mmap_io(&ucontext->ibucontext, vma, pfn, PAGE_SIZE,
- pgprot_noncached(vma->vm_page_prot));
-#endif
}
-#if __FreeBSD_version >= 1400026
static void
irdma_mmap_free(struct rdma_user_mmap_entry *rdma_entry)
{
@@ -149,104 +142,6 @@
return &entry->rdma_entry;
}
-#else
-static inline bool
-find_key_in_mmap_tbl(struct irdma_ucontext *ucontext, u64 key)
-{
- struct irdma_user_mmap_entry *entry;
-
- HASH_FOR_EACH_POSSIBLE(ucontext->mmap_hash_tbl, entry, hlist, key) {
- if (entry->pgoff_key == key)
- return true;
- }
-
- return false;
-}
-
-struct irdma_user_mmap_entry *
-irdma_user_mmap_entry_add_hash(struct irdma_ucontext *ucontext, u64 bar_offset,
- enum irdma_mmap_flag mmap_flag, u64 *mmap_offset)
-{
- struct irdma_user_mmap_entry *entry = kzalloc(sizeof(*entry), GFP_KERNEL);
- unsigned long flags;
- int retry_cnt = 0;
-
- if (!entry)
- return NULL;
-
- entry->bar_offset = bar_offset;
- entry->mmap_flag = mmap_flag;
- entry->ucontext = ucontext;
- do {
- get_random_bytes(&entry->pgoff_key, sizeof(entry->pgoff_key));
-
- /* The key is a page offset */
- entry->pgoff_key >>= PAGE_SHIFT;
-
- /* In the event of a collision in the hash table, retry a new key */
- spin_lock_irqsave(&ucontext->mmap_tbl_lock, flags);
- if (!find_key_in_mmap_tbl(ucontext, entry->pgoff_key)) {
- HASH_ADD(ucontext->mmap_hash_tbl, &entry->hlist, entry->pgoff_key);
- spin_unlock_irqrestore(&ucontext->mmap_tbl_lock, flags);
- goto hash_add_done;
- }
- spin_unlock_irqrestore(&ucontext->mmap_tbl_lock, flags);
- } while (retry_cnt++ < 10);
-
- irdma_debug(&ucontext->iwdev->rf->sc_dev, IRDMA_DEBUG_VERBS,
- "mmap table add failed: Cannot find a unique key\n");
- kfree(entry);
- return NULL;
-
-hash_add_done:
- /* libc mmap uses a byte offset */
- *mmap_offset = entry->pgoff_key << PAGE_SHIFT;
-
- return entry;
-}
-
-static struct irdma_user_mmap_entry *
-irdma_find_user_mmap_entry(struct irdma_ucontext *ucontext,
- struct vm_area_struct *vma)
-{
- struct irdma_user_mmap_entry *entry;
- unsigned long flags;
-
- if (vma->vm_end - vma->vm_start != PAGE_SIZE)
- return NULL;
-
- spin_lock_irqsave(&ucontext->mmap_tbl_lock, flags);
- HASH_FOR_EACH_POSSIBLE(ucontext->mmap_hash_tbl, entry, hlist, vma->vm_pgoff) {
- if (entry->pgoff_key == vma->vm_pgoff) {
- spin_unlock_irqrestore(&ucontext->mmap_tbl_lock, flags);
- return entry;
- }
- }
-
- spin_unlock_irqrestore(&ucontext->mmap_tbl_lock, flags);
-
- return NULL;
-}
-
-void
-irdma_user_mmap_entry_del_hash(struct irdma_user_mmap_entry *entry)
-{
- struct irdma_ucontext *ucontext;
- unsigned long flags;
-
- if (!entry)
- return;
-
- ucontext = entry->ucontext;
-
- spin_lock_irqsave(&ucontext->mmap_tbl_lock, flags);
- HASH_DEL(ucontext->mmap_hash_tbl, &entry->hlist);
- spin_unlock_irqrestore(&ucontext->mmap_tbl_lock, flags);
-
- kfree(entry);
-}
-
-#endif
/**
* irdma_mmap - user memory map
* @context: context created during alloc
@@ -255,9 +150,7 @@
static int
irdma_mmap(struct ib_ucontext *context, struct vm_area_struct *vma)
{
-#if __FreeBSD_version >= 1400026
struct rdma_user_mmap_entry *rdma_entry;
-#endif
struct irdma_user_mmap_entry *entry;
struct irdma_ucontext *ucontext;
u64 pfn;
@@ -269,7 +162,6 @@
if (ucontext->legacy_mode)
return irdma_mmap_legacy(ucontext, vma);
-#if __FreeBSD_version >= 1400026
rdma_entry = rdma_user_mmap_entry_get(&ucontext->ibucontext, vma);
if (!rdma_entry) {
irdma_debug(&ucontext->iwdev->rf->sc_dev, IRDMA_DEBUG_VERBS,
@@ -279,15 +171,6 @@
}
entry = to_irdma_mmap_entry(rdma_entry);
-#else
- entry = irdma_find_user_mmap_entry(ucontext, vma);
- if (!entry) {
- irdma_debug(&ucontext->iwdev->rf->sc_dev, IRDMA_DEBUG_VERBS,
- "pgoff[0x%lx] does not have valid entry\n",
- vma->vm_pgoff);
- return -EINVAL;
- }
-#endif
irdma_debug(&ucontext->iwdev->rf->sc_dev, IRDMA_DEBUG_VERBS,
"bar_offset [0x%lx] mmap_flag [%d]\n", entry->bar_offset,
entry->mmap_flag);
@@ -297,24 +180,14 @@
switch (entry->mmap_flag) {
case IRDMA_MMAP_IO_NC:
-#if __FreeBSD_version >= 1400026
ret = rdma_user_mmap_io(context, vma, pfn, PAGE_SIZE,
pgprot_noncached(vma->vm_page_prot),
rdma_entry);
-#else
- ret = rdma_user_mmap_io(context, vma, pfn, PAGE_SIZE,
- pgprot_noncached(vma->vm_page_prot));
-#endif
break;
case IRDMA_MMAP_IO_WC:
-#if __FreeBSD_version >= 1400026
ret = rdma_user_mmap_io(context, vma, pfn, PAGE_SIZE,
pgprot_writecombine(vma->vm_page_prot),
rdma_entry);
-#else
- ret = rdma_user_mmap_io(context, vma, pfn, PAGE_SIZE,
- pgprot_writecombine(vma->vm_page_prot));
-#endif
break;
default:
ret = -EINVAL;
@@ -324,9 +197,7 @@
irdma_debug(&ucontext->iwdev->rf->sc_dev, IRDMA_DEBUG_VERBS,
"bar_offset [0x%lx] mmap_flag[%d] err[%d]\n",
entry->bar_offset, entry->mmap_flag, ret);
-#if __FreeBSD_version >= 1400026
rdma_user_mmap_entry_put(rdma_entry);
-#endif
return ret;
}
@@ -428,19 +299,11 @@
irdma_remove_push_mmap_entries(struct irdma_qp *iwqp)
{
if (iwqp->push_db_mmap_entry) {
-#if __FreeBSD_version >= 1400026
rdma_user_mmap_entry_remove(iwqp->push_db_mmap_entry);
-#else
- irdma_user_mmap_entry_del_hash(iwqp->push_db_mmap_entry);
-#endif
iwqp->push_db_mmap_entry = NULL;
}
if (iwqp->push_wqe_mmap_entry) {
-#if __FreeBSD_version >= 1400026
rdma_user_mmap_entry_remove(iwqp->push_wqe_mmap_entry);
-#else
- irdma_user_mmap_entry_del_hash(iwqp->push_wqe_mmap_entry);
-#endif
iwqp->push_wqe_mmap_entry = NULL;
}
}
@@ -458,36 +321,19 @@
bar_off = irdma_compute_push_wqe_offset(iwdev, iwqp->sc_qp.push_idx);
-#if __FreeBSD_version >= 1400026
iwqp->push_wqe_mmap_entry = irdma_user_mmap_entry_insert(ucontext,
bar_off, IRDMA_MMAP_IO_WC,
push_wqe_mmap_key);
-#else
- iwqp->push_wqe_mmap_entry = irdma_user_mmap_entry_add_hash(ucontext, bar_off,
- IRDMA_MMAP_IO_WC,
- push_wqe_mmap_key);
-#endif
if (!iwqp->push_wqe_mmap_entry)
return -ENOMEM;
/* push doorbell page */
bar_off += IRDMA_HW_PAGE_SIZE;
-#if __FreeBSD_version >= 1400026
iwqp->push_db_mmap_entry = irdma_user_mmap_entry_insert(ucontext,
bar_off, IRDMA_MMAP_IO_NC,
push_db_mmap_key);
-#else
-
- iwqp->push_db_mmap_entry = irdma_user_mmap_entry_add_hash(ucontext, bar_off,
- IRDMA_MMAP_IO_NC,
- push_db_mmap_key);
-#endif
if (!iwqp->push_db_mmap_entry) {
-#if __FreeBSD_version >= 1400026
rdma_user_mmap_entry_remove(iwqp->push_wqe_mmap_entry);
-#else
- irdma_user_mmap_entry_del_hash(iwqp->push_wqe_mmap_entry);
-#endif
return -ENOMEM;
}
@@ -535,11 +381,7 @@
struct irdma_qp_init_info *info,
struct ib_qp_init_attr *init_attr)
{
-#if __FreeBSD_version >= 1400026
struct irdma_ucontext *ucontext = rdma_udata_to_drv_context(udata, struct irdma_ucontext, ibucontext);
-#else
- struct irdma_ucontext *ucontext = to_ucontext(iwqp->iwpd->ibpd.uobject->context);
-#endif
struct irdma_qp_uk_init_info *ukinfo = &info->qp_uk_init_info;
struct irdma_create_qp_req req = {0};
unsigned long flags;
@@ -598,6 +440,9 @@
ukinfo->sq_size = ukinfo->sq_depth >> ukinfo->sq_shift;
ukinfo->rq_size = ukinfo->rq_depth >> ukinfo->rq_shift;
}
+ if (req.comp_mask & IRDMA_CREATE_QP_USE_START_WQE_IDX &&
+ iwdev->rf->sc_dev.hw_attrs.uk_attrs.feature_flags & IRDMA_FEATURE_RTS_AE)
+ ukinfo->start_wqe_idx = 4;
irdma_setup_virt_qp(iwdev, iwqp, info);
return 0;
@@ -740,7 +585,6 @@
roce_info->rd_en = true;
roce_info->wr_rdresp_en = true;
- roce_info->bind_en = true;
roce_info->dcqcn_en = false;
roce_info->rtomin = iwdev->roce_rtomin;
@@ -772,7 +616,6 @@
ether_addr_copy(iwarp_info->mac_addr, if_getlladdr(iwdev->netdev));
iwarp_info->rd_en = true;
iwarp_info->wr_rdresp_en = true;
- iwarp_info->bind_en = true;
iwarp_info->ecn_en = true;
iwarp_info->rtomin = 5;
@@ -803,6 +646,8 @@
if (init_attr->cap.max_inline_data > uk_attrs->max_hw_inline ||
init_attr->cap.max_send_sge > uk_attrs->max_hw_wq_frags ||
+ init_attr->cap.max_send_wr > uk_attrs->max_hw_wq_quanta ||
+ init_attr->cap.max_recv_wr > uk_attrs->max_hw_rq_quanta ||
init_attr->cap.max_recv_sge > uk_attrs->max_hw_wq_frags)
return -EINVAL;
@@ -857,8 +702,6 @@
}
if (iwqp->roce_info.rd_en)
acc_flags |= IB_ACCESS_REMOTE_READ;
- if (iwqp->roce_info.bind_en)
- acc_flags |= IB_ACCESS_MW_BIND;
} else {
if (iwqp->iwarp_info.wr_rdresp_en) {
acc_flags |= IB_ACCESS_LOCAL_WRITE;
@@ -866,8 +709,6 @@
}
if (iwqp->iwarp_info.rd_en)
acc_flags |= IB_ACCESS_REMOTE_READ;
- if (iwqp->iwarp_info.bind_en)
- acc_flags |= IB_ACCESS_MW_BIND;
}
return acc_flags;
}
@@ -1267,11 +1108,7 @@
if (udata && udata->outlen && dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2) {
struct irdma_ucontext *ucontext;
-#if __FreeBSD_version >= 1400026
ucontext = rdma_udata_to_drv_context(udata, struct irdma_ucontext, ibucontext);
-#else
- ucontext = to_ucontext(ibqp->uobject->context);
-#endif
if (iwqp->sc_qp.push_idx != IRDMA_INVALID_PUSH_PAGE_INDEX &&
!iwqp->push_wqe_mmap_entry &&
!irdma_setup_push_mmap_entries(ucontext, iwqp,
@@ -1284,7 +1121,8 @@
udata->outlen));
if (ret) {
irdma_remove_push_mmap_entries(iwqp);
- irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_VERBS,
+ irdma_debug(&iwdev->rf->sc_dev,
+ IRDMA_DEBUG_VERBS,
"copy_to_udata failed\n");
return ret;
}
@@ -1517,11 +1355,7 @@
dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2) {
struct irdma_ucontext *ucontext;
-#if __FreeBSD_version >= 1400026
ucontext = rdma_udata_to_drv_context(udata, struct irdma_ucontext, ibucontext);
-#else
- ucontext = to_ucontext(ibqp->uobject->context);
-#endif
if (iwqp->sc_qp.push_idx != IRDMA_INVALID_PUSH_PAGE_INDEX &&
!iwqp->push_wqe_mmap_entry &&
!irdma_setup_push_mmap_entries(ucontext, iwqp,
@@ -1660,11 +1494,7 @@
if (udata) {
struct irdma_resize_cq_req req = {};
struct irdma_ucontext *ucontext =
-#if __FreeBSD_version >= 1400026
rdma_udata_to_drv_context(udata, struct irdma_ucontext, ibucontext);
-#else
- to_ucontext(ibcq->uobject->context);
-#endif
/* CQ resize not supported with legacy GEN_1 libi40iw */
if (ucontext->legacy_mode)
@@ -1758,8 +1588,10 @@
/**
* irdma_get_mr_access - get hw MR access permissions from IB access flags
* @access: IB access flags
+ * @hw_rev: Hardware version
*/
-static inline u16 irdma_get_mr_access(int access){
+static inline u16 irdma_get_mr_access(int access, u8 hw_rev)
+{
u16 hw_access = 0;
hw_access |= (access & IB_ACCESS_LOCAL_WRITE) ?
@@ -1768,8 +1600,6 @@
IRDMA_ACCESS_FLAGS_REMOTEWRITE : 0;
hw_access |= (access & IB_ACCESS_REMOTE_READ) ?
IRDMA_ACCESS_FLAGS_REMOTEREAD : 0;
- hw_access |= (access & IB_ACCESS_MW_BIND) ?
- IRDMA_ACCESS_FLAGS_BIND_WINDOW : 0;
hw_access |= (access & IB_ZERO_BASED) ?
IRDMA_ACCESS_FLAGS_ZERO_BASED : 0;
hw_access |= IRDMA_ACCESS_FLAGS_LOCALREAD;
@@ -2004,81 +1834,6 @@
return err;
}
-/**
- * irdma_hw_alloc_mw - create the hw memory window
- * @iwdev: irdma device
- * @iwmr: pointer to memory window info
- */
-int
-irdma_hw_alloc_mw(struct irdma_device *iwdev, struct irdma_mr *iwmr)
-{
- struct irdma_mw_alloc_info *info;
- struct irdma_pd *iwpd = to_iwpd(iwmr->ibmr.pd);
- struct irdma_cqp_request *cqp_request;
- struct cqp_cmds_info *cqp_info;
- int status;
-
- cqp_request = irdma_alloc_and_get_cqp_request(&iwdev->rf->cqp, true);
- if (!cqp_request)
- return -ENOMEM;
-
- cqp_info = &cqp_request->info;
- info = &cqp_info->in.u.mw_alloc.info;
- memset(info, 0, sizeof(*info));
- if (iwmr->ibmw.type == IB_MW_TYPE_1)
- info->mw_wide = true;
-
- info->page_size = PAGE_SIZE;
- info->mw_stag_index = iwmr->stag >> IRDMA_CQPSQ_STAG_IDX_S;
- info->pd_id = iwpd->sc_pd.pd_id;
- info->remote_access = true;
- cqp_info->cqp_cmd = IRDMA_OP_MW_ALLOC;
- cqp_info->post_sq = 1;
- cqp_info->in.u.mw_alloc.dev = &iwdev->rf->sc_dev;
- cqp_info->in.u.mw_alloc.scratch = (uintptr_t)cqp_request;
- status = irdma_handle_cqp_op(iwdev->rf, cqp_request);
- irdma_put_cqp_request(&iwdev->rf->cqp, cqp_request);
-
- return status;
-}
-
-/**
- * irdma_dealloc_mw - Dealloc memory window
- * @ibmw: memory window structure.
- */
-static int
-irdma_dealloc_mw(struct ib_mw *ibmw)
-{
- struct ib_pd *ibpd = ibmw->pd;
- struct irdma_pd *iwpd = to_iwpd(ibpd);
- struct irdma_mr *iwmr = to_iwmr((struct ib_mr *)ibmw);
- struct irdma_device *iwdev = to_iwdev(ibmw->device);
- struct irdma_cqp_request *cqp_request;
- struct cqp_cmds_info *cqp_info;
- struct irdma_dealloc_stag_info *info;
-
- cqp_request = irdma_alloc_and_get_cqp_request(&iwdev->rf->cqp, true);
- if (!cqp_request)
- return -ENOMEM;
-
- cqp_info = &cqp_request->info;
- info = &cqp_info->in.u.dealloc_stag.info;
- memset(info, 0, sizeof(*info));
- info->pd_id = iwpd->sc_pd.pd_id;
- info->stag_idx = RS_64_1(ibmw->rkey, IRDMA_CQPSQ_STAG_IDX_S);
- info->mr = false;
- cqp_info->cqp_cmd = IRDMA_OP_DEALLOC_STAG;
- cqp_info->post_sq = 1;
- cqp_info->in.u.dealloc_stag.dev = &iwdev->rf->sc_dev;
- cqp_info->in.u.dealloc_stag.scratch = (uintptr_t)cqp_request;
- irdma_handle_cqp_op(iwdev->rf, cqp_request);
- irdma_put_cqp_request(&iwdev->rf->cqp, cqp_request);
- irdma_free_stag(iwdev, iwmr->stag);
- kfree(iwmr);
-
- return 0;
-}
-
/**
* irdma_hw_alloc_stag - cqp command to allocate stag
* @iwdev: irdma device
@@ -2199,7 +1954,8 @@
stag_info->stag_key = (u8)iwmr->stag;
stag_info->total_len = iwmr->len;
stag_info->all_memory = (pd->flags & IB_PD_UNSAFE_GLOBAL_RKEY) ? true : false;
- stag_info->access_rights = irdma_get_mr_access(access);
+ stag_info->access_rights = irdma_get_mr_access(access,
+ iwdev->rf->sc_dev.hw_attrs.uk_attrs.hw_rev);
stag_info->pd_id = iwpd->sc_pd.pd_id;
if (stag_info->access_rights & IRDMA_ACCESS_FLAGS_ZERO_BASED)
stag_info->addr_type = IRDMA_ADDR_TYPE_ZERO_BASED;
@@ -2273,14 +2029,18 @@
}
/*
- * irdma_reg_user_mr_type_mem - Handle memory registration @iwmr - irdma mr @access - access rights
+ * irdma_reg_user_mr_type_mem - Handle memory registration
+ * @iwmr - irdma mr
+ * @access - access rights
+ * @create_stag - flag to create stag or not
*/
static int
-irdma_reg_user_mr_type_mem(struct irdma_mr *iwmr, int access)
+irdma_reg_user_mr_type_mem(struct irdma_mr *iwmr, int access,
+ bool create_stag)
{
struct irdma_device *iwdev = to_iwdev(iwmr->ibmr.device);
struct irdma_pbl *iwpbl = &iwmr->iwpbl;
- u32 stag;
+ u32 stag = 0;
int err;
u8 lvl;
@@ -2299,15 +2059,17 @@
}
}
- stag = irdma_create_stag(iwdev);
- if (!stag) {
- err = -ENOMEM;
- goto free_pble;
- }
+ if (create_stag) {
+ stag = irdma_create_stag(iwdev);
+ if (!stag) {
+ err = -ENOMEM;
+ goto free_pble;
+ }
- iwmr->stag = stag;
- iwmr->ibmr.rkey = stag;
- iwmr->ibmr.lkey = stag;
+ iwmr->stag = stag;
+ iwmr->ibmr.rkey = stag;
+ iwmr->ibmr.lkey = stag;
+ }
iwmr->access = access;
err = irdma_hwreg_mr(iwdev, iwmr, access);
if (err)
@@ -2316,7 +2078,8 @@
return 0;
err_hwreg:
- irdma_free_stag(iwdev, stag);
+ if (stag)
+ irdma_free_stag(iwdev, stag);
free_pble:
if (iwpbl->pble_alloc.level != PBLE_LEVEL_0 && iwpbl->pbl_allocated)
@@ -2351,11 +2114,7 @@
if (err)
return err;
-#if __FreeBSD_version >= 1400026
ucontext = rdma_udata_to_drv_context(udata, struct irdma_ucontext, ibucontext);
-#else
- ucontext = to_ucontext(iwmr->ibpd.pd->uobject->context);
-#endif
spin_lock_irqsave(&ucontext->qp_reg_mem_list_lock, flags);
list_add_tail(&iwpbl->list, &ucontext->qp_reg_mem_list);
iwpbl->on_list = true;
@@ -2390,11 +2149,7 @@
if (err)
return err;
-#if __FreeBSD_version >= 1400026
ucontext = rdma_udata_to_drv_context(udata, struct irdma_ucontext, ibucontext);
-#else
- ucontext = to_ucontext(iwmr->ibmr.pd->uobject->context);
-#endif
spin_lock_irqsave(&ucontext->cq_reg_mem_list_lock, flags);
list_add_tail(&iwpbl->list, &ucontext->cq_reg_mem_list);
iwpbl->on_list = true;
@@ -2463,7 +2218,7 @@
break;
case IRDMA_MEMREG_TYPE_MEM:
- err = irdma_reg_user_mr_type_mem(iwmr, access);
+ err = irdma_reg_user_mr_type_mem(iwmr, access, true);
if (err)
goto error;
@@ -2540,10 +2295,8 @@
{
struct irdma_device *iwdev = to_iwdev(iwmr->ibmr.device);
struct irdma_pbl *iwpbl = &iwmr->iwpbl;
- struct irdma_pble_alloc *palloc = &iwpbl->pble_alloc;
struct ib_pd *pd = iwmr->ibmr.pd;
struct ib_umem *region;
- u8 lvl;
int err;
region = ib_umem_get(pd->uobject->context, start, len, iwmr->access, 0);
@@ -2564,35 +2317,14 @@
iwmr->page_cnt = irdma_ib_umem_num_dma_blocks(region, iwmr->page_size,
virt);
- lvl = iwmr->page_cnt != 1 ? PBLE_LEVEL_1 | PBLE_LEVEL_2 : PBLE_LEVEL_0;
-
- err = irdma_setup_pbles(iwdev->rf, iwmr, lvl);
- if (err)
- goto error;
-
- if (lvl) {
- err = irdma_check_mr_contiguous(palloc,
- iwmr->page_size);
- if (err) {
- irdma_free_pble(iwdev->rf->pble_rsrc, palloc);
- iwpbl->pbl_allocated = false;
- }
- }
-
- err = irdma_hwreg_mr(iwdev, iwmr, iwmr->access);
+ err = irdma_reg_user_mr_type_mem(iwmr, iwmr->access, false);
if (err)
- goto error;
+ goto err;
return &iwmr->ibmr;
-error:
- if (palloc->level != PBLE_LEVEL_0 && iwpbl->pbl_allocated) {
- irdma_free_pble(iwdev->rf->pble_rsrc, palloc);
- iwpbl->pbl_allocated = false;
- }
+err:
ib_umem_release(region);
- iwmr->region = NULL;
-
return ERR_PTR(err);
}
@@ -2699,25 +2431,6 @@
}
}
-/**
- * irdma_copy_sg_list - copy sg list for qp
- * @sg_list: copied into sg_list
- * @sgl: copy from sgl
- * @num_sges: count of sg entries
- */
-static void
-irdma_copy_sg_list(struct irdma_sge *sg_list, struct ib_sge *sgl,
- int num_sges)
-{
- unsigned int i;
-
- for (i = 0; i < num_sges; i++) {
- sg_list[i].tag_off = sgl[i].addr;
- sg_list[i].len = sgl[i].length;
- sg_list[i].stag = sgl[i].lkey;
- }
-}
-
/**
* irdma_post_send - kernel application wr
* @ibqp: qp ptr for wr
@@ -2778,7 +2491,7 @@
}
info.op.send.num_sges = ib_wr->num_sge;
- info.op.send.sg_list = (struct irdma_sge *)ib_wr->sg_list;
+ info.op.send.sg_list = ib_wr->sg_list;
if (iwqp->ibqp.qp_type == IB_QPT_UD ||
iwqp->ibqp.qp_type == IB_QPT_GSI) {
ah = to_iwah(ud_wr(ib_wr)->ah);
@@ -2809,8 +2522,8 @@
info.op.rdma_write.num_lo_sges = ib_wr->num_sge;
info.op.rdma_write.lo_sg_list = (void *)ib_wr->sg_list;
- info.op.rdma_write.rem_addr.tag_off = rdma_wr(ib_wr)->remote_addr;
- info.op.rdma_write.rem_addr.stag = rdma_wr(ib_wr)->rkey;
+ info.op.rdma_write.rem_addr.addr = rdma_wr(ib_wr)->remote_addr;
+ info.op.rdma_write.rem_addr.lkey = rdma_wr(ib_wr)->rkey;
if (ib_wr->send_flags & IB_SEND_INLINE)
err = irdma_uk_inline_rdma_write(ukqp, &info, false);
else
@@ -2826,8 +2539,8 @@
break;
}
info.op_type = IRDMA_OP_TYPE_RDMA_READ;
- info.op.rdma_read.rem_addr.tag_off = rdma_wr(ib_wr)->remote_addr;
- info.op.rdma_read.rem_addr.stag = rdma_wr(ib_wr)->rkey;
+ info.op.rdma_read.rem_addr.addr = rdma_wr(ib_wr)->remote_addr;
+ info.op.rdma_read.rem_addr.lkey = rdma_wr(ib_wr)->rkey;
info.op.rdma_read.lo_sg_list = (void *)ib_wr->sg_list;
info.op.rdma_read.num_lo_sges = ib_wr->num_sge;
err = irdma_uk_rdma_read(ukqp, &info, inv_stag, false);
@@ -2845,7 +2558,9 @@
stag_info.signaled = info.signaled;
stag_info.read_fence = info.read_fence;
- stag_info.access_rights = irdma_get_mr_access(reg_wr(ib_wr)->access);
+ stag_info.access_rights =
+ irdma_get_mr_access(reg_wr(ib_wr)->access,
+ dev->hw_attrs.uk_attrs.hw_rev);
stag_info.stag_key = reg_wr(ib_wr)->key & 0xff;
stag_info.stag_idx = reg_wr(ib_wr)->key >> 8;
stag_info.page_size = reg_wr(ib_wr)->mr->page_size;
@@ -2907,7 +2622,6 @@
struct irdma_qp *iwqp = to_iwqp(ibqp);
struct irdma_qp_uk *ukqp = &iwqp->sc_qp.qp_uk;
struct irdma_post_rq_info post_recv = {0};
- struct irdma_sge *sg_list = iwqp->sg_list;
unsigned long flags;
int err = 0;
@@ -2920,8 +2634,7 @@
}
post_recv.num_sges = ib_wr->num_sge;
post_recv.wr_id = ib_wr->wr_id;
- irdma_copy_sg_list(sg_list, ib_wr->sg_list, ib_wr->num_sge);
- post_recv.sg_list = sg_list;
+ post_recv.sg_list = ib_wr->sg_list;
err = irdma_uk_post_receive(ukqp, &post_recv);
if (err) {
irdma_debug(&iwqp->iwdev->rf->sc_dev, IRDMA_DEBUG_VERBS,
@@ -3336,7 +3049,7 @@
if (!ipv6_addr_v4mapped((struct in6_addr *)ibgid)) {
irdma_copy_ip_ntohl(ip_addr,
sgid_addr.saddr_in6.sin6_addr.__u6_addr.__u6_addr32);
- irdma_netdev_vlan_ipv6(iwqp->cm_id, ip_addr, &vlan_id, NULL);
+ irdma_get_vlan_mac_ipv6(iwqp->cm_id, ip_addr, &vlan_id, NULL);
ipv4 = false;
irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_VERBS,
"qp_id=%d, IP6address=%x:%x:%x:%x\n", ibqp->qp_num,
@@ -3539,9 +3252,7 @@
return 0;
}
-static if_t
-irdma_get_netdev(struct ib_device *ibdev, u8 port_num)
-{
+static if_t irdma_get_netdev(struct ib_device *ibdev, u8 port_num){
struct irdma_device *iwdev = to_iwdev(ibdev);
if (iwdev->netdev) {
@@ -3557,7 +3268,6 @@
{
struct ib_device *dev_ops = ibdev;
-#if __FreeBSD_version >= 1400000
dev_ops->ops.driver_id = RDMA_DRIVER_I40IW;
dev_ops->ops.size_ib_ah = IRDMA_SET_RDMA_OBJ_SIZE(ib_ah, irdma_ah, ibah);
dev_ops->ops.size_ib_cq = IRDMA_SET_RDMA_OBJ_SIZE(ib_cq, irdma_cq, ibcq);
@@ -3566,15 +3276,12 @@
irdma_ucontext,
ibucontext);
-#endif /* __FreeBSD_version >= 1400000 */
dev_ops->alloc_hw_stats = irdma_alloc_hw_stats;
dev_ops->alloc_mr = irdma_alloc_mr;
- dev_ops->alloc_mw = irdma_alloc_mw;
dev_ops->alloc_pd = irdma_alloc_pd;
dev_ops->alloc_ucontext = irdma_alloc_ucontext;
dev_ops->create_cq = irdma_create_cq;
dev_ops->create_qp = irdma_create_qp;
- dev_ops->dealloc_mw = irdma_dealloc_mw;
dev_ops->dealloc_pd = irdma_dealloc_pd;
dev_ops->dealloc_ucontext = irdma_dealloc_ucontext;
dev_ops->dereg_mr = irdma_dereg_mr;
@@ -3587,9 +3294,7 @@
dev_ops->get_netdev = irdma_get_netdev;
dev_ops->map_mr_sg = irdma_map_mr_sg;
dev_ops->mmap = irdma_mmap;
-#if __FreeBSD_version >= 1400026
dev_ops->mmap_free = irdma_mmap_free;
-#endif
dev_ops->poll_cq = irdma_poll_cq;
dev_ops->post_recv = irdma_post_recv;
dev_ops->post_send = irdma_post_send;
@@ -3607,6 +3312,7 @@
irdma_set_device_mcast_ops(struct ib_device *ibdev)
{
struct ib_device *dev_ops = ibdev;
+
dev_ops->attach_mcast = irdma_attach_mcast;
dev_ops->detach_mcast = irdma_detach_mcast;
}
@@ -3615,6 +3321,7 @@
irdma_set_device_roce_ops(struct ib_device *ibdev)
{
struct ib_device *dev_ops = ibdev;
+
dev_ops->create_ah = irdma_create_ah;
dev_ops->destroy_ah = irdma_destroy_ah;
dev_ops->get_link_layer = irdma_get_link_layer;
diff --git a/sys/dev/irdma/irdma_ws.h b/sys/dev/irdma/irdma_ws.h
--- a/sys/dev/irdma/irdma_ws.h
+++ b/sys/dev/irdma/irdma_ws.h
@@ -1,7 +1,7 @@
/*-
* SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB
*
- * Copyright (c) 2015 - 2022 Intel Corporation
+ * Copyright (c) 2015 - 2023 Intel Corporation
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
diff --git a/sys/dev/irdma/irdma_ws.c b/sys/dev/irdma/irdma_ws.c
--- a/sys/dev/irdma/irdma_ws.c
+++ b/sys/dev/irdma/irdma_ws.c
@@ -57,7 +57,7 @@
struct irdma_ws_node *node;
u16 node_index = 0;
- ws_mem.size = sizeof(struct irdma_ws_node);
+ ws_mem.size = sizeof(*node);
ws_mem.va = kzalloc(ws_mem.size, GFP_KERNEL);
if (!ws_mem.va)
return NULL;
@@ -109,7 +109,7 @@
irdma_free_ws_node_id(vsi->dev, node->index);
ws_mem.va = node;
- ws_mem.size = sizeof(struct irdma_ws_node);
+ ws_mem.size = sizeof(*node);
kfree(ws_mem.va);
}
diff --git a/sys/dev/irdma/osdep.h b/sys/dev/irdma/osdep.h
--- a/sys/dev/irdma/osdep.h
+++ b/sys/dev/irdma/osdep.h
@@ -85,7 +85,7 @@
#define STATS_TIMER_DELAY 60000
/* a couple of linux size defines */
-#define SZ_128 128
+#define SZ_128 128
#define SPEED_1000 1000
#define SPEED_10000 10000
#define SPEED_20000 20000
@@ -95,17 +95,11 @@
#define irdma_mb() mb()
#define irdma_wmb() wmb()
-#ifndef smp_mb
-#define smp_mb() mb()
-#endif
#define irdma_get_virt_to_phy vtophys
#define __aligned_u64 uint64_t __aligned(8)
#define VLAN_PRIO_SHIFT 13
-#if __FreeBSD_version < 1400000
-#define IB_USER_VERBS_EX_CMD_MODIFY_QP IB_USER_VERBS_CMD_MODIFY_QP
-#endif
/*
* debug definition section