Changeset View
Changeset View
Standalone View
Standalone View
contrib/ofed/libmlx5/verbs.c
Show First 20 Lines • Show All 425 Lines • ▼ Show 20 Lines | static struct ibv_cq_ex *create_cq(struct ibv_context *context, | ||||
if (cq_attr->comp_mask & IBV_CQ_INIT_ATTR_MASK_FLAGS && | if (cq_attr->comp_mask & IBV_CQ_INIT_ATTR_MASK_FLAGS && | ||||
cq_attr->flags & IBV_CREATE_CQ_ATTR_SINGLE_THREADED) | cq_attr->flags & IBV_CREATE_CQ_ATTR_SINGLE_THREADED) | ||||
cq->flags |= MLX5_CQ_FLAGS_SINGLE_THREADED; | cq->flags |= MLX5_CQ_FLAGS_SINGLE_THREADED; | ||||
cmd.buf_addr = (uintptr_t) cq->buf_a.buf; | cmd.buf_addr = (uintptr_t) cq->buf_a.buf; | ||||
cmd.db_addr = (uintptr_t) cq->dbrec; | cmd.db_addr = (uintptr_t) cq->dbrec; | ||||
cmd.cqe_size = cqe_sz; | cmd.cqe_size = cqe_sz; | ||||
if (mlx5cq_attr) { | if (mlx5cq_attr) { | ||||
if (mlx5cq_attr->comp_mask & ~(MLX5DV_CQ_INIT_ATTR_MASK_RESERVED - 1)) { | if (!check_comp_mask(mlx5cq_attr->comp_mask, | ||||
MLX5DV_CQ_INIT_ATTR_MASK_RESERVED - 1)) { | |||||
mlx5_dbg(fp, MLX5_DBG_CQ, | mlx5_dbg(fp, MLX5_DBG_CQ, | ||||
"Unsupported vendor comp_mask for create_cq\n"); | "Unsupported vendor comp_mask for create_cq\n"); | ||||
errno = EINVAL; | errno = EINVAL; | ||||
goto err_db; | goto err_db; | ||||
} | } | ||||
if (mlx5cq_attr->comp_mask & MLX5DV_CQ_INIT_ATTR_MASK_COMPRESSED_CQE) { | if (mlx5cq_attr->comp_mask & MLX5DV_CQ_INIT_ATTR_MASK_COMPRESSED_CQE) { | ||||
if (mctx->cqe_comp_caps.max_num && | if (mctx->cqe_comp_caps.max_num && | ||||
▲ Show 20 Lines • Show All 471 Lines • ▼ Show 20 Lines | static int mlx5_calc_sq_size(struct mlx5_context *ctx, | ||||
qp->sq.wqe_shift = mlx5_ilog2(MLX5_SEND_WQE_BB); | qp->sq.wqe_shift = mlx5_ilog2(MLX5_SEND_WQE_BB); | ||||
qp->sq.max_gs = attr->cap.max_send_sge; | qp->sq.max_gs = attr->cap.max_send_sge; | ||||
qp->sq.max_post = wq_size / wqe_size; | qp->sq.max_post = wq_size / wqe_size; | ||||
return wq_size; | return wq_size; | ||||
} | } | ||||
enum { | |||||
DV_CREATE_WQ_SUPPORTED_COMP_MASK = MLX5DV_WQ_INIT_ATTR_MASK_STRIDING_RQ | |||||
}; | |||||
static int mlx5_calc_rwq_size(struct mlx5_context *ctx, | static int mlx5_calc_rwq_size(struct mlx5_context *ctx, | ||||
struct mlx5_rwq *rwq, | struct mlx5_rwq *rwq, | ||||
struct ibv_wq_init_attr *attr) | struct ibv_wq_init_attr *attr, | ||||
struct mlx5dv_wq_init_attr *mlx5wq_attr) | |||||
{ | { | ||||
size_t wqe_size; | size_t wqe_size; | ||||
int wq_size; | int wq_size; | ||||
uint32_t num_scatter; | uint32_t num_scatter; | ||||
int is_mprq = 0; | |||||
int scat_spc; | int scat_spc; | ||||
if (!attr->max_wr) | if (!attr->max_wr) | ||||
return -EINVAL; | return -EINVAL; | ||||
if (mlx5wq_attr) { | |||||
if (!check_comp_mask(mlx5wq_attr->comp_mask, | |||||
DV_CREATE_WQ_SUPPORTED_COMP_MASK)) | |||||
return -EINVAL; | |||||
is_mprq = !!(mlx5wq_attr->comp_mask & | |||||
MLX5DV_WQ_INIT_ATTR_MASK_STRIDING_RQ); | |||||
} | |||||
/* TBD: check caps for RQ */ | /* TBD: check caps for RQ */ | ||||
num_scatter = max_t(uint32_t, attr->max_sge, 1); | num_scatter = max_t(uint32_t, attr->max_sge, 1); | ||||
wqe_size = sizeof(struct mlx5_wqe_data_seg) * num_scatter; | wqe_size = sizeof(struct mlx5_wqe_data_seg) * num_scatter + | ||||
sizeof(struct mlx5_wqe_srq_next_seg) * is_mprq; | |||||
if (rwq->wq_sig) | if (rwq->wq_sig) | ||||
wqe_size += sizeof(struct mlx5_rwqe_sig); | wqe_size += sizeof(struct mlx5_rwqe_sig); | ||||
if (wqe_size <= 0 || wqe_size > ctx->max_rq_desc_sz) | if (wqe_size <= 0 || wqe_size > ctx->max_rq_desc_sz) | ||||
return -EINVAL; | return -EINVAL; | ||||
wqe_size = mlx5_round_up_power_of_two(wqe_size); | wqe_size = mlx5_round_up_power_of_two(wqe_size); | ||||
wq_size = mlx5_round_up_power_of_two(attr->max_wr) * wqe_size; | wq_size = mlx5_round_up_power_of_two(attr->max_wr) * wqe_size; | ||||
wq_size = max(wq_size, MLX5_SEND_WQE_BB); | wq_size = max(wq_size, MLX5_SEND_WQE_BB); | ||||
rwq->rq.wqe_cnt = wq_size / wqe_size; | rwq->rq.wqe_cnt = wq_size / wqe_size; | ||||
rwq->rq.wqe_shift = mlx5_ilog2(wqe_size); | rwq->rq.wqe_shift = mlx5_ilog2(wqe_size); | ||||
rwq->rq.max_post = 1 << mlx5_ilog2(wq_size / wqe_size); | rwq->rq.max_post = 1 << mlx5_ilog2(wq_size / wqe_size); | ||||
scat_spc = wqe_size - | scat_spc = wqe_size - | ||||
((rwq->wq_sig) ? sizeof(struct mlx5_rwqe_sig) : 0); | ((rwq->wq_sig) ? sizeof(struct mlx5_rwqe_sig) : 0) - | ||||
is_mprq * sizeof(struct mlx5_wqe_srq_next_seg); | |||||
rwq->rq.max_gs = scat_spc / sizeof(struct mlx5_wqe_data_seg); | rwq->rq.max_gs = scat_spc / sizeof(struct mlx5_wqe_data_seg); | ||||
return wq_size; | return wq_size; | ||||
} | } | ||||
static int mlx5_calc_rq_size(struct mlx5_context *ctx, | static int mlx5_calc_rq_size(struct mlx5_context *ctx, | ||||
struct ibv_qp_init_attr_ex *attr, | struct ibv_qp_init_attr_ex *attr, | ||||
struct mlx5_qp *qp) | struct mlx5_qp *qp) | ||||
{ | { | ||||
▲ Show 20 Lines • Show All 1,175 Lines • ▼ Show 20 Lines | if (err) { | ||||
free(rwq->rq.wrid); | free(rwq->rq.wrid); | ||||
errno = ENOMEM; | errno = ENOMEM; | ||||
return -1; | return -1; | ||||
} | } | ||||
return 0; | return 0; | ||||
} | } | ||||
struct ibv_wq *mlx5_create_wq(struct ibv_context *context, | static struct ibv_wq *create_wq(struct ibv_context *context, | ||||
struct ibv_wq_init_attr *attr) | struct ibv_wq_init_attr *attr, | ||||
struct mlx5dv_wq_init_attr *mlx5wq_attr) | |||||
{ | { | ||||
struct mlx5_create_wq cmd; | struct mlx5_create_wq cmd; | ||||
struct mlx5_create_wq_resp resp; | struct mlx5_create_wq_resp resp; | ||||
int err; | int err; | ||||
struct mlx5_rwq *rwq; | struct mlx5_rwq *rwq; | ||||
struct mlx5_context *ctx = to_mctx(context); | struct mlx5_context *ctx = to_mctx(context); | ||||
int ret; | int ret; | ||||
int32_t usr_idx = 0; | int32_t usr_idx = 0; | ||||
FILE *fp = ctx->dbg_fp; | FILE *fp = ctx->dbg_fp; | ||||
if (attr->wq_type != IBV_WQT_RQ) | if (attr->wq_type != IBV_WQT_RQ) | ||||
return NULL; | return NULL; | ||||
memset(&cmd, 0, sizeof(cmd)); | memset(&cmd, 0, sizeof(cmd)); | ||||
memset(&resp, 0, sizeof(resp)); | memset(&resp, 0, sizeof(resp)); | ||||
rwq = calloc(1, sizeof(*rwq)); | rwq = calloc(1, sizeof(*rwq)); | ||||
if (!rwq) | if (!rwq) | ||||
return NULL; | return NULL; | ||||
rwq->wq_sig = rwq_sig_enabled(context); | rwq->wq_sig = rwq_sig_enabled(context); | ||||
if (rwq->wq_sig) | if (rwq->wq_sig) | ||||
cmd.drv.flags = MLX5_RWQ_FLAG_SIGNATURE; | cmd.drv.flags = MLX5_RWQ_FLAG_SIGNATURE; | ||||
ret = mlx5_calc_rwq_size(ctx, rwq, attr); | ret = mlx5_calc_rwq_size(ctx, rwq, attr, mlx5wq_attr); | ||||
if (ret < 0) { | if (ret < 0) { | ||||
errno = -ret; | errno = -ret; | ||||
goto err; | goto err; | ||||
} | } | ||||
rwq->buf_size = ret; | rwq->buf_size = ret; | ||||
if (mlx5_alloc_rwq_buf(context, rwq, ret)) | if (mlx5_alloc_rwq_buf(context, rwq, ret)) | ||||
goto err; | goto err; | ||||
Show All 17 Lines | static struct ibv_wq *create_wq(struct ibv_context *context, | ||||
cmd.drv.rq_wqe_shift = rwq->rq.wqe_shift; | cmd.drv.rq_wqe_shift = rwq->rq.wqe_shift; | ||||
usr_idx = mlx5_store_uidx(ctx, rwq); | usr_idx = mlx5_store_uidx(ctx, rwq); | ||||
if (usr_idx < 0) { | if (usr_idx < 0) { | ||||
mlx5_dbg(fp, MLX5_DBG_QP, "Couldn't find free user index\n"); | mlx5_dbg(fp, MLX5_DBG_QP, "Couldn't find free user index\n"); | ||||
goto err_free_db_rec; | goto err_free_db_rec; | ||||
} | } | ||||
cmd.drv.user_index = usr_idx; | cmd.drv.user_index = usr_idx; | ||||
if (mlx5wq_attr) { | |||||
if (mlx5wq_attr->comp_mask & MLX5DV_WQ_INIT_ATTR_MASK_STRIDING_RQ) { | |||||
if ((mlx5wq_attr->striding_rq_attrs.single_stride_log_num_of_bytes < | |||||
ctx->striding_rq_caps.min_single_stride_log_num_of_bytes) || | |||||
(mlx5wq_attr->striding_rq_attrs.single_stride_log_num_of_bytes > | |||||
ctx->striding_rq_caps.max_single_stride_log_num_of_bytes)) { | |||||
errno = EINVAL; | |||||
goto err_create; | |||||
} | |||||
if ((mlx5wq_attr->striding_rq_attrs.single_wqe_log_num_of_strides < | |||||
ctx->striding_rq_caps.min_single_wqe_log_num_of_strides) || | |||||
(mlx5wq_attr->striding_rq_attrs.single_wqe_log_num_of_strides > | |||||
ctx->striding_rq_caps.max_single_wqe_log_num_of_strides)) { | |||||
errno = EINVAL; | |||||
goto err_create; | |||||
} | |||||
cmd.drv.single_stride_log_num_of_bytes = | |||||
mlx5wq_attr->striding_rq_attrs.single_stride_log_num_of_bytes; | |||||
cmd.drv.single_wqe_log_num_of_strides = | |||||
mlx5wq_attr->striding_rq_attrs.single_wqe_log_num_of_strides; | |||||
cmd.drv.two_byte_shift_en = | |||||
mlx5wq_attr->striding_rq_attrs.two_byte_shift_en; | |||||
cmd.drv.comp_mask |= MLX5_IB_CREATE_WQ_STRIDING_RQ; | |||||
} | |||||
} | |||||
err = ibv_cmd_create_wq(context, attr, &rwq->wq, &cmd.ibv_cmd, | err = ibv_cmd_create_wq(context, attr, &rwq->wq, &cmd.ibv_cmd, | ||||
sizeof(cmd.ibv_cmd), | sizeof(cmd.ibv_cmd), | ||||
sizeof(cmd), | sizeof(cmd), | ||||
&resp.ibv_resp, sizeof(resp.ibv_resp), | &resp.ibv_resp, sizeof(resp.ibv_resp), | ||||
sizeof(resp)); | sizeof(resp)); | ||||
if (err) | if (err) | ||||
goto err_create; | goto err_create; | ||||
rwq->rsc.type = MLX5_RSC_TYPE_RWQ; | rwq->rsc.type = MLX5_RSC_TYPE_RWQ; | ||||
rwq->rsc.rsn = cmd.drv.user_index; | rwq->rsc.rsn = cmd.drv.user_index; | ||||
rwq->wq.post_recv = mlx5_post_wq_recv; | rwq->wq.post_recv = mlx5_post_wq_recv; | ||||
return &rwq->wq; | return &rwq->wq; | ||||
err_create: | err_create: | ||||
mlx5_clear_uidx(ctx, cmd.drv.user_index); | mlx5_clear_uidx(ctx, cmd.drv.user_index); | ||||
err_free_db_rec: | err_free_db_rec: | ||||
mlx5_free_db(to_mctx(context), rwq->db); | mlx5_free_db(to_mctx(context), rwq->db); | ||||
err_free_rwq_buf: | err_free_rwq_buf: | ||||
mlx5_free_rwq_buf(rwq, context); | mlx5_free_rwq_buf(rwq, context); | ||||
err: | err: | ||||
free(rwq); | free(rwq); | ||||
return NULL; | return NULL; | ||||
} | |||||
struct ibv_wq *mlx5_create_wq(struct ibv_context *context, | |||||
struct ibv_wq_init_attr *attr) | |||||
{ | |||||
return create_wq(context, attr, NULL); | |||||
} | |||||
struct ibv_wq *mlx5dv_create_wq(struct ibv_context *context, | |||||
struct ibv_wq_init_attr *attr, | |||||
struct mlx5dv_wq_init_attr *mlx5_wq_attr) | |||||
{ | |||||
return create_wq(context, attr, mlx5_wq_attr); | |||||
} | } | ||||
int mlx5_modify_wq(struct ibv_wq *wq, struct ibv_wq_attr *attr) | int mlx5_modify_wq(struct ibv_wq *wq, struct ibv_wq_attr *attr) | ||||
{ | { | ||||
struct mlx5_modify_wq cmd = {}; | struct mlx5_modify_wq cmd = {}; | ||||
struct mlx5_rwq *rwq = to_mrwq(wq); | struct mlx5_rwq *rwq = to_mrwq(wq); | ||||
if ((attr->attr_mask & IBV_WQ_ATTR_STATE) && | if ((attr->attr_mask & IBV_WQ_ATTR_STATE) && | ||||
▲ Show 20 Lines • Show All 93 Lines • Show Last 20 Lines |