diff --git a/contrib/ofed/libibverbs/driver.h b/contrib/ofed/libibverbs/driver.h --- a/contrib/ofed/libibverbs/driver.h +++ b/contrib/ofed/libibverbs/driver.h @@ -35,6 +35,7 @@ #ifndef INFINIBAND_DRIVER_H #define INFINIBAND_DRIVER_H +#include #include #include @@ -320,6 +321,11 @@ return ENOSYS; } +static inline bool check_comp_mask(uint64_t input, uint64_t supported) +{ + return (input & ~supported) == 0; +} + int ibv_query_gid_type(struct ibv_context *context, uint8_t port_num, unsigned int index, enum ibv_gid_type *type); #endif /* INFINIBAND_DRIVER_H */ diff --git a/contrib/ofed/libmlx5/libmlx5.map b/contrib/ofed/libmlx5/libmlx5.map --- a/contrib/ofed/libmlx5/libmlx5.map +++ b/contrib/ofed/libmlx5/libmlx5.map @@ -16,3 +16,8 @@ mlx5dv_init_obj; mlx5dv_set_context_attr; } MLX5_1.1; + +MLX5_1.3 { + global: + mlx5dv_create_wq; +} MLX5_1.2; diff --git a/contrib/ofed/libmlx5/mlx5-abi.h b/contrib/ofed/libmlx5/mlx5-abi.h --- a/contrib/ofed/libmlx5/mlx5-abi.h +++ b/contrib/ofed/libmlx5/mlx5-abi.h @@ -206,6 +206,10 @@ __u32 uuar_index; }; +enum mlx5_create_wq_comp_mask { + MLX5_IB_CREATE_WQ_STRIDING_RQ = 1 << 0, +}; + struct mlx5_drv_create_wq { __u64 buf_addr; __u64 db_addr; @@ -214,7 +218,9 @@ __u32 user_index; __u32 flags; __u32 comp_mask; - __u32 reserved; + __u32 single_stride_log_num_of_bytes; + __u32 single_wqe_log_num_of_strides; + __u32 two_byte_shift_en; }; struct mlx5_create_wq { diff --git a/contrib/ofed/libmlx5/mlx5dv.h b/contrib/ofed/libmlx5/mlx5dv.h --- a/contrib/ofed/libmlx5/mlx5dv.h +++ b/contrib/ofed/libmlx5/mlx5dv.h @@ -202,6 +202,44 @@ MLX5DV_OBJ_RWQ = 1 << 3, }; +enum mlx5dv_wq_init_attr_mask { + MLX5DV_WQ_INIT_ATTR_MASK_STRIDING_RQ = 1 << 0, +}; + +struct mlx5dv_striding_rq_init_attr { + uint32_t single_stride_log_num_of_bytes; + uint32_t single_wqe_log_num_of_strides; + uint8_t two_byte_shift_en; +}; + +struct mlx5dv_wq_init_attr { + uint64_t comp_mask; /* Use enum mlx5dv_wq_init_attr_mask */ + struct mlx5dv_striding_rq_init_attr striding_rq_attrs; +}; + +/* + * This function creates a work queue object with extra properties + * defined by mlx5dv_wq_init_attr struct. + * + * For each bit in the comp_mask, a field in mlx5dv_wq_init_attr + * should follow. + * + * MLX5DV_WQ_INIT_ATTR_MASK_STRIDING_RQ: Create a work queue with + * striding RQ capabilities. + * - single_stride_log_num_of_bytes represents the size of each stride in the + * WQE and its value should be between min_single_stride_log_num_of_bytes + * and max_single_stride_log_num_of_bytes that are reported in + * mlx5dv_query_device. + * - single_wqe_log_num_of_strides represents the number of strides in each WQE. + * Its value should be between min_single_wqe_log_num_of_strides and + * max_single_wqe_log_num_of_strides that are reported in mlx5dv_query_device. + * - two_byte_shift_en: When enabled, hardware pads 2 bytes of zeroes + * before writing the message to memory (e.g. for IP alignment) + */ +struct ibv_wq *mlx5dv_create_wq(struct ibv_context *context, + struct ibv_wq_init_attr *wq_init_attr, + struct mlx5dv_wq_init_attr *mlx5_wq_attr); + /* * This function will initialize mlx5dv_xxx structs based on supplied type. * The information for initialization is taken from ibv_xx structs supplied @@ -307,7 +345,9 @@ }; struct mlx5_cqe64 { - uint8_t rsvd0[17]; + uint8_t rsvd0[2]; + __be16 wqe_id; + uint8_t rsvd4[13]; uint8_t ml_path; uint8_t rsvd20[4]; __be16 slid; @@ -423,6 +463,11 @@ __be32 imm; }; +struct mlx5_mprq_wqe { + struct mlx5_wqe_srq_next_seg nseg; + struct mlx5_wqe_data_seg dseg; +}; + struct mlx5_wqe_av { union { struct { diff --git a/contrib/ofed/libmlx5/verbs.c b/contrib/ofed/libmlx5/verbs.c --- a/contrib/ofed/libmlx5/verbs.c +++ b/contrib/ofed/libmlx5/verbs.c @@ -431,7 +431,8 @@ cmd.cqe_size = cqe_sz; if (mlx5cq_attr) { - if (mlx5cq_attr->comp_mask & ~(MLX5DV_CQ_INIT_ATTR_MASK_RESERVED - 1)) { + if (!check_comp_mask(mlx5cq_attr->comp_mask, + MLX5DV_CQ_INIT_ATTR_MASK_RESERVED - 1)) { mlx5_dbg(fp, MLX5_DBG_CQ, "Unsupported vendor comp_mask for create_cq\n"); errno = EINVAL; @@ -919,21 +920,36 @@ return wq_size; } +enum { + DV_CREATE_WQ_SUPPORTED_COMP_MASK = MLX5DV_WQ_INIT_ATTR_MASK_STRIDING_RQ +}; + static int mlx5_calc_rwq_size(struct mlx5_context *ctx, struct mlx5_rwq *rwq, - struct ibv_wq_init_attr *attr) + struct ibv_wq_init_attr *attr, + struct mlx5dv_wq_init_attr *mlx5wq_attr) { size_t wqe_size; int wq_size; uint32_t num_scatter; + int is_mprq = 0; int scat_spc; if (!attr->max_wr) return -EINVAL; + if (mlx5wq_attr) { + if (!check_comp_mask(mlx5wq_attr->comp_mask, + DV_CREATE_WQ_SUPPORTED_COMP_MASK)) + return -EINVAL; + + is_mprq = !!(mlx5wq_attr->comp_mask & + MLX5DV_WQ_INIT_ATTR_MASK_STRIDING_RQ); + } /* TBD: check caps for RQ */ num_scatter = max_t(uint32_t, attr->max_sge, 1); - wqe_size = sizeof(struct mlx5_wqe_data_seg) * num_scatter; + wqe_size = sizeof(struct mlx5_wqe_data_seg) * num_scatter + + sizeof(struct mlx5_wqe_srq_next_seg) * is_mprq; if (rwq->wq_sig) wqe_size += sizeof(struct mlx5_rwqe_sig); @@ -948,7 +964,8 @@ rwq->rq.wqe_shift = mlx5_ilog2(wqe_size); rwq->rq.max_post = 1 << mlx5_ilog2(wq_size / wqe_size); scat_spc = wqe_size - - ((rwq->wq_sig) ? sizeof(struct mlx5_rwqe_sig) : 0); + ((rwq->wq_sig) ? sizeof(struct mlx5_rwqe_sig) : 0) - + is_mprq * sizeof(struct mlx5_wqe_srq_next_seg); rwq->rq.max_gs = scat_spc / sizeof(struct mlx5_wqe_data_seg); return wq_size; } @@ -2140,8 +2157,9 @@ return 0; } -struct ibv_wq *mlx5_create_wq(struct ibv_context *context, - struct ibv_wq_init_attr *attr) +static struct ibv_wq *create_wq(struct ibv_context *context, + struct ibv_wq_init_attr *attr, + struct mlx5dv_wq_init_attr *mlx5wq_attr) { struct mlx5_create_wq cmd; struct mlx5_create_wq_resp resp; @@ -2166,7 +2184,7 @@ if (rwq->wq_sig) cmd.drv.flags = MLX5_RWQ_FLAG_SIGNATURE; - ret = mlx5_calc_rwq_size(ctx, rwq, attr); + ret = mlx5_calc_rwq_size(ctx, rwq, attr, mlx5wq_attr); if (ret < 0) { errno = -ret; goto err; @@ -2200,6 +2218,33 @@ } cmd.drv.user_index = usr_idx; + if (mlx5wq_attr) { + if (mlx5wq_attr->comp_mask & MLX5DV_WQ_INIT_ATTR_MASK_STRIDING_RQ) { + if ((mlx5wq_attr->striding_rq_attrs.single_stride_log_num_of_bytes < + ctx->striding_rq_caps.min_single_stride_log_num_of_bytes) || + (mlx5wq_attr->striding_rq_attrs.single_stride_log_num_of_bytes > + ctx->striding_rq_caps.max_single_stride_log_num_of_bytes)) { + errno = EINVAL; + goto err_create; + } + + if ((mlx5wq_attr->striding_rq_attrs.single_wqe_log_num_of_strides < + ctx->striding_rq_caps.min_single_wqe_log_num_of_strides) || + (mlx5wq_attr->striding_rq_attrs.single_wqe_log_num_of_strides > + ctx->striding_rq_caps.max_single_wqe_log_num_of_strides)) { + errno = EINVAL; + goto err_create; + } + + cmd.drv.single_stride_log_num_of_bytes = + mlx5wq_attr->striding_rq_attrs.single_stride_log_num_of_bytes; + cmd.drv.single_wqe_log_num_of_strides = + mlx5wq_attr->striding_rq_attrs.single_wqe_log_num_of_strides; + cmd.drv.two_byte_shift_en = + mlx5wq_attr->striding_rq_attrs.two_byte_shift_en; + cmd.drv.comp_mask |= MLX5_IB_CREATE_WQ_STRIDING_RQ; + } + } err = ibv_cmd_create_wq(context, attr, &rwq->wq, &cmd.ibv_cmd, sizeof(cmd.ibv_cmd), sizeof(cmd), @@ -2225,6 +2270,19 @@ return NULL; } +struct ibv_wq *mlx5_create_wq(struct ibv_context *context, + struct ibv_wq_init_attr *attr) +{ + return create_wq(context, attr, NULL); +} + +struct ibv_wq *mlx5dv_create_wq(struct ibv_context *context, + struct ibv_wq_init_attr *attr, + struct mlx5dv_wq_init_attr *mlx5_wq_attr) +{ + return create_wq(context, attr, mlx5_wq_attr); +} + int mlx5_modify_wq(struct ibv_wq *wq, struct ibv_wq_attr *attr) { struct mlx5_modify_wq cmd = {};