diff --git a/contrib/ofed/libibverbs/man/ibv_create_cq_ex.3 b/contrib/ofed/libibverbs/man/ibv_create_cq_ex.3 --- a/contrib/ofed/libibverbs/man/ibv_create_cq_ex.3 +++ b/contrib/ofed/libibverbs/man/ibv_create_cq_ex.3 @@ -104,9 +104,12 @@ .BI "uint32_t ibv_wc_read_byte_len(struct ibv_cq_ex " "*cq"); \c Get the vendor error from the current completion. -.BI "uint32_t ibv_wc_read_imm_data(struct ibv_cq_ex " "*cq"); \c +.BI "__be32 ibv_wc_read_imm_data(struct ibv_cq_ex " "*cq"); \c Get the immediate data field from the current completion. +.BI "uint32_t ibv_wc_read_invalidated_rkey(struct ibv_cq_ex " "*cq"); \c + Get the rkey invalided by the SEND_INVAL from the current completion. + .BI "uint32_t ibv_wc_read_qp_num(struct ibv_cq_ex " "*cq"); \c Get the QP number field from the current completion. diff --git a/contrib/ofed/libibverbs/man/ibv_post_send.3 b/contrib/ofed/libibverbs/man/ibv_post_send.3 --- a/contrib/ofed/libibverbs/man/ibv_post_send.3 +++ b/contrib/ofed/libibverbs/man/ibv_post_send.3 @@ -35,7 +35,12 @@ int num_sge; /* Size of the s/g array */ enum ibv_wr_opcode opcode; /* Operation type */ int send_flags; /* Flags of the WR properties */ -uint32_t imm_data; /* Immediate data (in network byte order) */ +union { +.in +8 +__be32 imm_data; /* Immediate data (in network byte order) */ +uint32_t invalidate_rkey; /* Remote rkey to invalidate */ +.in -8 +}; union { .in +8 struct { diff --git a/contrib/ofed/libibverbs/verbs.h b/contrib/ofed/libibverbs/verbs.h --- a/contrib/ofed/libibverbs/verbs.h +++ b/contrib/ofed/libibverbs/verbs.h @@ -945,7 +945,13 @@ int num_sge; enum ibv_wr_opcode opcode; int send_flags; - __be32 imm_data; + /* When opcode is *_WITH_IMM: Immediate data in network byte order. + * When opcode is *_INV: Stores the rkey to invalidate + */ + union { + __be32 imm_data; + uint32_t invalidate_rkey; + }; union { struct { uint64_t remote_addr; @@ -1097,7 +1103,7 @@ enum ibv_wc_opcode (*read_opcode)(struct ibv_cq_ex *current); uint32_t (*read_vendor_err)(struct ibv_cq_ex *current); uint32_t (*read_byte_len)(struct ibv_cq_ex *current); - uint32_t (*read_imm_data)(struct ibv_cq_ex *current); + __be32 (*read_imm_data)(struct ibv_cq_ex *current); uint32_t (*read_qp_num)(struct ibv_cq_ex *current); uint32_t (*read_src_qp)(struct ibv_cq_ex *current); int (*read_wc_flags)(struct ibv_cq_ex *current); @@ -1145,11 +1151,20 @@ return cq->read_byte_len(cq); } -static inline uint32_t ibv_wc_read_imm_data(struct ibv_cq_ex *cq) +static inline __be32 ibv_wc_read_imm_data(struct ibv_cq_ex *cq) { return cq->read_imm_data(cq); } +static inline uint32_t ibv_wc_read_invalidated_rkey(struct ibv_cq_ex *cq) +{ +#ifdef __CHECKER__ + return (__attribute__((force)) uint32_t)cq->read_imm_data(cq); +#else + return cq->read_imm_data(cq); +#endif +} + static inline uint32_t ibv_wc_read_qp_num(struct ibv_cq_ex *cq) { return cq->read_qp_num(cq); diff --git a/contrib/ofed/libmlx5/cq.c b/contrib/ofed/libmlx5/cq.c --- a/contrib/ofed/libmlx5/cq.c +++ b/contrib/ofed/libmlx5/cq.c @@ -181,6 +181,15 @@ return err; } +/* Returns IBV_WC_IP_CSUM_OK or 0 */ +static inline int get_csum_ok(struct mlx5_cqe64 *cqe) +{ + return (((cqe->hds_ip_ext & (MLX5_CQE_L4_OK | MLX5_CQE_L3_OK)) == + (MLX5_CQE_L4_OK | MLX5_CQE_L3_OK)) & + (get_cqe_l3_hdr_type(cqe) == MLX5_CQE_L3_HDR_TYPE_IPV4)) + << IBV_WC_IP_CSUM_OK_SHIFT; +} + static inline int handle_responder(struct ibv_wc *wc, struct mlx5_cqe64 *cqe, struct mlx5_resource *cur_rsc, struct mlx5_srq *srq) { @@ -205,11 +214,7 @@ if (likely(cur_rsc->type == MLX5_RSC_TYPE_QP)) { wq = &qp->rq; if (qp->qp_cap_cache & MLX5_RX_CSUM_VALID) - wc->wc_flags |= (!!(cqe->hds_ip_ext & MLX5_CQE_L4_OK) & - !!(cqe->hds_ip_ext & MLX5_CQE_L3_OK) & - (get_cqe_l3_hdr_type(cqe) == - MLX5_CQE_L3_HDR_TYPE_IPV4)) << - IBV_WC_IP_CSUM_OK_SHIFT; + wc->wc_flags |= get_csum_ok(cqe); } else { wq = &(rsc_to_mrwq(cur_rsc)->rq); } @@ -244,7 +249,7 @@ case MLX5_CQE_RESP_SEND_INV: wc->opcode = IBV_WC_RECV; wc->wc_flags |= IBV_WC_WITH_INV; - wc->imm_data = be32toh(cqe->imm_inval_pkey); + wc->invalidated_rkey = be32toh(cqe->imm_inval_pkey); break; } wc->slid = be16toh(cqe->slid); @@ -260,7 +265,7 @@ static void dump_cqe(FILE *fp, void *buf) { - uint32_t *p = buf; + __be32 *p = buf; int i; for (i = 0; i < 16; i += 4) @@ -1139,13 +1144,16 @@ return ecqe->vendor_err_synd; } -static inline uint32_t mlx5_cq_read_wc_imm_data(struct ibv_cq_ex *ibcq) +static inline __be32 mlx5_cq_read_wc_imm_data(struct ibv_cq_ex *ibcq) { struct mlx5_cq *cq = to_mcq(ibv_cq_ex_to_cq(ibcq)); switch (mlx5dv_get_cqe_opcode(cq->cqe64)) { case MLX5_CQE_RESP_SEND_INV: - return be32toh(cq->cqe64->imm_inval_pkey); + /* This is returning invalidate_rkey which is in host order, + * see ibv_wc_read_invalidated_rkey + */ + return (__be32)be32toh(cq->cqe64->imm_inval_pkey); default: return cq->cqe64->imm_inval_pkey; } diff --git a/contrib/ofed/libmlx5/dbrec.c b/contrib/ofed/libmlx5/dbrec.c --- a/contrib/ofed/libmlx5/dbrec.c +++ b/contrib/ofed/libmlx5/dbrec.c @@ -80,10 +80,10 @@ return page; } -uint32_t *mlx5_alloc_dbrec(struct mlx5_context *context) +__be32 *mlx5_alloc_dbrec(struct mlx5_context *context) { struct mlx5_db_page *page; - uint32_t *db = NULL; + __be32 *db = NULL; int i, j; pthread_mutex_lock(&context->db_list_mutex); @@ -113,7 +113,7 @@ return db; } -void mlx5_free_db(struct mlx5_context *context, uint32_t *db) +void mlx5_free_db(struct mlx5_context *context, __be32 *db) { struct mlx5_db_page *page; uintptr_t ps = to_mdev(context->ibv_ctx.device)->page_size; diff --git a/contrib/ofed/libmlx5/mlx5.h b/contrib/ofed/libmlx5/mlx5.h --- a/contrib/ofed/libmlx5/mlx5.h +++ b/contrib/ofed/libmlx5/mlx5.h @@ -325,7 +325,7 @@ struct mlx5_spinlock lock; uint32_t cqn; uint32_t cons_index; - uint32_t *dbrec; + __be32 *dbrec; int arm_sn; int cqe_sz; int resize_cqe_sz; @@ -353,7 +353,7 @@ int wqe_shift; int head; int tail; - uint32_t *db; + __be32 *db; uint16_t counter; int wq_sig; }; @@ -412,7 +412,7 @@ uint8_t sq_signal_bits; struct mlx5_wq sq; - uint32_t *db; + __be32 *db; struct mlx5_wq rq; int wq_sig; uint32_t qp_cap_cache; @@ -434,9 +434,9 @@ struct mlx5_buf buf; int buf_size; struct mlx5_wq rq; - uint32_t *db; + __be32 *db; void *pbuff; - uint32_t *recv_db; + __be32 *recv_db; int wq_sig; }; @@ -563,8 +563,8 @@ enum mlx5_alloc_type default_alloc_type); int mlx5_use_huge(const char *key); -uint32_t *mlx5_alloc_dbrec(struct mlx5_context *context); -void mlx5_free_db(struct mlx5_context *context, uint32_t *db); +__be32 *mlx5_alloc_dbrec(struct mlx5_context *context); +void mlx5_free_db(struct mlx5_context *context, __be32 *db); int mlx5_query_device(struct ibv_context *context, struct ibv_device_attr *attr); diff --git a/contrib/ofed/libmlx5/mlx5dv.h b/contrib/ofed/libmlx5/mlx5dv.h --- a/contrib/ofed/libmlx5/mlx5dv.h +++ b/contrib/ofed/libmlx5/mlx5dv.h @@ -111,7 +111,7 @@ }; struct mlx5dv_qp { - uint32_t *dbrec; + __be32 *dbrec; struct { void *buf; uint32_t wqe_cnt; @@ -132,7 +132,7 @@ struct mlx5dv_cq { void *buf; - uint32_t *dbrec; + __be32 *dbrec; uint32_t cqe_cnt; uint32_t cqe_size; void *uar; @@ -142,7 +142,7 @@ struct mlx5dv_srq { void *buf; - uint32_t *dbrec; + __be32 *dbrec; uint32_t stride; uint32_t head; uint32_t tail; @@ -151,7 +151,7 @@ struct mlx5dv_rwq { void *buf; - uint32_t *dbrec; + __be32 *dbrec; uint32_t wqe_cnt; uint32_t stride; uint64_t comp_mask; @@ -291,18 +291,18 @@ uint8_t rsvd0[17]; uint8_t ml_path; uint8_t rsvd20[4]; - uint16_t slid; - uint32_t flags_rqpn; + __be16 slid; + __be32 flags_rqpn; uint8_t hds_ip_ext; uint8_t l4_hdr_type_etc; - uint16_t vlan_info; - uint32_t srqn_uidx; - uint32_t imm_inval_pkey; + __be16 vlan_info; + __be32 srqn_uidx; + __be32 imm_inval_pkey; uint8_t rsvd40[4]; - uint32_t byte_cnt; + __be32 byte_cnt; __be64 timestamp; - uint32_t sop_drop_qpn; - uint16_t wqe_counter; + __be32 sop_drop_qpn; + __be16 wqe_counter; uint8_t signature; uint8_t op_own; }; @@ -378,43 +378,43 @@ struct mlx5_wqe_srq_next_seg { uint8_t rsvd0[2]; - uint16_t next_wqe_index; + __be16 next_wqe_index; uint8_t signature; uint8_t rsvd1[11]; }; struct mlx5_wqe_data_seg { - uint32_t byte_count; - uint32_t lkey; - uint64_t addr; + __be32 byte_count; + __be32 lkey; + __be64 addr; }; struct mlx5_wqe_ctrl_seg { - uint32_t opmod_idx_opcode; - uint32_t qpn_ds; + __be32 opmod_idx_opcode; + __be32 qpn_ds; uint8_t signature; uint8_t rsvd[2]; uint8_t fm_ce_se; - uint32_t imm; + __be32 imm; }; struct mlx5_wqe_av { union { struct { - uint32_t qkey; - uint32_t reserved; + __be32 qkey; + __be32 reserved; } qkey; - uint64_t dc_key; + __be64 dc_key; } key; - uint32_t dqp_dct; + __be32 dqp_dct; uint8_t stat_rate_sl; uint8_t fl_mlid; - uint16_t rlid; + __be16 rlid; uint8_t reserved0[4]; uint8_t rmac[6]; uint8_t tclass; uint8_t hop_limit; - uint32_t grh_gid_fl; + __be32 grh_gid_fl; uint8_t rgid[16]; }; @@ -423,14 +423,14 @@ }; struct mlx5_wqe_raddr_seg { - uint64_t raddr; - uint32_t rkey; - uint32_t reserved; + __be64 raddr; + __be32 rkey; + __be32 reserved; }; struct mlx5_wqe_atomic_seg { - uint64_t swap_add; - uint64_t compare; + __be64 swap_add; + __be64 compare; }; struct mlx5_wqe_inl_data_seg { @@ -438,12 +438,12 @@ }; struct mlx5_wqe_eth_seg { - uint32_t rsvd0; + __be32 rsvd0; uint8_t cs_flags; uint8_t rsvd1; - uint16_t mss; - uint32_t rsvd2; - uint16_t inline_hdr_sz; + __be16 mss; + __be32 rsvd2; + __be16 inline_hdr_sz; uint8_t inline_hdr_start[2]; uint8_t inline_hdr[16]; }; diff --git a/contrib/ofed/libmlx5/qp.c b/contrib/ofed/libmlx5/qp.c --- a/contrib/ofed/libmlx5/qp.c +++ b/contrib/ofed/libmlx5/qp.c @@ -253,14 +253,14 @@ } } -static uint32_t send_ieth(struct ibv_send_wr *wr) +static __be32 send_ieth(struct ibv_send_wr *wr) { switch (wr->opcode) { case IBV_WR_SEND_WITH_IMM: case IBV_WR_RDMA_WRITE_WITH_IMM: return wr->imm_data; case IBV_WR_SEND_WITH_INV: - return htobe32(wr->imm_data); + return htobe32(wr->invalidate_rkey); default: return 0; } @@ -413,7 +413,7 @@ #undef ALIGN #define ALIGN(x, log_a) ((((x) + (1 << (log_a)) - 1)) & ~((1 << (log_a)) - 1)) -static inline uint16_t get_klm_octo(int nentries) +static inline __be16 get_klm_octo(int nentries) { return htobe16(ALIGN(nentries, 3) / 2); } @@ -742,7 +742,7 @@ struct ibv_mw_bind_info bind_info = {}; next_fence = MLX5_WQE_CTRL_INITIATOR_SMALL_FENCE; - ctrl->imm = htobe32(wr->imm_data); + ctrl->imm = htobe32(wr->invalidate_rkey); err = set_bind_wr(qp, IBV_MW_TYPE_2, 0, &bind_info, ibqp->qp_num, &seg, &size); @@ -787,7 +787,7 @@ struct ibv_mw_bind_info bind_info = {}; next_fence = MLX5_WQE_CTRL_INITIATOR_SMALL_FENCE; - ctrl->imm = htobe32(wr->imm_data); + ctrl->imm = htobe32(wr->invalidate_rkey); err = set_bind_wr(qp, IBV_MW_TYPE_2, 0, &bind_info, ibqp->qp_num, &seg, &size); diff --git a/contrib/ofed/libmlx5/verbs.c b/contrib/ofed/libmlx5/verbs.c --- a/contrib/ofed/libmlx5/verbs.c +++ b/contrib/ofed/libmlx5/verbs.c @@ -1553,7 +1553,7 @@ struct mlx5_qp *mqp = to_mqp(qp); struct mlx5_context *context = to_mctx(qp->context); int ret; - uint32_t *db; + __be32 *db; if (mqp->rss_qp) return ENOSYS; @@ -1636,7 +1636,7 @@ struct ibv_port_attr port_attr; struct mlx5_ah *ah; uint32_t gid_type; - uint32_t tmp; + __be32 tmp; uint8_t grh; int is_eth; diff --git a/contrib/ofed/libmlx5/wqe.h b/contrib/ofed/libmlx5/wqe.h --- a/contrib/ofed/libmlx5/wqe.h +++ b/contrib/ofed/libmlx5/wqe.h @@ -51,7 +51,7 @@ }; struct mlx5_wqe_xrc_seg { - uint32_t xrc_srqn; + __be32 xrc_srqn; uint8_t rsvd[12]; }; @@ -89,17 +89,17 @@ struct mlx5_wqe_umr_ctrl_seg { uint8_t flags; uint8_t rsvd0[3]; - uint16_t klm_octowords; - uint16_t translation_offset; - uint64_t mkey_mask; + __be16 klm_octowords; + __be16 translation_offset; + __be64 mkey_mask; uint8_t rsvd1[32]; }; struct mlx5_wqe_umr_klm_seg { /* up to 2GB */ - uint32_t byte_count; - uint32_t mkey; - uint64_t address; + __be32 byte_count; + __be32 mkey; + __be64 address; }; union mlx5_wqe_umr_inline_seg { @@ -123,17 +123,17 @@ uint8_t reserved1; uint8_t access_flags; uint8_t sf; - uint32_t qpn_mkey; - uint32_t reserved2; - uint32_t flags_pd; - uint64_t start_addr; - uint64_t len; - uint32_t bsf_octword_size; - uint32_t reserved3[4]; - uint32_t translations_octword_size; + __be32 qpn_mkey; + __be32 reserved2; + __be32 flags_pd; + __be64 start_addr; + __be64 len; + __be32 bsf_octword_size; + __be32 reserved3[4]; + __be32 translations_octword_size; uint8_t reserved4[3]; uint8_t log_page_size; - uint32_t reserved; + __be32 reserved; union mlx5_wqe_umr_inline_seg inseg[0]; }; @@ -183,7 +183,7 @@ }; struct mlx5_wqe_inline_seg { - uint32_t byte_count; + __be32 byte_count; };