Changeset View
Changeset View
Standalone View
Standalone View
sys/dev/irdma/irdma_uk.c
/*- | /*- | ||||
* SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB | * SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB | ||||
* | * | ||||
* Copyright (c) 2015 - 2021 Intel Corporation | * Copyright (c) 2015 - 2022 Intel Corporation | ||||
* | * | ||||
* This software is available to you under a choice of one of two | * This software is available to you under a choice of one of two | ||||
* licenses. You may choose to be licensed under the terms of the GNU | * licenses. You may choose to be licensed under the terms of the GNU | ||||
Context not available. | |||||
{ | { | ||||
if (sge) { | if (sge) { | ||||
set_64bit_val(wqe, offset, | set_64bit_val(wqe, offset, | ||||
LS_64(sge->tag_off, IRDMAQPSQ_FRAG_TO)); | FIELD_PREP(IRDMAQPSQ_FRAG_TO, sge->tag_off)); | ||||
set_64bit_val(wqe, offset + IRDMA_BYTE_8, | set_64bit_val(wqe, offset + IRDMA_BYTE_8, | ||||
LS_64(valid, IRDMAQPSQ_VALID) | | FIELD_PREP(IRDMAQPSQ_VALID, valid) | | ||||
LS_64(sge->len, IRDMAQPSQ_FRAG_LEN) | | FIELD_PREP(IRDMAQPSQ_FRAG_LEN, sge->len) | | ||||
LS_64(sge->stag, IRDMAQPSQ_FRAG_STAG)); | FIELD_PREP(IRDMAQPSQ_FRAG_STAG, sge->stag)); | ||||
} else { | } else { | ||||
set_64bit_val(wqe, offset, 0); | set_64bit_val(wqe, offset, 0); | ||||
set_64bit_val(wqe, offset + IRDMA_BYTE_8, | set_64bit_val(wqe, offset + IRDMA_BYTE_8, | ||||
LS_64(valid, IRDMAQPSQ_VALID)); | FIELD_PREP(IRDMAQPSQ_VALID, valid)); | ||||
} | } | ||||
} | } | ||||
Context not available. | |||||
{ | { | ||||
if (sge) { | if (sge) { | ||||
set_64bit_val(wqe, offset, | set_64bit_val(wqe, offset, | ||||
LS_64(sge->tag_off, IRDMAQPSQ_FRAG_TO)); | FIELD_PREP(IRDMAQPSQ_FRAG_TO, sge->tag_off)); | ||||
set_64bit_val(wqe, offset + IRDMA_BYTE_8, | set_64bit_val(wqe, offset + IRDMA_BYTE_8, | ||||
LS_64(sge->len, IRDMAQPSQ_GEN1_FRAG_LEN) | | FIELD_PREP(IRDMAQPSQ_GEN1_FRAG_LEN, sge->len) | | ||||
LS_64(sge->stag, IRDMAQPSQ_GEN1_FRAG_STAG)); | FIELD_PREP(IRDMAQPSQ_GEN1_FRAG_STAG, sge->stag)); | ||||
} else { | } else { | ||||
set_64bit_val(wqe, offset, 0); | set_64bit_val(wqe, offset, 0); | ||||
set_64bit_val(wqe, offset + IRDMA_BYTE_8, 0); | set_64bit_val(wqe, offset + IRDMA_BYTE_8, 0); | ||||
} | } | ||||
} | } | ||||
/** | |||||
* irdma_nop_hdr - Format header section of noop WQE | |||||
* @qp: hw qp ptr | |||||
*/ | |||||
static inline u64 irdma_nop_hdr(struct irdma_qp_uk *qp){ | |||||
return FIELD_PREP(IRDMAQPSQ_OPCODE, IRDMAQP_OP_NOP) | | |||||
FIELD_PREP(IRDMAQPSQ_SIGCOMPL, false) | | |||||
FIELD_PREP(IRDMAQPSQ_VALID, qp->swqe_polarity); | |||||
} | |||||
/** | /** | ||||
* irdma_nop_1 - insert a NOP wqe | * irdma_nop_1 - insert a NOP wqe | ||||
* @qp: hw qp ptr | * @qp: hw qp ptr | ||||
Context not available. | |||||
static int | static int | ||||
irdma_nop_1(struct irdma_qp_uk *qp) | irdma_nop_1(struct irdma_qp_uk *qp) | ||||
{ | { | ||||
u64 hdr; | |||||
__le64 *wqe; | __le64 *wqe; | ||||
u32 wqe_idx; | u32 wqe_idx; | ||||
bool signaled = false; | |||||
if (!qp->sq_ring.head) | if (!qp->sq_ring.head) | ||||
return -EINVAL; | return -EINVAL; | ||||
Context not available. | |||||
set_64bit_val(wqe, IRDMA_BYTE_8, 0); | set_64bit_val(wqe, IRDMA_BYTE_8, 0); | ||||
set_64bit_val(wqe, IRDMA_BYTE_16, 0); | set_64bit_val(wqe, IRDMA_BYTE_16, 0); | ||||
hdr = LS_64(IRDMAQP_OP_NOP, IRDMAQPSQ_OPCODE) | | |||||
LS_64(signaled, IRDMAQPSQ_SIGCOMPL) | | |||||
LS_64(qp->swqe_polarity, IRDMAQPSQ_VALID); | |||||
/* make sure WQE is written before valid bit is set */ | /* make sure WQE is written before valid bit is set */ | ||||
irdma_wmb(); | irdma_wmb(); | ||||
set_64bit_val(wqe, IRDMA_BYTE_24, hdr); | set_64bit_val(wqe, IRDMA_BYTE_24, irdma_nop_hdr(qp)); | ||||
return 0; | return 0; | ||||
} | } | ||||
Context not available. | |||||
/* read the doorbell shadow area */ | /* read the doorbell shadow area */ | ||||
get_64bit_val(qp->shadow_area, IRDMA_BYTE_0, &temp); | get_64bit_val(qp->shadow_area, IRDMA_BYTE_0, &temp); | ||||
hw_sq_tail = (u32)RS_64(temp, IRDMA_QP_DBSA_HW_SQ_TAIL); | hw_sq_tail = (u32)FIELD_GET(IRDMA_QP_DBSA_HW_SQ_TAIL, temp); | ||||
sw_sq_head = IRDMA_RING_CURRENT_HEAD(qp->sq_ring); | sw_sq_head = IRDMA_RING_CURRENT_HEAD(qp->sq_ring); | ||||
if (sw_sq_head != qp->initial_ring.head) { | if (sw_sq_head != qp->initial_ring.head) { | ||||
if (qp->push_dropped) { | if (qp->push_dropped) { | ||||
Context not available. | |||||
irdma_qp_ring_push_db(struct irdma_qp_uk *qp, u32 wqe_idx) | irdma_qp_ring_push_db(struct irdma_qp_uk *qp, u32 wqe_idx) | ||||
{ | { | ||||
set_32bit_val(qp->push_db, 0, | set_32bit_val(qp->push_db, 0, | ||||
LS_32(wqe_idx >> 3, IRDMA_WQEALLOC_WQE_DESC_INDEX) | qp->qp_id); | FIELD_PREP(IRDMA_WQEALLOC_WQE_DESC_INDEX, wqe_idx >> 3) | qp->qp_id); | ||||
qp->initial_ring.head = qp->sq_ring.head; | qp->initial_ring.head = qp->sq_ring.head; | ||||
qp->push_mode = true; | qp->push_mode = true; | ||||
qp->push_dropped = false; | qp->push_dropped = false; | ||||
Context not available. | |||||
* irdma_qp_get_next_send_wqe - pad with NOP if needed, return where next WR should go | * irdma_qp_get_next_send_wqe - pad with NOP if needed, return where next WR should go | ||||
* @qp: hw qp ptr | * @qp: hw qp ptr | ||||
* @wqe_idx: return wqe index | * @wqe_idx: return wqe index | ||||
* @quanta: size of WR in quanta | * @quanta: (in/out) ptr to size of WR in quanta. Modified in case pad is needed | ||||
* @total_size: size of WR in bytes | * @total_size: size of WR in bytes | ||||
* @info: info on WR | * @info: info on WR | ||||
*/ | */ | ||||
__le64 * | __le64 * | ||||
irdma_qp_get_next_send_wqe(struct irdma_qp_uk *qp, u32 *wqe_idx, | irdma_qp_get_next_send_wqe(struct irdma_qp_uk *qp, u32 *wqe_idx, | ||||
u16 quanta, u32 total_size, | u16 *quanta, u32 total_size, | ||||
struct irdma_post_sq_info *info) | struct irdma_post_sq_info *info) | ||||
{ | { | ||||
__le64 *wqe; | __le64 *wqe; | ||||
__le64 *wqe_0 = NULL; | __le64 *wqe_0 = NULL; | ||||
u32 nop_wqe_idx; | u32 nop_wqe_idx; | ||||
u16 avail_quanta; | u16 avail_quanta, wqe_quanta = *quanta; | ||||
u16 i; | u16 i; | ||||
avail_quanta = qp->uk_attrs->max_hw_sq_chunk - | avail_quanta = qp->uk_attrs->max_hw_sq_chunk - | ||||
(IRDMA_RING_CURRENT_HEAD(qp->sq_ring) % | (IRDMA_RING_CURRENT_HEAD(qp->sq_ring) % | ||||
qp->uk_attrs->max_hw_sq_chunk); | qp->uk_attrs->max_hw_sq_chunk); | ||||
if (quanta <= avail_quanta) { | |||||
if (*quanta <= avail_quanta) { | |||||
/* WR fits in current chunk */ | /* WR fits in current chunk */ | ||||
if (quanta > IRDMA_SQ_RING_FREE_QUANTA(qp->sq_ring)) | if (*quanta > IRDMA_SQ_RING_FREE_QUANTA(qp->sq_ring)) | ||||
return NULL; | return NULL; | ||||
} else { | } else { | ||||
/* Need to pad with NOP */ | /* Need to pad with NOP */ | ||||
if (quanta + avail_quanta > | if (*quanta + avail_quanta > | ||||
IRDMA_SQ_RING_FREE_QUANTA(qp->sq_ring)) | IRDMA_SQ_RING_FREE_QUANTA(qp->sq_ring)) | ||||
return NULL; | return NULL; | ||||
Context not available. | |||||
if (!*wqe_idx) | if (!*wqe_idx) | ||||
qp->swqe_polarity = !qp->swqe_polarity; | qp->swqe_polarity = !qp->swqe_polarity; | ||||
IRDMA_RING_MOVE_HEAD_BY_COUNT_NOCHECK(qp->sq_ring, quanta); | IRDMA_RING_MOVE_HEAD_BY_COUNT_NOCHECK(qp->sq_ring, *quanta); | ||||
irdma_clr_wqes(qp, *wqe_idx); | |||||
wqe = qp->sq_base[*wqe_idx].elem; | wqe = qp->sq_base[*wqe_idx].elem; | ||||
if (qp->uk_attrs->hw_rev == IRDMA_GEN_1 && quanta == 1 && | if (qp->uk_attrs->hw_rev == IRDMA_GEN_1 && wqe_quanta == 1 && | ||||
(IRDMA_RING_CURRENT_HEAD(qp->sq_ring) & 1)) { | (IRDMA_RING_CURRENT_HEAD(qp->sq_ring) & 1)) { | ||||
wqe_0 = qp->sq_base[IRDMA_RING_CURRENT_HEAD(qp->sq_ring)].elem; | wqe_0 = qp->sq_base[IRDMA_RING_CURRENT_HEAD(qp->sq_ring)].elem; | ||||
wqe_0[3] = cpu_to_le64(LS_64(!qp->swqe_polarity, IRDMAQPSQ_VALID)); | wqe_0[3] = cpu_to_le64(FIELD_PREP(IRDMAQPSQ_VALID, !qp->swqe_polarity)); | ||||
} | } | ||||
qp->sq_wrtrk_array[*wqe_idx].wrid = info->wr_id; | qp->sq_wrtrk_array[*wqe_idx].wrid = info->wr_id; | ||||
qp->sq_wrtrk_array[*wqe_idx].wr_len = total_size; | qp->sq_wrtrk_array[*wqe_idx].wr_len = total_size; | ||||
qp->sq_wrtrk_array[*wqe_idx].quanta = quanta; | qp->sq_wrtrk_array[*wqe_idx].quanta = wqe_quanta; | ||||
qp->sq_wrtrk_array[*wqe_idx].signaled = info->signaled; | qp->sq_wrtrk_array[*wqe_idx].signaled = info->signaled; | ||||
return wqe; | return wqe; | ||||
Context not available. | |||||
if (ret_code) | if (ret_code) | ||||
return ret_code; | return ret_code; | ||||
wqe = irdma_qp_get_next_send_wqe(qp, &wqe_idx, quanta, total_size, | wqe = irdma_qp_get_next_send_wqe(qp, &wqe_idx, &quanta, total_size, info); | ||||
info); | |||||
if (!wqe) | if (!wqe) | ||||
return -ENOSPC; | return -ENOSPC; | ||||
irdma_clr_wqes(qp, wqe_idx); | |||||
qp->sq_wrtrk_array[wqe_idx].signaled = info->signaled; | qp->sq_wrtrk_array[wqe_idx].signaled = info->signaled; | ||||
set_64bit_val(wqe, IRDMA_BYTE_16, | set_64bit_val(wqe, IRDMA_BYTE_16, | ||||
LS_64(op_info->rem_addr.tag_off, IRDMAQPSQ_FRAG_TO)); | FIELD_PREP(IRDMAQPSQ_FRAG_TO, op_info->rem_addr.tag_off)); | ||||
if (info->imm_data_valid) { | if (info->imm_data_valid) { | ||||
set_64bit_val(wqe, IRDMA_BYTE_0, | set_64bit_val(wqe, IRDMA_BYTE_0, | ||||
LS_64(info->imm_data, IRDMAQPSQ_IMMDATA)); | FIELD_PREP(IRDMAQPSQ_IMMDATA, info->imm_data)); | ||||
i = 0; | i = 0; | ||||
} else { | } else { | ||||
qp->wqe_ops.iw_set_fragment(wqe, IRDMA_BYTE_0, | qp->wqe_ops.iw_set_fragment(wqe, IRDMA_BYTE_0, | ||||
Context not available. | |||||
++addl_frag_cnt; | ++addl_frag_cnt; | ||||
} | } | ||||
if (!op_info->rem_addr.stag && !total_size) | hdr = FIELD_PREP(IRDMAQPSQ_REMSTAG, op_info->rem_addr.stag) | | ||||
op_info->rem_addr.stag = 0x1234; | FIELD_PREP(IRDMAQPSQ_OPCODE, info->op_type) | | ||||
hdr = LS_64(op_info->rem_addr.stag, IRDMAQPSQ_REMSTAG) | | FIELD_PREP(IRDMAQPSQ_IMMDATAFLAG, info->imm_data_valid) | | ||||
LS_64(info->op_type, IRDMAQPSQ_OPCODE) | | FIELD_PREP(IRDMAQPSQ_REPORTRTT, info->report_rtt) | | ||||
LS_64((info->imm_data_valid ? 1 : 0), IRDMAQPSQ_IMMDATAFLAG) | | FIELD_PREP(IRDMAQPSQ_ADDFRAGCNT, addl_frag_cnt) | | ||||
LS_64((info->report_rtt ? 1 : 0), IRDMAQPSQ_REPORTRTT) | | FIELD_PREP(IRDMAQPSQ_PUSHWQE, info->push_wqe) | | ||||
LS_64(addl_frag_cnt, IRDMAQPSQ_ADDFRAGCNT) | | FIELD_PREP(IRDMAQPSQ_READFENCE, read_fence) | | ||||
LS_64((info->push_wqe ? 1 : 0), IRDMAQPSQ_PUSHWQE) | | FIELD_PREP(IRDMAQPSQ_LOCALFENCE, info->local_fence) | | ||||
LS_64(read_fence, IRDMAQPSQ_READFENCE) | | FIELD_PREP(IRDMAQPSQ_SIGCOMPL, info->signaled) | | ||||
LS_64(info->local_fence, IRDMAQPSQ_LOCALFENCE) | | FIELD_PREP(IRDMAQPSQ_VALID, qp->swqe_polarity); | ||||
LS_64(info->signaled, IRDMAQPSQ_SIGCOMPL) | | |||||
LS_64(qp->swqe_polarity, IRDMAQPSQ_VALID); | |||||
irdma_wmb(); /* make sure WQE is populated before valid bit is set */ | irdma_wmb(); /* make sure WQE is populated before valid bit is set */ | ||||
set_64bit_val(wqe, IRDMA_BYTE_24, hdr); | set_64bit_val(wqe, IRDMA_BYTE_24, hdr); | ||||
if (info->push_wqe) { | if (info->push_wqe) | ||||
irdma_qp_push_wqe(qp, wqe, quanta, wqe_idx, post_sq); | irdma_qp_push_wqe(qp, wqe, quanta, wqe_idx, post_sq); | ||||
} else { | else if (post_sq) | ||||
if (post_sq) | irdma_uk_qp_post_wr(qp); | ||||
irdma_uk_qp_post_wr(qp); | |||||
} | |||||
return 0; | return 0; | ||||
} | } | ||||
Context not available. | |||||
if (ret_code) | if (ret_code) | ||||
return ret_code; | return ret_code; | ||||
wqe = irdma_qp_get_next_send_wqe(qp, &wqe_idx, quanta, total_size, | wqe = irdma_qp_get_next_send_wqe(qp, &wqe_idx, &quanta, total_size, info); | ||||
info); | |||||
if (!wqe) | if (!wqe) | ||||
return -ENOSPC; | return -ENOSPC; | ||||
Context not available. | |||||
qp->ord_cnt = 0; | qp->ord_cnt = 0; | ||||
} | } | ||||
irdma_clr_wqes(qp, wqe_idx); | |||||
qp->sq_wrtrk_array[wqe_idx].signaled = info->signaled; | qp->sq_wrtrk_array[wqe_idx].signaled = info->signaled; | ||||
addl_frag_cnt = op_info->num_lo_sges > 1 ? | addl_frag_cnt = op_info->num_lo_sges > 1 ? | ||||
(op_info->num_lo_sges - 1) : 0; | (op_info->num_lo_sges - 1) : 0; | ||||
Context not available. | |||||
++addl_frag_cnt; | ++addl_frag_cnt; | ||||
} | } | ||||
set_64bit_val(wqe, IRDMA_BYTE_16, | set_64bit_val(wqe, IRDMA_BYTE_16, | ||||
LS_64(op_info->rem_addr.tag_off, IRDMAQPSQ_FRAG_TO)); | FIELD_PREP(IRDMAQPSQ_FRAG_TO, op_info->rem_addr.tag_off)); | ||||
hdr = LS_64(op_info->rem_addr.stag, IRDMAQPSQ_REMSTAG) | | hdr = FIELD_PREP(IRDMAQPSQ_REMSTAG, op_info->rem_addr.stag) | | ||||
LS_64((info->report_rtt ? 1 : 0), IRDMAQPSQ_REPORTRTT) | | FIELD_PREP(IRDMAQPSQ_REPORTRTT, (info->report_rtt ? 1 : 0)) | | ||||
LS_64(addl_frag_cnt, IRDMAQPSQ_ADDFRAGCNT) | | FIELD_PREP(IRDMAQPSQ_ADDFRAGCNT, addl_frag_cnt) | | ||||
LS_64((inv_stag ? IRDMAQP_OP_RDMA_READ_LOC_INV : IRDMAQP_OP_RDMA_READ), | FIELD_PREP(IRDMAQPSQ_OPCODE, | ||||
IRDMAQPSQ_OPCODE) | | (inv_stag ? IRDMAQP_OP_RDMA_READ_LOC_INV : IRDMAQP_OP_RDMA_READ)) | | ||||
LS_64((info->push_wqe ? 1 : 0), IRDMAQPSQ_PUSHWQE) | | FIELD_PREP(IRDMAQPSQ_PUSHWQE, info->push_wqe) | | ||||
LS_64(info->read_fence || qp->force_fence || ord_fence ? 1 : 0, | FIELD_PREP(IRDMAQPSQ_READFENCE, | ||||
IRDMAQPSQ_READFENCE) | | info->read_fence || ord_fence ? 1 : 0) | | ||||
LS_64(local_fence, IRDMAQPSQ_LOCALFENCE) | | FIELD_PREP(IRDMAQPSQ_LOCALFENCE, local_fence) | | ||||
LS_64(info->signaled, IRDMAQPSQ_SIGCOMPL) | | FIELD_PREP(IRDMAQPSQ_SIGCOMPL, info->signaled) | | ||||
LS_64(qp->swqe_polarity, IRDMAQPSQ_VALID); | FIELD_PREP(IRDMAQPSQ_VALID, qp->swqe_polarity); | ||||
irdma_wmb(); /* make sure WQE is populated before valid bit is set */ | irdma_wmb(); /* make sure WQE is populated before valid bit is set */ | ||||
set_64bit_val(wqe, IRDMA_BYTE_24, hdr); | set_64bit_val(wqe, IRDMA_BYTE_24, hdr); | ||||
if (info->push_wqe) { | if (info->push_wqe) | ||||
irdma_qp_push_wqe(qp, wqe, quanta, wqe_idx, post_sq); | irdma_qp_push_wqe(qp, wqe, quanta, wqe_idx, post_sq); | ||||
} else { | else if (post_sq) | ||||
if (post_sq) | irdma_uk_qp_post_wr(qp); | ||||
irdma_uk_qp_post_wr(qp); | |||||
} | |||||
return 0; | return 0; | ||||
} | } | ||||
Context not available. | |||||
if (ret_code) | if (ret_code) | ||||
return ret_code; | return ret_code; | ||||
wqe = irdma_qp_get_next_send_wqe(qp, &wqe_idx, quanta, total_size, | wqe = irdma_qp_get_next_send_wqe(qp, &wqe_idx, &quanta, total_size, info); | ||||
info); | |||||
if (!wqe) | if (!wqe) | ||||
return -ENOSPC; | return -ENOSPC; | ||||
irdma_clr_wqes(qp, wqe_idx); | |||||
read_fence |= info->read_fence; | read_fence |= info->read_fence; | ||||
addl_frag_cnt = frag_cnt > 1 ? (frag_cnt - 1) : 0; | addl_frag_cnt = frag_cnt > 1 ? (frag_cnt - 1) : 0; | ||||
if (info->imm_data_valid) { | if (info->imm_data_valid) { | ||||
set_64bit_val(wqe, IRDMA_BYTE_0, | set_64bit_val(wqe, IRDMA_BYTE_0, | ||||
LS_64(info->imm_data, IRDMAQPSQ_IMMDATA)); | FIELD_PREP(IRDMAQPSQ_IMMDATA, info->imm_data)); | ||||
i = 0; | i = 0; | ||||
} else { | } else { | ||||
qp->wqe_ops.iw_set_fragment(wqe, IRDMA_BYTE_0, op_info->sg_list, | qp->wqe_ops.iw_set_fragment(wqe, IRDMA_BYTE_0, | ||||
frag_cnt ? op_info->sg_list : NULL, | |||||
qp->swqe_polarity); | qp->swqe_polarity); | ||||
i = 1; | i = 1; | ||||
} | } | ||||
Context not available. | |||||
} | } | ||||
set_64bit_val(wqe, IRDMA_BYTE_16, | set_64bit_val(wqe, IRDMA_BYTE_16, | ||||
LS_64(op_info->qkey, IRDMAQPSQ_DESTQKEY) | | FIELD_PREP(IRDMAQPSQ_DESTQKEY, op_info->qkey) | | ||||
LS_64(op_info->dest_qp, IRDMAQPSQ_DESTQPN)); | FIELD_PREP(IRDMAQPSQ_DESTQPN, op_info->dest_qp)); | ||||
hdr = LS_64(info->stag_to_inv, IRDMAQPSQ_REMSTAG) | | hdr = FIELD_PREP(IRDMAQPSQ_REMSTAG, info->stag_to_inv) | | ||||
LS_64(op_info->ah_id, IRDMAQPSQ_AHID) | | FIELD_PREP(IRDMAQPSQ_AHID, op_info->ah_id) | | ||||
LS_64((info->imm_data_valid ? 1 : 0), IRDMAQPSQ_IMMDATAFLAG) | | FIELD_PREP(IRDMAQPSQ_IMMDATAFLAG, | ||||
LS_64((info->report_rtt ? 1 : 0), IRDMAQPSQ_REPORTRTT) | | (info->imm_data_valid ? 1 : 0)) | | ||||
LS_64(info->op_type, IRDMAQPSQ_OPCODE) | | FIELD_PREP(IRDMAQPSQ_REPORTRTT, (info->report_rtt ? 1 : 0)) | | ||||
LS_64(addl_frag_cnt, IRDMAQPSQ_ADDFRAGCNT) | | FIELD_PREP(IRDMAQPSQ_OPCODE, info->op_type) | | ||||
LS_64((info->push_wqe ? 1 : 0), IRDMAQPSQ_PUSHWQE) | | FIELD_PREP(IRDMAQPSQ_ADDFRAGCNT, addl_frag_cnt) | | ||||
LS_64(read_fence, IRDMAQPSQ_READFENCE) | | FIELD_PREP(IRDMAQPSQ_PUSHWQE, info->push_wqe) | | ||||
LS_64(info->local_fence, IRDMAQPSQ_LOCALFENCE) | | FIELD_PREP(IRDMAQPSQ_READFENCE, read_fence) | | ||||
LS_64(info->signaled, IRDMAQPSQ_SIGCOMPL) | | FIELD_PREP(IRDMAQPSQ_LOCALFENCE, info->local_fence) | | ||||
LS_64(info->udp_hdr, IRDMAQPSQ_UDPHEADER) | | FIELD_PREP(IRDMAQPSQ_SIGCOMPL, info->signaled) | | ||||
LS_64(info->l4len, IRDMAQPSQ_L4LEN) | | FIELD_PREP(IRDMAQPSQ_UDPHEADER, info->udp_hdr) | | ||||
LS_64(qp->swqe_polarity, IRDMAQPSQ_VALID); | FIELD_PREP(IRDMAQPSQ_L4LEN, info->l4len) | | ||||
FIELD_PREP(IRDMAQPSQ_VALID, qp->swqe_polarity); | |||||
irdma_wmb(); /* make sure WQE is populated before valid bit is set */ | irdma_wmb(); /* make sure WQE is populated before valid bit is set */ | ||||
set_64bit_val(wqe, IRDMA_BYTE_24, hdr); | set_64bit_val(wqe, IRDMA_BYTE_24, hdr); | ||||
if (info->push_wqe) { | if (info->push_wqe) | ||||
irdma_qp_push_wqe(qp, wqe, quanta, wqe_idx, post_sq); | irdma_qp_push_wqe(qp, wqe, quanta, wqe_idx, post_sq); | ||||
} else { | else if (post_sq) | ||||
if (post_sq) | irdma_uk_qp_post_wr(qp); | ||||
irdma_uk_qp_post_wr(qp); | |||||
} | |||||
return 0; | return 0; | ||||
} | } | ||||
Context not available. | |||||
{ | { | ||||
set_64bit_val(wqe, IRDMA_BYTE_0, (uintptr_t)op_info->va); | set_64bit_val(wqe, IRDMA_BYTE_0, (uintptr_t)op_info->va); | ||||
set_64bit_val(wqe, IRDMA_BYTE_8, | set_64bit_val(wqe, IRDMA_BYTE_8, | ||||
LS_64(op_info->mw_stag, IRDMAQPSQ_PARENTMRSTAG) | | FIELD_PREP(IRDMAQPSQ_PARENTMRSTAG, op_info->mw_stag) | | ||||
LS_64(op_info->mr_stag, IRDMAQPSQ_MWSTAG)); | FIELD_PREP(IRDMAQPSQ_MWSTAG, op_info->mr_stag)); | ||||
set_64bit_val(wqe, IRDMA_BYTE_16, op_info->bind_len); | set_64bit_val(wqe, IRDMA_BYTE_16, op_info->bind_len); | ||||
} | } | ||||
/** | /** | ||||
* irdma_copy_inline_data_gen_1 - Copy inline data to wqe | * irdma_copy_inline_data_gen_1 - Copy inline data to wqe | ||||
* @dest: pointer to wqe | * @wqe: pointer to wqe | ||||
* @src: pointer to inline data | * @sge_list: table of pointers to inline data | ||||
* @len: length of inline data to copy | * @num_sges: Total inline data length | ||||
* @polarity: compatibility parameter | * @polarity: compatibility parameter | ||||
*/ | */ | ||||
static void | static void | ||||
irdma_copy_inline_data_gen_1(u8 *dest, u8 *src, u32 len, | irdma_copy_inline_data_gen_1(u8 *wqe, struct irdma_sge *sge_list, | ||||
u8 polarity) | u32 num_sges, u8 polarity) | ||||
{ | { | ||||
if (len <= IRDMA_BYTE_16) { | u32 quanta_bytes_remaining = 16; | ||||
irdma_memcpy(dest, src, len); | u32 i; | ||||
} else { | |||||
irdma_memcpy(dest, src, IRDMA_BYTE_16); | for (i = 0; i < num_sges; i++) { | ||||
src += IRDMA_BYTE_16; | u8 *cur_sge = (u8 *)(uintptr_t)sge_list[i].tag_off; | ||||
dest = dest + IRDMA_BYTE_32; | u32 sge_len = sge_list[i].len; | ||||
irdma_memcpy(dest, src, len - IRDMA_BYTE_16); | |||||
while (sge_len) { | |||||
u32 bytes_copied; | |||||
bytes_copied = min(sge_len, quanta_bytes_remaining); | |||||
irdma_memcpy(wqe, cur_sge, bytes_copied); | |||||
wqe += bytes_copied; | |||||
cur_sge += bytes_copied; | |||||
quanta_bytes_remaining -= bytes_copied; | |||||
sge_len -= bytes_copied; | |||||
if (!quanta_bytes_remaining) { | |||||
/* Remaining inline bytes reside after the hdr */ | |||||
wqe += 16; | |||||
quanta_bytes_remaining = 32; | |||||
} | |||||
} | |||||
} | } | ||||
} | } | ||||
Context not available. | |||||
{ | { | ||||
set_64bit_val(wqe, IRDMA_BYTE_0, (uintptr_t)op_info->va); | set_64bit_val(wqe, IRDMA_BYTE_0, (uintptr_t)op_info->va); | ||||
set_64bit_val(wqe, IRDMA_BYTE_8, | set_64bit_val(wqe, IRDMA_BYTE_8, | ||||
LS_64(op_info->mr_stag, IRDMAQPSQ_PARENTMRSTAG) | | FIELD_PREP(IRDMAQPSQ_PARENTMRSTAG, op_info->mr_stag) | | ||||
LS_64(op_info->mw_stag, IRDMAQPSQ_MWSTAG)); | FIELD_PREP(IRDMAQPSQ_MWSTAG, op_info->mw_stag)); | ||||
set_64bit_val(wqe, IRDMA_BYTE_16, op_info->bind_len); | set_64bit_val(wqe, IRDMA_BYTE_16, op_info->bind_len); | ||||
} | } | ||||
/** | /** | ||||
* irdma_copy_inline_data - Copy inline data to wqe | * irdma_copy_inline_data - Copy inline data to wqe | ||||
* @dest: pointer to wqe | * @wqe: pointer to wqe | ||||
* @src: pointer to inline data | * @sge_list: table of pointers to inline data | ||||
* @len: length of inline data to copy | * @num_sges: number of SGE's | ||||
* @polarity: polarity of wqe valid bit | * @polarity: polarity of wqe valid bit | ||||
*/ | */ | ||||
static void | static void | ||||
irdma_copy_inline_data(u8 *dest, u8 *src, u32 len, u8 polarity) | irdma_copy_inline_data(u8 *wqe, struct irdma_sge *sge_list, u32 num_sges, | ||||
u8 polarity) | |||||
{ | { | ||||
u8 inline_valid = polarity << IRDMA_INLINE_VALID_S; | u8 inline_valid = polarity << IRDMA_INLINE_VALID_S; | ||||
u32 copy_size; | u32 quanta_bytes_remaining = 8; | ||||
u32 i; | |||||
dest += IRDMA_BYTE_8; | bool first_quanta = true; | ||||
if (len <= IRDMA_BYTE_8) { | |||||
irdma_memcpy(dest, src, len); | wqe += 8; | ||||
return; | |||||
} | for (i = 0; i < num_sges; i++) { | ||||
u8 *cur_sge = (u8 *)(uintptr_t)sge_list[i].tag_off; | |||||
*((u64 *)dest) = *((u64 *)src); | u32 sge_len = sge_list[i].len; | ||||
len -= IRDMA_BYTE_8; | |||||
src += IRDMA_BYTE_8; | while (sge_len) { | ||||
dest += IRDMA_BYTE_24; /* point to additional 32 byte quanta */ | u32 bytes_copied; | ||||
while (len) { | bytes_copied = min(sge_len, quanta_bytes_remaining); | ||||
copy_size = len < 31 ? len : 31; | irdma_memcpy(wqe, cur_sge, bytes_copied); | ||||
irdma_memcpy(dest, src, copy_size); | wqe += bytes_copied; | ||||
*(dest + 31) = inline_valid; | cur_sge += bytes_copied; | ||||
len -= copy_size; | quanta_bytes_remaining -= bytes_copied; | ||||
dest += IRDMA_BYTE_32; | sge_len -= bytes_copied; | ||||
src += copy_size; | |||||
if (!quanta_bytes_remaining) { | |||||
quanta_bytes_remaining = 31; | |||||
/* Remaining inline bytes reside after the hdr */ | |||||
if (first_quanta) { | |||||
first_quanta = false; | |||||
wqe += 16; | |||||
} else { | |||||
*wqe = inline_valid; | |||||
wqe++; | |||||
} | |||||
} | |||||
} | |||||
} | } | ||||
if (!first_quanta && quanta_bytes_remaining < 31) | |||||
*(wqe + quanta_bytes_remaining) = inline_valid; | |||||
} | } | ||||
/** | /** | ||||
Context not available. | |||||
struct irdma_post_sq_info *info, bool post_sq) | struct irdma_post_sq_info *info, bool post_sq) | ||||
{ | { | ||||
__le64 *wqe; | __le64 *wqe; | ||||
struct irdma_inline_rdma_write *op_info; | struct irdma_rdma_write *op_info; | ||||
u64 hdr = 0; | u64 hdr = 0; | ||||
u32 wqe_idx; | u32 wqe_idx; | ||||
bool read_fence = false; | bool read_fence = false; | ||||
u16 quanta; | u16 quanta; | ||||
u32 i, total_size = 0; | |||||
info->push_wqe = qp->push_db ? true : false; | info->push_wqe = qp->push_db ? true : false; | ||||
op_info = &info->op.inline_rdma_write; | op_info = &info->op.rdma_write; | ||||
if (op_info->len > qp->max_inline_data) | if (unlikely(qp->max_sq_frag_cnt < op_info->num_lo_sges)) | ||||
return -EINVAL; | return -EINVAL; | ||||
quanta = qp->wqe_ops.iw_inline_data_size_to_quanta(op_info->len); | for (i = 0; i < op_info->num_lo_sges; i++) | ||||
wqe = irdma_qp_get_next_send_wqe(qp, &wqe_idx, quanta, op_info->len, | total_size += op_info->lo_sg_list[i].len; | ||||
info); | |||||
if (unlikely(total_size > qp->max_inline_data)) | |||||
return -EINVAL; | |||||
quanta = qp->wqe_ops.iw_inline_data_size_to_quanta(total_size); | |||||
wqe = irdma_qp_get_next_send_wqe(qp, &wqe_idx, &quanta, total_size, info); | |||||
if (!wqe) | if (!wqe) | ||||
return -ENOSPC; | return -ENOSPC; | ||||
irdma_clr_wqes(qp, wqe_idx); | |||||
qp->sq_wrtrk_array[wqe_idx].signaled = info->signaled; | qp->sq_wrtrk_array[wqe_idx].signaled = info->signaled; | ||||
read_fence |= info->read_fence; | read_fence |= info->read_fence; | ||||
set_64bit_val(wqe, IRDMA_BYTE_16, | set_64bit_val(wqe, IRDMA_BYTE_16, | ||||
LS_64(op_info->rem_addr.tag_off, IRDMAQPSQ_FRAG_TO)); | FIELD_PREP(IRDMAQPSQ_FRAG_TO, op_info->rem_addr.tag_off)); | ||||
hdr = LS_64(op_info->rem_addr.stag, IRDMAQPSQ_REMSTAG) | | hdr = FIELD_PREP(IRDMAQPSQ_REMSTAG, op_info->rem_addr.stag) | | ||||
LS_64(info->op_type, IRDMAQPSQ_OPCODE) | | FIELD_PREP(IRDMAQPSQ_OPCODE, info->op_type) | | ||||
LS_64(op_info->len, IRDMAQPSQ_INLINEDATALEN) | | FIELD_PREP(IRDMAQPSQ_INLINEDATALEN, total_size) | | ||||
LS_64(info->report_rtt ? 1 : 0, IRDMAQPSQ_REPORTRTT) | | FIELD_PREP(IRDMAQPSQ_REPORTRTT, info->report_rtt ? 1 : 0) | | ||||
LS_64(1, IRDMAQPSQ_INLINEDATAFLAG) | | FIELD_PREP(IRDMAQPSQ_INLINEDATAFLAG, 1) | | ||||
LS_64(info->imm_data_valid ? 1 : 0, IRDMAQPSQ_IMMDATAFLAG) | | FIELD_PREP(IRDMAQPSQ_IMMDATAFLAG, info->imm_data_valid ? 1 : 0) | | ||||
LS_64(info->push_wqe ? 1 : 0, IRDMAQPSQ_PUSHWQE) | | FIELD_PREP(IRDMAQPSQ_PUSHWQE, info->push_wqe ? 1 : 0) | | ||||
LS_64(read_fence, IRDMAQPSQ_READFENCE) | | FIELD_PREP(IRDMAQPSQ_READFENCE, read_fence) | | ||||
LS_64(info->local_fence, IRDMAQPSQ_LOCALFENCE) | | FIELD_PREP(IRDMAQPSQ_LOCALFENCE, info->local_fence) | | ||||
LS_64(info->signaled, IRDMAQPSQ_SIGCOMPL) | | FIELD_PREP(IRDMAQPSQ_SIGCOMPL, info->signaled) | | ||||
LS_64(qp->swqe_polarity, IRDMAQPSQ_VALID); | FIELD_PREP(IRDMAQPSQ_VALID, qp->swqe_polarity); | ||||
if (info->imm_data_valid) | if (info->imm_data_valid) | ||||
set_64bit_val(wqe, IRDMA_BYTE_0, | set_64bit_val(wqe, IRDMA_BYTE_0, | ||||
LS_64(info->imm_data, IRDMAQPSQ_IMMDATA)); | FIELD_PREP(IRDMAQPSQ_IMMDATA, info->imm_data)); | ||||
qp->wqe_ops.iw_copy_inline_data((u8 *)wqe, op_info->lo_sg_list, | |||||
op_info->num_lo_sges, qp->swqe_polarity); | |||||
qp->wqe_ops.iw_copy_inline_data((u8 *)wqe, op_info->data, op_info->len, | |||||
qp->swqe_polarity); | |||||
irdma_wmb(); /* make sure WQE is populated before valid bit is set */ | irdma_wmb(); /* make sure WQE is populated before valid bit is set */ | ||||
set_64bit_val(wqe, IRDMA_BYTE_24, hdr); | set_64bit_val(wqe, IRDMA_BYTE_24, hdr); | ||||
if (info->push_wqe) { | if (info->push_wqe) | ||||
irdma_qp_push_wqe(qp, wqe, quanta, wqe_idx, post_sq); | irdma_qp_push_wqe(qp, wqe, quanta, wqe_idx, post_sq); | ||||
} else { | else if (post_sq) | ||||
if (post_sq) | irdma_uk_qp_post_wr(qp); | ||||
irdma_uk_qp_post_wr(qp); | |||||
} | |||||
return 0; | return 0; | ||||
} | } | ||||
Context not available. | |||||
struct irdma_post_sq_info *info, bool post_sq) | struct irdma_post_sq_info *info, bool post_sq) | ||||
{ | { | ||||
__le64 *wqe; | __le64 *wqe; | ||||
struct irdma_post_inline_send *op_info; | struct irdma_post_send *op_info; | ||||
u64 hdr; | u64 hdr; | ||||
u32 wqe_idx; | u32 wqe_idx; | ||||
bool read_fence = false; | bool read_fence = false; | ||||
u16 quanta; | u16 quanta; | ||||
u32 i, total_size = 0; | |||||
info->push_wqe = qp->push_db ? true : false; | info->push_wqe = qp->push_db ? true : false; | ||||
op_info = &info->op.inline_send; | op_info = &info->op.send; | ||||
if (op_info->len > qp->max_inline_data) | if (unlikely(qp->max_sq_frag_cnt < op_info->num_sges)) | ||||
return -EINVAL; | return -EINVAL; | ||||
quanta = qp->wqe_ops.iw_inline_data_size_to_quanta(op_info->len); | for (i = 0; i < op_info->num_sges; i++) | ||||
wqe = irdma_qp_get_next_send_wqe(qp, &wqe_idx, quanta, op_info->len, | total_size += op_info->sg_list[i].len; | ||||
info); | |||||
if (unlikely(total_size > qp->max_inline_data)) | |||||
return -EINVAL; | |||||
quanta = qp->wqe_ops.iw_inline_data_size_to_quanta(total_size); | |||||
wqe = irdma_qp_get_next_send_wqe(qp, &wqe_idx, &quanta, total_size, info); | |||||
if (!wqe) | if (!wqe) | ||||
return -ENOSPC; | return -ENOSPC; | ||||
irdma_clr_wqes(qp, wqe_idx); | |||||
set_64bit_val(wqe, IRDMA_BYTE_16, | set_64bit_val(wqe, IRDMA_BYTE_16, | ||||
LS_64(op_info->qkey, IRDMAQPSQ_DESTQKEY) | | FIELD_PREP(IRDMAQPSQ_DESTQKEY, op_info->qkey) | | ||||
LS_64(op_info->dest_qp, IRDMAQPSQ_DESTQPN)); | FIELD_PREP(IRDMAQPSQ_DESTQPN, op_info->dest_qp)); | ||||
read_fence |= info->read_fence; | read_fence |= info->read_fence; | ||||
hdr = LS_64(info->stag_to_inv, IRDMAQPSQ_REMSTAG) | | hdr = FIELD_PREP(IRDMAQPSQ_REMSTAG, info->stag_to_inv) | | ||||
LS_64(op_info->ah_id, IRDMAQPSQ_AHID) | | FIELD_PREP(IRDMAQPSQ_AHID, op_info->ah_id) | | ||||
LS_64(info->op_type, IRDMAQPSQ_OPCODE) | | FIELD_PREP(IRDMAQPSQ_OPCODE, info->op_type) | | ||||
LS_64(op_info->len, IRDMAQPSQ_INLINEDATALEN) | | FIELD_PREP(IRDMAQPSQ_INLINEDATALEN, total_size) | | ||||
LS_64((info->imm_data_valid ? 1 : 0), IRDMAQPSQ_IMMDATAFLAG) | | FIELD_PREP(IRDMAQPSQ_IMMDATAFLAG, | ||||
LS_64((info->report_rtt ? 1 : 0), IRDMAQPSQ_REPORTRTT) | | (info->imm_data_valid ? 1 : 0)) | | ||||
LS_64(1, IRDMAQPSQ_INLINEDATAFLAG) | | FIELD_PREP(IRDMAQPSQ_REPORTRTT, (info->report_rtt ? 1 : 0)) | | ||||
LS_64((info->push_wqe ? 1 : 0), IRDMAQPSQ_PUSHWQE) | | FIELD_PREP(IRDMAQPSQ_INLINEDATAFLAG, 1) | | ||||
LS_64(read_fence, IRDMAQPSQ_READFENCE) | | FIELD_PREP(IRDMAQPSQ_PUSHWQE, info->push_wqe) | | ||||
LS_64(info->local_fence, IRDMAQPSQ_LOCALFENCE) | | FIELD_PREP(IRDMAQPSQ_READFENCE, read_fence) | | ||||
LS_64(info->signaled, IRDMAQPSQ_SIGCOMPL) | | FIELD_PREP(IRDMAQPSQ_LOCALFENCE, info->local_fence) | | ||||
LS_64(info->udp_hdr, IRDMAQPSQ_UDPHEADER) | | FIELD_PREP(IRDMAQPSQ_SIGCOMPL, info->signaled) | | ||||
LS_64(info->l4len, IRDMAQPSQ_L4LEN) | | FIELD_PREP(IRDMAQPSQ_UDPHEADER, info->udp_hdr) | | ||||
LS_64(qp->swqe_polarity, IRDMAQPSQ_VALID); | FIELD_PREP(IRDMAQPSQ_L4LEN, info->l4len) | | ||||
FIELD_PREP(IRDMAQPSQ_VALID, qp->swqe_polarity); | |||||
if (info->imm_data_valid) | if (info->imm_data_valid) | ||||
set_64bit_val(wqe, IRDMA_BYTE_0, | set_64bit_val(wqe, IRDMA_BYTE_0, | ||||
LS_64(info->imm_data, IRDMAQPSQ_IMMDATA)); | FIELD_PREP(IRDMAQPSQ_IMMDATA, info->imm_data)); | ||||
qp->wqe_ops.iw_copy_inline_data((u8 *)wqe, op_info->data, op_info->len, | qp->wqe_ops.iw_copy_inline_data((u8 *)wqe, op_info->sg_list, | ||||
qp->swqe_polarity); | op_info->num_sges, qp->swqe_polarity); | ||||
irdma_wmb(); /* make sure WQE is populated before valid bit is set */ | irdma_wmb(); /* make sure WQE is populated before valid bit is set */ | ||||
set_64bit_val(wqe, IRDMA_BYTE_24, hdr); | set_64bit_val(wqe, IRDMA_BYTE_24, hdr); | ||||
if (info->push_wqe) { | if (info->push_wqe) | ||||
irdma_qp_push_wqe(qp, wqe, quanta, wqe_idx, post_sq); | irdma_qp_push_wqe(qp, wqe, quanta, wqe_idx, post_sq); | ||||
} else { | else if (post_sq) | ||||
if (post_sq) | irdma_uk_qp_post_wr(qp); | ||||
irdma_uk_qp_post_wr(qp); | |||||
} | |||||
return 0; | return 0; | ||||
} | } | ||||
Context not available. | |||||
u32 wqe_idx; | u32 wqe_idx; | ||||
bool local_fence = false; | bool local_fence = false; | ||||
struct irdma_sge sge = {0}; | struct irdma_sge sge = {0}; | ||||
u16 quanta = IRDMA_QP_WQE_MIN_QUANTA; | |||||
info->push_wqe = qp->push_db ? true : false; | info->push_wqe = qp->push_db ? true : false; | ||||
op_info = &info->op.inv_local_stag; | op_info = &info->op.inv_local_stag; | ||||
local_fence = info->local_fence; | local_fence = info->local_fence; | ||||
wqe = irdma_qp_get_next_send_wqe(qp, &wqe_idx, IRDMA_QP_WQE_MIN_QUANTA, | wqe = irdma_qp_get_next_send_wqe(qp, &wqe_idx, &quanta, 0, info); | ||||
0, info); | |||||
if (!wqe) | if (!wqe) | ||||
return -ENOSPC; | return -ENOSPC; | ||||
irdma_clr_wqes(qp, wqe_idx); | |||||
sge.stag = op_info->target_stag; | sge.stag = op_info->target_stag; | ||||
qp->wqe_ops.iw_set_fragment(wqe, IRDMA_BYTE_0, &sge, 0); | qp->wqe_ops.iw_set_fragment(wqe, IRDMA_BYTE_0, &sge, 0); | ||||
set_64bit_val(wqe, IRDMA_BYTE_16, 0); | set_64bit_val(wqe, IRDMA_BYTE_16, 0); | ||||
hdr = LS_64(IRDMA_OP_TYPE_INV_STAG, IRDMAQPSQ_OPCODE) | | hdr = FIELD_PREP(IRDMAQPSQ_OPCODE, IRDMA_OP_TYPE_INV_STAG) | | ||||
LS_64((info->push_wqe ? 1 : 0), IRDMAQPSQ_PUSHWQE) | | FIELD_PREP(IRDMAQPSQ_PUSHWQE, info->push_wqe) | | ||||
LS_64(info->read_fence, IRDMAQPSQ_READFENCE) | | FIELD_PREP(IRDMAQPSQ_READFENCE, info->read_fence) | | ||||
LS_64(local_fence, IRDMAQPSQ_LOCALFENCE) | | FIELD_PREP(IRDMAQPSQ_LOCALFENCE, local_fence) | | ||||
LS_64(info->signaled, IRDMAQPSQ_SIGCOMPL) | | FIELD_PREP(IRDMAQPSQ_SIGCOMPL, info->signaled) | | ||||
LS_64(qp->swqe_polarity, IRDMAQPSQ_VALID); | FIELD_PREP(IRDMAQPSQ_VALID, qp->swqe_polarity); | ||||
irdma_wmb(); /* make sure WQE is populated before valid bit is set */ | irdma_wmb(); /* make sure WQE is populated before valid bit is set */ | ||||
set_64bit_val(wqe, IRDMA_BYTE_24, hdr); | set_64bit_val(wqe, IRDMA_BYTE_24, hdr); | ||||
if (info->push_wqe) { | if (info->push_wqe) | ||||
irdma_qp_push_wqe(qp, wqe, IRDMA_QP_WQE_MIN_QUANTA, wqe_idx, | irdma_qp_push_wqe(qp, wqe, quanta, wqe_idx, post_sq); | ||||
post_sq); | else if (post_sq) | ||||
} else { | irdma_uk_qp_post_wr(qp); | ||||
if (post_sq) | |||||
irdma_uk_qp_post_wr(qp); | |||||
} | |||||
return 0; | return 0; | ||||
} | } | ||||
Context not available. | |||||
u64 hdr; | u64 hdr; | ||||
u32 wqe_idx; | u32 wqe_idx; | ||||
bool local_fence; | bool local_fence; | ||||
u16 quanta = IRDMA_QP_WQE_MIN_QUANTA; | |||||
info->push_wqe = qp->push_db ? true : false; | info->push_wqe = qp->push_db ? true : false; | ||||
op_info = &info->op.bind_window; | op_info = &info->op.bind_window; | ||||
local_fence = info->local_fence; | local_fence = info->local_fence; | ||||
wqe = irdma_qp_get_next_send_wqe(qp, &wqe_idx, IRDMA_QP_WQE_MIN_QUANTA, | wqe = irdma_qp_get_next_send_wqe(qp, &wqe_idx, &quanta, 0, info); | ||||
0, info); | |||||
if (!wqe) | if (!wqe) | ||||
return -ENOSPC; | return -ENOSPC; | ||||
irdma_clr_wqes(qp, wqe_idx); | |||||
qp->wqe_ops.iw_set_mw_bind_wqe(wqe, op_info); | qp->wqe_ops.iw_set_mw_bind_wqe(wqe, op_info); | ||||
hdr = LS_64(IRDMA_OP_TYPE_BIND_MW, IRDMAQPSQ_OPCODE) | | hdr = FIELD_PREP(IRDMAQPSQ_OPCODE, IRDMA_OP_TYPE_BIND_MW) | | ||||
LS_64(((op_info->ena_reads << 2) | (op_info->ena_writes << 3)), | FIELD_PREP(IRDMAQPSQ_STAGRIGHTS, | ||||
IRDMAQPSQ_STAGRIGHTS) | | ((op_info->ena_reads << 2) | (op_info->ena_writes << 3))) | | ||||
LS_64((op_info->addressing_type == IRDMA_ADDR_TYPE_VA_BASED ? 1 : 0), | FIELD_PREP(IRDMAQPSQ_VABASEDTO, | ||||
IRDMAQPSQ_VABASEDTO) | | (op_info->addressing_type == IRDMA_ADDR_TYPE_VA_BASED ? 1 : 0)) | | ||||
LS_64((op_info->mem_window_type_1 ? 1 : 0), | FIELD_PREP(IRDMAQPSQ_MEMWINDOWTYPE, | ||||
IRDMAQPSQ_MEMWINDOWTYPE) | | (op_info->mem_window_type_1 ? 1 : 0)) | | ||||
LS_64((info->push_wqe ? 1 : 0), IRDMAQPSQ_PUSHWQE) | | FIELD_PREP(IRDMAQPSQ_PUSHWQE, info->push_wqe) | | ||||
LS_64(info->read_fence, IRDMAQPSQ_READFENCE) | | FIELD_PREP(IRDMAQPSQ_READFENCE, info->read_fence) | | ||||
LS_64(local_fence, IRDMAQPSQ_LOCALFENCE) | | FIELD_PREP(IRDMAQPSQ_LOCALFENCE, local_fence) | | ||||
LS_64(info->signaled, IRDMAQPSQ_SIGCOMPL) | | FIELD_PREP(IRDMAQPSQ_SIGCOMPL, info->signaled) | | ||||
LS_64(qp->swqe_polarity, IRDMAQPSQ_VALID); | FIELD_PREP(IRDMAQPSQ_VALID, qp->swqe_polarity); | ||||
irdma_wmb(); /* make sure WQE is populated before valid bit is set */ | irdma_wmb(); /* make sure WQE is populated before valid bit is set */ | ||||
set_64bit_val(wqe, IRDMA_BYTE_24, hdr); | set_64bit_val(wqe, IRDMA_BYTE_24, hdr); | ||||
if (info->push_wqe) { | if (info->push_wqe) | ||||
irdma_qp_push_wqe(qp, wqe, IRDMA_QP_WQE_MIN_QUANTA, wqe_idx, | irdma_qp_push_wqe(qp, wqe, quanta, wqe_idx, post_sq); | ||||
post_sq); | else if (post_sq) | ||||
} else { | irdma_uk_qp_post_wr(qp); | ||||
if (post_sq) | |||||
irdma_uk_qp_post_wr(qp); | |||||
} | |||||
return 0; | return 0; | ||||
} | } | ||||
Context not available. | |||||
} | } | ||||
set_64bit_val(wqe, IRDMA_BYTE_16, 0); | set_64bit_val(wqe, IRDMA_BYTE_16, 0); | ||||
hdr = LS_64(addl_frag_cnt, IRDMAQPSQ_ADDFRAGCNT) | | hdr = FIELD_PREP(IRDMAQPSQ_ADDFRAGCNT, addl_frag_cnt) | | ||||
LS_64(qp->rwqe_polarity, IRDMAQPSQ_VALID); | FIELD_PREP(IRDMAQPSQ_VALID, qp->rwqe_polarity); | ||||
irdma_wmb(); /* make sure WQE is populated before valid bit is set */ | irdma_wmb(); /* make sure WQE is populated before valid bit is set */ | ||||
Context not available. | |||||
get_64bit_val(cq->shadow_area, 32, &temp_val); | get_64bit_val(cq->shadow_area, 32, &temp_val); | ||||
sw_cq_sel = (u16)RS_64(temp_val, IRDMA_CQ_DBSA_SW_CQ_SELECT); | sw_cq_sel = (u16)FIELD_GET(IRDMA_CQ_DBSA_SW_CQ_SELECT, temp_val); | ||||
sw_cq_sel += cq_cnt; | sw_cq_sel += cq_cnt; | ||||
arm_seq_num = (u8)RS_64(temp_val, IRDMA_CQ_DBSA_ARM_SEQ_NUM); | arm_seq_num = (u8)FIELD_GET(IRDMA_CQ_DBSA_ARM_SEQ_NUM, temp_val); | ||||
arm_next_se = (u8)RS_64(temp_val, IRDMA_CQ_DBSA_ARM_NEXT_SE); | arm_next_se = (u8)FIELD_GET(IRDMA_CQ_DBSA_ARM_NEXT_SE, temp_val); | ||||
arm_next = (u8)RS_64(temp_val, IRDMA_CQ_DBSA_ARM_NEXT); | arm_next = (u8)FIELD_GET(IRDMA_CQ_DBSA_ARM_NEXT, temp_val); | ||||
temp_val = LS_64(arm_seq_num, IRDMA_CQ_DBSA_ARM_SEQ_NUM) | | temp_val = FIELD_PREP(IRDMA_CQ_DBSA_ARM_SEQ_NUM, arm_seq_num) | | ||||
LS_64(sw_cq_sel, IRDMA_CQ_DBSA_SW_CQ_SELECT) | | FIELD_PREP(IRDMA_CQ_DBSA_SW_CQ_SELECT, sw_cq_sel) | | ||||
LS_64(arm_next_se, IRDMA_CQ_DBSA_ARM_NEXT_SE) | | FIELD_PREP(IRDMA_CQ_DBSA_ARM_NEXT_SE, arm_next_se) | | ||||
LS_64(arm_next, IRDMA_CQ_DBSA_ARM_NEXT); | FIELD_PREP(IRDMA_CQ_DBSA_ARM_NEXT, arm_next); | ||||
set_64bit_val(cq->shadow_area, 32, temp_val); | set_64bit_val(cq->shadow_area, 32, temp_val); | ||||
} | } | ||||
Context not available. | |||||
cq->armed = true; | cq->armed = true; | ||||
get_64bit_val(cq->shadow_area, IRDMA_BYTE_32, &temp_val); | get_64bit_val(cq->shadow_area, IRDMA_BYTE_32, &temp_val); | ||||
arm_seq_num = (u8)RS_64(temp_val, IRDMA_CQ_DBSA_ARM_SEQ_NUM); | arm_seq_num = (u8)FIELD_GET(IRDMA_CQ_DBSA_ARM_SEQ_NUM, temp_val); | ||||
arm_seq_num++; | arm_seq_num++; | ||||
sw_cq_sel = (u16)RS_64(temp_val, IRDMA_CQ_DBSA_SW_CQ_SELECT); | sw_cq_sel = (u16)FIELD_GET(IRDMA_CQ_DBSA_SW_CQ_SELECT, temp_val); | ||||
arm_next_se = (u8)RS_64(temp_val, IRDMA_CQ_DBSA_ARM_NEXT_SE); | arm_next_se = (u8)FIELD_GET(IRDMA_CQ_DBSA_ARM_NEXT_SE, temp_val); | ||||
arm_next_se |= 1; | arm_next_se |= 1; | ||||
if (cq_notify == IRDMA_CQ_COMPL_EVENT) | if (cq_notify == IRDMA_CQ_COMPL_EVENT) | ||||
arm_next = 1; | arm_next = 1; | ||||
temp_val = LS_64(arm_seq_num, IRDMA_CQ_DBSA_ARM_SEQ_NUM) | | temp_val = FIELD_PREP(IRDMA_CQ_DBSA_ARM_SEQ_NUM, arm_seq_num) | | ||||
LS_64(sw_cq_sel, IRDMA_CQ_DBSA_SW_CQ_SELECT) | | FIELD_PREP(IRDMA_CQ_DBSA_SW_CQ_SELECT, sw_cq_sel) | | ||||
LS_64(arm_next_se, IRDMA_CQ_DBSA_ARM_NEXT_SE) | | FIELD_PREP(IRDMA_CQ_DBSA_ARM_NEXT_SE, arm_next_se) | | ||||
LS_64(arm_next, IRDMA_CQ_DBSA_ARM_NEXT); | FIELD_PREP(IRDMA_CQ_DBSA_ARM_NEXT, arm_next); | ||||
set_64bit_val(cq->shadow_area, IRDMA_BYTE_32, temp_val); | set_64bit_val(cq->shadow_area, IRDMA_BYTE_32, temp_val); | ||||
Context not available. | |||||
__le64 *cqe; | __le64 *cqe; | ||||
struct irdma_qp_uk *qp; | struct irdma_qp_uk *qp; | ||||
struct irdma_ring *pring = NULL; | struct irdma_ring *pring = NULL; | ||||
u32 wqe_idx, q_type; | u32 wqe_idx; | ||||
int ret_code; | int ret_code; | ||||
bool move_cq_head = true; | bool move_cq_head = true; | ||||
u8 polarity; | u8 polarity; | ||||
Context not available. | |||||
cqe = IRDMA_GET_CURRENT_CQ_ELEM(cq); | cqe = IRDMA_GET_CURRENT_CQ_ELEM(cq); | ||||
get_64bit_val(cqe, IRDMA_BYTE_24, &qword3); | get_64bit_val(cqe, IRDMA_BYTE_24, &qword3); | ||||
polarity = (u8)RS_64(qword3, IRDMA_CQ_VALID); | polarity = (u8)FIELD_GET(IRDMA_CQ_VALID, qword3); | ||||
if (polarity != cq->polarity) | if (polarity != cq->polarity) | ||||
return -ENOENT; | return -ENOENT; | ||||
/* Ensure CQE contents are read after valid bit is checked */ | /* Ensure CQE contents are read after valid bit is checked */ | ||||
rmb(); | rmb(); | ||||
ext_valid = (bool)RS_64(qword3, IRDMA_CQ_EXTCQE); | ext_valid = (bool)FIELD_GET(IRDMA_CQ_EXTCQE, qword3); | ||||
if (ext_valid) { | if (ext_valid) { | ||||
u64 qword6, qword7; | u64 qword6, qword7; | ||||
u32 peek_head; | u32 peek_head; | ||||
Context not available. | |||||
if (cq->avoid_mem_cflct) { | if (cq->avoid_mem_cflct) { | ||||
ext_cqe = (__le64 *) ((u8 *)cqe + 32); | ext_cqe = (__le64 *) ((u8 *)cqe + 32); | ||||
get_64bit_val(ext_cqe, IRDMA_BYTE_24, &qword7); | get_64bit_val(ext_cqe, IRDMA_BYTE_24, &qword7); | ||||
polarity = (u8)RS_64(qword7, IRDMA_CQ_VALID); | polarity = (u8)FIELD_GET(IRDMA_CQ_VALID, qword7); | ||||
} else { | } else { | ||||
peek_head = (cq->cq_ring.head + 1) % cq->cq_ring.size; | peek_head = (cq->cq_ring.head + 1) % cq->cq_ring.size; | ||||
ext_cqe = cq->cq_base[peek_head].buf; | ext_cqe = cq->cq_base[peek_head].buf; | ||||
get_64bit_val(ext_cqe, IRDMA_BYTE_24, &qword7); | get_64bit_val(ext_cqe, IRDMA_BYTE_24, &qword7); | ||||
polarity = (u8)RS_64(qword7, IRDMA_CQ_VALID); | polarity = (u8)FIELD_GET(IRDMA_CQ_VALID, qword7); | ||||
if (!peek_head) | if (!peek_head) | ||||
polarity ^= 1; | polarity ^= 1; | ||||
} | } | ||||
Context not available. | |||||
/* Ensure ext CQE contents are read after ext valid bit is checked */ | /* Ensure ext CQE contents are read after ext valid bit is checked */ | ||||
rmb(); | rmb(); | ||||
info->imm_valid = (bool)RS_64(qword7, IRDMA_CQ_IMMVALID); | info->imm_valid = (bool)FIELD_GET(IRDMA_CQ_IMMVALID, qword7); | ||||
if (info->imm_valid) { | if (info->imm_valid) { | ||||
u64 qword4; | u64 qword4; | ||||
get_64bit_val(ext_cqe, IRDMA_BYTE_0, &qword4); | get_64bit_val(ext_cqe, IRDMA_BYTE_0, &qword4); | ||||
info->imm_data = (u32)RS_64(qword4, IRDMA_CQ_IMMDATALOW32); | info->imm_data = (u32)FIELD_GET(IRDMA_CQ_IMMDATALOW32, qword4); | ||||
} | } | ||||
info->ud_smac_valid = (bool)RS_64(qword7, IRDMA_CQ_UDSMACVALID); | info->ud_smac_valid = (bool)FIELD_GET(IRDMA_CQ_UDSMACVALID, qword7); | ||||
info->ud_vlan_valid = (bool)RS_64(qword7, IRDMA_CQ_UDVLANVALID); | info->ud_vlan_valid = (bool)FIELD_GET(IRDMA_CQ_UDVLANVALID, qword7); | ||||
if (info->ud_smac_valid || info->ud_vlan_valid) { | if (info->ud_smac_valid || info->ud_vlan_valid) { | ||||
get_64bit_val(ext_cqe, IRDMA_BYTE_16, &qword6); | get_64bit_val(ext_cqe, IRDMA_BYTE_16, &qword6); | ||||
if (info->ud_vlan_valid) | if (info->ud_vlan_valid) | ||||
info->ud_vlan = (u16)RS_64(qword6, IRDMA_CQ_UDVLAN); | info->ud_vlan = (u16)FIELD_GET(IRDMA_CQ_UDVLAN, qword6); | ||||
if (info->ud_smac_valid) { | if (info->ud_smac_valid) { | ||||
info->ud_smac[5] = qword6 & 0xFF; | info->ud_smac[5] = qword6 & 0xFF; | ||||
info->ud_smac[4] = (qword6 >> 8) & 0xFF; | info->ud_smac[4] = (qword6 >> 8) & 0xFF; | ||||
Context not available. | |||||
info->ud_vlan_valid = false; | info->ud_vlan_valid = false; | ||||
} | } | ||||
q_type = (u8)RS_64(qword3, IRDMA_CQ_SQ); | info->q_type = (u8)FIELD_GET(IRDMA_CQ_SQ, qword3); | ||||
info->error = (bool)RS_64(qword3, IRDMA_CQ_ERROR); | info->error = (bool)FIELD_GET(IRDMA_CQ_ERROR, qword3); | ||||
info->push_dropped = (bool)RS_64(qword3, IRDMACQ_PSHDROP); | info->push_dropped = (bool)FIELD_GET(IRDMACQ_PSHDROP, qword3); | ||||
info->ipv4 = (bool)RS_64(qword3, IRDMACQ_IPV4); | info->ipv4 = (bool)FIELD_GET(IRDMACQ_IPV4, qword3); | ||||
if (info->error) { | if (info->error) { | ||||
info->major_err = RS_64(qword3, IRDMA_CQ_MAJERR); | info->major_err = FIELD_GET(IRDMA_CQ_MAJERR, qword3); | ||||
info->minor_err = RS_64(qword3, IRDMA_CQ_MINERR); | info->minor_err = FIELD_GET(IRDMA_CQ_MINERR, qword3); | ||||
if (info->major_err == IRDMA_FLUSH_MAJOR_ERR) { | switch (info->major_err) { | ||||
info->comp_status = IRDMA_COMPL_STATUS_FLUSHED; | case IRDMA_FLUSH_MAJOR_ERR: | ||||
/* Set the min error to standard flush error code for remaining cqes */ | /* Set the min error to standard flush error code for remaining cqes */ | ||||
if (info->minor_err != FLUSH_GENERAL_ERR) { | if (info->minor_err != FLUSH_GENERAL_ERR) { | ||||
qword3 &= ~IRDMA_CQ_MINERR_M; | qword3 &= ~IRDMA_CQ_MINERR; | ||||
qword3 |= LS_64(FLUSH_GENERAL_ERR, IRDMA_CQ_MINERR); | qword3 |= FIELD_PREP(IRDMA_CQ_MINERR, FLUSH_GENERAL_ERR); | ||||
set_64bit_val(cqe, IRDMA_BYTE_24, qword3); | set_64bit_val(cqe, IRDMA_BYTE_24, qword3); | ||||
} | } | ||||
} else { | info->comp_status = IRDMA_COMPL_STATUS_FLUSHED; | ||||
break; | |||||
default: | |||||
info->comp_status = IRDMA_COMPL_STATUS_UNKNOWN; | info->comp_status = IRDMA_COMPL_STATUS_UNKNOWN; | ||||
break; | |||||
} | } | ||||
} else { | } else { | ||||
info->comp_status = IRDMA_COMPL_STATUS_SUCCESS; | info->comp_status = IRDMA_COMPL_STATUS_SUCCESS; | ||||
Context not available. | |||||
get_64bit_val(cqe, IRDMA_BYTE_0, &qword0); | get_64bit_val(cqe, IRDMA_BYTE_0, &qword0); | ||||
get_64bit_val(cqe, IRDMA_BYTE_16, &qword2); | get_64bit_val(cqe, IRDMA_BYTE_16, &qword2); | ||||
info->tcp_seq_num_rtt = (u32)RS_64(qword0, IRDMACQ_TCPSEQNUMRTT); | info->stat.raw = (u32)FIELD_GET(IRDMACQ_TCPSQN_ROCEPSN_RTT_TS, qword0); | ||||
info->qp_id = (u32)RS_64(qword2, IRDMACQ_QPID); | info->qp_id = (u32)FIELD_GET(IRDMACQ_QPID, qword2); | ||||
info->ud_src_qpn = (u32)RS_64(qword2, IRDMACQ_UDSRCQPN); | info->ud_src_qpn = (u32)FIELD_GET(IRDMACQ_UDSRCQPN, qword2); | ||||
get_64bit_val(cqe, IRDMA_BYTE_8, &comp_ctx); | get_64bit_val(cqe, IRDMA_BYTE_8, &comp_ctx); | ||||
info->solicited_event = (bool)RS_64(qword3, IRDMACQ_SOEVENT); | info->solicited_event = (bool)FIELD_GET(IRDMACQ_SOEVENT, qword3); | ||||
qp = (struct irdma_qp_uk *)(irdma_uintptr) comp_ctx; | qp = (struct irdma_qp_uk *)(irdma_uintptr) comp_ctx; | ||||
if (!qp || qp->destroy_pending) { | if (!qp || qp->destroy_pending) { | ||||
ret_code = -EFAULT; | ret_code = -EFAULT; | ||||
goto exit; | goto exit; | ||||
} | } | ||||
wqe_idx = (u32)RS_64(qword3, IRDMA_CQ_WQEIDX); | wqe_idx = (u32)FIELD_GET(IRDMA_CQ_WQEIDX, qword3); | ||||
info->qp_handle = (irdma_qp_handle) (irdma_uintptr) qp; | info->qp_handle = (irdma_qp_handle) (irdma_uintptr) qp; | ||||
info->op_type = (u8)FIELD_GET(IRDMACQ_OP, qword3); | |||||
if (q_type == IRDMA_CQE_QTYPE_RQ) { | if (info->q_type == IRDMA_CQE_QTYPE_RQ) { | ||||
u32 array_idx; | u32 array_idx; | ||||
ret_code = irdma_skip_duplicate_flush_cmpl(qp->rq_ring, | ret_code = irdma_skip_duplicate_flush_cmpl(qp->rq_ring, | ||||
Context not available. | |||||
} | } | ||||
} | } | ||||
info->bytes_xfered = (u32)RS_64(qword0, IRDMACQ_PAYLDLEN); | info->bytes_xfered = (u32)FIELD_GET(IRDMACQ_PAYLDLEN, qword0); | ||||
if (info->imm_valid) | if (qword3 & IRDMACQ_STAG) { | ||||
info->op_type = IRDMA_OP_TYPE_REC_IMM; | |||||
else | |||||
info->op_type = IRDMA_OP_TYPE_REC; | |||||
if (qword3 & IRDMACQ_STAG_M) { | |||||
info->stag_invalid_set = true; | info->stag_invalid_set = true; | ||||
info->inv_stag = (u32)RS_64(qword2, IRDMACQ_INVSTAG); | info->inv_stag = (u32)FIELD_GET(IRDMACQ_INVSTAG, qword2); | ||||
} else { | } else { | ||||
info->stag_invalid_set = false; | info->stag_invalid_set = false; | ||||
} | } | ||||
Context not available. | |||||
info->signaled = qp->sq_wrtrk_array[wqe_idx].signaled; | info->signaled = qp->sq_wrtrk_array[wqe_idx].signaled; | ||||
if (!info->comp_status) | if (!info->comp_status) | ||||
info->bytes_xfered = qp->sq_wrtrk_array[wqe_idx].wr_len; | info->bytes_xfered = qp->sq_wrtrk_array[wqe_idx].wr_len; | ||||
info->op_type = (u8)RS_64(qword3, IRDMACQ_OP); | info->op_type = (u8)FIELD_GET(IRDMACQ_OP, qword3); | ||||
IRDMA_RING_SET_TAIL(qp->sq_ring, | IRDMA_RING_SET_TAIL(qp->sq_ring, | ||||
wqe_idx + qp->sq_wrtrk_array[wqe_idx].quanta); | wqe_idx + qp->sq_wrtrk_array[wqe_idx].quanta); | ||||
} else { | } else { | ||||
unsigned long flags; | |||||
spin_lock_irqsave(qp->lock, flags); | |||||
if (!IRDMA_RING_MORE_WORK(qp->sq_ring)) { | if (!IRDMA_RING_MORE_WORK(qp->sq_ring)) { | ||||
spin_unlock_irqrestore(qp->lock, flags); | |||||
ret_code = -ENOENT; | ret_code = -ENOENT; | ||||
goto exit; | goto exit; | ||||
} | } | ||||
Context not available. | |||||
do { | do { | ||||
__le64 *sw_wqe; | __le64 *sw_wqe; | ||||
u64 wqe_qword; | u64 wqe_qword; | ||||
u8 op_type; | |||||
u32 tail; | u32 tail; | ||||
tail = qp->sq_ring.tail; | tail = qp->sq_ring.tail; | ||||
sw_wqe = qp->sq_base[tail].elem; | sw_wqe = qp->sq_base[tail].elem; | ||||
get_64bit_val(sw_wqe, IRDMA_BYTE_24, | get_64bit_val(sw_wqe, IRDMA_BYTE_24, | ||||
&wqe_qword); | &wqe_qword); | ||||
op_type = (u8)RS_64(wqe_qword, IRDMAQPSQ_OPCODE); | info->op_type = (u8)FIELD_GET(IRDMAQPSQ_OPCODE, wqe_qword); | ||||
info->op_type = op_type; | |||||
IRDMA_RING_SET_TAIL(qp->sq_ring, | IRDMA_RING_SET_TAIL(qp->sq_ring, | ||||
tail + qp->sq_wrtrk_array[tail].quanta); | tail + qp->sq_wrtrk_array[tail].quanta); | ||||
if (op_type != IRDMAQP_OP_NOP) { | if (info->op_type != IRDMAQP_OP_NOP) { | ||||
info->wr_id = qp->sq_wrtrk_array[tail].wrid; | info->wr_id = qp->sq_wrtrk_array[tail].wrid; | ||||
info->signaled = qp->sq_wrtrk_array[tail].signaled; | info->signaled = qp->sq_wrtrk_array[tail].signaled; | ||||
info->bytes_xfered = qp->sq_wrtrk_array[tail].wr_len; | info->bytes_xfered = qp->sq_wrtrk_array[tail].wr_len; | ||||
break; | break; | ||||
} | } | ||||
} while (1); | } while (1); | ||||
if (info->op_type == IRDMA_OP_TYPE_BIND_MW && info->minor_err == FLUSH_PROT_ERR) | |||||
info->minor_err = FLUSH_MW_BIND_ERR; | |||||
qp->sq_flush_seen = true; | qp->sq_flush_seen = true; | ||||
if (!IRDMA_RING_MORE_WORK(qp->sq_ring)) | if (!IRDMA_RING_MORE_WORK(qp->sq_ring)) | ||||
qp->sq_flush_complete = true; | qp->sq_flush_complete = true; | ||||
spin_unlock_irqrestore(qp->lock, flags); | |||||
} | } | ||||
pring = &qp->sq_ring; | pring = &qp->sq_ring; | ||||
} | } | ||||
Context not available. | |||||
ret_code = 0; | ret_code = 0; | ||||
exit: | exit: | ||||
if (!ret_code && info->comp_status == IRDMA_COMPL_STATUS_FLUSHED) | if (!ret_code && info->comp_status == IRDMA_COMPL_STATUS_FLUSHED) { | ||||
if (pring && IRDMA_RING_MORE_WORK(*pring)) | if (pring && IRDMA_RING_MORE_WORK(*pring)) | ||||
move_cq_head = false; | move_cq_head = false; | ||||
} | |||||
if (move_cq_head) { | if (move_cq_head) { | ||||
IRDMA_RING_MOVE_HEAD_NOCHECK(cq->cq_ring); | IRDMA_RING_MOVE_HEAD_NOCHECK(cq->cq_ring); | ||||
Context not available. | |||||
set_64bit_val(cq->shadow_area, IRDMA_BYTE_0, | set_64bit_val(cq->shadow_area, IRDMA_BYTE_0, | ||||
IRDMA_RING_CURRENT_HEAD(cq->cq_ring)); | IRDMA_RING_CURRENT_HEAD(cq->cq_ring)); | ||||
} else { | } else { | ||||
qword3 &= ~IRDMA_CQ_WQEIDX_M; | qword3 &= ~IRDMA_CQ_WQEIDX; | ||||
qword3 |= LS_64(pring->tail, IRDMA_CQ_WQEIDX); | qword3 |= FIELD_PREP(IRDMA_CQ_WQEIDX, pring->tail); | ||||
set_64bit_val(cqe, IRDMA_BYTE_24, qword3); | set_64bit_val(cqe, IRDMA_BYTE_24, qword3); | ||||
} | } | ||||
Context not available. | |||||
} | } | ||||
/** | /** | ||||
* irdma_qp_round_up - return round up qp wq depth | * irdma_round_up_wq - return round up qp wq depth | ||||
* @wqdepth: wq depth in quanta to round up | * @wqdepth: wq depth in quanta to round up | ||||
*/ | */ | ||||
static int | static int | ||||
irdma_qp_round_up(u32 wqdepth) | irdma_round_up_wq(u32 wqdepth) | ||||
{ | { | ||||
int scount = 1; | int scount = 1; | ||||
Context not available. | |||||
} | } | ||||
/* | /* | ||||
* irdma_get_sqdepth - get SQ depth (quanta) @max_hw_wq_quanta: HW SQ size limit @sq_size: SQ size @shift: shift which | * irdma_get_sqdepth - get SQ depth (quanta) @uk_attrs: qp HW attributes @sq_size: SQ size @shift: shift which | ||||
* determines size of WQE @sqdepth: depth of SQ | * determines size of WQE @sqdepth: depth of SQ | ||||
*/ | */ | ||||
int | int | ||||
irdma_get_sqdepth(u32 max_hw_wq_quanta, u32 sq_size, u8 shift, u32 *sqdepth) | irdma_get_sqdepth(struct irdma_uk_attrs *uk_attrs, u32 sq_size, u8 shift, u32 *sqdepth) | ||||
{ | { | ||||
*sqdepth = irdma_qp_round_up((sq_size << shift) + IRDMA_SQ_RSVD); | *sqdepth = irdma_round_up_wq((sq_size << shift) + IRDMA_SQ_RSVD); | ||||
if (*sqdepth < (IRDMA_QP_SW_MIN_WQSIZE << shift)) | if (*sqdepth < ((u32)uk_attrs->min_hw_wq_size << shift)) | ||||
*sqdepth = IRDMA_QP_SW_MIN_WQSIZE << shift; | *sqdepth = uk_attrs->min_hw_wq_size << shift; | ||||
else if (*sqdepth > max_hw_wq_quanta) | else if (*sqdepth > uk_attrs->max_hw_wq_quanta) | ||||
return -EINVAL; | return -EINVAL; | ||||
return 0; | return 0; | ||||
} | } | ||||
/* | /* | ||||
* irdma_get_rqdepth - get RQ/SRQ depth (quanta) @max_hw_rq_quanta: HW RQ/SRQ size limit @rq_size: RQ/SRQ size @shift: | * irdma_get_rqdepth - get RQ depth (quanta) @uk_attrs: qp HW attributes @rq_size: SRQ size @shift: shift which | ||||
* shift which determines size of WQE @rqdepth: depth of RQ/SRQ | * determines size of WQE @rqdepth: depth of RQ/SRQ | ||||
*/ | */ | ||||
int | int | ||||
irdma_get_rqdepth(u32 max_hw_rq_quanta, u32 rq_size, u8 shift, u32 *rqdepth) | irdma_get_rqdepth(struct irdma_uk_attrs *uk_attrs, u32 rq_size, u8 shift, u32 *rqdepth) | ||||
{ | { | ||||
*rqdepth = irdma_qp_round_up((rq_size << shift) + IRDMA_RQ_RSVD); | *rqdepth = irdma_round_up_wq((rq_size << shift) + IRDMA_RQ_RSVD); | ||||
if (*rqdepth < (IRDMA_QP_SW_MIN_WQSIZE << shift)) | if (*rqdepth < ((u32)uk_attrs->min_hw_wq_size << shift)) | ||||
*rqdepth = IRDMA_QP_SW_MIN_WQSIZE << shift; | *rqdepth = uk_attrs->min_hw_wq_size << shift; | ||||
else if (*rqdepth > max_hw_rq_quanta) | else if (*rqdepth > uk_attrs->max_hw_rq_quanta) | ||||
return -EINVAL; | return -EINVAL; | ||||
return 0; | return 0; | ||||
Context not available. | |||||
IRDMA_RING_MOVE_HEAD_BY_COUNT_NOCHECK(qp->initial_ring, move_cnt); | IRDMA_RING_MOVE_HEAD_BY_COUNT_NOCHECK(qp->initial_ring, move_cnt); | ||||
} | } | ||||
/** | |||||
* irdma_uk_calc_shift_wq - calculate WQE shift for both SQ and RQ | |||||
* @ukinfo: qp initialization info | |||||
* @sq_shift: Returns shift of SQ | |||||
* @rq_shift: Returns shift of RQ | |||||
*/ | |||||
void | |||||
irdma_uk_calc_shift_wq(struct irdma_qp_uk_init_info *ukinfo, u8 *sq_shift, | |||||
u8 *rq_shift) | |||||
{ | |||||
bool imm_support = ukinfo->uk_attrs->hw_rev >= IRDMA_GEN_2 ? true : false; | |||||
irdma_get_wqe_shift(ukinfo->uk_attrs, | |||||
imm_support ? ukinfo->max_sq_frag_cnt + 1 : | |||||
ukinfo->max_sq_frag_cnt, | |||||
ukinfo->max_inline_data, sq_shift); | |||||
irdma_get_wqe_shift(ukinfo->uk_attrs, ukinfo->max_rq_frag_cnt, 0, | |||||
rq_shift); | |||||
if (ukinfo->uk_attrs->hw_rev == IRDMA_GEN_1) { | |||||
if (ukinfo->abi_ver > 4) | |||||
*rq_shift = IRDMA_MAX_RQ_WQE_SHIFT_GEN1; | |||||
} | |||||
} | |||||
/** | |||||
* irdma_uk_calc_depth_shift_sq - calculate depth and shift for SQ size. | |||||
* @ukinfo: qp initialization info | |||||
* @sq_depth: Returns depth of SQ | |||||
* @sq_shift: Returns shift of SQ | |||||
*/ | |||||
int | |||||
irdma_uk_calc_depth_shift_sq(struct irdma_qp_uk_init_info *ukinfo, | |||||
u32 *sq_depth, u8 *sq_shift) | |||||
{ | |||||
bool imm_support = ukinfo->uk_attrs->hw_rev >= IRDMA_GEN_2 ? true : false; | |||||
int status; | |||||
irdma_get_wqe_shift(ukinfo->uk_attrs, | |||||
imm_support ? ukinfo->max_sq_frag_cnt + 1 : | |||||
ukinfo->max_sq_frag_cnt, | |||||
ukinfo->max_inline_data, sq_shift); | |||||
status = irdma_get_sqdepth(ukinfo->uk_attrs, ukinfo->sq_size, | |||||
*sq_shift, sq_depth); | |||||
return status; | |||||
} | |||||
/** | |||||
* irdma_uk_calc_depth_shift_rq - calculate depth and shift for RQ size. | |||||
* @ukinfo: qp initialization info | |||||
* @rq_depth: Returns depth of RQ | |||||
* @rq_shift: Returns shift of RQ | |||||
*/ | |||||
int | |||||
irdma_uk_calc_depth_shift_rq(struct irdma_qp_uk_init_info *ukinfo, | |||||
u32 *rq_depth, u8 *rq_shift) | |||||
{ | |||||
int status; | |||||
irdma_get_wqe_shift(ukinfo->uk_attrs, ukinfo->max_rq_frag_cnt, 0, | |||||
rq_shift); | |||||
if (ukinfo->uk_attrs->hw_rev == IRDMA_GEN_1) { | |||||
if (ukinfo->abi_ver > 4) | |||||
*rq_shift = IRDMA_MAX_RQ_WQE_SHIFT_GEN1; | |||||
} | |||||
status = irdma_get_rqdepth(ukinfo->uk_attrs, ukinfo->rq_size, | |||||
*rq_shift, rq_depth); | |||||
return status; | |||||
} | |||||
/** | /** | ||||
* irdma_uk_qp_init - initialize shared qp | * irdma_uk_qp_init - initialize shared qp | ||||
* @qp: hw qp (user and kernel) | * @qp: hw qp (user and kernel) | ||||
Context not available. | |||||
{ | { | ||||
int ret_code = 0; | int ret_code = 0; | ||||
u32 sq_ring_size; | u32 sq_ring_size; | ||||
u8 sqshift, rqshift; | |||||
qp->uk_attrs = info->uk_attrs; | qp->uk_attrs = info->uk_attrs; | ||||
if (info->max_sq_frag_cnt > qp->uk_attrs->max_hw_wq_frags || | if (info->max_sq_frag_cnt > qp->uk_attrs->max_hw_wq_frags || | ||||
info->max_rq_frag_cnt > qp->uk_attrs->max_hw_wq_frags) | info->max_rq_frag_cnt > qp->uk_attrs->max_hw_wq_frags) | ||||
return -EINVAL; | return -EINVAL; | ||||
irdma_get_wqe_shift(qp->uk_attrs, info->max_rq_frag_cnt, 0, &rqshift); | |||||
if (qp->uk_attrs->hw_rev == IRDMA_GEN_1) { | |||||
irdma_get_wqe_shift(qp->uk_attrs, info->max_sq_frag_cnt, | |||||
info->max_inline_data, &sqshift); | |||||
if (info->abi_ver > 4) | |||||
rqshift = IRDMA_MAX_RQ_WQE_SHIFT_GEN1; | |||||
} else { | |||||
irdma_get_wqe_shift(qp->uk_attrs, info->max_sq_frag_cnt + 1, | |||||
info->max_inline_data, &sqshift); | |||||
} | |||||
qp->qp_caps = info->qp_caps; | qp->qp_caps = info->qp_caps; | ||||
qp->sq_base = info->sq; | qp->sq_base = info->sq; | ||||
qp->rq_base = info->rq; | qp->rq_base = info->rq; | ||||
Context not available. | |||||
qp->sq_size = info->sq_size; | qp->sq_size = info->sq_size; | ||||
qp->push_mode = false; | qp->push_mode = false; | ||||
qp->max_sq_frag_cnt = info->max_sq_frag_cnt; | qp->max_sq_frag_cnt = info->max_sq_frag_cnt; | ||||
sq_ring_size = qp->sq_size << sqshift; | sq_ring_size = qp->sq_size << info->sq_shift; | ||||
IRDMA_RING_INIT(qp->sq_ring, sq_ring_size); | IRDMA_RING_INIT(qp->sq_ring, sq_ring_size); | ||||
IRDMA_RING_INIT(qp->initial_ring, sq_ring_size); | IRDMA_RING_INIT(qp->initial_ring, sq_ring_size); | ||||
if (info->first_sq_wq) { | if (info->first_sq_wq) { | ||||
Context not available. | |||||
qp->rq_size = info->rq_size; | qp->rq_size = info->rq_size; | ||||
qp->max_rq_frag_cnt = info->max_rq_frag_cnt; | qp->max_rq_frag_cnt = info->max_rq_frag_cnt; | ||||
qp->max_inline_data = info->max_inline_data; | qp->max_inline_data = info->max_inline_data; | ||||
qp->rq_wqe_size = rqshift; | qp->rq_wqe_size = info->rq_shift; | ||||
IRDMA_RING_INIT(qp->rq_ring, qp->rq_size); | IRDMA_RING_INIT(qp->rq_ring, qp->rq_size); | ||||
qp->rq_wqe_size_multiplier = 1 << rqshift; | qp->rq_wqe_size_multiplier = 1 << info->rq_shift; | ||||
if (qp->uk_attrs->hw_rev == IRDMA_GEN_1) | if (qp->uk_attrs->hw_rev == IRDMA_GEN_1) | ||||
qp->wqe_ops = iw_wqe_uk_ops_gen_1; | qp->wqe_ops = iw_wqe_uk_ops_gen_1; | ||||
else | else | ||||
Context not available. | |||||
else | else | ||||
cqe = cq->cq_base[cq_head].buf; | cqe = cq->cq_base[cq_head].buf; | ||||
get_64bit_val(cqe, IRDMA_BYTE_24, &qword3); | get_64bit_val(cqe, IRDMA_BYTE_24, &qword3); | ||||
polarity = (u8)RS_64(qword3, IRDMA_CQ_VALID); | polarity = (u8)FIELD_GET(IRDMA_CQ_VALID, qword3); | ||||
if (polarity != temp) | if (polarity != temp) | ||||
break; | break; | ||||
Context not available. | |||||
u64 hdr; | u64 hdr; | ||||
u32 wqe_idx; | u32 wqe_idx; | ||||
struct irdma_post_sq_info info = {0}; | struct irdma_post_sq_info info = {0}; | ||||
u16 quanta = IRDMA_QP_WQE_MIN_QUANTA; | |||||
info.push_wqe = false; | info.push_wqe = qp->push_db ? true : false; | ||||
info.wr_id = wr_id; | info.wr_id = wr_id; | ||||
wqe = irdma_qp_get_next_send_wqe(qp, &wqe_idx, IRDMA_QP_WQE_MIN_QUANTA, | wqe = irdma_qp_get_next_send_wqe(qp, &wqe_idx, &quanta, 0, &info); | ||||
0, &info); | |||||
if (!wqe) | if (!wqe) | ||||
return -ENOSPC; | return -ENOSPC; | ||||
irdma_clr_wqes(qp, wqe_idx); | |||||
set_64bit_val(wqe, IRDMA_BYTE_0, 0); | set_64bit_val(wqe, IRDMA_BYTE_0, 0); | ||||
set_64bit_val(wqe, IRDMA_BYTE_8, 0); | set_64bit_val(wqe, IRDMA_BYTE_8, 0); | ||||
set_64bit_val(wqe, IRDMA_BYTE_16, 0); | set_64bit_val(wqe, IRDMA_BYTE_16, 0); | ||||
hdr = LS_64(IRDMAQP_OP_NOP, IRDMAQPSQ_OPCODE) | | hdr = FIELD_PREP(IRDMAQPSQ_OPCODE, IRDMAQP_OP_NOP) | | ||||
LS_64(signaled, IRDMAQPSQ_SIGCOMPL) | | FIELD_PREP(IRDMAQPSQ_SIGCOMPL, signaled) | | ||||
LS_64(qp->swqe_polarity, IRDMAQPSQ_VALID); | FIELD_PREP(IRDMAQPSQ_VALID, qp->swqe_polarity); | ||||
irdma_wmb(); /* make sure WQE is populated before valid bit is set */ | irdma_wmb(); /* make sure WQE is populated before valid bit is set */ | ||||
set_64bit_val(wqe, IRDMA_BYTE_24, hdr); | set_64bit_val(wqe, IRDMA_BYTE_24, hdr); | ||||
if (post_sq) | |||||
if (info.push_wqe) | |||||
irdma_qp_push_wqe(qp, wqe, quanta, wqe_idx, post_sq); | |||||
else if (post_sq) | |||||
irdma_uk_qp_post_wr(qp); | irdma_uk_qp_post_wr(qp); | ||||
return 0; | return 0; | ||||
Context not available. |