Index: head/sys/dev/qlnx/qlnxe/ecore_iwarp.c =================================================================== --- head/sys/dev/qlnx/qlnxe/ecore_iwarp.c (nonexistent) +++ head/sys/dev/qlnx/qlnxe/ecore_iwarp.c (revision 343598) @@ -0,0 +1,3970 @@ +/* + * Copyright (c) 2018-2019 Cavium, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +/* + * File : ecore_iwarp.c + */ +#include +__FBSDID("$FreeBSD$"); + +#include "bcm_osal.h" +#include "ecore.h" +#include "ecore_status.h" +#include "ecore_sp_commands.h" +#include "ecore_cxt.h" +#include "ecore_rdma.h" +#include "reg_addr.h" +#include "ecore_hw.h" +#include "ecore_hsi_iwarp.h" +#include "ecore_ll2.h" +#include "ecore_ooo.h" +#ifndef LINUX_REMOVE +#include "ecore_tcp_ip.h" +#endif + +#ifdef _NTDDK_ +#pragma warning(push) +#pragma warning(disable : 28123) +#pragma warning(disable : 28167) +#endif + +/* Default values used for MPA Rev 1 */ +#define ECORE_IWARP_ORD_DEFAULT 32 +#define ECORE_IWARP_IRD_DEFAULT 32 + +#define ECORE_IWARP_MAX_FW_MSS 4120 + +struct mpa_v2_hdr { + __be16 ird; + __be16 ord; +}; + +#define MPA_V2_PEER2PEER_MODEL 0x8000 +#define MPA_V2_SEND_RTR 0x4000 /* on ird */ +#define MPA_V2_READ_RTR 0x4000 /* on ord */ +#define MPA_V2_WRITE_RTR 0x8000 +#define MPA_V2_IRD_ORD_MASK 0x3FFF + +#define MPA_REV2(_mpa_rev) (_mpa_rev == MPA_NEGOTIATION_TYPE_ENHANCED) + +#define ECORE_IWARP_INVALID_TCP_CID 0xffffffff +/* How many times fin will be sent before FW aborts and send RST */ +#define ECORE_IWARP_MAX_FIN_RT_DEFAULT 2 +#define ECORE_IWARP_RCV_WND_SIZE_MIN (0xffff) +/* INTERNAL: These numbers are derived from BRB buffer sizes to obtain optimal performance */ +#define ECORE_IWARP_RCV_WND_SIZE_BB_DEF_2_PORTS (200*1024) +#define ECORE_IWARP_RCV_WND_SIZE_BB_DEF_4_PORTS (100*1024) +#define ECORE_IWARP_RCV_WND_SIZE_AH_DEF_2_PORTS (150*1024) +#define ECORE_IWARP_RCV_WND_SIZE_AH_DEF_4_PORTS (90*1024) +#define ECORE_IWARP_MAX_WND_SCALE (14) +/* Timestamp header is the length of the timestamp option (10): + * kind:8 bit, length:8 bit, timestamp:32 bit, ack: 32bit + * rounded up to a multiple of 4 + */ +#define TIMESTAMP_HEADER_SIZE (12) + +static enum _ecore_status_t +ecore_iwarp_async_event(struct ecore_hwfn *p_hwfn, + u8 fw_event_code, + u16 OSAL_UNUSED echo, + union event_ring_data *data, + u8 fw_return_code); + +static enum _ecore_status_t +ecore_iwarp_empty_ramrod(struct ecore_hwfn *p_hwfn, + struct ecore_iwarp_listener *listener); + +static OSAL_INLINE struct ecore_iwarp_fpdu * +ecore_iwarp_get_curr_fpdu(struct ecore_hwfn *p_hwfn, u16 cid); + +/* Override devinfo with iWARP specific values */ +void +ecore_iwarp_init_devinfo(struct ecore_hwfn *p_hwfn) +{ + struct ecore_rdma_device *dev = p_hwfn->p_rdma_info->dev; + + dev->max_inline = IWARP_REQ_MAX_INLINE_DATA_SIZE; + dev->max_qp = OSAL_MIN_T(u64, + IWARP_MAX_QPS, + p_hwfn->p_rdma_info->num_qps) - + ECORE_IWARP_PREALLOC_CNT; + + dev->max_cq = dev->max_qp; + + dev->max_qp_resp_rd_atomic_resc = ECORE_IWARP_IRD_DEFAULT; + dev->max_qp_req_rd_atomic_resc = ECORE_IWARP_ORD_DEFAULT; +} + +enum _ecore_status_t +ecore_iwarp_init_hw(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt) +{ + p_hwfn->rdma_prs_search_reg = PRS_REG_SEARCH_TCP; + ecore_wr(p_hwfn, p_ptt, p_hwfn->rdma_prs_search_reg, 1); + p_hwfn->b_rdma_enabled_in_prs = true; + + return 0; +} + +void +ecore_iwarp_init_fw_ramrod(struct ecore_hwfn *p_hwfn, + struct iwarp_init_func_ramrod_data *p_ramrod) +{ + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, + "ooo handle = %d\n", + p_hwfn->p_rdma_info->iwarp.ll2_ooo_handle); + + p_ramrod->iwarp.ll2_ooo_q_index = + p_hwfn->hw_info.resc_start[ECORE_LL2_QUEUE] + + p_hwfn->p_rdma_info->iwarp.ll2_ooo_handle; + + p_ramrod->tcp.max_fin_rt = ECORE_IWARP_MAX_FIN_RT_DEFAULT; + return; +} + +static enum _ecore_status_t +ecore_iwarp_alloc_cid(struct ecore_hwfn *p_hwfn, u32 *cid) +{ + enum _ecore_status_t rc; + + OSAL_SPIN_LOCK(&p_hwfn->p_rdma_info->lock); + + rc = ecore_rdma_bmap_alloc_id(p_hwfn, + &p_hwfn->p_rdma_info->cid_map, + cid); + + OSAL_SPIN_UNLOCK(&p_hwfn->p_rdma_info->lock); + *cid += ecore_cxt_get_proto_cid_start(p_hwfn, + p_hwfn->p_rdma_info->proto); + if (rc != ECORE_SUCCESS) { + DP_NOTICE(p_hwfn, false, "Failed in allocating iwarp cid\n"); + return rc; + } + + rc = ecore_cxt_dynamic_ilt_alloc(p_hwfn, ECORE_ELEM_CXT, *cid); + + if (rc != ECORE_SUCCESS) { + OSAL_SPIN_LOCK(&p_hwfn->p_rdma_info->lock); + *cid -= ecore_cxt_get_proto_cid_start(p_hwfn, + p_hwfn->p_rdma_info->proto); + + ecore_bmap_release_id(p_hwfn, + &p_hwfn->p_rdma_info->cid_map, + *cid); + + OSAL_SPIN_UNLOCK(&p_hwfn->p_rdma_info->lock); + } + + return rc; +} + +static void +ecore_iwarp_set_tcp_cid(struct ecore_hwfn *p_hwfn, u32 cid) +{ + cid -= ecore_cxt_get_proto_cid_start(p_hwfn, + p_hwfn->p_rdma_info->proto); + + OSAL_SPIN_LOCK(&p_hwfn->p_rdma_info->lock); + ecore_bmap_set_id(p_hwfn, + &p_hwfn->p_rdma_info->tcp_cid_map, + cid); + OSAL_SPIN_UNLOCK(&p_hwfn->p_rdma_info->lock); +} + +/* This function allocates a cid for passive tcp ( called from syn receive) + * the reason it's separate from the regular cid allocation is because it + * is assured that these cids already have ilt alloacted. They are preallocated + * to ensure that we won't need to allocate memory during syn processing + */ +static enum _ecore_status_t +ecore_iwarp_alloc_tcp_cid(struct ecore_hwfn *p_hwfn, u32 *cid) +{ + enum _ecore_status_t rc; + + OSAL_SPIN_LOCK(&p_hwfn->p_rdma_info->lock); + + rc = ecore_rdma_bmap_alloc_id(p_hwfn, + &p_hwfn->p_rdma_info->tcp_cid_map, + cid); + + OSAL_SPIN_UNLOCK(&p_hwfn->p_rdma_info->lock); + + *cid += ecore_cxt_get_proto_cid_start(p_hwfn, + p_hwfn->p_rdma_info->proto); + if (rc != ECORE_SUCCESS) { + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, + "can't allocate iwarp tcp cid max-count=%d\n", + p_hwfn->p_rdma_info->tcp_cid_map.max_count); + + *cid = ECORE_IWARP_INVALID_TCP_CID; + } + + return rc; +} + +/* We have two cid maps, one for tcp which should be used only from passive + * syn processing and replacing a pre-allocated ep in the list. the second + * for active tcp and for QPs. + */ +static void ecore_iwarp_cid_cleaned(struct ecore_hwfn *p_hwfn, u32 cid) +{ + cid -= ecore_cxt_get_proto_cid_start(p_hwfn, + p_hwfn->p_rdma_info->proto); + + OSAL_SPIN_LOCK(&p_hwfn->p_rdma_info->lock); + + if (cid < ECORE_IWARP_PREALLOC_CNT) { + ecore_bmap_release_id(p_hwfn, + &p_hwfn->p_rdma_info->tcp_cid_map, + cid); + } else { + ecore_bmap_release_id(p_hwfn, + &p_hwfn->p_rdma_info->cid_map, + cid); + } + + OSAL_SPIN_UNLOCK(&p_hwfn->p_rdma_info->lock); +} + +enum _ecore_status_t +ecore_iwarp_create_qp(struct ecore_hwfn *p_hwfn, + struct ecore_rdma_qp *qp, + struct ecore_rdma_create_qp_out_params *out_params) +{ + struct iwarp_create_qp_ramrod_data *p_ramrod; + struct ecore_sp_init_data init_data; + struct ecore_spq_entry *p_ent; + enum _ecore_status_t rc; + u16 physical_queue; + u32 cid; + + qp->shared_queue = + OSAL_DMA_ALLOC_COHERENT(p_hwfn->p_dev, + &qp->shared_queue_phys_addr, + IWARP_SHARED_QUEUE_PAGE_SIZE); + if (!qp->shared_queue) { + DP_NOTICE(p_hwfn, false, + "ecore iwarp create qp failed: cannot allocate memory (shared queue).\n"); + return ECORE_NOMEM; + } else { + out_params->sq_pbl_virt = (u8 *)qp->shared_queue + + IWARP_SHARED_QUEUE_PAGE_SQ_PBL_OFFSET; + out_params->sq_pbl_phys = qp->shared_queue_phys_addr + + IWARP_SHARED_QUEUE_PAGE_SQ_PBL_OFFSET; + out_params->rq_pbl_virt = (u8 *)qp->shared_queue + + IWARP_SHARED_QUEUE_PAGE_RQ_PBL_OFFSET; + out_params->rq_pbl_phys = qp->shared_queue_phys_addr + + IWARP_SHARED_QUEUE_PAGE_RQ_PBL_OFFSET; + } + + rc = ecore_iwarp_alloc_cid(p_hwfn, &cid); + if (rc != ECORE_SUCCESS) + goto err1; + + qp->icid = (u16)cid; + + OSAL_MEMSET(&init_data, 0, sizeof(init_data)); + init_data.opaque_fid = p_hwfn->hw_info.opaque_fid; + init_data.cid = qp->icid; + init_data.comp_mode = ECORE_SPQ_MODE_EBLOCK; + + rc = ecore_sp_init_request(p_hwfn, &p_ent, + IWARP_RAMROD_CMD_ID_CREATE_QP, + PROTOCOLID_IWARP, &init_data); + if (rc != ECORE_SUCCESS) + return rc; + + p_ramrod = &p_ent->ramrod.iwarp_create_qp; + + SET_FIELD(p_ramrod->flags, + IWARP_CREATE_QP_RAMROD_DATA_FMR_AND_RESERVED_EN, + qp->fmr_and_reserved_lkey); + + SET_FIELD(p_ramrod->flags, + IWARP_CREATE_QP_RAMROD_DATA_SIGNALED_COMP, + qp->signal_all); + + SET_FIELD(p_ramrod->flags, + IWARP_CREATE_QP_RAMROD_DATA_RDMA_RD_EN, + qp->incoming_rdma_read_en); + + SET_FIELD(p_ramrod->flags, + IWARP_CREATE_QP_RAMROD_DATA_RDMA_WR_EN, + qp->incoming_rdma_write_en); + + SET_FIELD(p_ramrod->flags, + IWARP_CREATE_QP_RAMROD_DATA_ATOMIC_EN, + qp->incoming_atomic_en); + + SET_FIELD(p_ramrod->flags, + IWARP_CREATE_QP_RAMROD_DATA_SRQ_FLG, + qp->use_srq); + + p_ramrod->pd = qp->pd; + p_ramrod->sq_num_pages = qp->sq_num_pages; + p_ramrod->rq_num_pages = qp->rq_num_pages; + + p_ramrod->qp_handle_for_cqe.hi = OSAL_CPU_TO_LE32(qp->qp_handle.hi); + p_ramrod->qp_handle_for_cqe.lo = OSAL_CPU_TO_LE32(qp->qp_handle.lo); + + p_ramrod->cq_cid_for_sq = + OSAL_CPU_TO_LE32((p_hwfn->hw_info.opaque_fid << 16) | + qp->sq_cq_id); + p_ramrod->cq_cid_for_rq = + OSAL_CPU_TO_LE32((p_hwfn->hw_info.opaque_fid << 16) | + qp->rq_cq_id); + + p_ramrod->dpi = OSAL_CPU_TO_LE16(qp->dpi); + + physical_queue = ecore_get_cm_pq_idx(p_hwfn, PQ_FLAGS_OFLD); + p_ramrod->physical_q0 = OSAL_CPU_TO_LE16(physical_queue); + physical_queue = ecore_get_cm_pq_idx(p_hwfn, PQ_FLAGS_ACK); + p_ramrod->physical_q1 = OSAL_CPU_TO_LE16(physical_queue); + + rc = ecore_spq_post(p_hwfn, p_ent, OSAL_NULL); + + if (rc != ECORE_SUCCESS) + goto err1; + + return rc; + +err1: + OSAL_DMA_FREE_COHERENT(p_hwfn->p_dev, + qp->shared_queue, + qp->shared_queue_phys_addr, + IWARP_SHARED_QUEUE_PAGE_SIZE); + + return rc; +} + +static enum _ecore_status_t +ecore_iwarp_modify_fw(struct ecore_hwfn *p_hwfn, + struct ecore_rdma_qp *qp) +{ + struct iwarp_modify_qp_ramrod_data *p_ramrod; + struct ecore_sp_init_data init_data; + struct ecore_spq_entry *p_ent; + enum _ecore_status_t rc; + + /* Get SPQ entry */ + OSAL_MEMSET(&init_data, 0, sizeof(init_data)); + init_data.cid = qp->icid; + init_data.opaque_fid = p_hwfn->hw_info.opaque_fid; + init_data.comp_mode = ECORE_SPQ_MODE_EBLOCK; + + rc = ecore_sp_init_request(p_hwfn, &p_ent, + IWARP_RAMROD_CMD_ID_MODIFY_QP, + p_hwfn->p_rdma_info->proto, + &init_data); + if (rc != ECORE_SUCCESS) + return rc; + + p_ramrod = &p_ent->ramrod.iwarp_modify_qp; + SET_FIELD(p_ramrod->flags, IWARP_MODIFY_QP_RAMROD_DATA_STATE_TRANS_EN, + 0x1); + if (qp->iwarp_state == ECORE_IWARP_QP_STATE_CLOSING) + p_ramrod->transition_to_state = IWARP_MODIFY_QP_STATE_CLOSING; + else + p_ramrod->transition_to_state = IWARP_MODIFY_QP_STATE_ERROR; + + rc = ecore_spq_post(p_hwfn, p_ent, OSAL_NULL); + + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "QP(0x%x)rc=%d\n", + qp->icid, rc); + + return rc; +} + +enum ecore_iwarp_qp_state +ecore_roce2iwarp_state(enum ecore_roce_qp_state state) +{ + switch (state) { + case ECORE_ROCE_QP_STATE_RESET: + case ECORE_ROCE_QP_STATE_INIT: + case ECORE_ROCE_QP_STATE_RTR: + return ECORE_IWARP_QP_STATE_IDLE; + case ECORE_ROCE_QP_STATE_RTS: + return ECORE_IWARP_QP_STATE_RTS; + case ECORE_ROCE_QP_STATE_SQD: + return ECORE_IWARP_QP_STATE_CLOSING; + case ECORE_ROCE_QP_STATE_ERR: + return ECORE_IWARP_QP_STATE_ERROR; + case ECORE_ROCE_QP_STATE_SQE: + return ECORE_IWARP_QP_STATE_TERMINATE; + } + return ECORE_IWARP_QP_STATE_ERROR; +} + +static enum ecore_roce_qp_state +ecore_iwarp2roce_state(enum ecore_iwarp_qp_state state) +{ + switch (state) { + case ECORE_IWARP_QP_STATE_IDLE: + return ECORE_ROCE_QP_STATE_INIT; + case ECORE_IWARP_QP_STATE_RTS: + return ECORE_ROCE_QP_STATE_RTS; + case ECORE_IWARP_QP_STATE_TERMINATE: + return ECORE_ROCE_QP_STATE_SQE; + case ECORE_IWARP_QP_STATE_CLOSING: + return ECORE_ROCE_QP_STATE_SQD; + case ECORE_IWARP_QP_STATE_ERROR: + return ECORE_ROCE_QP_STATE_ERR; + } + return ECORE_ROCE_QP_STATE_ERR; +} + +const char *iwarp_state_names[] = { + "IDLE", + "RTS", + "TERMINATE", + "CLOSING", + "ERROR", +}; + +enum _ecore_status_t +ecore_iwarp_modify_qp(struct ecore_hwfn *p_hwfn, + struct ecore_rdma_qp *qp, + enum ecore_iwarp_qp_state new_state, + bool internal) +{ + enum ecore_iwarp_qp_state prev_iw_state; + enum _ecore_status_t rc = 0; + bool modify_fw = false; + + /* modify QP can be called from upper-layer or as a result of async + * RST/FIN... therefore need to protect + */ + OSAL_SPIN_LOCK(&p_hwfn->p_rdma_info->iwarp.qp_lock); + prev_iw_state = qp->iwarp_state; + + if (prev_iw_state == new_state) { + OSAL_SPIN_UNLOCK(&p_hwfn->p_rdma_info->iwarp.qp_lock); + return ECORE_SUCCESS; + } + + switch (prev_iw_state) { + case ECORE_IWARP_QP_STATE_IDLE: + switch (new_state) { + case ECORE_IWARP_QP_STATE_RTS: + qp->iwarp_state = ECORE_IWARP_QP_STATE_RTS; + break; + case ECORE_IWARP_QP_STATE_ERROR: + qp->iwarp_state = ECORE_IWARP_QP_STATE_ERROR; + if (!internal) + modify_fw = true; + break; + default: + break; + } + break; + case ECORE_IWARP_QP_STATE_RTS: + switch (new_state) { + case ECORE_IWARP_QP_STATE_CLOSING: + if (!internal) + modify_fw = true; + + qp->iwarp_state = ECORE_IWARP_QP_STATE_CLOSING; + break; + case ECORE_IWARP_QP_STATE_ERROR: + if (!internal) + modify_fw = true; + qp->iwarp_state = ECORE_IWARP_QP_STATE_ERROR; + break; + default: + break; + } + break; + case ECORE_IWARP_QP_STATE_ERROR: + switch (new_state) { + case ECORE_IWARP_QP_STATE_IDLE: + /* TODO: destroy flow -> need to destroy EP&QP */ + qp->iwarp_state = new_state; + break; + case ECORE_IWARP_QP_STATE_CLOSING: + /* could happen due to race... do nothing.... */ + break; + default: + rc = ECORE_INVAL; + } + break; + case ECORE_IWARP_QP_STATE_TERMINATE: + case ECORE_IWARP_QP_STATE_CLOSING: + qp->iwarp_state = new_state; + break; + default: + break; + } + + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "QP(0x%x) %s --> %s %s\n", + qp->icid, + iwarp_state_names[prev_iw_state], + iwarp_state_names[qp->iwarp_state], + internal ? "internal" : " "); + + OSAL_SPIN_UNLOCK(&p_hwfn->p_rdma_info->iwarp.qp_lock); + + if (modify_fw) + ecore_iwarp_modify_fw(p_hwfn, qp); + + return rc; +} + +enum _ecore_status_t +ecore_iwarp_fw_destroy(struct ecore_hwfn *p_hwfn, + struct ecore_rdma_qp *qp) +{ + struct ecore_sp_init_data init_data; + struct ecore_spq_entry *p_ent; + enum _ecore_status_t rc; + + /* Get SPQ entry */ + OSAL_MEMSET(&init_data, 0, sizeof(init_data)); + init_data.cid = qp->icid; + init_data.opaque_fid = p_hwfn->hw_info.opaque_fid; + init_data.comp_mode = ECORE_SPQ_MODE_EBLOCK; + + rc = ecore_sp_init_request(p_hwfn, &p_ent, + IWARP_RAMROD_CMD_ID_DESTROY_QP, + p_hwfn->p_rdma_info->proto, + &init_data); + if (rc != ECORE_SUCCESS) + return rc; + + rc = ecore_spq_post(p_hwfn, p_ent, OSAL_NULL); + + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "QP(0x%x) rc = %d\n", qp->icid, rc); + + return rc; +} + +static void ecore_iwarp_destroy_ep(struct ecore_hwfn *p_hwfn, + struct ecore_iwarp_ep *ep, + bool remove_from_active_list) +{ + OSAL_DMA_FREE_COHERENT(p_hwfn->p_dev, + ep->ep_buffer_virt, + ep->ep_buffer_phys, + sizeof(*ep->ep_buffer_virt)); + + if (remove_from_active_list) { + OSAL_SPIN_LOCK(&p_hwfn->p_rdma_info->iwarp.iw_lock); + + OSAL_LIST_REMOVE_ENTRY(&ep->list_entry, + &p_hwfn->p_rdma_info->iwarp.ep_list); + + OSAL_SPIN_UNLOCK(&p_hwfn->p_rdma_info->iwarp.iw_lock); + } + + if (ep->qp) + ep->qp->ep = OSAL_NULL; + + OSAL_FREE(p_hwfn->p_dev, ep); +} + +enum _ecore_status_t +ecore_iwarp_destroy_qp(struct ecore_hwfn *p_hwfn, + struct ecore_rdma_qp *qp) +{ + enum _ecore_status_t rc = ECORE_SUCCESS; + struct ecore_iwarp_ep *ep = qp->ep; + struct ecore_iwarp_fpdu *fpdu; + int wait_count = 0; + + fpdu = ecore_iwarp_get_curr_fpdu(p_hwfn, qp->icid); + if (fpdu && fpdu->incomplete_bytes) + DP_NOTICE(p_hwfn, false, + "Pending Partial fpdu with incomplete bytes=%d\n", + fpdu->incomplete_bytes); + + if (qp->iwarp_state != ECORE_IWARP_QP_STATE_ERROR) { + + rc = ecore_iwarp_modify_qp(p_hwfn, qp, + ECORE_IWARP_QP_STATE_ERROR, + false); + + if (rc != ECORE_SUCCESS) + return rc; + } + + /* Make sure ep is closed before returning and freeing memory. */ + if (ep) { + while (ep->state != ECORE_IWARP_EP_CLOSED) { + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, + "Waiting for ep->state to be closed...state=%x\n", + ep->state); + + OSAL_MSLEEP(100); + if (wait_count++ > 200) { + DP_NOTICE(p_hwfn, false, "ep state close timeout state=%x\n", + ep->state); + break; + } + } + + ecore_iwarp_destroy_ep(p_hwfn, ep, false); + } + + rc = ecore_iwarp_fw_destroy(p_hwfn, qp); + + if (qp->shared_queue) + OSAL_DMA_FREE_COHERENT(p_hwfn->p_dev, + qp->shared_queue, + qp->shared_queue_phys_addr, + IWARP_SHARED_QUEUE_PAGE_SIZE); + + return rc; +} + +static enum _ecore_status_t +ecore_iwarp_create_ep(struct ecore_hwfn *p_hwfn, + struct ecore_iwarp_ep **ep_out) +{ + struct ecore_iwarp_ep *ep; + enum _ecore_status_t rc; + + ep = OSAL_ZALLOC(p_hwfn->p_dev, GFP_KERNEL, sizeof(*ep)); + if (!ep) { + DP_NOTICE(p_hwfn, false, + "ecore create ep failed: cannot allocate memory (ep). rc = %d\n", + ECORE_NOMEM); + return ECORE_NOMEM; + } + + ep->state = ECORE_IWARP_EP_INIT; + + /* ep_buffer is allocated once and is structured as follows: + * [MAX_PRIV_DATA_LEN][MAX_PRIV_DATA_LEN][union async_output] + * We could have allocated this in three calls but since all together + * it is less than a page, we do one allocation and initialize pointers + * accordingly + */ + ep->ep_buffer_virt = OSAL_DMA_ALLOC_COHERENT( + p_hwfn->p_dev, + &ep->ep_buffer_phys, + sizeof(*ep->ep_buffer_virt)); + + if (!ep->ep_buffer_virt) { + DP_NOTICE(p_hwfn, false, + "ecore create ep failed: cannot allocate memory (ulp buffer). rc = %d\n", + ECORE_NOMEM); + rc = ECORE_NOMEM; + goto err; + } + + ep->sig = 0xdeadbeef; + + *ep_out = ep; + + return ECORE_SUCCESS; + +err: + OSAL_FREE(p_hwfn->p_dev, ep); + return rc; +} + +static void +ecore_iwarp_print_tcp_ramrod(struct ecore_hwfn *p_hwfn, + struct iwarp_tcp_offload_ramrod_data *p_tcp_ramrod) +{ + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, ">>> PRINT TCP RAMROD\n"); + + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "local_mac=%x %x %x\n", + p_tcp_ramrod->tcp.local_mac_addr_lo, + p_tcp_ramrod->tcp.local_mac_addr_mid, + p_tcp_ramrod->tcp.local_mac_addr_hi); + + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "remote_mac=%x %x %x\n", + p_tcp_ramrod->tcp.remote_mac_addr_lo, + p_tcp_ramrod->tcp.remote_mac_addr_mid, + p_tcp_ramrod->tcp.remote_mac_addr_hi); + + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "vlan_id=%x\n", + p_tcp_ramrod->tcp.vlan_id); + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "flags=%x\n", + p_tcp_ramrod->tcp.flags); + + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "ip_version=%x\n", + p_tcp_ramrod->tcp.ip_version); + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "local_ip=%x.%x.%x.%x\n", + p_tcp_ramrod->tcp.local_ip[0], + p_tcp_ramrod->tcp.local_ip[1], + p_tcp_ramrod->tcp.local_ip[2], + p_tcp_ramrod->tcp.local_ip[3]); + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "remote_ip=%x.%x.%x.%x\n", + p_tcp_ramrod->tcp.remote_ip[0], + p_tcp_ramrod->tcp.remote_ip[1], + p_tcp_ramrod->tcp.remote_ip[2], + p_tcp_ramrod->tcp.remote_ip[3]); + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "flow_label=%x\n", + p_tcp_ramrod->tcp.flow_label); + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "ttl=%x\n", + p_tcp_ramrod->tcp.ttl); + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "tos_or_tc=%x\n", + p_tcp_ramrod->tcp.tos_or_tc); + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "local_port=%x\n", + p_tcp_ramrod->tcp.local_port); + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "remote_port=%x\n", + p_tcp_ramrod->tcp.remote_port); + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "mss=%x\n", + p_tcp_ramrod->tcp.mss); + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "rcv_wnd_scale=%x\n", + p_tcp_ramrod->tcp.rcv_wnd_scale); + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "connect_mode=%x\n", + p_tcp_ramrod->tcp.connect_mode); + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "syn_ip_payload_length=%x\n", + p_tcp_ramrod->tcp.syn_ip_payload_length); + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "syn_phy_addr_lo=%x\n", + p_tcp_ramrod->tcp.syn_phy_addr_lo); + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "syn_phy_addr_hi=%x\n", + p_tcp_ramrod->tcp.syn_phy_addr_hi); + + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "<<p_rdma_info->iwarp; + struct iwarp_tcp_offload_ramrod_data *p_tcp_ramrod; + struct ecore_sp_init_data init_data; + struct ecore_spq_entry *p_ent; + dma_addr_t async_output_phys; + dma_addr_t in_pdata_phys; + enum _ecore_status_t rc; + u16 physical_q; + u8 tcp_flags; + int i; + + OSAL_MEMSET(&init_data, 0, sizeof(init_data)); + init_data.cid = ep->tcp_cid; + init_data.opaque_fid = p_hwfn->hw_info.opaque_fid; + + if (ep->connect_mode == TCP_CONNECT_PASSIVE) { + init_data.comp_mode = ECORE_SPQ_MODE_CB; + } else { + init_data.comp_mode = ECORE_SPQ_MODE_EBLOCK; + } + + rc = ecore_sp_init_request(p_hwfn, &p_ent, + IWARP_RAMROD_CMD_ID_TCP_OFFLOAD, + PROTOCOLID_IWARP, &init_data); + if (rc != ECORE_SUCCESS) + return rc; + + p_tcp_ramrod = &p_ent->ramrod.iwarp_tcp_offload; + + /* Point to the "second half" of the ulp buffer */ + in_pdata_phys = ep->ep_buffer_phys + + OFFSETOF(struct ecore_iwarp_ep_memory, in_pdata); + p_tcp_ramrod->iwarp.incoming_ulp_buffer.addr.hi = + DMA_HI_LE(in_pdata_phys); + p_tcp_ramrod->iwarp.incoming_ulp_buffer.addr.lo = + DMA_LO_LE(in_pdata_phys); + p_tcp_ramrod->iwarp.incoming_ulp_buffer.len = + OSAL_CPU_TO_LE16(sizeof(ep->ep_buffer_virt->in_pdata)); + + async_output_phys = ep->ep_buffer_phys + + OFFSETOF(struct ecore_iwarp_ep_memory, async_output); + + p_tcp_ramrod->iwarp.async_eqe_output_buf.hi = + DMA_HI_LE(async_output_phys); + p_tcp_ramrod->iwarp.async_eqe_output_buf.lo = + DMA_LO_LE(async_output_phys); + p_tcp_ramrod->iwarp.handle_for_async.hi = OSAL_CPU_TO_LE32(PTR_HI(ep)); + p_tcp_ramrod->iwarp.handle_for_async.lo = OSAL_CPU_TO_LE32(PTR_LO(ep)); + + physical_q = ecore_get_cm_pq_idx(p_hwfn, PQ_FLAGS_OFLD); + p_tcp_ramrod->iwarp.physical_q0 = OSAL_CPU_TO_LE16(physical_q); + physical_q = ecore_get_cm_pq_idx(p_hwfn, PQ_FLAGS_ACK); + p_tcp_ramrod->iwarp.physical_q1 = OSAL_CPU_TO_LE16(physical_q); + p_tcp_ramrod->iwarp.mpa_mode = iwarp_info->mpa_rev; + + ecore_set_fw_mac_addr(&p_tcp_ramrod->tcp.remote_mac_addr_hi, + &p_tcp_ramrod->tcp.remote_mac_addr_mid, + &p_tcp_ramrod->tcp.remote_mac_addr_lo, + ep->remote_mac_addr); + ecore_set_fw_mac_addr(&p_tcp_ramrod->tcp.local_mac_addr_hi, + &p_tcp_ramrod->tcp.local_mac_addr_mid, + &p_tcp_ramrod->tcp.local_mac_addr_lo, + ep->local_mac_addr); + + p_tcp_ramrod->tcp.vlan_id = OSAL_CPU_TO_LE16(ep->cm_info.vlan); + + tcp_flags = p_hwfn->p_rdma_info->iwarp.tcp_flags; + p_tcp_ramrod->tcp.flags = 0; + SET_FIELD(p_tcp_ramrod->tcp.flags, + TCP_OFFLOAD_PARAMS_OPT2_TS_EN, + !!(tcp_flags & ECORE_IWARP_TS_EN)); + + SET_FIELD(p_tcp_ramrod->tcp.flags, + TCP_OFFLOAD_PARAMS_OPT2_DA_EN, + !!(tcp_flags & ECORE_IWARP_DA_EN)); + + p_tcp_ramrod->tcp.ip_version = ep->cm_info.ip_version; + + for (i = 0; i < 4; i++) { + p_tcp_ramrod->tcp.remote_ip[i] = + OSAL_CPU_TO_LE32(ep->cm_info.remote_ip[i]); + p_tcp_ramrod->tcp.local_ip[i] = + OSAL_CPU_TO_LE32(ep->cm_info.local_ip[i]); + } + + p_tcp_ramrod->tcp.remote_port = + OSAL_CPU_TO_LE16(ep->cm_info.remote_port); + p_tcp_ramrod->tcp.local_port = OSAL_CPU_TO_LE16(ep->cm_info.local_port); + p_tcp_ramrod->tcp.mss = OSAL_CPU_TO_LE16(ep->mss); + p_tcp_ramrod->tcp.flow_label = 0; + p_tcp_ramrod->tcp.ttl = 0x40; + p_tcp_ramrod->tcp.tos_or_tc = 0; + + p_tcp_ramrod->tcp.max_rt_time = ECORE_IWARP_DEF_MAX_RT_TIME; + p_tcp_ramrod->tcp.cwnd = ECORE_IWARP_DEF_CWND_FACTOR * p_tcp_ramrod->tcp.mss; + p_tcp_ramrod->tcp.ka_max_probe_cnt = ECORE_IWARP_DEF_KA_MAX_PROBE_CNT; + p_tcp_ramrod->tcp.ka_timeout = ECORE_IWARP_DEF_KA_TIMEOUT; + p_tcp_ramrod->tcp.ka_interval = ECORE_IWARP_DEF_KA_INTERVAL; + + p_tcp_ramrod->tcp.rcv_wnd_scale = + (u8)p_hwfn->p_rdma_info->iwarp.rcv_wnd_scale; + p_tcp_ramrod->tcp.connect_mode = ep->connect_mode; + + if (ep->connect_mode == TCP_CONNECT_PASSIVE) { + p_tcp_ramrod->tcp.syn_ip_payload_length = + OSAL_CPU_TO_LE16(ep->syn_ip_payload_length); + p_tcp_ramrod->tcp.syn_phy_addr_hi = + DMA_HI_LE(ep->syn_phy_addr); + p_tcp_ramrod->tcp.syn_phy_addr_lo = + DMA_LO_LE(ep->syn_phy_addr); + } + + ecore_iwarp_print_tcp_ramrod(p_hwfn, p_tcp_ramrod); + + rc = ecore_spq_post(p_hwfn, p_ent, OSAL_NULL); + + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, + "EP(0x%x) Offload completed rc=%d\n" , ep->tcp_cid, rc); + + return rc; +} + +/* This function should be called after IWARP_EVENT_TYPE_ASYNC_CONNECT_COMPLETE + * is received. it will be called from the dpc context. + */ +static enum _ecore_status_t +ecore_iwarp_mpa_offload(struct ecore_hwfn *p_hwfn, + struct ecore_iwarp_ep *ep) +{ + struct iwarp_mpa_offload_ramrod_data *p_mpa_ramrod; + struct ecore_iwarp_info *iwarp_info; + struct ecore_sp_init_data init_data; + struct ecore_spq_entry *p_ent; + dma_addr_t async_output_phys; + dma_addr_t out_pdata_phys; + dma_addr_t in_pdata_phys; + struct ecore_rdma_qp *qp; + bool reject; + enum _ecore_status_t rc; + + if (!ep) + return ECORE_INVAL; + + qp = ep->qp; + reject = (qp == OSAL_NULL); + + OSAL_MEMSET(&init_data, 0, sizeof(init_data)); + init_data.cid = reject ? ep->tcp_cid : qp->icid; + init_data.opaque_fid = p_hwfn->hw_info.opaque_fid; + + if (ep->connect_mode == TCP_CONNECT_ACTIVE || !ep->event_cb) + init_data.comp_mode = ECORE_SPQ_MODE_CB; + else + init_data.comp_mode = ECORE_SPQ_MODE_EBLOCK; + + rc = ecore_sp_init_request(p_hwfn, &p_ent, + IWARP_RAMROD_CMD_ID_MPA_OFFLOAD, + PROTOCOLID_IWARP, &init_data); + + if (rc != ECORE_SUCCESS) + return rc; + + p_mpa_ramrod = &p_ent->ramrod.iwarp_mpa_offload; + out_pdata_phys = ep->ep_buffer_phys + + OFFSETOF(struct ecore_iwarp_ep_memory, out_pdata); + p_mpa_ramrod->common.outgoing_ulp_buffer.addr.hi = + DMA_HI_LE(out_pdata_phys); + p_mpa_ramrod->common.outgoing_ulp_buffer.addr.lo = + DMA_LO_LE(out_pdata_phys); + p_mpa_ramrod->common.outgoing_ulp_buffer.len = + ep->cm_info.private_data_len; + p_mpa_ramrod->common.crc_needed = p_hwfn->p_rdma_info->iwarp.crc_needed; + + p_mpa_ramrod->common.out_rq.ord = ep->cm_info.ord; + p_mpa_ramrod->common.out_rq.ird = ep->cm_info.ird; + + p_mpa_ramrod->tcp_cid = p_hwfn->hw_info.opaque_fid << 16 | ep->tcp_cid; + + in_pdata_phys = ep->ep_buffer_phys + + OFFSETOF(struct ecore_iwarp_ep_memory, in_pdata); + p_mpa_ramrod->tcp_connect_side = ep->connect_mode; + p_mpa_ramrod->incoming_ulp_buffer.addr.hi = + DMA_HI_LE(in_pdata_phys); + p_mpa_ramrod->incoming_ulp_buffer.addr.lo = + DMA_LO_LE(in_pdata_phys); + p_mpa_ramrod->incoming_ulp_buffer.len = + OSAL_CPU_TO_LE16(sizeof(ep->ep_buffer_virt->in_pdata)); + async_output_phys = ep->ep_buffer_phys + + OFFSETOF(struct ecore_iwarp_ep_memory, async_output); + p_mpa_ramrod->async_eqe_output_buf.hi = + DMA_HI_LE(async_output_phys); + p_mpa_ramrod->async_eqe_output_buf.lo = + DMA_LO_LE(async_output_phys); + p_mpa_ramrod->handle_for_async.hi = OSAL_CPU_TO_LE32(PTR_HI(ep)); + p_mpa_ramrod->handle_for_async.lo = OSAL_CPU_TO_LE32(PTR_LO(ep)); + + if (!reject) { + p_mpa_ramrod->shared_queue_addr.hi = + DMA_HI_LE(qp->shared_queue_phys_addr); + p_mpa_ramrod->shared_queue_addr.lo = + DMA_LO_LE(qp->shared_queue_phys_addr); + + p_mpa_ramrod->stats_counter_id = + RESC_START(p_hwfn, ECORE_RDMA_STATS_QUEUE) + + qp->stats_queue; + } else { + p_mpa_ramrod->common.reject = 1; + } + + iwarp_info = &p_hwfn->p_rdma_info->iwarp; + p_mpa_ramrod->rcv_wnd = iwarp_info->rcv_wnd_size; + p_mpa_ramrod->mode = ep->mpa_rev; + SET_FIELD(p_mpa_ramrod->rtr_pref, + IWARP_MPA_OFFLOAD_RAMROD_DATA_RTR_SUPPORTED, + ep->rtr_type); + + ep->state = ECORE_IWARP_EP_MPA_OFFLOADED; + rc = ecore_spq_post(p_hwfn, p_ent, OSAL_NULL); + if (!reject) + ep->cid = qp->icid; /* Now they're migrated. */ + + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, + "QP(0x%x) EP(0x%x) MPA Offload rc = %d IRD=0x%x ORD=0x%x rtr_type=%d mpa_rev=%d reject=%d\n", + reject ? 0xffff : qp->icid, ep->tcp_cid, rc, ep->cm_info.ird, + ep->cm_info.ord, ep->rtr_type, ep->mpa_rev, reject); + return rc; +} + +static void +ecore_iwarp_mpa_received(struct ecore_hwfn *p_hwfn, + struct ecore_iwarp_ep *ep) +{ + struct ecore_iwarp_info *iwarp_info = &p_hwfn->p_rdma_info->iwarp; + struct ecore_iwarp_cm_event_params params; + struct mpa_v2_hdr *mpa_v2_params; + union async_output *async_data; + u16 mpa_ord, mpa_ird; + u8 mpa_hdr_size = 0; + u8 mpa_rev; + + async_data = &ep->ep_buffer_virt->async_output; + + mpa_rev = async_data->mpa_request.mpa_handshake_mode; + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, + "private_data_len=%x handshake_mode=%x private_data=(%x)\n", + async_data->mpa_request.ulp_data_len, + mpa_rev, + *((u32 *)((u8 *)ep->ep_buffer_virt->in_pdata))); + + if (ep->listener->state > ECORE_IWARP_LISTENER_STATE_UNPAUSE) { + /* MPA reject initiated by ecore */ + OSAL_MEMSET(&ep->cm_info, 0, sizeof(ep->cm_info)); + ep->event_cb = OSAL_NULL; + ecore_iwarp_mpa_offload(p_hwfn, ep); + return; + } + + if (mpa_rev == MPA_NEGOTIATION_TYPE_ENHANCED) { + if (iwarp_info->mpa_rev == MPA_NEGOTIATION_TYPE_BASIC) { + DP_ERR(p_hwfn, "MPA_NEGOTIATE Received MPA rev 2 on driver supporting only MPA rev 1\n"); + /* MPA_REV2 ToDo: close the tcp connection. */ + return; + } + + /* Read ord/ird values from private data buffer */ + mpa_v2_params = + (struct mpa_v2_hdr *)(ep->ep_buffer_virt->in_pdata); + mpa_hdr_size = sizeof(*mpa_v2_params); + + mpa_ord = ntohs(mpa_v2_params->ord); + mpa_ird = ntohs(mpa_v2_params->ird); + + /* Temprary store in cm_info incoming ord/ird requested, later + * replace with negotiated value during accept + */ + ep->cm_info.ord = (u8)OSAL_MIN_T(u16, + (mpa_ord & MPA_V2_IRD_ORD_MASK), + ECORE_IWARP_ORD_DEFAULT); + + ep->cm_info.ird = (u8)OSAL_MIN_T(u16, + (mpa_ird & MPA_V2_IRD_ORD_MASK), + ECORE_IWARP_IRD_DEFAULT); + + /* Peer2Peer negotiation */ + ep->rtr_type = MPA_RTR_TYPE_NONE; + if (mpa_ird & MPA_V2_PEER2PEER_MODEL) { + if (mpa_ord & MPA_V2_WRITE_RTR) + ep->rtr_type |= MPA_RTR_TYPE_ZERO_WRITE; + + if (mpa_ord & MPA_V2_READ_RTR) + ep->rtr_type |= MPA_RTR_TYPE_ZERO_READ; + + if (mpa_ird & MPA_V2_SEND_RTR) + ep->rtr_type |= MPA_RTR_TYPE_ZERO_SEND; + + ep->rtr_type &= iwarp_info->rtr_type; + /* if we're left with no match send our capabilities */ + if (ep->rtr_type == MPA_RTR_TYPE_NONE) + ep->rtr_type = iwarp_info->rtr_type; + + /* prioritize write over send and read */ + if (ep->rtr_type & MPA_RTR_TYPE_ZERO_WRITE) + ep->rtr_type = MPA_RTR_TYPE_ZERO_WRITE; + } + + ep->mpa_rev = MPA_NEGOTIATION_TYPE_ENHANCED; + } else { + ep->cm_info.ord = ECORE_IWARP_ORD_DEFAULT; + ep->cm_info.ird = ECORE_IWARP_IRD_DEFAULT; + ep->mpa_rev = MPA_NEGOTIATION_TYPE_BASIC; + } + + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, + "MPA_NEGOTIATE (v%d): ORD: 0x%x IRD: 0x%x rtr:0x%x ulp_data_len = %x mpa_hdr_size = %x\n", + mpa_rev, ep->cm_info.ord, ep->cm_info.ird, ep->rtr_type, + async_data->mpa_request.ulp_data_len, + mpa_hdr_size); + + /* Strip mpa v2 hdr from private data before sending to upper layer */ + ep->cm_info.private_data = + ep->ep_buffer_virt->in_pdata + mpa_hdr_size; + + ep->cm_info.private_data_len = + async_data->mpa_request.ulp_data_len - mpa_hdr_size; + + params.event = ECORE_IWARP_EVENT_MPA_REQUEST; + params.cm_info = &ep->cm_info; + params.ep_context = ep; + params.status = ECORE_SUCCESS; + + ep->state = ECORE_IWARP_EP_MPA_REQ_RCVD; + ep->event_cb(ep->cb_context, ¶ms); +} + +static void +ecore_iwarp_move_to_ep_list(struct ecore_hwfn *p_hwfn, + osal_list_t *list, struct ecore_iwarp_ep *ep) +{ + OSAL_SPIN_LOCK(&ep->listener->lock); + OSAL_LIST_REMOVE_ENTRY(&ep->list_entry, &ep->listener->ep_list); + OSAL_SPIN_UNLOCK(&ep->listener->lock); + OSAL_SPIN_LOCK(&p_hwfn->p_rdma_info->iwarp.iw_lock); + OSAL_LIST_PUSH_TAIL(&ep->list_entry, list); + OSAL_SPIN_UNLOCK(&p_hwfn->p_rdma_info->iwarp.iw_lock); +} + +static void +ecore_iwarp_return_ep(struct ecore_hwfn *p_hwfn, + struct ecore_iwarp_ep *ep) +{ + ep->state = ECORE_IWARP_EP_INIT; + if (ep->qp) + ep->qp->ep = OSAL_NULL; + ep->qp = OSAL_NULL; + OSAL_MEMSET(&ep->cm_info, 0, sizeof(ep->cm_info)); + + if (ep->tcp_cid == ECORE_IWARP_INVALID_TCP_CID) { + /* We don't care about the return code, it's ok if tcp_cid + * remains invalid...in this case we'll defer allocation + */ + ecore_iwarp_alloc_tcp_cid(p_hwfn, &ep->tcp_cid); + } + + ecore_iwarp_move_to_ep_list(p_hwfn, + &p_hwfn->p_rdma_info->iwarp.ep_free_list, + ep); +} + +static void +ecore_iwarp_parse_private_data(struct ecore_hwfn *p_hwfn, + struct ecore_iwarp_ep *ep) +{ + struct mpa_v2_hdr *mpa_v2_params; + union async_output *async_data; + u16 mpa_ird, mpa_ord; + u8 mpa_data_size = 0; + + if (MPA_REV2(p_hwfn->p_rdma_info->iwarp.mpa_rev)) { + mpa_v2_params = (struct mpa_v2_hdr *) + ((u8 *)ep->ep_buffer_virt->in_pdata); + mpa_data_size = sizeof(*mpa_v2_params); + mpa_ird = ntohs(mpa_v2_params->ird); + mpa_ord = ntohs(mpa_v2_params->ord); + + ep->cm_info.ird = (u8)(mpa_ord & MPA_V2_IRD_ORD_MASK); + ep->cm_info.ord = (u8)(mpa_ird & MPA_V2_IRD_ORD_MASK); + } /* else: Ord / Ird already configured */ + + async_data = &ep->ep_buffer_virt->async_output; + + ep->cm_info.private_data = ep->ep_buffer_virt->in_pdata + mpa_data_size; + ep->cm_info.private_data_len = + async_data->mpa_response.ulp_data_len - mpa_data_size; +} + +static void +ecore_iwarp_mpa_reply_arrived(struct ecore_hwfn *p_hwfn, + struct ecore_iwarp_ep *ep) +{ + struct ecore_iwarp_cm_event_params params; + + if (ep->connect_mode == TCP_CONNECT_PASSIVE) { + DP_NOTICE(p_hwfn, true, "MPA reply event not expected on passive side!\n"); + return; + } + + params.event = ECORE_IWARP_EVENT_ACTIVE_MPA_REPLY; + + ecore_iwarp_parse_private_data(p_hwfn, ep); + + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, + "MPA_NEGOTIATE (v%d): ORD: 0x%x IRD: 0x%x\n", + ep->mpa_rev, ep->cm_info.ord, ep->cm_info.ird); + + params.cm_info = &ep->cm_info; + params.ep_context = ep; + params.status = ECORE_SUCCESS; + + ep->mpa_reply_processed = true; + + ep->event_cb(ep->cb_context, ¶ms); +} + +#define ECORE_IWARP_CONNECT_MODE_STRING(ep) \ + (ep->connect_mode == TCP_CONNECT_PASSIVE) ? "Passive" : "Active" + +/* Called as a result of the event: + * IWARP_EVENT_TYPE_ASYNC_MPA_HANDSHAKE_COMPLETE + */ +static void +ecore_iwarp_mpa_complete(struct ecore_hwfn *p_hwfn, + struct ecore_iwarp_ep *ep, + u8 fw_return_code) +{ + struct ecore_iwarp_cm_event_params params; + + if (ep->connect_mode == TCP_CONNECT_ACTIVE) + params.event = ECORE_IWARP_EVENT_ACTIVE_COMPLETE; + else + params.event = ECORE_IWARP_EVENT_PASSIVE_COMPLETE; + + if (ep->connect_mode == TCP_CONNECT_ACTIVE && + !ep->mpa_reply_processed) { + ecore_iwarp_parse_private_data(p_hwfn, ep); + } + + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, + "MPA_NEGOTIATE (v%d): ORD: 0x%x IRD: 0x%x\n", + ep->mpa_rev, ep->cm_info.ord, ep->cm_info.ird); + + params.cm_info = &ep->cm_info; + + params.ep_context = ep; + + if ((ep->connect_mode == TCP_CONNECT_PASSIVE) && + (ep->state != ECORE_IWARP_EP_MPA_OFFLOADED)) { + /* This is a FW bug. Shouldn't get complete without offload */ + DP_NOTICE(p_hwfn, false, "%s(0x%x) ERROR: Got MPA complete without MPA offload fw_return_code=%d ep->state=%d\n", + ECORE_IWARP_CONNECT_MODE_STRING(ep), ep->tcp_cid, + fw_return_code, ep->state); + ep->state = ECORE_IWARP_EP_CLOSED; + return; + } + + if ((ep->connect_mode == TCP_CONNECT_PASSIVE) && + (ep->state == ECORE_IWARP_EP_ABORTING)) + return; + + ep->state = ECORE_IWARP_EP_CLOSED; + + switch (fw_return_code) { + case RDMA_RETURN_OK: + ep->qp->max_rd_atomic_req = ep->cm_info.ord; + ep->qp->max_rd_atomic_resp = ep->cm_info.ird; + ecore_iwarp_modify_qp(p_hwfn, ep->qp, + ECORE_IWARP_QP_STATE_RTS, + 1); + ep->state = ECORE_IWARP_EP_ESTABLISHED; + params.status = ECORE_SUCCESS; + break; + case IWARP_CONN_ERROR_MPA_TIMEOUT: + DP_NOTICE(p_hwfn, false, "%s(0x%x) MPA timeout\n", + ECORE_IWARP_CONNECT_MODE_STRING(ep), ep->cid); + params.status = ECORE_TIMEOUT; + break; + case IWARP_CONN_ERROR_MPA_ERROR_REJECT: + DP_NOTICE(p_hwfn, false, "%s(0x%x) MPA Reject\n", + ECORE_IWARP_CONNECT_MODE_STRING(ep), ep->cid); + params.status = ECORE_CONN_REFUSED; + break; + case IWARP_CONN_ERROR_MPA_RST: + DP_NOTICE(p_hwfn, false, "%s(0x%x) MPA reset(tcp cid: 0x%x)\n", + ECORE_IWARP_CONNECT_MODE_STRING(ep), ep->cid, + ep->tcp_cid); + params.status = ECORE_CONN_RESET; + break; + case IWARP_CONN_ERROR_MPA_FIN: + DP_NOTICE(p_hwfn, false, "%s(0x%x) MPA received FIN\n", + ECORE_IWARP_CONNECT_MODE_STRING(ep), ep->cid); + params.status = ECORE_CONN_REFUSED; + break; + case IWARP_CONN_ERROR_MPA_INSUF_IRD: + DP_NOTICE(p_hwfn, false, "%s(0x%x) MPA insufficient ird\n", + ECORE_IWARP_CONNECT_MODE_STRING(ep), ep->cid); + params.status = ECORE_CONN_REFUSED; + break; + case IWARP_CONN_ERROR_MPA_RTR_MISMATCH: + DP_NOTICE(p_hwfn, false, "%s(0x%x) MPA RTR MISMATCH\n", + ECORE_IWARP_CONNECT_MODE_STRING(ep), ep->cid); + params.status = ECORE_CONN_REFUSED; + break; + case IWARP_CONN_ERROR_MPA_INVALID_PACKET: + DP_NOTICE(p_hwfn, false, "%s(0x%x) MPA Invalid Packet\n", + ECORE_IWARP_CONNECT_MODE_STRING(ep), ep->cid); + params.status = ECORE_CONN_REFUSED; + break; + case IWARP_CONN_ERROR_MPA_LOCAL_ERROR: + DP_NOTICE(p_hwfn, false, "%s(0x%x) MPA Local Error\n", + ECORE_IWARP_CONNECT_MODE_STRING(ep), ep->cid); + params.status = ECORE_CONN_REFUSED; + break; + case IWARP_CONN_ERROR_MPA_TERMINATE: + DP_NOTICE(p_hwfn, false, "%s(0x%x) MPA TERMINATE\n", + ECORE_IWARP_CONNECT_MODE_STRING(ep), ep->cid); + params.status = ECORE_CONN_REFUSED; + break; + default: + params.status = ECORE_CONN_RESET; + break; + } + + if (ep->event_cb) + ep->event_cb(ep->cb_context, ¶ms); + + /* on passive side, if there is no associated QP (REJECT) we need to + * return the ep to the pool, otherwise we wait for QP to release it. + * Since we add an element in accept instead of this one. in anycase + * we need to remove it from the ep_list (active connections)... + */ + if (fw_return_code != RDMA_RETURN_OK) { + ep->tcp_cid = ECORE_IWARP_INVALID_TCP_CID; + if ((ep->connect_mode == TCP_CONNECT_PASSIVE) && + (ep->qp == OSAL_NULL)) { /* Rejected */ + ecore_iwarp_return_ep(p_hwfn, ep); + } else { + OSAL_SPIN_LOCK(&p_hwfn->p_rdma_info->iwarp.iw_lock); + OSAL_LIST_REMOVE_ENTRY( + &ep->list_entry, + &p_hwfn->p_rdma_info->iwarp.ep_list); + OSAL_SPIN_UNLOCK(&p_hwfn->p_rdma_info->iwarp.iw_lock); + } + } +} + +static void +ecore_iwarp_mpa_v2_set_private(struct ecore_hwfn *p_hwfn, + struct ecore_iwarp_ep *ep, + u8 *mpa_data_size) +{ + struct mpa_v2_hdr *mpa_v2_params; + u16 mpa_ird, mpa_ord; + + *mpa_data_size = 0; + if (MPA_REV2(ep->mpa_rev)) { + mpa_v2_params = + (struct mpa_v2_hdr *)ep->ep_buffer_virt->out_pdata; + *mpa_data_size = sizeof(*mpa_v2_params); + + mpa_ird = (u16)ep->cm_info.ird; + mpa_ord = (u16)ep->cm_info.ord; + + if (ep->rtr_type != MPA_RTR_TYPE_NONE) { + mpa_ird |= MPA_V2_PEER2PEER_MODEL; + + if (ep->rtr_type & MPA_RTR_TYPE_ZERO_SEND) + mpa_ird |= MPA_V2_SEND_RTR; + + if (ep->rtr_type & MPA_RTR_TYPE_ZERO_WRITE) + mpa_ord |= MPA_V2_WRITE_RTR; + + if (ep->rtr_type & MPA_RTR_TYPE_ZERO_READ) + mpa_ord |= MPA_V2_READ_RTR; + } + + mpa_v2_params->ird = htons(mpa_ird); + mpa_v2_params->ord = htons(mpa_ord); + + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, + "MPA_NEGOTIATE Header: [%x ord:%x ird] %x ord:%x ird:%x peer2peer:%x rtr_send:%x rtr_write:%x rtr_read:%x\n", + mpa_v2_params->ird, + mpa_v2_params->ord, + *((u32 *)mpa_v2_params), + mpa_ord & MPA_V2_IRD_ORD_MASK, + mpa_ird & MPA_V2_IRD_ORD_MASK, + !!(mpa_ird & MPA_V2_PEER2PEER_MODEL), + !!(mpa_ird & MPA_V2_SEND_RTR), + !!(mpa_ord & MPA_V2_WRITE_RTR), + !!(mpa_ord & MPA_V2_READ_RTR)); + } +} + +enum _ecore_status_t +ecore_iwarp_connect(void *rdma_cxt, + struct ecore_iwarp_connect_in *iparams, + struct ecore_iwarp_connect_out *oparams) +{ + struct ecore_hwfn *p_hwfn = (struct ecore_hwfn *)rdma_cxt; + struct ecore_iwarp_info *iwarp_info; + struct ecore_iwarp_ep *ep; + enum _ecore_status_t rc; + u8 mpa_data_size = 0; + u8 ts_hdr_size = 0; + u32 cid; + + if ((iparams->cm_info.ord > ECORE_IWARP_ORD_DEFAULT) || + (iparams->cm_info.ird > ECORE_IWARP_IRD_DEFAULT)) { + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, + "QP(0x%x) ERROR: Invalid ord(0x%x)/ird(0x%x)\n", + iparams->qp->icid, iparams->cm_info.ord, + iparams->cm_info.ird); + + return ECORE_INVAL; + } + + iwarp_info = &p_hwfn->p_rdma_info->iwarp; + + /* Allocate ep object */ + rc = ecore_iwarp_alloc_cid(p_hwfn, &cid); + if (rc != ECORE_SUCCESS) + return rc; + + if (iparams->qp->ep == OSAL_NULL) { + rc = ecore_iwarp_create_ep(p_hwfn, &ep); + if (rc != ECORE_SUCCESS) + return rc; + } else { + ep = iparams->qp->ep; + DP_ERR(p_hwfn, "Note re-use of QP for different connect\n"); + ep->state = ECORE_IWARP_EP_INIT; + } + + ep->tcp_cid = cid; + + OSAL_SPIN_LOCK(&p_hwfn->p_rdma_info->iwarp.iw_lock); + OSAL_LIST_PUSH_TAIL(&ep->list_entry, + &p_hwfn->p_rdma_info->iwarp.ep_list); + OSAL_SPIN_UNLOCK(&p_hwfn->p_rdma_info->iwarp.iw_lock); + + ep->qp = iparams->qp; + ep->qp->ep = ep; + OSAL_MEMCPY(ep->remote_mac_addr, + iparams->remote_mac_addr, + ETH_ALEN); + OSAL_MEMCPY(ep->local_mac_addr, + iparams->local_mac_addr, + ETH_ALEN); + OSAL_MEMCPY(&ep->cm_info, &iparams->cm_info, sizeof(ep->cm_info)); + + ep->cm_info.ord = iparams->cm_info.ord; + ep->cm_info.ird = iparams->cm_info.ird; + + ep->rtr_type = iwarp_info->rtr_type; + if (iwarp_info->peer2peer == 0) + ep->rtr_type = MPA_RTR_TYPE_NONE; + + if ((ep->rtr_type & MPA_RTR_TYPE_ZERO_READ) && + (ep->cm_info.ord == 0)) + ep->cm_info.ord = 1; + + ep->mpa_rev = iwarp_info->mpa_rev; + + ecore_iwarp_mpa_v2_set_private(p_hwfn, ep, &mpa_data_size); + + ep->cm_info.private_data = (u8 *)ep->ep_buffer_virt->out_pdata; + ep->cm_info.private_data_len = + iparams->cm_info.private_data_len + mpa_data_size; + + OSAL_MEMCPY((u8 *)(u8 *)ep->ep_buffer_virt->out_pdata + mpa_data_size, + iparams->cm_info.private_data, + iparams->cm_info.private_data_len); + + if (p_hwfn->p_rdma_info->iwarp.tcp_flags & ECORE_IWARP_TS_EN) + ts_hdr_size = TIMESTAMP_HEADER_SIZE; + + ep->mss = iparams->mss - ts_hdr_size; + ep->mss = OSAL_MIN_T(u16, ECORE_IWARP_MAX_FW_MSS, ep->mss); + + ep->event_cb = iparams->event_cb; + ep->cb_context = iparams->cb_context; + ep->connect_mode = TCP_CONNECT_ACTIVE; + + oparams->ep_context = ep; + + rc = ecore_iwarp_tcp_offload(p_hwfn, ep); + + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "QP(0x%x) EP(0x%x) rc = %d\n", + iparams->qp->icid, ep->tcp_cid, rc); + + if (rc != ECORE_SUCCESS) + ecore_iwarp_destroy_ep(p_hwfn, ep, true); + + return rc; +} + +static struct ecore_iwarp_ep * +ecore_iwarp_get_free_ep(struct ecore_hwfn *p_hwfn) +{ + struct ecore_iwarp_ep *ep = OSAL_NULL; + enum _ecore_status_t rc; + + OSAL_SPIN_LOCK(&p_hwfn->p_rdma_info->iwarp.iw_lock); + + if (OSAL_LIST_IS_EMPTY(&p_hwfn->p_rdma_info->iwarp.ep_free_list)) { + DP_ERR(p_hwfn, "Ep list is empty\n"); + goto out; + } + + ep = OSAL_LIST_FIRST_ENTRY(&p_hwfn->p_rdma_info->iwarp.ep_free_list, + struct ecore_iwarp_ep, + list_entry); + + /* in some cases we could have failed allocating a tcp cid when added + * from accept / failure... retry now..this is not the common case. + */ + if (ep->tcp_cid == ECORE_IWARP_INVALID_TCP_CID) { + rc = ecore_iwarp_alloc_tcp_cid(p_hwfn, &ep->tcp_cid); + /* if we fail we could look for another entry with a valid + * tcp_cid, but since we don't expect to reach this anyway + * it's not worth the handling + */ + if (rc) { + ep->tcp_cid = ECORE_IWARP_INVALID_TCP_CID; + ep = OSAL_NULL; + goto out; + } + } + + OSAL_LIST_REMOVE_ENTRY(&ep->list_entry, + &p_hwfn->p_rdma_info->iwarp.ep_free_list); + +out: + OSAL_SPIN_UNLOCK(&p_hwfn->p_rdma_info->iwarp.iw_lock); + return ep; +} + +/* takes into account timer scan ~20 ms and interrupt/dpc overhead */ +#define ECORE_IWARP_MAX_CID_CLEAN_TIME 100 +/* Technically we shouldn't reach this count with 100 ms iteration sleep */ +#define ECORE_IWARP_MAX_NO_PROGRESS_CNT 5 + +/* This function waits for all the bits of a bmap to be cleared, as long as + * there is progress ( i.e. the number of bits left to be cleared decreases ) + * the function continues. + */ +static enum _ecore_status_t +ecore_iwarp_wait_cid_map_cleared(struct ecore_hwfn *p_hwfn, + struct ecore_bmap *bmap) +{ + int prev_weight = 0; + int wait_count = 0; + int weight = 0; + + weight = OSAL_BITMAP_WEIGHT(bmap->bitmap, bmap->max_count); + prev_weight = weight; + + while (weight) { + OSAL_MSLEEP(ECORE_IWARP_MAX_CID_CLEAN_TIME); + + weight = OSAL_BITMAP_WEIGHT(bmap->bitmap, bmap->max_count); + + if (prev_weight == weight) { + wait_count++; + } else { + prev_weight = weight; + wait_count = 0; + } + + if (wait_count > ECORE_IWARP_MAX_NO_PROGRESS_CNT) { + DP_NOTICE(p_hwfn, false, + "%s bitmap wait timed out (%d cids pending)\n", + bmap->name, weight); + return ECORE_TIMEOUT; + } + } + return ECORE_SUCCESS; +} + +static enum _ecore_status_t +ecore_iwarp_wait_for_all_cids(struct ecore_hwfn *p_hwfn) +{ + enum _ecore_status_t rc; + int i; + + rc = ecore_iwarp_wait_cid_map_cleared( + p_hwfn, &p_hwfn->p_rdma_info->tcp_cid_map); + if (rc) + return rc; + + /* Now free the tcp cids from the main cid map */ + for (i = 0; i < ECORE_IWARP_PREALLOC_CNT; i++) { + ecore_bmap_release_id(p_hwfn, + &p_hwfn->p_rdma_info->cid_map, + i); + } + + /* Now wait for all cids to be completed */ + rc = ecore_iwarp_wait_cid_map_cleared( + p_hwfn, &p_hwfn->p_rdma_info->cid_map); + + return rc; +} + +static void +ecore_iwarp_free_prealloc_ep(struct ecore_hwfn *p_hwfn) +{ + struct ecore_iwarp_ep *ep; + u32 cid; + + while (!OSAL_LIST_IS_EMPTY(&p_hwfn->p_rdma_info->iwarp.ep_free_list)) { + OSAL_SPIN_LOCK(&p_hwfn->p_rdma_info->iwarp.iw_lock); + + ep = OSAL_LIST_FIRST_ENTRY( + &p_hwfn->p_rdma_info->iwarp.ep_free_list, + struct ecore_iwarp_ep, list_entry); + + if (ep == OSAL_NULL) { + OSAL_SPIN_UNLOCK(&p_hwfn->p_rdma_info->iwarp.iw_lock); + break; + } + +#ifdef _NTDDK_ +#pragma warning(suppress : 6011) +#endif + OSAL_LIST_REMOVE_ENTRY( + &ep->list_entry, + &p_hwfn->p_rdma_info->iwarp.ep_free_list); + + OSAL_SPIN_UNLOCK(&p_hwfn->p_rdma_info->iwarp.iw_lock); + + if (ep->tcp_cid != ECORE_IWARP_INVALID_TCP_CID) { + cid = ep->tcp_cid - ecore_cxt_get_proto_cid_start( + p_hwfn, p_hwfn->p_rdma_info->proto); + + OSAL_SPIN_LOCK(&p_hwfn->p_rdma_info->lock); + + ecore_bmap_release_id(p_hwfn, + &p_hwfn->p_rdma_info->tcp_cid_map, + cid); + + OSAL_SPIN_UNLOCK(&p_hwfn->p_rdma_info->lock); + } + + ecore_iwarp_destroy_ep(p_hwfn, ep, false); + } +} + +static enum _ecore_status_t +ecore_iwarp_prealloc_ep(struct ecore_hwfn *p_hwfn, bool init) +{ + struct ecore_iwarp_ep *ep; + int rc = ECORE_SUCCESS; + u32 cid; + int count; + int i; + + if (init) + count = ECORE_IWARP_PREALLOC_CNT; + else + count = 1; + + for (i = 0; i < count; i++) { + rc = ecore_iwarp_create_ep(p_hwfn, &ep); + if (rc != ECORE_SUCCESS) + return rc; + + /* During initialization we allocate from the main pool, + * afterwards we allocate only from the tcp_cid. + */ + if (init) { + rc = ecore_iwarp_alloc_cid(p_hwfn, &cid); + if (rc != ECORE_SUCCESS) + goto err; + ecore_iwarp_set_tcp_cid(p_hwfn, cid); + } else { + /* We don't care about the return code, it's ok if + * tcp_cid remains invalid...in this case we'll + * defer allocation + */ + ecore_iwarp_alloc_tcp_cid(p_hwfn, &cid); + } + + ep->tcp_cid = cid; + + OSAL_SPIN_LOCK(&p_hwfn->p_rdma_info->iwarp.iw_lock); + OSAL_LIST_PUSH_TAIL(&ep->list_entry, + &p_hwfn->p_rdma_info->iwarp.ep_free_list); + OSAL_SPIN_UNLOCK(&p_hwfn->p_rdma_info->iwarp.iw_lock); + } + + return rc; + +err: + ecore_iwarp_destroy_ep(p_hwfn, ep, false); + + return rc; +} + +enum _ecore_status_t +ecore_iwarp_alloc(struct ecore_hwfn *p_hwfn) +{ + enum _ecore_status_t rc; + +#ifdef CONFIG_ECORE_LOCK_ALLOC + OSAL_SPIN_LOCK_ALLOC(p_hwfn, &p_hwfn->p_rdma_info->iwarp.iw_lock); + OSAL_SPIN_LOCK_ALLOC(p_hwfn, &p_hwfn->p_rdma_info->iwarp.qp_lock); +#endif + OSAL_SPIN_LOCK_INIT(&p_hwfn->p_rdma_info->iwarp.iw_lock); + OSAL_SPIN_LOCK_INIT(&p_hwfn->p_rdma_info->iwarp.qp_lock); + + /* Allocate bitmap for tcp cid. These are used by passive side + * to ensure it can allocate a tcp cid during dpc that was + * pre-acquired and doesn't require dynamic allocation of ilt + */ + rc = ecore_rdma_bmap_alloc(p_hwfn, &p_hwfn->p_rdma_info->tcp_cid_map, + ECORE_IWARP_PREALLOC_CNT, + "TCP_CID"); + if (rc != ECORE_SUCCESS) { + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, + "Failed to allocate tcp cid, rc = %d\n", + rc); + return rc; + } + + OSAL_LIST_INIT(&p_hwfn->p_rdma_info->iwarp.ep_free_list); +//DAVIDS OSAL_SPIN_LOCK_INIT(&p_hwfn->p_rdma_info->iwarp.iw_lock); + rc = ecore_iwarp_prealloc_ep(p_hwfn, true); + if (rc != ECORE_SUCCESS) { + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, + "ecore_iwarp_prealloc_ep failed, rc = %d\n", + rc); + return rc; + } + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, + "ecore_iwarp_prealloc_ep success, rc = %d\n", + rc); + + return ecore_ooo_alloc(p_hwfn); +} + +void +ecore_iwarp_resc_free(struct ecore_hwfn *p_hwfn) +{ + struct ecore_iwarp_info *iwarp_info = &p_hwfn->p_rdma_info->iwarp; + +#ifdef CONFIG_ECORE_LOCK_ALLOC + OSAL_SPIN_LOCK_DEALLOC(iwarp_info->iw_lock); + OSAL_SPIN_LOCK_DEALLOC(iwarp_info->qp_lock); +#endif + ecore_ooo_free(p_hwfn); + if (iwarp_info->partial_fpdus) + OSAL_FREE(p_hwfn->p_dev, iwarp_info->partial_fpdus); + if (iwarp_info->mpa_bufs) + OSAL_FREE(p_hwfn->p_dev, iwarp_info->mpa_bufs); + if (iwarp_info->mpa_intermediate_buf) + OSAL_FREE(p_hwfn->p_dev, iwarp_info->mpa_intermediate_buf); + + ecore_rdma_bmap_free(p_hwfn, &p_hwfn->p_rdma_info->tcp_cid_map, 1); +} + + +enum _ecore_status_t +ecore_iwarp_accept(void *rdma_cxt, + struct ecore_iwarp_accept_in *iparams) +{ + struct ecore_hwfn *p_hwfn = (struct ecore_hwfn *)rdma_cxt; + struct ecore_iwarp_ep *ep; + u8 mpa_data_size = 0; + enum _ecore_status_t rc; + + ep = (struct ecore_iwarp_ep *)iparams->ep_context; + if (!ep) { + DP_ERR(p_hwfn, "Ep Context receive in accept is NULL\n"); + return ECORE_INVAL; + } + + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "QP(0x%x) EP(0x%x)\n", + iparams->qp->icid, ep->tcp_cid); + + if ((iparams->ord > ECORE_IWARP_ORD_DEFAULT) || + (iparams->ird > ECORE_IWARP_IRD_DEFAULT)) { + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, + "QP(0x%x) EP(0x%x) ERROR: Invalid ord(0x%x)/ird(0x%x)\n", + iparams->qp->icid, ep->tcp_cid, + iparams->ord, iparams->ord); + return ECORE_INVAL; + } + + /* We could reach qp->ep != OSAL NULL if we do accept on the same qp */ + if (iparams->qp->ep == OSAL_NULL) { + /* We need to add a replacement for the ep to the free list */ + ecore_iwarp_prealloc_ep(p_hwfn, false); + } else { + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, + "Note re-use of QP for different connect\n"); + /* Return the old ep to the free_pool */ + ecore_iwarp_return_ep(p_hwfn, iparams->qp->ep); + } + + ecore_iwarp_move_to_ep_list(p_hwfn, + &p_hwfn->p_rdma_info->iwarp.ep_list, + ep); + ep->listener = OSAL_NULL; + ep->cb_context = iparams->cb_context; + ep->qp = iparams->qp; + ep->qp->ep = ep; + + if (ep->mpa_rev == MPA_NEGOTIATION_TYPE_ENHANCED) { + /* Negotiate ord/ird: if upperlayer requested ord larger than + * ird advertised by remote, we need to decrease our ord + * to match remote ord + */ + if (iparams->ord > ep->cm_info.ird) { + iparams->ord = ep->cm_info.ird; + } + + /* For chelsio compatability, if rtr_zero read is requested + * we can't set ird to zero + */ + if ((ep->rtr_type & MPA_RTR_TYPE_ZERO_READ) && + (iparams->ird == 0)) + iparams->ird = 1; + } + + /* Update cm_info ord/ird to be negotiated values */ + ep->cm_info.ord = iparams->ord; + ep->cm_info.ird = iparams->ird; + + ecore_iwarp_mpa_v2_set_private(p_hwfn, ep, &mpa_data_size); + + ep->cm_info.private_data = ep->ep_buffer_virt->out_pdata; + ep->cm_info.private_data_len = + iparams->private_data_len + mpa_data_size; + + OSAL_MEMCPY((u8 *)ep->ep_buffer_virt->out_pdata + mpa_data_size, + iparams->private_data, + iparams->private_data_len); + + if (ep->state == ECORE_IWARP_EP_CLOSED) { + DP_NOTICE(p_hwfn, false, + "(0x%x) Accept called on EP in CLOSED state\n", + ep->tcp_cid); + ep->tcp_cid = ECORE_IWARP_INVALID_TCP_CID; + ecore_iwarp_return_ep(p_hwfn, ep); + return ECORE_CONN_RESET; + } + + rc = ecore_iwarp_mpa_offload(p_hwfn, ep); + if (rc) { + ecore_iwarp_modify_qp(p_hwfn, + iparams->qp, + ECORE_IWARP_QP_STATE_ERROR, + 1); + } + + return rc; +} + +enum _ecore_status_t +ecore_iwarp_reject(void *rdma_cxt, + struct ecore_iwarp_reject_in *iparams) +{ + struct ecore_hwfn *p_hwfn = (struct ecore_hwfn *)rdma_cxt; + struct ecore_iwarp_ep *ep; + u8 mpa_data_size = 0; + enum _ecore_status_t rc; + + ep = (struct ecore_iwarp_ep *)iparams->ep_context; + if (!ep) { + DP_ERR(p_hwfn, "Ep Context receive in reject is NULL\n"); + return ECORE_INVAL; + } + + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "EP(0x%x)\n", ep->tcp_cid); + + ep->cb_context = iparams->cb_context; + ep->qp = OSAL_NULL; + + ecore_iwarp_mpa_v2_set_private(p_hwfn, ep, &mpa_data_size); + + ep->cm_info.private_data = ep->ep_buffer_virt->out_pdata; + ep->cm_info.private_data_len = + iparams->private_data_len + mpa_data_size; + + OSAL_MEMCPY((u8 *)ep->ep_buffer_virt->out_pdata + mpa_data_size, + iparams->private_data, + iparams->private_data_len); + + if (ep->state == ECORE_IWARP_EP_CLOSED) { + DP_NOTICE(p_hwfn, false, + "(0x%x) Reject called on EP in CLOSED state\n", + ep->tcp_cid); + ep->tcp_cid = ECORE_IWARP_INVALID_TCP_CID; + ecore_iwarp_return_ep(p_hwfn, ep); + return ECORE_CONN_RESET; + } + + rc = ecore_iwarp_mpa_offload(p_hwfn, ep); + return rc; +} + +static void +ecore_iwarp_print_cm_info(struct ecore_hwfn *p_hwfn, + struct ecore_iwarp_cm_info *cm_info) +{ + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "ip_version = %d\n", + cm_info->ip_version); + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "remote_ip %x.%x.%x.%x\n", + cm_info->remote_ip[0], + cm_info->remote_ip[1], + cm_info->remote_ip[2], + cm_info->remote_ip[3]); + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "local_ip %x.%x.%x.%x\n", + cm_info->local_ip[0], + cm_info->local_ip[1], + cm_info->local_ip[2], + cm_info->local_ip[3]); + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "remote_port = %x\n", + cm_info->remote_port); + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "local_port = %x\n", + cm_info->local_port); + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "vlan = %x\n", + cm_info->vlan); + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "private_data_len = %x\n", + cm_info->private_data_len); + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "ord = %d\n", + cm_info->ord); + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "ird = %d\n", + cm_info->ird); +} + +static int +ecore_iwarp_ll2_post_rx(struct ecore_hwfn *p_hwfn, + struct ecore_iwarp_ll2_buff *buf, + u8 handle) +{ + enum _ecore_status_t rc; + + rc = ecore_ll2_post_rx_buffer( + p_hwfn, + handle, + buf->data_phys_addr, + (u16)buf->buff_size, + buf, 1); + + if (rc) { + DP_NOTICE(p_hwfn, false, + "Failed to repost rx buffer to ll2 rc = %d, handle=%d\n", + rc, handle); + OSAL_DMA_FREE_COHERENT( + p_hwfn->p_dev, + buf->data, + buf->data_phys_addr, + buf->buff_size); + OSAL_FREE(p_hwfn->p_dev, buf); + } + + return rc; +} + +static bool +ecore_iwarp_ep_exists(struct ecore_hwfn *p_hwfn, + struct ecore_iwarp_listener *listener, + struct ecore_iwarp_cm_info *cm_info) +{ + struct ecore_iwarp_ep *ep = OSAL_NULL; + bool found = false; + + OSAL_SPIN_LOCK(&listener->lock); + OSAL_LIST_FOR_EACH_ENTRY(ep, &listener->ep_list, + list_entry, struct ecore_iwarp_ep) { + if ((ep->cm_info.local_port == cm_info->local_port) && + (ep->cm_info.remote_port == cm_info->remote_port) && + (ep->cm_info.vlan == cm_info->vlan) && + !OSAL_MEMCMP(&(ep->cm_info.local_ip), cm_info->local_ip, + sizeof(cm_info->local_ip)) && + !OSAL_MEMCMP(&(ep->cm_info.remote_ip), cm_info->remote_ip, + sizeof(cm_info->remote_ip))) { + found = true; + break; + } + } + + OSAL_SPIN_UNLOCK(&listener->lock); + + if (found) { + DP_NOTICE(p_hwfn, false, "SYN received on active connection - dropping\n"); + ecore_iwarp_print_cm_info(p_hwfn, cm_info); + + return true; + } + + return false; +} + +static struct ecore_iwarp_listener * +ecore_iwarp_get_listener(struct ecore_hwfn *p_hwfn, + struct ecore_iwarp_cm_info *cm_info) +{ + struct ecore_iwarp_listener *listener = OSAL_NULL; + static const u32 ip_zero[4] = {0, 0, 0, 0}; + bool found = false; + + ecore_iwarp_print_cm_info(p_hwfn, cm_info); + + OSAL_LIST_FOR_EACH_ENTRY(listener, + &p_hwfn->p_rdma_info->iwarp.listen_list, + list_entry, struct ecore_iwarp_listener) { + + if (listener->port == cm_info->local_port) { + /* Any IP (i.e. 0.0.0.0 ) will be treated as any vlan */ + if (!OSAL_MEMCMP(listener->ip_addr, + ip_zero, + sizeof(ip_zero))) { + found = true; + break; + } + + /* If not any IP -> check vlan as well */ + if (!OSAL_MEMCMP(listener->ip_addr, + cm_info->local_ip, + sizeof(cm_info->local_ip)) && + + (listener->vlan == cm_info->vlan)) { + found = true; + break; + } + } + } + + if (found && listener->state == ECORE_IWARP_LISTENER_STATE_ACTIVE) { + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "listener found = %p\n", + listener); + return listener; + } + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "listener not found\n"); + return OSAL_NULL; +} + +static enum _ecore_status_t +ecore_iwarp_parse_rx_pkt(struct ecore_hwfn *p_hwfn, + struct ecore_iwarp_cm_info *cm_info, + void *buf, + u8 *remote_mac_addr, + u8 *local_mac_addr, + int *payload_len, + int *tcp_start_offset) +{ + struct ecore_vlan_ethhdr *vethh; + struct ecore_ethhdr *ethh; + struct ecore_iphdr *iph; + struct ecore_ipv6hdr *ip6h; + struct ecore_tcphdr *tcph; + bool vlan_valid = false; + int eth_hlen, ip_hlen; + u16 eth_type; + int i; + + ethh = (struct ecore_ethhdr *)buf; + eth_type = ntohs(ethh->h_proto); + if (eth_type == ETH_P_8021Q) { + vlan_valid = true; + vethh = (struct ecore_vlan_ethhdr *)ethh; + cm_info->vlan = ntohs(vethh->h_vlan_TCI) & VLAN_VID_MASK; + eth_type = ntohs(vethh->h_vlan_encapsulated_proto); + } + + eth_hlen = ETH_HLEN + (vlan_valid ? sizeof(u32) : 0); + + OSAL_MEMCPY(remote_mac_addr, + ethh->h_source, + ETH_ALEN); + + OSAL_MEMCPY(local_mac_addr, + ethh->h_dest, + ETH_ALEN); + + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "eth_type =%d Source mac: [0x%x]:[0x%x]:[0x%x]:[0x%x]:[0x%x]:[0x%x]\n", + eth_type, ethh->h_source[0], ethh->h_source[1], + ethh->h_source[2], ethh->h_source[3], + ethh->h_source[4], ethh->h_source[5]); + + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "eth_hlen=%d destination mac: [0x%x]:[0x%x]:[0x%x]:[0x%x]:[0x%x]:[0x%x]\n", + eth_hlen, ethh->h_dest[0], ethh->h_dest[1], + ethh->h_dest[2], ethh->h_dest[3], + ethh->h_dest[4], ethh->h_dest[5]); + + iph = (struct ecore_iphdr *)((u8 *)(ethh) + eth_hlen); + + if (eth_type == ETH_P_IP) { + if (iph->protocol != IPPROTO_TCP) { + DP_NOTICE(p_hwfn, false, + "Unexpected ip protocol on ll2 %x\n", + iph->protocol); + return ECORE_INVAL; + } + + cm_info->local_ip[0] = ntohl(iph->daddr); + cm_info->remote_ip[0] = ntohl(iph->saddr); + cm_info->ip_version = (enum ecore_tcp_ip_version)TCP_IPV4; + + ip_hlen = (iph->ihl)*sizeof(u32); + *payload_len = ntohs(iph->tot_len) - ip_hlen; + + } else if (eth_type == ETH_P_IPV6) { + ip6h = (struct ecore_ipv6hdr *)iph; + + if (ip6h->nexthdr != IPPROTO_TCP) { + DP_NOTICE(p_hwfn, false, + "Unexpected ip protocol on ll2 %x\n", + iph->protocol); + return ECORE_INVAL; + } + + for (i = 0; i < 4; i++) { + cm_info->local_ip[i] = + ntohl(ip6h->daddr.in6_u.u6_addr32[i]); + cm_info->remote_ip[i] = + ntohl(ip6h->saddr.in6_u.u6_addr32[i]); + } + cm_info->ip_version = (enum ecore_tcp_ip_version)TCP_IPV6; + + ip_hlen = sizeof(*ip6h); + *payload_len = ntohs(ip6h->payload_len); + } else { + DP_NOTICE(p_hwfn, false, + "Unexpected ethertype on ll2 %x\n", eth_type); + return ECORE_INVAL; + } + + tcph = (struct ecore_tcphdr *)((u8 *)iph + ip_hlen); + + if (!tcph->syn) { + DP_NOTICE(p_hwfn, false, + "Only SYN type packet expected on this ll2 conn, iph->ihl=%d source=%d dest=%d\n", + iph->ihl, tcph->source, tcph->dest); + return ECORE_INVAL; + } + + cm_info->local_port = ntohs(tcph->dest); + cm_info->remote_port = ntohs(tcph->source); + + ecore_iwarp_print_cm_info(p_hwfn, cm_info); + + *tcp_start_offset = eth_hlen + ip_hlen; + + return ECORE_SUCCESS; +} + +static struct ecore_iwarp_fpdu * +ecore_iwarp_get_curr_fpdu(struct ecore_hwfn *p_hwfn, u16 cid) +{ + struct ecore_iwarp_info *iwarp_info = &p_hwfn->p_rdma_info->iwarp; + struct ecore_iwarp_fpdu *partial_fpdu; + u32 idx = cid - ecore_cxt_get_proto_cid_start(p_hwfn, PROTOCOLID_IWARP); + + if (idx >= iwarp_info->max_num_partial_fpdus) { + DP_ERR(p_hwfn, "Invalid cid %x max_num_partial_fpdus=%x\n", cid, + iwarp_info->max_num_partial_fpdus); + return OSAL_NULL; + } + + partial_fpdu = &iwarp_info->partial_fpdus[idx]; + + return partial_fpdu; +} + +enum ecore_iwarp_mpa_pkt_type { + ECORE_IWARP_MPA_PKT_PACKED, + ECORE_IWARP_MPA_PKT_PARTIAL, + ECORE_IWARP_MPA_PKT_UNALIGNED +}; + +#define ECORE_IWARP_INVALID_FPDU_LENGTH 0xffff +#define ECORE_IWARP_MPA_FPDU_LENGTH_SIZE (2) +#define ECORE_IWARP_MPA_CRC32_DIGEST_SIZE (4) + +/* Pad to multiple of 4 */ +#define ECORE_IWARP_PDU_DATA_LEN_WITH_PAD(data_len) (((data_len) + 3) & ~3) + +#define ECORE_IWARP_FPDU_LEN_WITH_PAD(_mpa_len) \ + (ECORE_IWARP_PDU_DATA_LEN_WITH_PAD(_mpa_len + \ + ECORE_IWARP_MPA_FPDU_LENGTH_SIZE) + \ + ECORE_IWARP_MPA_CRC32_DIGEST_SIZE) + +/* fpdu can be fragmented over maximum 3 bds: header, partial mpa, unaligned */ +#define ECORE_IWARP_MAX_BDS_PER_FPDU 3 + +char *pkt_type_str[] = { + "ECORE_IWARP_MPA_PKT_PACKED", + "ECORE_IWARP_MPA_PKT_PARTIAL", + "ECORE_IWARP_MPA_PKT_UNALIGNED" +}; + +static enum _ecore_status_t +ecore_iwarp_recycle_pkt(struct ecore_hwfn *p_hwfn, + struct ecore_iwarp_fpdu *fpdu, + struct ecore_iwarp_ll2_buff *buf); + +static enum ecore_iwarp_mpa_pkt_type +ecore_iwarp_mpa_classify(struct ecore_hwfn *p_hwfn, + struct ecore_iwarp_fpdu *fpdu, + u16 tcp_payload_len, + u8 *mpa_data) + +{ + enum ecore_iwarp_mpa_pkt_type pkt_type; + u16 mpa_len; + + if (fpdu->incomplete_bytes) { + pkt_type = ECORE_IWARP_MPA_PKT_UNALIGNED; + goto out; + } + + /* special case of one byte remaining... */ + if (tcp_payload_len == 1) { + /* lower byte will be read next packet */ + fpdu->fpdu_length = *mpa_data << 8; + pkt_type = ECORE_IWARP_MPA_PKT_PARTIAL; + goto out; + } + + mpa_len = ntohs(*((u16 *)(mpa_data))); + fpdu->fpdu_length = ECORE_IWARP_FPDU_LEN_WITH_PAD(mpa_len); + + if (fpdu->fpdu_length <= tcp_payload_len) + pkt_type = ECORE_IWARP_MPA_PKT_PACKED; + else + pkt_type = ECORE_IWARP_MPA_PKT_PARTIAL; + +out: + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, + "MPA_ALIGN: %s: fpdu_length=0x%x tcp_payload_len:0x%x\n", + pkt_type_str[pkt_type], fpdu->fpdu_length, tcp_payload_len); + + return pkt_type; +} + +static void +ecore_iwarp_init_fpdu(struct ecore_iwarp_ll2_buff *buf, + struct ecore_iwarp_fpdu *fpdu, + struct unaligned_opaque_data *pkt_data, + u16 tcp_payload_size, u8 placement_offset) +{ + fpdu->mpa_buf = buf; + fpdu->pkt_hdr = buf->data_phys_addr + placement_offset; + fpdu->pkt_hdr_size = pkt_data->tcp_payload_offset; + + fpdu->mpa_frag = buf->data_phys_addr + pkt_data->first_mpa_offset; + fpdu->mpa_frag_virt = (u8 *)(buf->data) + pkt_data->first_mpa_offset; + + if (tcp_payload_size == 1) + fpdu->incomplete_bytes = ECORE_IWARP_INVALID_FPDU_LENGTH; + else if (tcp_payload_size < fpdu->fpdu_length) + fpdu->incomplete_bytes = fpdu->fpdu_length - tcp_payload_size; + else + fpdu->incomplete_bytes = 0; /* complete fpdu */ + + fpdu->mpa_frag_len = fpdu->fpdu_length - fpdu->incomplete_bytes; +} + +static enum _ecore_status_t +ecore_iwarp_copy_fpdu(struct ecore_hwfn *p_hwfn, + struct ecore_iwarp_fpdu *fpdu, + struct unaligned_opaque_data *pkt_data, + struct ecore_iwarp_ll2_buff *buf, + u16 tcp_payload_size) + +{ + u8 *tmp_buf = p_hwfn->p_rdma_info->iwarp.mpa_intermediate_buf; + enum _ecore_status_t rc; + + /* need to copy the data from the partial packet stored in fpdu + * to the new buf, for this we also need to move the data currently + * placed on the buf. The assumption is that the buffer is big enough + * since fpdu_length <= mss, we use an intermediate buffer since + * we may need to copy the new data to an overlapping location + */ + if ((fpdu->mpa_frag_len + tcp_payload_size) > (u16)buf->buff_size) { + DP_ERR(p_hwfn, + "MPA ALIGN: Unexpected: buffer is not large enough for split fpdu buff_size = %d mpa_frag_len = %d, tcp_payload_size = %d, incomplete_bytes = %d\n", + buf->buff_size, fpdu->mpa_frag_len, tcp_payload_size, + fpdu->incomplete_bytes); + return ECORE_INVAL; + } + + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, + "MPA ALIGN Copying fpdu: [%p, %d] [%p, %d]\n", + fpdu->mpa_frag_virt, fpdu->mpa_frag_len, + (u8 *)(buf->data) + pkt_data->first_mpa_offset, + tcp_payload_size); + + OSAL_MEMCPY(tmp_buf, fpdu->mpa_frag_virt, fpdu->mpa_frag_len); + OSAL_MEMCPY(tmp_buf + fpdu->mpa_frag_len, + (u8 *)(buf->data) + pkt_data->first_mpa_offset, + tcp_payload_size); + + rc = ecore_iwarp_recycle_pkt(p_hwfn, fpdu, fpdu->mpa_buf); + if (rc) + return rc; + + /* If we managed to post the buffer copy the data to the new buffer + * o/w this will occur in the next round... + */ + OSAL_MEMCPY((u8 *)(buf->data), tmp_buf, + fpdu->mpa_frag_len + tcp_payload_size); + + fpdu->mpa_buf = buf; + /* fpdu->pkt_hdr remains as is */ + /* fpdu->mpa_frag is overriden with new buf */ + fpdu->mpa_frag = buf->data_phys_addr; + fpdu->mpa_frag_virt = buf->data; + fpdu->mpa_frag_len += tcp_payload_size; + + fpdu->incomplete_bytes -= tcp_payload_size; + + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, + "MPA ALIGN: split fpdu buff_size = %d mpa_frag_len = %d, tcp_payload_size = %d, incomplete_bytes = %d\n", + buf->buff_size, fpdu->mpa_frag_len, tcp_payload_size, + fpdu->incomplete_bytes); + + return 0; +} + +static void +ecore_iwarp_update_fpdu_length(struct ecore_hwfn *p_hwfn, + struct ecore_iwarp_fpdu *fpdu, + u8 *mpa_data) +{ + u16 mpa_len; + + /* Update incomplete packets if needed */ + if (fpdu->incomplete_bytes == ECORE_IWARP_INVALID_FPDU_LENGTH) { + mpa_len = fpdu->fpdu_length | *mpa_data; + fpdu->fpdu_length = ECORE_IWARP_FPDU_LEN_WITH_PAD(mpa_len); + fpdu->mpa_frag_len = fpdu->fpdu_length; + /* one byte of hdr */ + fpdu->incomplete_bytes = fpdu->fpdu_length - 1; + DP_VERBOSE(p_hwfn, + ECORE_MSG_RDMA, + "MPA_ALIGN: Partial header mpa_len=%x fpdu_length=%x incomplete_bytes=%x\n", + mpa_len, fpdu->fpdu_length, fpdu->incomplete_bytes); + } +} + +#define ECORE_IWARP_IS_RIGHT_EDGE(_curr_pkt) \ + (GET_FIELD(_curr_pkt->flags, \ + UNALIGNED_OPAQUE_DATA_PKT_REACHED_WIN_RIGHT_EDGE)) + +/* This function is used to recycle a buffer using the ll2 drop option. It + * uses the mechanism to ensure that all buffers posted to tx before this one + * were completed. The buffer sent here will be sent as a cookie in the tx + * completion function and can then be reposted to rx chain when done. The flow + * that requires this is the flow where a FPDU splits over more than 3 tcp + * segments. In this case the driver needs to re-post a rx buffer instead of + * the one received, but driver can't simply repost a buffer it copied from + * as there is a case where the buffer was originally a packed FPDU, and is + * partially posted to FW. Driver needs to ensure FW is done with it. + */ +static enum _ecore_status_t +ecore_iwarp_recycle_pkt(struct ecore_hwfn *p_hwfn, + struct ecore_iwarp_fpdu *fpdu, + struct ecore_iwarp_ll2_buff *buf) +{ + struct ecore_ll2_tx_pkt_info tx_pkt; + enum _ecore_status_t rc; + u8 ll2_handle; + + OSAL_MEM_ZERO(&tx_pkt, sizeof(tx_pkt)); + tx_pkt.num_of_bds = 1; + tx_pkt.tx_dest = ECORE_LL2_TX_DEST_DROP; + tx_pkt.l4_hdr_offset_w = fpdu->pkt_hdr_size >> 2; + tx_pkt.first_frag = fpdu->pkt_hdr; + tx_pkt.first_frag_len = fpdu->pkt_hdr_size; + buf->piggy_buf = OSAL_NULL; + tx_pkt.cookie = buf; + + ll2_handle = p_hwfn->p_rdma_info->iwarp.ll2_mpa_handle; + + rc = ecore_ll2_prepare_tx_packet(p_hwfn, + ll2_handle, + &tx_pkt, true); + + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, + "MPA_ALIGN: send drop tx packet [%lx, 0x%x], buf=%p, rc=%d\n", + (long unsigned int)tx_pkt.first_frag, + tx_pkt.first_frag_len, buf, rc); + + if (rc) + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, + "Can't drop packet rc=%d\n", rc); + + return rc; +} + +static enum _ecore_status_t +ecore_iwarp_win_right_edge(struct ecore_hwfn *p_hwfn, + struct ecore_iwarp_fpdu *fpdu) +{ + struct ecore_ll2_tx_pkt_info tx_pkt; + enum _ecore_status_t rc; + u8 ll2_handle; + + OSAL_MEM_ZERO(&tx_pkt, sizeof(tx_pkt)); + tx_pkt.num_of_bds = 1; + tx_pkt.tx_dest = ECORE_LL2_TX_DEST_LB; + tx_pkt.l4_hdr_offset_w = fpdu->pkt_hdr_size >> 2; + + tx_pkt.first_frag = fpdu->pkt_hdr; + tx_pkt.first_frag_len = fpdu->pkt_hdr_size; + tx_pkt.enable_ip_cksum = true; + tx_pkt.enable_l4_cksum = true; + tx_pkt.calc_ip_len = true; + /* vlan overload with enum iwarp_ll2_tx_queues */ + tx_pkt.vlan = IWARP_LL2_ALIGNED_RIGHT_TRIMMED_TX_QUEUE; + + ll2_handle = p_hwfn->p_rdma_info->iwarp.ll2_mpa_handle; + + rc = ecore_ll2_prepare_tx_packet(p_hwfn, + ll2_handle, + &tx_pkt, true); + + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, + "MPA_ALIGN: Sent right edge FPDU num_bds=%d [%lx, 0x%x], rc=%d\n", + tx_pkt.num_of_bds, (long unsigned int)tx_pkt.first_frag, + tx_pkt.first_frag_len, rc); + + if (rc) + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, + "Can't send right edge rc=%d\n", rc); + + return rc; +} + +static enum _ecore_status_t +ecore_iwarp_send_fpdu(struct ecore_hwfn *p_hwfn, + struct ecore_iwarp_fpdu *fpdu, + struct unaligned_opaque_data *curr_pkt, + struct ecore_iwarp_ll2_buff *buf, + u16 tcp_payload_size, + enum ecore_iwarp_mpa_pkt_type pkt_type) +{ + struct ecore_ll2_tx_pkt_info tx_pkt; + enum _ecore_status_t rc; + u8 ll2_handle; + + OSAL_MEM_ZERO(&tx_pkt, sizeof(tx_pkt)); + + tx_pkt.num_of_bds = (pkt_type == ECORE_IWARP_MPA_PKT_UNALIGNED) ? 3 : 2; + tx_pkt.tx_dest = ECORE_LL2_TX_DEST_LB; + tx_pkt.l4_hdr_offset_w = fpdu->pkt_hdr_size >> 2; + + /* Send the mpa_buf only with the last fpdu (in case of packed) */ + if ((pkt_type == ECORE_IWARP_MPA_PKT_UNALIGNED) || + (tcp_payload_size <= fpdu->fpdu_length)) + tx_pkt.cookie = fpdu->mpa_buf; + + tx_pkt.first_frag = fpdu->pkt_hdr; + tx_pkt.first_frag_len = fpdu->pkt_hdr_size; + tx_pkt.enable_ip_cksum = true; + tx_pkt.enable_l4_cksum = true; + tx_pkt.calc_ip_len = true; + /* vlan overload with enum iwarp_ll2_tx_queues */ + tx_pkt.vlan = IWARP_LL2_ALIGNED_TX_QUEUE; + + /* special case of unaligned packet and not packed, need to send + * both buffers as cookie to release. + */ + if (tcp_payload_size == fpdu->incomplete_bytes) { + fpdu->mpa_buf->piggy_buf = buf; + } + + ll2_handle = p_hwfn->p_rdma_info->iwarp.ll2_mpa_handle; + + rc = ecore_ll2_prepare_tx_packet(p_hwfn, + ll2_handle, + &tx_pkt, true); + if (rc) + goto err; + + rc = ecore_ll2_set_fragment_of_tx_packet(p_hwfn, ll2_handle, + fpdu->mpa_frag, + fpdu->mpa_frag_len); + if (rc) + goto err; + + if (fpdu->incomplete_bytes) { + rc = ecore_ll2_set_fragment_of_tx_packet( + p_hwfn, ll2_handle, + buf->data_phys_addr + curr_pkt->first_mpa_offset, + fpdu->incomplete_bytes); + + if (rc) + goto err; + } + +err: + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, + "MPA_ALIGN: Sent FPDU num_bds=%d [%lx, 0x%x], [0x%lx, 0x%x], [0x%lx, 0x%x] (cookie %p) rc=%d\n", + tx_pkt.num_of_bds, (long unsigned int)tx_pkt.first_frag, + tx_pkt.first_frag_len, (long unsigned int)fpdu->mpa_frag, + fpdu->mpa_frag_len, (long unsigned int)buf->data_phys_addr + + curr_pkt->first_mpa_offset, fpdu->incomplete_bytes, + tx_pkt.cookie, rc); + + return rc; +} + +static void +ecore_iwarp_mpa_get_data(struct ecore_hwfn *p_hwfn, + struct unaligned_opaque_data *curr_pkt, + u32 opaque_data0, u32 opaque_data1) +{ + u64 opaque_data; + + opaque_data = HILO_64(opaque_data1, opaque_data0); + *curr_pkt = *((struct unaligned_opaque_data *)&opaque_data); + + /* fix endianity */ + curr_pkt->first_mpa_offset = curr_pkt->tcp_payload_offset + + OSAL_LE16_TO_CPU(curr_pkt->first_mpa_offset); + curr_pkt->cid = OSAL_LE32_TO_CPU(curr_pkt->cid); + + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, + "OPAQUE0=0x%x OPAQUE1=0x%x first_mpa_offset:0x%x\ttcp_payload_offset:0x%x\tflags:0x%x\tcid:0x%x\n", + opaque_data0, opaque_data1, curr_pkt->first_mpa_offset, + curr_pkt->tcp_payload_offset, curr_pkt->flags, + curr_pkt->cid); +} + +static void +ecore_iwarp_mpa_print_tcp_seq(struct ecore_hwfn *p_hwfn, + void *buf) +{ + struct ecore_vlan_ethhdr *vethh; + struct ecore_ethhdr *ethh; + struct ecore_iphdr *iph; + struct ecore_ipv6hdr *ip6h; + struct ecore_tcphdr *tcph; + bool vlan_valid = false; + int eth_hlen, ip_hlen; + u16 eth_type; + + if ((p_hwfn->dp_level > ECORE_LEVEL_VERBOSE) || + !(p_hwfn->dp_module & ECORE_MSG_RDMA)) + return; + + ethh = (struct ecore_ethhdr *)buf; + eth_type = ntohs(ethh->h_proto); + if (eth_type == ETH_P_8021Q) { + vlan_valid = true; + vethh = (struct ecore_vlan_ethhdr *)ethh; + eth_type = ntohs(vethh->h_vlan_encapsulated_proto); + } + + eth_hlen = ETH_HLEN + (vlan_valid ? sizeof(u32) : 0); + + iph = (struct ecore_iphdr *)((u8 *)(ethh) + eth_hlen); + + if (eth_type == ETH_P_IP) { + ip_hlen = (iph->ihl)*sizeof(u32); + } else if (eth_type == ETH_P_IPV6) { + ip6h = (struct ecore_ipv6hdr *)iph; + ip_hlen = sizeof(*ip6h); + } else { + DP_ERR(p_hwfn, "Unexpected ethertype on ll2 %x\n", eth_type); + return; + } + + tcph = (struct ecore_tcphdr *)((u8 *)iph + ip_hlen); + + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "Processing MPA PKT: tcp_seq=0x%x tcp_ack_seq=0x%x\n", + ntohl(tcph->seq), ntohl(tcph->ack_seq)); + + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "eth_type =%d Source mac: [0x%x]:[0x%x]:[0x%x]:[0x%x]:[0x%x]:[0x%x]\n", + eth_type, ethh->h_source[0], ethh->h_source[1], + ethh->h_source[2], ethh->h_source[3], + ethh->h_source[4], ethh->h_source[5]); + + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "eth_hlen=%d destination mac: [0x%x]:[0x%x]:[0x%x]:[0x%x]:[0x%x]:[0x%x]\n", + eth_hlen, ethh->h_dest[0], ethh->h_dest[1], + ethh->h_dest[2], ethh->h_dest[3], + ethh->h_dest[4], ethh->h_dest[5]); + + return; +} + +/* This function is called when an unaligned or incomplete MPA packet arrives + * driver needs to align the packet, perhaps using previous data and send + * it down to FW once it is aligned. + */ +static enum _ecore_status_t +ecore_iwarp_process_mpa_pkt(struct ecore_hwfn *p_hwfn, + struct ecore_iwarp_ll2_mpa_buf *mpa_buf) +{ + struct ecore_iwarp_ll2_buff *buf = mpa_buf->ll2_buf; + enum ecore_iwarp_mpa_pkt_type pkt_type; + struct unaligned_opaque_data *curr_pkt = &mpa_buf->data; + struct ecore_iwarp_fpdu *fpdu; + u8 *mpa_data; + enum _ecore_status_t rc = ECORE_SUCCESS; + + ecore_iwarp_mpa_print_tcp_seq( + p_hwfn, (u8 *)(buf->data) + mpa_buf->placement_offset); + + fpdu = ecore_iwarp_get_curr_fpdu(p_hwfn, curr_pkt->cid & 0xffff); + if (!fpdu) {/* something corrupt with cid, post rx back */ + DP_ERR(p_hwfn, "Invalid cid, drop and post back to rx cid=%x\n", + curr_pkt->cid); + rc = ecore_iwarp_ll2_post_rx( + p_hwfn, buf, p_hwfn->p_rdma_info->iwarp.ll2_mpa_handle); + + if (rc) { /* not much we can do here except log and free */ + DP_ERR(p_hwfn, "Post rx buffer failed\n"); + + /* we don't expect any failures from rx, not even + * busy since we allocate #bufs=#descs + */ + rc = ECORE_UNKNOWN_ERROR; + } + return rc; + } + + do { + mpa_data = ((u8 *)(buf->data) + curr_pkt->first_mpa_offset); + + pkt_type = ecore_iwarp_mpa_classify(p_hwfn, fpdu, + mpa_buf->tcp_payload_len, + mpa_data); + + switch (pkt_type) { + case ECORE_IWARP_MPA_PKT_PARTIAL: + ecore_iwarp_init_fpdu(buf, fpdu, + curr_pkt, + mpa_buf->tcp_payload_len, + mpa_buf->placement_offset); + + if (!ECORE_IWARP_IS_RIGHT_EDGE(curr_pkt)) { + mpa_buf->tcp_payload_len = 0; + break; + } + + rc = ecore_iwarp_win_right_edge(p_hwfn, fpdu); + + if (rc) { + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, + "Can't send FPDU:reset rc=%d\n", rc); + OSAL_MEM_ZERO(fpdu, sizeof(*fpdu)); + break; + } + + mpa_buf->tcp_payload_len = 0; + break; + case ECORE_IWARP_MPA_PKT_PACKED: + if (fpdu->fpdu_length == 8) { + DP_ERR(p_hwfn, "SUSPICIOUS fpdu_length = 0x%x: assuming bug...aborting this packet...\n", + fpdu->fpdu_length); + mpa_buf->tcp_payload_len = 0; + break; + } + + ecore_iwarp_init_fpdu(buf, fpdu, + curr_pkt, + mpa_buf->tcp_payload_len, + mpa_buf->placement_offset); + + rc = ecore_iwarp_send_fpdu(p_hwfn, fpdu, curr_pkt, buf, + mpa_buf->tcp_payload_len, + pkt_type); + if (rc) { + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, + "Can't send FPDU:reset rc=%d\n", rc); + OSAL_MEM_ZERO(fpdu, sizeof(*fpdu)); + break; + } + mpa_buf->tcp_payload_len -= fpdu->fpdu_length; + curr_pkt->first_mpa_offset += fpdu->fpdu_length; + break; + case ECORE_IWARP_MPA_PKT_UNALIGNED: + ecore_iwarp_update_fpdu_length(p_hwfn, fpdu, mpa_data); + if (mpa_buf->tcp_payload_len < fpdu->incomplete_bytes) { + /* special handling of fpdu split over more + * than 2 segments + */ + if (ECORE_IWARP_IS_RIGHT_EDGE(curr_pkt)) { + rc = ecore_iwarp_win_right_edge(p_hwfn, + fpdu); + /* packet will be re-processed later */ + if (rc) + return rc; + } + + rc = ecore_iwarp_copy_fpdu( + p_hwfn, fpdu, curr_pkt, + buf, mpa_buf->tcp_payload_len); + + /* packet will be re-processed later */ + if (rc) + return rc; + + mpa_buf->tcp_payload_len = 0; + + break; + } + + rc = ecore_iwarp_send_fpdu(p_hwfn, fpdu, curr_pkt, buf, + mpa_buf->tcp_payload_len, + pkt_type); + if (rc) { + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, + "Can't send FPDU:delay rc=%d\n", rc); + /* don't reset fpdu -> we need it for next + * classify + */ + break; + } + mpa_buf->tcp_payload_len -= fpdu->incomplete_bytes; + curr_pkt->first_mpa_offset += fpdu->incomplete_bytes; + /* The framed PDU was sent - no more incomplete bytes */ + fpdu->incomplete_bytes = 0; + break; + } + + } while (mpa_buf->tcp_payload_len && !rc); + + return rc; +} + +static void +ecore_iwarp_process_pending_pkts(struct ecore_hwfn *p_hwfn) +{ + struct ecore_iwarp_info *iwarp_info = &p_hwfn->p_rdma_info->iwarp; + struct ecore_iwarp_ll2_mpa_buf *mpa_buf = OSAL_NULL; + enum _ecore_status_t rc; + + while (!OSAL_LIST_IS_EMPTY(&iwarp_info->mpa_buf_pending_list)) { + mpa_buf = OSAL_LIST_FIRST_ENTRY( + &iwarp_info->mpa_buf_pending_list, + struct ecore_iwarp_ll2_mpa_buf, + list_entry); + + rc = ecore_iwarp_process_mpa_pkt(p_hwfn, mpa_buf); + + /* busy means break and continue processing later, don't + * remove the buf from the pending list. + */ + if (rc == ECORE_BUSY) + break; + +#ifdef _NTDDK_ +#pragma warning(suppress : 6011) +#pragma warning(suppress : 28182) +#endif + OSAL_LIST_REMOVE_ENTRY( + &mpa_buf->list_entry, + &iwarp_info->mpa_buf_pending_list); + + OSAL_LIST_PUSH_TAIL(&mpa_buf->list_entry, + &iwarp_info->mpa_buf_list); + + if (rc) { /* different error, don't continue */ + DP_NOTICE(p_hwfn, false, "process pkts failed rc=%d\n", + rc); + break; + } + } +} + +static void +ecore_iwarp_ll2_comp_mpa_pkt(void *cxt, + struct ecore_ll2_comp_rx_data *data) +{ + struct ecore_hwfn *p_hwfn = (struct ecore_hwfn *)cxt; + struct ecore_iwarp_info *iwarp_info = &p_hwfn->p_rdma_info->iwarp; + struct ecore_iwarp_ll2_mpa_buf *mpa_buf; + + iwarp_info->unalign_rx_comp++; + + mpa_buf = OSAL_LIST_FIRST_ENTRY(&iwarp_info->mpa_buf_list, + struct ecore_iwarp_ll2_mpa_buf, + list_entry); + + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, + "LL2 MPA CompRx buf=%p placement_offset=%d, payload_len=0x%x mpa_buf=%p\n", + data->cookie, data->u.placement_offset, + data->length.packet_length, mpa_buf); + + if (!mpa_buf) { + DP_ERR(p_hwfn, "no free mpa buf. this is a driver bug.\n"); + return; + } + OSAL_LIST_REMOVE_ENTRY(&mpa_buf->list_entry, &iwarp_info->mpa_buf_list); + + ecore_iwarp_mpa_get_data(p_hwfn, &mpa_buf->data, + data->opaque_data_0, data->opaque_data_1); + + mpa_buf->tcp_payload_len = data->length.packet_length - + mpa_buf->data.first_mpa_offset; + mpa_buf->ll2_buf = (struct ecore_iwarp_ll2_buff *)data->cookie; + mpa_buf->data.first_mpa_offset += data->u.placement_offset; + mpa_buf->placement_offset = data->u.placement_offset; + + OSAL_LIST_PUSH_TAIL(&mpa_buf->list_entry, + &iwarp_info->mpa_buf_pending_list); + + ecore_iwarp_process_pending_pkts(p_hwfn); +} + +static void +ecore_iwarp_ll2_comp_syn_pkt(void *cxt, struct ecore_ll2_comp_rx_data *data) +{ + struct ecore_hwfn *p_hwfn = (struct ecore_hwfn *)cxt; + struct ecore_iwarp_ll2_buff *buf = + (struct ecore_iwarp_ll2_buff *)data->cookie; + struct ecore_iwarp_listener *listener; + struct ecore_iwarp_cm_info cm_info; + struct ecore_ll2_tx_pkt_info tx_pkt; + u8 remote_mac_addr[ETH_ALEN]; + u8 local_mac_addr[ETH_ALEN]; + struct ecore_iwarp_ep *ep; + enum _ecore_status_t rc; + int tcp_start_offset; + u8 ts_hdr_size = 0; + int payload_len; + u32 hdr_size; + + OSAL_MEM_ZERO(&cm_info, sizeof(cm_info)); + + /* Check if packet was received with errors... */ + if (data->err_flags != 0) { + DP_NOTICE(p_hwfn, false, "Error received on SYN packet: 0x%x\n", + data->err_flags); + goto err; + } + + if (GET_FIELD(data->parse_flags, + PARSING_AND_ERR_FLAGS_L4CHKSMWASCALCULATED) && + GET_FIELD(data->parse_flags, + PARSING_AND_ERR_FLAGS_L4CHKSMERROR)) { + DP_NOTICE(p_hwfn, false, "Syn packet received with checksum error\n"); + goto err; + } + + rc = ecore_iwarp_parse_rx_pkt( + p_hwfn, &cm_info, (u8 *)(buf->data) + data->u.placement_offset, + remote_mac_addr, local_mac_addr, &payload_len, + &tcp_start_offset); + if (rc) + goto err; + + /* Check if there is a listener for this 4-tuple */ + listener = ecore_iwarp_get_listener(p_hwfn, &cm_info); + if (!listener) { + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, + "SYN received on tuple not listened on parse_flags=%d packet len=%d\n", + data->parse_flags, data->length.packet_length); + + OSAL_MEMSET(&tx_pkt, 0, sizeof(tx_pkt)); + tx_pkt.num_of_bds = 1; + tx_pkt.bd_flags = 0; + tx_pkt.l4_hdr_offset_w = (data->length.packet_length) >> 2; + tx_pkt.tx_dest = ECORE_LL2_TX_DEST_LB; + tx_pkt.first_frag = buf->data_phys_addr + + data->u.placement_offset; + tx_pkt.first_frag_len = data->length.packet_length; + tx_pkt.cookie = buf; + + rc = ecore_ll2_prepare_tx_packet( + p_hwfn, + p_hwfn->p_rdma_info->iwarp.ll2_syn_handle, + &tx_pkt, true); + + if (rc) { + DP_NOTICE(p_hwfn, false, + "Can't post SYN back to chip rc=%d\n", rc); + goto err; + } + return; + } + + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "Received syn on listening port\n"); + + /* For debugging purpose... */ + if (listener->drop) + goto err; + + /* There may be an open ep on this connection if this is a syn + * retrasnmit... need to make sure there isn't... + */ + if (ecore_iwarp_ep_exists(p_hwfn, listener, &cm_info)) + goto err; + + ep = ecore_iwarp_get_free_ep(p_hwfn); + if (ep == OSAL_NULL) + goto err; + + OSAL_SPIN_LOCK(&listener->lock); + OSAL_LIST_PUSH_TAIL(&ep->list_entry, &listener->ep_list); + OSAL_SPIN_UNLOCK(&listener->lock); + + OSAL_MEMCPY(ep->remote_mac_addr, + remote_mac_addr, + ETH_ALEN); + OSAL_MEMCPY(ep->local_mac_addr, + local_mac_addr, + ETH_ALEN); + + OSAL_MEMCPY(&ep->cm_info, &cm_info, sizeof(ep->cm_info)); + + if (p_hwfn->p_rdma_info->iwarp.tcp_flags & ECORE_IWARP_TS_EN) + ts_hdr_size = TIMESTAMP_HEADER_SIZE; + + hdr_size = ((cm_info.ip_version == ECORE_TCP_IPV4) ? 40 : 60) + + ts_hdr_size; + ep->mss = p_hwfn->p_rdma_info->iwarp.max_mtu - hdr_size; + ep->mss = OSAL_MIN_T(u16, ECORE_IWARP_MAX_FW_MSS, ep->mss); + + ep->listener = listener; + ep->event_cb = listener->event_cb; + ep->cb_context = listener->cb_context; + ep->connect_mode = TCP_CONNECT_PASSIVE; + + ep->syn = buf; + ep->syn_ip_payload_length = (u16)payload_len; + ep->syn_phy_addr = buf->data_phys_addr + data->u.placement_offset + + tcp_start_offset; + + rc = ecore_iwarp_tcp_offload(p_hwfn, ep); + if (rc != ECORE_SUCCESS) { + ecore_iwarp_return_ep(p_hwfn, ep); + goto err; + } + return; + +err: + ecore_iwarp_ll2_post_rx( + p_hwfn, buf, p_hwfn->p_rdma_info->iwarp.ll2_syn_handle); +} + +static void +ecore_iwarp_ll2_rel_rx_pkt(void *cxt, + u8 OSAL_UNUSED connection_handle, + void *cookie, + dma_addr_t OSAL_UNUSED rx_buf_addr, + bool OSAL_UNUSED b_last_packet) +{ + struct ecore_hwfn *p_hwfn = (struct ecore_hwfn *)cxt; + struct ecore_iwarp_ll2_buff *buffer = + (struct ecore_iwarp_ll2_buff *)cookie; + + OSAL_DMA_FREE_COHERENT(p_hwfn->p_dev, + buffer->data, + buffer->data_phys_addr, + buffer->buff_size); + + OSAL_FREE(p_hwfn->p_dev, buffer); +} + +static void +ecore_iwarp_ll2_comp_tx_pkt(void *cxt, + u8 connection_handle, + void *cookie, + dma_addr_t OSAL_UNUSED first_frag_addr, + bool OSAL_UNUSED b_last_fragment, + bool OSAL_UNUSED b_last_packet) +{ + struct ecore_hwfn *p_hwfn = (struct ecore_hwfn *)cxt; + struct ecore_iwarp_ll2_buff *buffer = + (struct ecore_iwarp_ll2_buff *)cookie; + struct ecore_iwarp_ll2_buff *piggy; + + if (!buffer) /* can happen in packed mpa unaligned... */ + return; + + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, + "LL2 CompTX buf=%p piggy_buf=%p handle=%d\n", + buffer, buffer->piggy_buf, connection_handle); + + /* we got a tx packet -> this was originally a rx packet... now we + * can post it back... + */ + piggy = buffer->piggy_buf; + if (piggy) { + buffer->piggy_buf = OSAL_NULL; + ecore_iwarp_ll2_post_rx(p_hwfn, piggy, + connection_handle); + } + + ecore_iwarp_ll2_post_rx(p_hwfn, buffer, + connection_handle); + + if (connection_handle == p_hwfn->p_rdma_info->iwarp.ll2_mpa_handle) + ecore_iwarp_process_pending_pkts(p_hwfn); + + return; +} + +static void +ecore_iwarp_ll2_rel_tx_pkt(void *cxt, + u8 OSAL_UNUSED connection_handle, + void *cookie, + dma_addr_t OSAL_UNUSED first_frag_addr, + bool OSAL_UNUSED b_last_fragment, + bool OSAL_UNUSED b_last_packet) +{ + struct ecore_hwfn *p_hwfn = (struct ecore_hwfn *)cxt; + struct ecore_iwarp_ll2_buff *buffer = + (struct ecore_iwarp_ll2_buff *)cookie; + + if (!buffer) + return; + + if (buffer->piggy_buf) { + OSAL_DMA_FREE_COHERENT( + p_hwfn->p_dev, + buffer->piggy_buf->data, + buffer->piggy_buf->data_phys_addr, + buffer->piggy_buf->buff_size); + + OSAL_FREE(p_hwfn->p_dev, buffer->piggy_buf); + } + + OSAL_DMA_FREE_COHERENT(p_hwfn->p_dev, + buffer->data, + buffer->data_phys_addr, + buffer->buff_size); + + OSAL_FREE(p_hwfn->p_dev, buffer); + return; +} + +/* Current known slowpath for iwarp ll2 is unalign flush. When this completion + * is received, need to reset the FPDU. + */ +static void +ecore_iwarp_ll2_slowpath(void *cxt, + u8 OSAL_UNUSED connection_handle, + u32 opaque_data_0, + u32 opaque_data_1) +{ + struct ecore_hwfn *p_hwfn = (struct ecore_hwfn *)cxt; + struct unaligned_opaque_data unalign_data; + struct ecore_iwarp_fpdu *fpdu; + + ecore_iwarp_mpa_get_data(p_hwfn, &unalign_data, + opaque_data_0, opaque_data_1); + + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "(0x%x) Flush fpdu\n", + unalign_data.cid); + + fpdu = ecore_iwarp_get_curr_fpdu(p_hwfn, (u16)unalign_data.cid); + if (fpdu) + OSAL_MEM_ZERO(fpdu, sizeof(*fpdu)); +} + +static int +ecore_iwarp_ll2_stop(struct ecore_hwfn *p_hwfn) +{ + struct ecore_iwarp_info *iwarp_info = &p_hwfn->p_rdma_info->iwarp; + int rc = 0; + + if (iwarp_info->ll2_syn_handle != ECORE_IWARP_HANDLE_INVAL) { + + rc = ecore_ll2_terminate_connection(p_hwfn, + iwarp_info->ll2_syn_handle); + if (rc) + DP_INFO(p_hwfn, "Failed to terminate syn connection\n"); + + ecore_ll2_release_connection(p_hwfn, + iwarp_info->ll2_syn_handle); + iwarp_info->ll2_syn_handle = ECORE_IWARP_HANDLE_INVAL; + } + + if (iwarp_info->ll2_ooo_handle != ECORE_IWARP_HANDLE_INVAL) { + rc = ecore_ll2_terminate_connection(p_hwfn, + iwarp_info->ll2_ooo_handle); + if (rc) + DP_INFO(p_hwfn, "Failed to terminate ooo connection\n"); + + ecore_ll2_release_connection(p_hwfn, + iwarp_info->ll2_ooo_handle); + iwarp_info->ll2_ooo_handle = ECORE_IWARP_HANDLE_INVAL; + } + + if (iwarp_info->ll2_mpa_handle != ECORE_IWARP_HANDLE_INVAL) { + rc = ecore_ll2_terminate_connection(p_hwfn, + iwarp_info->ll2_mpa_handle); + if (rc) + DP_INFO(p_hwfn, "Failed to terminate mpa connection\n"); + + ecore_ll2_release_connection(p_hwfn, + iwarp_info->ll2_mpa_handle); + iwarp_info->ll2_mpa_handle = ECORE_IWARP_HANDLE_INVAL; + } + + ecore_llh_remove_mac_filter(p_hwfn->p_dev, 0, + p_hwfn->p_rdma_info->iwarp.mac_addr); + + return rc; +} + +static int +ecore_iwarp_ll2_alloc_buffers(struct ecore_hwfn *p_hwfn, + int num_rx_bufs, + int buff_size, + u8 ll2_handle) +{ + struct ecore_iwarp_ll2_buff *buffer; + int rc = 0; + int i; + + for (i = 0; i < num_rx_bufs; i++) { + buffer = OSAL_ZALLOC(p_hwfn->p_dev, + GFP_KERNEL, sizeof(*buffer)); + if (!buffer) { + DP_INFO(p_hwfn, "Failed to allocate LL2 buffer desc\n"); + break; + } + + buffer->data = + OSAL_DMA_ALLOC_COHERENT(p_hwfn->p_dev, + &buffer->data_phys_addr, + buff_size); + + if (!buffer->data) { + DP_INFO(p_hwfn, "Failed to allocate LL2 buffers\n"); + OSAL_FREE(p_hwfn->p_dev, buffer); + rc = ECORE_NOMEM; + break; + } + + buffer->buff_size = buff_size; + rc = ecore_iwarp_ll2_post_rx(p_hwfn, buffer, ll2_handle); + + if (rc) + break; /* buffers will be deallocated by ecore_ll2 */ + } + return rc; +} + +#define ECORE_IWARP_CACHE_PADDING(size) \ + (((size) + ETH_CACHE_LINE_SIZE - 1) & ~(ETH_CACHE_LINE_SIZE - 1)) + +#define ECORE_IWARP_MAX_BUF_SIZE(mtu) \ + ECORE_IWARP_CACHE_PADDING(mtu + ETH_HLEN + 2*VLAN_HLEN + 2 +\ + ETH_CACHE_LINE_SIZE) + +static int +ecore_iwarp_ll2_start(struct ecore_hwfn *p_hwfn, + struct ecore_rdma_start_in_params *params) +{ + struct ecore_iwarp_info *iwarp_info; + struct ecore_ll2_acquire_data data; + struct ecore_ll2_cbs cbs; + u32 mpa_buff_size; + int rc = ECORE_SUCCESS; + u16 n_ooo_bufs; + int i; + + iwarp_info = &p_hwfn->p_rdma_info->iwarp; + iwarp_info->ll2_syn_handle = ECORE_IWARP_HANDLE_INVAL; + iwarp_info->ll2_ooo_handle = ECORE_IWARP_HANDLE_INVAL; + iwarp_info->ll2_mpa_handle = ECORE_IWARP_HANDLE_INVAL; + + iwarp_info->max_mtu = params->max_mtu; + + OSAL_MEMCPY(p_hwfn->p_rdma_info->iwarp.mac_addr, params->mac_addr, + ETH_ALEN); + + rc = ecore_llh_add_mac_filter(p_hwfn->p_dev, 0, params->mac_addr); + if (rc != ECORE_SUCCESS) + return rc; + + /* Start SYN connection */ + cbs.rx_comp_cb = ecore_iwarp_ll2_comp_syn_pkt; + cbs.rx_release_cb = ecore_iwarp_ll2_rel_rx_pkt; + cbs.tx_comp_cb = ecore_iwarp_ll2_comp_tx_pkt; + cbs.tx_release_cb = ecore_iwarp_ll2_rel_tx_pkt; + cbs.cookie = p_hwfn; + + OSAL_MEMSET(&data, 0, sizeof(data)); + data.input.conn_type = ECORE_LL2_TYPE_IWARP; + data.input.mtu = ECORE_IWARP_MAX_SYN_PKT_SIZE; + data.input.rx_num_desc = ECORE_IWARP_LL2_SYN_RX_SIZE; + data.input.tx_num_desc = ECORE_IWARP_LL2_SYN_TX_SIZE; + data.input.tx_max_bds_per_packet = 1; /* will never be fragmented */ + data.input.tx_tc = PKT_LB_TC; + data.input.tx_dest = ECORE_LL2_TX_DEST_LB; + data.p_connection_handle = &iwarp_info->ll2_syn_handle; + data.cbs = &cbs; + + rc = ecore_ll2_acquire_connection(p_hwfn, &data); + if (rc) { + DP_NOTICE(p_hwfn, false, "Failed to acquire LL2 connection\n"); + ecore_llh_remove_mac_filter(p_hwfn->p_dev, 0, params->mac_addr); + return rc; + } + + rc = ecore_ll2_establish_connection(p_hwfn, iwarp_info->ll2_syn_handle); + if (rc) { + DP_NOTICE(p_hwfn, false, + "Failed to establish LL2 connection\n"); + goto err; + } + + rc = ecore_iwarp_ll2_alloc_buffers(p_hwfn, + ECORE_IWARP_LL2_SYN_RX_SIZE, + ECORE_IWARP_MAX_SYN_PKT_SIZE, + iwarp_info->ll2_syn_handle); + if (rc) + goto err; + + /* Start OOO connection */ + data.input.conn_type = ECORE_LL2_TYPE_OOO; + data.input.mtu = params->max_mtu; + + n_ooo_bufs = params->iwarp.ooo_num_rx_bufs; + + if (n_ooo_bufs > ECORE_IWARP_LL2_OOO_MAX_RX_SIZE) + n_ooo_bufs = ECORE_IWARP_LL2_OOO_MAX_RX_SIZE; + + data.input.rx_num_desc = n_ooo_bufs; + data.input.rx_num_ooo_buffers = n_ooo_bufs; + + p_hwfn->p_rdma_info->iwarp.num_ooo_rx_bufs = data.input.rx_num_desc; + data.input.tx_max_bds_per_packet = 1; /* will never be fragmented */ + data.input.tx_num_desc = ECORE_IWARP_LL2_OOO_DEF_TX_SIZE; + data.p_connection_handle = &iwarp_info->ll2_ooo_handle; + data.input.secondary_queue = true; + + rc = ecore_ll2_acquire_connection(p_hwfn, &data); + if (rc) + goto err; + + rc = ecore_ll2_establish_connection(p_hwfn, iwarp_info->ll2_ooo_handle); + if (rc) + goto err; + + /* Start MPA connection */ + cbs.rx_comp_cb = ecore_iwarp_ll2_comp_mpa_pkt; + cbs.slowpath_cb = ecore_iwarp_ll2_slowpath; + + OSAL_MEMSET(&data, 0, sizeof(data)); + data.input.conn_type = ECORE_LL2_TYPE_IWARP; + data.input.mtu = params->max_mtu; + data.input.rx_num_desc = n_ooo_bufs * 2; + /* we allocate the same amount for TX to reduce the chance we + * run out of tx descriptors + */ + data.input.tx_num_desc = data.input.rx_num_desc; + data.input.tx_max_bds_per_packet = ECORE_IWARP_MAX_BDS_PER_FPDU; + data.p_connection_handle = &iwarp_info->ll2_mpa_handle; + data.input.secondary_queue = true; + data.cbs = &cbs; + + rc = ecore_ll2_acquire_connection(p_hwfn, &data); + if (rc) + goto err; + + rc = ecore_ll2_establish_connection(p_hwfn, iwarp_info->ll2_mpa_handle); + if (rc) + goto err; + + mpa_buff_size = ECORE_IWARP_MAX_BUF_SIZE(params->max_mtu); + rc = ecore_iwarp_ll2_alloc_buffers(p_hwfn, + data.input.rx_num_desc, + mpa_buff_size, + iwarp_info->ll2_mpa_handle); + if (rc) + goto err; + + iwarp_info->partial_fpdus = + OSAL_ZALLOC(p_hwfn->p_dev, GFP_KERNEL, + sizeof(*iwarp_info->partial_fpdus) * + (u16)p_hwfn->p_rdma_info->num_qps); + + if (!iwarp_info->partial_fpdus) { + DP_NOTICE(p_hwfn, false, + "Failed to allocate ecore_iwarp_info(partial_fpdus)\n"); + goto err; + } + + iwarp_info->max_num_partial_fpdus = (u16)p_hwfn->p_rdma_info->num_qps; + + /* The mpa_bufs array serves for pending RX packets received on the + * mpa ll2 that don't have place on the tx ring and require later + * processing. We can't fail on allocation of such a struct therefore + * we allocate enough to take care of all rx packets + */ + iwarp_info->mpa_bufs = + OSAL_ZALLOC(p_hwfn->p_dev, GFP_KERNEL, + sizeof(*iwarp_info->mpa_bufs) * + data.input.rx_num_desc); + + if (!iwarp_info->mpa_bufs) { + DP_NOTICE(p_hwfn, false, + "Failed to allocate mpa_bufs array mem_size=%d\n", + (u32)(sizeof(*iwarp_info->mpa_bufs) * + data.input.rx_num_desc)); + goto err; + } + + iwarp_info->mpa_intermediate_buf = + OSAL_ZALLOC(p_hwfn->p_dev, GFP_KERNEL, mpa_buff_size); + if (!iwarp_info->mpa_intermediate_buf) { + DP_NOTICE(p_hwfn, false, + "Failed to allocate mpa_intermediate_buf mem_size=%d\n", + mpa_buff_size); + goto err; + } + + OSAL_LIST_INIT(&iwarp_info->mpa_buf_pending_list); + OSAL_LIST_INIT(&iwarp_info->mpa_buf_list); + for (i = 0; i < data.input.rx_num_desc; i++) { + OSAL_LIST_PUSH_TAIL(&iwarp_info->mpa_bufs[i].list_entry, + &iwarp_info->mpa_buf_list); + } + + return rc; + +err: + ecore_iwarp_ll2_stop(p_hwfn); + + return rc; +} + +static void +ecore_iwarp_set_defaults(struct ecore_hwfn *p_hwfn, + struct ecore_rdma_start_in_params *params) +{ + u32 rcv_wnd_size; + u32 n_ooo_bufs; + + /* rcv_wnd_size = 0: use defaults */ + rcv_wnd_size = params->iwarp.rcv_wnd_size; + if (!rcv_wnd_size) { + if (ecore_device_num_ports(p_hwfn->p_dev) == 4) { + rcv_wnd_size = ECORE_IS_AH(p_hwfn->p_dev) ? + ECORE_IWARP_RCV_WND_SIZE_AH_DEF_4_PORTS : + ECORE_IWARP_RCV_WND_SIZE_BB_DEF_4_PORTS; + } else { + rcv_wnd_size = ECORE_IS_AH(p_hwfn->p_dev) ? + ECORE_IWARP_RCV_WND_SIZE_AH_DEF_2_PORTS : + ECORE_IWARP_RCV_WND_SIZE_BB_DEF_2_PORTS; + } + params->iwarp.rcv_wnd_size = rcv_wnd_size; + } + + n_ooo_bufs = params->iwarp.ooo_num_rx_bufs; + if (!n_ooo_bufs) { + n_ooo_bufs = (u32)(((u64)ECORE_MAX_OOO * + params->iwarp.rcv_wnd_size) / + params->max_mtu); + n_ooo_bufs = OSAL_MIN_T(u32, n_ooo_bufs, USHRT_MAX); + params->iwarp.ooo_num_rx_bufs = (u16)n_ooo_bufs; + } +} + +enum _ecore_status_t +ecore_iwarp_setup(struct ecore_hwfn *p_hwfn, + struct ecore_rdma_start_in_params *params) +{ + enum _ecore_status_t rc = ECORE_SUCCESS; + struct ecore_iwarp_info *iwarp_info; + u32 rcv_wnd_size; + + iwarp_info = &(p_hwfn->p_rdma_info->iwarp); + + if (!params->iwarp.rcv_wnd_size || !params->iwarp.ooo_num_rx_bufs) + ecore_iwarp_set_defaults(p_hwfn, params); + + /* Scale 0 will set window of 0xFFFC (64K -4). + * Scale x will set window of 0xFFFC << (x) + * Therefore we subtract log2(64K) so that result is 0 + */ + rcv_wnd_size = params->iwarp.rcv_wnd_size; + if (rcv_wnd_size < ECORE_IWARP_RCV_WND_SIZE_MIN) + rcv_wnd_size = ECORE_IWARP_RCV_WND_SIZE_MIN; + + iwarp_info->rcv_wnd_scale = OSAL_MIN_T(u32, OSAL_LOG2(rcv_wnd_size) - + OSAL_LOG2(ECORE_IWARP_RCV_WND_SIZE_MIN), ECORE_IWARP_MAX_WND_SCALE); + iwarp_info->rcv_wnd_size = rcv_wnd_size >> iwarp_info->rcv_wnd_scale; + + iwarp_info->tcp_flags = params->iwarp.flags; + iwarp_info->crc_needed = params->iwarp.crc_needed; + switch (params->iwarp.mpa_rev) { + case ECORE_MPA_REV1: + iwarp_info->mpa_rev = MPA_NEGOTIATION_TYPE_BASIC; + break; + case ECORE_MPA_REV2: + iwarp_info->mpa_rev = MPA_NEGOTIATION_TYPE_ENHANCED; + break; + } + + iwarp_info->peer2peer = params->iwarp.mpa_peer2peer; + iwarp_info->rtr_type = MPA_RTR_TYPE_NONE; + + if (params->iwarp.mpa_rtr & ECORE_MPA_RTR_TYPE_ZERO_SEND) + iwarp_info->rtr_type |= MPA_RTR_TYPE_ZERO_SEND; + + if (params->iwarp.mpa_rtr & ECORE_MPA_RTR_TYPE_ZERO_WRITE) + iwarp_info->rtr_type |= MPA_RTR_TYPE_ZERO_WRITE; + + if (params->iwarp.mpa_rtr & ECORE_MPA_RTR_TYPE_ZERO_READ) + iwarp_info->rtr_type |= MPA_RTR_TYPE_ZERO_READ; + + //DAVIDS OSAL_SPIN_LOCK_INIT(&p_hwfn->p_rdma_info->iwarp.qp_lock); + OSAL_LIST_INIT(&p_hwfn->p_rdma_info->iwarp.ep_list); + OSAL_LIST_INIT(&p_hwfn->p_rdma_info->iwarp.listen_list); + + ecore_spq_register_async_cb(p_hwfn, PROTOCOLID_IWARP, + ecore_iwarp_async_event); + ecore_ooo_setup(p_hwfn); + + rc = ecore_iwarp_ll2_start(p_hwfn, params); + + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, + "MPA_REV = %d. peer2peer=%d rtr=%x\n", + iwarp_info->mpa_rev, + iwarp_info->peer2peer, + iwarp_info->rtr_type); + + return rc; +} + +enum _ecore_status_t +ecore_iwarp_stop(struct ecore_hwfn *p_hwfn) +{ + enum _ecore_status_t rc; + + ecore_iwarp_free_prealloc_ep(p_hwfn); + rc = ecore_iwarp_wait_for_all_cids(p_hwfn); + if (rc != ECORE_SUCCESS) + return rc; + + ecore_spq_unregister_async_cb(p_hwfn, PROTOCOLID_IWARP); + + return ecore_iwarp_ll2_stop(p_hwfn); +} + +static void +ecore_iwarp_qp_in_error(struct ecore_hwfn *p_hwfn, + struct ecore_iwarp_ep *ep, + u8 fw_return_code) +{ + struct ecore_iwarp_cm_event_params params; + + ecore_iwarp_modify_qp(p_hwfn, ep->qp, ECORE_IWARP_QP_STATE_ERROR, true); + + params.event = ECORE_IWARP_EVENT_CLOSE; + params.ep_context = ep; + params.cm_info = &ep->cm_info; + params.status = (fw_return_code == IWARP_QP_IN_ERROR_GOOD_CLOSE) ? + ECORE_SUCCESS : ECORE_CONN_RESET; + + ep->state = ECORE_IWARP_EP_CLOSED; + OSAL_SPIN_LOCK(&p_hwfn->p_rdma_info->iwarp.iw_lock); + OSAL_LIST_REMOVE_ENTRY(&ep->list_entry, + &p_hwfn->p_rdma_info->iwarp.ep_list); + OSAL_SPIN_UNLOCK(&p_hwfn->p_rdma_info->iwarp.iw_lock); + + ep->event_cb(ep->cb_context, ¶ms); +} + +static void +ecore_iwarp_exception_received(struct ecore_hwfn *p_hwfn, + struct ecore_iwarp_ep *ep, + int fw_ret_code) +{ + struct ecore_iwarp_cm_event_params params; + bool event_cb = false; + + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "EP(0x%x) fw_ret_code=%d\n", + ep->cid, fw_ret_code); + + switch (fw_ret_code) { + case IWARP_EXCEPTION_DETECTED_LLP_CLOSED: + params.status = ECORE_SUCCESS; + params.event = ECORE_IWARP_EVENT_DISCONNECT; + event_cb = true; + break; + case IWARP_EXCEPTION_DETECTED_LLP_RESET: + params.status = ECORE_CONN_RESET; + params.event = ECORE_IWARP_EVENT_DISCONNECT; + event_cb = true; + break; + case IWARP_EXCEPTION_DETECTED_RQ_EMPTY: + params.event = ECORE_IWARP_EVENT_RQ_EMPTY; + event_cb = true; + break; + case IWARP_EXCEPTION_DETECTED_IRQ_FULL: + params.event = ECORE_IWARP_EVENT_IRQ_FULL; + event_cb = true; + break; + case IWARP_EXCEPTION_DETECTED_LLP_TIMEOUT: + params.event = ECORE_IWARP_EVENT_LLP_TIMEOUT; + event_cb = true; + break; + case IWARP_EXCEPTION_DETECTED_REMOTE_PROTECTION_ERROR: + params.event = ECORE_IWARP_EVENT_REMOTE_PROTECTION_ERROR; + event_cb = true; + break; + case IWARP_EXCEPTION_DETECTED_CQ_OVERFLOW: + params.event = ECORE_IWARP_EVENT_CQ_OVERFLOW; + event_cb = true; + break; + case IWARP_EXCEPTION_DETECTED_LOCAL_CATASTROPHIC: + params.event = ECORE_IWARP_EVENT_QP_CATASTROPHIC; + event_cb = true; + break; + case IWARP_EXCEPTION_DETECTED_LOCAL_ACCESS_ERROR: + params.event = ECORE_IWARP_EVENT_LOCAL_ACCESS_ERROR; + event_cb = true; + break; + case IWARP_EXCEPTION_DETECTED_REMOTE_OPERATION_ERROR: + params.event = ECORE_IWARP_EVENT_REMOTE_OPERATION_ERROR; + event_cb = true; + break; + case IWARP_EXCEPTION_DETECTED_TERMINATE_RECEIVED: + params.event = ECORE_IWARP_EVENT_TERMINATE_RECEIVED; + event_cb = true; + break; + default: + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, + "Unhandled exception received...\n"); + break; + } + + if (event_cb) { + params.ep_context = ep; + params.cm_info = &ep->cm_info; + ep->event_cb(ep->cb_context, ¶ms); + } +} + +static void +ecore_iwarp_tcp_connect_unsuccessful(struct ecore_hwfn *p_hwfn, + struct ecore_iwarp_ep *ep, + u8 fw_return_code) +{ + struct ecore_iwarp_cm_event_params params; + + OSAL_MEM_ZERO(¶ms, sizeof(params)); + params.event = ECORE_IWARP_EVENT_ACTIVE_COMPLETE; + params.ep_context = ep; + params.cm_info = &ep->cm_info; + ep->state = ECORE_IWARP_EP_CLOSED; + + switch (fw_return_code) { + case IWARP_CONN_ERROR_TCP_CONNECT_INVALID_PACKET: + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, + "%s(0x%x) TCP connect got invalid packet\n", + ECORE_IWARP_CONNECT_MODE_STRING(ep), + ep->tcp_cid); + params.status = ECORE_CONN_RESET; + break; + case IWARP_CONN_ERROR_TCP_CONNECTION_RST: + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, + "%s(0x%x) TCP Connection Reset\n", + ECORE_IWARP_CONNECT_MODE_STRING(ep), + ep->tcp_cid); + params.status = ECORE_CONN_RESET; + break; + case IWARP_CONN_ERROR_TCP_CONNECT_TIMEOUT: + DP_NOTICE(p_hwfn, false, "%s(0x%x) TCP timeout\n", + ECORE_IWARP_CONNECT_MODE_STRING(ep), + ep->tcp_cid); + params.status = ECORE_TIMEOUT; + break; + case IWARP_CONN_ERROR_MPA_NOT_SUPPORTED_VER: + DP_NOTICE(p_hwfn, false, "%s(0x%x) MPA not supported VER\n", + ECORE_IWARP_CONNECT_MODE_STRING(ep), + ep->tcp_cid); + params.status = ECORE_CONN_REFUSED; + break; + case IWARP_CONN_ERROR_MPA_INVALID_PACKET: + DP_NOTICE(p_hwfn, false, "%s(0x%x) MPA Invalid Packet\n", + ECORE_IWARP_CONNECT_MODE_STRING(ep), ep->tcp_cid); + params.status = ECORE_CONN_RESET; + break; + default: + DP_ERR(p_hwfn, "%s(0x%x) Unexpected return code tcp connect: %d\n", + ECORE_IWARP_CONNECT_MODE_STRING(ep), ep->tcp_cid, + fw_return_code); + params.status = ECORE_CONN_RESET; + break; + } + + if (ep->connect_mode == TCP_CONNECT_PASSIVE) { + ep->tcp_cid = ECORE_IWARP_INVALID_TCP_CID; + ecore_iwarp_return_ep(p_hwfn, ep); + } else { + ep->event_cb(ep->cb_context, ¶ms); + OSAL_SPIN_LOCK(&p_hwfn->p_rdma_info->iwarp.iw_lock); + OSAL_LIST_REMOVE_ENTRY(&ep->list_entry, + &p_hwfn->p_rdma_info->iwarp.ep_list); + OSAL_SPIN_UNLOCK(&p_hwfn->p_rdma_info->iwarp.iw_lock); + } +} + +static void +ecore_iwarp_connect_complete(struct ecore_hwfn *p_hwfn, + struct ecore_iwarp_ep *ep, + u8 fw_return_code) +{ + if (ep->connect_mode == TCP_CONNECT_PASSIVE) { + /* Done with the SYN packet, post back to ll2 rx */ + ecore_iwarp_ll2_post_rx( + p_hwfn, ep->syn, + p_hwfn->p_rdma_info->iwarp.ll2_syn_handle); + + ep->syn = OSAL_NULL; + + if (ep->state == ECORE_IWARP_EP_ABORTING) + return; + + /* If connect failed - upper layer doesn't know about it */ + if (fw_return_code == RDMA_RETURN_OK) + ecore_iwarp_mpa_received(p_hwfn, ep); + else + ecore_iwarp_tcp_connect_unsuccessful(p_hwfn, ep, + fw_return_code); + + } else { + if (fw_return_code == RDMA_RETURN_OK) + ecore_iwarp_mpa_offload(p_hwfn, ep); + else + ecore_iwarp_tcp_connect_unsuccessful(p_hwfn, ep, + fw_return_code); + } +} + +static OSAL_INLINE bool +ecore_iwarp_check_ep_ok(struct ecore_hwfn *p_hwfn, + struct ecore_iwarp_ep *ep) +{ + if (ep == OSAL_NULL) { + DP_ERR(p_hwfn, "ERROR ON ASYNC ep=%p\n", ep); + return false; + } + + if (ep->sig != 0xdeadbeef) { + DP_ERR(p_hwfn, "ERROR ON ASYNC ep=%p\n", ep); + return false; + } + + return true; +} + +static enum _ecore_status_t +ecore_iwarp_async_event(struct ecore_hwfn *p_hwfn, + u8 fw_event_code, + u16 OSAL_UNUSED echo, + union event_ring_data *data, + u8 fw_return_code) +{ + struct regpair *fw_handle = &data->rdma_data.async_handle; + struct ecore_iwarp_ep *ep = OSAL_NULL; + u16 cid; + + ep = (struct ecore_iwarp_ep *)(osal_uintptr_t)HILO_64(fw_handle->hi, + fw_handle->lo); + + switch (fw_event_code) { + /* Async completion after TCP 3-way handshake */ + case IWARP_EVENT_TYPE_ASYNC_CONNECT_COMPLETE: + if (!ecore_iwarp_check_ep_ok(p_hwfn, ep)) + return ECORE_INVAL; + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, + "EP(0x%x) IWARP_EVENT_TYPE_ASYNC_CONNECT_COMPLETE fw_ret_code=%d\n", + ep->tcp_cid, fw_return_code); + ecore_iwarp_connect_complete(p_hwfn, ep, fw_return_code); + break; + case IWARP_EVENT_TYPE_ASYNC_EXCEPTION_DETECTED: + if (!ecore_iwarp_check_ep_ok(p_hwfn, ep)) + return ECORE_INVAL; + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, + "QP(0x%x) IWARP_EVENT_TYPE_ASYNC_EXCEPTION_DETECTED fw_ret_code=%d\n", + ep->cid, fw_return_code); + ecore_iwarp_exception_received(p_hwfn, ep, fw_return_code); + break; + /* Async completion for Close Connection ramrod */ + case IWARP_EVENT_TYPE_ASYNC_QP_IN_ERROR_STATE: + if (!ecore_iwarp_check_ep_ok(p_hwfn, ep)) + return ECORE_INVAL; + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, + "QP(0x%x) IWARP_EVENT_TYPE_ASYNC_QP_IN_ERROR_STATE fw_ret_code=%d\n", + ep->cid, fw_return_code); + ecore_iwarp_qp_in_error(p_hwfn, ep, fw_return_code); + break; + /* Async event for active side only */ + case IWARP_EVENT_TYPE_ASYNC_ENHANCED_MPA_REPLY_ARRIVED: + if (!ecore_iwarp_check_ep_ok(p_hwfn, ep)) + return ECORE_INVAL; + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, + "QP(0x%x) IWARP_EVENT_TYPE_ASYNC_MPA_HANDSHAKE_MPA_REPLY_ARRIVED fw_ret_code=%d\n", + ep->cid, fw_return_code); + ecore_iwarp_mpa_reply_arrived(p_hwfn, ep); + break; + /* MPA Negotiations completed */ + case IWARP_EVENT_TYPE_ASYNC_MPA_HANDSHAKE_COMPLETE: + if (!ecore_iwarp_check_ep_ok(p_hwfn, ep)) + return ECORE_INVAL; + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, + "QP(0x%x) IWARP_EVENT_TYPE_ASYNC_MPA_HANDSHAKE_COMPLETE fw_ret_code=%d\n", + ep->cid, fw_return_code); + ecore_iwarp_mpa_complete(p_hwfn, ep, fw_return_code); + break; + case IWARP_EVENT_TYPE_ASYNC_CID_CLEANED: + cid = (u16)OSAL_LE32_TO_CPU(fw_handle->lo); + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, + "(0x%x)IWARP_EVENT_TYPE_ASYNC_CID_CLEANED\n", + cid); + ecore_iwarp_cid_cleaned(p_hwfn, cid); + + break; + case IWARP_EVENT_TYPE_ASYNC_CQ_OVERFLOW: + DP_NOTICE(p_hwfn, false, + "IWARP_EVENT_TYPE_ASYNC_CQ_OVERFLOW\n"); + + p_hwfn->p_rdma_info->events.affiliated_event( + p_hwfn->p_rdma_info->events.context, + ECORE_IWARP_EVENT_CQ_OVERFLOW, + (void *)fw_handle); + break; + default: + DP_ERR(p_hwfn, "Received unexpected async iwarp event %d\n", + fw_event_code); + return ECORE_INVAL; + } + return ECORE_SUCCESS; +} + +enum _ecore_status_t +ecore_iwarp_create_listen(void *rdma_cxt, + struct ecore_iwarp_listen_in *iparams, + struct ecore_iwarp_listen_out *oparams) +{ + struct ecore_hwfn *p_hwfn = (struct ecore_hwfn *)rdma_cxt; + struct ecore_iwarp_listener *listener; + + listener = OSAL_ZALLOC(p_hwfn->p_dev, GFP_KERNEL, sizeof(*listener)); + + if (!listener) { + DP_NOTICE(p_hwfn, + false, + "ecore iwarp create listener failed: cannot allocate memory (listener). rc = %d\n", + ECORE_NOMEM); + return ECORE_NOMEM; + } + listener->ip_version = iparams->ip_version; + OSAL_MEMCPY(listener->ip_addr, + iparams->ip_addr, + sizeof(listener->ip_addr)); + listener->port = iparams->port; + listener->vlan = iparams->vlan; + + listener->event_cb = iparams->event_cb; + listener->cb_context = iparams->cb_context; + listener->max_backlog = iparams->max_backlog; + listener->state = ECORE_IWARP_LISTENER_STATE_ACTIVE; + oparams->handle = listener; + + OSAL_SPIN_LOCK_INIT(&listener->lock); + OSAL_LIST_INIT(&listener->ep_list); + OSAL_SPIN_LOCK(&p_hwfn->p_rdma_info->iwarp.iw_lock); + OSAL_LIST_PUSH_TAIL(&listener->list_entry, + &p_hwfn->p_rdma_info->iwarp.listen_list); + OSAL_SPIN_UNLOCK(&p_hwfn->p_rdma_info->iwarp.iw_lock); + + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "callback=%p handle=%p ip=%x:%x:%x:%x port=0x%x vlan=0x%x\n", + listener->event_cb, + listener, + listener->ip_addr[0], + listener->ip_addr[1], + listener->ip_addr[2], + listener->ip_addr[3], + listener->port, + listener->vlan); + + return ECORE_SUCCESS; +} + +static void +ecore_iwarp_pause_complete(struct ecore_iwarp_listener *listener) +{ + struct ecore_iwarp_cm_event_params params; + + if (listener->state == ECORE_IWARP_LISTENER_STATE_UNPAUSE) + listener->state = ECORE_IWARP_LISTENER_STATE_ACTIVE; + + params.event = ECORE_IWARP_EVENT_LISTEN_PAUSE_COMP; + listener->event_cb(listener->cb_context, ¶ms); +} + +static void +ecore_iwarp_tcp_abort_comp(struct ecore_hwfn *p_hwfn, void *cookie, + union event_ring_data OSAL_UNUSED *data, + u8 OSAL_UNUSED fw_return_code) +{ + struct ecore_iwarp_ep *ep = (struct ecore_iwarp_ep *)cookie; + struct ecore_iwarp_listener *listener = ep->listener; + + ecore_iwarp_return_ep(p_hwfn, ep); + + if (OSAL_LIST_IS_EMPTY(&listener->ep_list)) + listener->done = true; +} + +static void +ecore_iwarp_abort_inflight_connections(struct ecore_hwfn *p_hwfn, + struct ecore_iwarp_listener *listener) +{ + struct ecore_spq_entry *p_ent = OSAL_NULL; + struct ecore_iwarp_ep *ep = OSAL_NULL; + struct ecore_sp_init_data init_data; + struct ecore_spq_comp_cb comp_data; + enum _ecore_status_t rc; + + /* remove listener from list before destroying listener */ + OSAL_LIST_REMOVE_ENTRY(&listener->list_entry, + &p_hwfn->p_rdma_info->iwarp.listen_list); + if (OSAL_LIST_IS_EMPTY(&listener->ep_list)) { + listener->done = true; + return; + } + OSAL_MEMSET(&init_data, 0, sizeof(init_data)); + init_data.p_comp_data = &comp_data; + init_data.opaque_fid = p_hwfn->hw_info.opaque_fid; + init_data.comp_mode = ECORE_SPQ_MODE_CB; + init_data.p_comp_data->function = ecore_iwarp_tcp_abort_comp; + + OSAL_LIST_FOR_EACH_ENTRY(ep, &listener->ep_list, + list_entry, struct ecore_iwarp_ep) { + ep->state = ECORE_IWARP_EP_ABORTING; + init_data.p_comp_data->cookie = ep; + init_data.cid = ep->tcp_cid; + rc = ecore_sp_init_request(p_hwfn, &p_ent, + IWARP_RAMROD_CMD_ID_ABORT_TCP_OFFLOAD, + PROTOCOLID_IWARP, + &init_data); + if (rc == ECORE_SUCCESS) + ecore_spq_post(p_hwfn, p_ent, OSAL_NULL); + } +} + +static void +ecore_iwarp_listener_state_transition(struct ecore_hwfn *p_hwfn, void *cookie, + union event_ring_data OSAL_UNUSED *data, + u8 OSAL_UNUSED fw_return_code) +{ + struct ecore_iwarp_listener *listener = (struct ecore_iwarp_listener *)cookie; + + switch (listener->state) { + case ECORE_IWARP_LISTENER_STATE_PAUSE: + case ECORE_IWARP_LISTENER_STATE_UNPAUSE: + ecore_iwarp_pause_complete(listener); + break; + case ECORE_IWARP_LISTENER_STATE_DESTROYING: + ecore_iwarp_abort_inflight_connections(p_hwfn, listener); + break; + default: + break; + } +} + +static enum _ecore_status_t +ecore_iwarp_empty_ramrod(struct ecore_hwfn *p_hwfn, + struct ecore_iwarp_listener *listener) +{ + struct ecore_spq_entry *p_ent = OSAL_NULL; + struct ecore_spq_comp_cb comp_data; + struct ecore_sp_init_data init_data; + enum _ecore_status_t rc; + + OSAL_MEMSET(&init_data, 0, sizeof(init_data)); + init_data.p_comp_data = &comp_data; + init_data.cid = ecore_spq_get_cid(p_hwfn); + init_data.opaque_fid = p_hwfn->hw_info.opaque_fid; + init_data.comp_mode = ECORE_SPQ_MODE_CB; + init_data.p_comp_data->function = ecore_iwarp_listener_state_transition; + init_data.p_comp_data->cookie = listener; + rc = ecore_sp_init_request(p_hwfn, &p_ent, + COMMON_RAMROD_EMPTY, + PROTOCOLID_COMMON, + &init_data); + if (rc != ECORE_SUCCESS) + return rc; + + rc = ecore_spq_post(p_hwfn, p_ent, OSAL_NULL); + if (rc != ECORE_SUCCESS) + return rc; + + return rc; +} + +enum _ecore_status_t +ecore_iwarp_pause_listen(void *rdma_cxt, void *handle, + bool pause, bool comp) +{ + struct ecore_hwfn *p_hwfn = (struct ecore_hwfn *)rdma_cxt; + struct ecore_iwarp_listener *listener = + (struct ecore_iwarp_listener *)handle; + enum _ecore_status_t rc; + + listener->state = pause ? + ECORE_IWARP_LISTENER_STATE_PAUSE : + ECORE_IWARP_LISTENER_STATE_UNPAUSE; + if (!comp) + return ECORE_SUCCESS; + + rc = ecore_iwarp_empty_ramrod(p_hwfn, listener); + if (rc != ECORE_SUCCESS) + return rc; + + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "listener=%p, state=%d\n", + listener, listener->state); + + return ECORE_PENDING; +} + +enum _ecore_status_t +ecore_iwarp_destroy_listen(void *rdma_cxt, void *handle) +{ + struct ecore_hwfn *p_hwfn = (struct ecore_hwfn *)rdma_cxt; + struct ecore_iwarp_listener *listener = + (struct ecore_iwarp_listener *)handle; + enum _ecore_status_t rc; + int wait_count = 0; + + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "handle=%p\n", handle); + + listener->state = ECORE_IWARP_LISTENER_STATE_DESTROYING; + rc = ecore_iwarp_empty_ramrod(p_hwfn, listener); + if (rc != ECORE_SUCCESS) + return rc; + + while (!listener->done) { + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, + "Waiting for ep list to be empty...\n"); + OSAL_MSLEEP(100); + if (wait_count++ > 200) { + DP_NOTICE(p_hwfn, false, "ep list close timeout\n"); + break; + } + } + + OSAL_FREE(p_hwfn->p_dev, listener); + + return ECORE_SUCCESS; +} + +enum _ecore_status_t +ecore_iwarp_send_rtr(void *rdma_cxt, struct ecore_iwarp_send_rtr_in *iparams) +{ + struct ecore_hwfn *p_hwfn = (struct ecore_hwfn *)rdma_cxt; + struct ecore_sp_init_data init_data; + struct ecore_spq_entry *p_ent; + struct ecore_rdma_qp *qp; + struct ecore_iwarp_ep *ep; + enum _ecore_status_t rc; + + ep = (struct ecore_iwarp_ep *)iparams->ep_context; + if (!ep) { + DP_ERR(p_hwfn, "Ep Context receive in send_rtr is NULL\n"); + return ECORE_INVAL; + } + + qp = ep->qp; + + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "QP(0x%x) EP(0x%x)\n", + qp->icid, ep->tcp_cid); + + OSAL_MEMSET(&init_data, 0, sizeof(init_data)); + init_data.cid = qp->icid; + init_data.opaque_fid = p_hwfn->hw_info.opaque_fid; + init_data.comp_mode = ECORE_SPQ_MODE_CB; + + rc = ecore_sp_init_request(p_hwfn, &p_ent, + IWARP_RAMROD_CMD_ID_MPA_OFFLOAD_SEND_RTR, + PROTOCOLID_IWARP, &init_data); + + if (rc != ECORE_SUCCESS) + return rc; + + rc = ecore_spq_post(p_hwfn, p_ent, OSAL_NULL); + + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "ecore_iwarp_send_rtr, rc = 0x%x\n", + rc); + + return rc; +} + +enum _ecore_status_t +ecore_iwarp_query_qp(struct ecore_rdma_qp *qp, + struct ecore_rdma_query_qp_out_params *out_params) +{ + out_params->state = ecore_iwarp2roce_state(qp->iwarp_state); + return ECORE_SUCCESS; +} + +#ifdef _NTDDK_ +#pragma warning(pop) +#endif Property changes on: head/sys/dev/qlnx/qlnxe/ecore_iwarp.c ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Added: svn:mime-type ## -0,0 +1 ## +text/plain \ No newline at end of property Index: head/sys/dev/qlnx/qlnxe/ecore_ll2.c =================================================================== --- head/sys/dev/qlnx/qlnxe/ecore_ll2.c (nonexistent) +++ head/sys/dev/qlnx/qlnxe/ecore_ll2.c (revision 343598) @@ -0,0 +1,2211 @@ +/* + * Copyright (c) 2018-2019 Cavium, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +/* + * File : ecore_ll2.c + */ +#include +__FBSDID("$FreeBSD$"); + +#include "bcm_osal.h" + +#include "ecore.h" +#include "ecore_status.h" +#include "ecore_ll2.h" +#include "reg_addr.h" +#include "ecore_int.h" +#include "ecore_cxt.h" +#include "ecore_sp_commands.h" +#include "ecore_hw.h" +#include "reg_addr.h" +#include "ecore_dev_api.h" +#include "ecore_iro.h" +#include "ecore_gtt_reg_addr.h" +#include "ecore_ooo.h" +#include "ecore_hw.h" +#include "ecore_mcp.h" + +#define ECORE_LL2_RX_REGISTERED(ll2) ((ll2)->rx_queue.b_cb_registred) +#define ECORE_LL2_TX_REGISTERED(ll2) ((ll2)->tx_queue.b_cb_registred) + +#ifdef _NTDDK_ +#pragma warning(push) +#pragma warning(disable : 28167) +#pragma warning(disable : 28123) +#pragma warning(disable : 28121) +#endif + +static struct ecore_ll2_info * +__ecore_ll2_handle_sanity(struct ecore_hwfn *p_hwfn, + u8 connection_handle, + bool b_lock, bool b_only_active) +{ + struct ecore_ll2_info *p_ll2_conn, *p_ret = OSAL_NULL; + + if (connection_handle >= ECORE_MAX_NUM_OF_LL2_CONNECTIONS) + return OSAL_NULL; + + if (!p_hwfn->p_ll2_info) + return OSAL_NULL; + + /* TODO - is there really need for the locked vs. unlocked + * variant? I simply used what was already there. + */ + p_ll2_conn = &p_hwfn->p_ll2_info[connection_handle]; + + if (b_only_active) { + if (b_lock) + OSAL_MUTEX_ACQUIRE(&p_ll2_conn->mutex); + if (p_ll2_conn->b_active) + p_ret = p_ll2_conn; + if (b_lock) + OSAL_MUTEX_RELEASE(&p_ll2_conn->mutex); + } else { + p_ret = p_ll2_conn; + } + + return p_ret; +} + +static struct ecore_ll2_info * +ecore_ll2_handle_sanity(struct ecore_hwfn *p_hwfn, + u8 connection_handle) +{ + return __ecore_ll2_handle_sanity(p_hwfn, connection_handle, + false, true); +} + +static struct ecore_ll2_info * +ecore_ll2_handle_sanity_lock(struct ecore_hwfn *p_hwfn, + u8 connection_handle) +{ + return __ecore_ll2_handle_sanity(p_hwfn, connection_handle, + true, true); +} + +static struct ecore_ll2_info * +ecore_ll2_handle_sanity_inactive(struct ecore_hwfn *p_hwfn, + u8 connection_handle) +{ + return __ecore_ll2_handle_sanity(p_hwfn, connection_handle, + false, false); +} + +#ifndef LINUX_REMOVE +/* TODO - is this really been used by anyone? Is it a on future todo list? */ +enum _ecore_status_t +ecore_ll2_get_fragment_of_tx_packet(struct ecore_hwfn *p_hwfn, + u8 connection_handle, + dma_addr_t *p_addr, + bool *b_last_fragment) +{ + struct ecore_ll2_tx_packet *p_pkt; + struct ecore_ll2_info *p_ll2_conn; + u16 cur_frag_idx = 0; + + p_ll2_conn = ecore_ll2_handle_sanity(p_hwfn, connection_handle); + if (p_ll2_conn == OSAL_NULL) + return ECORE_INVAL; + p_pkt = &p_ll2_conn->tx_queue.cur_completing_packet; + + if (!p_ll2_conn->tx_queue.b_completing_packet || !p_addr) + return ECORE_INVAL; + + if (p_ll2_conn->tx_queue.cur_completing_bd_idx == p_pkt->bd_used) + return ECORE_INVAL; + + /* Packet is available and has at least one more frag - provide it */ + cur_frag_idx = p_ll2_conn->tx_queue.cur_completing_bd_idx++; + *p_addr = p_pkt->bds_set[cur_frag_idx].tx_frag; + if (b_last_fragment) + *b_last_fragment = p_pkt->bd_used == + p_ll2_conn->tx_queue.cur_completing_bd_idx; + + return ECORE_SUCCESS; +} +#endif + +static void ecore_ll2_txq_flush(struct ecore_hwfn *p_hwfn, + u8 connection_handle) +{ + bool b_last_packet = false, b_last_frag = false; + struct ecore_ll2_tx_packet *p_pkt = OSAL_NULL; + struct ecore_ll2_info *p_ll2_conn; + struct ecore_ll2_tx_queue *p_tx; + unsigned long flags = 0; + dma_addr_t tx_frag; + + p_ll2_conn = ecore_ll2_handle_sanity_inactive(p_hwfn, + connection_handle); + if (p_ll2_conn == OSAL_NULL) + return; + p_tx = &p_ll2_conn->tx_queue; + + OSAL_SPIN_LOCK_IRQSAVE(&p_tx->lock, flags); + while (!OSAL_LIST_IS_EMPTY(&p_tx->active_descq)) { + p_pkt = OSAL_LIST_FIRST_ENTRY(&p_tx->active_descq, + struct ecore_ll2_tx_packet, + list_entry); + + if (p_pkt == OSAL_NULL) + break; + +#if defined(_NTDDK_) +#pragma warning(suppress : 6011 28182) +#endif + OSAL_LIST_REMOVE_ENTRY(&p_pkt->list_entry, + &p_tx->active_descq); + b_last_packet = OSAL_LIST_IS_EMPTY(&p_tx->active_descq); + OSAL_LIST_PUSH_TAIL(&p_pkt->list_entry, + &p_tx->free_descq); + OSAL_SPIN_UNLOCK_IRQSAVE(&p_tx->lock, flags); + if (p_ll2_conn->input.conn_type == ECORE_LL2_TYPE_OOO) { + struct ecore_ooo_buffer *p_buffer; + + p_buffer = (struct ecore_ooo_buffer *)p_pkt->cookie; + ecore_ooo_put_free_buffer(p_hwfn->p_ooo_info, p_buffer); + } else { + p_tx->cur_completing_packet = *p_pkt; + p_tx->cur_completing_bd_idx = 1; + b_last_frag = p_tx->cur_completing_bd_idx == + p_pkt->bd_used; + + tx_frag = p_pkt->bds_set[0].tx_frag; + p_ll2_conn->cbs.tx_release_cb(p_ll2_conn->cbs.cookie, + p_ll2_conn->my_id, + p_pkt->cookie, + tx_frag, + b_last_frag, + b_last_packet); + } + OSAL_SPIN_LOCK_IRQSAVE(&p_tx->lock, flags); + } + OSAL_SPIN_UNLOCK_IRQSAVE(&p_tx->lock, flags); +} + +static enum _ecore_status_t +ecore_ll2_txq_completion(struct ecore_hwfn *p_hwfn, + void *p_cookie) +{ + struct ecore_ll2_info *p_ll2_conn = (struct ecore_ll2_info*)p_cookie; + struct ecore_ll2_tx_queue *p_tx = &p_ll2_conn->tx_queue; + u16 new_idx = 0, num_bds = 0, num_bds_in_packet = 0; + struct ecore_ll2_tx_packet *p_pkt; + bool b_last_frag = false; + unsigned long flags; + enum _ecore_status_t rc = ECORE_INVAL; + + OSAL_SPIN_LOCK_IRQSAVE(&p_tx->lock, flags); + if (p_tx->b_completing_packet) { + /* TODO - this looks completely unnecessary to me - the only + * way we can re-enter is by the DPC calling us again, but this + * would only happen AFTER we return, and we unset this at end + * of the function. + */ + rc = ECORE_BUSY; + goto out; + } + + new_idx = OSAL_LE16_TO_CPU(*p_tx->p_fw_cons); + num_bds = ((s16)new_idx - (s16)p_tx->bds_idx); + while (num_bds) { + if (OSAL_LIST_IS_EMPTY(&p_tx->active_descq)) + goto out; + + p_pkt = OSAL_LIST_FIRST_ENTRY(&p_tx->active_descq, + struct ecore_ll2_tx_packet, + list_entry); + if (!p_pkt) + goto out; + + p_tx->b_completing_packet = true; + p_tx->cur_completing_packet = *p_pkt; + num_bds_in_packet = p_pkt->bd_used; +#if defined(_NTDDK_) +#pragma warning(suppress : 6011 28182) +#endif + OSAL_LIST_REMOVE_ENTRY(&p_pkt->list_entry, + &p_tx->active_descq); + + if (num_bds < num_bds_in_packet) { + DP_NOTICE(p_hwfn, true, + "Rest of BDs does not cover whole packet\n"); + goto out; + } + + num_bds -= num_bds_in_packet; + p_tx->bds_idx += num_bds_in_packet; + while (num_bds_in_packet--) + ecore_chain_consume(&p_tx->txq_chain); + + p_tx->cur_completing_bd_idx = 1; + b_last_frag = p_tx->cur_completing_bd_idx == + p_pkt->bd_used; + OSAL_LIST_PUSH_TAIL(&p_pkt->list_entry, + &p_tx->free_descq); + + OSAL_SPIN_UNLOCK_IRQSAVE(&p_tx->lock, flags); + + p_ll2_conn->cbs.tx_comp_cb(p_ll2_conn->cbs.cookie, + p_ll2_conn->my_id, + p_pkt->cookie, + p_pkt->bds_set[0].tx_frag, + b_last_frag, + !num_bds); + + OSAL_SPIN_LOCK_IRQSAVE(&p_tx->lock, flags); + } + + p_tx->b_completing_packet = false; + rc = ECORE_SUCCESS; +out: + OSAL_SPIN_UNLOCK_IRQSAVE(&p_tx->lock, flags); + return rc; +} + +static void ecore_ll2_rxq_parse_gsi(union core_rx_cqe_union *p_cqe, + struct ecore_ll2_comp_rx_data *data) +{ + data->parse_flags = + OSAL_LE16_TO_CPU(p_cqe->rx_cqe_gsi.parse_flags.flags); + data->length.data_length = + OSAL_LE16_TO_CPU(p_cqe->rx_cqe_gsi.data_length); + data->vlan = + OSAL_LE16_TO_CPU(p_cqe->rx_cqe_gsi.vlan); + data->opaque_data_0 = + OSAL_LE32_TO_CPU(p_cqe->rx_cqe_gsi.src_mac_addrhi); + data->opaque_data_1 = + OSAL_LE16_TO_CPU(p_cqe->rx_cqe_gsi.src_mac_addrlo); + data->u.data_length_error = + p_cqe->rx_cqe_gsi.data_length_error; + data->qp_id = OSAL_LE16_TO_CPU(p_cqe->rx_cqe_gsi.qp_id); + + data->src_qp = OSAL_LE32_TO_CPU(p_cqe->rx_cqe_gsi.src_qp); +} + +static void ecore_ll2_rxq_parse_reg(union core_rx_cqe_union *p_cqe, + struct ecore_ll2_comp_rx_data *data) +{ + data->parse_flags = + OSAL_LE16_TO_CPU(p_cqe->rx_cqe_fp.parse_flags.flags); + data->err_flags = + OSAL_LE16_TO_CPU(p_cqe->rx_cqe_fp.err_flags.flags); + data->length.packet_length = + OSAL_LE16_TO_CPU(p_cqe->rx_cqe_fp.packet_length); + data->vlan = + OSAL_LE16_TO_CPU(p_cqe->rx_cqe_fp.vlan); + data->opaque_data_0 = + OSAL_LE32_TO_CPU(p_cqe->rx_cqe_fp.opaque_data.data[0]); + data->opaque_data_1 = + OSAL_LE32_TO_CPU(p_cqe->rx_cqe_fp.opaque_data.data[1]); + data->u.placement_offset = + p_cqe->rx_cqe_fp.placement_offset; +} + +#if defined(_NTDDK_) +#pragma warning(suppress : 28167 26110) +#endif +static enum _ecore_status_t +ecore_ll2_handle_slowpath(struct ecore_hwfn *p_hwfn, + struct ecore_ll2_info *p_ll2_conn, + union core_rx_cqe_union *p_cqe, + unsigned long *p_lock_flags) +{ + struct ecore_ll2_rx_queue *p_rx = &p_ll2_conn->rx_queue; + struct core_rx_slow_path_cqe *sp_cqe; + + sp_cqe = &p_cqe->rx_cqe_sp; + if (sp_cqe->ramrod_cmd_id != CORE_RAMROD_RX_QUEUE_FLUSH) { + DP_NOTICE(p_hwfn, true, + "LL2 - unexpected Rx CQE slowpath ramrod_cmd_id:%d\n", + sp_cqe->ramrod_cmd_id); + return ECORE_INVAL; + } + + if (p_ll2_conn->cbs.slowpath_cb == OSAL_NULL) { + DP_NOTICE(p_hwfn, true, + "LL2 - received RX_QUEUE_FLUSH but no callback was provided\n"); + return ECORE_INVAL; + } + + OSAL_SPIN_UNLOCK_IRQSAVE(&p_rx->lock, *p_lock_flags); + + p_ll2_conn->cbs.slowpath_cb(p_ll2_conn->cbs.cookie, + p_ll2_conn->my_id, + OSAL_LE32_TO_CPU(sp_cqe->opaque_data.data[0]), + OSAL_LE32_TO_CPU(sp_cqe->opaque_data.data[1])); + + OSAL_SPIN_LOCK_IRQSAVE(&p_rx->lock, *p_lock_flags); + + return ECORE_SUCCESS; +} + +static enum _ecore_status_t +ecore_ll2_rxq_handle_completion(struct ecore_hwfn *p_hwfn, + struct ecore_ll2_info *p_ll2_conn, + union core_rx_cqe_union *p_cqe, + unsigned long *p_lock_flags, + bool b_last_cqe) +{ + struct ecore_ll2_rx_queue *p_rx = &p_ll2_conn->rx_queue; + struct ecore_ll2_rx_packet *p_pkt = OSAL_NULL; + struct ecore_ll2_comp_rx_data data; + + if (!OSAL_LIST_IS_EMPTY(&p_rx->active_descq)) + p_pkt = OSAL_LIST_FIRST_ENTRY(&p_rx->active_descq, + struct ecore_ll2_rx_packet, + list_entry); + if (!p_pkt) { + DP_NOTICE(p_hwfn, false, + "[%d] LL2 Rx completion but active_descq is empty\n", + p_ll2_conn->input.conn_type); + + return ECORE_IO; + } + + OSAL_LIST_REMOVE_ENTRY(&p_pkt->list_entry, &p_rx->active_descq); + + if (p_cqe->rx_cqe_sp.type == CORE_RX_CQE_TYPE_REGULAR) + ecore_ll2_rxq_parse_reg(p_cqe, &data); + else + ecore_ll2_rxq_parse_gsi(p_cqe, &data); + + if (ecore_chain_consume(&p_rx->rxq_chain) != p_pkt->rxq_bd) { + DP_NOTICE(p_hwfn, false, + "Mismatch between active_descq and the LL2 Rx chain\n"); + /* TODO - didn't return error value since this wasn't handled + * before, but this is obviously lacking. + */ + } + + OSAL_LIST_PUSH_TAIL(&p_pkt->list_entry, &p_rx->free_descq); + + data.connection_handle = p_ll2_conn->my_id; + data.cookie = p_pkt->cookie; + data.rx_buf_addr = p_pkt->rx_buf_addr; + data.b_last_packet = b_last_cqe; + + OSAL_SPIN_UNLOCK_IRQSAVE(&p_rx->lock, *p_lock_flags); + p_ll2_conn->cbs.rx_comp_cb(p_ll2_conn->cbs.cookie, + &data); + + OSAL_SPIN_LOCK_IRQSAVE(&p_rx->lock, *p_lock_flags); + + return ECORE_SUCCESS; +} + +static enum _ecore_status_t ecore_ll2_rxq_completion(struct ecore_hwfn *p_hwfn, + void *cookie) +{ + struct ecore_ll2_info *p_ll2_conn = (struct ecore_ll2_info*)cookie; + struct ecore_ll2_rx_queue *p_rx = &p_ll2_conn->rx_queue; + union core_rx_cqe_union *cqe = OSAL_NULL; + u16 cq_new_idx = 0, cq_old_idx = 0; + unsigned long flags = 0; + enum _ecore_status_t rc = ECORE_SUCCESS; + + OSAL_SPIN_LOCK_IRQSAVE(&p_rx->lock, flags); + cq_new_idx = OSAL_LE16_TO_CPU(*p_rx->p_fw_cons); + cq_old_idx = ecore_chain_get_cons_idx(&p_rx->rcq_chain); + + while (cq_new_idx != cq_old_idx) { + bool b_last_cqe = (cq_new_idx == cq_old_idx); + + cqe = (union core_rx_cqe_union *)ecore_chain_consume(&p_rx->rcq_chain); + cq_old_idx = ecore_chain_get_cons_idx(&p_rx->rcq_chain); + + DP_VERBOSE(p_hwfn, ECORE_MSG_LL2, + "LL2 [sw. cons %04x, fw. at %04x] - Got Packet of type %02x\n", + cq_old_idx, cq_new_idx, cqe->rx_cqe_sp.type); + + switch (cqe->rx_cqe_sp.type) { + case CORE_RX_CQE_TYPE_SLOW_PATH: + rc = ecore_ll2_handle_slowpath(p_hwfn, p_ll2_conn, + cqe, &flags); + break; + case CORE_RX_CQE_TYPE_GSI_OFFLOAD: + case CORE_RX_CQE_TYPE_REGULAR: + rc = ecore_ll2_rxq_handle_completion(p_hwfn, p_ll2_conn, + cqe, &flags, + b_last_cqe); + break; + default: + rc = ECORE_IO; + } + } + + OSAL_SPIN_UNLOCK_IRQSAVE(&p_rx->lock, flags); + return rc; +} + +static void ecore_ll2_rxq_flush(struct ecore_hwfn *p_hwfn, + u8 connection_handle) +{ + struct ecore_ll2_info *p_ll2_conn = OSAL_NULL; + struct ecore_ll2_rx_packet *p_pkt = OSAL_NULL; + struct ecore_ll2_rx_queue *p_rx; + unsigned long flags = 0; + + p_ll2_conn = ecore_ll2_handle_sanity_inactive(p_hwfn, + connection_handle); + if (p_ll2_conn == OSAL_NULL) + return; + p_rx = &p_ll2_conn->rx_queue; + + OSAL_SPIN_LOCK_IRQSAVE(&p_rx->lock, flags); + while (!OSAL_LIST_IS_EMPTY(&p_rx->active_descq)) { + bool b_last; + p_pkt = OSAL_LIST_FIRST_ENTRY(&p_rx->active_descq, + struct ecore_ll2_rx_packet, + list_entry); + if (p_pkt == OSAL_NULL) + break; +#if defined(_NTDDK_) +#pragma warning(suppress : 6011 28182) +#endif + OSAL_LIST_REMOVE_ENTRY(&p_pkt->list_entry, + &p_rx->active_descq); + OSAL_LIST_PUSH_TAIL(&p_pkt->list_entry, + &p_rx->free_descq); + b_last = OSAL_LIST_IS_EMPTY(&p_rx->active_descq); + OSAL_SPIN_UNLOCK_IRQSAVE(&p_rx->lock, flags); + + if (p_ll2_conn->input.conn_type == ECORE_LL2_TYPE_OOO) { + struct ecore_ooo_buffer *p_buffer; + + p_buffer = (struct ecore_ooo_buffer *)p_pkt->cookie; + ecore_ooo_put_free_buffer(p_hwfn->p_ooo_info, p_buffer); + } else { + dma_addr_t rx_buf_addr = p_pkt->rx_buf_addr; + void *cookie = p_pkt->cookie; + + p_ll2_conn->cbs.rx_release_cb(p_ll2_conn->cbs.cookie, + p_ll2_conn->my_id, + cookie, + rx_buf_addr, + b_last); + } + OSAL_SPIN_LOCK_IRQSAVE(&p_rx->lock, flags); + } + OSAL_SPIN_UNLOCK_IRQSAVE(&p_rx->lock, flags); +} + +static bool +ecore_ll2_lb_rxq_handler_slowpath(struct ecore_hwfn *p_hwfn, + struct core_rx_slow_path_cqe *p_cqe) +{ + struct ooo_opaque *iscsi_ooo; + u32 cid; + + if (p_cqe->ramrod_cmd_id != CORE_RAMROD_RX_QUEUE_FLUSH) + return false; + + iscsi_ooo = (struct ooo_opaque *)&p_cqe->opaque_data; + if (iscsi_ooo->ooo_opcode != TCP_EVENT_DELETE_ISLES) + return false; + + /* Need to make a flush */ + cid = OSAL_LE32_TO_CPU(iscsi_ooo->cid); + ecore_ooo_release_connection_isles(p_hwfn->p_ooo_info, cid); + + return true; +} + +static enum _ecore_status_t +ecore_ll2_lb_rxq_handler(struct ecore_hwfn *p_hwfn, + struct ecore_ll2_info *p_ll2_conn) +{ + struct ecore_ll2_rx_queue *p_rx = &p_ll2_conn->rx_queue; + u16 packet_length = 0, parse_flags = 0, vlan = 0; + struct ecore_ll2_rx_packet *p_pkt = OSAL_NULL; + u32 num_ooo_add_to_peninsula = 0, cid; + union core_rx_cqe_union *cqe = OSAL_NULL; + u16 cq_new_idx = 0, cq_old_idx = 0; + struct ecore_ooo_buffer *p_buffer; + struct ooo_opaque *iscsi_ooo; + u8 placement_offset = 0; + u8 cqe_type; + + cq_new_idx = OSAL_LE16_TO_CPU(*p_rx->p_fw_cons); + cq_old_idx = ecore_chain_get_cons_idx(&p_rx->rcq_chain); + if (cq_new_idx == cq_old_idx) + return ECORE_SUCCESS; + + while (cq_new_idx != cq_old_idx) { + struct core_rx_fast_path_cqe *p_cqe_fp; + + cqe = (union core_rx_cqe_union *)ecore_chain_consume(&p_rx->rcq_chain); + cq_old_idx = ecore_chain_get_cons_idx(&p_rx->rcq_chain); + cqe_type = cqe->rx_cqe_sp.type; + + if (cqe_type == CORE_RX_CQE_TYPE_SLOW_PATH) + if (ecore_ll2_lb_rxq_handler_slowpath(p_hwfn, + &cqe->rx_cqe_sp)) + continue; + + if (cqe_type != CORE_RX_CQE_TYPE_REGULAR) { + DP_NOTICE(p_hwfn, true, + "Got a non-regular LB LL2 completion [type 0x%02x]\n", + cqe_type); + return ECORE_INVAL; + } + p_cqe_fp = &cqe->rx_cqe_fp; + + placement_offset = p_cqe_fp->placement_offset; + parse_flags = OSAL_LE16_TO_CPU(p_cqe_fp->parse_flags.flags); + packet_length = OSAL_LE16_TO_CPU(p_cqe_fp->packet_length); + vlan = OSAL_LE16_TO_CPU(p_cqe_fp->vlan); + iscsi_ooo = (struct ooo_opaque *)&p_cqe_fp->opaque_data; + ecore_ooo_save_history_entry(p_hwfn->p_ooo_info, iscsi_ooo); + cid = OSAL_LE32_TO_CPU(iscsi_ooo->cid); + + /* Process delete isle first*/ + if (iscsi_ooo->drop_size) + ecore_ooo_delete_isles(p_hwfn, p_hwfn->p_ooo_info, cid, + iscsi_ooo->drop_isle, + iscsi_ooo->drop_size); + + if (iscsi_ooo->ooo_opcode == TCP_EVENT_NOP) + continue; + + /* Now process create/add/join isles */ + if (OSAL_LIST_IS_EMPTY(&p_rx->active_descq)) { + DP_NOTICE(p_hwfn, true, + "LL2 OOO RX chain has no submitted buffers\n"); + return ECORE_IO; + } + + p_pkt = OSAL_LIST_FIRST_ENTRY(&p_rx->active_descq, + struct ecore_ll2_rx_packet, + list_entry); + + if ((iscsi_ooo->ooo_opcode == TCP_EVENT_ADD_NEW_ISLE) || + (iscsi_ooo->ooo_opcode == TCP_EVENT_ADD_ISLE_RIGHT) || + (iscsi_ooo->ooo_opcode == TCP_EVENT_ADD_ISLE_LEFT) || + (iscsi_ooo->ooo_opcode == TCP_EVENT_ADD_PEN) || + (iscsi_ooo->ooo_opcode == TCP_EVENT_JOIN)) { + if (!p_pkt) { + DP_NOTICE(p_hwfn, true, + "LL2 OOO RX packet is not valid\n"); + return ECORE_IO; + } +#if defined(_NTDDK_) +#pragma warning(suppress : 6011 28182) +#endif + OSAL_LIST_REMOVE_ENTRY(&p_pkt->list_entry, + &p_rx->active_descq); + p_buffer = (struct ecore_ooo_buffer *)p_pkt->cookie; + p_buffer->packet_length = packet_length; + p_buffer->parse_flags = parse_flags; + p_buffer->vlan = vlan; + p_buffer->placement_offset = placement_offset; + if (ecore_chain_consume(&p_rx->rxq_chain) != + p_pkt->rxq_bd) { + /**/ + } + ecore_ooo_dump_rx_event(p_hwfn, iscsi_ooo, p_buffer); + OSAL_LIST_PUSH_TAIL(&p_pkt->list_entry, + &p_rx->free_descq); + + switch (iscsi_ooo->ooo_opcode) { + case TCP_EVENT_ADD_NEW_ISLE: + ecore_ooo_add_new_isle(p_hwfn, + p_hwfn->p_ooo_info, + cid, + iscsi_ooo->ooo_isle, + p_buffer); + break; + case TCP_EVENT_ADD_ISLE_RIGHT: + ecore_ooo_add_new_buffer(p_hwfn, + p_hwfn->p_ooo_info, + cid, + iscsi_ooo->ooo_isle, + p_buffer, + ECORE_OOO_RIGHT_BUF); + break; + case TCP_EVENT_ADD_ISLE_LEFT: + ecore_ooo_add_new_buffer(p_hwfn, + p_hwfn->p_ooo_info, + cid, + iscsi_ooo->ooo_isle, + p_buffer, + ECORE_OOO_LEFT_BUF); + break; + case TCP_EVENT_JOIN: + ecore_ooo_add_new_buffer(p_hwfn, + p_hwfn->p_ooo_info, + cid, + iscsi_ooo->ooo_isle + + 1, + p_buffer, + ECORE_OOO_LEFT_BUF); + ecore_ooo_join_isles(p_hwfn, + p_hwfn->p_ooo_info, + cid, + iscsi_ooo->ooo_isle); + break; + case TCP_EVENT_ADD_PEN: + num_ooo_add_to_peninsula++; + ecore_ooo_put_ready_buffer(p_hwfn->p_ooo_info, + p_buffer, true); + break; + } + } else { + DP_NOTICE(p_hwfn, true, + "Unexpected event (%d) TX OOO completion\n", + iscsi_ooo->ooo_opcode); + } + } + + return ECORE_SUCCESS; +} + +static void +ecore_ooo_submit_tx_buffers(struct ecore_hwfn *p_hwfn, + struct ecore_ll2_info *p_ll2_conn) +{ + struct ecore_ll2_tx_pkt_info tx_pkt; + struct ecore_ooo_buffer *p_buffer; + dma_addr_t first_frag; + u16 l4_hdr_offset_w; + u8 bd_flags; + enum _ecore_status_t rc; + + /* Submit Tx buffers here */ + while ((p_buffer = ecore_ooo_get_ready_buffer(p_hwfn->p_ooo_info))) { + l4_hdr_offset_w = 0; + bd_flags = 0; + + first_frag = p_buffer->rx_buffer_phys_addr + + p_buffer->placement_offset; + SET_FIELD(bd_flags, CORE_TX_BD_DATA_FORCE_VLAN_MODE, 1); + SET_FIELD(bd_flags, CORE_TX_BD_DATA_L4_PROTOCOL, 1); + + OSAL_MEM_ZERO(&tx_pkt, sizeof(tx_pkt)); + tx_pkt.num_of_bds = 1; + tx_pkt.vlan = p_buffer->vlan; + tx_pkt.bd_flags = bd_flags; + tx_pkt.l4_hdr_offset_w = l4_hdr_offset_w; + tx_pkt.tx_dest = (enum ecore_ll2_tx_dest)p_ll2_conn->tx_dest; + tx_pkt.first_frag = first_frag; + tx_pkt.first_frag_len = p_buffer->packet_length; + tx_pkt.cookie = p_buffer; + + rc = ecore_ll2_prepare_tx_packet(p_hwfn, p_ll2_conn->my_id, + &tx_pkt, true); + if (rc != ECORE_SUCCESS) { + ecore_ooo_put_ready_buffer(p_hwfn->p_ooo_info, + p_buffer, false); + break; + } + } +} + +static void +ecore_ooo_submit_rx_buffers(struct ecore_hwfn *p_hwfn, + struct ecore_ll2_info *p_ll2_conn) +{ + struct ecore_ooo_buffer *p_buffer; + enum _ecore_status_t rc; + + while ((p_buffer = ecore_ooo_get_free_buffer(p_hwfn->p_ooo_info))) { + rc = ecore_ll2_post_rx_buffer(p_hwfn, + p_ll2_conn->my_id, + p_buffer->rx_buffer_phys_addr, + 0, p_buffer, true); + if (rc != ECORE_SUCCESS) { + ecore_ooo_put_free_buffer(p_hwfn->p_ooo_info, p_buffer); + break; + } + } +} + +static enum _ecore_status_t +ecore_ll2_lb_rxq_completion(struct ecore_hwfn *p_hwfn, + void *p_cookie) +{ + struct ecore_ll2_info *p_ll2_conn = (struct ecore_ll2_info *)p_cookie; + enum _ecore_status_t rc; + + rc = ecore_ll2_lb_rxq_handler(p_hwfn, p_ll2_conn); + if (rc != ECORE_SUCCESS) + return rc; + + ecore_ooo_submit_rx_buffers(p_hwfn, p_ll2_conn); + ecore_ooo_submit_tx_buffers(p_hwfn, p_ll2_conn); + + return 0; +} + +static enum _ecore_status_t +ecore_ll2_lb_txq_completion(struct ecore_hwfn *p_hwfn, + void *p_cookie) +{ + struct ecore_ll2_info *p_ll2_conn = (struct ecore_ll2_info *)p_cookie; + struct ecore_ll2_tx_queue *p_tx = &p_ll2_conn->tx_queue; + struct ecore_ll2_tx_packet *p_pkt = OSAL_NULL; + struct ecore_ooo_buffer *p_buffer; + bool b_dont_submit_rx = false; + u16 new_idx = 0, num_bds = 0; + enum _ecore_status_t rc; + + new_idx = OSAL_LE16_TO_CPU(*p_tx->p_fw_cons); + num_bds = ((s16)new_idx - (s16)p_tx->bds_idx); + + if (!num_bds) + return ECORE_SUCCESS; + + while (num_bds) { + + if (OSAL_LIST_IS_EMPTY(&p_tx->active_descq)) + return ECORE_INVAL; + + p_pkt = OSAL_LIST_FIRST_ENTRY(&p_tx->active_descq, + struct ecore_ll2_tx_packet, + list_entry); + if (!p_pkt) + return ECORE_INVAL; + + if (p_pkt->bd_used != 1) { + DP_NOTICE(p_hwfn, true, + "Unexpectedly many BDs(%d) in TX OOO completion\n", + p_pkt->bd_used); + return ECORE_INVAL; + } + + OSAL_LIST_REMOVE_ENTRY(&p_pkt->list_entry, + &p_tx->active_descq); + + num_bds--; + p_tx->bds_idx++; + ecore_chain_consume(&p_tx->txq_chain); + + p_buffer = (struct ecore_ooo_buffer *)p_pkt->cookie; + OSAL_LIST_PUSH_TAIL(&p_pkt->list_entry, + &p_tx->free_descq); + + if (b_dont_submit_rx) { + ecore_ooo_put_free_buffer(p_hwfn->p_ooo_info, p_buffer); + continue; + } + + rc = ecore_ll2_post_rx_buffer(p_hwfn, p_ll2_conn->my_id, + p_buffer->rx_buffer_phys_addr, 0, + p_buffer, true); + if (rc != ECORE_SUCCESS) { + ecore_ooo_put_free_buffer(p_hwfn->p_ooo_info, p_buffer); + b_dont_submit_rx = true; + } + } + + ecore_ooo_submit_tx_buffers(p_hwfn, p_ll2_conn); + + return ECORE_SUCCESS; +} + +static enum _ecore_status_t ecore_sp_ll2_rx_queue_start(struct ecore_hwfn *p_hwfn, + struct ecore_ll2_info *p_ll2_conn, + u8 action_on_error) +{ + enum ecore_ll2_conn_type conn_type = p_ll2_conn->input.conn_type; + struct ecore_ll2_rx_queue *p_rx = &p_ll2_conn->rx_queue; + struct core_rx_start_ramrod_data *p_ramrod = OSAL_NULL; + struct ecore_spq_entry *p_ent = OSAL_NULL; + struct ecore_sp_init_data init_data; + u16 cqe_pbl_size; + enum _ecore_status_t rc = ECORE_SUCCESS; + + /* Get SPQ entry */ + OSAL_MEMSET(&init_data, 0, sizeof(init_data)); + init_data.cid = p_ll2_conn->cid; + init_data.opaque_fid = p_hwfn->hw_info.opaque_fid; + init_data.comp_mode = ECORE_SPQ_MODE_EBLOCK; + + rc = ecore_sp_init_request(p_hwfn, &p_ent, + CORE_RAMROD_RX_QUEUE_START, + PROTOCOLID_CORE, &init_data); + if (rc != ECORE_SUCCESS) + return rc; + + p_ramrod = &p_ent->ramrod.core_rx_queue_start; + + p_ramrod->sb_id = OSAL_CPU_TO_LE16(ecore_int_get_sp_sb_id(p_hwfn)); + p_ramrod->sb_index = p_rx->rx_sb_index; + p_ramrod->complete_event_flg = 1; + + p_ramrod->mtu = OSAL_CPU_TO_LE16(p_ll2_conn->input.mtu); + DMA_REGPAIR_LE(p_ramrod->bd_base, + p_rx->rxq_chain.p_phys_addr); + cqe_pbl_size = (u16)ecore_chain_get_page_cnt(&p_rx->rcq_chain); + p_ramrod->num_of_pbl_pages = OSAL_CPU_TO_LE16(cqe_pbl_size); + DMA_REGPAIR_LE(p_ramrod->cqe_pbl_addr, + ecore_chain_get_pbl_phys(&p_rx->rcq_chain)); + + p_ramrod->drop_ttl0_flg = p_ll2_conn->input.rx_drop_ttl0_flg; + p_ramrod->inner_vlan_stripping_en = + p_ll2_conn->input.rx_vlan_removal_en; + + if (OSAL_TEST_BIT(ECORE_MF_UFP_SPECIFIC, &p_hwfn->p_dev->mf_bits) && + (p_ll2_conn->input.conn_type == ECORE_LL2_TYPE_FCOE)) + p_ramrod->report_outer_vlan = 1; + p_ramrod->queue_id = p_ll2_conn->queue_id; + p_ramrod->main_func_queue = p_ll2_conn->main_func_queue; + + if (OSAL_TEST_BIT(ECORE_MF_LL2_NON_UNICAST, + &p_hwfn->p_dev->mf_bits) && + p_ramrod->main_func_queue && + ((conn_type != ECORE_LL2_TYPE_ROCE) && + (conn_type != ECORE_LL2_TYPE_IWARP))) { + p_ramrod->mf_si_bcast_accept_all = 1; + p_ramrod->mf_si_mcast_accept_all = 1; + } else { + p_ramrod->mf_si_bcast_accept_all = 0; + p_ramrod->mf_si_mcast_accept_all = 0; + } + + p_ramrod->action_on_error.error_type = action_on_error; + p_ramrod->gsi_offload_flag = p_ll2_conn->input.gsi_enable; + return ecore_spq_post(p_hwfn, p_ent, OSAL_NULL); +} + +static enum _ecore_status_t ecore_sp_ll2_tx_queue_start(struct ecore_hwfn *p_hwfn, + struct ecore_ll2_info *p_ll2_conn) +{ + enum ecore_ll2_conn_type conn_type = p_ll2_conn->input.conn_type; + struct ecore_ll2_tx_queue *p_tx = &p_ll2_conn->tx_queue; + struct core_tx_start_ramrod_data *p_ramrod = OSAL_NULL; + struct ecore_spq_entry *p_ent = OSAL_NULL; + struct ecore_sp_init_data init_data; + u16 pq_id = 0, pbl_size; + enum _ecore_status_t rc = ECORE_NOTIMPL; + + if (!ECORE_LL2_TX_REGISTERED(p_ll2_conn)) + return ECORE_SUCCESS; + + if (p_ll2_conn->input.conn_type == ECORE_LL2_TYPE_OOO) + p_ll2_conn->tx_stats_en = 0; + else + p_ll2_conn->tx_stats_en = 1; + + /* Get SPQ entry */ + OSAL_MEMSET(&init_data, 0, sizeof(init_data)); + init_data.cid = p_ll2_conn->cid; + init_data.opaque_fid = p_hwfn->hw_info.opaque_fid; + init_data.comp_mode = ECORE_SPQ_MODE_EBLOCK; + + rc = ecore_sp_init_request(p_hwfn, &p_ent, + CORE_RAMROD_TX_QUEUE_START, + PROTOCOLID_CORE, &init_data); + if (rc != ECORE_SUCCESS) + return rc; + + p_ramrod = &p_ent->ramrod.core_tx_queue_start; + + p_ramrod->sb_id = OSAL_CPU_TO_LE16(ecore_int_get_sp_sb_id(p_hwfn)); + p_ramrod->sb_index = p_tx->tx_sb_index; + p_ramrod->mtu = OSAL_CPU_TO_LE16(p_ll2_conn->input.mtu); + p_ramrod->stats_en = p_ll2_conn->tx_stats_en; + p_ramrod->stats_id = p_ll2_conn->tx_stats_id; + + DMA_REGPAIR_LE(p_ramrod->pbl_base_addr, + ecore_chain_get_pbl_phys(&p_tx->txq_chain)); + pbl_size = (u16)ecore_chain_get_page_cnt(&p_tx->txq_chain); + p_ramrod->pbl_size = OSAL_CPU_TO_LE16(pbl_size); + + /* TODO RESC_ALLOC pq for ll2 */ + switch (p_ll2_conn->input.tx_tc) { + case PURE_LB_TC: + pq_id = ecore_get_cm_pq_idx(p_hwfn, PQ_FLAGS_LB); + break; + case PKT_LB_TC: + pq_id = ecore_get_cm_pq_idx(p_hwfn, PQ_FLAGS_OOO); + break; + default: + pq_id = ecore_get_cm_pq_idx(p_hwfn, PQ_FLAGS_OFLD); + } + + p_ramrod->qm_pq_id = OSAL_CPU_TO_LE16(pq_id); + + switch (conn_type) { + case ECORE_LL2_TYPE_FCOE: + p_ramrod->conn_type = PROTOCOLID_FCOE; + break; + case ECORE_LL2_TYPE_ISCSI: + p_ramrod->conn_type = PROTOCOLID_ISCSI; + break; + case ECORE_LL2_TYPE_ROCE: + p_ramrod->conn_type = PROTOCOLID_ROCE; + break; + case ECORE_LL2_TYPE_IWARP: + p_ramrod->conn_type = PROTOCOLID_IWARP; + break; + case ECORE_LL2_TYPE_OOO: + if (p_hwfn->hw_info.personality == ECORE_PCI_ISCSI) { + p_ramrod->conn_type = PROTOCOLID_ISCSI; + } else { + p_ramrod->conn_type = PROTOCOLID_IWARP; + } + break; + default: + p_ramrod->conn_type = PROTOCOLID_ETH; + DP_NOTICE(p_hwfn, false, "Unknown connection type: %d\n", + conn_type); + } + + p_ramrod->gsi_offload_flag = p_ll2_conn->input.gsi_enable; + + rc = ecore_spq_post(p_hwfn, p_ent, OSAL_NULL); + if (rc != ECORE_SUCCESS) + return rc; + + rc = ecore_db_recovery_add(p_hwfn->p_dev, p_tx->doorbell_addr, + &p_tx->db_msg, DB_REC_WIDTH_32B, + DB_REC_KERNEL); + return rc; +} + +static enum _ecore_status_t ecore_sp_ll2_rx_queue_stop(struct ecore_hwfn *p_hwfn, + struct ecore_ll2_info *p_ll2_conn) +{ + struct core_rx_stop_ramrod_data *p_ramrod = OSAL_NULL; + struct ecore_spq_entry *p_ent = OSAL_NULL; + struct ecore_sp_init_data init_data; + enum _ecore_status_t rc = ECORE_NOTIMPL; + + /* Get SPQ entry */ + OSAL_MEMSET(&init_data, 0, sizeof(init_data)); + init_data.cid = p_ll2_conn->cid; + init_data.opaque_fid = p_hwfn->hw_info.opaque_fid; + init_data.comp_mode = ECORE_SPQ_MODE_EBLOCK; + + rc = ecore_sp_init_request(p_hwfn, &p_ent, + CORE_RAMROD_RX_QUEUE_STOP, + PROTOCOLID_CORE, &init_data); + if (rc != ECORE_SUCCESS) + return rc; + + p_ramrod = &p_ent->ramrod.core_rx_queue_stop; + + p_ramrod->complete_event_flg = 1; + p_ramrod->queue_id = p_ll2_conn->queue_id; + + return ecore_spq_post(p_hwfn, p_ent, OSAL_NULL); +} + +static enum _ecore_status_t ecore_sp_ll2_tx_queue_stop(struct ecore_hwfn *p_hwfn, + struct ecore_ll2_info *p_ll2_conn) +{ + struct ecore_ll2_tx_queue *p_tx = &p_ll2_conn->tx_queue; + struct ecore_spq_entry *p_ent = OSAL_NULL; + struct ecore_sp_init_data init_data; + enum _ecore_status_t rc = ECORE_NOTIMPL; + + ecore_db_recovery_del(p_hwfn->p_dev, p_tx->doorbell_addr, + &p_tx->db_msg); + + /* Get SPQ entry */ + OSAL_MEMSET(&init_data, 0, sizeof(init_data)); + init_data.cid = p_ll2_conn->cid; + init_data.opaque_fid = p_hwfn->hw_info.opaque_fid; + init_data.comp_mode = ECORE_SPQ_MODE_EBLOCK; + + rc = ecore_sp_init_request(p_hwfn, &p_ent, + CORE_RAMROD_TX_QUEUE_STOP, + PROTOCOLID_CORE, &init_data); + if (rc != ECORE_SUCCESS) + return rc; + + return ecore_spq_post(p_hwfn, p_ent, OSAL_NULL); +} + +static enum _ecore_status_t +ecore_ll2_acquire_connection_rx(struct ecore_hwfn *p_hwfn, + struct ecore_ll2_info *p_ll2_info) +{ + struct ecore_ll2_rx_packet *p_descq; + u32 capacity; + enum _ecore_status_t rc = ECORE_SUCCESS; + + if (!p_ll2_info->input.rx_num_desc) + goto out; + + rc = ecore_chain_alloc(p_hwfn->p_dev, + ECORE_CHAIN_USE_TO_CONSUME_PRODUCE, + ECORE_CHAIN_MODE_NEXT_PTR, + ECORE_CHAIN_CNT_TYPE_U16, + p_ll2_info->input.rx_num_desc, + sizeof(struct core_rx_bd), + &p_ll2_info->rx_queue.rxq_chain, OSAL_NULL); + if (rc) { + DP_NOTICE(p_hwfn, false, + "Failed to allocate ll2 rxq chain\n"); + goto out; + } + + capacity = ecore_chain_get_capacity(&p_ll2_info->rx_queue.rxq_chain); + p_descq = OSAL_ZALLOC(p_hwfn->p_dev, GFP_KERNEL, + capacity * sizeof(struct ecore_ll2_rx_packet)); + if (!p_descq) { + rc = ECORE_NOMEM; + DP_NOTICE(p_hwfn, false, + "Failed to allocate ll2 Rx desc\n"); + goto out; + } + p_ll2_info->rx_queue.descq_array = p_descq; + + rc = ecore_chain_alloc(p_hwfn->p_dev, + ECORE_CHAIN_USE_TO_CONSUME_PRODUCE, + ECORE_CHAIN_MODE_PBL, + ECORE_CHAIN_CNT_TYPE_U16, + p_ll2_info->input.rx_num_desc, + sizeof(struct core_rx_fast_path_cqe), + &p_ll2_info->rx_queue.rcq_chain, OSAL_NULL); + if (rc != ECORE_SUCCESS) { + DP_NOTICE(p_hwfn, false, + "Failed to allocate ll2 rcq chain\n"); + goto out; + } + + DP_VERBOSE(p_hwfn, ECORE_MSG_LL2, + "Allocated LL2 Rxq [Type %08x] with 0x%08x buffers\n", + p_ll2_info->input.conn_type, + p_ll2_info->input.rx_num_desc); + +out: + return rc; +} + +static enum _ecore_status_t +ecore_ll2_acquire_connection_tx(struct ecore_hwfn *p_hwfn, + struct ecore_ll2_info *p_ll2_info) +{ + struct ecore_ll2_tx_packet *p_descq; + u32 capacity; + enum _ecore_status_t rc = ECORE_SUCCESS; + u32 desc_size; + + if (!p_ll2_info->input.tx_num_desc) + goto out; + + rc = ecore_chain_alloc(p_hwfn->p_dev, + ECORE_CHAIN_USE_TO_CONSUME_PRODUCE, + ECORE_CHAIN_MODE_PBL, + ECORE_CHAIN_CNT_TYPE_U16, + p_ll2_info->input.tx_num_desc, + sizeof(struct core_tx_bd), + &p_ll2_info->tx_queue.txq_chain, OSAL_NULL); + if (rc != ECORE_SUCCESS) + goto out; + + capacity = ecore_chain_get_capacity(&p_ll2_info->tx_queue.txq_chain); + desc_size = (sizeof(*p_descq) + + (p_ll2_info->input.tx_max_bds_per_packet - 1) * + sizeof(p_descq->bds_set)); + + p_descq = OSAL_ZALLOC(p_hwfn->p_dev, GFP_KERNEL, + capacity * desc_size); + if (!p_descq) { + rc = ECORE_NOMEM; + goto out; + } + p_ll2_info->tx_queue.descq_array = p_descq; + + DP_VERBOSE(p_hwfn, ECORE_MSG_LL2, + "Allocated LL2 Txq [Type %08x] with 0x%08x buffers\n", + p_ll2_info->input.conn_type, + p_ll2_info->input.tx_num_desc); + +out: + if (rc != ECORE_SUCCESS) + DP_NOTICE(p_hwfn, false, + "Can't allocate memory for Tx LL2 with 0x%08x buffers\n", + p_ll2_info->input.tx_num_desc); + return rc; +} + +static enum _ecore_status_t +ecore_ll2_acquire_connection_ooo(struct ecore_hwfn *p_hwfn, + struct ecore_ll2_info *p_ll2_info, u16 mtu) +{ + struct ecore_ooo_buffer *p_buf = OSAL_NULL; + u32 rx_buffer_size = 0; + void *p_virt; + u16 buf_idx; + enum _ecore_status_t rc = ECORE_SUCCESS; + + if (p_ll2_info->input.conn_type != ECORE_LL2_TYPE_OOO) + return rc; + + /* Correct number of requested OOO buffers if needed */ + if (!p_ll2_info->input.rx_num_ooo_buffers) { + u16 num_desc = p_ll2_info->input.rx_num_desc; + + if (!num_desc) + return ECORE_INVAL; + p_ll2_info->input.rx_num_ooo_buffers = num_desc * 2; + } + + /* TODO - use some defines for buffer size */ + rx_buffer_size = mtu + 14 + 4 + 8 + ETH_CACHE_LINE_SIZE; + rx_buffer_size = (rx_buffer_size + ETH_CACHE_LINE_SIZE - 1) & + ~(ETH_CACHE_LINE_SIZE - 1); + + for (buf_idx = 0; buf_idx < p_ll2_info->input.rx_num_ooo_buffers; + buf_idx++) { + p_buf = OSAL_ZALLOC(p_hwfn->p_dev, GFP_KERNEL, sizeof(*p_buf)); + if (!p_buf) { + DP_NOTICE(p_hwfn, false, + "Failed to allocate ooo descriptor\n"); + rc = ECORE_NOMEM; + goto out; + } + + p_buf->rx_buffer_size = rx_buffer_size; + p_virt = OSAL_DMA_ALLOC_COHERENT(p_hwfn->p_dev, + &p_buf->rx_buffer_phys_addr, + p_buf->rx_buffer_size); + if (!p_virt) { + DP_NOTICE(p_hwfn, false, + "Failed to allocate ooo buffer\n"); + OSAL_FREE(p_hwfn->p_dev, p_buf); + rc = ECORE_NOMEM; + goto out; + } + p_buf->rx_buffer_virt_addr = p_virt; + ecore_ooo_put_free_buffer(p_hwfn->p_ooo_info, p_buf); + } + + DP_VERBOSE(p_hwfn, ECORE_MSG_LL2, + "Allocated [%04x] LL2 OOO buffers [each of size 0x%08x]\n", + p_ll2_info->input.rx_num_ooo_buffers, rx_buffer_size); + +out: + return rc; +} + +static enum _ecore_status_t +ecore_ll2_set_cbs(struct ecore_ll2_info *p_ll2_info, + const struct ecore_ll2_cbs *cbs) +{ + if (!cbs || (!cbs->rx_comp_cb || + !cbs->rx_release_cb || + !cbs->tx_comp_cb || + !cbs->tx_release_cb || + !cbs->cookie)) + return ECORE_INVAL; + + p_ll2_info->cbs.rx_comp_cb = cbs->rx_comp_cb; + p_ll2_info->cbs.rx_release_cb = cbs->rx_release_cb; + p_ll2_info->cbs.tx_comp_cb = cbs->tx_comp_cb; + p_ll2_info->cbs.tx_release_cb = cbs->tx_release_cb; + p_ll2_info->cbs.slowpath_cb = cbs->slowpath_cb; + p_ll2_info->cbs.cookie = cbs->cookie; + + return ECORE_SUCCESS; +} + +static enum core_error_handle +ecore_ll2_get_error_choice(enum ecore_ll2_error_handle err) +{ + switch (err) { + case ECORE_LL2_DROP_PACKET: + return LL2_DROP_PACKET; + case ECORE_LL2_DO_NOTHING: + return LL2_DO_NOTHING; + case ECORE_LL2_ASSERT: + return LL2_ASSERT; + default: + return LL2_DO_NOTHING; + } +} + +enum _ecore_status_t +ecore_ll2_acquire_connection(void *cxt, + struct ecore_ll2_acquire_data *data) +{ + struct ecore_hwfn *p_hwfn = (struct ecore_hwfn *)cxt; + ecore_int_comp_cb_t comp_rx_cb, comp_tx_cb; + struct ecore_ll2_info *p_ll2_info = OSAL_NULL; + enum _ecore_status_t rc; + u8 i, *p_tx_max; + + if (!data->p_connection_handle || !p_hwfn->p_ll2_info) { + DP_NOTICE(p_hwfn, false, "Invalid connection handle, ll2_info not allocated\n"); + return ECORE_INVAL; + } + + /* Find a free connection to be used */ + for (i = 0; (i < ECORE_MAX_NUM_OF_LL2_CONNECTIONS); i++) { + OSAL_MUTEX_ACQUIRE(&p_hwfn->p_ll2_info[i].mutex); + if (p_hwfn->p_ll2_info[i].b_active) { + OSAL_MUTEX_RELEASE(&p_hwfn->p_ll2_info[i].mutex); + continue; + } + + p_hwfn->p_ll2_info[i].b_active = true; + p_ll2_info = &p_hwfn->p_ll2_info[i]; + OSAL_MUTEX_RELEASE(&p_hwfn->p_ll2_info[i].mutex); + break; + } + if (p_ll2_info == OSAL_NULL) { + DP_NOTICE(p_hwfn, false, "No available ll2 connection\n"); + return ECORE_BUSY; + } + + OSAL_MEMCPY(&p_ll2_info->input, &data->input, + sizeof(p_ll2_info->input)); + + switch (data->input.tx_dest) { + case ECORE_LL2_TX_DEST_NW: + p_ll2_info->tx_dest = CORE_TX_DEST_NW; + break; + case ECORE_LL2_TX_DEST_LB: + p_ll2_info->tx_dest = CORE_TX_DEST_LB; + break; + case ECORE_LL2_TX_DEST_DROP: + p_ll2_info->tx_dest = CORE_TX_DEST_DROP; + break; + default: + return ECORE_INVAL; + } + + if ((data->input.conn_type == ECORE_LL2_TYPE_OOO) || + data->input.secondary_queue) + p_ll2_info->main_func_queue = false; + else + p_ll2_info->main_func_queue = true; + + /* Correct maximum number of Tx BDs */ + p_tx_max = &p_ll2_info->input.tx_max_bds_per_packet; + if (*p_tx_max == 0) + *p_tx_max = CORE_LL2_TX_MAX_BDS_PER_PACKET; + else + *p_tx_max = OSAL_MIN_T(u8, *p_tx_max, + CORE_LL2_TX_MAX_BDS_PER_PACKET); + + rc = ecore_ll2_set_cbs(p_ll2_info, data->cbs); + if (rc) { + DP_NOTICE(p_hwfn, false, "Invalid callback functions\n"); + goto q_allocate_fail; + } + + rc = ecore_ll2_acquire_connection_rx(p_hwfn, p_ll2_info); + if (rc != ECORE_SUCCESS) { + DP_NOTICE(p_hwfn, false, "ll2 acquire rx connection failed\n"); + goto q_allocate_fail; + } + + rc = ecore_ll2_acquire_connection_tx(p_hwfn, p_ll2_info); + if (rc != ECORE_SUCCESS) { + DP_NOTICE(p_hwfn, false, "ll2 acquire tx connection failed\n"); + goto q_allocate_fail; + } + + rc = ecore_ll2_acquire_connection_ooo(p_hwfn, p_ll2_info, + data->input.mtu); + if (rc != ECORE_SUCCESS) { + DP_NOTICE(p_hwfn, false, "ll2 acquire ooo connection failed\n"); + goto q_allocate_fail; + } + + /* Register callbacks for the Rx/Tx queues */ + if (data->input.conn_type == ECORE_LL2_TYPE_OOO) { + comp_rx_cb = ecore_ll2_lb_rxq_completion; + comp_tx_cb = ecore_ll2_lb_txq_completion; + + } else { + comp_rx_cb = ecore_ll2_rxq_completion; + comp_tx_cb = ecore_ll2_txq_completion; + } + + if (data->input.rx_num_desc) { + ecore_int_register_cb(p_hwfn, comp_rx_cb, + &p_hwfn->p_ll2_info[i], + &p_ll2_info->rx_queue.rx_sb_index, + &p_ll2_info->rx_queue.p_fw_cons); + p_ll2_info->rx_queue.b_cb_registred = true; + } + + if (data->input.tx_num_desc) { + ecore_int_register_cb(p_hwfn, + comp_tx_cb, + &p_hwfn->p_ll2_info[i], + &p_ll2_info->tx_queue.tx_sb_index, + &p_ll2_info->tx_queue.p_fw_cons); + p_ll2_info->tx_queue.b_cb_registred = true; + } + + *(data->p_connection_handle) = i; + return rc; + +q_allocate_fail: + ecore_ll2_release_connection(p_hwfn, i); + return ECORE_NOMEM; +} + +static enum _ecore_status_t ecore_ll2_establish_connection_rx(struct ecore_hwfn *p_hwfn, + struct ecore_ll2_info *p_ll2_conn) +{ + enum ecore_ll2_error_handle error_input; + enum core_error_handle error_mode; + u8 action_on_error = 0; + + if (!ECORE_LL2_RX_REGISTERED(p_ll2_conn)) + return ECORE_SUCCESS; + + DIRECT_REG_WR(p_hwfn, p_ll2_conn->rx_queue.set_prod_addr, 0x0); + error_input = p_ll2_conn->input.ai_err_packet_too_big; + error_mode = ecore_ll2_get_error_choice(error_input); + SET_FIELD(action_on_error, + CORE_RX_ACTION_ON_ERROR_PACKET_TOO_BIG, error_mode); + error_input = p_ll2_conn->input.ai_err_no_buf; + error_mode = ecore_ll2_get_error_choice(error_input); + SET_FIELD(action_on_error, + CORE_RX_ACTION_ON_ERROR_NO_BUFF, error_mode); + + return ecore_sp_ll2_rx_queue_start(p_hwfn, p_ll2_conn, action_on_error); +} + +static void +ecore_ll2_establish_connection_ooo(struct ecore_hwfn *p_hwfn, + struct ecore_ll2_info *p_ll2_conn) +{ + if (p_ll2_conn->input.conn_type != ECORE_LL2_TYPE_OOO) + return; + + ecore_ooo_release_all_isles(p_hwfn->p_ooo_info); + ecore_ooo_submit_rx_buffers(p_hwfn, p_ll2_conn); +} + +enum _ecore_status_t ecore_ll2_establish_connection(void *cxt, + u8 connection_handle) +{ + struct ecore_hwfn *p_hwfn = (struct ecore_hwfn *)cxt; + struct e4_core_conn_context *p_cxt; + struct ecore_ll2_info *p_ll2_conn; + struct ecore_cxt_info cxt_info; + struct ecore_ll2_rx_queue *p_rx; + struct ecore_ll2_tx_queue *p_tx; + struct ecore_ll2_tx_packet *p_pkt; + struct ecore_ptt *p_ptt; + enum _ecore_status_t rc = ECORE_NOTIMPL; + u32 i, capacity; + u32 desc_size; + u8 qid; + + p_ptt = ecore_ptt_acquire(p_hwfn); + if (!p_ptt) + return ECORE_AGAIN; + + p_ll2_conn = ecore_ll2_handle_sanity_lock(p_hwfn, connection_handle); + if (p_ll2_conn == OSAL_NULL) { + rc = ECORE_INVAL; + goto out; + } + + p_rx = &p_ll2_conn->rx_queue; + p_tx = &p_ll2_conn->tx_queue; + + ecore_chain_reset(&p_rx->rxq_chain); + ecore_chain_reset(&p_rx->rcq_chain); + OSAL_LIST_INIT(&p_rx->active_descq); + OSAL_LIST_INIT(&p_rx->free_descq); + OSAL_LIST_INIT(&p_rx->posting_descq); + OSAL_SPIN_LOCK_INIT(&p_rx->lock); + capacity = ecore_chain_get_capacity(&p_rx->rxq_chain); + for (i = 0; i < capacity; i++) + OSAL_LIST_PUSH_TAIL(&p_rx->descq_array[i].list_entry, + &p_rx->free_descq); + *p_rx->p_fw_cons = 0; + + ecore_chain_reset(&p_tx->txq_chain); + OSAL_LIST_INIT(&p_tx->active_descq); + OSAL_LIST_INIT(&p_tx->free_descq); + OSAL_LIST_INIT(&p_tx->sending_descq); + OSAL_SPIN_LOCK_INIT(&p_tx->lock); + capacity = ecore_chain_get_capacity(&p_tx->txq_chain); + /* The size of the element in descq_array is flexible */ + desc_size = (sizeof(*p_pkt) + + (p_ll2_conn->input.tx_max_bds_per_packet - 1) * + sizeof(p_pkt->bds_set)); + + for (i = 0; i < capacity; i++) { + p_pkt = (struct ecore_ll2_tx_packet *)((u8 *)p_tx->descq_array + + desc_size*i); + OSAL_LIST_PUSH_TAIL(&p_pkt->list_entry, + &p_tx->free_descq); + } + p_tx->cur_completing_bd_idx = 0; + p_tx->bds_idx = 0; + p_tx->b_completing_packet = false; + p_tx->cur_send_packet = OSAL_NULL; + p_tx->cur_send_frag_num = 0; + p_tx->cur_completing_frag_num = 0; + *p_tx->p_fw_cons = 0; + + rc = ecore_cxt_acquire_cid(p_hwfn, PROTOCOLID_CORE, &p_ll2_conn->cid); + if (rc) + goto out; + cxt_info.iid = p_ll2_conn->cid; + rc = ecore_cxt_get_cid_info(p_hwfn, &cxt_info); + if (rc) { + DP_NOTICE(p_hwfn, true, "Cannot find context info for cid=%d\n", + p_ll2_conn->cid); + goto out; + } + + p_cxt = cxt_info.p_cxt; + + /* @@@TBD we zero the context until we have ilt_reset implemented. */ + OSAL_MEM_ZERO(p_cxt, sizeof(*p_cxt)); + + qid = ecore_ll2_handle_to_queue_id(p_hwfn, connection_handle); + p_ll2_conn->queue_id = qid; + p_ll2_conn->tx_stats_id = qid; + p_rx->set_prod_addr = (u8 OSAL_IOMEM*)p_hwfn->regview + + GTT_BAR0_MAP_REG_TSDM_RAM + + TSTORM_LL2_RX_PRODS_OFFSET(qid); + p_tx->doorbell_addr = (u8 OSAL_IOMEM*)p_hwfn->doorbells + + DB_ADDR(p_ll2_conn->cid, + DQ_DEMS_LEGACY); + + /* prepare db data */ + SET_FIELD(p_tx->db_msg.params, CORE_DB_DATA_DEST, DB_DEST_XCM); + SET_FIELD(p_tx->db_msg.params, CORE_DB_DATA_AGG_CMD, + DB_AGG_CMD_SET); + SET_FIELD(p_tx->db_msg.params, CORE_DB_DATA_AGG_VAL_SEL, + DQ_XCM_CORE_TX_BD_PROD_CMD); + p_tx->db_msg.agg_flags = DQ_XCM_CORE_DQ_CF_CMD; + + rc = ecore_ll2_establish_connection_rx(p_hwfn, p_ll2_conn); + if (rc) + goto out; + + rc = ecore_sp_ll2_tx_queue_start(p_hwfn, p_ll2_conn); + if (rc) + goto out; + + if (!ECORE_IS_RDMA_PERSONALITY(p_hwfn)) + ecore_wr(p_hwfn, p_ptt, PRS_REG_USE_LIGHT_L2, 1); + + ecore_ll2_establish_connection_ooo(p_hwfn, p_ll2_conn); + + if (p_ll2_conn->input.conn_type == ECORE_LL2_TYPE_FCOE) { + if (!OSAL_TEST_BIT(ECORE_MF_UFP_SPECIFIC, + &p_hwfn->p_dev->mf_bits)) + ecore_llh_add_protocol_filter(p_hwfn->p_dev, 0, + ECORE_LLH_FILTER_ETHERTYPE, + 0x8906, 0); + ecore_llh_add_protocol_filter(p_hwfn->p_dev, 0, + ECORE_LLH_FILTER_ETHERTYPE, + 0x8914, 0); + } + +out: + ecore_ptt_release(p_hwfn, p_ptt); + + return rc; +} + +static void ecore_ll2_post_rx_buffer_notify_fw(struct ecore_hwfn *p_hwfn, + struct ecore_ll2_rx_queue *p_rx, + struct ecore_ll2_rx_packet *p_curp) +{ + struct ecore_ll2_rx_packet *p_posting_packet = OSAL_NULL; + struct core_ll2_rx_prod rx_prod = {0, 0, 0}; + bool b_notify_fw = false; + u16 bd_prod, cq_prod; + + /* This handles the flushing of already posted buffers */ + while (!OSAL_LIST_IS_EMPTY(&p_rx->posting_descq)) { + p_posting_packet = OSAL_LIST_FIRST_ENTRY(&p_rx->posting_descq, + struct ecore_ll2_rx_packet, + list_entry); +#if defined(_NTDDK_) +#pragma warning(suppress : 6011 28182) +#endif + OSAL_LIST_REMOVE_ENTRY(&p_posting_packet->list_entry, &p_rx->posting_descq); + OSAL_LIST_PUSH_TAIL(&p_posting_packet->list_entry, &p_rx->active_descq); + b_notify_fw = true; + } + + /* This handles the supplied packet [if there is one] */ + if (p_curp) { + OSAL_LIST_PUSH_TAIL(&p_curp->list_entry, + &p_rx->active_descq); + b_notify_fw = true; + } + + if (!b_notify_fw) + return; + + bd_prod = ecore_chain_get_prod_idx(&p_rx->rxq_chain); + cq_prod = ecore_chain_get_prod_idx(&p_rx->rcq_chain); + rx_prod.bd_prod = OSAL_CPU_TO_LE16(bd_prod); + rx_prod.cqe_prod = OSAL_CPU_TO_LE16(cq_prod); + DIRECT_REG_WR(p_hwfn, p_rx->set_prod_addr, *((u32 *)&rx_prod)); +} + +enum _ecore_status_t ecore_ll2_post_rx_buffer(void *cxt, + u8 connection_handle, + dma_addr_t addr, + u16 buf_len, + void *cookie, + u8 notify_fw) +{ + struct ecore_hwfn *p_hwfn = (struct ecore_hwfn *)cxt; + struct core_rx_bd_with_buff_len *p_curb = OSAL_NULL; + struct ecore_ll2_rx_packet *p_curp = OSAL_NULL; + struct ecore_ll2_info *p_ll2_conn; + struct ecore_ll2_rx_queue *p_rx; + unsigned long flags; + void *p_data; + enum _ecore_status_t rc = ECORE_SUCCESS; + + p_ll2_conn = ecore_ll2_handle_sanity(p_hwfn, connection_handle); + if (p_ll2_conn == OSAL_NULL) + return ECORE_INVAL; + p_rx = &p_ll2_conn->rx_queue; + if (p_rx->set_prod_addr == OSAL_NULL) + return ECORE_IO; + + OSAL_SPIN_LOCK_IRQSAVE(&p_rx->lock, flags); + if (!OSAL_LIST_IS_EMPTY(&p_rx->free_descq)) + p_curp = OSAL_LIST_FIRST_ENTRY(&p_rx->free_descq, + struct ecore_ll2_rx_packet, + list_entry); + if (p_curp) { + if (ecore_chain_get_elem_left(&p_rx->rxq_chain) && + ecore_chain_get_elem_left(&p_rx->rcq_chain)) { + p_data = ecore_chain_produce(&p_rx->rxq_chain); + p_curb = (struct core_rx_bd_with_buff_len *)p_data; + ecore_chain_produce(&p_rx->rcq_chain); + } + } + + /* If we're lacking entires, let's try to flush buffers to FW */ + if (!p_curp || !p_curb) { + rc = ECORE_BUSY; + p_curp = OSAL_NULL; + goto out_notify; + } + + /* We have an Rx packet we can fill */ + DMA_REGPAIR_LE(p_curb->addr, addr); + p_curb->buff_length = OSAL_CPU_TO_LE16(buf_len); + p_curp->rx_buf_addr = addr; + p_curp->cookie = cookie; + p_curp->rxq_bd = p_curb; + p_curp->buf_length = buf_len; + OSAL_LIST_REMOVE_ENTRY(&p_curp->list_entry, + &p_rx->free_descq); + + /* Check if we only want to enqueue this packet without informing FW */ + if (!notify_fw) { + OSAL_LIST_PUSH_TAIL(&p_curp->list_entry, + &p_rx->posting_descq); + goto out; + } + +out_notify: + ecore_ll2_post_rx_buffer_notify_fw(p_hwfn, p_rx, p_curp); +out: + OSAL_SPIN_UNLOCK_IRQSAVE(&p_rx->lock, flags); + return rc; +} + +static void ecore_ll2_prepare_tx_packet_set(struct ecore_ll2_tx_queue *p_tx, + struct ecore_ll2_tx_packet *p_curp, + struct ecore_ll2_tx_pkt_info *pkt, + u8 notify_fw) +{ + OSAL_LIST_REMOVE_ENTRY(&p_curp->list_entry, + &p_tx->free_descq); + p_curp->cookie = pkt->cookie; + p_curp->bd_used = pkt->num_of_bds; + p_curp->notify_fw = notify_fw; + p_tx->cur_send_packet = p_curp; + p_tx->cur_send_frag_num = 0; + + p_curp->bds_set[p_tx->cur_send_frag_num].tx_frag = pkt->first_frag; + p_curp->bds_set[p_tx->cur_send_frag_num].frag_len = pkt->first_frag_len; + p_tx->cur_send_frag_num++; +} + +static void ecore_ll2_prepare_tx_packet_set_bd( + struct ecore_hwfn *p_hwfn, + struct ecore_ll2_info *p_ll2, + struct ecore_ll2_tx_packet *p_curp, + struct ecore_ll2_tx_pkt_info *pkt) +{ + struct ecore_chain *p_tx_chain = &p_ll2->tx_queue.txq_chain; + u16 prod_idx = ecore_chain_get_prod_idx(p_tx_chain); + struct core_tx_bd *start_bd = OSAL_NULL; + enum core_roce_flavor_type roce_flavor; + enum core_tx_dest tx_dest; + u16 bd_data = 0, frag_idx; + + roce_flavor = (pkt->ecore_roce_flavor == ECORE_LL2_ROCE) ? + CORE_ROCE : CORE_RROCE; + + switch (pkt->tx_dest) { + case ECORE_LL2_TX_DEST_NW: + tx_dest = CORE_TX_DEST_NW; + break; + case ECORE_LL2_TX_DEST_LB: + tx_dest = CORE_TX_DEST_LB; + break; + case ECORE_LL2_TX_DEST_DROP: + tx_dest = CORE_TX_DEST_DROP; + break; + default: + tx_dest = CORE_TX_DEST_LB; + break; + } + + start_bd = (struct core_tx_bd*)ecore_chain_produce(p_tx_chain); + + if (ECORE_IS_IWARP_PERSONALITY(p_hwfn) && + (p_ll2->input.conn_type == ECORE_LL2_TYPE_OOO)) { + start_bd->nw_vlan_or_lb_echo = + OSAL_CPU_TO_LE16(IWARP_LL2_IN_ORDER_TX_QUEUE); + } else { + start_bd->nw_vlan_or_lb_echo = OSAL_CPU_TO_LE16(pkt->vlan); + if (OSAL_TEST_BIT(ECORE_MF_UFP_SPECIFIC, &p_hwfn->p_dev->mf_bits) && + (p_ll2->input.conn_type == ECORE_LL2_TYPE_FCOE)) + pkt->remove_stag = true; + } + + SET_FIELD(start_bd->bitfield1, CORE_TX_BD_L4_HDR_OFFSET_W, + OSAL_CPU_TO_LE16(pkt->l4_hdr_offset_w)); + SET_FIELD(start_bd->bitfield1, CORE_TX_BD_TX_DST, tx_dest); + bd_data |= pkt->bd_flags; + SET_FIELD(bd_data, CORE_TX_BD_DATA_START_BD, 0x1); + SET_FIELD(bd_data, CORE_TX_BD_DATA_NBDS, pkt->num_of_bds); + SET_FIELD(bd_data, CORE_TX_BD_DATA_ROCE_FLAV, roce_flavor); + SET_FIELD(bd_data, CORE_TX_BD_DATA_IP_CSUM, !!(pkt->enable_ip_cksum)); + SET_FIELD(bd_data, CORE_TX_BD_DATA_L4_CSUM, !!(pkt->enable_l4_cksum)); + SET_FIELD(bd_data, CORE_TX_BD_DATA_IP_LEN, !!(pkt->calc_ip_len)); + SET_FIELD(bd_data, CORE_TX_BD_DATA_DISABLE_STAG_INSERTION, + !!(pkt->remove_stag)); + + start_bd->bd_data.as_bitfield = OSAL_CPU_TO_LE16(bd_data); + DMA_REGPAIR_LE(start_bd->addr, pkt->first_frag); + start_bd->nbytes = OSAL_CPU_TO_LE16(pkt->first_frag_len); + + DP_VERBOSE(p_hwfn, (ECORE_MSG_TX_QUEUED | ECORE_MSG_LL2), + "LL2 [q 0x%02x cid 0x%08x type 0x%08x] Tx Producer at [0x%04x] - set with a %04x bytes %02x BDs buffer at %08x:%08x\n", + p_ll2->queue_id, p_ll2->cid, p_ll2->input.conn_type, + prod_idx, pkt->first_frag_len, pkt->num_of_bds, + OSAL_LE32_TO_CPU(start_bd->addr.hi), + OSAL_LE32_TO_CPU(start_bd->addr.lo)); + + if (p_ll2->tx_queue.cur_send_frag_num == pkt->num_of_bds) + return; + + /* Need to provide the packet with additional BDs for frags */ + for (frag_idx = p_ll2->tx_queue.cur_send_frag_num; + frag_idx < pkt->num_of_bds; frag_idx++) { + struct core_tx_bd **p_bd = &p_curp->bds_set[frag_idx].txq_bd; + + *p_bd = (struct core_tx_bd *)ecore_chain_produce(p_tx_chain); + (*p_bd)->bd_data.as_bitfield = 0; + (*p_bd)->bitfield1 = 0; + p_curp->bds_set[frag_idx].tx_frag = 0; + p_curp->bds_set[frag_idx].frag_len = 0; + } +} + +/* This should be called while the Txq spinlock is being held */ +static void ecore_ll2_tx_packet_notify(struct ecore_hwfn *p_hwfn, + struct ecore_ll2_info *p_ll2_conn) +{ + bool b_notify = p_ll2_conn->tx_queue.cur_send_packet->notify_fw; + struct ecore_ll2_tx_queue *p_tx = &p_ll2_conn->tx_queue; + struct ecore_ll2_tx_packet *p_pkt = OSAL_NULL; + u16 bd_prod; + + /* If there are missing BDs, don't do anything now */ + if (p_ll2_conn->tx_queue.cur_send_frag_num != + p_ll2_conn->tx_queue.cur_send_packet->bd_used) + return; + + + /* Push the current packet to the list and clean after it */ + OSAL_LIST_PUSH_TAIL(&p_ll2_conn->tx_queue.cur_send_packet->list_entry, + &p_ll2_conn->tx_queue.sending_descq); + p_ll2_conn->tx_queue.cur_send_packet = OSAL_NULL; + p_ll2_conn->tx_queue.cur_send_frag_num = 0; + + /* Notify FW of packet only if requested to */ + if (!b_notify) + return; + + bd_prod = ecore_chain_get_prod_idx(&p_ll2_conn->tx_queue.txq_chain); + + while (!OSAL_LIST_IS_EMPTY(&p_tx->sending_descq)) { + p_pkt = OSAL_LIST_FIRST_ENTRY(&p_tx->sending_descq, + struct ecore_ll2_tx_packet, + list_entry); + if (p_pkt == OSAL_NULL) + break; +#if defined(_NTDDK_) +#pragma warning(suppress : 6011 28182) +#endif + OSAL_LIST_REMOVE_ENTRY(&p_pkt->list_entry, + &p_tx->sending_descq); + OSAL_LIST_PUSH_TAIL(&p_pkt->list_entry, &p_tx->active_descq); + } + + p_tx->db_msg.spq_prod = OSAL_CPU_TO_LE16(bd_prod); + + /* Make sure the BDs data is updated before ringing the doorbell */ + OSAL_WMB(p_hwfn->p_dev); + + //DIRECT_REG_WR(p_hwfn, p_tx->doorbell_addr, *((u32 *)&p_tx->db_msg)); + DIRECT_REG_WR_DB(p_hwfn, p_tx->doorbell_addr, *((u32 *)&p_tx->db_msg)); + + DP_VERBOSE(p_hwfn, (ECORE_MSG_TX_QUEUED | ECORE_MSG_LL2), + "LL2 [q 0x%02x cid 0x%08x type 0x%08x] Doorbelled [producer 0x%04x]\n", + p_ll2_conn->queue_id, p_ll2_conn->cid, + p_ll2_conn->input.conn_type, + p_tx->db_msg.spq_prod); +} + +enum _ecore_status_t ecore_ll2_prepare_tx_packet( + void *cxt, + u8 connection_handle, + struct ecore_ll2_tx_pkt_info *pkt, + bool notify_fw) +{ + struct ecore_hwfn *p_hwfn = (struct ecore_hwfn *)cxt; + struct ecore_ll2_tx_packet *p_curp = OSAL_NULL; + struct ecore_ll2_info *p_ll2_conn = OSAL_NULL; + struct ecore_ll2_tx_queue *p_tx; + struct ecore_chain *p_tx_chain; + unsigned long flags; + enum _ecore_status_t rc = ECORE_SUCCESS; + + p_ll2_conn = ecore_ll2_handle_sanity(p_hwfn, connection_handle); + if (p_ll2_conn == OSAL_NULL) + return ECORE_INVAL; + p_tx = &p_ll2_conn->tx_queue; + p_tx_chain = &p_tx->txq_chain; + + if (pkt->num_of_bds > p_ll2_conn->input.tx_max_bds_per_packet) + return ECORE_IO; /* coalescing is requireed */ + + OSAL_SPIN_LOCK_IRQSAVE(&p_tx->lock, flags); + if (p_tx->cur_send_packet) { + rc = ECORE_EXISTS; + goto out; + } + + /* Get entry, but only if we have tx elements for it */ + if (!OSAL_LIST_IS_EMPTY(&p_tx->free_descq)) + p_curp = OSAL_LIST_FIRST_ENTRY(&p_tx->free_descq, + struct ecore_ll2_tx_packet, + list_entry); + if (p_curp && ecore_chain_get_elem_left(p_tx_chain) < pkt->num_of_bds) + p_curp = OSAL_NULL; + + if (!p_curp) { + rc = ECORE_BUSY; + goto out; + } + + /* Prepare packet and BD, and perhaps send a doorbell to FW */ + ecore_ll2_prepare_tx_packet_set(p_tx, p_curp, pkt, notify_fw); + + ecore_ll2_prepare_tx_packet_set_bd(p_hwfn, p_ll2_conn, p_curp, + pkt); + + ecore_ll2_tx_packet_notify(p_hwfn, p_ll2_conn); + +out: + OSAL_SPIN_UNLOCK_IRQSAVE(&p_tx->lock, flags); + return rc; +} + +enum _ecore_status_t ecore_ll2_set_fragment_of_tx_packet(void *cxt, + u8 connection_handle, + dma_addr_t addr, + u16 nbytes) +{ + struct ecore_ll2_tx_packet *p_cur_send_packet = OSAL_NULL; + struct ecore_hwfn *p_hwfn = (struct ecore_hwfn *)cxt; + struct ecore_ll2_info *p_ll2_conn = OSAL_NULL; + u16 cur_send_frag_num = 0; + struct core_tx_bd *p_bd; + unsigned long flags; + + p_ll2_conn = ecore_ll2_handle_sanity(p_hwfn, connection_handle); + if (p_ll2_conn == OSAL_NULL) + return ECORE_INVAL; + + if (!p_ll2_conn->tx_queue.cur_send_packet) + return ECORE_INVAL; + + p_cur_send_packet = p_ll2_conn->tx_queue.cur_send_packet; + cur_send_frag_num = p_ll2_conn->tx_queue.cur_send_frag_num; + + if (cur_send_frag_num >= p_cur_send_packet->bd_used) + return ECORE_INVAL; + + /* Fill the BD information, and possibly notify FW */ + p_bd = p_cur_send_packet->bds_set[cur_send_frag_num].txq_bd; + DMA_REGPAIR_LE(p_bd->addr, addr); + p_bd->nbytes = OSAL_CPU_TO_LE16(nbytes); + p_cur_send_packet->bds_set[cur_send_frag_num].tx_frag = addr; + p_cur_send_packet->bds_set[cur_send_frag_num].frag_len = nbytes; + + p_ll2_conn->tx_queue.cur_send_frag_num++; + + OSAL_SPIN_LOCK_IRQSAVE(&p_ll2_conn->tx_queue.lock, flags); + ecore_ll2_tx_packet_notify(p_hwfn, p_ll2_conn); + OSAL_SPIN_UNLOCK_IRQSAVE(&p_ll2_conn->tx_queue.lock, flags); + + return ECORE_SUCCESS; +} + +enum _ecore_status_t ecore_ll2_terminate_connection(void *cxt, + u8 connection_handle) +{ + struct ecore_hwfn *p_hwfn = (struct ecore_hwfn *)cxt; + struct ecore_ll2_info *p_ll2_conn = OSAL_NULL; + enum _ecore_status_t rc = ECORE_NOTIMPL; + struct ecore_ptt *p_ptt; + + p_ptt = ecore_ptt_acquire(p_hwfn); + if (!p_ptt) + return ECORE_AGAIN; + + p_ll2_conn = ecore_ll2_handle_sanity_lock(p_hwfn, connection_handle); + if (p_ll2_conn == OSAL_NULL) { + rc = ECORE_INVAL; + goto out; + } + + /* Stop Tx & Rx of connection, if needed */ + if (ECORE_LL2_TX_REGISTERED(p_ll2_conn)) { + rc = ecore_sp_ll2_tx_queue_stop(p_hwfn, p_ll2_conn); + if (rc != ECORE_SUCCESS) + goto out; + ecore_ll2_txq_flush(p_hwfn, connection_handle); + } + + if (ECORE_LL2_RX_REGISTERED(p_ll2_conn)) { + rc = ecore_sp_ll2_rx_queue_stop(p_hwfn, p_ll2_conn); + if (rc) + goto out; + ecore_ll2_rxq_flush(p_hwfn, connection_handle); + } + + if (p_ll2_conn->input.conn_type == ECORE_LL2_TYPE_OOO) + ecore_ooo_release_all_isles(p_hwfn->p_ooo_info); + + if (p_ll2_conn->input.conn_type == ECORE_LL2_TYPE_FCOE) { + if (!OSAL_TEST_BIT(ECORE_MF_UFP_SPECIFIC, + &p_hwfn->p_dev->mf_bits)) + ecore_llh_remove_protocol_filter(p_hwfn->p_dev, 0, + ECORE_LLH_FILTER_ETHERTYPE, + 0x8906, 0); + ecore_llh_remove_protocol_filter(p_hwfn->p_dev, 0, + ECORE_LLH_FILTER_ETHERTYPE, + 0x8914, 0); + } + +out: + ecore_ptt_release(p_hwfn, p_ptt); + + return rc; +} + +static void ecore_ll2_release_connection_ooo(struct ecore_hwfn *p_hwfn, + struct ecore_ll2_info *p_ll2_conn) +{ + struct ecore_ooo_buffer *p_buffer; + + if (p_ll2_conn->input.conn_type != ECORE_LL2_TYPE_OOO) + return; + + ecore_ooo_release_all_isles(p_hwfn->p_ooo_info); + while ((p_buffer = ecore_ooo_get_free_buffer(p_hwfn->p_ooo_info))) { + OSAL_DMA_FREE_COHERENT(p_hwfn->p_dev, + p_buffer->rx_buffer_virt_addr, + p_buffer->rx_buffer_phys_addr, + p_buffer->rx_buffer_size); + OSAL_FREE(p_hwfn->p_dev, p_buffer); + } +} + +void ecore_ll2_release_connection(void *cxt, + u8 connection_handle) +{ + struct ecore_hwfn *p_hwfn = (struct ecore_hwfn *)cxt; + struct ecore_ll2_info *p_ll2_conn = OSAL_NULL; + + p_ll2_conn = ecore_ll2_handle_sanity(p_hwfn, connection_handle); + if (p_ll2_conn == OSAL_NULL) + return; + + if (ECORE_LL2_RX_REGISTERED(p_ll2_conn)) { + p_ll2_conn->rx_queue.b_cb_registred = false; + ecore_int_unregister_cb(p_hwfn, + p_ll2_conn->rx_queue.rx_sb_index); + } + + if (ECORE_LL2_TX_REGISTERED(p_ll2_conn)) { + p_ll2_conn->tx_queue.b_cb_registred = false; + ecore_int_unregister_cb(p_hwfn, + p_ll2_conn->tx_queue.tx_sb_index); + } + + OSAL_FREE(p_hwfn->p_dev, p_ll2_conn->tx_queue.descq_array); + ecore_chain_free(p_hwfn->p_dev, &p_ll2_conn->tx_queue.txq_chain); + + OSAL_FREE(p_hwfn->p_dev, p_ll2_conn->rx_queue.descq_array); + ecore_chain_free(p_hwfn->p_dev, &p_ll2_conn->rx_queue.rxq_chain); + ecore_chain_free(p_hwfn->p_dev, &p_ll2_conn->rx_queue.rcq_chain); + + ecore_cxt_release_cid(p_hwfn, p_ll2_conn->cid); + + ecore_ll2_release_connection_ooo(p_hwfn, p_ll2_conn); + + OSAL_MUTEX_ACQUIRE(&p_ll2_conn->mutex); + p_ll2_conn->b_active = false; + OSAL_MUTEX_RELEASE(&p_ll2_conn->mutex); +} + +/* ECORE LL2: internal functions */ + +enum _ecore_status_t ecore_ll2_alloc(struct ecore_hwfn *p_hwfn) +{ + struct ecore_ll2_info *p_ll2_info; + u8 i; + + /* Allocate LL2's set struct */ + p_ll2_info = OSAL_ZALLOC(p_hwfn->p_dev, GFP_KERNEL, + sizeof(struct ecore_ll2_info) * + ECORE_MAX_NUM_OF_LL2_CONNECTIONS); + if (!p_ll2_info) { + DP_NOTICE(p_hwfn, false, + "Failed to allocate `struct ecore_ll2'\n"); + return ECORE_NOMEM; + } + + p_hwfn->p_ll2_info = p_ll2_info; + + for (i = 0; i < ECORE_MAX_NUM_OF_LL2_CONNECTIONS; i++) { +#ifdef CONFIG_ECORE_LOCK_ALLOC + if (OSAL_MUTEX_ALLOC(p_hwfn, &p_ll2_info[i].mutex)) + goto handle_err; + if (OSAL_SPIN_LOCK_ALLOC(p_hwfn, &p_ll2_info[i].rx_queue.lock)) + goto handle_err; + if (OSAL_SPIN_LOCK_ALLOC(p_hwfn, &p_ll2_info[i].tx_queue.lock)) + goto handle_err; +#endif + p_ll2_info[i].my_id = i; + } + + return ECORE_SUCCESS; +#ifdef CONFIG_ECORE_LOCK_ALLOC +handle_err: + ecore_ll2_free(p_hwfn); + return ECORE_NOMEM; +#endif +} + +void ecore_ll2_setup(struct ecore_hwfn *p_hwfn) +{ + int i; + + for (i = 0; i < ECORE_MAX_NUM_OF_LL2_CONNECTIONS; i++) + OSAL_MUTEX_INIT(&p_hwfn->p_ll2_info[i].mutex); +} + +void ecore_ll2_free(struct ecore_hwfn *p_hwfn) +{ +#ifdef CONFIG_ECORE_LOCK_ALLOC + int i; +#endif + if (!p_hwfn->p_ll2_info) + return; + +#ifdef CONFIG_ECORE_LOCK_ALLOC + for (i = 0; i < ECORE_MAX_NUM_OF_LL2_CONNECTIONS; i++) { + OSAL_SPIN_LOCK_DEALLOC(&p_hwfn->p_ll2_info[i].rx_queue.lock); + OSAL_SPIN_LOCK_DEALLOC(&p_hwfn->p_ll2_info[i].tx_queue.lock); + OSAL_MUTEX_DEALLOC(&p_hwfn->p_ll2_info[i].mutex); + } +#endif + OSAL_FREE(p_hwfn->p_dev, p_hwfn->p_ll2_info); + p_hwfn->p_ll2_info = OSAL_NULL; +} + +static void _ecore_ll2_get_port_stats(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt, + struct ecore_ll2_stats *p_stats) +{ + struct core_ll2_port_stats port_stats; + + OSAL_MEMSET(&port_stats, 0, sizeof(port_stats)); + ecore_memcpy_from(p_hwfn, p_ptt, &port_stats, + BAR0_MAP_REG_TSDM_RAM + + TSTORM_LL2_PORT_STAT_OFFSET(MFW_PORT(p_hwfn)), + sizeof(port_stats)); + + p_stats->gsi_invalid_hdr += + HILO_64_REGPAIR(port_stats.gsi_invalid_hdr); + p_stats->gsi_invalid_pkt_length += + HILO_64_REGPAIR(port_stats.gsi_invalid_pkt_length); + p_stats->gsi_unsupported_pkt_typ += + HILO_64_REGPAIR(port_stats.gsi_unsupported_pkt_typ); + p_stats->gsi_crcchksm_error += + HILO_64_REGPAIR(port_stats.gsi_crcchksm_error); +} + +static void _ecore_ll2_get_tstats(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt, + struct ecore_ll2_info *p_ll2_conn, + struct ecore_ll2_stats *p_stats) +{ + struct core_ll2_tstorm_per_queue_stat tstats; + u8 qid = p_ll2_conn->queue_id; + u32 tstats_addr; + + OSAL_MEMSET(&tstats, 0, sizeof(tstats)); + tstats_addr = BAR0_MAP_REG_TSDM_RAM + + CORE_LL2_TSTORM_PER_QUEUE_STAT_OFFSET(qid); + ecore_memcpy_from(p_hwfn, p_ptt, &tstats, + tstats_addr, + sizeof(tstats)); + + p_stats->packet_too_big_discard += + HILO_64_REGPAIR(tstats.packet_too_big_discard); + p_stats->no_buff_discard += + HILO_64_REGPAIR(tstats.no_buff_discard); +} + +static void _ecore_ll2_get_ustats(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt, + struct ecore_ll2_info *p_ll2_conn, + struct ecore_ll2_stats *p_stats) +{ + struct core_ll2_ustorm_per_queue_stat ustats; + u8 qid = p_ll2_conn->queue_id; + u32 ustats_addr; + + OSAL_MEMSET(&ustats, 0, sizeof(ustats)); + ustats_addr = BAR0_MAP_REG_USDM_RAM + + CORE_LL2_USTORM_PER_QUEUE_STAT_OFFSET(qid); + ecore_memcpy_from(p_hwfn, p_ptt, &ustats, + ustats_addr, + sizeof(ustats)); + + p_stats->rcv_ucast_bytes += HILO_64_REGPAIR(ustats.rcv_ucast_bytes); + p_stats->rcv_mcast_bytes += HILO_64_REGPAIR(ustats.rcv_mcast_bytes); + p_stats->rcv_bcast_bytes += HILO_64_REGPAIR(ustats.rcv_bcast_bytes); + p_stats->rcv_ucast_pkts += HILO_64_REGPAIR(ustats.rcv_ucast_pkts); + p_stats->rcv_mcast_pkts += HILO_64_REGPAIR(ustats.rcv_mcast_pkts); + p_stats->rcv_bcast_pkts += HILO_64_REGPAIR(ustats.rcv_bcast_pkts); +} + +static void _ecore_ll2_get_pstats(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt, + struct ecore_ll2_info *p_ll2_conn, + struct ecore_ll2_stats *p_stats) +{ + struct core_ll2_pstorm_per_queue_stat pstats; + u8 stats_id = p_ll2_conn->tx_stats_id; + u32 pstats_addr; + + OSAL_MEMSET(&pstats, 0, sizeof(pstats)); + pstats_addr = BAR0_MAP_REG_PSDM_RAM + + CORE_LL2_PSTORM_PER_QUEUE_STAT_OFFSET(stats_id); + ecore_memcpy_from(p_hwfn, p_ptt, &pstats, + pstats_addr, + sizeof(pstats)); + + p_stats->sent_ucast_bytes += HILO_64_REGPAIR(pstats.sent_ucast_bytes); + p_stats->sent_mcast_bytes += HILO_64_REGPAIR(pstats.sent_mcast_bytes); + p_stats->sent_bcast_bytes += HILO_64_REGPAIR(pstats.sent_bcast_bytes); + p_stats->sent_ucast_pkts += HILO_64_REGPAIR(pstats.sent_ucast_pkts); + p_stats->sent_mcast_pkts += HILO_64_REGPAIR(pstats.sent_mcast_pkts); + p_stats->sent_bcast_pkts += HILO_64_REGPAIR(pstats.sent_bcast_pkts); +} + +enum _ecore_status_t __ecore_ll2_get_stats(void *cxt, + u8 connection_handle, + struct ecore_ll2_stats *p_stats) +{ + struct ecore_hwfn *p_hwfn = (struct ecore_hwfn *)cxt; + struct ecore_ll2_info *p_ll2_conn = OSAL_NULL; + struct ecore_ptt *p_ptt; + + if ((connection_handle >= ECORE_MAX_NUM_OF_LL2_CONNECTIONS) || + !p_hwfn->p_ll2_info) { + return ECORE_INVAL; + } + + p_ll2_conn = &p_hwfn->p_ll2_info[connection_handle]; + + p_ptt = ecore_ptt_acquire(p_hwfn); + if (!p_ptt) { + DP_ERR(p_hwfn, "Failed to acquire ptt\n"); + return ECORE_INVAL; + } + + if (p_ll2_conn->input.gsi_enable) + _ecore_ll2_get_port_stats(p_hwfn, p_ptt, p_stats); + + _ecore_ll2_get_tstats(p_hwfn, p_ptt, p_ll2_conn, p_stats); + + _ecore_ll2_get_ustats(p_hwfn, p_ptt, p_ll2_conn, p_stats); + + if (p_ll2_conn->tx_stats_en) + _ecore_ll2_get_pstats(p_hwfn, p_ptt, p_ll2_conn, p_stats); + + ecore_ptt_release(p_hwfn, p_ptt); + + return ECORE_SUCCESS; +} + +enum _ecore_status_t ecore_ll2_get_stats(void *cxt, + u8 connection_handle, + struct ecore_ll2_stats *p_stats) +{ + OSAL_MEMSET(p_stats, 0, sizeof(*p_stats)); + + return __ecore_ll2_get_stats(cxt, connection_handle, p_stats); +} + +/**/ + +#ifdef _NTDDK_ +#pragma warning(pop) +#endif Property changes on: head/sys/dev/qlnx/qlnxe/ecore_ll2.c ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Added: svn:mime-type ## -0,0 +1 ## +text/plain \ No newline at end of property Index: head/sys/dev/qlnx/qlnxe/ecore_ooo.c =================================================================== --- head/sys/dev/qlnx/qlnxe/ecore_ooo.c (nonexistent) +++ head/sys/dev/qlnx/qlnxe/ecore_ooo.c (revision 343598) @@ -0,0 +1,603 @@ +/* + * Copyright (c) 2018-2019 Cavium, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +/* + * File : ecore_ooo.c + */ +#include +__FBSDID("$FreeBSD$"); + +#include "bcm_osal.h" + +#include "ecore.h" +#include "ecore_status.h" +#include "ecore_ll2.h" +#include "ecore_ooo.h" +#include "ecore_iscsi.h" +#include "ecore_cxt.h" +/* + * Static OOO functions + */ + +static struct ecore_ooo_archipelago * +ecore_ooo_seek_archipelago(struct ecore_ooo_info *p_ooo_info, u32 cid) +{ + u32 idx = (cid & 0xffff) - p_ooo_info->cid_base; + struct ecore_ooo_archipelago *p_archipelago; + + if (idx >= p_ooo_info->max_num_archipelagos) + return OSAL_NULL; + + p_archipelago = &p_ooo_info->p_archipelagos_mem[idx]; + + if (OSAL_LIST_IS_EMPTY(&p_archipelago->isles_list)) + return OSAL_NULL; + + return p_archipelago; +} + +static struct ecore_ooo_isle *ecore_ooo_seek_isle(struct ecore_hwfn *p_hwfn, + struct ecore_ooo_info *p_ooo_info, + u32 cid, u8 isle) +{ + struct ecore_ooo_archipelago *p_archipelago = OSAL_NULL; + struct ecore_ooo_isle *p_isle = OSAL_NULL; + u8 the_num_of_isle = 1; + + p_archipelago = ecore_ooo_seek_archipelago(p_ooo_info, cid); + if (!p_archipelago) { + DP_NOTICE(p_hwfn, true, + "Connection %d is not found in OOO list\n", cid); + return OSAL_NULL; + } + + OSAL_LIST_FOR_EACH_ENTRY(p_isle, + &p_archipelago->isles_list, + list_entry, struct ecore_ooo_isle) { + if (the_num_of_isle == isle) + return p_isle; + the_num_of_isle++; + } + + return OSAL_NULL; +} + +void ecore_ooo_save_history_entry(struct ecore_ooo_info *p_ooo_info, + struct ooo_opaque *p_cqe) +{ + struct ecore_ooo_history *p_history = &p_ooo_info->ooo_history; + + if (p_history->head_idx == p_history->num_of_cqes) + p_history->head_idx = 0; + p_history->p_cqes[p_history->head_idx] = *p_cqe; + p_history->head_idx++; +} + +//#ifdef CONFIG_ECORE_ISCSI +#if defined(CONFIG_ECORE_ISCSI) || defined(CONFIG_ECORE_IWARP) +enum _ecore_status_t ecore_ooo_alloc(struct ecore_hwfn *p_hwfn) +{ + u16 max_num_archipelagos = 0, cid_base; + struct ecore_ooo_info *p_ooo_info; + u16 max_num_isles = 0; + u32 i; + + switch (p_hwfn->hw_info.personality) { + case ECORE_PCI_ISCSI: + max_num_archipelagos = + p_hwfn->pf_params.iscsi_pf_params.num_cons; + cid_base =(u16)ecore_cxt_get_proto_cid_start(p_hwfn, + PROTOCOLID_ISCSI); + break; + case ECORE_PCI_ETH_RDMA: + case ECORE_PCI_ETH_IWARP: + max_num_archipelagos = + (u16)ecore_cxt_get_proto_cid_count(p_hwfn, + PROTOCOLID_IWARP, + OSAL_NULL); + cid_base = (u16)ecore_cxt_get_proto_cid_start(p_hwfn, + PROTOCOLID_IWARP); + break; + default: + DP_NOTICE(p_hwfn, true, + "Failed to allocate ecore_ooo_info: unknown personalization\n"); + return ECORE_INVAL; + } + + max_num_isles = ECORE_MAX_NUM_ISLES + max_num_archipelagos; + + if (!max_num_archipelagos) { + DP_NOTICE(p_hwfn, true, + "Failed to allocate ecore_ooo_info: unknown amount of connections\n"); + return ECORE_INVAL; + } + + p_ooo_info = OSAL_ZALLOC(p_hwfn->p_dev, GFP_KERNEL, + sizeof(*p_ooo_info)); + if (!p_ooo_info) { + DP_NOTICE(p_hwfn, true, "Failed to allocate ecore_ooo_info\n"); + return ECORE_NOMEM; + } + p_ooo_info->cid_base = cid_base; /* We look only at the icid */ + p_ooo_info->max_num_archipelagos = max_num_archipelagos; + + OSAL_LIST_INIT(&p_ooo_info->free_buffers_list); + OSAL_LIST_INIT(&p_ooo_info->ready_buffers_list); + OSAL_LIST_INIT(&p_ooo_info->free_isles_list); + + p_ooo_info->p_isles_mem = + OSAL_ZALLOC(p_hwfn->p_dev, GFP_KERNEL, + sizeof(struct ecore_ooo_isle) * + max_num_isles); + if (!p_ooo_info->p_isles_mem) { + DP_NOTICE(p_hwfn,true, + "Failed to allocate ecore_ooo_info (isles)\n"); + goto no_isles_mem; + } + + for (i = 0; i < max_num_isles; i++) { + OSAL_LIST_INIT(&p_ooo_info->p_isles_mem[i].buffers_list); + OSAL_LIST_PUSH_TAIL(&p_ooo_info->p_isles_mem[i].list_entry, + &p_ooo_info->free_isles_list); + } + + p_ooo_info->p_archipelagos_mem = + OSAL_ZALLOC(p_hwfn->p_dev, GFP_KERNEL, + sizeof(struct ecore_ooo_archipelago) * + max_num_archipelagos); + if (!p_ooo_info->p_archipelagos_mem) { + DP_NOTICE(p_hwfn,true, + "Failed to allocate ecore_ooo_info(archpelagos)\n"); + goto no_archipelagos_mem; + } + + for (i = 0; i < max_num_archipelagos; i++) { + OSAL_LIST_INIT(&p_ooo_info->p_archipelagos_mem[i].isles_list); + } + + p_ooo_info->ooo_history.p_cqes = + OSAL_ZALLOC(p_hwfn->p_dev, GFP_KERNEL, + sizeof(struct ooo_opaque) * + ECORE_MAX_NUM_OOO_HISTORY_ENTRIES); + if (!p_ooo_info->ooo_history.p_cqes) { + DP_NOTICE(p_hwfn,true, + "Failed to allocate ecore_ooo_info(history)\n"); + goto no_history_mem; + } + p_ooo_info->ooo_history.num_of_cqes = + ECORE_MAX_NUM_OOO_HISTORY_ENTRIES; + + p_hwfn->p_ooo_info = p_ooo_info; + return ECORE_SUCCESS; + +no_history_mem: + OSAL_FREE(p_hwfn->p_dev, p_ooo_info->p_archipelagos_mem); +no_archipelagos_mem: + OSAL_FREE(p_hwfn->p_dev, p_ooo_info->p_isles_mem); +no_isles_mem: + OSAL_FREE(p_hwfn->p_dev, p_ooo_info); + return ECORE_NOMEM; +} +#endif + +void ecore_ooo_release_connection_isles(struct ecore_ooo_info *p_ooo_info, + u32 cid) +{ + struct ecore_ooo_archipelago *p_archipelago; + struct ecore_ooo_buffer *p_buffer; + struct ecore_ooo_isle *p_isle; + + p_archipelago = ecore_ooo_seek_archipelago(p_ooo_info, cid); + if (!p_archipelago) + return; + + while (!OSAL_LIST_IS_EMPTY(&p_archipelago->isles_list)) { + p_isle = OSAL_LIST_FIRST_ENTRY( + &p_archipelago->isles_list, + struct ecore_ooo_isle, list_entry); + +#if defined(_NTDDK_) +#pragma warning(suppress : 6011 28182) +#endif + OSAL_LIST_REMOVE_ENTRY(&p_isle->list_entry, + &p_archipelago->isles_list); + + while (!OSAL_LIST_IS_EMPTY(&p_isle->buffers_list)) { + p_buffer = + OSAL_LIST_FIRST_ENTRY( + &p_isle->buffers_list , + struct ecore_ooo_buffer, list_entry); + + if (p_buffer == OSAL_NULL) + break; +#if defined(_NTDDK_) +#pragma warning(suppress : 6011 28182) +#endif + OSAL_LIST_REMOVE_ENTRY(&p_buffer->list_entry, + &p_isle->buffers_list); + OSAL_LIST_PUSH_TAIL(&p_buffer->list_entry, + &p_ooo_info->free_buffers_list); + } + OSAL_LIST_PUSH_TAIL(&p_isle->list_entry, + &p_ooo_info->free_isles_list); + } + +} + +void ecore_ooo_release_all_isles(struct ecore_ooo_info *p_ooo_info) +{ + struct ecore_ooo_archipelago *p_archipelago; + struct ecore_ooo_buffer *p_buffer; + struct ecore_ooo_isle *p_isle; + u32 i; + + for (i = 0; i < p_ooo_info->max_num_archipelagos; i++) { + p_archipelago = &(p_ooo_info->p_archipelagos_mem[i]); + +#if defined(_NTDDK_) +#pragma warning(suppress : 6011 28182) +#endif + while (!OSAL_LIST_IS_EMPTY(&p_archipelago->isles_list)) { + p_isle = OSAL_LIST_FIRST_ENTRY( + &p_archipelago->isles_list, + struct ecore_ooo_isle, list_entry); + +#if defined(_NTDDK_) +#pragma warning(suppress : 6011 28182) +#endif + OSAL_LIST_REMOVE_ENTRY(&p_isle->list_entry, + &p_archipelago->isles_list); + + while (!OSAL_LIST_IS_EMPTY(&p_isle->buffers_list)) { + p_buffer = + OSAL_LIST_FIRST_ENTRY( + &p_isle->buffers_list , + struct ecore_ooo_buffer, list_entry); + + if (p_buffer == OSAL_NULL) + break; +#if defined(_NTDDK_) +#pragma warning(suppress : 6011 28182) +#endif + OSAL_LIST_REMOVE_ENTRY(&p_buffer->list_entry, + &p_isle->buffers_list); + OSAL_LIST_PUSH_TAIL(&p_buffer->list_entry, + &p_ooo_info->free_buffers_list); + } + OSAL_LIST_PUSH_TAIL(&p_isle->list_entry, + &p_ooo_info->free_isles_list); + } + } + if (!OSAL_LIST_IS_EMPTY(&p_ooo_info->ready_buffers_list)) { + OSAL_LIST_SPLICE_TAIL_INIT(&p_ooo_info->ready_buffers_list, + &p_ooo_info->free_buffers_list); + } +} + +//#ifdef CONFIG_ECORE_ISCSI +#if defined(CONFIG_ECORE_ISCSI) || defined(CONFIG_ECORE_IWARP) +void ecore_ooo_setup(struct ecore_hwfn *p_hwfn) +{ + ecore_ooo_release_all_isles(p_hwfn->p_ooo_info); + OSAL_MEM_ZERO(p_hwfn->p_ooo_info->ooo_history.p_cqes, + p_hwfn->p_ooo_info->ooo_history.num_of_cqes * + sizeof(struct ooo_opaque)); + p_hwfn->p_ooo_info->ooo_history.head_idx = 0; +} + +void ecore_ooo_free(struct ecore_hwfn *p_hwfn) +{ + struct ecore_ooo_info *p_ooo_info = p_hwfn->p_ooo_info; + struct ecore_ooo_buffer *p_buffer; + + if (!p_ooo_info) + return; + + ecore_ooo_release_all_isles(p_ooo_info); + while (!OSAL_LIST_IS_EMPTY(&p_ooo_info->free_buffers_list)) { + p_buffer = OSAL_LIST_FIRST_ENTRY(&p_ooo_info-> + free_buffers_list, + struct ecore_ooo_buffer, + list_entry); + if (p_buffer == OSAL_NULL) + break; +#if defined(_NTDDK_) +#pragma warning(suppress : 6011 28182) +#endif + OSAL_LIST_REMOVE_ENTRY(&p_buffer->list_entry, + &p_ooo_info->free_buffers_list); + OSAL_DMA_FREE_COHERENT(p_hwfn->p_dev, + p_buffer->rx_buffer_virt_addr, + p_buffer->rx_buffer_phys_addr, + p_buffer->rx_buffer_size); + OSAL_FREE(p_hwfn->p_dev, p_buffer); + } + + OSAL_FREE(p_hwfn->p_dev, p_ooo_info->p_isles_mem); + OSAL_FREE(p_hwfn->p_dev, p_ooo_info->p_archipelagos_mem); + OSAL_FREE(p_hwfn->p_dev, p_ooo_info->ooo_history.p_cqes); + OSAL_FREE(p_hwfn->p_dev, p_ooo_info); + p_hwfn->p_ooo_info = OSAL_NULL; +} +#endif + +void ecore_ooo_put_free_buffer(struct ecore_ooo_info *p_ooo_info, + struct ecore_ooo_buffer *p_buffer) +{ + OSAL_LIST_PUSH_TAIL(&p_buffer->list_entry, + &p_ooo_info->free_buffers_list); +} + +struct ecore_ooo_buffer * +ecore_ooo_get_free_buffer(struct ecore_ooo_info *p_ooo_info) +{ + struct ecore_ooo_buffer *p_buffer = OSAL_NULL; + + if (!OSAL_LIST_IS_EMPTY(&p_ooo_info->free_buffers_list)) { + p_buffer = + OSAL_LIST_FIRST_ENTRY( + &p_ooo_info->free_buffers_list, + struct ecore_ooo_buffer, list_entry); + + OSAL_LIST_REMOVE_ENTRY(&p_buffer->list_entry, + &p_ooo_info->free_buffers_list); + } + + return p_buffer; +} + +void ecore_ooo_put_ready_buffer(struct ecore_ooo_info *p_ooo_info, + struct ecore_ooo_buffer *p_buffer, u8 on_tail) +{ + if (on_tail) { + OSAL_LIST_PUSH_TAIL(&p_buffer->list_entry, + &p_ooo_info->ready_buffers_list); + } else { + OSAL_LIST_PUSH_HEAD(&p_buffer->list_entry, + &p_ooo_info->ready_buffers_list); + } +} + +struct ecore_ooo_buffer * +ecore_ooo_get_ready_buffer(struct ecore_ooo_info *p_ooo_info) +{ + struct ecore_ooo_buffer *p_buffer = OSAL_NULL; + + if (!OSAL_LIST_IS_EMPTY(&p_ooo_info->ready_buffers_list)) { + p_buffer = + OSAL_LIST_FIRST_ENTRY( + &p_ooo_info->ready_buffers_list, + struct ecore_ooo_buffer, list_entry); + + OSAL_LIST_REMOVE_ENTRY(&p_buffer->list_entry, + &p_ooo_info->ready_buffers_list); + } + + return p_buffer; +} + +void ecore_ooo_delete_isles(struct ecore_hwfn *p_hwfn, + struct ecore_ooo_info *p_ooo_info, + u32 cid, + u8 drop_isle, + u8 drop_size) +{ + struct ecore_ooo_archipelago *p_archipelago = OSAL_NULL; + struct ecore_ooo_isle *p_isle = OSAL_NULL; + u8 isle_idx; + + p_archipelago = ecore_ooo_seek_archipelago(p_ooo_info, cid); + for (isle_idx = 0; isle_idx < drop_size; isle_idx++) { + p_isle = ecore_ooo_seek_isle(p_hwfn, p_ooo_info, + cid, drop_isle); + if (!p_isle) { + DP_NOTICE(p_hwfn, true, + "Isle %d is not found(cid %d)\n", + drop_isle, cid); + return; + } + if (OSAL_LIST_IS_EMPTY(&p_isle->buffers_list)) { + DP_NOTICE(p_hwfn, true, + "Isle %d is empty(cid %d)\n", + drop_isle, cid); + } else { + OSAL_LIST_SPLICE_TAIL_INIT(&p_isle->buffers_list, + &p_ooo_info->free_buffers_list); + } +#if defined(_NTDDK_) +#pragma warning(suppress : 6011) +#endif + OSAL_LIST_REMOVE_ENTRY(&p_isle->list_entry, + &p_archipelago->isles_list); + p_ooo_info->cur_isles_number--; + OSAL_LIST_PUSH_HEAD(&p_isle->list_entry, + &p_ooo_info->free_isles_list); + } +} + +void ecore_ooo_add_new_isle(struct ecore_hwfn *p_hwfn, + struct ecore_ooo_info *p_ooo_info, + u32 cid, u8 ooo_isle, + struct ecore_ooo_buffer *p_buffer) +{ + struct ecore_ooo_archipelago *p_archipelago = OSAL_NULL; + struct ecore_ooo_isle *p_prev_isle = OSAL_NULL; + struct ecore_ooo_isle *p_isle = OSAL_NULL; + + if (ooo_isle > 1) { + p_prev_isle = ecore_ooo_seek_isle(p_hwfn, p_ooo_info, cid, ooo_isle - 1); + if (!p_prev_isle) { + DP_NOTICE(p_hwfn, true, + "Isle %d is not found(cid %d)\n", + ooo_isle - 1, cid); + return; + } + } + p_archipelago = ecore_ooo_seek_archipelago(p_ooo_info, cid); + if (!p_archipelago && (ooo_isle != 1)) { + DP_NOTICE(p_hwfn, true, + "Connection %d is not found in OOO list\n", cid); + return; + } + + if (!OSAL_LIST_IS_EMPTY(&p_ooo_info->free_isles_list)) { + p_isle = + OSAL_LIST_FIRST_ENTRY( + &p_ooo_info->free_isles_list, + struct ecore_ooo_isle, list_entry); + + OSAL_LIST_REMOVE_ENTRY(&p_isle->list_entry, + &p_ooo_info->free_isles_list); + if (!OSAL_LIST_IS_EMPTY(&p_isle->buffers_list)) { + DP_NOTICE(p_hwfn, true, "Free isle is not empty\n"); + OSAL_LIST_INIT(&p_isle->buffers_list); + } + } else { + DP_NOTICE(p_hwfn, true, "No more free isles\n"); + return; + } + + if (!p_archipelago) { + u32 idx = (cid & 0xffff) - p_ooo_info->cid_base; + + p_archipelago = &p_ooo_info->p_archipelagos_mem[idx]; + } + OSAL_LIST_PUSH_HEAD(&p_buffer->list_entry, &p_isle->buffers_list); + p_ooo_info->cur_isles_number++; + p_ooo_info->gen_isles_number++; + if (p_ooo_info->cur_isles_number > p_ooo_info->max_isles_number) + p_ooo_info->max_isles_number = p_ooo_info->cur_isles_number; + if (!p_prev_isle) { + OSAL_LIST_PUSH_HEAD(&p_isle->list_entry, &p_archipelago->isles_list); + } else { + OSAL_LIST_INSERT_ENTRY_AFTER(&p_isle->list_entry, + &p_prev_isle->list_entry, + &p_archipelago->isles_list); + } +} + +void ecore_ooo_add_new_buffer(struct ecore_hwfn *p_hwfn, + struct ecore_ooo_info *p_ooo_info, + u32 cid, + u8 ooo_isle, + struct ecore_ooo_buffer *p_buffer, + u8 buffer_side) +{ + struct ecore_ooo_isle * p_isle = OSAL_NULL; + p_isle = ecore_ooo_seek_isle(p_hwfn, p_ooo_info, cid, ooo_isle); + if (!p_isle) { + DP_NOTICE(p_hwfn, true, + "Isle %d is not found(cid %d)\n", + ooo_isle, cid); + return; + } + if (buffer_side == ECORE_OOO_LEFT_BUF) { + OSAL_LIST_PUSH_HEAD(&p_buffer->list_entry, + &p_isle->buffers_list); + } else { + OSAL_LIST_PUSH_TAIL(&p_buffer->list_entry, + &p_isle->buffers_list); + } +} + +void ecore_ooo_join_isles(struct ecore_hwfn *p_hwfn, + struct ecore_ooo_info *p_ooo_info, + u32 cid, u8 left_isle) +{ + struct ecore_ooo_archipelago *p_archipelago = OSAL_NULL; + struct ecore_ooo_isle *p_right_isle = OSAL_NULL; + struct ecore_ooo_isle *p_left_isle = OSAL_NULL; + + p_right_isle = ecore_ooo_seek_isle(p_hwfn, p_ooo_info, cid, + left_isle + 1); + if (!p_right_isle) { + DP_NOTICE(p_hwfn, true, + "Right isle %d is not found(cid %d)\n", + left_isle + 1, cid); + return; + } + p_archipelago = ecore_ooo_seek_archipelago(p_ooo_info, cid); + OSAL_LIST_REMOVE_ENTRY(&p_right_isle->list_entry, + &p_archipelago->isles_list); + p_ooo_info->cur_isles_number--; + if (left_isle) { + p_left_isle = ecore_ooo_seek_isle(p_hwfn, p_ooo_info, cid, + left_isle); + if (!p_left_isle) { + DP_NOTICE(p_hwfn, true, + "Left isle %d is not found(cid %d)\n", + left_isle, cid); + return; + } + OSAL_LIST_SPLICE_TAIL_INIT(&p_right_isle->buffers_list, + &p_left_isle->buffers_list); + } else { + OSAL_LIST_SPLICE_TAIL_INIT(&p_right_isle->buffers_list, + &p_ooo_info->ready_buffers_list); + } + OSAL_LIST_PUSH_TAIL(&p_right_isle->list_entry, + &p_ooo_info->free_isles_list); +} + +void ecore_ooo_dump_rx_event(struct ecore_hwfn *p_hwfn, + struct ooo_opaque *iscsi_ooo, + struct ecore_ooo_buffer *p_buffer) +{ + int i; + u32 dp_module = ECORE_MSG_OOO; + u32 ph_hi, ph_lo; + u8 *packet_buffer = 0; + + if (p_hwfn->dp_level > ECORE_LEVEL_VERBOSE) + return; + if (!(p_hwfn->dp_module & dp_module)) + return; + + packet_buffer = (u8 *)p_buffer->rx_buffer_virt_addr + + p_buffer->placement_offset; + DP_VERBOSE(p_hwfn, dp_module, + "******************************************************\n"); + ph_hi = DMA_HI(p_buffer->rx_buffer_phys_addr); + ph_lo = DMA_LO(p_buffer->rx_buffer_phys_addr); + DP_VERBOSE(p_hwfn, dp_module, + "0x%x-%x: CID 0x%x, OP 0x%x, ISLE 0x%x\n", + ph_hi, ph_lo, + iscsi_ooo->cid, iscsi_ooo->ooo_opcode, iscsi_ooo->ooo_isle); + for (i = 0; i < 64; i = i + 8) { + DP_VERBOSE(p_hwfn, dp_module, + "0x%x-%x: 0x%x 0x%x 0x%x 0x%x 0x%x 0x%x 0x%x 0x%x\n", + ph_hi, ph_lo, + packet_buffer[i], + packet_buffer[i + 1], + packet_buffer[i + 2], + packet_buffer[i + 3], + packet_buffer[i + 4], + packet_buffer[i + 5], + packet_buffer[i + 6], + packet_buffer[i + 7]); + } +} Property changes on: head/sys/dev/qlnx/qlnxe/ecore_ooo.c ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Added: svn:mime-type ## -0,0 +1 ## +text/plain \ No newline at end of property Index: head/sys/dev/qlnx/qlnxe/ecore_rdma.c =================================================================== --- head/sys/dev/qlnx/qlnxe/ecore_rdma.c (nonexistent) +++ head/sys/dev/qlnx/qlnxe/ecore_rdma.c (revision 343598) @@ -0,0 +1,2697 @@ +/* + * Copyright (c) 2018-2019 Cavium, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +/* + * File : ecore_rdma.c + */ +#include +__FBSDID("$FreeBSD$"); + +#include "bcm_osal.h" +#include "ecore.h" +#include "ecore_status.h" +#include "ecore_sp_commands.h" +#include "ecore_cxt.h" +#include "ecore_rdma.h" +#include "reg_addr.h" +#include "ecore_rt_defs.h" +#include "ecore_init_ops.h" +#include "ecore_hw.h" +#include "ecore_mcp.h" +#include "ecore_init_fw_funcs.h" +#include "ecore_int.h" +#include "pcics_reg_driver.h" +#include "ecore_iro.h" +#include "ecore_gtt_reg_addr.h" +#include "ecore_hsi_iwarp.h" +#include "ecore_ll2.h" +#include "ecore_ooo.h" +#ifndef LINUX_REMOVE +#include "ecore_tcp_ip.h" +#endif + +enum _ecore_status_t ecore_rdma_bmap_alloc(struct ecore_hwfn *p_hwfn, + struct ecore_bmap *bmap, + u32 max_count, + char *name) +{ + u32 size_in_bytes; + + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "max_count = %08x\n", max_count); + + bmap->max_count = max_count; + + if (!max_count) { + bmap->bitmap = OSAL_NULL; + return ECORE_SUCCESS; + } + + size_in_bytes = sizeof(unsigned long) * + DIV_ROUND_UP(max_count, (sizeof(unsigned long) * 8)); + + bmap->bitmap = OSAL_ZALLOC(p_hwfn->p_dev, GFP_KERNEL, size_in_bytes); + if (!bmap->bitmap) + { + DP_NOTICE(p_hwfn, false, + "ecore bmap alloc failed: cannot allocate memory (bitmap). rc = %d\n", + ECORE_NOMEM); + return ECORE_NOMEM; + } + + OSAL_SNPRINTF(bmap->name, QEDR_MAX_BMAP_NAME, "%s", name); + + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "ECORE_SUCCESS\n"); + return ECORE_SUCCESS; +} + +enum _ecore_status_t ecore_rdma_bmap_alloc_id(struct ecore_hwfn *p_hwfn, + struct ecore_bmap *bmap, + u32 *id_num) +{ + *id_num = OSAL_FIND_FIRST_ZERO_BIT(bmap->bitmap, bmap->max_count); + if (*id_num >= bmap->max_count) + return ECORE_INVAL; + + OSAL_SET_BIT(*id_num, bmap->bitmap); + + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "%s bitmap: allocated id %d\n", + bmap->name, *id_num); + + return ECORE_SUCCESS; +} + +void ecore_bmap_set_id(struct ecore_hwfn *p_hwfn, + struct ecore_bmap *bmap, + u32 id_num) +{ + if (id_num >= bmap->max_count) { + DP_NOTICE(p_hwfn, true, + "%s bitmap: cannot set id %d max is %d\n", + bmap->name, id_num, bmap->max_count); + + return; + } + + OSAL_SET_BIT(id_num, bmap->bitmap); + + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "%s bitmap: set id %d\n", + bmap->name, id_num); +} + +void ecore_bmap_release_id(struct ecore_hwfn *p_hwfn, + struct ecore_bmap *bmap, + u32 id_num) +{ + bool b_acquired; + + if (id_num >= bmap->max_count) + return; + + b_acquired = OSAL_TEST_AND_CLEAR_BIT(id_num, bmap->bitmap); + if (!b_acquired) + { + DP_NOTICE(p_hwfn, false, "%s bitmap: id %d already released\n", + bmap->name, id_num); + return; + } + + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "%s bitmap: released id %d\n", + bmap->name, id_num); +} + +int ecore_bmap_test_id(struct ecore_hwfn *p_hwfn, + struct ecore_bmap *bmap, + u32 id_num) +{ + if (id_num >= bmap->max_count) { + DP_NOTICE(p_hwfn, true, + "%s bitmap: id %d too high. max is %d\n", + bmap->name, id_num, bmap->max_count); + return -1; + } + + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "%s bitmap: tested id %d\n", + bmap->name, id_num); + + return OSAL_TEST_BIT(id_num, bmap->bitmap); +} + +static bool ecore_bmap_is_empty(struct ecore_bmap *bmap) +{ + return (bmap->max_count == + OSAL_FIND_FIRST_BIT(bmap->bitmap, bmap->max_count)); +} + +#ifndef LINUX_REMOVE +u32 ecore_rdma_get_sb_id(struct ecore_hwfn *p_hwfn, u32 rel_sb_id) +{ + /* first sb id for RoCE is after all the l2 sb */ + return FEAT_NUM(p_hwfn, ECORE_PF_L2_QUE) + rel_sb_id; +} + +u32 ecore_rdma_query_cau_timer_res(void) +{ + return ECORE_CAU_DEF_RX_TIMER_RES; +} +#endif + +enum _ecore_status_t ecore_rdma_info_alloc(struct ecore_hwfn *p_hwfn) +{ + struct ecore_rdma_info *p_rdma_info; + + p_rdma_info = OSAL_ZALLOC(p_hwfn->p_dev, GFP_KERNEL, sizeof(*p_rdma_info)); + if (!p_rdma_info) { + DP_NOTICE(p_hwfn, false, + "ecore rdma alloc failed: cannot allocate memory (rdma info).\n"); + return ECORE_NOMEM; + } + p_hwfn->p_rdma_info = p_rdma_info; + +#ifdef CONFIG_ECORE_LOCK_ALLOC + if (OSAL_SPIN_LOCK_ALLOC(p_hwfn, &p_rdma_info->lock)) { + ecore_rdma_info_free(p_hwfn); + return ECORE_NOMEM; + } +#endif + OSAL_SPIN_LOCK_INIT(&p_rdma_info->lock); + + return ECORE_SUCCESS; +} + +void ecore_rdma_info_free(struct ecore_hwfn *p_hwfn) +{ +#ifdef CONFIG_ECORE_LOCK_ALLOC + OSAL_SPIN_LOCK_DEALLOC(&p_hwfn->p_rdma_info->lock); +#endif + OSAL_FREE(p_hwfn->p_dev, p_hwfn->p_rdma_info); + p_hwfn->p_rdma_info = OSAL_NULL; +} + +static enum _ecore_status_t ecore_rdma_inc_ref_cnt(struct ecore_hwfn *p_hwfn) +{ + enum _ecore_status_t rc = ECORE_INVAL; + + OSAL_SPIN_LOCK(&p_hwfn->p_rdma_info->lock); + if (p_hwfn->p_rdma_info->active) { + p_hwfn->p_rdma_info->ref_cnt++; + rc = ECORE_SUCCESS; + } else { + DP_INFO(p_hwfn, "Ref cnt requested for inactive rdma\n"); + } + OSAL_SPIN_UNLOCK(&p_hwfn->p_rdma_info->lock); + return rc; +} + +static void ecore_rdma_dec_ref_cnt(struct ecore_hwfn *p_hwfn) +{ + OSAL_SPIN_LOCK(&p_hwfn->p_rdma_info->lock); + p_hwfn->p_rdma_info->ref_cnt--; + OSAL_SPIN_UNLOCK(&p_hwfn->p_rdma_info->lock); +} + +static void ecore_rdma_activate(struct ecore_hwfn *p_hwfn) +{ + OSAL_SPIN_LOCK(&p_hwfn->p_rdma_info->lock); + p_hwfn->p_rdma_info->active = true; + OSAL_SPIN_UNLOCK(&p_hwfn->p_rdma_info->lock); +} + +/* Part of deactivating rdma is letting all the relevant flows complete before + * we start shutting down: Currently query-stats which can be called from MCP + * context. + */ +/* The longest time it can take a rdma flow to complete */ +#define ECORE_RDMA_MAX_FLOW_TIME (100) +static enum _ecore_status_t ecore_rdma_deactivate(struct ecore_hwfn *p_hwfn) +{ + int wait_count; + + OSAL_SPIN_LOCK(&p_hwfn->p_rdma_info->lock); + p_hwfn->p_rdma_info->active = false; + OSAL_SPIN_UNLOCK(&p_hwfn->p_rdma_info->lock); + + /* We'll give each flow it's time to complete... */ + wait_count = p_hwfn->p_rdma_info->ref_cnt; + + while (p_hwfn->p_rdma_info->ref_cnt) { + OSAL_MSLEEP(ECORE_RDMA_MAX_FLOW_TIME); + if (--wait_count == 0) { + DP_NOTICE(p_hwfn, false, + "Timeout on refcnt=%d\n", + p_hwfn->p_rdma_info->ref_cnt); + return ECORE_TIMEOUT; + } + } + return ECORE_SUCCESS; +} + +static enum _ecore_status_t ecore_rdma_alloc(struct ecore_hwfn *p_hwfn) +{ + struct ecore_rdma_info *p_rdma_info = p_hwfn->p_rdma_info; + u32 num_cons, num_tasks; + enum _ecore_status_t rc; + + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "Allocating RDMA\n"); + + if (!p_rdma_info) + return ECORE_INVAL; + + if (p_hwfn->hw_info.personality == ECORE_PCI_ETH_IWARP) + p_rdma_info->proto = PROTOCOLID_IWARP; + else + p_rdma_info->proto = PROTOCOLID_ROCE; + + num_cons = ecore_cxt_get_proto_cid_count(p_hwfn, p_rdma_info->proto, + OSAL_NULL); + + if (IS_IWARP(p_hwfn)) + p_rdma_info->num_qps = num_cons; + else + p_rdma_info->num_qps = num_cons / 2; + + /* INTERNAL: RoCE & iWARP use the same taskid */ + num_tasks = ecore_cxt_get_proto_tid_count(p_hwfn, PROTOCOLID_ROCE); + + /* Each MR uses a single task */ + p_rdma_info->num_mrs = num_tasks; + + /* Queue zone lines are shared between RoCE and L2 in such a way that + * they can be used by each without obstructing the other. + */ + p_rdma_info->queue_zone_base = (u16) RESC_START(p_hwfn, ECORE_L2_QUEUE); + p_rdma_info->max_queue_zones = (u16) RESC_NUM(p_hwfn, ECORE_L2_QUEUE); + + /* Allocate a struct with device params and fill it */ + p_rdma_info->dev = OSAL_ZALLOC(p_hwfn->p_dev, GFP_KERNEL, sizeof(*p_rdma_info->dev)); + if (!p_rdma_info->dev) + { + rc = ECORE_NOMEM; + DP_NOTICE(p_hwfn, false, + "ecore rdma alloc failed: cannot allocate memory (rdma info dev). rc = %d\n", + rc); + return rc; + } + + /* Allocate a struct with port params and fill it */ + p_rdma_info->port = OSAL_ZALLOC(p_hwfn->p_dev, GFP_KERNEL, sizeof(*p_rdma_info->port)); + if (!p_rdma_info->port) + { + DP_NOTICE(p_hwfn, false, + "ecore rdma alloc failed: cannot allocate memory (rdma info port)\n"); + return ECORE_NOMEM; + } + + /* Allocate bit map for pd's */ + rc = ecore_rdma_bmap_alloc(p_hwfn, &p_rdma_info->pd_map, RDMA_MAX_PDS, + "PD"); + if (rc != ECORE_SUCCESS) + { + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, + "Failed to allocate pd_map,rc = %d\n", + rc); + return rc; + } + + /* Allocate bit map for XRC Domains */ + rc = ecore_rdma_bmap_alloc(p_hwfn, &p_rdma_info->xrcd_map, + ECORE_RDMA_MAX_XRCDS, "XRCD"); + if (rc != ECORE_SUCCESS) + { + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, + "Failed to allocate xrcd_map,rc = %d\n", + rc); + return rc; + } + + /* Allocate DPI bitmap */ + rc = ecore_rdma_bmap_alloc(p_hwfn, &p_rdma_info->dpi_map, + p_hwfn->dpi_count, "DPI"); + if (rc != ECORE_SUCCESS) + { + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, + "Failed to allocate DPI bitmap, rc = %d\n", rc); + return rc; + } + + /* Allocate bitmap for cq's. The maximum number of CQs is bounded to + * twice the number of QPs. + */ + rc = ecore_rdma_bmap_alloc(p_hwfn, &p_rdma_info->cq_map, + num_cons, "CQ"); + if (rc != ECORE_SUCCESS) + { + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, + "Failed to allocate cq bitmap, rc = %d\n", rc); + return rc; + } + + /* Allocate bitmap for toggle bit for cq icids + * We toggle the bit every time we create or resize cq for a given icid. + * The maximum number of CQs is bounded to the number of connections we + * support. (num_qps in iWARP or num_qps/2 in RoCE). + */ + rc = ecore_rdma_bmap_alloc(p_hwfn, &p_rdma_info->toggle_bits, + num_cons, "Toggle"); + if (rc != ECORE_SUCCESS) + { + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, + "Failed to allocate toogle bits, rc = %d\n", rc); + return rc; + } + + /* Allocate bitmap for itids */ + rc = ecore_rdma_bmap_alloc(p_hwfn, &p_rdma_info->tid_map, + p_rdma_info->num_mrs, "MR"); + if (rc != ECORE_SUCCESS) + { + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, + "Failed to allocate itids bitmaps, rc = %d\n", rc); + return rc; + } + + /* Allocate bitmap for qps. */ + rc = ecore_rdma_bmap_alloc(p_hwfn, &p_rdma_info->qp_map, + p_rdma_info->num_qps, "QP"); + if (rc != ECORE_SUCCESS) + { + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, + "Failed to allocate qp bitmap, rc = %d\n", rc); + return rc; + } + + /* Allocate bitmap for cids used for responders/requesters. */ + rc = ecore_rdma_bmap_alloc(p_hwfn, &p_rdma_info->cid_map, num_cons, + "REAL CID"); + if (rc != ECORE_SUCCESS) + { + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, + "Failed to allocate cid bitmap, rc = %d\n", rc); + return rc; + } + + /* The first SRQ follows the last XRC SRQ. This means that the + * SRQ IDs start from an offset equals to max_xrc_srqs. + */ + p_rdma_info->srq_id_offset = (u16)ecore_cxt_get_xrc_srq_count(p_hwfn); + rc = ecore_rdma_bmap_alloc(p_hwfn, &p_rdma_info->xrc_srq_map, + p_rdma_info->srq_id_offset, "XRC SRQ"); + if (rc != ECORE_SUCCESS) { + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, + "Failed to allocate xrc srq bitmap, rc = %d\n", rc); + return rc; + } + + /* Allocate bitmap for srqs */ + p_rdma_info->num_srqs = ecore_cxt_get_srq_count(p_hwfn); + rc = ecore_rdma_bmap_alloc(p_hwfn, &p_rdma_info->srq_map, + p_rdma_info->num_srqs, + "SRQ"); + if (rc != ECORE_SUCCESS) { + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, + "Failed to allocate srq bitmap, rc = %d\n", rc); + + return rc; + } + + if (IS_IWARP(p_hwfn)) + rc = ecore_iwarp_alloc(p_hwfn); + + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "rc = %d\n", rc); + + return rc; +} + +void ecore_rdma_bmap_free(struct ecore_hwfn *p_hwfn, + struct ecore_bmap *bmap, + bool check) +{ + int weight, line, item, last_line, last_item; + u64 *pmap; + + if (!bmap || !bmap->bitmap) + return; + + if (!check) + goto end; + + weight = OSAL_BITMAP_WEIGHT(bmap->bitmap, bmap->max_count); + if (!weight) + goto end; + + DP_NOTICE(p_hwfn, false, + "%s bitmap not free - size=%d, weight=%d, 512 bits per line\n", + bmap->name, bmap->max_count, weight); + + pmap = (u64 *)bmap->bitmap; + last_line = bmap->max_count / (64*8); + last_item = last_line * 8 + (((bmap->max_count % (64*8)) + 63) / 64); + + /* print aligned non-zero lines, if any */ + for (item = 0, line = 0; line < last_line; line++, item += 8) { + if (OSAL_BITMAP_WEIGHT((unsigned long *)&pmap[item], 64*8)) + DP_NOTICE(p_hwfn, false, + "line 0x%04x: 0x%016llx 0x%016llx 0x%016llx 0x%016llx 0x%016llx 0x%016llx 0x%016llx 0x%016llx\n", + line, (unsigned long long)pmap[item], + (unsigned long long)pmap[item+1], + (unsigned long long)pmap[item+2], + (unsigned long long)pmap[item+3], + (unsigned long long)pmap[item+4], + (unsigned long long)pmap[item+5], + (unsigned long long)pmap[item+6], + (unsigned long long)pmap[item+7]); + } + + /* print last unaligned non-zero line, if any */ + if ((bmap->max_count % (64*8)) && + (OSAL_BITMAP_WEIGHT((unsigned long *)&pmap[item], + bmap->max_count-item*64))) { + u8 str_last_line[200] = { 0 }; + int offset; + + offset = OSAL_SPRINTF(str_last_line, "line 0x%04x: ", line); + for (; item < last_item; item++) { + offset += OSAL_SPRINTF(str_last_line+offset, + "0x%016llx ", + (unsigned long long)pmap[item]); + } + DP_NOTICE(p_hwfn, false, "%s\n", str_last_line); + } + +end: + OSAL_FREE(p_hwfn->p_dev, bmap->bitmap); + bmap->bitmap = OSAL_NULL; +} + + +void ecore_rdma_resc_free(struct ecore_hwfn *p_hwfn) +{ + if (IS_IWARP(p_hwfn)) + ecore_iwarp_resc_free(p_hwfn); + + ecore_rdma_bmap_free(p_hwfn, &p_hwfn->p_rdma_info->cid_map, 1); + ecore_rdma_bmap_free(p_hwfn, &p_hwfn->p_rdma_info->qp_map, 1); + ecore_rdma_bmap_free(p_hwfn, &p_hwfn->p_rdma_info->pd_map, 1); + ecore_rdma_bmap_free(p_hwfn, &p_hwfn->p_rdma_info->xrcd_map, 1); + ecore_rdma_bmap_free(p_hwfn, &p_hwfn->p_rdma_info->dpi_map, 1); + ecore_rdma_bmap_free(p_hwfn, &p_hwfn->p_rdma_info->cq_map, 1); + ecore_rdma_bmap_free(p_hwfn, &p_hwfn->p_rdma_info->toggle_bits, 0); + ecore_rdma_bmap_free(p_hwfn, &p_hwfn->p_rdma_info->tid_map, 1); + ecore_rdma_bmap_free(p_hwfn, &p_hwfn->p_rdma_info->srq_map, 1); + ecore_rdma_bmap_free(p_hwfn, &p_hwfn->p_rdma_info->xrc_srq_map, 1); + + OSAL_FREE(p_hwfn->p_dev, p_hwfn->p_rdma_info->port); + p_hwfn->p_rdma_info->port = OSAL_NULL; + + OSAL_FREE(p_hwfn->p_dev, p_hwfn->p_rdma_info->dev); + p_hwfn->p_rdma_info->dev = OSAL_NULL; +} + +static OSAL_INLINE void ecore_rdma_free_reserved_lkey(struct ecore_hwfn *p_hwfn) +{ + ecore_rdma_free_tid(p_hwfn, p_hwfn->p_rdma_info->dev->reserved_lkey); +} + +static void ecore_rdma_free_ilt(struct ecore_hwfn *p_hwfn) +{ + /* Free Connection CXT */ + ecore_cxt_free_ilt_range( + p_hwfn, ECORE_ELEM_CXT, + ecore_cxt_get_proto_cid_start(p_hwfn, + p_hwfn->p_rdma_info->proto), + ecore_cxt_get_proto_cid_count(p_hwfn, + p_hwfn->p_rdma_info->proto, + OSAL_NULL)); + + /* Free Task CXT ( Intentionally RoCE as task-id is shared between + * RoCE and iWARP + */ + ecore_cxt_free_ilt_range(p_hwfn, ECORE_ELEM_TASK, 0, + ecore_cxt_get_proto_tid_count( + p_hwfn, PROTOCOLID_ROCE)); + + /* Free TSDM CXT */ + ecore_cxt_free_ilt_range(p_hwfn, ECORE_ELEM_SRQ, 0, + ecore_cxt_get_srq_count(p_hwfn)); +} + +static void ecore_rdma_free(struct ecore_hwfn *p_hwfn) +{ + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "\n"); + + ecore_rdma_free_reserved_lkey(p_hwfn); + + ecore_rdma_resc_free(p_hwfn); + + ecore_rdma_free_ilt(p_hwfn); +} + +static void ecore_rdma_get_guid(struct ecore_hwfn *p_hwfn, u8 *guid) +{ + u8 mac_addr[6]; + + OSAL_MEMCPY(&mac_addr[0], &p_hwfn->hw_info.hw_mac_addr[0], ETH_ALEN); + guid[0] = mac_addr[0] ^ 2; + guid[1] = mac_addr[1]; + guid[2] = mac_addr[2]; + guid[3] = 0xff; + guid[4] = 0xfe; + guid[5] = mac_addr[3]; + guid[6] = mac_addr[4]; + guid[7] = mac_addr[5]; +} + + +static void ecore_rdma_init_events( + struct ecore_hwfn *p_hwfn, + struct ecore_rdma_start_in_params *params) +{ + struct ecore_rdma_events *events; + + events = &p_hwfn->p_rdma_info->events; + + events->unaffiliated_event = params->events->unaffiliated_event; + events->affiliated_event = params->events->affiliated_event; + events->context = params->events->context; +} + +static void ecore_rdma_init_devinfo( + struct ecore_hwfn *p_hwfn, + struct ecore_rdma_start_in_params *params) +{ + struct ecore_rdma_device *dev = p_hwfn->p_rdma_info->dev; + u32 pci_status_control; + + /* Vendor specific information */ + dev->vendor_id = p_hwfn->p_dev->vendor_id; + dev->vendor_part_id = p_hwfn->p_dev->device_id; + dev->hw_ver = 0; + dev->fw_ver = STORM_FW_VERSION; + + ecore_rdma_get_guid(p_hwfn, (u8 *)(&dev->sys_image_guid)); + dev->node_guid = dev->sys_image_guid; + + dev->max_sge = OSAL_MIN_T(u32, RDMA_MAX_SGE_PER_SQ_WQE, + RDMA_MAX_SGE_PER_RQ_WQE); + + if (p_hwfn->p_dev->rdma_max_sge) { + dev->max_sge = OSAL_MIN_T(u32, + p_hwfn->p_dev->rdma_max_sge, + dev->max_sge); + } + + /* Set these values according to configuration + * MAX SGE for SRQ is not defined by FW for now + * define it in driver. + * TODO: Get this value from FW. + */ + dev->max_srq_sge = ECORE_RDMA_MAX_SGE_PER_SRQ_WQE; + if (p_hwfn->p_dev->rdma_max_srq_sge) { + dev->max_srq_sge = OSAL_MIN_T(u32, + p_hwfn->p_dev->rdma_max_srq_sge, + dev->max_srq_sge); + } + + dev->max_inline = ROCE_REQ_MAX_INLINE_DATA_SIZE; + dev->max_inline = (p_hwfn->p_dev->rdma_max_inline) ? + OSAL_MIN_T(u32, + p_hwfn->p_dev->rdma_max_inline, + dev->max_inline) : + dev->max_inline; + + dev->max_wqe = ECORE_RDMA_MAX_WQE; + dev->max_cnq = (u8)FEAT_NUM(p_hwfn, ECORE_RDMA_CNQ); + + /* The number of QPs may be higher than ECORE_ROCE_MAX_QPS. because + * it is up-aligned to 16 and then to ILT page size within ecore cxt. + * This is OK in terms of ILT but we don't want to configure the FW + * above its abilities + */ + dev->max_qp = OSAL_MIN_T(u64, ROCE_MAX_QPS, + p_hwfn->p_rdma_info->num_qps); + + /* CQs uses the same icids that QPs use hence they are limited by the + * number of icids. There are two icids per QP. + */ + dev->max_cq = dev->max_qp * 2; + + /* The number of mrs is smaller by 1 since the first is reserved */ + dev->max_mr = p_hwfn->p_rdma_info->num_mrs - 1; + dev->max_mr_size = ECORE_RDMA_MAX_MR_SIZE; + /* The maximum CQE capacity per CQ supported */ + /* max number of cqes will be in two layer pbl, + * 8 is the pointer size in bytes + * 32 is the size of cq element in bytes + */ + if (params->roce.cq_mode == ECORE_RDMA_CQ_MODE_32_BITS) + dev->max_cqe = ECORE_RDMA_MAX_CQE_32_BIT; + else + dev->max_cqe = ECORE_RDMA_MAX_CQE_16_BIT; + + dev->max_mw = 0; + dev->max_fmr = ECORE_RDMA_MAX_FMR; + dev->max_mr_mw_fmr_pbl = (OSAL_PAGE_SIZE/8) * (OSAL_PAGE_SIZE/8); + dev->max_mr_mw_fmr_size = dev->max_mr_mw_fmr_pbl * OSAL_PAGE_SIZE; + dev->max_pkey = ECORE_RDMA_MAX_P_KEY; + /* Right now we dont take any parameters from user + * So assign predefined max_srq to num_srqs. + */ + dev->max_srq = p_hwfn->p_rdma_info->num_srqs; + + /* SRQ WQE size */ + dev->max_srq_wr = ECORE_RDMA_MAX_SRQ_WQE_ELEM; + + dev->max_qp_resp_rd_atomic_resc = RDMA_RING_PAGE_SIZE / + (RDMA_RESP_RD_ATOMIC_ELM_SIZE*2); + dev->max_qp_req_rd_atomic_resc = RDMA_RING_PAGE_SIZE / + RDMA_REQ_RD_ATOMIC_ELM_SIZE; + + dev->max_dev_resp_rd_atomic_resc = + dev->max_qp_resp_rd_atomic_resc * p_hwfn->p_rdma_info->num_qps; + dev->page_size_caps = ECORE_RDMA_PAGE_SIZE_CAPS; + dev->dev_ack_delay = ECORE_RDMA_ACK_DELAY; + dev->max_pd = RDMA_MAX_PDS; + dev->max_ah = dev->max_qp; + dev->max_stats_queues = (u8)RESC_NUM(p_hwfn, ECORE_RDMA_STATS_QUEUE); + + /* Set capablities */ + dev->dev_caps = 0; + SET_FIELD(dev->dev_caps, ECORE_RDMA_DEV_CAP_RNR_NAK, 1); + SET_FIELD(dev->dev_caps, ECORE_RDMA_DEV_CAP_PORT_ACTIVE_EVENT, 1); + SET_FIELD(dev->dev_caps, ECORE_RDMA_DEV_CAP_PORT_CHANGE_EVENT, 1); + SET_FIELD(dev->dev_caps, ECORE_RDMA_DEV_CAP_RESIZE_CQ, 1); + SET_FIELD(dev->dev_caps, ECORE_RDMA_DEV_CAP_BASE_MEMORY_EXT, 1); + SET_FIELD(dev->dev_caps, ECORE_RDMA_DEV_CAP_BASE_QUEUE_EXT, 1); + SET_FIELD(dev->dev_caps, ECORE_RDMA_DEV_CAP_ZBVA, 1); + SET_FIELD(dev->dev_caps, ECORE_RDMA_DEV_CAP_LOCAL_INV_FENCE, 1); + + /* Check atomic operations support in PCI configuration space. */ + OSAL_PCI_READ_CONFIG_DWORD(p_hwfn->p_dev, + PCICFG_DEVICE_STATUS_CONTROL_2, + &pci_status_control); + + if (pci_status_control & + PCICFG_DEVICE_STATUS_CONTROL_2_ATOMIC_REQ_ENABLE) + SET_FIELD(dev->dev_caps, ECORE_RDMA_DEV_CAP_ATOMIC_OP, 1); + + if (IS_IWARP(p_hwfn)) + ecore_iwarp_init_devinfo(p_hwfn); +} + +static void ecore_rdma_init_port( + struct ecore_hwfn *p_hwfn) +{ + struct ecore_rdma_port *port = p_hwfn->p_rdma_info->port; + struct ecore_rdma_device *dev = p_hwfn->p_rdma_info->dev; + + port->port_state = p_hwfn->mcp_info->link_output.link_up ? + ECORE_RDMA_PORT_UP : ECORE_RDMA_PORT_DOWN; + + port->max_msg_size = OSAL_MIN_T(u64, + (dev->max_mr_mw_fmr_size * + p_hwfn->p_dev->rdma_max_sge), + ((u64)1 << 31)); + + port->pkey_bad_counter = 0; +} + +static enum _ecore_status_t ecore_rdma_init_hw( + struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt) +{ + u32 ll2_ethertype_en; + + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "Initializing HW\n"); + p_hwfn->b_rdma_enabled_in_prs = false; + + if (IS_IWARP(p_hwfn)) + return ecore_iwarp_init_hw(p_hwfn, p_ptt); + + ecore_wr(p_hwfn, + p_ptt, + PRS_REG_ROCE_DEST_QP_MAX_PF, + 0); + + p_hwfn->rdma_prs_search_reg = PRS_REG_SEARCH_ROCE; + + /* We delay writing to this reg until first cid is allocated. See + * ecore_cxt_dynamic_ilt_alloc function for more details + */ + + ll2_ethertype_en = ecore_rd(p_hwfn, + p_ptt, + PRS_REG_LIGHT_L2_ETHERTYPE_EN); + ecore_wr(p_hwfn, p_ptt, PRS_REG_LIGHT_L2_ETHERTYPE_EN, + (ll2_ethertype_en | 0x01)); + +#ifndef REAL_ASIC_ONLY + if (ECORE_IS_BB_A0(p_hwfn->p_dev) && ECORE_IS_CMT(p_hwfn->p_dev)) { + ecore_wr(p_hwfn, + p_ptt, + NIG_REG_LLH_ENG_CLS_ENG_ID_TBL, + 0); + ecore_wr(p_hwfn, + p_ptt, + NIG_REG_LLH_ENG_CLS_ENG_ID_TBL + 4, + 0); + } +#endif + + if (ecore_cxt_get_proto_cid_start(p_hwfn, PROTOCOLID_ROCE) % 2) + { + DP_NOTICE(p_hwfn, + true, + "The first RoCE's cid should be even\n"); + return ECORE_UNKNOWN_ERROR; + } + + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "Initializing HW - Done\n"); + return ECORE_SUCCESS; +} + +static enum _ecore_status_t +ecore_rdma_start_fw(struct ecore_hwfn *p_hwfn, +#ifdef CONFIG_DCQCN + struct ecore_ptt *p_ptt, +#else + struct ecore_ptt OSAL_UNUSED *p_ptt, +#endif + struct ecore_rdma_start_in_params *params) +{ + struct rdma_init_func_ramrod_data *p_ramrod; + struct rdma_init_func_hdr *pheader; + struct ecore_rdma_info *p_rdma_info; + struct ecore_sp_init_data init_data; + struct ecore_spq_entry *p_ent; + u16 igu_sb_id, sb_id; + u8 ll2_queue_id; + u32 cnq_id; + enum _ecore_status_t rc; + + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "Starting FW\n"); + + p_rdma_info = p_hwfn->p_rdma_info; + + /* Save the number of cnqs for the function close ramrod */ + p_rdma_info->num_cnqs = params->desired_cnq; + + /* Get SPQ entry */ + OSAL_MEMSET(&init_data, 0, sizeof(init_data)); + init_data.opaque_fid = p_hwfn->hw_info.opaque_fid; + init_data.comp_mode = ECORE_SPQ_MODE_EBLOCK; + + rc = ecore_sp_init_request(p_hwfn, &p_ent, RDMA_RAMROD_FUNC_INIT, + p_rdma_info->proto, &init_data); + if (rc != ECORE_SUCCESS) + return rc; + + if (IS_IWARP(p_hwfn)) { + ecore_iwarp_init_fw_ramrod(p_hwfn, + &p_ent->ramrod.iwarp_init_func); + p_ramrod = &p_ent->ramrod.iwarp_init_func.rdma; + } else { + +#ifdef CONFIG_DCQCN + rc = ecore_roce_dcqcn_cfg(p_hwfn, ¶ms->roce.dcqcn_params, + &p_ent->ramrod.roce_init_func, p_ptt); + if (rc != ECORE_SUCCESS) { + DP_NOTICE(p_hwfn, false, + "Failed to configure DCQCN. rc = %d.\n", rc); + return rc; + } +#endif + p_ramrod = &p_ent->ramrod.roce_init_func.rdma; + + /* The ll2_queue_id is used only for UD QPs */ + ll2_queue_id = ecore_ll2_handle_to_queue_id( + p_hwfn, params->roce.ll2_handle); + p_ent->ramrod.roce_init_func.roce.ll2_queue_id = ll2_queue_id; + + } + + pheader = &p_ramrod->params_header; + pheader->cnq_start_offset = (u8)RESC_START(p_hwfn, ECORE_RDMA_CNQ_RAM); + pheader->num_cnqs = params->desired_cnq; + + /* The first SRQ ILT page is used for XRC SRQs and all the following + * pages contain regular SRQs. Hence the first regular SRQ ID is the + * maximum number XRC SRQs. + */ + pheader->first_reg_srq_id = p_rdma_info->srq_id_offset; + pheader->reg_srq_base_addr = + ecore_cxt_get_ilt_page_size(p_hwfn, ILT_CLI_TSDM); + + if (params->roce.cq_mode == ECORE_RDMA_CQ_MODE_16_BITS) + pheader->cq_ring_mode = 1; /* 1=16 bits */ + else + pheader->cq_ring_mode = 0; /* 0=32 bits */ + + for (cnq_id = 0; cnq_id < params->desired_cnq; cnq_id++) + { + sb_id = (u16)OSAL_GET_RDMA_SB_ID(p_hwfn, cnq_id); + igu_sb_id = ecore_get_igu_sb_id(p_hwfn, sb_id); + p_ramrod->cnq_params[cnq_id].sb_num = + OSAL_CPU_TO_LE16(igu_sb_id); + + p_ramrod->cnq_params[cnq_id].sb_index = + p_hwfn->pf_params.rdma_pf_params.gl_pi; + + p_ramrod->cnq_params[cnq_id].num_pbl_pages = + params->cnq_pbl_list[cnq_id].num_pbl_pages; + + p_ramrod->cnq_params[cnq_id].pbl_base_addr.hi = + DMA_HI_LE(params->cnq_pbl_list[cnq_id].pbl_ptr); + p_ramrod->cnq_params[cnq_id].pbl_base_addr.lo = + DMA_LO_LE(params->cnq_pbl_list[cnq_id].pbl_ptr); + + /* we arbitrarily decide that cnq_id will be as qz_offset */ + p_ramrod->cnq_params[cnq_id].queue_zone_num = + OSAL_CPU_TO_LE16(p_rdma_info->queue_zone_base + cnq_id); + } + + rc = ecore_spq_post(p_hwfn, p_ent, OSAL_NULL); + + return rc; +} + +enum _ecore_status_t ecore_rdma_alloc_tid(void *rdma_cxt, + u32 *itid) +{ + struct ecore_hwfn *p_hwfn = (struct ecore_hwfn *)rdma_cxt; + enum _ecore_status_t rc; + + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "Allocate TID\n"); + + OSAL_SPIN_LOCK(&p_hwfn->p_rdma_info->lock); + rc = ecore_rdma_bmap_alloc_id(p_hwfn, + &p_hwfn->p_rdma_info->tid_map, + itid); + OSAL_SPIN_UNLOCK(&p_hwfn->p_rdma_info->lock); + if (rc != ECORE_SUCCESS) { + DP_NOTICE(p_hwfn, false, "Failed in allocating tid\n"); + goto out; + } + + rc = ecore_cxt_dynamic_ilt_alloc(p_hwfn, ECORE_ELEM_TASK, *itid); +out: + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "Allocate TID - done, rc = %d\n", rc); + return rc; +} + +static OSAL_INLINE enum _ecore_status_t ecore_rdma_reserve_lkey( + struct ecore_hwfn *p_hwfn) +{ + struct ecore_rdma_device *dev = p_hwfn->p_rdma_info->dev; + + /* Tid 0 will be used as the key for "reserved MR". + * The driver should allocate memory for it so it can be loaded but no + * ramrod should be passed on it. + */ + ecore_rdma_alloc_tid(p_hwfn, &dev->reserved_lkey); + if (dev->reserved_lkey != RDMA_RESERVED_LKEY) + { + DP_NOTICE(p_hwfn, true, + "Reserved lkey should be equal to RDMA_RESERVED_LKEY\n"); + return ECORE_INVAL; + } + + return ECORE_SUCCESS; +} + +static enum _ecore_status_t ecore_rdma_setup(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt, + struct ecore_rdma_start_in_params *params) +{ + enum _ecore_status_t rc = 0; + + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "RDMA setup\n"); + + ecore_rdma_init_devinfo(p_hwfn, params); + ecore_rdma_init_port(p_hwfn); + ecore_rdma_init_events(p_hwfn, params); + + rc = ecore_rdma_reserve_lkey(p_hwfn); + if (rc != ECORE_SUCCESS) + return rc; + + rc = ecore_rdma_init_hw(p_hwfn, p_ptt); + if (rc != ECORE_SUCCESS) + return rc; + + if (IS_IWARP(p_hwfn)) { + rc = ecore_iwarp_setup(p_hwfn, params); + if (rc != ECORE_SUCCESS) + return rc; + } else { + rc = ecore_roce_setup(p_hwfn); + if (rc != ECORE_SUCCESS) + return rc; + } + + return ecore_rdma_start_fw(p_hwfn, p_ptt, params); +} + + +enum _ecore_status_t ecore_rdma_stop(void *rdma_cxt) +{ + struct ecore_hwfn *p_hwfn = (struct ecore_hwfn *)rdma_cxt; + struct rdma_close_func_ramrod_data *p_ramrod; + struct ecore_sp_init_data init_data; + struct ecore_spq_entry *p_ent; + struct ecore_ptt *p_ptt; + u32 ll2_ethertype_en; + enum _ecore_status_t rc = ECORE_TIMEOUT; + + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "RDMA stop\n"); + + rc = ecore_rdma_deactivate(p_hwfn); + if (rc != ECORE_SUCCESS) + return rc; + + p_ptt = ecore_ptt_acquire(p_hwfn); + if (!p_ptt) { + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "Failed to acquire PTT\n"); + return rc; + } + +#ifdef CONFIG_DCQCN + ecore_roce_stop_rl(p_hwfn); +#endif + + /* Disable RoCE search */ + ecore_wr(p_hwfn, p_ptt, p_hwfn->rdma_prs_search_reg, 0); + p_hwfn->b_rdma_enabled_in_prs = false; + + ecore_wr(p_hwfn, + p_ptt, + PRS_REG_ROCE_DEST_QP_MAX_PF, + 0); + + ll2_ethertype_en = ecore_rd(p_hwfn, + p_ptt, + PRS_REG_LIGHT_L2_ETHERTYPE_EN); + + ecore_wr(p_hwfn, p_ptt, PRS_REG_LIGHT_L2_ETHERTYPE_EN, + (ll2_ethertype_en & 0xFFFE)); + +#ifndef REAL_ASIC_ONLY + /* INTERNAL: In CMT mode, re-initialize nig to direct packets to both + * enginesfor L2 performance, Roce requires all traffic to go just to + * engine 0. + */ + if (ECORE_IS_BB_A0(p_hwfn->p_dev) && ECORE_IS_CMT(p_hwfn->p_dev)) { + DP_ERR(p_hwfn->p_dev, + "On Everest 4 Big Bear Board revision A0 when RoCE driver is loaded L2 performance is sub-optimal (all traffic is routed to engine 0). For optimal L2 results either remove RoCE driver or use board revision B0\n"); + + ecore_wr(p_hwfn, + p_ptt, + NIG_REG_LLH_ENG_CLS_ENG_ID_TBL, + 0x55555555); + ecore_wr(p_hwfn, + p_ptt, + NIG_REG_LLH_ENG_CLS_ENG_ID_TBL + 0x4, + 0x55555555); + } +#endif + + if (IS_IWARP(p_hwfn)) { + rc = ecore_iwarp_stop(p_hwfn); + if (rc != ECORE_SUCCESS) { + ecore_ptt_release(p_hwfn, p_ptt); + return 0; + } + } else { + rc = ecore_roce_stop(p_hwfn); + if (rc != ECORE_SUCCESS) { + ecore_ptt_release(p_hwfn, p_ptt); + return 0; + } + } + + ecore_ptt_release(p_hwfn, p_ptt); + + /* Get SPQ entry */ + OSAL_MEMSET(&init_data, 0, sizeof(init_data)); + init_data.opaque_fid = p_hwfn->hw_info.opaque_fid; + init_data.comp_mode = ECORE_SPQ_MODE_EBLOCK; + + /* Stop RoCE */ + rc = ecore_sp_init_request(p_hwfn, &p_ent, RDMA_RAMROD_FUNC_CLOSE, + p_hwfn->p_rdma_info->proto, &init_data); + if (rc != ECORE_SUCCESS) + goto out; + + p_ramrod = &p_ent->ramrod.rdma_close_func; + + p_ramrod->num_cnqs = p_hwfn->p_rdma_info->num_cnqs; + p_ramrod->cnq_start_offset = (u8)RESC_START(p_hwfn, ECORE_RDMA_CNQ_RAM); + + rc = ecore_spq_post(p_hwfn, p_ent, OSAL_NULL); + +out: + ecore_rdma_free(p_hwfn); + + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "RDMA stop done, rc = %d\n", rc); + return rc; +} + +enum _ecore_status_t ecore_rdma_add_user(void *rdma_cxt, + struct ecore_rdma_add_user_out_params *out_params) +{ + struct ecore_hwfn *p_hwfn = (struct ecore_hwfn *)rdma_cxt; + u32 dpi_start_offset; + u32 returned_id = 0; + enum _ecore_status_t rc; + + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "Adding User\n"); + + /* Allocate DPI */ + OSAL_SPIN_LOCK(&p_hwfn->p_rdma_info->lock); + rc = ecore_rdma_bmap_alloc_id(p_hwfn, &p_hwfn->p_rdma_info->dpi_map, + &returned_id); + OSAL_SPIN_UNLOCK(&p_hwfn->p_rdma_info->lock); + + if (rc != ECORE_SUCCESS) + DP_NOTICE(p_hwfn, false, "Failed in allocating dpi\n"); + + out_params->dpi = (u16)returned_id; + + /* Calculate the corresponding DPI address */ + dpi_start_offset = p_hwfn->dpi_start_offset; + + out_params->dpi_addr = (u64)(osal_int_ptr_t)((u8 OSAL_IOMEM*)p_hwfn->doorbells + + dpi_start_offset + + ((out_params->dpi) * p_hwfn->dpi_size)); + + out_params->dpi_phys_addr = p_hwfn->db_phys_addr + dpi_start_offset + + out_params->dpi * p_hwfn->dpi_size; + + out_params->dpi_size = p_hwfn->dpi_size; + out_params->wid_count = p_hwfn->wid_count; + + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "Adding user - done, rc = %d\n", rc); + return rc; +} + +struct ecore_rdma_port *ecore_rdma_query_port(void *rdma_cxt) +{ + struct ecore_hwfn *p_hwfn = (struct ecore_hwfn *)rdma_cxt; + struct ecore_rdma_port *p_port = p_hwfn->p_rdma_info->port; + struct ecore_mcp_link_state *p_link_output; + + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "RDMA Query port\n"); + + /* The link state is saved only for the leading hwfn */ + p_link_output = + &ECORE_LEADING_HWFN(p_hwfn->p_dev)->mcp_info->link_output; + + /* Link may have changed... */ + p_port->port_state = p_link_output->link_up ? ECORE_RDMA_PORT_UP + : ECORE_RDMA_PORT_DOWN; + + p_port->link_speed = p_link_output->speed; + + p_port->max_msg_size = RDMA_MAX_DATA_SIZE_IN_WQE; + + return p_port; +} + +struct ecore_rdma_device *ecore_rdma_query_device(void *rdma_cxt) +{ + struct ecore_hwfn *p_hwfn = (struct ecore_hwfn *)rdma_cxt; + + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "Query device\n"); + + /* Return struct with device parameters */ + return p_hwfn->p_rdma_info->dev; +} + +void ecore_rdma_free_tid(void *rdma_cxt, + u32 itid) +{ + struct ecore_hwfn *p_hwfn = (struct ecore_hwfn *)rdma_cxt; + + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "itid = %08x\n", itid); + + OSAL_SPIN_LOCK(&p_hwfn->p_rdma_info->lock); + ecore_bmap_release_id(p_hwfn, + &p_hwfn->p_rdma_info->tid_map, + itid); + OSAL_SPIN_UNLOCK(&p_hwfn->p_rdma_info->lock); +} + +void ecore_rdma_cnq_prod_update(void *rdma_cxt, u8 qz_offset, u16 prod) +{ + struct ecore_hwfn *p_hwfn; + u16 qz_num; + u32 addr; + + p_hwfn = (struct ecore_hwfn *)rdma_cxt; + + if (qz_offset > p_hwfn->p_rdma_info->max_queue_zones) { + DP_NOTICE(p_hwfn, false, + "queue zone offset %d is too large (max is %d)\n", + qz_offset, p_hwfn->p_rdma_info->max_queue_zones); + return; + } + + qz_num = p_hwfn->p_rdma_info->queue_zone_base + qz_offset; + addr = GTT_BAR0_MAP_REG_USDM_RAM + + USTORM_COMMON_QUEUE_CONS_OFFSET(qz_num); + + REG_WR16(p_hwfn, addr, prod); + + /* keep prod updates ordered */ + OSAL_WMB(p_hwfn->p_dev); +} + +enum _ecore_status_t ecore_rdma_alloc_pd(void *rdma_cxt, + u16 *pd) +{ + struct ecore_hwfn *p_hwfn = (struct ecore_hwfn *)rdma_cxt; + u32 returned_id; + enum _ecore_status_t rc; + + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "Alloc PD\n"); + + /* Allocates an unused protection domain */ + OSAL_SPIN_LOCK(&p_hwfn->p_rdma_info->lock); + rc = ecore_rdma_bmap_alloc_id(p_hwfn, + &p_hwfn->p_rdma_info->pd_map, + &returned_id); + OSAL_SPIN_UNLOCK(&p_hwfn->p_rdma_info->lock); + if (rc != ECORE_SUCCESS) + DP_NOTICE(p_hwfn, false, "Failed in allocating pd id\n"); + + *pd = (u16)returned_id; + + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "Alloc PD - done, rc = %d\n", rc); + return rc; +} + +void ecore_rdma_free_pd(void *rdma_cxt, + u16 pd) +{ + struct ecore_hwfn *p_hwfn = (struct ecore_hwfn *)rdma_cxt; + + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "pd = %08x\n", pd); + + /* Returns a previously allocated protection domain for reuse */ + OSAL_SPIN_LOCK(&p_hwfn->p_rdma_info->lock); + ecore_bmap_release_id(p_hwfn, &p_hwfn->p_rdma_info->pd_map, pd); + OSAL_SPIN_UNLOCK(&p_hwfn->p_rdma_info->lock); +} + +enum _ecore_status_t ecore_rdma_alloc_xrcd(void *rdma_cxt, + u16 *xrcd_id) +{ + struct ecore_hwfn *p_hwfn = (struct ecore_hwfn *)rdma_cxt; + u32 returned_id; + enum _ecore_status_t rc; + + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "Alloc XRCD\n"); + + /* Allocates an unused XRC domain */ + OSAL_SPIN_LOCK(&p_hwfn->p_rdma_info->lock); + rc = ecore_rdma_bmap_alloc_id(p_hwfn, + &p_hwfn->p_rdma_info->xrcd_map, + &returned_id); + OSAL_SPIN_UNLOCK(&p_hwfn->p_rdma_info->lock); + if (rc != ECORE_SUCCESS) + DP_NOTICE(p_hwfn, false, "Failed in allocating xrcd id\n"); + + *xrcd_id = (u16)returned_id; + + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "Alloc XRCD - done, rc = %d\n", rc); + return rc; +} + +void ecore_rdma_free_xrcd(void *rdma_cxt, + u16 xrcd_id) +{ + struct ecore_hwfn *p_hwfn = (struct ecore_hwfn *)rdma_cxt; + + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "xrcd_id = %08x\n", xrcd_id); + + /* Returns a previously allocated protection domain for reuse */ + OSAL_SPIN_LOCK(&p_hwfn->p_rdma_info->lock); + ecore_bmap_release_id(p_hwfn, &p_hwfn->p_rdma_info->xrcd_map, xrcd_id); + OSAL_SPIN_UNLOCK(&p_hwfn->p_rdma_info->lock); +} + +static enum ecore_rdma_toggle_bit +ecore_rdma_toggle_bit_create_resize_cq(struct ecore_hwfn *p_hwfn, + u16 icid) +{ + struct ecore_rdma_info *p_info = p_hwfn->p_rdma_info; + enum ecore_rdma_toggle_bit toggle_bit; + u32 bmap_id; + + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "icid = %08x\n", icid); + + /* the function toggle the bit that is related to a given icid + * and returns the new toggle bit's value + */ + bmap_id = icid - ecore_cxt_get_proto_cid_start(p_hwfn, p_info->proto); + + OSAL_SPIN_LOCK(&p_info->lock); + toggle_bit = !OSAL_TEST_AND_FLIP_BIT(bmap_id, p_info->toggle_bits.bitmap); + OSAL_SPIN_UNLOCK(&p_info->lock); + + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "ECORE_RDMA_TOGGLE_BIT_= %d\n", + toggle_bit); + + return toggle_bit; +} + +enum _ecore_status_t ecore_rdma_create_cq(void *rdma_cxt, + struct ecore_rdma_create_cq_in_params *params, + u16 *icid) +{ + struct ecore_hwfn *p_hwfn = (struct ecore_hwfn *)rdma_cxt; + struct ecore_rdma_info *p_info = p_hwfn->p_rdma_info; + struct rdma_create_cq_ramrod_data *p_ramrod; + enum ecore_rdma_toggle_bit toggle_bit; + struct ecore_sp_init_data init_data; + struct ecore_spq_entry *p_ent; + enum _ecore_status_t rc; + u32 returned_id; + + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "cq_handle = %08x%08x\n", + params->cq_handle_hi, params->cq_handle_lo); + + /* Allocate icid */ + OSAL_SPIN_LOCK(&p_info->lock); + rc = ecore_rdma_bmap_alloc_id(p_hwfn, &p_info->cq_map, &returned_id); + OSAL_SPIN_UNLOCK(&p_info->lock); + + if (rc != ECORE_SUCCESS) + { + DP_NOTICE(p_hwfn, false, "Can't create CQ, rc = %d\n", rc); + return rc; + } + + *icid = (u16)(returned_id + + ecore_cxt_get_proto_cid_start( + p_hwfn, p_info->proto)); + + /* Check if icid requires a page allocation */ + rc = ecore_cxt_dynamic_ilt_alloc(p_hwfn, ECORE_ELEM_CXT, *icid); + if (rc != ECORE_SUCCESS) + goto err; + + /* Get SPQ entry */ + OSAL_MEMSET(&init_data, 0, sizeof(init_data)); + init_data.cid = *icid; + init_data.opaque_fid = p_hwfn->hw_info.opaque_fid; + init_data.comp_mode = ECORE_SPQ_MODE_EBLOCK; + + /* Send create CQ ramrod */ + rc = ecore_sp_init_request(p_hwfn, &p_ent, + RDMA_RAMROD_CREATE_CQ, + p_info->proto, &init_data); + if (rc != ECORE_SUCCESS) + goto err; + + p_ramrod = &p_ent->ramrod.rdma_create_cq; + + p_ramrod->cq_handle.hi = OSAL_CPU_TO_LE32(params->cq_handle_hi); + p_ramrod->cq_handle.lo = OSAL_CPU_TO_LE32(params->cq_handle_lo); + p_ramrod->dpi = OSAL_CPU_TO_LE16(params->dpi); + p_ramrod->is_two_level_pbl = params->pbl_two_level; + p_ramrod->max_cqes = OSAL_CPU_TO_LE32(params->cq_size); + DMA_REGPAIR_LE(p_ramrod->pbl_addr, params->pbl_ptr); + p_ramrod->pbl_num_pages = OSAL_CPU_TO_LE16(params->pbl_num_pages); + p_ramrod->cnq_id = (u8)RESC_START(p_hwfn, ECORE_RDMA_CNQ_RAM) + + params->cnq_id; + p_ramrod->int_timeout = params->int_timeout; + /* INTERNAL: Two layer PBL is currently not supported, ignoring next line */ + /* INTERNAL: p_ramrod->pbl_log_page_size = params->pbl_page_size_log - 12; */ + + /* toggle the bit for every resize or create cq for a given icid */ + toggle_bit = ecore_rdma_toggle_bit_create_resize_cq(p_hwfn, *icid); + + p_ramrod->toggle_bit = toggle_bit; + + rc = ecore_spq_post(p_hwfn, p_ent, OSAL_NULL); + if (rc != ECORE_SUCCESS) { + /* restore toggle bit */ + ecore_rdma_toggle_bit_create_resize_cq(p_hwfn, *icid); + goto err; + } + + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "Created CQ, rc = %d\n", rc); + return rc; + +err: + /* release allocated icid */ + OSAL_SPIN_LOCK(&p_info->lock); + ecore_bmap_release_id(p_hwfn, &p_info->cq_map, returned_id); + OSAL_SPIN_UNLOCK(&p_info->lock); + + DP_NOTICE(p_hwfn, false, "Create CQ failed, rc = %d\n", rc); + + return rc; +} + +enum _ecore_status_t ecore_rdma_destroy_cq(void *rdma_cxt, + struct ecore_rdma_destroy_cq_in_params *in_params, + struct ecore_rdma_destroy_cq_out_params *out_params) +{ + struct ecore_hwfn *p_hwfn = (struct ecore_hwfn *)rdma_cxt; + struct rdma_destroy_cq_output_params *p_ramrod_res; + struct rdma_destroy_cq_ramrod_data *p_ramrod; + struct ecore_sp_init_data init_data; + struct ecore_spq_entry *p_ent; + dma_addr_t ramrod_res_phys; + enum _ecore_status_t rc = ECORE_NOMEM; + + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "icid = %08x\n", in_params->icid); + + p_ramrod_res = (struct rdma_destroy_cq_output_params *) + OSAL_DMA_ALLOC_COHERENT(p_hwfn->p_dev, &ramrod_res_phys, + sizeof(struct rdma_destroy_cq_output_params)); + if (!p_ramrod_res) + { + DP_NOTICE(p_hwfn, false, + "ecore destroy cq failed: cannot allocate memory (ramrod)\n"); + return rc; + } + + /* Get SPQ entry */ + OSAL_MEMSET(&init_data, 0, sizeof(init_data)); + init_data.cid = in_params->icid; + init_data.opaque_fid = p_hwfn->hw_info.opaque_fid; + init_data.comp_mode = ECORE_SPQ_MODE_EBLOCK; + + /* Send destroy CQ ramrod */ + rc = ecore_sp_init_request(p_hwfn, &p_ent, + RDMA_RAMROD_DESTROY_CQ, + p_hwfn->p_rdma_info->proto, &init_data); + if (rc != ECORE_SUCCESS) + goto err; + + p_ramrod = &p_ent->ramrod.rdma_destroy_cq; + DMA_REGPAIR_LE(p_ramrod->output_params_addr, ramrod_res_phys); + + rc = ecore_spq_post(p_hwfn, p_ent, OSAL_NULL); + if (rc != ECORE_SUCCESS) + goto err; + + out_params->num_cq_notif = + OSAL_LE16_TO_CPU(p_ramrod_res->cnq_num); + + OSAL_DMA_FREE_COHERENT(p_hwfn->p_dev, p_ramrod_res, ramrod_res_phys, + sizeof(struct rdma_destroy_cq_output_params)); + + /* Free icid */ + OSAL_SPIN_LOCK(&p_hwfn->p_rdma_info->lock); + + ecore_bmap_release_id(p_hwfn, + &p_hwfn->p_rdma_info->cq_map, + (in_params->icid - ecore_cxt_get_proto_cid_start( + p_hwfn, p_hwfn->p_rdma_info->proto))); + + OSAL_SPIN_UNLOCK(&p_hwfn->p_rdma_info->lock); + + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "Destroyed CQ, rc = %d\n", rc); + return rc; + +err: + OSAL_DMA_FREE_COHERENT(p_hwfn->p_dev, p_ramrod_res, ramrod_res_phys, + sizeof(struct rdma_destroy_cq_output_params)); + + return rc; +} + +void ecore_rdma_set_fw_mac(u16 *p_fw_mac, u8 *p_ecore_mac) +{ + p_fw_mac[0] = OSAL_CPU_TO_LE16((p_ecore_mac[0] << 8) + p_ecore_mac[1]); + p_fw_mac[1] = OSAL_CPU_TO_LE16((p_ecore_mac[2] << 8) + p_ecore_mac[3]); + p_fw_mac[2] = OSAL_CPU_TO_LE16((p_ecore_mac[4] << 8) + p_ecore_mac[5]); +} + + +enum _ecore_status_t ecore_rdma_query_qp(void *rdma_cxt, + struct ecore_rdma_qp *qp, + struct ecore_rdma_query_qp_out_params *out_params) + +{ + struct ecore_hwfn *p_hwfn = (struct ecore_hwfn *)rdma_cxt; + enum _ecore_status_t rc = ECORE_SUCCESS; + + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "icid = %08x\n", qp->icid); + + /* The following fields are filled in from qp and not FW as they can't + * be modified by FW + */ + out_params->mtu = qp->mtu; + out_params->dest_qp = qp->dest_qp; + out_params->incoming_atomic_en = qp->incoming_atomic_en; + out_params->e2e_flow_control_en = qp->e2e_flow_control_en; + out_params->incoming_rdma_read_en = qp->incoming_rdma_read_en; + out_params->incoming_rdma_write_en = qp->incoming_rdma_write_en; + out_params->dgid = qp->dgid; + out_params->flow_label = qp->flow_label; + out_params->hop_limit_ttl = qp->hop_limit_ttl; + out_params->traffic_class_tos = qp->traffic_class_tos; + out_params->timeout = qp->ack_timeout; + out_params->rnr_retry = qp->rnr_retry_cnt; + out_params->retry_cnt = qp->retry_cnt; + out_params->min_rnr_nak_timer = qp->min_rnr_nak_timer; + out_params->pkey_index = 0; + out_params->max_rd_atomic = qp->max_rd_atomic_req; + out_params->max_dest_rd_atomic = qp->max_rd_atomic_resp; + out_params->sqd_async = qp->sqd_async; + + if (IS_IWARP(p_hwfn)) + rc = ecore_iwarp_query_qp(qp, out_params); + else + rc = ecore_roce_query_qp(p_hwfn, qp, out_params); + + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "Query QP, rc = %d\n", rc); + return rc; +} + + +enum _ecore_status_t ecore_rdma_destroy_qp(void *rdma_cxt, + struct ecore_rdma_qp *qp, + struct ecore_rdma_destroy_qp_out_params *out_params) +{ + struct ecore_hwfn *p_hwfn = (struct ecore_hwfn *)rdma_cxt; + enum _ecore_status_t rc = ECORE_SUCCESS; + + if (!rdma_cxt || !qp) { + DP_ERR(p_hwfn, + "ecore rdma destroy qp failed: invalid NULL input. rdma_cxt=%p, qp=%p\n", + rdma_cxt, qp); + return ECORE_INVAL; + } + + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "QP(0x%x)\n", qp->icid); + + if (IS_IWARP(p_hwfn)) + rc = ecore_iwarp_destroy_qp(p_hwfn, qp); + else + rc = ecore_roce_destroy_qp(p_hwfn, qp, out_params); + + /* free qp params struct */ + OSAL_FREE(p_hwfn->p_dev, qp); + + return rc; +} + + +struct ecore_rdma_qp *ecore_rdma_create_qp(void *rdma_cxt, + struct ecore_rdma_create_qp_in_params *in_params, + struct ecore_rdma_create_qp_out_params *out_params) +{ + struct ecore_hwfn *p_hwfn = (struct ecore_hwfn *)rdma_cxt; + struct ecore_rdma_qp *qp; + u8 max_stats_queues; + enum _ecore_status_t rc = 0; + + if (!rdma_cxt || !in_params || !out_params || !p_hwfn->p_rdma_info) { + DP_ERR(p_hwfn->p_dev, + "ecore roce create qp failed due to NULL entry (rdma_cxt=%p, in=%p, out=%p, roce_info=?\n", + rdma_cxt, + in_params, + out_params); + return OSAL_NULL; + } + + /* Some sanity checks... */ + max_stats_queues = p_hwfn->p_rdma_info->dev->max_stats_queues; + if (in_params->stats_queue >= max_stats_queues) { + DP_ERR(p_hwfn->p_dev, + "ecore rdma create qp failed due to invalid statistics queue %d. maximum is %d\n", + in_params->stats_queue, max_stats_queues); + return OSAL_NULL; + } + + if (IS_IWARP(p_hwfn)) { + if (in_params->sq_num_pages*sizeof(struct regpair) > + IWARP_SHARED_QUEUE_PAGE_SQ_PBL_MAX_SIZE) { + DP_NOTICE(p_hwfn->p_dev, true, "Sq num pages: %d exceeds maximum\n", + in_params->sq_num_pages); + return OSAL_NULL; + } + if (in_params->rq_num_pages*sizeof(struct regpair) > + IWARP_SHARED_QUEUE_PAGE_RQ_PBL_MAX_SIZE) { + DP_NOTICE(p_hwfn->p_dev, true, + "Rq num pages: %d exceeds maximum\n", + in_params->rq_num_pages); + return OSAL_NULL; + } + } + + qp = OSAL_ZALLOC(p_hwfn->p_dev, + GFP_KERNEL, + sizeof(struct ecore_rdma_qp)); + if (!qp) + { + DP_NOTICE(p_hwfn, false, "Failed to allocate ecore_rdma_qp\n"); + return OSAL_NULL; + } + + qp->cur_state = ECORE_ROCE_QP_STATE_RESET; +#ifdef CONFIG_ECORE_IWARP + qp->iwarp_state = ECORE_IWARP_QP_STATE_IDLE; +#endif + qp->qp_handle.hi = OSAL_CPU_TO_LE32(in_params->qp_handle_hi); + qp->qp_handle.lo = OSAL_CPU_TO_LE32(in_params->qp_handle_lo); + qp->qp_handle_async.hi = OSAL_CPU_TO_LE32(in_params->qp_handle_async_hi); + qp->qp_handle_async.lo = OSAL_CPU_TO_LE32(in_params->qp_handle_async_lo); + qp->use_srq = in_params->use_srq; + qp->signal_all = in_params->signal_all; + qp->fmr_and_reserved_lkey = in_params->fmr_and_reserved_lkey; + qp->pd = in_params->pd; + qp->dpi = in_params->dpi; + qp->sq_cq_id = in_params->sq_cq_id; + qp->sq_num_pages = in_params->sq_num_pages; + qp->sq_pbl_ptr = in_params->sq_pbl_ptr; + qp->rq_cq_id = in_params->rq_cq_id; + qp->rq_num_pages = in_params->rq_num_pages; + qp->rq_pbl_ptr = in_params->rq_pbl_ptr; + qp->srq_id = in_params->srq_id; + qp->req_offloaded = false; + qp->resp_offloaded = false; + /* e2e_flow_control cannot be done in case of S-RQ. + * Refer to 9.7.7.2 End-to-End Flow Control section of IB spec + */ + qp->e2e_flow_control_en = qp->use_srq ? false : true; + qp->stats_queue = in_params->stats_queue; + qp->qp_type = in_params->qp_type; + qp->xrcd_id = in_params->xrcd_id; + + if (IS_IWARP(p_hwfn)) { + rc = ecore_iwarp_create_qp(p_hwfn, qp, out_params); + qp->qpid = qp->icid; + } else { + rc = ecore_roce_alloc_qp_idx(p_hwfn, &qp->qp_idx); + qp->icid = ECORE_ROCE_QP_TO_ICID(qp->qp_idx); + qp->qpid = ((0xFF << 16) | qp->icid); + } + + if (rc != ECORE_SUCCESS) { + OSAL_FREE(p_hwfn->p_dev, qp); + return OSAL_NULL; + } + + out_params->icid = qp->icid; + out_params->qp_id = qp->qpid; + + /* INTERNAL: max_sq_sges future use only*/ + + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "Create QP, rc = %d\n", rc); + return qp; +} + +#define ECORE_RDMA_ECN_SHIFT 0 +#define ECORE_RDMA_ECN_MASK 0x3 +#define ECORE_RDMA_DSCP_SHIFT 2 +#define ECORE_RDMA_DSCP_MASK 0x3f +#define ECORE_RDMA_VLAN_PRIO_SHIFT 13 +#define ECORE_RDMA_VLAN_PRIO_MASK 0x7 +enum _ecore_status_t ecore_rdma_modify_qp( + void *rdma_cxt, + struct ecore_rdma_qp *qp, + struct ecore_rdma_modify_qp_in_params *params) +{ + struct ecore_hwfn *p_hwfn = (struct ecore_hwfn *)rdma_cxt; + enum ecore_roce_qp_state prev_state; + enum _ecore_status_t rc = ECORE_SUCCESS; + + if (GET_FIELD(params->modify_flags, + ECORE_RDMA_MODIFY_QP_VALID_RDMA_OPS_EN)) + { + qp->incoming_rdma_read_en = params->incoming_rdma_read_en; + qp->incoming_rdma_write_en = params->incoming_rdma_write_en; + qp->incoming_atomic_en = params->incoming_atomic_en; + } + + /* Update QP structure with the updated values */ + if (GET_FIELD(params->modify_flags, + ECORE_ROCE_MODIFY_QP_VALID_ROCE_MODE)) + { + qp->roce_mode = params->roce_mode; + } + if (GET_FIELD(params->modify_flags, ECORE_ROCE_MODIFY_QP_VALID_PKEY)) + { + qp->pkey = params->pkey; + } + if (GET_FIELD(params->modify_flags, + ECORE_ROCE_MODIFY_QP_VALID_E2E_FLOW_CONTROL_EN)) + { + qp->e2e_flow_control_en = params->e2e_flow_control_en; + } + if (GET_FIELD(params->modify_flags, + ECORE_ROCE_MODIFY_QP_VALID_DEST_QP)) + { + qp->dest_qp = params->dest_qp; + } + if (GET_FIELD(params->modify_flags, + ECORE_ROCE_MODIFY_QP_VALID_ADDRESS_VECTOR)) + { + /* Indicates that the following parameters have changed: + * Traffic class, flow label, hop limit, source GID, + * destination GID, loopback indicator + */ + qp->flow_label = params->flow_label; + qp->hop_limit_ttl = params->hop_limit_ttl; + + qp->sgid = params->sgid; + qp->dgid = params->dgid; + qp->udp_src_port = params->udp_src_port; + qp->vlan_id = params->vlan_id; + qp->traffic_class_tos = params->traffic_class_tos; + + /* apply global override values */ + if (p_hwfn->p_rdma_info->glob_cfg.vlan_pri_en) + SET_FIELD(qp->vlan_id, ECORE_RDMA_VLAN_PRIO, + p_hwfn->p_rdma_info->glob_cfg.vlan_pri); + + if (p_hwfn->p_rdma_info->glob_cfg.ecn_en) + SET_FIELD(qp->traffic_class_tos, ECORE_RDMA_ECN, + p_hwfn->p_rdma_info->glob_cfg.ecn); + + if (p_hwfn->p_rdma_info->glob_cfg.dscp_en) + SET_FIELD(qp->traffic_class_tos, ECORE_RDMA_DSCP, + p_hwfn->p_rdma_info->glob_cfg.dscp); + + qp->mtu = params->mtu; + + OSAL_MEMCPY((u8 *)&qp->remote_mac_addr[0], + (u8 *)¶ms->remote_mac_addr[0], ETH_ALEN); + if (params->use_local_mac) { + OSAL_MEMCPY((u8 *)&qp->local_mac_addr[0], + (u8 *)¶ms->local_mac_addr[0], + ETH_ALEN); + } else { + OSAL_MEMCPY((u8 *)&qp->local_mac_addr[0], + (u8 *)&p_hwfn->hw_info.hw_mac_addr, + ETH_ALEN); + } + } + if (GET_FIELD(params->modify_flags, ECORE_ROCE_MODIFY_QP_VALID_RQ_PSN)) + { + qp->rq_psn = params->rq_psn; + } + if (GET_FIELD(params->modify_flags, ECORE_ROCE_MODIFY_QP_VALID_SQ_PSN)) + { + qp->sq_psn = params->sq_psn; + } + if (GET_FIELD(params->modify_flags, + ECORE_RDMA_MODIFY_QP_VALID_MAX_RD_ATOMIC_REQ)) + { + qp->max_rd_atomic_req = params->max_rd_atomic_req; + } + if (GET_FIELD(params->modify_flags, + ECORE_RDMA_MODIFY_QP_VALID_MAX_RD_ATOMIC_RESP)) + { + qp->max_rd_atomic_resp = params->max_rd_atomic_resp; + } + if (GET_FIELD(params->modify_flags, + ECORE_ROCE_MODIFY_QP_VALID_ACK_TIMEOUT)) + { + qp->ack_timeout = params->ack_timeout; + } + if (GET_FIELD(params->modify_flags, + ECORE_ROCE_MODIFY_QP_VALID_RETRY_CNT)) + { + qp->retry_cnt = params->retry_cnt; + } + if (GET_FIELD(params->modify_flags, + ECORE_ROCE_MODIFY_QP_VALID_RNR_RETRY_CNT)) + { + qp->rnr_retry_cnt = params->rnr_retry_cnt; + } + if (GET_FIELD(params->modify_flags, + ECORE_ROCE_MODIFY_QP_VALID_MIN_RNR_NAK_TIMER)) + { + qp->min_rnr_nak_timer = params->min_rnr_nak_timer; + } + + qp->sqd_async = params->sqd_async; + + prev_state = qp->cur_state; + if (GET_FIELD(params->modify_flags, + ECORE_RDMA_MODIFY_QP_VALID_NEW_STATE)) + { + qp->cur_state = params->new_state; + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "qp->cur_state=%d\n", + qp->cur_state); + } + + if (qp->qp_type == ECORE_RDMA_QP_TYPE_XRC_INI) { + qp->has_req = 1; + } else if (qp->qp_type == ECORE_RDMA_QP_TYPE_XRC_TGT) + { + qp->has_resp = 1; + } else { + qp->has_req = 1; + qp->has_resp = 1; + } + + if (IS_IWARP(p_hwfn)) { + enum ecore_iwarp_qp_state new_state = + ecore_roce2iwarp_state(qp->cur_state); + + rc = ecore_iwarp_modify_qp(p_hwfn, qp, new_state, 0); + } else { + rc = ecore_roce_modify_qp(p_hwfn, qp, prev_state, params); + } + + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "Modify QP, rc = %d\n", rc); + return rc; +} + +enum _ecore_status_t ecore_rdma_register_tid(void *rdma_cxt, + struct ecore_rdma_register_tid_in_params *params) +{ + struct ecore_hwfn *p_hwfn = (struct ecore_hwfn *)rdma_cxt; + struct rdma_register_tid_ramrod_data *p_ramrod; + struct ecore_sp_init_data init_data; + struct ecore_spq_entry *p_ent; + enum rdma_tid_type tid_type; + u8 fw_return_code; + enum _ecore_status_t rc; + + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "itid = %08x\n", params->itid); + + /* Get SPQ entry */ + OSAL_MEMSET(&init_data, 0, sizeof(init_data)); + init_data.opaque_fid = p_hwfn->hw_info.opaque_fid; + init_data.comp_mode = ECORE_SPQ_MODE_EBLOCK; + + rc = ecore_sp_init_request(p_hwfn, &p_ent, RDMA_RAMROD_REGISTER_MR, + p_hwfn->p_rdma_info->proto, &init_data); + if (rc != ECORE_SUCCESS) { + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "rc = %d\n", rc); + return rc; + } + + if (p_hwfn->p_rdma_info->last_tid < params->itid) { + p_hwfn->p_rdma_info->last_tid = params->itid; + } + + p_ramrod = &p_ent->ramrod.rdma_register_tid; + + p_ramrod->flags = 0; + SET_FIELD(p_ramrod->flags, + RDMA_REGISTER_TID_RAMROD_DATA_TWO_LEVEL_PBL, + params->pbl_two_level); + + SET_FIELD(p_ramrod->flags, + RDMA_REGISTER_TID_RAMROD_DATA_ZERO_BASED, + params->zbva); + + SET_FIELD(p_ramrod->flags, + RDMA_REGISTER_TID_RAMROD_DATA_PHY_MR, + params->phy_mr); + + /* Don't initialize D/C field, as it may override other bits. */ + if (!(params->tid_type == ECORE_RDMA_TID_FMR) && + !(params->dma_mr)) + SET_FIELD(p_ramrod->flags, + RDMA_REGISTER_TID_RAMROD_DATA_PAGE_SIZE_LOG, + params->page_size_log - 12); + + SET_FIELD(p_ramrod->flags, + RDMA_REGISTER_TID_RAMROD_DATA_REMOTE_READ, + params->remote_read); + + SET_FIELD(p_ramrod->flags, + RDMA_REGISTER_TID_RAMROD_DATA_REMOTE_WRITE, + params->remote_write); + + SET_FIELD(p_ramrod->flags, + RDMA_REGISTER_TID_RAMROD_DATA_REMOTE_ATOMIC, + params->remote_atomic); + + SET_FIELD(p_ramrod->flags, + RDMA_REGISTER_TID_RAMROD_DATA_LOCAL_WRITE, + params->local_write); + + SET_FIELD(p_ramrod->flags, + RDMA_REGISTER_TID_RAMROD_DATA_LOCAL_READ, + params->local_read); + + SET_FIELD(p_ramrod->flags, + RDMA_REGISTER_TID_RAMROD_DATA_ENABLE_MW_BIND, + params->mw_bind); + + SET_FIELD(p_ramrod->flags1, + RDMA_REGISTER_TID_RAMROD_DATA_PBL_PAGE_SIZE_LOG, + params->pbl_page_size_log - 12); + + SET_FIELD(p_ramrod->flags2, + RDMA_REGISTER_TID_RAMROD_DATA_DMA_MR, + params->dma_mr); + + switch (params->tid_type) + { + case ECORE_RDMA_TID_REGISTERED_MR: + tid_type = RDMA_TID_REGISTERED_MR; + break; + case ECORE_RDMA_TID_FMR: + tid_type = RDMA_TID_FMR; + break; + case ECORE_RDMA_TID_MW_TYPE1: + tid_type = RDMA_TID_MW_TYPE1; + break; + case ECORE_RDMA_TID_MW_TYPE2A: + tid_type = RDMA_TID_MW_TYPE2A; + break; + default: + rc = ECORE_INVAL; + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "rc = %d\n", rc); + return rc; + } + SET_FIELD(p_ramrod->flags1, + RDMA_REGISTER_TID_RAMROD_DATA_TID_TYPE, + tid_type); + + p_ramrod->itid = OSAL_CPU_TO_LE32(params->itid); + p_ramrod->key = params->key; + p_ramrod->pd = OSAL_CPU_TO_LE16(params->pd); + p_ramrod->length_hi = (u8)(params->length >> 32); + p_ramrod->length_lo = DMA_LO_LE(params->length); + if (params->zbva) + { + /* Lower 32 bits of the registered MR address. + * In case of zero based MR, will hold FBO + */ + p_ramrod->va.hi = 0; + p_ramrod->va.lo = OSAL_CPU_TO_LE32(params->fbo); + } else { + DMA_REGPAIR_LE(p_ramrod->va, params->vaddr); + } + DMA_REGPAIR_LE(p_ramrod->pbl_base, params->pbl_ptr); + + /* DIF */ + if (params->dif_enabled) { + SET_FIELD(p_ramrod->flags2, + RDMA_REGISTER_TID_RAMROD_DATA_DIF_ON_HOST_FLG, 1); + DMA_REGPAIR_LE(p_ramrod->dif_error_addr, + params->dif_error_addr); + DMA_REGPAIR_LE(p_ramrod->dif_runt_addr, params->dif_runt_addr); + } + + rc = ecore_spq_post(p_hwfn, p_ent, &fw_return_code); + if (rc) + return rc; + + if (fw_return_code != RDMA_RETURN_OK) { + DP_NOTICE(p_hwfn, true, "fw_return_code = %d\n", fw_return_code); + return ECORE_UNKNOWN_ERROR; + } + + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "Register TID, rc = %d\n", rc); + return rc; +} + +static OSAL_INLINE int ecore_rdma_send_deregister_tid_ramrod( + struct ecore_hwfn *p_hwfn, + u32 itid, + u8 *fw_return_code) +{ + struct ecore_sp_init_data init_data; + struct rdma_deregister_tid_ramrod_data *p_ramrod; + struct ecore_spq_entry *p_ent; + enum _ecore_status_t rc; + + /* Get SPQ entry */ + OSAL_MEMSET(&init_data, 0, sizeof(init_data)); + init_data.opaque_fid = p_hwfn->hw_info.opaque_fid; + init_data.comp_mode = ECORE_SPQ_MODE_EBLOCK; + + rc = ecore_sp_init_request(p_hwfn, &p_ent, + RDMA_RAMROD_DEREGISTER_MR, + p_hwfn->p_rdma_info->proto, &init_data); + if (rc != ECORE_SUCCESS) { + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "rc = %d\n", rc); + return rc; + } + + p_ramrod = &p_ent->ramrod.rdma_deregister_tid; + p_ramrod->itid = OSAL_CPU_TO_LE32(itid); + + rc = ecore_spq_post(p_hwfn, p_ent, fw_return_code); + if (rc != ECORE_SUCCESS) + { + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "rc = %d\n", rc); + return rc; + } + + return rc; +} + +#define ECORE_RDMA_DEREGISTER_TIMEOUT_MSEC (1) + +enum _ecore_status_t ecore_rdma_deregister_tid(void *rdma_cxt, + u32 itid) +{ + enum _ecore_status_t rc; + u8 fw_ret_code; + struct ecore_ptt *p_ptt; + struct ecore_hwfn *p_hwfn = (struct ecore_hwfn *)rdma_cxt; + + /* First attempt */ + rc = ecore_rdma_send_deregister_tid_ramrod(p_hwfn, itid, &fw_ret_code); + if (rc != ECORE_SUCCESS) + return rc; + + if (fw_ret_code != RDMA_RETURN_NIG_DRAIN_REQ) + goto done; + + /* Second attempt, after 1msec, if device still holds data. + * This can occur since 'destroy QP' returns to the caller rather fast. + * The synchronous part of it returns after freeing a few of the + * resources but not all of them, allowing the consumer to continue its + * flow. All of the resources will be freed after the asynchronous part + * of the destroy QP is complete. + */ + OSAL_MSLEEP(ECORE_RDMA_DEREGISTER_TIMEOUT_MSEC); + rc = ecore_rdma_send_deregister_tid_ramrod(p_hwfn, itid, &fw_ret_code); + if (rc != ECORE_SUCCESS) + return rc; + + if (fw_ret_code != RDMA_RETURN_NIG_DRAIN_REQ) + goto done; + + /* Third and last attempt, perform NIG drain and resend the ramrod */ + p_ptt = ecore_ptt_acquire(p_hwfn); + if (!p_ptt) + return ECORE_TIMEOUT; + + rc = ecore_mcp_drain(p_hwfn, p_ptt); + if (rc != ECORE_SUCCESS) { + ecore_ptt_release(p_hwfn, p_ptt); + return rc; + } + + ecore_ptt_release(p_hwfn, p_ptt); + + rc = ecore_rdma_send_deregister_tid_ramrod(p_hwfn, itid, &fw_ret_code); + if (rc != ECORE_SUCCESS) + return rc; + +done: + if (fw_ret_code == RDMA_RETURN_OK) { + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "De-registered itid=%d\n", + itid); + return ECORE_SUCCESS; + } else if (fw_ret_code == RDMA_RETURN_DEREGISTER_MR_BAD_STATE_ERR) { + /* INTERNAL: This error is returned in case trying to deregister + * a MR that is not allocated. We define "allocated" as either: + * 1. Registered. + * 2. This is an FMR MR type, which is not currently registered + * but can accept FMR WQEs on SQ. + */ + DP_NOTICE(p_hwfn, false, "itid=%d, fw_ret_code=%d\n", itid, + fw_ret_code); + return ECORE_INVAL; + } else { /* fw_ret_code == RDMA_RETURN_NIG_DRAIN_REQ */ + DP_NOTICE(p_hwfn, true, + "deregister failed after three attempts. itid=%d, fw_ret_code=%d\n", + itid, fw_ret_code); + return ECORE_UNKNOWN_ERROR; + } +} + +static struct ecore_bmap *ecore_rdma_get_srq_bmap(struct ecore_hwfn *p_hwfn, bool is_xrc) +{ + if (is_xrc) + return &p_hwfn->p_rdma_info->xrc_srq_map; + + return &p_hwfn->p_rdma_info->srq_map; +} + +u16 ecore_rdma_get_fw_srq_id(struct ecore_hwfn *p_hwfn, u16 id, bool is_xrc) +{ + if (is_xrc) + return id; + + return id + p_hwfn->p_rdma_info->srq_id_offset; +} + +enum _ecore_status_t +ecore_rdma_modify_srq(void *rdma_cxt, + struct ecore_rdma_modify_srq_in_params *in_params) +{ + struct ecore_hwfn *p_hwfn = (struct ecore_hwfn *)rdma_cxt; + struct rdma_srq_modify_ramrod_data *p_ramrod; + struct ecore_sp_init_data init_data; + struct ecore_spq_entry *p_ent; + u16 opaque_fid, fw_srq_id; + enum _ecore_status_t rc; + + OSAL_MEMSET(&init_data, 0, sizeof(init_data)); + init_data.opaque_fid = p_hwfn->hw_info.opaque_fid; + init_data.comp_mode = ECORE_SPQ_MODE_EBLOCK; + /* Send modify SRQ ramrod */ + rc = ecore_sp_init_request(p_hwfn, &p_ent, + RDMA_RAMROD_MODIFY_SRQ, + p_hwfn->p_rdma_info->proto, &init_data); + if (rc != ECORE_SUCCESS) + return rc; + + p_ramrod = &p_ent->ramrod.rdma_modify_srq; + + fw_srq_id = ecore_rdma_get_fw_srq_id(p_hwfn, in_params->srq_id, + in_params->is_xrc); + p_ramrod->srq_id.srq_idx = OSAL_CPU_TO_LE16(fw_srq_id); + opaque_fid = p_hwfn->hw_info.opaque_fid; + p_ramrod->srq_id.opaque_fid = OSAL_CPU_TO_LE16(opaque_fid); + p_ramrod->wqe_limit = OSAL_CPU_TO_LE16(in_params->wqe_limit); + + rc = ecore_spq_post(p_hwfn, p_ent, OSAL_NULL); + if (rc != ECORE_SUCCESS) + return rc; + + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "modified SRQ id = %x, is_xrc=%u\n", + in_params->srq_id, in_params->is_xrc); + + return rc; +} + +enum _ecore_status_t +ecore_rdma_destroy_srq(void *rdma_cxt, + struct ecore_rdma_destroy_srq_in_params *in_params) +{ + struct ecore_hwfn *p_hwfn = (struct ecore_hwfn *)rdma_cxt; + struct rdma_srq_destroy_ramrod_data *p_ramrod; + struct ecore_sp_init_data init_data; + struct ecore_spq_entry *p_ent; + u16 opaque_fid, fw_srq_id; + struct ecore_bmap *bmap; + enum _ecore_status_t rc; + + opaque_fid = p_hwfn->hw_info.opaque_fid; + + OSAL_MEMSET(&init_data, 0, sizeof(init_data)); + init_data.opaque_fid = opaque_fid; + init_data.comp_mode = ECORE_SPQ_MODE_EBLOCK; + + /* Send destroy SRQ ramrod */ + rc = ecore_sp_init_request(p_hwfn, &p_ent, + RDMA_RAMROD_DESTROY_SRQ, + p_hwfn->p_rdma_info->proto, &init_data); + if (rc != ECORE_SUCCESS) + return rc; + + p_ramrod = &p_ent->ramrod.rdma_destroy_srq; + + fw_srq_id = ecore_rdma_get_fw_srq_id(p_hwfn, in_params->srq_id, + in_params->is_xrc); + p_ramrod->srq_id.srq_idx = OSAL_CPU_TO_LE16(fw_srq_id); + p_ramrod->srq_id.opaque_fid = OSAL_CPU_TO_LE16(opaque_fid); + + rc = ecore_spq_post(p_hwfn, p_ent, OSAL_NULL); + + if (rc != ECORE_SUCCESS) + return rc; + + bmap = ecore_rdma_get_srq_bmap(p_hwfn, in_params->is_xrc); + + OSAL_SPIN_LOCK(&p_hwfn->p_rdma_info->lock); + ecore_bmap_release_id(p_hwfn, bmap, in_params->srq_id); + OSAL_SPIN_UNLOCK(&p_hwfn->p_rdma_info->lock); + + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, + "XRC/SRQ destroyed Id = %x, is_xrc=%u\n", + in_params->srq_id, in_params->is_xrc); + + return rc; +} + +enum _ecore_status_t +ecore_rdma_create_srq(void *rdma_cxt, + struct ecore_rdma_create_srq_in_params *in_params, + struct ecore_rdma_create_srq_out_params *out_params) +{ + struct ecore_hwfn *p_hwfn = (struct ecore_hwfn *)rdma_cxt; + struct rdma_srq_create_ramrod_data *p_ramrod; + struct ecore_sp_init_data init_data; + enum ecore_cxt_elem_type elem_type; + struct ecore_spq_entry *p_ent; + u16 opaque_fid, fw_srq_id; + struct ecore_bmap *bmap; + u32 returned_id; + enum _ecore_status_t rc; + + /* Allocate XRC/SRQ ID */ + bmap = ecore_rdma_get_srq_bmap(p_hwfn, in_params->is_xrc); + OSAL_SPIN_LOCK(&p_hwfn->p_rdma_info->lock); + rc = ecore_rdma_bmap_alloc_id(p_hwfn, bmap, &returned_id); + OSAL_SPIN_UNLOCK(&p_hwfn->p_rdma_info->lock); + + if (rc != ECORE_SUCCESS) { + DP_NOTICE(p_hwfn, false, + "failed to allocate xrc/srq id (is_xrc=%u)\n", + in_params->is_xrc); + return rc; + } + /* Allocate XRC/SRQ ILT page */ + elem_type = (in_params->is_xrc) ? (ECORE_ELEM_XRC_SRQ) : (ECORE_ELEM_SRQ); + rc = ecore_cxt_dynamic_ilt_alloc(p_hwfn, elem_type, returned_id); + if (rc != ECORE_SUCCESS) + goto err; + + OSAL_MEMSET(&init_data, 0, sizeof(init_data)); + opaque_fid = p_hwfn->hw_info.opaque_fid; + init_data.opaque_fid = opaque_fid; + init_data.comp_mode = ECORE_SPQ_MODE_EBLOCK; + + /* Create XRC/SRQ ramrod */ + rc = ecore_sp_init_request(p_hwfn, &p_ent, + RDMA_RAMROD_CREATE_SRQ, + p_hwfn->p_rdma_info->proto, &init_data); + if (rc != ECORE_SUCCESS) + goto err; + + p_ramrod = &p_ent->ramrod.rdma_create_srq; + + p_ramrod->pbl_base_addr.hi = DMA_HI_LE(in_params->pbl_base_addr); + p_ramrod->pbl_base_addr.lo = DMA_LO_LE(in_params->pbl_base_addr); + p_ramrod->pages_in_srq_pbl = OSAL_CPU_TO_LE16(in_params->num_pages); + p_ramrod->pd_id = OSAL_CPU_TO_LE16(in_params->pd_id); + p_ramrod->srq_id.opaque_fid = OSAL_CPU_TO_LE16(opaque_fid); + p_ramrod->page_size = OSAL_CPU_TO_LE16(in_params->page_size); + p_ramrod->producers_addr.hi = DMA_HI_LE(in_params->prod_pair_addr); + p_ramrod->producers_addr.lo = DMA_LO_LE(in_params->prod_pair_addr); + fw_srq_id = ecore_rdma_get_fw_srq_id(p_hwfn, (u16) returned_id, + in_params->is_xrc); + p_ramrod->srq_id.srq_idx = OSAL_CPU_TO_LE16(fw_srq_id); + + if (in_params->is_xrc) { + SET_FIELD(p_ramrod->flags, + RDMA_SRQ_CREATE_RAMROD_DATA_XRC_FLAG, + 1); + SET_FIELD(p_ramrod->flags, + RDMA_SRQ_CREATE_RAMROD_DATA_RESERVED_KEY_EN, + in_params->reserved_key_en); + p_ramrod->xrc_srq_cq_cid = OSAL_CPU_TO_LE32(in_params->cq_cid); + p_ramrod->xrc_domain = OSAL_CPU_TO_LE16(in_params->xrcd_id); + } + + rc = ecore_spq_post(p_hwfn, p_ent, OSAL_NULL); + + if (rc != ECORE_SUCCESS) + goto err; + + out_params->srq_id = (u16)returned_id; + + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "XRC/SRQ created Id = %x (is_xrc=%u)\n", + out_params->srq_id, in_params->is_xrc); + return rc; + +err: + OSAL_SPIN_LOCK(&p_hwfn->p_rdma_info->lock); + ecore_bmap_release_id(p_hwfn, bmap, returned_id); + OSAL_SPIN_UNLOCK(&p_hwfn->p_rdma_info->lock); + + return rc; +} + +bool ecore_rdma_allocated_qps(struct ecore_hwfn *p_hwfn) +{ + bool result; + + /* if rdma info has not been allocated, naturally there are no qps */ + if (!p_hwfn->p_rdma_info) + return false; + + OSAL_SPIN_LOCK(&p_hwfn->p_rdma_info->lock); + if (!p_hwfn->p_rdma_info->qp_map.bitmap) + result = false; + else + result = !ecore_bmap_is_empty(&p_hwfn->p_rdma_info->qp_map); + OSAL_SPIN_UNLOCK(&p_hwfn->p_rdma_info->lock); + return result; +} + +enum _ecore_status_t ecore_rdma_resize_cq(void *rdma_cxt, + struct ecore_rdma_resize_cq_in_params *in_params, + struct ecore_rdma_resize_cq_out_params *out_params) +{ + enum _ecore_status_t rc; + enum ecore_rdma_toggle_bit toggle_bit; + struct ecore_spq_entry *p_ent; + struct rdma_resize_cq_ramrod_data *p_ramrod; + u8 fw_return_code; + struct ecore_hwfn *p_hwfn = (struct ecore_hwfn *)rdma_cxt; + dma_addr_t ramrod_res_phys; + struct rdma_resize_cq_output_params *p_ramrod_res; + struct ecore_sp_init_data init_data; + + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "icid = %08x\n", in_params->icid); + + /* Send resize CQ ramrod */ + + p_ramrod_res = (struct rdma_resize_cq_output_params *) + OSAL_DMA_ALLOC_COHERENT(p_hwfn->p_dev, &ramrod_res_phys, + sizeof(*p_ramrod_res)); + if (!p_ramrod_res) + { + rc = ECORE_NOMEM; + DP_NOTICE(p_hwfn, false, + "ecore resize cq failed: cannot allocate memory (ramrod). rc = %d\n", + rc); + return rc; + } + + /* Get SPQ entry */ + OSAL_MEMSET(&init_data, 0, sizeof(init_data)); + init_data.cid = in_params->icid; + init_data.opaque_fid = p_hwfn->hw_info.opaque_fid; + init_data.comp_mode = ECORE_SPQ_MODE_EBLOCK; + + rc = ecore_sp_init_request(p_hwfn, &p_ent, + RDMA_RAMROD_RESIZE_CQ, + p_hwfn->p_rdma_info->proto, &init_data); + if (rc != ECORE_SUCCESS) + goto err; + + p_ramrod = &p_ent->ramrod.rdma_resize_cq; + + p_ramrod->flags = 0; + + /* toggle the bit for every resize or create cq for a given icid */ + toggle_bit = ecore_rdma_toggle_bit_create_resize_cq(p_hwfn, + in_params->icid); + + SET_FIELD(p_ramrod->flags, + RDMA_RESIZE_CQ_RAMROD_DATA_TOGGLE_BIT, + toggle_bit); + + SET_FIELD(p_ramrod->flags, + RDMA_RESIZE_CQ_RAMROD_DATA_IS_TWO_LEVEL_PBL, + in_params->pbl_two_level); + + p_ramrod->pbl_log_page_size = in_params->pbl_page_size_log - 12; + p_ramrod->pbl_num_pages = OSAL_CPU_TO_LE16(in_params->pbl_num_pages); + p_ramrod->max_cqes = OSAL_CPU_TO_LE32(in_params->cq_size); + p_ramrod->pbl_addr.hi = DMA_HI_LE(in_params->pbl_ptr); + p_ramrod->pbl_addr.lo = DMA_LO_LE(in_params->pbl_ptr); + + p_ramrod->output_params_addr.hi = DMA_HI_LE(ramrod_res_phys); + p_ramrod->output_params_addr.lo = DMA_LO_LE(ramrod_res_phys); + + rc = ecore_spq_post(p_hwfn, p_ent, &fw_return_code); + if (rc != ECORE_SUCCESS) + goto err; + + if (fw_return_code != RDMA_RETURN_OK) + { + DP_NOTICE(p_hwfn, fw_return_code != RDMA_RETURN_RESIZE_CQ_ERR, + "fw_return_code = %d\n", fw_return_code); + DP_NOTICE(p_hwfn, + true, "fw_return_code = %d\n", fw_return_code); + rc = ECORE_UNKNOWN_ERROR; + goto err; + } + + out_params->prod = OSAL_LE32_TO_CPU(p_ramrod_res->old_cq_prod); + out_params->cons = OSAL_LE32_TO_CPU(p_ramrod_res->old_cq_cons); + + OSAL_DMA_FREE_COHERENT(p_hwfn->p_dev, p_ramrod_res, ramrod_res_phys, + sizeof(*p_ramrod_res)); + + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "rc = %d\n", rc); + + return rc; + +err: + OSAL_DMA_FREE_COHERENT(p_hwfn->p_dev, p_ramrod_res, ramrod_res_phys, + sizeof(*p_ramrod_res)); + DP_NOTICE(p_hwfn, false, "rc = %d\n", rc); + + return rc; +} + +enum _ecore_status_t ecore_rdma_start(void *rdma_cxt, + struct ecore_rdma_start_in_params *params) +{ + struct ecore_hwfn *p_hwfn = (struct ecore_hwfn *)rdma_cxt; + struct ecore_ptt *p_ptt; + enum _ecore_status_t rc = ECORE_TIMEOUT; + + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, + "desired_cnq = %08x\n", params->desired_cnq); + + p_ptt = ecore_ptt_acquire(p_hwfn); + if (!p_ptt) + goto err; + + rc = ecore_rdma_alloc(p_hwfn); + if (rc) + goto err1; + + rc = ecore_rdma_setup(p_hwfn, p_ptt, params); + if (rc) + goto err2; + + ecore_ptt_release(p_hwfn, p_ptt); + + ecore_rdma_activate(p_hwfn); + return rc; + +err2: + ecore_rdma_free(p_hwfn); +err1: + ecore_ptt_release(p_hwfn, p_ptt); +err: + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "RDMA start - error, rc = %d\n", rc); + return rc; +} + +enum _ecore_status_t ecore_rdma_query_stats(void *rdma_cxt, u8 stats_queue, + struct ecore_rdma_stats_out_params *out_params) +{ + struct ecore_hwfn *p_hwfn = (struct ecore_hwfn *)rdma_cxt; + u8 abs_stats_queue, max_stats_queues; + u32 pstats_addr, tstats_addr, addr; + struct ecore_rdma_info *info; + struct ecore_ptt *p_ptt; +#ifdef CONFIG_ECORE_IWARP + u32 xstats_addr; +#endif + enum _ecore_status_t rc = ECORE_SUCCESS; + + if (!p_hwfn) + return ECORE_INVAL; + + if (!p_hwfn->p_rdma_info) { + DP_INFO(p_hwfn->p_dev, "ecore rdma query stats failed due to NULL rdma_info\n"); + return ECORE_INVAL; + } + + info = p_hwfn->p_rdma_info; + + rc = ecore_rdma_inc_ref_cnt(p_hwfn); + if (rc != ECORE_SUCCESS) + return rc; + + max_stats_queues = p_hwfn->p_rdma_info->dev->max_stats_queues; + if (stats_queue >= max_stats_queues) { + DP_ERR(p_hwfn->p_dev, + "ecore rdma query stats failed due to invalid statistics queue %d. maximum is %d\n", + stats_queue, max_stats_queues); + rc = ECORE_INVAL; + goto err; + } + + /* Statistics collected in statistics queues (for PF/VF) */ + abs_stats_queue = RESC_START(p_hwfn, ECORE_RDMA_STATS_QUEUE) + + stats_queue; + pstats_addr = BAR0_MAP_REG_PSDM_RAM + + PSTORM_RDMA_QUEUE_STAT_OFFSET(abs_stats_queue); + tstats_addr = BAR0_MAP_REG_TSDM_RAM + + TSTORM_RDMA_QUEUE_STAT_OFFSET(abs_stats_queue); + +#ifdef CONFIG_ECORE_IWARP + /* Statistics per PF ID */ + xstats_addr = BAR0_MAP_REG_XSDM_RAM + + XSTORM_IWARP_RXMIT_STATS_OFFSET(p_hwfn->rel_pf_id); +#endif + + OSAL_MEMSET(&info->rdma_sent_pstats, 0, sizeof(info->rdma_sent_pstats)); + OSAL_MEMSET(&info->rdma_rcv_tstats, 0, sizeof(info->rdma_rcv_tstats)); + OSAL_MEMSET(&info->roce.event_stats, 0, sizeof(info->roce.event_stats)); + OSAL_MEMSET(&info->roce.dcqcn_rx_stats, 0,sizeof(info->roce.dcqcn_rx_stats)); + OSAL_MEMSET(&info->roce.dcqcn_tx_stats, 0,sizeof(info->roce.dcqcn_tx_stats)); +#ifdef CONFIG_ECORE_IWARP + OSAL_MEMSET(&info->iwarp.stats, 0, sizeof(info->iwarp.stats)); +#endif + + p_ptt = ecore_ptt_acquire(p_hwfn); + + if (!p_ptt) { + rc = ECORE_TIMEOUT; + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "rc = %d\n", rc); + goto err; + } + + ecore_memcpy_from(p_hwfn, p_ptt, &info->rdma_sent_pstats, + pstats_addr, sizeof(struct rdma_sent_stats)); + + ecore_memcpy_from(p_hwfn, p_ptt, &info->rdma_rcv_tstats, + tstats_addr, sizeof(struct rdma_rcv_stats)); + + addr = BAR0_MAP_REG_TSDM_RAM + + TSTORM_ROCE_EVENTS_STAT_OFFSET(p_hwfn->rel_pf_id); + ecore_memcpy_from(p_hwfn, p_ptt, &info->roce.event_stats, addr, + sizeof(struct roce_events_stats)); + + addr = BAR0_MAP_REG_YSDM_RAM + + YSTORM_ROCE_DCQCN_RECEIVED_STATS_OFFSET(p_hwfn->rel_pf_id); + ecore_memcpy_from(p_hwfn, p_ptt, &info->roce.dcqcn_rx_stats, addr, + sizeof(struct roce_dcqcn_received_stats)); + + addr = BAR0_MAP_REG_PSDM_RAM + + PSTORM_ROCE_DCQCN_SENT_STATS_OFFSET(p_hwfn->rel_pf_id); + ecore_memcpy_from(p_hwfn, p_ptt, &info->roce.dcqcn_tx_stats, addr, + sizeof(struct roce_dcqcn_sent_stats)); + +#ifdef CONFIG_ECORE_IWARP + ecore_memcpy_from(p_hwfn, p_ptt, &info->iwarp.stats, + xstats_addr, sizeof(struct iwarp_rxmit_stats_drv)); +#endif + + ecore_ptt_release(p_hwfn, p_ptt); + + OSAL_MEMSET(out_params, 0, sizeof(*out_params)); + + out_params->sent_bytes = + HILO_64_REGPAIR(info->rdma_sent_pstats.sent_bytes); + out_params->sent_pkts = + HILO_64_REGPAIR(info->rdma_sent_pstats.sent_pkts); + out_params->rcv_bytes = + HILO_64_REGPAIR(info->rdma_rcv_tstats.rcv_bytes); + out_params->rcv_pkts = + HILO_64_REGPAIR(info->rdma_rcv_tstats.rcv_pkts); + + out_params->silent_drops = + OSAL_LE16_TO_CPU(info->roce.event_stats.silent_drops); + out_params->rnr_nacks_sent = + OSAL_LE16_TO_CPU(info->roce.event_stats.rnr_naks_sent); + out_params->icrc_errors = + OSAL_LE32_TO_CPU(info->roce.event_stats.icrc_error_count); + out_params->retransmit_events = + OSAL_LE32_TO_CPU(info->roce.event_stats.retransmit_count); + out_params->ecn_pkt_rcv = + HILO_64_REGPAIR(info->roce.dcqcn_rx_stats.ecn_pkt_rcv); + out_params->cnp_pkt_rcv = + HILO_64_REGPAIR(info->roce.dcqcn_rx_stats.cnp_pkt_rcv); + out_params->cnp_pkt_sent = + HILO_64_REGPAIR(info->roce.dcqcn_tx_stats.cnp_pkt_sent); + +#ifdef CONFIG_ECORE_IWARP + out_params->iwarp_tx_fast_rxmit_cnt = + HILO_64_REGPAIR(info->iwarp.stats.tx_fast_retransmit_event_cnt); + out_params->iwarp_tx_slow_start_cnt = + HILO_64_REGPAIR( + info->iwarp.stats.tx_go_to_slow_start_event_cnt); + out_params->unalign_rx_comp = info->iwarp.unalign_rx_comp; +#endif + +err: + ecore_rdma_dec_ref_cnt(p_hwfn); + + return rc; +} + +enum _ecore_status_t +ecore_rdma_query_counters(void *rdma_cxt, + struct ecore_rdma_counters_out_params *out_params) +{ + struct ecore_hwfn *p_hwfn = (struct ecore_hwfn *)rdma_cxt; + unsigned long *bitmap; + unsigned int nbits; + + if (!p_hwfn->p_rdma_info) + return ECORE_INVAL; + + OSAL_MEMSET(out_params, 0, sizeof(*out_params)); + + bitmap = p_hwfn->p_rdma_info->pd_map.bitmap; + nbits = p_hwfn->p_rdma_info->pd_map.max_count; + out_params->pd_count = OSAL_BITMAP_WEIGHT(bitmap, nbits); + out_params->max_pd = nbits; + + bitmap = p_hwfn->p_rdma_info->dpi_map.bitmap; + nbits = p_hwfn->p_rdma_info->dpi_map.max_count; + out_params->dpi_count = OSAL_BITMAP_WEIGHT(bitmap, nbits); + out_params->max_dpi = nbits; + + bitmap = p_hwfn->p_rdma_info->cq_map.bitmap; + nbits = p_hwfn->p_rdma_info->cq_map.max_count; + out_params->cq_count = OSAL_BITMAP_WEIGHT(bitmap, nbits); + out_params->max_cq = nbits; + + bitmap = p_hwfn->p_rdma_info->qp_map.bitmap; + nbits = p_hwfn->p_rdma_info->qp_map.max_count; + out_params->qp_count = OSAL_BITMAP_WEIGHT(bitmap, nbits); + out_params->max_qp = nbits; + + bitmap = p_hwfn->p_rdma_info->tid_map.bitmap; + nbits = p_hwfn->p_rdma_info->tid_map.max_count; + out_params->tid_count = OSAL_BITMAP_WEIGHT(bitmap, nbits); + out_params->max_tid = nbits; + + bitmap = p_hwfn->p_rdma_info->srq_map.bitmap; + nbits = p_hwfn->p_rdma_info->srq_map.max_count; + out_params->srq_count = OSAL_BITMAP_WEIGHT(bitmap, nbits); + out_params->max_srq = nbits; + + bitmap = p_hwfn->p_rdma_info->xrc_srq_map.bitmap; + nbits = p_hwfn->p_rdma_info->xrc_srq_map.max_count; + out_params->xrc_srq_count = OSAL_BITMAP_WEIGHT(bitmap, nbits); + out_params->max_xrc_srq = nbits; + + bitmap = p_hwfn->p_rdma_info->xrcd_map.bitmap; + nbits = p_hwfn->p_rdma_info->xrcd_map.max_count; + out_params->xrcd_count = OSAL_BITMAP_WEIGHT(bitmap, nbits); + out_params->max_xrcd = nbits; + + return ECORE_SUCCESS; +} + +enum _ecore_status_t ecore_rdma_resize_cnq(void *rdma_cxt, + struct ecore_rdma_resize_cnq_in_params *params) +{ + struct ecore_hwfn *p_hwfn = (struct ecore_hwfn *)rdma_cxt; + + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "cnq_id = %08x\n", params->cnq_id); + + /* @@@TBD: waiting for fw (there is no ramrod yet) */ + return ECORE_NOTIMPL; +} + +void ecore_rdma_remove_user(void *rdma_cxt, + u16 dpi) +{ + struct ecore_hwfn *p_hwfn = (struct ecore_hwfn *)rdma_cxt; + + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "dpi = %08x\n", dpi); + + OSAL_SPIN_LOCK(&p_hwfn->p_rdma_info->lock); + ecore_bmap_release_id(p_hwfn, &p_hwfn->p_rdma_info->dpi_map, dpi); + OSAL_SPIN_UNLOCK(&p_hwfn->p_rdma_info->lock); +} + +#ifndef LINUX_REMOVE +enum _ecore_status_t +ecore_rdma_set_glob_cfg(struct ecore_hwfn *p_hwfn, + struct ecore_rdma_glob_cfg *in_params, + u32 glob_cfg_bits) +{ + struct ecore_rdma_glob_cfg glob_cfg; + enum _ecore_status_t rc = ECORE_SUCCESS; + + DP_VERBOSE(p_hwfn->p_dev, ECORE_MSG_RDMA, + "dscp %d dscp en %d ecn %d ecn en %d vlan pri %d vlan_pri_en %d\n", + in_params->dscp, in_params->dscp_en, + in_params->ecn, in_params->ecn_en, in_params->vlan_pri, + in_params->vlan_pri_en); + + /* Read global cfg to local */ + OSAL_MEMCPY(&glob_cfg, &p_hwfn->p_rdma_info->glob_cfg, + sizeof(glob_cfg)); + + if (glob_cfg_bits & ECORE_RDMA_DCSP_BIT_MASK) { + if (in_params->dscp > MAX_DSCP) { + DP_ERR(p_hwfn->p_dev, "invalid glob dscp %d\n", + in_params->dscp); + return ECORE_INVAL; + } + glob_cfg.dscp = in_params->dscp; + } + + if (glob_cfg_bits & ECORE_RDMA_DCSP_EN_BIT_MASK) { + if (in_params->dscp_en > 1) { + DP_ERR(p_hwfn->p_dev, "invalid glob_dscp_en %d\n", + in_params->dscp_en); + return ECORE_INVAL; + } + glob_cfg.dscp_en = in_params->dscp_en; + } + + if (glob_cfg_bits & ECORE_RDMA_ECN_BIT_MASK) { + if (in_params->ecn > INET_ECN_ECT_0) { + DP_ERR(p_hwfn->p_dev, "invalid glob ecn %d\n", + in_params->ecn); + return ECORE_INVAL; + } + glob_cfg.ecn = in_params->ecn; + } + + if (glob_cfg_bits & ECORE_RDMA_ECN_EN_BIT_MASK) { + if (in_params->ecn_en > 1) { + DP_ERR(p_hwfn->p_dev, "invalid glob ecn en %d\n", + in_params->ecn_en); + return ECORE_INVAL; + } + glob_cfg.ecn_en = in_params->ecn_en; + } + + if (glob_cfg_bits & ECORE_RDMA_VLAN_PRIO_BIT_MASK) { + if (in_params->vlan_pri > MAX_VLAN_PRIO) { + DP_ERR(p_hwfn->p_dev, "invalid glob vlan pri %d\n", + in_params->vlan_pri); + return ECORE_INVAL; + } + glob_cfg.vlan_pri = in_params->vlan_pri; + } + + if (glob_cfg_bits & ECORE_RDMA_VLAN_PRIO_EN_BIT_MASK) { + if (in_params->vlan_pri_en > 1) { + DP_ERR(p_hwfn->p_dev, "invalid glob vlan pri en %d\n", + in_params->vlan_pri_en); + return ECORE_INVAL; + } + glob_cfg.vlan_pri_en = in_params->vlan_pri_en; + } + + /* Write back local cfg to global */ + OSAL_MEMCPY(&p_hwfn->p_rdma_info->glob_cfg, &glob_cfg, + sizeof(glob_cfg)); + + return rc; +} + +enum _ecore_status_t +ecore_rdma_get_glob_cfg(struct ecore_hwfn *p_hwfn, + struct ecore_rdma_glob_cfg *out_params) +{ + OSAL_MEMCPY(out_params, &p_hwfn->p_rdma_info->glob_cfg, + sizeof(struct ecore_rdma_glob_cfg)); + + return ECORE_SUCCESS; +} +#endif /* LINUX_REMOVE */ Property changes on: head/sys/dev/qlnx/qlnxe/ecore_rdma.c ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Added: svn:mime-type ## -0,0 +1 ## +text/plain \ No newline at end of property Index: head/sys/dev/qlnx/qlnxe/ecore_roce.c =================================================================== --- head/sys/dev/qlnx/qlnxe/ecore_roce.c (nonexistent) +++ head/sys/dev/qlnx/qlnxe/ecore_roce.c (revision 343598) @@ -0,0 +1,1579 @@ +/* + * Copyright (c) 2018-2019 Cavium, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +/* + * File : ecore_roce.c + */ +#include +__FBSDID("$FreeBSD$"); + +#include "bcm_osal.h" +#include "ecore.h" +#include "ecore_status.h" +#include "ecore_sp_commands.h" +#include "ecore_cxt.h" +#include "ecore_rdma.h" +#include "reg_addr.h" +#include "ecore_rt_defs.h" +#include "ecore_init_ops.h" +#include "ecore_hw.h" +#include "ecore_mcp.h" +#include "ecore_init_fw_funcs.h" +#include "ecore_int.h" +#include "pcics_reg_driver.h" +#include "ecore_iro.h" +#include "ecore_gtt_reg_addr.h" +#ifndef LINUX_REMOVE +#include "ecore_tcp_ip.h" +#endif + +#ifdef _NTDDK_ +#pragma warning(push) +#pragma warning(disable : 28167) +#pragma warning(disable : 28123) +#pragma warning(disable : 28182) +#pragma warning(disable : 6011) +#endif + +static void ecore_roce_free_icid(struct ecore_hwfn *p_hwfn, u16 icid); + +static enum _ecore_status_t +ecore_roce_async_event(struct ecore_hwfn *p_hwfn, + u8 fw_event_code, + u16 OSAL_UNUSED echo, + union event_ring_data *data, + u8 OSAL_UNUSED fw_return_code) +{ + if (fw_event_code == ROCE_ASYNC_EVENT_DESTROY_QP_DONE) { + u16 icid = (u16)OSAL_LE32_TO_CPU( + data->rdma_data.rdma_destroy_qp_data.cid); + + /* icid release in this async event can occur only if the icid + * was offloaded to the FW. In case it wasn't offloaded this is + * handled in ecore_roce_sp_destroy_qp. + */ + ecore_roce_free_icid(p_hwfn, icid); + } else + p_hwfn->p_rdma_info->events.affiliated_event( + p_hwfn->p_rdma_info->events.context, + fw_event_code, + (void *)&data->rdma_data.async_handle); + + return ECORE_SUCCESS; +} + + + +#ifdef CONFIG_DCQCN +static enum _ecore_status_t ecore_roce_start_rl( + struct ecore_hwfn *p_hwfn, + struct ecore_roce_dcqcn_params *dcqcn_params) +{ + struct ecore_rl_update_params params; + + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "\n"); + OSAL_MEMSET(¶ms, 0, sizeof(params)); + + params.rl_id_first = (u8)RESC_START(p_hwfn, ECORE_RL); + params.rl_id_last = RESC_START(p_hwfn, ECORE_RL) + + ecore_init_qm_get_num_pf_rls(p_hwfn); + params.dcqcn_update_param_flg = 1; + params.rl_init_flg = 1; + params.rl_start_flg = 1; + params.rl_stop_flg = 0; + params.rl_dc_qcn_flg = 1; + + params.rl_bc_rate = dcqcn_params->rl_bc_rate; + params.rl_max_rate = dcqcn_params->rl_max_rate; + params.rl_r_ai = dcqcn_params->rl_r_ai; + params.rl_r_hai = dcqcn_params->rl_r_hai; + params.dcqcn_gd = dcqcn_params->dcqcn_gd; + params.dcqcn_k_us = dcqcn_params->dcqcn_k_us; + params.dcqcn_timeuot_us = dcqcn_params->dcqcn_timeout_us; + + return ecore_sp_rl_update(p_hwfn, ¶ms); +} + +enum _ecore_status_t ecore_roce_stop_rl(struct ecore_hwfn *p_hwfn) +{ + struct ecore_rl_update_params params; + + if (!p_hwfn->p_rdma_info->roce.dcqcn_reaction_point) + return ECORE_SUCCESS; + + OSAL_MEMSET(¶ms, 0, sizeof(params)); + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "\n"); + + params.rl_id_first = (u8)RESC_START(p_hwfn, ECORE_RL); + params.rl_id_last = RESC_START(p_hwfn, ECORE_RL) + + ecore_init_qm_get_num_pf_rls(p_hwfn); + params.rl_stop_flg = 1; + + return ecore_sp_rl_update(p_hwfn, ¶ms); +} + +#define NIG_REG_ROCE_DUPLICATE_TO_HOST_BTH 2 +#define NIG_REG_ROCE_DUPLICATE_TO_HOST_ECN 1 + +enum _ecore_status_t ecore_roce_dcqcn_cfg( + struct ecore_hwfn *p_hwfn, + struct ecore_roce_dcqcn_params *params, + struct roce_init_func_ramrod_data *p_ramrod, + struct ecore_ptt *p_ptt) +{ + u32 val = 0; + enum _ecore_status_t rc = ECORE_SUCCESS; + + if (!p_hwfn->pf_params.rdma_pf_params.enable_dcqcn || + p_hwfn->p_rdma_info->proto == PROTOCOLID_IWARP) + return rc; + + p_hwfn->p_rdma_info->roce.dcqcn_enabled = 0; + if (params->notification_point) { + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, + "Configuring dcqcn notification point: timeout = 0x%x\n", + params->cnp_send_timeout); + p_ramrod->roce.cnp_send_timeout = params->cnp_send_timeout; + p_hwfn->p_rdma_info->roce.dcqcn_enabled = 1; + /* Configure NIG to duplicate to host and storm when: + * - (ECN == 2'b11 (notification point) + */ + val |= 1 << NIG_REG_ROCE_DUPLICATE_TO_HOST_ECN; + } + + if (params->reaction_point) { + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, + "Configuring dcqcn reaction point\n"); + p_hwfn->p_rdma_info->roce.dcqcn_enabled = 1; + p_hwfn->p_rdma_info->roce.dcqcn_reaction_point = 1; + /* Configure NIG to duplicate to host and storm when: + * - BTH opcode equals bth_hdr_flow_ctrl_opcode_2 + * (reaction point) + */ + val |= 1 << NIG_REG_ROCE_DUPLICATE_TO_HOST_BTH; + + rc = ecore_roce_start_rl(p_hwfn, params); + } + + if (rc) + return rc; + + p_ramrod->roce.cnp_dscp = params->cnp_dscp; + p_ramrod->roce.cnp_vlan_priority = params->cnp_vlan_priority; + + ecore_wr(p_hwfn, + p_ptt, + NIG_REG_ROCE_DUPLICATE_TO_HOST, + val); + + return rc; +} +#endif + + +enum _ecore_status_t ecore_roce_stop(struct ecore_hwfn *p_hwfn) +{ + struct ecore_bmap *cid_map = &p_hwfn->p_rdma_info->cid_map; + int wait_count = 0; + + /* when destroying a_RoCE QP the control is returned to the + * user after the synchronous part. The asynchronous part may + * take a little longer. We delay for a short while if an + * asyn destroy QP is still expected. Beyond the added delay + * we clear the bitmap anyway. + */ + while (OSAL_BITMAP_WEIGHT(cid_map->bitmap, cid_map->max_count)) { + OSAL_MSLEEP(100); + if (wait_count++ > 20) { + DP_NOTICE(p_hwfn, false, + "cid bitmap wait timed out\n"); + break; + } + } + + ecore_spq_unregister_async_cb(p_hwfn, PROTOCOLID_ROCE); + + return ECORE_SUCCESS; +} + + +static void ecore_rdma_copy_gids(struct ecore_rdma_qp *qp, __le32 *src_gid, + __le32 *dst_gid) { + u32 i; + + if (qp->roce_mode == ROCE_V2_IPV4) { + /* The IPv4 addresses shall be aligned to the highest word. + * The lower words must be zero. + */ + OSAL_MEMSET(src_gid, 0, sizeof(union ecore_gid)); + OSAL_MEMSET(dst_gid, 0, sizeof(union ecore_gid)); + src_gid[3] = OSAL_CPU_TO_LE32(qp->sgid.ipv4_addr); + dst_gid[3] = OSAL_CPU_TO_LE32(qp->dgid.ipv4_addr); + } else { + /* RoCE, and RoCE v2 - IPv6: GIDs and IPv6 addresses coincide in + * location and size + */ + for (i = 0; i < OSAL_ARRAY_SIZE(qp->sgid.dwords); i++) { + src_gid[i] = OSAL_CPU_TO_LE32(qp->sgid.dwords[i]); + dst_gid[i] = OSAL_CPU_TO_LE32(qp->dgid.dwords[i]); + } + } +} + +static enum roce_flavor ecore_roce_mode_to_flavor(enum roce_mode roce_mode) +{ + enum roce_flavor flavor; + + switch (roce_mode) { + case ROCE_V1: + flavor = PLAIN_ROCE; + break; + case ROCE_V2_IPV4: + flavor = RROCE_IPV4; + break; + case ROCE_V2_IPV6: + flavor = (enum roce_flavor)ROCE_V2_IPV6; + break; + default: + flavor = (enum roce_flavor)MAX_ROCE_MODE; + break; + } + return flavor; +} + +#if 0 +static void ecore_roce_free_cid_pair(struct ecore_hwfn *p_hwfn, u16 cid) +{ + OSAL_SPIN_LOCK(&p_hwfn->p_rdma_info->lock); + ecore_bmap_release_id(p_hwfn, &p_hwfn->p_rdma_info->qp_map, cid); + ecore_bmap_release_id(p_hwfn, &p_hwfn->p_rdma_info->qp_map, cid + 1); + OSAL_SPIN_UNLOCK(&p_hwfn->p_rdma_info->lock); +} +#endif + +static void ecore_roce_free_qp(struct ecore_hwfn *p_hwfn, u16 qp_idx) +{ + OSAL_SPIN_LOCK(&p_hwfn->p_rdma_info->lock); + ecore_bmap_release_id(p_hwfn, &p_hwfn->p_rdma_info->qp_map, qp_idx); + OSAL_SPIN_UNLOCK(&p_hwfn->p_rdma_info->lock); +} + +#define ECORE_ROCE_CREATE_QP_ATTEMPTS (20) +#define ECORE_ROCE_CREATE_QP_MSLEEP (10) + +static enum _ecore_status_t ecore_roce_wait_free_cids(struct ecore_hwfn *p_hwfn, u32 qp_idx) +{ + struct ecore_rdma_info *p_rdma_info = p_hwfn->p_rdma_info; + bool cids_free = false; + u32 icid, iter = 0; + int req, resp; + + icid = ECORE_ROCE_QP_TO_ICID(qp_idx); + + /* Make sure that the cids that were used by the QP index are free. + * This is necessary because the destroy flow returns to the user before + * the device finishes clean up. + * It can happen in the following flows: + * (1) ib_destroy_qp followed by an ib_create_qp + * (2) ib_modify_qp to RESET followed (not immediately), by an + * ib_modify_qp to RTR + */ + + do { + OSAL_SPIN_LOCK(&p_rdma_info->lock); + resp = ecore_bmap_test_id(p_hwfn, &p_rdma_info->cid_map, icid); + req = ecore_bmap_test_id(p_hwfn, &p_rdma_info->cid_map, icid + 1); + if (!resp && !req) + cids_free = true; + + OSAL_SPIN_UNLOCK(&p_rdma_info->lock); + + if (!cids_free) { + OSAL_MSLEEP(ECORE_ROCE_CREATE_QP_MSLEEP); + iter++; + } + } while (!cids_free && iter < ECORE_ROCE_CREATE_QP_ATTEMPTS); + + if (!cids_free) { + DP_ERR(p_hwfn->p_dev, + "responder and/or requester CIDs are still in use. resp=%d, req=%d\n", + resp, req); + return ECORE_AGAIN; + } + + return ECORE_SUCCESS; +} + +enum _ecore_status_t ecore_roce_alloc_qp_idx( + struct ecore_hwfn *p_hwfn, u16 *qp_idx16) +{ + struct ecore_rdma_info *p_rdma_info = p_hwfn->p_rdma_info; + u32 start_cid, icid, cid, qp_idx; + enum _ecore_status_t rc; + + OSAL_SPIN_LOCK(&p_rdma_info->lock); + rc = ecore_rdma_bmap_alloc_id(p_hwfn, &p_rdma_info->qp_map, &qp_idx); + if (rc != ECORE_SUCCESS) { + DP_NOTICE(p_hwfn, false, "failed to allocate qp\n"); + OSAL_SPIN_UNLOCK(&p_rdma_info->lock); + return rc; + } + + OSAL_SPIN_UNLOCK(&p_rdma_info->lock); + + /* Verify the cid bits that of this qp index are clear */ + rc = ecore_roce_wait_free_cids(p_hwfn, qp_idx); + if (rc) { + rc = ECORE_UNKNOWN_ERROR; + goto err; + } + + /* Allocate a DMA-able context for an ILT page, if not existing, for the + * associated iids. + * Note: If second allocation fails there's no need to free the first as + * it will be used in the future. + */ + icid = ECORE_ROCE_QP_TO_ICID(qp_idx); + start_cid = ecore_cxt_get_proto_cid_start(p_hwfn, p_rdma_info->proto); + cid = start_cid + icid; + + rc = ecore_cxt_dynamic_ilt_alloc(p_hwfn, ECORE_ELEM_CXT, cid); + if (rc != ECORE_SUCCESS) + goto err; + + rc = ecore_cxt_dynamic_ilt_alloc(p_hwfn, ECORE_ELEM_CXT, cid + 1); + if (rc != ECORE_SUCCESS) + goto err; + + /* qp index is under 2^16 */ + *qp_idx16 = (u16)qp_idx; + + return ECORE_SUCCESS; + +err: + ecore_roce_free_qp(p_hwfn, (u16)qp_idx); + + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "rc = %d\n", rc); + + return rc; +} + +static void ecore_roce_set_cid(struct ecore_hwfn *p_hwfn, + u32 cid) +{ + OSAL_SPIN_LOCK(&p_hwfn->p_rdma_info->lock); + ecore_bmap_set_id(p_hwfn, + &p_hwfn->p_rdma_info->cid_map, + cid); + OSAL_SPIN_UNLOCK(&p_hwfn->p_rdma_info->lock); +} + +static enum _ecore_status_t ecore_roce_sp_create_responder( + struct ecore_hwfn *p_hwfn, + struct ecore_rdma_qp *qp) +{ + struct roce_create_qp_resp_ramrod_data *p_ramrod; + u16 regular_latency_queue, low_latency_queue; + struct ecore_sp_init_data init_data; + enum roce_flavor roce_flavor; + struct ecore_spq_entry *p_ent; + enum _ecore_status_t rc; + u32 cid_start; + u16 fw_srq_id; + bool is_xrc; + + if (!qp->has_resp) + return ECORE_SUCCESS; + + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "qp_idx = %08x\n", qp->qp_idx); + + /* Allocate DMA-able memory for IRQ */ + qp->irq_num_pages = 1; + qp->irq = OSAL_DMA_ALLOC_COHERENT(p_hwfn->p_dev, + &qp->irq_phys_addr, + RDMA_RING_PAGE_SIZE); + if (!qp->irq) { + rc = ECORE_NOMEM; + DP_NOTICE(p_hwfn, false, + "ecore create responder failed: cannot allocate memory (irq). rc = %d\n", + rc); + return rc; + } + + /* Get SPQ entry */ + OSAL_MEMSET(&init_data, 0, sizeof(init_data)); + init_data.cid = qp->icid; + init_data.opaque_fid = p_hwfn->hw_info.opaque_fid; + init_data.comp_mode = ECORE_SPQ_MODE_EBLOCK; + + rc = ecore_sp_init_request(p_hwfn, &p_ent, ROCE_RAMROD_CREATE_QP, + PROTOCOLID_ROCE, &init_data); + if (rc != ECORE_SUCCESS) + goto err; + + p_ramrod = &p_ent->ramrod.roce_create_qp_resp; + + p_ramrod->flags = 0; + + roce_flavor = ecore_roce_mode_to_flavor(qp->roce_mode); + SET_FIELD(p_ramrod->flags, + ROCE_CREATE_QP_RESP_RAMROD_DATA_ROCE_FLAVOR, + roce_flavor); + + SET_FIELD(p_ramrod->flags, + ROCE_CREATE_QP_RESP_RAMROD_DATA_RDMA_RD_EN, + qp->incoming_rdma_read_en); + + SET_FIELD(p_ramrod->flags, + ROCE_CREATE_QP_RESP_RAMROD_DATA_RDMA_WR_EN, + qp->incoming_rdma_write_en); + + SET_FIELD(p_ramrod->flags, + ROCE_CREATE_QP_RESP_RAMROD_DATA_ATOMIC_EN, + qp->incoming_atomic_en); + + SET_FIELD(p_ramrod->flags, + ROCE_CREATE_QP_RESP_RAMROD_DATA_E2E_FLOW_CONTROL_EN, + qp->e2e_flow_control_en); + + SET_FIELD(p_ramrod->flags, + ROCE_CREATE_QP_RESP_RAMROD_DATA_SRQ_FLG, + qp->use_srq); + + SET_FIELD(p_ramrod->flags, + ROCE_CREATE_QP_RESP_RAMROD_DATA_RESERVED_KEY_EN, + qp->fmr_and_reserved_lkey); + + SET_FIELD(p_ramrod->flags, + ROCE_CREATE_QP_RESP_RAMROD_DATA_XRC_FLAG, + ecore_rdma_is_xrc_qp(qp)); + + /* TBD: future use only + * #define ROCE_CREATE_QP_RESP_RAMROD_DATA_PRI_MASK + * #define ROCE_CREATE_QP_RESP_RAMROD_DATA_PRI_SHIFT + */ + SET_FIELD(p_ramrod->flags, + ROCE_CREATE_QP_RESP_RAMROD_DATA_MIN_RNR_NAK_TIMER, + qp->min_rnr_nak_timer); + + p_ramrod->max_ird = + qp->max_rd_atomic_resp; + p_ramrod->traffic_class = qp->traffic_class_tos; + p_ramrod->hop_limit = qp->hop_limit_ttl; + p_ramrod->irq_num_pages = qp->irq_num_pages; + p_ramrod->p_key = OSAL_CPU_TO_LE16(qp->pkey); + p_ramrod->flow_label = OSAL_CPU_TO_LE32(qp->flow_label); + p_ramrod->dst_qp_id = OSAL_CPU_TO_LE32(qp->dest_qp); + p_ramrod->mtu = OSAL_CPU_TO_LE16(qp->mtu); + p_ramrod->initial_psn = OSAL_CPU_TO_LE32(qp->rq_psn); + p_ramrod->pd = OSAL_CPU_TO_LE16(qp->pd); + p_ramrod->rq_num_pages = OSAL_CPU_TO_LE16(qp->rq_num_pages); + DMA_REGPAIR_LE(p_ramrod->rq_pbl_addr, qp->rq_pbl_ptr); + DMA_REGPAIR_LE(p_ramrod->irq_pbl_addr, qp->irq_phys_addr); + ecore_rdma_copy_gids(qp, p_ramrod->src_gid, p_ramrod->dst_gid); + p_ramrod->qp_handle_for_async.hi = + OSAL_CPU_TO_LE32(qp->qp_handle_async.hi); + p_ramrod->qp_handle_for_async.lo = + OSAL_CPU_TO_LE32(qp->qp_handle_async.lo); + p_ramrod->qp_handle_for_cqe.hi = OSAL_CPU_TO_LE32(qp->qp_handle.hi); + p_ramrod->qp_handle_for_cqe.lo = OSAL_CPU_TO_LE32(qp->qp_handle.lo); + p_ramrod->cq_cid = OSAL_CPU_TO_LE32((p_hwfn->hw_info.opaque_fid << 16) | qp->rq_cq_id); + p_ramrod->xrc_domain = OSAL_CPU_TO_LE16(qp->xrcd_id); + +#ifdef CONFIG_DCQCN + /* when dcqcn is enabled physical queues are determined accoridng to qp id */ + if (p_hwfn->p_rdma_info->roce.dcqcn_enabled) + regular_latency_queue = + ecore_get_cm_pq_idx_rl(p_hwfn, + (qp->icid >> 1) % + ROCE_DCQCN_RP_MAX_QPS); + else +#endif + regular_latency_queue = ecore_get_cm_pq_idx(p_hwfn, PQ_FLAGS_OFLD); + low_latency_queue = ecore_get_cm_pq_idx(p_hwfn, PQ_FLAGS_LLT); + + p_ramrod->regular_latency_phy_queue = OSAL_CPU_TO_LE16(regular_latency_queue); + p_ramrod->low_latency_phy_queue = OSAL_CPU_TO_LE16(low_latency_queue); + p_ramrod->dpi = OSAL_CPU_TO_LE16(qp->dpi); + + ecore_rdma_set_fw_mac(p_ramrod->remote_mac_addr, qp->remote_mac_addr); + ecore_rdma_set_fw_mac(p_ramrod->local_mac_addr, qp->local_mac_addr); + + p_ramrod->udp_src_port = qp->udp_src_port; + p_ramrod->vlan_id = OSAL_CPU_TO_LE16(qp->vlan_id); + is_xrc = ecore_rdma_is_xrc_qp(qp); + fw_srq_id = ecore_rdma_get_fw_srq_id(p_hwfn, qp->srq_id, is_xrc); + p_ramrod->srq_id.srq_idx = OSAL_CPU_TO_LE16(fw_srq_id); + p_ramrod->srq_id.opaque_fid = OSAL_CPU_TO_LE16(p_hwfn->hw_info.opaque_fid); + + p_ramrod->stats_counter_id = RESC_START(p_hwfn, ECORE_RDMA_STATS_QUEUE) + + qp->stats_queue; + + rc = ecore_spq_post(p_hwfn, p_ent, OSAL_NULL); + + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "rc = %d regular physical queue = 0x%x, low latency physical queue 0x%x\n", + rc, regular_latency_queue, low_latency_queue); + + if (rc != ECORE_SUCCESS) + goto err; + + qp->resp_offloaded = true; + qp->cq_prod.resp = 0; + + cid_start = ecore_cxt_get_proto_cid_start(p_hwfn, + p_hwfn->p_rdma_info->proto); + ecore_roce_set_cid(p_hwfn, qp->icid - cid_start); + + return rc; + +err: + DP_NOTICE(p_hwfn, false, "create responder - failed, rc = %d\n", rc); + OSAL_DMA_FREE_COHERENT(p_hwfn->p_dev, + qp->irq, + qp->irq_phys_addr, + qp->irq_num_pages * + RDMA_RING_PAGE_SIZE); + + return rc; +} + +static enum _ecore_status_t ecore_roce_sp_create_requester( + struct ecore_hwfn *p_hwfn, + struct ecore_rdma_qp *qp) +{ + struct roce_create_qp_req_ramrod_data *p_ramrod; + u16 regular_latency_queue, low_latency_queue; + struct ecore_sp_init_data init_data; + enum roce_flavor roce_flavor; + struct ecore_spq_entry *p_ent; + enum _ecore_status_t rc; + u32 cid_start; + + if (!qp->has_req) + return ECORE_SUCCESS; + + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "icid = %08x\n", qp->icid); + + /* Allocate DMA-able memory for ORQ */ + qp->orq_num_pages = 1; + qp->orq = OSAL_DMA_ALLOC_COHERENT(p_hwfn->p_dev, + &qp->orq_phys_addr, + RDMA_RING_PAGE_SIZE); + if (!qp->orq) + { + rc = ECORE_NOMEM; + DP_NOTICE(p_hwfn, false, + "ecore create requester failed: cannot allocate memory (orq). rc = %d\n", + rc); + return rc; + } + + /* Get SPQ entry */ + OSAL_MEMSET(&init_data, 0, sizeof(init_data)); + init_data.cid = qp->icid + 1; + init_data.opaque_fid = p_hwfn->hw_info.opaque_fid; + init_data.comp_mode = ECORE_SPQ_MODE_EBLOCK; + + rc = ecore_sp_init_request(p_hwfn, &p_ent, + ROCE_RAMROD_CREATE_QP, + PROTOCOLID_ROCE, &init_data); + if (rc != ECORE_SUCCESS) + goto err; + + p_ramrod = &p_ent->ramrod.roce_create_qp_req; + + p_ramrod->flags = 0; + + roce_flavor = ecore_roce_mode_to_flavor(qp->roce_mode); + SET_FIELD(p_ramrod->flags, + ROCE_CREATE_QP_REQ_RAMROD_DATA_ROCE_FLAVOR, + roce_flavor); + + SET_FIELD(p_ramrod->flags, + ROCE_CREATE_QP_REQ_RAMROD_DATA_FMR_AND_RESERVED_EN, + qp->fmr_and_reserved_lkey); + + SET_FIELD(p_ramrod->flags, + ROCE_CREATE_QP_REQ_RAMROD_DATA_SIGNALED_COMP, + qp->signal_all); + + /* TBD: + * future use only + * #define ROCE_CREATE_QP_REQ_RAMROD_DATA_PRI_MASK + * #define ROCE_CREATE_QP_REQ_RAMROD_DATA_PRI_SHIFT + */ + SET_FIELD(p_ramrod->flags, + ROCE_CREATE_QP_REQ_RAMROD_DATA_ERR_RETRY_CNT, + qp->retry_cnt); + + SET_FIELD(p_ramrod->flags, + ROCE_CREATE_QP_REQ_RAMROD_DATA_RNR_NAK_CNT, + qp->rnr_retry_cnt); + + SET_FIELD(p_ramrod->flags, + ROCE_CREATE_QP_REQ_RAMROD_DATA_XRC_FLAG, + ecore_rdma_is_xrc_qp(qp)); + + p_ramrod->max_ord = qp->max_rd_atomic_req; + p_ramrod->traffic_class = qp->traffic_class_tos; + p_ramrod->hop_limit = qp->hop_limit_ttl; + p_ramrod->orq_num_pages = qp->orq_num_pages; + p_ramrod->p_key = OSAL_CPU_TO_LE16(qp->pkey); + p_ramrod->flow_label = OSAL_CPU_TO_LE32(qp->flow_label); + p_ramrod->dst_qp_id = OSAL_CPU_TO_LE32(qp->dest_qp); + p_ramrod->ack_timeout_val = OSAL_CPU_TO_LE32(qp->ack_timeout); + p_ramrod->mtu = OSAL_CPU_TO_LE16(qp->mtu); + p_ramrod->initial_psn = OSAL_CPU_TO_LE32(qp->sq_psn); + p_ramrod->pd = OSAL_CPU_TO_LE16(qp->pd); + p_ramrod->sq_num_pages = OSAL_CPU_TO_LE16(qp->sq_num_pages); + DMA_REGPAIR_LE(p_ramrod->sq_pbl_addr, qp->sq_pbl_ptr); + DMA_REGPAIR_LE(p_ramrod->orq_pbl_addr, qp->orq_phys_addr); + ecore_rdma_copy_gids(qp, p_ramrod->src_gid, p_ramrod->dst_gid); + p_ramrod->qp_handle_for_async.hi = + OSAL_CPU_TO_LE32(qp->qp_handle_async.hi); + p_ramrod->qp_handle_for_async.lo = + OSAL_CPU_TO_LE32(qp->qp_handle_async.lo); + p_ramrod->qp_handle_for_cqe.hi = OSAL_CPU_TO_LE32(qp->qp_handle.hi); + p_ramrod->qp_handle_for_cqe.lo = OSAL_CPU_TO_LE32(qp->qp_handle.lo); + p_ramrod->cq_cid = OSAL_CPU_TO_LE32((p_hwfn->hw_info.opaque_fid << 16) | + qp->sq_cq_id); + +#ifdef CONFIG_DCQCN + /* when dcqcn is enabled physical queues are determined accoridng to qp id */ + if (p_hwfn->p_rdma_info->roce.dcqcn_enabled) + regular_latency_queue = + ecore_get_cm_pq_idx_rl(p_hwfn, + (qp->icid >> 1) % + ROCE_DCQCN_RP_MAX_QPS); + else +#endif + regular_latency_queue = ecore_get_cm_pq_idx(p_hwfn, PQ_FLAGS_OFLD); + low_latency_queue = ecore_get_cm_pq_idx(p_hwfn, PQ_FLAGS_LLT); + + p_ramrod->regular_latency_phy_queue = OSAL_CPU_TO_LE16(regular_latency_queue); + p_ramrod->low_latency_phy_queue = OSAL_CPU_TO_LE16(low_latency_queue); + p_ramrod->dpi = OSAL_CPU_TO_LE16(qp->dpi); + + ecore_rdma_set_fw_mac(p_ramrod->remote_mac_addr, qp->remote_mac_addr); + ecore_rdma_set_fw_mac(p_ramrod->local_mac_addr, qp->local_mac_addr); + + p_ramrod->udp_src_port = qp->udp_src_port; + p_ramrod->vlan_id = OSAL_CPU_TO_LE16(qp->vlan_id); + p_ramrod->stats_counter_id = RESC_START(p_hwfn, ECORE_RDMA_STATS_QUEUE) + + qp->stats_queue; + + rc = ecore_spq_post(p_hwfn, p_ent, OSAL_NULL); + + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "rc = %d\n", rc); + + if (rc != ECORE_SUCCESS) + goto err; + + qp->req_offloaded = true; + qp->cq_prod.req = 0; + + cid_start = ecore_cxt_get_proto_cid_start(p_hwfn, + p_hwfn->p_rdma_info->proto); + ecore_roce_set_cid(p_hwfn, qp->icid + 1 - cid_start); + + return rc; + +err: + DP_NOTICE(p_hwfn, false, "Create requested - failed, rc = %d\n", rc); + OSAL_DMA_FREE_COHERENT(p_hwfn->p_dev, + qp->orq, + qp->orq_phys_addr, + qp->orq_num_pages * + RDMA_RING_PAGE_SIZE); + return rc; +} + +static enum _ecore_status_t ecore_roce_sp_modify_responder( + struct ecore_hwfn *p_hwfn, + struct ecore_rdma_qp *qp, + bool move_to_err, + u32 modify_flags) +{ + struct roce_modify_qp_resp_ramrod_data *p_ramrod; + struct ecore_sp_init_data init_data; + struct ecore_spq_entry *p_ent; + enum _ecore_status_t rc; + + if (!qp->has_resp) + return ECORE_SUCCESS; + + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "icid = %08x\n", qp->icid); + + if (move_to_err && !qp->resp_offloaded) + return ECORE_SUCCESS; + + /* Get SPQ entry */ + OSAL_MEMSET(&init_data, 0, sizeof(init_data)); + init_data.cid = qp->icid; + init_data.opaque_fid = p_hwfn->hw_info.opaque_fid; + init_data.comp_mode = ECORE_SPQ_MODE_EBLOCK; + + rc = ecore_sp_init_request(p_hwfn, &p_ent, + ROCE_EVENT_MODIFY_QP, + PROTOCOLID_ROCE, &init_data); + if (rc != ECORE_SUCCESS) + { + DP_NOTICE(p_hwfn, false, "rc = %d\n", rc); + return rc; + } + + p_ramrod = &p_ent->ramrod.roce_modify_qp_resp; + + p_ramrod->flags = 0; + + SET_FIELD(p_ramrod->flags, + ROCE_MODIFY_QP_RESP_RAMROD_DATA_MOVE_TO_ERR_FLG, + move_to_err); + + SET_FIELD(p_ramrod->flags, + ROCE_MODIFY_QP_RESP_RAMROD_DATA_RDMA_RD_EN, + qp->incoming_rdma_read_en); + + SET_FIELD(p_ramrod->flags, + ROCE_MODIFY_QP_RESP_RAMROD_DATA_RDMA_WR_EN, + qp->incoming_rdma_write_en); + + SET_FIELD(p_ramrod->flags, + ROCE_MODIFY_QP_RESP_RAMROD_DATA_ATOMIC_EN, + qp->incoming_atomic_en); + + SET_FIELD(p_ramrod->flags, + ROCE_CREATE_QP_RESP_RAMROD_DATA_E2E_FLOW_CONTROL_EN, + qp->e2e_flow_control_en); + + SET_FIELD(p_ramrod->flags, + ROCE_MODIFY_QP_RESP_RAMROD_DATA_RDMA_OPS_EN_FLG, + GET_FIELD(modify_flags, + ECORE_RDMA_MODIFY_QP_VALID_RDMA_OPS_EN)); + + SET_FIELD(p_ramrod->flags, + ROCE_MODIFY_QP_RESP_RAMROD_DATA_P_KEY_FLG, + GET_FIELD(modify_flags, ECORE_ROCE_MODIFY_QP_VALID_PKEY)); + + SET_FIELD(p_ramrod->flags, + ROCE_MODIFY_QP_RESP_RAMROD_DATA_ADDRESS_VECTOR_FLG, + GET_FIELD(modify_flags, + ECORE_ROCE_MODIFY_QP_VALID_ADDRESS_VECTOR)); + + SET_FIELD(p_ramrod->flags, + ROCE_MODIFY_QP_RESP_RAMROD_DATA_MAX_IRD_FLG, + GET_FIELD(modify_flags, + ECORE_RDMA_MODIFY_QP_VALID_MAX_RD_ATOMIC_RESP)); + + /* TBD: future use only + * #define ROCE_MODIFY_QP_RESP_RAMROD_DATA_PRI_FLG_MASK + * #define ROCE_MODIFY_QP_RESP_RAMROD_DATA_PRI_FLG_SHIFT + */ + + SET_FIELD(p_ramrod->flags, + ROCE_MODIFY_QP_RESP_RAMROD_DATA_MIN_RNR_NAK_TIMER_FLG, + GET_FIELD(modify_flags, + ECORE_ROCE_MODIFY_QP_VALID_MIN_RNR_NAK_TIMER)); + + p_ramrod->fields = 0; + SET_FIELD(p_ramrod->fields, + ROCE_MODIFY_QP_RESP_RAMROD_DATA_MIN_RNR_NAK_TIMER, + qp->min_rnr_nak_timer); + + p_ramrod->max_ird = qp->max_rd_atomic_resp; + p_ramrod->traffic_class = qp->traffic_class_tos; + p_ramrod->hop_limit = qp->hop_limit_ttl; + p_ramrod->p_key = OSAL_CPU_TO_LE16(qp->pkey); + p_ramrod->flow_label = OSAL_CPU_TO_LE32(qp->flow_label); + p_ramrod->mtu = OSAL_CPU_TO_LE16(qp->mtu); + ecore_rdma_copy_gids(qp, p_ramrod->src_gid, p_ramrod->dst_gid); + rc = ecore_spq_post(p_hwfn, p_ent, OSAL_NULL); + + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "Modify responder, rc = %d\n", rc); + return rc; +} + +static enum _ecore_status_t ecore_roce_sp_modify_requester( + struct ecore_hwfn *p_hwfn, + struct ecore_rdma_qp *qp, + bool move_to_sqd, + bool move_to_err, + u32 modify_flags) +{ + struct roce_modify_qp_req_ramrod_data *p_ramrod; + struct ecore_sp_init_data init_data; + struct ecore_spq_entry *p_ent; + enum _ecore_status_t rc; + + if (!qp->has_req) + return ECORE_SUCCESS; + + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "icid = %08x\n", qp->icid); + + if (move_to_err && !(qp->req_offloaded)) + return ECORE_SUCCESS; + + /* Get SPQ entry */ + OSAL_MEMSET(&init_data, 0, sizeof(init_data)); + init_data.cid = qp->icid + 1; + init_data.opaque_fid = p_hwfn->hw_info.opaque_fid; + init_data.comp_mode = ECORE_SPQ_MODE_EBLOCK; + + rc = ecore_sp_init_request(p_hwfn, &p_ent, + ROCE_EVENT_MODIFY_QP, + PROTOCOLID_ROCE, &init_data); + if (rc != ECORE_SUCCESS) { + DP_NOTICE(p_hwfn, false, "rc = %d\n", rc); + return rc; + } + + p_ramrod = &p_ent->ramrod.roce_modify_qp_req; + + p_ramrod->flags = 0; + + SET_FIELD(p_ramrod->flags, + ROCE_MODIFY_QP_REQ_RAMROD_DATA_MOVE_TO_ERR_FLG, + move_to_err); + + SET_FIELD(p_ramrod->flags, + ROCE_MODIFY_QP_REQ_RAMROD_DATA_MOVE_TO_SQD_FLG, + move_to_sqd); + + SET_FIELD(p_ramrod->flags, + ROCE_MODIFY_QP_REQ_RAMROD_DATA_EN_SQD_ASYNC_NOTIFY, + qp->sqd_async); + + SET_FIELD(p_ramrod->flags, + ROCE_MODIFY_QP_REQ_RAMROD_DATA_P_KEY_FLG, + GET_FIELD(modify_flags, ECORE_ROCE_MODIFY_QP_VALID_PKEY)); + + SET_FIELD(p_ramrod->flags, + ROCE_MODIFY_QP_REQ_RAMROD_DATA_ADDRESS_VECTOR_FLG, + GET_FIELD(modify_flags, + ECORE_ROCE_MODIFY_QP_VALID_ADDRESS_VECTOR)); + + SET_FIELD(p_ramrod->flags, + ROCE_MODIFY_QP_REQ_RAMROD_DATA_MAX_ORD_FLG, + GET_FIELD(modify_flags, + ECORE_RDMA_MODIFY_QP_VALID_MAX_RD_ATOMIC_REQ)); + + SET_FIELD(p_ramrod->flags, + ROCE_MODIFY_QP_REQ_RAMROD_DATA_RNR_NAK_CNT_FLG, + GET_FIELD(modify_flags, + ECORE_ROCE_MODIFY_QP_VALID_RNR_RETRY_CNT)); + + SET_FIELD(p_ramrod->flags, + ROCE_MODIFY_QP_REQ_RAMROD_DATA_ERR_RETRY_CNT_FLG, + GET_FIELD(modify_flags, + ECORE_ROCE_MODIFY_QP_VALID_RETRY_CNT)); + + SET_FIELD(p_ramrod->flags, + ROCE_MODIFY_QP_REQ_RAMROD_DATA_ACK_TIMEOUT_FLG, + GET_FIELD(modify_flags, + ECORE_ROCE_MODIFY_QP_VALID_ACK_TIMEOUT)); + + /* TBD: future use only + * #define ROCE_MODIFY_QP_REQ_RAMROD_DATA_PRI_FLG_MASK + * #define ROCE_MODIFY_QP_REQ_RAMROD_DATA_PRI_FLG_SHIFT + */ + + p_ramrod->fields = 0; + SET_FIELD(p_ramrod->fields, + ROCE_MODIFY_QP_REQ_RAMROD_DATA_ERR_RETRY_CNT, + qp->retry_cnt); + + SET_FIELD(p_ramrod->fields, + ROCE_MODIFY_QP_REQ_RAMROD_DATA_RNR_NAK_CNT, + qp->rnr_retry_cnt); + + p_ramrod->max_ord = qp->max_rd_atomic_req; + p_ramrod->traffic_class = qp->traffic_class_tos; + p_ramrod->hop_limit = qp->hop_limit_ttl; + p_ramrod->p_key = OSAL_CPU_TO_LE16(qp->pkey); + p_ramrod->flow_label = OSAL_CPU_TO_LE32(qp->flow_label); + p_ramrod->ack_timeout_val = OSAL_CPU_TO_LE32(qp->ack_timeout); + p_ramrod->mtu = OSAL_CPU_TO_LE16(qp->mtu); + ecore_rdma_copy_gids(qp, p_ramrod->src_gid, p_ramrod->dst_gid); + rc = ecore_spq_post(p_hwfn, p_ent, OSAL_NULL); + + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "Modify requester, rc = %d\n", rc); + return rc; +} + +static enum _ecore_status_t ecore_roce_sp_destroy_qp_responder( + struct ecore_hwfn *p_hwfn, + struct ecore_rdma_qp *qp, + u32 *num_invalidated_mw, + u32 *cq_prod) +{ + struct roce_destroy_qp_resp_output_params *p_ramrod_res; + struct roce_destroy_qp_resp_ramrod_data *p_ramrod; + struct ecore_sp_init_data init_data; + struct ecore_spq_entry *p_ent; + dma_addr_t ramrod_res_phys; + enum _ecore_status_t rc; + + if (!qp->has_resp) { + *num_invalidated_mw = 0; + *cq_prod = 0; + return ECORE_SUCCESS; + } + + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "icid = %08x\n", qp->icid); + + *num_invalidated_mw = 0; + + if (!qp->resp_offloaded) { + *cq_prod = qp->cq_prod.resp; + return ECORE_SUCCESS; + } + + /* Get SPQ entry */ + OSAL_MEMSET(&init_data, 0, sizeof(init_data)); + init_data.cid = qp->icid; + init_data.opaque_fid = p_hwfn->hw_info.opaque_fid; + init_data.comp_mode = ECORE_SPQ_MODE_EBLOCK; + + rc = ecore_sp_init_request(p_hwfn, &p_ent, + ROCE_RAMROD_DESTROY_QP, + PROTOCOLID_ROCE, &init_data); + if (rc != ECORE_SUCCESS) + return rc; + + p_ramrod = &p_ent->ramrod.roce_destroy_qp_resp; + + p_ramrod_res = (struct roce_destroy_qp_resp_output_params *)OSAL_DMA_ALLOC_COHERENT(p_hwfn->p_dev, + &ramrod_res_phys, sizeof(*p_ramrod_res)); + + if (!p_ramrod_res) + { + rc = ECORE_NOMEM; + DP_NOTICE(p_hwfn, false, + "ecore destroy responder failed: cannot allocate memory (ramrod). rc = %d\n", + rc); + return rc; + } + + DMA_REGPAIR_LE(p_ramrod->output_params_addr, ramrod_res_phys); + + rc = ecore_spq_post(p_hwfn, p_ent, OSAL_NULL); + if (rc != ECORE_SUCCESS) + goto err; + + *num_invalidated_mw + = OSAL_LE32_TO_CPU(p_ramrod_res->num_invalidated_mw); + *cq_prod = OSAL_LE32_TO_CPU(p_ramrod_res->cq_prod); + qp->cq_prod.resp = *cq_prod; + + /* Free IRQ - only if ramrod succeeded, in case FW is still using it */ + OSAL_DMA_FREE_COHERENT(p_hwfn->p_dev, + qp->irq, + qp->irq_phys_addr, + qp->irq_num_pages * + RDMA_RING_PAGE_SIZE); + + qp->resp_offloaded = false; + + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "Destroy responder, rc = %d\n", rc); + + /* "fall through" */ + +err: + OSAL_DMA_FREE_COHERENT(p_hwfn->p_dev, p_ramrod_res, ramrod_res_phys, + sizeof(*p_ramrod_res)); + + return rc; +} + +static enum _ecore_status_t ecore_roce_sp_destroy_qp_requester( + struct ecore_hwfn *p_hwfn, + struct ecore_rdma_qp *qp, + u32 *num_bound_mw, + u32 *cq_prod) +{ + struct roce_destroy_qp_req_output_params *p_ramrod_res; + struct roce_destroy_qp_req_ramrod_data *p_ramrod; + struct ecore_sp_init_data init_data; + struct ecore_spq_entry *p_ent; + dma_addr_t ramrod_res_phys; + enum _ecore_status_t rc; + + if (!qp->has_req) { + *num_bound_mw = 0; + *cq_prod = 0; + return ECORE_SUCCESS; + } + + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "icid = %08x\n", qp->icid); + + if (!qp->req_offloaded) { + *cq_prod = qp->cq_prod.req; + return ECORE_SUCCESS; + } + + p_ramrod_res = (struct roce_destroy_qp_req_output_params *) + OSAL_DMA_ALLOC_COHERENT(p_hwfn->p_dev, &ramrod_res_phys, + sizeof(*p_ramrod_res)); + if (!p_ramrod_res) + { + DP_NOTICE(p_hwfn, false, + "ecore destroy requester failed: cannot allocate memory (ramrod)\n"); + return ECORE_NOMEM; + } + + /* Get SPQ entry */ + OSAL_MEMSET(&init_data, 0, sizeof(init_data)); + init_data.cid = qp->icid + 1; + init_data.opaque_fid = p_hwfn->hw_info.opaque_fid; + init_data.comp_mode = ECORE_SPQ_MODE_EBLOCK; + + rc = ecore_sp_init_request(p_hwfn, &p_ent, ROCE_RAMROD_DESTROY_QP, + PROTOCOLID_ROCE, &init_data); + if (rc != ECORE_SUCCESS) + goto err; + + p_ramrod = &p_ent->ramrod.roce_destroy_qp_req; + DMA_REGPAIR_LE(p_ramrod->output_params_addr, ramrod_res_phys); + + rc = ecore_spq_post(p_hwfn, p_ent, OSAL_NULL); + if (rc != ECORE_SUCCESS) + goto err; + + *num_bound_mw = OSAL_LE32_TO_CPU(p_ramrod_res->num_bound_mw); + *cq_prod = OSAL_LE32_TO_CPU(p_ramrod_res->cq_prod); + qp->cq_prod.req = *cq_prod; + + /* Free ORQ - only if ramrod succeeded, in case FW is still using it */ + OSAL_DMA_FREE_COHERENT(p_hwfn->p_dev, + qp->orq, + qp->orq_phys_addr, + qp->orq_num_pages * + RDMA_RING_PAGE_SIZE); + + qp->req_offloaded = false; + + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "Destroy requester, rc = %d\n", rc); + + /* "fall through" */ + +err: + OSAL_DMA_FREE_COHERENT(p_hwfn->p_dev, p_ramrod_res, ramrod_res_phys, + sizeof(*p_ramrod_res)); + + return rc; +} + +static OSAL_INLINE enum _ecore_status_t ecore_roce_sp_query_responder( + struct ecore_hwfn *p_hwfn, + struct ecore_rdma_qp *qp, + struct ecore_rdma_query_qp_out_params *out_params) +{ + struct roce_query_qp_resp_output_params *p_resp_ramrod_res; + struct roce_query_qp_resp_ramrod_data *p_resp_ramrod; + struct ecore_sp_init_data init_data; + dma_addr_t resp_ramrod_res_phys; + struct ecore_spq_entry *p_ent; + enum _ecore_status_t rc = ECORE_SUCCESS; + bool error_flag; + + if (!qp->resp_offloaded) { + /* Don't send query qp for the responder */ + out_params->rq_psn = qp->rq_psn; + + return ECORE_SUCCESS; + } + + /* Send a query responder ramrod to the FW */ + p_resp_ramrod_res = (struct roce_query_qp_resp_output_params *) + OSAL_DMA_ALLOC_COHERENT(p_hwfn->p_dev, &resp_ramrod_res_phys, + sizeof(*p_resp_ramrod_res)); + if (!p_resp_ramrod_res) + { + DP_NOTICE(p_hwfn, false, + "ecore query qp failed: cannot allocate memory (ramrod)\n"); + return ECORE_NOMEM; + } + + /* Get SPQ entry */ + OSAL_MEMSET(&init_data, 0, sizeof(init_data)); + init_data.cid = qp->icid; + init_data.opaque_fid = p_hwfn->hw_info.opaque_fid; + init_data.comp_mode = ECORE_SPQ_MODE_EBLOCK; + rc = ecore_sp_init_request(p_hwfn, &p_ent, ROCE_RAMROD_QUERY_QP, + PROTOCOLID_ROCE, &init_data); + if (rc != ECORE_SUCCESS) + goto err; + + p_resp_ramrod = &p_ent->ramrod.roce_query_qp_resp; + DMA_REGPAIR_LE(p_resp_ramrod->output_params_addr, resp_ramrod_res_phys); + + rc = ecore_spq_post(p_hwfn, p_ent, OSAL_NULL); + if (rc != ECORE_SUCCESS) + goto err; + + out_params->rq_psn = OSAL_LE32_TO_CPU(p_resp_ramrod_res->psn); + error_flag = GET_FIELD( + OSAL_LE32_TO_CPU(p_resp_ramrod_res->err_flag), + ROCE_QUERY_QP_RESP_OUTPUT_PARAMS_ERROR_FLG); + if (error_flag) + qp->cur_state = ECORE_ROCE_QP_STATE_ERR; + +err: + OSAL_DMA_FREE_COHERENT(p_hwfn->p_dev, p_resp_ramrod_res, + resp_ramrod_res_phys, + sizeof(*p_resp_ramrod_res)); + + return rc; +} + +static OSAL_INLINE enum _ecore_status_t ecore_roce_sp_query_requester( + struct ecore_hwfn *p_hwfn, + struct ecore_rdma_qp *qp, + struct ecore_rdma_query_qp_out_params *out_params, + bool *sq_draining) +{ + struct roce_query_qp_req_output_params *p_req_ramrod_res; + struct roce_query_qp_req_ramrod_data *p_req_ramrod; + struct ecore_sp_init_data init_data; + dma_addr_t req_ramrod_res_phys; + struct ecore_spq_entry *p_ent; + enum _ecore_status_t rc = ECORE_SUCCESS; + bool error_flag; + + if (!qp->req_offloaded) + { + /* Don't send query qp for the requester */ + out_params->sq_psn = qp->sq_psn; + out_params->draining = false; + + *sq_draining = 0; + + return ECORE_SUCCESS; + } + + /* Send a query requester ramrod to the FW */ + p_req_ramrod_res = (struct roce_query_qp_req_output_params *) + OSAL_DMA_ALLOC_COHERENT(p_hwfn->p_dev, &req_ramrod_res_phys, + sizeof(*p_req_ramrod_res)); + if (!p_req_ramrod_res) + { + DP_NOTICE(p_hwfn, false, + "ecore query qp failed: cannot allocate memory (ramrod). rc = %d\n", + rc); + return ECORE_NOMEM; + } + + /* Get SPQ entry */ + init_data.cid = qp->icid + 1; + rc = ecore_sp_init_request(p_hwfn, &p_ent, ROCE_RAMROD_QUERY_QP, + PROTOCOLID_ROCE, &init_data); + if (rc != ECORE_SUCCESS) + goto err; + + p_req_ramrod = &p_ent->ramrod.roce_query_qp_req; + DMA_REGPAIR_LE(p_req_ramrod->output_params_addr, req_ramrod_res_phys); + + rc = ecore_spq_post(p_hwfn, p_ent, OSAL_NULL); + if (rc != ECORE_SUCCESS) + goto err; + + out_params->sq_psn = OSAL_LE32_TO_CPU(p_req_ramrod_res->psn); + error_flag = GET_FIELD(OSAL_LE32_TO_CPU(p_req_ramrod_res->flags), + ROCE_QUERY_QP_REQ_OUTPUT_PARAMS_ERR_FLG); + if (error_flag) + qp->cur_state = ECORE_ROCE_QP_STATE_ERR; + else + *sq_draining = GET_FIELD( + OSAL_LE32_TO_CPU(p_req_ramrod_res->flags), + ROCE_QUERY_QP_REQ_OUTPUT_PARAMS_SQ_DRAINING_FLG); + +err: + OSAL_DMA_FREE_COHERENT(p_hwfn->p_dev, p_req_ramrod_res, + req_ramrod_res_phys, sizeof(*p_req_ramrod_res)); + + return rc; +} + +enum _ecore_status_t ecore_roce_query_qp( + struct ecore_hwfn *p_hwfn, + struct ecore_rdma_qp *qp, + struct ecore_rdma_query_qp_out_params *out_params) +{ + enum _ecore_status_t rc; + + rc = ecore_roce_sp_query_responder(p_hwfn, qp, out_params); + if (rc) + return rc; + + rc = ecore_roce_sp_query_requester(p_hwfn, qp, out_params, + &out_params->draining); + if (rc) + return rc; + + out_params->state = qp->cur_state; + + return ECORE_SUCCESS; +} + +enum _ecore_status_t ecore_roce_destroy_qp(struct ecore_hwfn *p_hwfn, + struct ecore_rdma_qp *qp, + struct ecore_rdma_destroy_qp_out_params *out_params) +{ + u32 cq_prod_resp = qp->cq_prod.resp, cq_prod_req = qp->cq_prod.req; + u32 num_invalidated_mw = 0; + u32 num_bound_mw = 0; + enum _ecore_status_t rc; + + /* Destroys the specified QP + * Note: if qp state != RESET/ERR/INIT then upper driver first need to + * call modify qp to move the qp to ERR state + */ + if ((qp->cur_state != ECORE_ROCE_QP_STATE_RESET) && + (qp->cur_state != ECORE_ROCE_QP_STATE_ERR) && + (qp->cur_state != ECORE_ROCE_QP_STATE_INIT)) + { + DP_NOTICE(p_hwfn, + true, + "QP must be in error, reset or init state before destroying it\n"); + return ECORE_INVAL; + } + + if (qp->cur_state != ECORE_ROCE_QP_STATE_RESET) { + rc = ecore_roce_sp_destroy_qp_responder(p_hwfn, + qp, + &num_invalidated_mw, + &cq_prod_resp); + if (rc != ECORE_SUCCESS) + return rc; + + /* Send destroy requester ramrod */ + rc = ecore_roce_sp_destroy_qp_requester(p_hwfn, qp, + &num_bound_mw, + &cq_prod_req); + if (rc != ECORE_SUCCESS) + return rc; + + /* resp_ofload was true, num_invalidated_mw is valid */ + if (num_invalidated_mw != num_bound_mw) { + DP_NOTICE(p_hwfn, + true, + "number of invalidate memory windows is different from bounded ones\n"); + return ECORE_INVAL; + } + } + + ecore_roce_free_qp(p_hwfn, qp->qp_idx); + + out_params->rq_cq_prod = cq_prod_resp; + out_params->sq_cq_prod = cq_prod_req; + + return ECORE_SUCCESS; +} + +enum _ecore_status_t ecore_roce_destroy_ud_qp(void *rdma_cxt, u16 cid) +{ + struct ecore_hwfn *p_hwfn = (struct ecore_hwfn *)rdma_cxt; + struct ecore_sp_init_data init_data; + struct ecore_spq_entry *p_ent; + enum _ecore_status_t rc; + + if (!rdma_cxt) { + DP_ERR(p_hwfn->p_dev, + "destroy ud qp failed due to NULL rdma_cxt\n"); + return ECORE_INVAL; + } + + /* Get SPQ entry */ + OSAL_MEMSET(&init_data, 0, sizeof(init_data)); + init_data.cid = cid; + init_data.opaque_fid = p_hwfn->hw_info.opaque_fid; + init_data.comp_mode = ECORE_SPQ_MODE_EBLOCK; + rc = ecore_sp_init_request(p_hwfn, &p_ent, ROCE_RAMROD_DESTROY_UD_QP, + PROTOCOLID_ROCE, &init_data); + if (rc != ECORE_SUCCESS) + goto err; + + rc = ecore_spq_post(p_hwfn, p_ent, OSAL_NULL); + if (rc != ECORE_SUCCESS) + goto err; + + ecore_roce_free_qp(p_hwfn, ECORE_ROCE_ICID_TO_QP(cid)); + + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "freed a ud qp with cid=%d\n", cid); + + return ECORE_SUCCESS; + +err: + DP_ERR(p_hwfn, "failed destroying a ud qp with cid=%d\n", cid); + + return rc; +} + + +enum _ecore_status_t ecore_roce_create_ud_qp(void *rdma_cxt, + struct ecore_rdma_create_qp_out_params *out_params) +{ + struct ecore_hwfn *p_hwfn = (struct ecore_hwfn *)rdma_cxt; + struct ecore_sp_init_data init_data; + struct ecore_spq_entry *p_ent; + enum _ecore_status_t rc; + u16 icid, qp_idx; + + if (!rdma_cxt || !out_params) { + DP_ERR(p_hwfn->p_dev, + "ecore roce create ud qp failed due to NULL entry (rdma_cxt=%p, out=%p)\n", + rdma_cxt, out_params); + return ECORE_INVAL; + } + + rc = ecore_roce_alloc_qp_idx(p_hwfn, &qp_idx); + if (rc != ECORE_SUCCESS) + goto err; + + icid = ECORE_ROCE_QP_TO_ICID(qp_idx); + + /* Get SPQ entry */ + OSAL_MEMSET(&init_data, 0, sizeof(init_data)); + init_data.cid = icid; + init_data.opaque_fid = p_hwfn->hw_info.opaque_fid; + init_data.comp_mode = ECORE_SPQ_MODE_EBLOCK; + rc = ecore_sp_init_request(p_hwfn, &p_ent, ROCE_RAMROD_CREATE_UD_QP, + PROTOCOLID_ROCE, &init_data); + if (rc != ECORE_SUCCESS) + goto err1; + + rc = ecore_spq_post(p_hwfn, p_ent, OSAL_NULL); + if (rc != ECORE_SUCCESS) + goto err1; + + out_params->icid = icid; + out_params->qp_id = ((0xFF << 16) | icid); + + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "created a ud qp with icid=%d\n", + icid); + + return ECORE_SUCCESS; + +err1: + ecore_roce_free_qp(p_hwfn, qp_idx); + +err: + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "failed creating a ud qp\n"); + + return rc; +} + + +enum _ecore_status_t +ecore_roce_modify_qp(struct ecore_hwfn *p_hwfn, + struct ecore_rdma_qp *qp, + enum ecore_roce_qp_state prev_state, + struct ecore_rdma_modify_qp_in_params *params) +{ + u32 num_invalidated_mw = 0, num_bound_mw = 0; + enum _ecore_status_t rc = ECORE_SUCCESS; + + /* Perform additional operations according to the current state and the + * next state + */ + if (((prev_state == ECORE_ROCE_QP_STATE_INIT) || + (prev_state == ECORE_ROCE_QP_STATE_RESET)) && + (qp->cur_state == ECORE_ROCE_QP_STATE_RTR)) + { + /* Init->RTR or Reset->RTR */ + + /* Verify the cid bits that of this qp index are clear */ + rc = ecore_roce_wait_free_cids(p_hwfn, qp->qp_idx); + if (rc) + return rc; + + rc = ecore_roce_sp_create_responder(p_hwfn, qp); + return rc; + + } else if ((prev_state == ECORE_ROCE_QP_STATE_RTR) && + (qp->cur_state == ECORE_ROCE_QP_STATE_RTS)) + { + /* RTR-> RTS */ + rc = ecore_roce_sp_create_requester(p_hwfn, qp); + if (rc != ECORE_SUCCESS) + return rc; + + /* Send modify responder ramrod */ + rc = ecore_roce_sp_modify_responder(p_hwfn, qp, false, + params->modify_flags); + return rc; + + } else if ((prev_state == ECORE_ROCE_QP_STATE_RTS) && + (qp->cur_state == ECORE_ROCE_QP_STATE_RTS)) + { + /* RTS->RTS */ + rc = ecore_roce_sp_modify_responder(p_hwfn, qp, false, + params->modify_flags); + if (rc != ECORE_SUCCESS) + return rc; + + rc = ecore_roce_sp_modify_requester(p_hwfn, qp, false, false, + params->modify_flags); + return rc; + + } else if ((prev_state == ECORE_ROCE_QP_STATE_RTS) && + (qp->cur_state == ECORE_ROCE_QP_STATE_SQD)) + { + /* RTS->SQD */ + rc = ecore_roce_sp_modify_requester(p_hwfn, qp, true, false, + params->modify_flags); + return rc; + + } else if ((prev_state == ECORE_ROCE_QP_STATE_SQD) && + (qp->cur_state == ECORE_ROCE_QP_STATE_SQD)) + { + /* SQD->SQD */ + rc = ecore_roce_sp_modify_responder(p_hwfn, qp, false, + params->modify_flags); + if (rc != ECORE_SUCCESS) + return rc; + + rc = ecore_roce_sp_modify_requester(p_hwfn, qp, false, false, + params->modify_flags); + return rc; + + } else if ((prev_state == ECORE_ROCE_QP_STATE_SQD) && + (qp->cur_state == ECORE_ROCE_QP_STATE_RTS)) + { + /* SQD->RTS */ + rc = ecore_roce_sp_modify_responder(p_hwfn, qp, false, + params->modify_flags); + if (rc != ECORE_SUCCESS) + return rc; + + rc = ecore_roce_sp_modify_requester(p_hwfn, qp, false, false, + params->modify_flags); + + return rc; + } else if (qp->cur_state == ECORE_ROCE_QP_STATE_ERR) { + /* ->ERR */ + rc = ecore_roce_sp_modify_responder(p_hwfn, qp, true, + params->modify_flags); + if (rc != ECORE_SUCCESS) + return rc; + + rc = ecore_roce_sp_modify_requester(p_hwfn, qp, false, true, + params->modify_flags); + return rc; + + } else if (qp->cur_state == ECORE_ROCE_QP_STATE_RESET) { + /* Any state -> RESET */ + + /* Send destroy responder ramrod */ + rc = ecore_roce_sp_destroy_qp_responder(p_hwfn, qp, + &num_invalidated_mw, + &qp->cq_prod.resp); + + if (rc != ECORE_SUCCESS) + return rc; + + rc = ecore_roce_sp_destroy_qp_requester(p_hwfn, qp, + &num_bound_mw, + &qp->cq_prod.req); + + + if (rc != ECORE_SUCCESS) + return rc; + + if (num_invalidated_mw != num_bound_mw) { + DP_NOTICE(p_hwfn, + true, + "number of invalidate memory windows is different from bounded ones\n"); + return ECORE_INVAL; + } + } else { + DP_VERBOSE(p_hwfn, ECORE_MSG_RDMA, "ECORE_SUCCESS\n"); + } + + return rc; +} + +static void ecore_roce_free_icid(struct ecore_hwfn *p_hwfn, u16 icid) +{ + struct ecore_rdma_info *p_rdma_info = p_hwfn->p_rdma_info; + u32 start_cid, cid; + + start_cid = ecore_cxt_get_proto_cid_start(p_hwfn, p_rdma_info->proto); + cid = icid - start_cid; + + OSAL_SPIN_LOCK(&p_rdma_info->lock); + + ecore_bmap_release_id(p_hwfn, &p_rdma_info->cid_map, cid); + + OSAL_SPIN_UNLOCK(&p_hwfn->p_rdma_info->lock); +} + +static void ecore_rdma_dpm_conf(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt) +{ + u32 val; + + val = (p_hwfn->dcbx_no_edpm || p_hwfn->db_bar_no_edpm) ? 0 : 1; + + ecore_wr(p_hwfn, p_ptt, DORQ_REG_PF_DPM_ENABLE, val); + DP_VERBOSE(p_hwfn, (ECORE_MSG_DCB | ECORE_MSG_RDMA), + "Changing DPM_EN state to %d (DCBX=%d, DB_BAR=%d)\n", + val, p_hwfn->dcbx_no_edpm, p_hwfn->db_bar_no_edpm); +} + +/* This function disables EDPM due to DCBx considerations */ +void ecore_roce_dpm_dcbx(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt) +{ + u8 val; + + /* if any QPs are already active, we want to disable DPM, since their + * context information contains information from before the latest DCBx + * update. Otherwise enable it. + */ + val = (ecore_rdma_allocated_qps(p_hwfn)) ? true : false; + p_hwfn->dcbx_no_edpm = (u8)val; + + ecore_rdma_dpm_conf(p_hwfn, p_ptt); +} + +/* This function disables EDPM due to doorbell bar considerations */ +void ecore_rdma_dpm_bar(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt) +{ + p_hwfn->db_bar_no_edpm = true; + + ecore_rdma_dpm_conf(p_hwfn, p_ptt); +} + +enum _ecore_status_t ecore_roce_setup(struct ecore_hwfn *p_hwfn) +{ + return ecore_spq_register_async_cb(p_hwfn, PROTOCOLID_ROCE, + ecore_roce_async_event); +} + +#ifdef _NTDDK_ +#pragma warning(pop) +#endif Property changes on: head/sys/dev/qlnx/qlnxe/ecore_roce.c ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Added: svn:mime-type ## -0,0 +1 ## +text/plain \ No newline at end of property Index: head/sys/dev/qlnx/qlnxe/qlnx_rdma.c =================================================================== --- head/sys/dev/qlnx/qlnxe/qlnx_rdma.c (nonexistent) +++ head/sys/dev/qlnx/qlnxe/qlnx_rdma.c (revision 343598) @@ -0,0 +1,347 @@ +/* + * Copyright (c) 2018-2019 Cavium, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +/* + * File : qlnx_rdma.c + * Author: David C Somayajulu + */ +#include +__FBSDID("$FreeBSD$"); + + +#include "qlnx_os.h" +#include "bcm_osal.h" + +#include "reg_addr.h" +#include "ecore_gtt_reg_addr.h" +#include "ecore.h" +#include "ecore_chain.h" +#include "ecore_status.h" +#include "ecore_hw.h" +#include "ecore_rt_defs.h" +#include "ecore_init_ops.h" +#include "ecore_int.h" +#include "ecore_cxt.h" +#include "ecore_spq.h" +#include "ecore_init_fw_funcs.h" +#include "ecore_sp_commands.h" +#include "ecore_dev_api.h" +#include "ecore_l2_api.h" +#ifdef CONFIG_ECORE_SRIOV +#include "ecore_sriov.h" +#include "ecore_vf.h" +#endif +#ifdef CONFIG_ECORE_LL2 +#include "ecore_ll2.h" +#endif +#ifdef CONFIG_ECORE_FCOE +#include "ecore_fcoe.h" +#endif +#ifdef CONFIG_ECORE_ISCSI +#include "ecore_iscsi.h" +#endif +#include "ecore_mcp.h" +#include "ecore_hw_defs.h" +#include "mcp_public.h" + +#ifdef CONFIG_ECORE_RDMA +#include "ecore_rdma.h" +#endif + +#ifdef CONFIG_ECORE_ROCE +#include "ecore_roce.h" +#endif + +#ifdef CONFIG_ECORE_IWARP +#include "ecore_iwarp.h" +#endif + +#include "ecore_iro.h" +#include "nvm_cfg.h" +#include "ecore_dev_api.h" +#include "ecore_dbg_fw_funcs.h" + +#include "qlnx_ioctl.h" +#include "qlnx_def.h" +#include "qlnx_rdma.h" +#include "qlnx_ver.h" +#include + +struct mtx qlnx_rdma_dev_lock; +struct qlnx_rdma_if *qlnx_rdma_if = NULL; + +qlnx_host_t *qlnx_host_list = NULL; + +void +qlnx_rdma_init(void) +{ + if (!mtx_initialized(&qlnx_rdma_dev_lock)) { + mtx_init(&qlnx_rdma_dev_lock, "qlnx_rdma_dev_lock", NULL, MTX_DEF); + } + return; +} + +void +qlnx_rdma_deinit(void) +{ + if (mtx_initialized(&qlnx_rdma_dev_lock) && (qlnx_host_list == NULL)) { + mtx_destroy(&qlnx_rdma_dev_lock); + } + return; +} + +static void +_qlnx_rdma_dev_add(struct qlnx_host *ha) +{ + QL_DPRINT12(ha, "enter ha = %p qlnx_rdma_if = %p\n", ha, qlnx_rdma_if); + + if (qlnx_rdma_if == NULL) + return; + + if (ha->personality != ECORE_PCI_ETH_IWARP && + ha->personality != ECORE_PCI_ETH_ROCE) + return; + + ha->qlnx_rdma = qlnx_rdma_if->add(ha); + + QL_DPRINT12(ha, "exit (ha = %p, qlnx_rdma = %p)\n", ha, ha->qlnx_rdma); + return; +} + +void +qlnx_rdma_dev_add(struct qlnx_host *ha) +{ + QL_DPRINT12(ha, "enter ha = %p\n", ha); + + if (ha->personality != ECORE_PCI_ETH_IWARP && + ha->personality != ECORE_PCI_ETH_ROCE) + return; + + mtx_lock(&qlnx_rdma_dev_lock); + + if (qlnx_host_list == NULL) { + qlnx_host_list = ha; + ha->next = NULL; + } else { + ha->next = qlnx_host_list; + qlnx_host_list = ha; + } + + mtx_unlock(&qlnx_rdma_dev_lock); + + _qlnx_rdma_dev_add(ha); + + QL_DPRINT12(ha, "exit (%p)\n", ha); + + return; +} + +static int +_qlnx_rdma_dev_remove(struct qlnx_host *ha) +{ + int ret = 0; + + QL_DPRINT12(ha, "enter ha = %p qlnx_rdma_if = %p\n", ha, qlnx_rdma_if); + + if (qlnx_rdma_if == NULL) + return (ret); + + if (ha->personality != ECORE_PCI_ETH_IWARP && + ha->personality != ECORE_PCI_ETH_ROCE) + return (ret); + + ret = qlnx_rdma_if->remove(ha, ha->qlnx_rdma); + + QL_DPRINT12(ha, "exit ha = %p qlnx_rdma_if = %p\n", ha, qlnx_rdma_if); + return (ret); +} + +int +qlnx_rdma_dev_remove(struct qlnx_host *ha) +{ + int ret = 0; + qlnx_host_t *ha_prev; + qlnx_host_t *ha_cur; + + QL_DPRINT12(ha, "enter ha = %p\n", ha); + + if ((qlnx_host_list == NULL) || (ha == NULL)) + return (ret); + + if (ha->personality != ECORE_PCI_ETH_IWARP && + ha->personality != ECORE_PCI_ETH_ROCE) + return (ret); + + ret = _qlnx_rdma_dev_remove(ha); + + if (ret) + return (ret); + + mtx_lock(&qlnx_rdma_dev_lock); + + if (qlnx_host_list == ha) { + qlnx_host_list = ha->next; + ha->next = NULL; + mtx_unlock(&qlnx_rdma_dev_lock); + QL_DPRINT12(ha, "exit0 ha = %p\n", ha); + return (ret); + } + + ha_prev = ha_cur = qlnx_host_list; + + while ((ha_cur != ha) && (ha_cur != NULL)) { + ha_prev = ha_cur; + ha_cur = ha_cur->next; + } + + if (ha_cur == ha) { + ha_prev = ha->next; + ha->next = NULL; + } + + mtx_unlock(&qlnx_rdma_dev_lock); + + QL_DPRINT12(ha, "exit1 ha = %p\n", ha); + return (ret); +} + +int +qlnx_rdma_register_if(qlnx_rdma_if_t *rdma_if) +{ + qlnx_host_t *ha; + + if (mtx_initialized(&qlnx_rdma_dev_lock)) { + + mtx_lock(&qlnx_rdma_dev_lock); + qlnx_rdma_if = rdma_if; + + ha = qlnx_host_list; + + while (ha != NULL) { + _qlnx_rdma_dev_add(ha); + ha = ha->next; + } + + mtx_unlock(&qlnx_rdma_dev_lock); + + return (0); + } + + return (-1); +} + +int +qlnx_rdma_deregister_if(qlnx_rdma_if_t *rdma_if) +{ + int ret = 0; + qlnx_host_t *ha; + + printf("%s: enter rdma_if = %p\n", __func__, rdma_if); + + if (mtx_initialized(&qlnx_rdma_dev_lock)) { + + mtx_lock(&qlnx_rdma_dev_lock); + + ha = qlnx_host_list; + + while (ha != NULL) { + + mtx_unlock(&qlnx_rdma_dev_lock); + + if (ha->dbg_level & 0xF000) + ret = EBUSY; + else + ret = _qlnx_rdma_dev_remove(ha); + + device_printf(ha->pci_dev, "%s [%d]: ret = 0x%x\n", + __func__, __LINE__, ret); + if (ret) + return (ret); + + mtx_lock(&qlnx_rdma_dev_lock); + + ha->qlnx_rdma = NULL; + + ha = ha->next; + } + + if (!ret) + qlnx_rdma_if = NULL; + + mtx_unlock(&qlnx_rdma_dev_lock); + + } + printf("%s: exit rdma_if = %p\n", __func__, rdma_if); + + return (ret); +} + + +void +qlnx_rdma_dev_open(struct qlnx_host *ha) +{ + QL_DPRINT12(ha, "enter ha = %p qlnx_rdma_if = %p\n", ha, qlnx_rdma_if); + + if (qlnx_rdma_if == NULL) + return; + + if (ha->personality != ECORE_PCI_ETH_IWARP && + ha->personality != ECORE_PCI_ETH_ROCE) + return; + + qlnx_rdma_if->notify(ha, ha->qlnx_rdma, QLNX_ETHDEV_UP); + + QL_DPRINT12(ha, "exit ha = %p qlnx_rdma_if = %p\n", ha, qlnx_rdma_if); + return; +} + + +void +qlnx_rdma_dev_close(struct qlnx_host *ha) +{ + QL_DPRINT12(ha, "enter ha = %p qlnx_rdma_if = %p\n", ha, qlnx_rdma_if); + + if (qlnx_rdma_if == NULL) + return; + + if (ha->personality != ECORE_PCI_ETH_IWARP && + ha->personality != ECORE_PCI_ETH_ROCE) + return; + + qlnx_rdma_if->notify(ha, ha->qlnx_rdma, QLNX_ETHDEV_DOWN); + + QL_DPRINT12(ha, "exit ha = %p qlnx_rdma_if = %p\n", ha, qlnx_rdma_if); + return; +} + +int +qlnx_rdma_get_num_irqs(struct qlnx_host *ha) +{ + return (QLNX_NUM_CNQ + ecore_rdma_get_sb_id(&ha->cdev.hwfns[0], 0) + 2); +} + + Property changes on: head/sys/dev/qlnx/qlnxe/qlnx_rdma.c ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Added: svn:mime-type ## -0,0 +1 ## +text/plain \ No newline at end of property Index: head/sys/dev/qlnx/qlnxe/qlnx_rdma.h =================================================================== --- head/sys/dev/qlnx/qlnxe/qlnx_rdma.h (nonexistent) +++ head/sys/dev/qlnx/qlnxe/qlnx_rdma.h (revision 343598) @@ -0,0 +1,69 @@ +/* + * Copyright (c) 2018-2019 Cavium, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * + * $FreeBSD$ + */ + + +/* + * File: qlnx_rdma.h + * Author: David C Somayajulu + */ + +#ifndef _QLNX_RDMA_H_ +#define _QLNX_RDMA_H_ + +enum qlnx_rdma_event { + QLNX_ETHDEV_UP = 0x10, + QLNX_ETHDEV_DOWN = 0x11, + QLNX_ETHDEV_CHANGE_ADDR = 0x12 +}; + +struct qlnx_rdma_if { + void * (*add)(void *ha); + int (*remove)(void *ha, void *qlnx_rdma_dev); + void (*notify)(void *ha, void *qlnx_rdma_dev, enum qlnx_rdma_event); +}; +typedef struct qlnx_rdma_if qlnx_rdma_if_t; + +extern int qlnx_rdma_register_if(qlnx_rdma_if_t *rdma_if); +extern int qlnx_rdma_deregister_if(qlnx_rdma_if_t *rdma_if); +extern int qlnx_rdma_ll2_set_mac_filter(void *rdma_ctx, uint8_t *old_mac_address, + uint8_t *new_mac_address); + +#define QLNX_NUM_CNQ 1 + +extern int qlnx_rdma_get_num_irqs(struct qlnx_host *ha); +extern void qlnx_rdma_dev_add(struct qlnx_host *ha); +extern void qlnx_rdma_dev_open(struct qlnx_host *ha); +extern void qlnx_rdma_dev_close(struct qlnx_host *ha); +extern int qlnx_rdma_dev_remove(struct qlnx_host *ha); +extern void qlnx_rdma_changeaddr(struct qlnx_host *ha); + +extern void qlnx_rdma_init(void); +extern void qlnx_rdma_deinit(void); + +#endif /* #ifndef _QLNX_RDMA_H_ */ Property changes on: head/sys/dev/qlnx/qlnxe/qlnx_rdma.h ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Added: svn:mime-type ## -0,0 +1 ## +text/plain \ No newline at end of property Index: head/sys/dev/qlnx/qlnxr/qlnxr_cm.c =================================================================== --- head/sys/dev/qlnx/qlnxr/qlnxr_cm.c (nonexistent) +++ head/sys/dev/qlnx/qlnxr/qlnxr_cm.c (revision 343598) @@ -0,0 +1,887 @@ +/* + * Copyright (c) 2018-2019 Cavium, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include +__FBSDID("$FreeBSD$"); + +#include "qlnxr_def.h" +#include "rdma_common.h" +#include "qlnxr_cm.h" + +void +qlnxr_inc_sw_gsi_cons(struct qlnxr_qp_hwq_info *info) +{ + info->gsi_cons = (info->gsi_cons + 1) % info->max_wr; +} + +void +qlnxr_store_gsi_qp_cq(struct qlnxr_dev *dev, + struct qlnxr_qp *qp, + struct ib_qp_init_attr *attrs) +{ + QL_DPRINT12(dev->ha, "enter\n"); + + dev->gsi_qp_created = 1; + dev->gsi_sqcq = get_qlnxr_cq((attrs->send_cq)); + dev->gsi_rqcq = get_qlnxr_cq((attrs->recv_cq)); + dev->gsi_qp = qp; + + QL_DPRINT12(dev->ha, "exit\n"); + + return; +} + +void +qlnxr_ll2_complete_tx_packet(void *cxt, + uint8_t connection_handle, + void *cookie, + dma_addr_t first_frag_addr, + bool b_last_fragment, + bool b_last_packet) +{ + struct qlnxr_dev *dev = (struct qlnxr_dev *)cxt; + struct ecore_roce_ll2_packet *pkt = cookie; + struct qlnxr_cq *cq = dev->gsi_sqcq; + struct qlnxr_qp *qp = dev->gsi_qp; + unsigned long flags; + + QL_DPRINT12(dev->ha, "enter\n"); + + qlnx_dma_free_coherent(&dev->ha->cdev, pkt->header.vaddr, + pkt->header.baddr, pkt->header.len); + kfree(pkt); + + spin_lock_irqsave(&qp->q_lock, flags); + + qlnxr_inc_sw_gsi_cons(&qp->sq); + + spin_unlock_irqrestore(&qp->q_lock, flags); + + if (cq->ibcq.comp_handler) + (*cq->ibcq.comp_handler) (&cq->ibcq, cq->ibcq.cq_context); + + QL_DPRINT12(dev->ha, "exit\n"); + + return; +} + +void +qlnxr_ll2_complete_rx_packet(void *cxt, + struct ecore_ll2_comp_rx_data *data) +{ + struct qlnxr_dev *dev = (struct qlnxr_dev *)cxt; + struct qlnxr_cq *cq = dev->gsi_rqcq; + // struct qlnxr_qp *qp = dev->gsi_qp; + struct qlnxr_qp *qp = NULL; + unsigned long flags; + uint32_t qp_num = 0; + // uint32_t delay_count = 0, gsi_cons = 0; + //void * dest_va; + + QL_DPRINT12(dev->ha, "enter\n"); + + if (data->u.data_length_error) { + /* TODO: add statistic */ + } + + if (data->cookie == NULL) { + QL_DPRINT12(dev->ha, "cookie is NULL, bad sign\n"); + } + + qp_num = (0xFF << 16) | data->qp_id; + + if (data->qp_id == 1) { + qp = dev->gsi_qp; + } else { + /* TODO: This will be needed for UD QP support */ + /* For RoCEv1 this is invalid */ + QL_DPRINT12(dev->ha, "invalid QP\n"); + return; + } + /* note: currently only one recv sg is supported */ + QL_DPRINT12(dev->ha, "MAD received on QP : %x\n", data->rx_buf_addr); + + spin_lock_irqsave(&qp->q_lock, flags); + + qp->rqe_wr_id[qp->rq.gsi_cons].rc = + data->u.data_length_error ? -EINVAL : 0; + qp->rqe_wr_id[qp->rq.gsi_cons].vlan_id = data->vlan; + /* note: length stands for data length i.e. GRH is excluded */ + qp->rqe_wr_id[qp->rq.gsi_cons].sg_list[0].length = + data->length.data_length; + *((u32 *)&qp->rqe_wr_id[qp->rq.gsi_cons].smac[0]) = + ntohl(data->opaque_data_0); + *((u16 *)&qp->rqe_wr_id[qp->rq.gsi_cons].smac[4]) = + ntohs((u16)data->opaque_data_1); + + qlnxr_inc_sw_gsi_cons(&qp->rq); + + spin_unlock_irqrestore(&qp->q_lock, flags); + + if (cq->ibcq.comp_handler) + (*cq->ibcq.comp_handler) (&cq->ibcq, cq->ibcq.cq_context); + + QL_DPRINT12(dev->ha, "exit\n"); + + return; +} + +void qlnxr_ll2_release_rx_packet(void *cxt, + u8 connection_handle, + void *cookie, + dma_addr_t rx_buf_addr, + bool b_last_packet) +{ + /* Do nothing... */ +} + +static void +qlnxr_destroy_gsi_cq(struct qlnxr_dev *dev, + struct ib_qp_init_attr *attrs) +{ + struct ecore_rdma_destroy_cq_in_params iparams; + struct ecore_rdma_destroy_cq_out_params oparams; + struct qlnxr_cq *cq; + + QL_DPRINT12(dev->ha, "enter\n"); + + cq = get_qlnxr_cq((attrs->send_cq)); + iparams.icid = cq->icid; + ecore_rdma_destroy_cq(dev->rdma_ctx, &iparams, &oparams); + ecore_chain_free(&dev->ha->cdev, &cq->pbl); + + cq = get_qlnxr_cq((attrs->recv_cq)); + /* if a dedicated recv_cq was used, delete it too */ + if (iparams.icid != cq->icid) { + iparams.icid = cq->icid; + ecore_rdma_destroy_cq(dev->rdma_ctx, &iparams, &oparams); + ecore_chain_free(&dev->ha->cdev, &cq->pbl); + } + + QL_DPRINT12(dev->ha, "exit\n"); + + return; +} + +static inline int +qlnxr_check_gsi_qp_attrs(struct qlnxr_dev *dev, + struct ib_qp_init_attr *attrs) +{ + QL_DPRINT12(dev->ha, "enter\n"); + + if (attrs->cap.max_recv_sge > QLNXR_GSI_MAX_RECV_SGE) { + QL_DPRINT11(dev->ha, + "(attrs->cap.max_recv_sge > QLNXR_GSI_MAX_RECV_SGE)\n"); + return -EINVAL; + } + + if (attrs->cap.max_recv_wr > QLNXR_GSI_MAX_RECV_WR) { + QL_DPRINT11(dev->ha, + "(attrs->cap.max_recv_wr > QLNXR_GSI_MAX_RECV_WR)\n"); + return -EINVAL; + } + + if (attrs->cap.max_send_wr > QLNXR_GSI_MAX_SEND_WR) { + QL_DPRINT11(dev->ha, + "(attrs->cap.max_send_wr > QLNXR_GSI_MAX_SEND_WR)\n"); + return -EINVAL; + } + + QL_DPRINT12(dev->ha, "exit\n"); + + return 0; +} + + +static int +qlnxr_ll2_post_tx(struct qlnxr_dev *dev, struct ecore_roce_ll2_packet *pkt) +{ + enum ecore_ll2_roce_flavor_type roce_flavor; + struct ecore_ll2_tx_pkt_info ll2_tx_pkt; + int rc; + int i; + + QL_DPRINT12(dev->ha, "enter\n"); + + memset(&ll2_tx_pkt, 0, sizeof(ll2_tx_pkt)); + + if (pkt->roce_mode != ROCE_V1) { + QL_DPRINT11(dev->ha, "roce_mode != ROCE_V1\n"); + return (-1); + } + + roce_flavor = (pkt->roce_mode == ROCE_V1) ? + ECORE_LL2_ROCE : ECORE_LL2_RROCE; + + ll2_tx_pkt.num_of_bds = 1 /* hdr */ + pkt->n_seg; + ll2_tx_pkt.vlan = 0; /* ??? */ + ll2_tx_pkt.tx_dest = ECORE_LL2_TX_DEST_NW; + ll2_tx_pkt.ecore_roce_flavor = roce_flavor; + ll2_tx_pkt.first_frag = pkt->header.baddr; + ll2_tx_pkt.first_frag_len = pkt->header.len; + ll2_tx_pkt.cookie = pkt; + ll2_tx_pkt.enable_ip_cksum = 1; // Only for RoCEv2:IPv4 + + /* tx header */ + rc = ecore_ll2_prepare_tx_packet(dev->rdma_ctx, + dev->gsi_ll2_handle, + &ll2_tx_pkt, + 1); + if (rc) { + + QL_DPRINT11(dev->ha, "ecore_ll2_prepare_tx_packet failed\n"); + + /* TX failed while posting header - release resources*/ + qlnx_dma_free_coherent(&dev->ha->cdev, + pkt->header.vaddr, + pkt->header.baddr, + pkt->header.len); + + kfree(pkt); + + return rc; + } + + /* tx payload */ + for (i = 0; i < pkt->n_seg; i++) { + rc = ecore_ll2_set_fragment_of_tx_packet(dev->rdma_ctx, + dev->gsi_ll2_handle, + pkt->payload[i].baddr, + pkt->payload[i].len); + if (rc) { + /* if failed not much to do here, partial packet has + * been posted we can't free memory, will need to wait + * for completion + */ + QL_DPRINT11(dev->ha, + "ecore_ll2_set_fragment_of_tx_packet failed\n"); + return rc; + } + } + struct ecore_ll2_stats stats = {0}; + rc = ecore_ll2_get_stats(dev->rdma_ctx, dev->gsi_ll2_handle, &stats); + if (rc) { + QL_DPRINT11(dev->ha, "failed to obtain ll2 stats\n"); + } + QL_DPRINT12(dev->ha, "exit\n"); + + return 0; +} + +int +qlnxr_ll2_stop(struct qlnxr_dev *dev) +{ + int rc; + + QL_DPRINT12(dev->ha, "enter\n"); + + if (dev->gsi_ll2_handle == 0xFF) + return 0; + + /* remove LL2 MAC address filter */ + rc = qlnx_rdma_ll2_set_mac_filter(dev->rdma_ctx, + dev->gsi_ll2_mac_address, NULL); + + rc = ecore_ll2_terminate_connection(dev->rdma_ctx, + dev->gsi_ll2_handle); + + ecore_ll2_release_connection(dev->rdma_ctx, dev->gsi_ll2_handle); + + dev->gsi_ll2_handle = 0xFF; + + QL_DPRINT12(dev->ha, "exit rc = %d\n", rc); + return rc; +} + +int qlnxr_ll2_start(struct qlnxr_dev *dev, + struct ib_qp_init_attr *attrs, + struct qlnxr_qp *qp) +{ + struct ecore_ll2_acquire_data data; + struct ecore_ll2_cbs cbs; + int rc; + + QL_DPRINT12(dev->ha, "enter\n"); + + /* configure and start LL2 */ + cbs.rx_comp_cb = qlnxr_ll2_complete_rx_packet; + cbs.tx_comp_cb = qlnxr_ll2_complete_tx_packet; + cbs.rx_release_cb = qlnxr_ll2_release_rx_packet; + cbs.tx_release_cb = qlnxr_ll2_complete_tx_packet; + cbs.cookie = dev; + dev->gsi_ll2_handle = 0xFF; + + memset(&data, 0, sizeof(data)); + data.input.conn_type = ECORE_LL2_TYPE_ROCE; + data.input.mtu = dev->ha->ifp->if_mtu; + data.input.rx_num_desc = 8 * 1024; + data.input.rx_drop_ttl0_flg = 1; + data.input.rx_vlan_removal_en = 0; + data.input.tx_num_desc = 8 * 1024; + data.input.tx_tc = 0; + data.input.tx_dest = ECORE_LL2_TX_DEST_NW; + data.input.ai_err_packet_too_big = ECORE_LL2_DROP_PACKET; + data.input.ai_err_no_buf = ECORE_LL2_DROP_PACKET; + data.input.gsi_enable = 1; + data.p_connection_handle = &dev->gsi_ll2_handle; + data.cbs = &cbs; + + rc = ecore_ll2_acquire_connection(dev->rdma_ctx, &data); + + if (rc) { + QL_DPRINT11(dev->ha, + "ecore_ll2_acquire_connection failed: %d\n", + rc); + return rc; + } + + QL_DPRINT11(dev->ha, + "ll2 connection acquired successfully\n"); + rc = ecore_ll2_establish_connection(dev->rdma_ctx, + dev->gsi_ll2_handle); + + if (rc) { + QL_DPRINT11(dev->ha, + "ecore_ll2_establish_connection failed\n", rc); + goto err1; + } + + QL_DPRINT11(dev->ha, + "ll2 connection established successfully\n"); + rc = qlnx_rdma_ll2_set_mac_filter(dev->rdma_ctx, NULL, + dev->ha->primary_mac); + if (rc) { + QL_DPRINT11(dev->ha, "qlnx_rdma_ll2_set_mac_filter failed\n", rc); + goto err2; + } + + QL_DPRINT12(dev->ha, "exit rc = %d\n", rc); + return 0; + +err2: + ecore_ll2_terminate_connection(dev->rdma_ctx, dev->gsi_ll2_handle); +err1: + ecore_ll2_release_connection(dev->rdma_ctx, dev->gsi_ll2_handle); + + QL_DPRINT12(dev->ha, "exit rc = %d\n", rc); + return rc; +} + +struct ib_qp* +qlnxr_create_gsi_qp(struct qlnxr_dev *dev, + struct ib_qp_init_attr *attrs, + struct qlnxr_qp *qp) +{ + int rc; + + QL_DPRINT12(dev->ha, "enter\n"); + + rc = qlnxr_check_gsi_qp_attrs(dev, attrs); + + if (rc) { + QL_DPRINT11(dev->ha, "qlnxr_check_gsi_qp_attrs failed\n"); + return ERR_PTR(rc); + } + + rc = qlnxr_ll2_start(dev, attrs, qp); + if (rc) { + QL_DPRINT11(dev->ha, "qlnxr_ll2_start failed\n"); + return ERR_PTR(rc); + } + + /* create QP */ + qp->ibqp.qp_num = 1; + qp->rq.max_wr = attrs->cap.max_recv_wr; + qp->sq.max_wr = attrs->cap.max_send_wr; + + qp->rqe_wr_id = kzalloc(qp->rq.max_wr * sizeof(*qp->rqe_wr_id), + GFP_KERNEL); + if (!qp->rqe_wr_id) { + QL_DPRINT11(dev->ha, "(!qp->rqe_wr_id)\n"); + goto err; + } + + qp->wqe_wr_id = kzalloc(qp->sq.max_wr * sizeof(*qp->wqe_wr_id), + GFP_KERNEL); + if (!qp->wqe_wr_id) { + QL_DPRINT11(dev->ha, "(!qp->wqe_wr_id)\n"); + goto err; + } + + qlnxr_store_gsi_qp_cq(dev, qp, attrs); + memcpy(dev->gsi_ll2_mac_address, dev->ha->primary_mac, ETH_ALEN); + + /* the GSI CQ is handled by the driver so remove it from the FW */ + qlnxr_destroy_gsi_cq(dev, attrs); + dev->gsi_rqcq->cq_type = QLNXR_CQ_TYPE_GSI; + dev->gsi_rqcq->cq_type = QLNXR_CQ_TYPE_GSI; + + QL_DPRINT12(dev->ha, "exit &qp->ibqp = %p\n", &qp->ibqp); + + return &qp->ibqp; +err: + kfree(qp->rqe_wr_id); + + rc = qlnxr_ll2_stop(dev); + + QL_DPRINT12(dev->ha, "exit with error\n"); + + return ERR_PTR(-ENOMEM); +} + +int +qlnxr_destroy_gsi_qp(struct qlnxr_dev *dev) +{ + int rc = 0; + + QL_DPRINT12(dev->ha, "enter\n"); + + rc = qlnxr_ll2_stop(dev); + + QL_DPRINT12(dev->ha, "exit rc = %d\n", rc); + return (rc); +} + + +static inline bool +qlnxr_get_vlan_id_gsi(struct ib_ah_attr *ah_attr, u16 *vlan_id) +{ + u16 tmp_vlan_id; + union ib_gid *dgid = &ah_attr->grh.dgid; + + tmp_vlan_id = (dgid->raw[11] << 8) | dgid->raw[12]; + if (tmp_vlan_id < 0x1000) { + *vlan_id = tmp_vlan_id; + return true; + } else { + *vlan_id = 0; + return false; + } +} + +#define QLNXR_MAX_UD_HEADER_SIZE (100) +#define QLNXR_GSI_QPN (1) +static inline int +qlnxr_gsi_build_header(struct qlnxr_dev *dev, + struct qlnxr_qp *qp, + struct ib_send_wr *swr, + struct ib_ud_header *udh, + int *roce_mode) +{ + bool has_vlan = false, has_grh_ipv6 = true; + struct ib_ah_attr *ah_attr = &get_qlnxr_ah((ud_wr(swr)->ah))->attr; + struct ib_global_route *grh = &ah_attr->grh; + union ib_gid sgid; + int send_size = 0; + u16 vlan_id = 0; + u16 ether_type; + +#if __FreeBSD_version >= 1102000 + int rc = 0; + int ip_ver = 0; + bool has_udp = false; +#endif /* #if __FreeBSD_version >= 1102000 */ + + +#if !DEFINE_IB_AH_ATTR_WITH_DMAC + u8 mac[ETH_ALEN]; +#endif + int i; + + send_size = 0; + for (i = 0; i < swr->num_sge; ++i) + send_size += swr->sg_list[i].length; + + has_vlan = qlnxr_get_vlan_id_gsi(ah_attr, &vlan_id); + ether_type = ETH_P_ROCE; + *roce_mode = ROCE_V1; + if (grh->sgid_index < QLNXR_MAX_SGID) + sgid = dev->sgid_tbl[grh->sgid_index]; + else + sgid = dev->sgid_tbl[0]; + +#if __FreeBSD_version >= 1102000 + + rc = ib_ud_header_init(send_size, false /* LRH */, true /* ETH */, + has_vlan, has_grh_ipv6, ip_ver, has_udp, + 0 /* immediate */, udh); + + if (rc) { + QL_DPRINT11(dev->ha, "gsi post send: failed to init header\n"); + return rc; + } + +#else + ib_ud_header_init(send_size, false /* LRH */, true /* ETH */, + has_vlan, has_grh_ipv6, 0 /* immediate */, udh); + +#endif /* #if __FreeBSD_version >= 1102000 */ + + /* ENET + VLAN headers*/ +#if DEFINE_IB_AH_ATTR_WITH_DMAC + memcpy(udh->eth.dmac_h, ah_attr->dmac, ETH_ALEN); +#else + qlnxr_get_dmac(dev, ah_attr, mac); + memcpy(udh->eth.dmac_h, mac, ETH_ALEN); +#endif + memcpy(udh->eth.smac_h, dev->ha->primary_mac, ETH_ALEN); + if (has_vlan) { + udh->eth.type = htons(ETH_P_8021Q); + udh->vlan.tag = htons(vlan_id); + udh->vlan.type = htons(ether_type); + } else { + udh->eth.type = htons(ether_type); + } + + for (int j = 0; j < 4; j++) { + QL_DPRINT12(dev->ha, "destination mac: %x\n", + udh->eth.dmac_h[j]); + } + for (int j = 0; j < 4; j++) { + QL_DPRINT12(dev->ha, "source mac: %x\n", + udh->eth.smac_h[j]); + } + + QL_DPRINT12(dev->ha, "QP: %p, opcode: %d, wq: %lx, roce: %x, hops:%d," + "imm : %d, vlan :%d, AH: %p\n", + qp, swr->opcode, swr->wr_id, *roce_mode, grh->hop_limit, + 0, has_vlan, get_qlnxr_ah((ud_wr(swr)->ah))); + + if (has_grh_ipv6) { + /* GRH / IPv6 header */ + udh->grh.traffic_class = grh->traffic_class; + udh->grh.flow_label = grh->flow_label; + udh->grh.hop_limit = grh->hop_limit; + udh->grh.destination_gid = grh->dgid; + memcpy(&udh->grh.source_gid.raw, &sgid.raw, + sizeof(udh->grh.source_gid.raw)); + QL_DPRINT12(dev->ha, "header: tc: %x, flow_label : %x, " + "hop_limit: %x \n", udh->grh.traffic_class, + udh->grh.flow_label, udh->grh.hop_limit); + for (i = 0; i < 16; i++) { + QL_DPRINT12(dev->ha, "udh dgid = %x\n", udh->grh.destination_gid.raw[i]); + } + for (i = 0; i < 16; i++) { + QL_DPRINT12(dev->ha, "udh sgid = %x\n", udh->grh.source_gid.raw[i]); + } + udh->grh.next_header = 0x1b; + } +#ifdef DEFINE_IB_UD_HEADER_INIT_UDP_PRESENT + /* This is for RoCEv2 */ + else { + /* IPv4 header */ + u32 ipv4_addr; + + udh->ip4.protocol = IPPROTO_UDP; + udh->ip4.tos = htonl(grh->flow_label); + udh->ip4.frag_off = htons(IP_DF); + udh->ip4.ttl = grh->hop_limit; + + ipv4_addr = qedr_get_ipv4_from_gid(sgid.raw); + udh->ip4.saddr = ipv4_addr; + ipv4_addr = qedr_get_ipv4_from_gid(grh->dgid.raw); + udh->ip4.daddr = ipv4_addr; + /* note: checksum is calculated by the device */ + } +#endif + + /* BTH */ + udh->bth.solicited_event = !!(swr->send_flags & IB_SEND_SOLICITED); + udh->bth.pkey = QLNXR_ROCE_PKEY_DEFAULT;/* TODO: ib_get_cahced_pkey?! */ + //udh->bth.destination_qpn = htonl(ud_wr(swr)->remote_qpn); + udh->bth.destination_qpn = OSAL_CPU_TO_BE32(ud_wr(swr)->remote_qpn); + //udh->bth.psn = htonl((qp->sq_psn++) & ((1 << 24) - 1)); + udh->bth.psn = OSAL_CPU_TO_BE32((qp->sq_psn++) & ((1 << 24) - 1)); + udh->bth.opcode = IB_OPCODE_UD_SEND_ONLY; + + /* DETH */ + //udh->deth.qkey = htonl(0x80010000); /* qp->qkey */ /* TODO: what is?! */ + //udh->deth.source_qpn = htonl(QLNXR_GSI_QPN); + udh->deth.qkey = OSAL_CPU_TO_BE32(0x80010000); /* qp->qkey */ /* TODO: what is?! */ + udh->deth.source_qpn = OSAL_CPU_TO_BE32(QLNXR_GSI_QPN); + QL_DPRINT12(dev->ha, "exit\n"); + return 0; +} + +static inline int +qlnxr_gsi_build_packet(struct qlnxr_dev *dev, + struct qlnxr_qp *qp, struct ib_send_wr *swr, + struct ecore_roce_ll2_packet **p_packet) +{ + u8 ud_header_buffer[QLNXR_MAX_UD_HEADER_SIZE]; + struct ecore_roce_ll2_packet *packet; + int roce_mode, header_size; + struct ib_ud_header udh; + int i, rc; + + QL_DPRINT12(dev->ha, "enter\n"); + + *p_packet = NULL; + + rc = qlnxr_gsi_build_header(dev, qp, swr, &udh, &roce_mode); + if (rc) { + QL_DPRINT11(dev->ha, + "qlnxr_gsi_build_header failed rc = %d\n", rc); + return rc; + } + + header_size = ib_ud_header_pack(&udh, &ud_header_buffer); + + packet = kzalloc(sizeof(*packet), GFP_ATOMIC); + if (!packet) { + QL_DPRINT11(dev->ha, "packet == NULL\n"); + return -ENOMEM; + } + + packet->header.vaddr = qlnx_dma_alloc_coherent(&dev->ha->cdev, + &packet->header.baddr, + header_size); + if (!packet->header.vaddr) { + QL_DPRINT11(dev->ha, "packet->header.vaddr == NULL\n"); + kfree(packet); + return -ENOMEM; + } + + if (memcmp(udh.eth.smac_h, udh.eth.dmac_h, ETH_ALEN)) + packet->tx_dest = ECORE_ROCE_LL2_TX_DEST_NW; + else + packet->tx_dest = ECORE_ROCE_LL2_TX_DEST_LB; + + packet->roce_mode = roce_mode; + memcpy(packet->header.vaddr, ud_header_buffer, header_size); + packet->header.len = header_size; + packet->n_seg = swr->num_sge; + qp->wqe_wr_id[qp->sq.prod].bytes_len = IB_GRH_BYTES; //RDMA_GRH_BYTES + for (i = 0; i < packet->n_seg; i++) { + packet->payload[i].baddr = swr->sg_list[i].addr; + packet->payload[i].len = swr->sg_list[i].length; + qp->wqe_wr_id[qp->sq.prod].bytes_len += + packet->payload[i].len; + QL_DPRINT11(dev->ha, "baddr: %p, len: %d\n", + packet->payload[i].baddr, + packet->payload[i].len); + } + + *p_packet = packet; + + QL_DPRINT12(dev->ha, "exit, packet->n_seg: %d\n", packet->n_seg); + return 0; +} + +int +qlnxr_gsi_post_send(struct ib_qp *ibqp, + struct ib_send_wr *wr, + struct ib_send_wr **bad_wr) +{ + struct ecore_roce_ll2_packet *pkt = NULL; + struct qlnxr_qp *qp = get_qlnxr_qp(ibqp); + struct qlnxr_dev *dev = qp->dev; + unsigned long flags; + int rc; + + QL_DPRINT12(dev->ha, "exit\n"); + + if (qp->state != ECORE_ROCE_QP_STATE_RTS) { + QL_DPRINT11(dev->ha, + "(qp->state != ECORE_ROCE_QP_STATE_RTS)\n"); + *bad_wr = wr; + return -EINVAL; + } + + if (wr->num_sge > RDMA_MAX_SGE_PER_SQ_WQE) { + QL_DPRINT11(dev->ha, + "(wr->num_sge > RDMA_MAX_SGE_PER_SQ_WQE)\n"); + rc = -EINVAL; + goto err; + } + + if (wr->opcode != IB_WR_SEND) { + QL_DPRINT11(dev->ha, "(wr->opcode > IB_WR_SEND)\n"); + rc = -EINVAL; + goto err; + } + + spin_lock_irqsave(&qp->q_lock, flags); + + rc = qlnxr_gsi_build_packet(dev, qp, wr, &pkt); + if(rc) { + spin_unlock_irqrestore(&qp->q_lock, flags); + QL_DPRINT11(dev->ha, "qlnxr_gsi_build_packet failed\n"); + goto err; + } + + rc = qlnxr_ll2_post_tx(dev, pkt); + + if (!rc) { + qp->wqe_wr_id[qp->sq.prod].wr_id = wr->wr_id; + qp->wqe_wr_id[qp->sq.prod].signaled = + !!(wr->send_flags & IB_SEND_SIGNALED); + qp->wqe_wr_id[qp->sq.prod].opcode = IB_WC_SEND; + qlnxr_inc_sw_prod(&qp->sq); + QL_DPRINT11(dev->ha, "packet sent over gsi qp\n"); + } else { + QL_DPRINT11(dev->ha, "qlnxr_ll2_post_tx failed\n"); + rc = -EAGAIN; + *bad_wr = wr; + } + + spin_unlock_irqrestore(&qp->q_lock, flags); + + if (wr->next != NULL) { + *bad_wr = wr->next; + rc=-EINVAL; + } + + QL_DPRINT12(dev->ha, "exit\n"); + return rc; + +err: + *bad_wr = wr; + QL_DPRINT12(dev->ha, "exit error\n"); + return rc; +} + +#define QLNXR_LL2_RX_BUFFER_SIZE (4 * 1024) +int +qlnxr_gsi_post_recv(struct ib_qp *ibqp, + struct ib_recv_wr *wr, + struct ib_recv_wr **bad_wr) +{ + struct qlnxr_dev *dev = get_qlnxr_dev((ibqp->device)); + struct qlnxr_qp *qp = get_qlnxr_qp(ibqp); + unsigned long flags; + int rc = 0; + + QL_DPRINT12(dev->ha, "enter, wr: %p\n", wr); + + if ((qp->state != ECORE_ROCE_QP_STATE_RTR) && + (qp->state != ECORE_ROCE_QP_STATE_RTS)) { + *bad_wr = wr; + QL_DPRINT11(dev->ha, "exit 0\n"); + return -EINVAL; + } + + spin_lock_irqsave(&qp->q_lock, flags); + + while (wr) { + if (wr->num_sge > QLNXR_GSI_MAX_RECV_SGE) { + QL_DPRINT11(dev->ha, "exit 1\n"); + goto err; + } + + rc = ecore_ll2_post_rx_buffer(dev->rdma_ctx, + dev->gsi_ll2_handle, + wr->sg_list[0].addr, + wr->sg_list[0].length, + 0 /* cookie */, + 1 /* notify_fw */); + if (rc) { + QL_DPRINT11(dev->ha, "exit 2\n"); + goto err; + } + + memset(&qp->rqe_wr_id[qp->rq.prod], 0, + sizeof(qp->rqe_wr_id[qp->rq.prod])); + qp->rqe_wr_id[qp->rq.prod].sg_list[0] = wr->sg_list[0]; + qp->rqe_wr_id[qp->rq.prod].wr_id = wr->wr_id; + + qlnxr_inc_sw_prod(&qp->rq); + + wr = wr->next; + } + + spin_unlock_irqrestore(&qp->q_lock, flags); + + QL_DPRINT12(dev->ha, "exit rc = %d\n", rc); + return rc; +err: + + spin_unlock_irqrestore(&qp->q_lock, flags); + *bad_wr = wr; + + QL_DPRINT12(dev->ha, "exit with -ENOMEM\n"); + return -ENOMEM; +} + +int +qlnxr_gsi_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc) +{ + struct qlnxr_dev *dev = get_qlnxr_dev((ibcq->device)); + struct qlnxr_cq *cq = get_qlnxr_cq(ibcq); + struct qlnxr_qp *qp = dev->gsi_qp; + unsigned long flags; + int i = 0; + + QL_DPRINT12(dev->ha, "enter\n"); + + spin_lock_irqsave(&cq->cq_lock, flags); + + while (i < num_entries && qp->rq.cons != qp->rq.gsi_cons) { + memset(&wc[i], 0, sizeof(*wc)); + + wc[i].qp = &qp->ibqp; + wc[i].wr_id = qp->rqe_wr_id[qp->rq.cons].wr_id; + wc[i].opcode = IB_WC_RECV; + wc[i].pkey_index = 0; + wc[i].status = (qp->rqe_wr_id[qp->rq.cons].rc)? + IB_WC_GENERAL_ERR:IB_WC_SUCCESS; + /* 0 - currently only one recv sg is supported */ + wc[i].byte_len = qp->rqe_wr_id[qp->rq.cons].sg_list[0].length; + wc[i].wc_flags |= IB_WC_GRH | IB_WC_IP_CSUM_OK; + +#if __FreeBSD_version >= 1100000 + memcpy(&wc[i].smac, qp->rqe_wr_id[qp->rq.cons].smac, ETH_ALEN); + wc[i].wc_flags |= IB_WC_WITH_SMAC; + + if (qp->rqe_wr_id[qp->rq.cons].vlan_id) { + wc[i].wc_flags |= IB_WC_WITH_VLAN; + wc[i].vlan_id = qp->rqe_wr_id[qp->rq.cons].vlan_id; + } + +#endif + qlnxr_inc_sw_cons(&qp->rq); + i++; + } + + while (i < num_entries && qp->sq.cons != qp->sq.gsi_cons) { + memset(&wc[i], 0, sizeof(*wc)); + + wc[i].qp = &qp->ibqp; + wc[i].wr_id = qp->wqe_wr_id[qp->sq.cons].wr_id; + wc[i].opcode = IB_WC_SEND; + wc[i].status = IB_WC_SUCCESS; + + qlnxr_inc_sw_cons(&qp->sq); + i++; + } + + spin_unlock_irqrestore(&cq->cq_lock, flags); + + QL_DPRINT12(dev->ha, "exit i = %d\n", i); + return i; +} + Property changes on: head/sys/dev/qlnx/qlnxr/qlnxr_cm.c ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Added: svn:mime-type ## -0,0 +1 ## +text/plain \ No newline at end of property Index: head/sys/dev/qlnx/qlnxr/qlnxr_cm.h =================================================================== --- head/sys/dev/qlnx/qlnxr/qlnxr_cm.h (nonexistent) +++ head/sys/dev/qlnx/qlnxr/qlnxr_cm.h (revision 343598) @@ -0,0 +1,112 @@ +/* + * Copyright (c) 2018-2019 Cavium, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * + * $FreeBSD$ + */ + + + +#ifndef __QLNXR_CM_H__ +#define __QLNXR_CM_H__ + + +/* ECORE LL2 has a limit to the number of buffers it can handle. + * FYI, OFED used 512 and 128 for recv and send. + */ +#define QLNXR_GSI_MAX_RECV_WR (4096) +#define QLNXR_GSI_MAX_SEND_WR (4096) + +#define QLNXR_GSI_MAX_RECV_SGE (1) /* LL2 FW limitation */ + +/* future OFED/kernel will have these */ +#define ETH_P_ROCE (0x8915) +#define QLNXR_ROCE_V2_UDP_SPORT (0000) + +#if __FreeBSD_version >= 1102000 + +#define rdma_wr(_wr) rdma_wr(_wr) +#define ud_wr(_wr) ud_wr(_wr) +#define atomic_wr(_wr) atomic_wr(_wr) + +#else + +#define rdma_wr(_wr) (&(_wr->wr.rdma)) +#define ud_wr(_wr) (&(_wr->wr.ud)) +#define atomic_wr(_wr) (&(_wr->wr.atomic)) + +#endif /* #if __FreeBSD_version >= 1102000 */ + +static inline u32 qlnxr_get_ipv4_from_gid(u8 *gid) +{ + return *(u32 *)(void *)&gid[12]; +} + +struct ecore_roce_ll2_header { + void *vaddr; + dma_addr_t baddr; + size_t len; +}; + +struct ecore_roce_ll2_buffer { + dma_addr_t baddr; + size_t len; +}; + +struct ecore_roce_ll2_packet { + struct ecore_roce_ll2_header header; + int n_seg; + struct ecore_roce_ll2_buffer payload[RDMA_MAX_SGE_PER_SQ_WQE]; + int roce_mode; + enum ecore_roce_ll2_tx_dest tx_dest; +}; + +/* RDMA CM */ + +extern int qlnxr_gsi_poll_cq(struct ib_cq *ibcq, + int num_entries, + struct ib_wc *wc); + +extern int qlnxr_gsi_post_recv(struct ib_qp *ibqp, + struct ib_recv_wr *wr, + struct ib_recv_wr **bad_wr); + +extern int qlnxr_gsi_post_send(struct ib_qp *ibqp, + struct ib_send_wr *wr, + struct ib_send_wr **bad_wr); + +extern struct ib_qp* qlnxr_create_gsi_qp(struct qlnxr_dev *dev, + struct ib_qp_init_attr *attrs, + struct qlnxr_qp *qp); + +extern void qlnxr_store_gsi_qp_cq(struct qlnxr_dev *dev, + struct qlnxr_qp *qp, + struct ib_qp_init_attr *attrs); + +extern void qlnxr_inc_sw_gsi_cons(struct qlnxr_qp_hwq_info *info); + +extern int qlnxr_destroy_gsi_qp(struct qlnxr_dev *dev); + +#endif /* #ifndef __QLNXR_CM_H__ */ Property changes on: head/sys/dev/qlnx/qlnxr/qlnxr_cm.h ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Added: svn:mime-type ## -0,0 +1 ## +text/plain \ No newline at end of property Index: head/sys/dev/qlnx/qlnxr/qlnxr_def.h =================================================================== --- head/sys/dev/qlnx/qlnxr/qlnxr_def.h (nonexistent) +++ head/sys/dev/qlnx/qlnxr/qlnxr_def.h (revision 343598) @@ -0,0 +1,924 @@ +/* + * Copyright (c) 2018-2019 Cavium, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * + * $FreeBSD$ + */ + + +/* + * File: qlnxr_def.h + * Author: David C Somayajulu + */ + +#ifndef __QLNX_DEF_H_ +#define __QLNX_DEF_H_ + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#if __FreeBSD_version < 1100000 +#undef MODULE_VERSION +#endif + +#include "qlnx_os.h" +#include "bcm_osal.h" + +#include "reg_addr.h" +#include "ecore_gtt_reg_addr.h" +#include "ecore.h" +#include "ecore_chain.h" +#include "ecore_status.h" +#include "ecore_hw.h" +#include "ecore_rt_defs.h" +#include "ecore_init_ops.h" +#include "ecore_int.h" +#include "ecore_cxt.h" +#include "ecore_spq.h" +#include "ecore_init_fw_funcs.h" +#include "ecore_sp_commands.h" +#include "ecore_dev_api.h" +#include "ecore_l2_api.h" +#ifdef CONFIG_ECORE_SRIOV +#include "ecore_sriov.h" +#include "ecore_vf.h" +#endif +#ifdef CONFIG_ECORE_LL2 +#include "ecore_ll2.h" +#endif +#ifdef CONFIG_ECORE_FCOE +#include "ecore_fcoe.h" +#endif +#ifdef CONFIG_ECORE_ISCSI +#include "ecore_iscsi.h" +#endif +#include "ecore_mcp.h" +#include "ecore_hw_defs.h" +#include "mcp_public.h" + +#ifdef CONFIG_ECORE_RDMA +#include "ecore_rdma.h" +#include "ecore_rdma_api.h" +#endif + +#ifdef CONFIG_ECORE_ROCE +#include "ecore_roce.h" +#endif + +#ifdef CONFIG_ECORE_IWARP +#include "ecore_iwarp.h" +#endif + +#include "ecore_iro.h" +#include "nvm_cfg.h" + +#include "ecore_dbg_fw_funcs.h" +#include "rdma_common.h" + +#include "qlnx_ioctl.h" +#include "qlnx_def.h" +#include "qlnx_rdma.h" +#include "qlnxr_verbs.h" +#include "qlnxr_user.h" +#include "qlnx_ver.h" +#include + +#define QLNXR_ROCE_INTERFACE_VERSION 1801 + +#define QLNXR_MODULE_VERSION "8.18.1.0" +#define QLNXR_NODE_DESC "QLogic 579xx RoCE HCA" + +#define OC_SKH_DEVICE_PF 0x720 +#define OC_SKH_DEVICE_VF 0x728 +#define QLNXR_MAX_AH 512 + +/* QLNXR Limitations */ + +/* SQ/RQ Limitations + * An S/RQ PBL contains a list a pointers to pages. Each page contains S/RQE + * elements. Several S/RQE elements make an S/RQE, up to a certain maximum that + * is different between SQ and RQ. The size of the PBL was chosen such as not to + * limit the MAX_WR supported by ECORE, and rounded up to a power of two. + */ +/* SQ */ +#define QLNXR_MAX_SQ_PBL (0x8000) /* 2^15 bytes */ +#define QLNXR_MAX_SQ_PBL_ENTRIES (0x10000 / sizeof(void *)) /* number */ +#define QLNXR_SQE_ELEMENT_SIZE (sizeof(struct rdma_sq_sge)) /* bytes */ +#define QLNXR_MAX_SQE_ELEMENTS_PER_SQE (ROCE_REQ_MAX_SINGLE_SQ_WQE_SIZE / \ + QLNXR_SQE_ELEMENT_SIZE) /* number */ +#define QLNXR_MAX_SQE_ELEMENTS_PER_PAGE ((RDMA_RING_PAGE_SIZE) / \ + QLNXR_SQE_ELEMENT_SIZE) /* number */ +#define QLNXR_MAX_SQE ((QLNXR_MAX_SQ_PBL_ENTRIES) * (RDMA_RING_PAGE_SIZE) / \ + (QLNXR_SQE_ELEMENT_SIZE) / (QLNXR_MAX_SQE_ELEMENTS_PER_SQE)) +/* RQ */ +#define QLNXR_MAX_RQ_PBL (0x2000) /* 2^13 bytes */ +#define QLNXR_MAX_RQ_PBL_ENTRIES (0x10000 / sizeof(void *)) /* number */ +#define QLNXR_RQE_ELEMENT_SIZE (sizeof(struct rdma_rq_sge)) /* bytes */ +#define QLNXR_MAX_RQE_ELEMENTS_PER_RQE (RDMA_MAX_SGE_PER_RQ_WQE) /* number */ +#define QLNXR_MAX_RQE_ELEMENTS_PER_PAGE ((RDMA_RING_PAGE_SIZE) / \ + QLNXR_RQE_ELEMENT_SIZE) /* number */ +#define QLNXR_MAX_RQE ((QLNXR_MAX_RQ_PBL_ENTRIES) * (RDMA_RING_PAGE_SIZE) / \ + (QLNXR_RQE_ELEMENT_SIZE) / (QLNXR_MAX_RQE_ELEMENTS_PER_RQE)) + +/* CQE Limitation + * Although FW supports two layer PBL we use single layer since it is more + * than enough. For that layer we use a maximum size of 512 kB, again, because + * it reaches the maximum number of page pointers. Notice is the '-1' in the + * calculation that comes from having a u16 for the number of pages i.e. 0xffff + * is the maximum number of pages (in single layer). + */ +#define QLNXR_CQE_SIZE (sizeof(union rdma_cqe)) +#define QLNXR_MAX_CQE_PBL_SIZE (512*1024) /* 512kB */ +#define QLNXR_MAX_CQE_PBL_ENTRIES (((QLNXR_MAX_CQE_PBL_SIZE) / \ + sizeof(u64)) - 1) /* 64k -1 */ +#define QLNXR_MAX_CQES ((u32)((QLNXR_MAX_CQE_PBL_ENTRIES) * (ECORE_CHAIN_PAGE_SIZE)\ + / QLNXR_CQE_SIZE)) /* 8M -4096/32 = 8,388,480 */ + +/* CNQ size Limitation + * The maximum CNQ size is not reachable because the FW supports a chain of u16 + * (specifically 64k-1). The FW can buffer CNQ elements avoiding an overflow, on + * the expense of performance. Hence we set it to an arbitrarily smaller value + * than the maximum. + */ +#define QLNXR_ROCE_MAX_CNQ_SIZE (0x4000) /* 2^16 */ + +#define QLNXR_MAX_PORT (1) +#define QLNXR_PORT (1) + +#define QLNXR_UVERBS(CMD_NAME) (1ull << IB_USER_VERBS_CMD_##CMD_NAME) + +#define convert_to_64bit(lo, hi) ((u64)hi << 32 | (u64)lo) + +/* The following number is used to determine if a handle recevied from the FW + * actually point to a CQ/QP. + */ +#define QLNXR_CQ_MAGIC_NUMBER (0x11223344) +#define QLNXR_QP_MAGIC_NUMBER (0x77889900) + +/* Fast path debug prints */ +#define FP_DP_VERBOSE(...) +/* #define FP_DP_VERBOSE(...) DP_VERBOSE(__VA_ARGS__) */ + +#define FW_PAGE_SIZE (RDMA_RING_PAGE_SIZE) + +#define QLNXR_MSG_INIT 0x10000, +#define QLNXR_MSG_FAIL 0x10000, +#define QLNXR_MSG_CQ 0x20000, +#define QLNXR_MSG_RQ 0x40000, +#define QLNXR_MSG_SQ 0x80000, +#define QLNXR_MSG_QP (QLNXR_MSG_SQ | QLNXR_MSG_RQ), +#define QLNXR_MSG_MR 0x100000, +#define QLNXR_MSG_GSI 0x200000, +#define QLNXR_MSG_MISC 0x400000, +#define QLNXR_MSG_SRQ 0x800000, +#define QLNXR_MSG_IWARP 0x1000000, + +#define QLNXR_ROCE_PKEY_MAX 1 +#define QLNXR_ROCE_PKEY_TABLE_LEN 1 +#define QLNXR_ROCE_PKEY_DEFAULT 0xffff + +#define QLNXR_MAX_SGID 128 /* TBD - add more source gids... */ + +#define QLNXR_ENET_STATE_BIT (0) + +#define QLNXR_MAX_MSIX (16) + + +struct qlnxr_cnq { + struct qlnxr_dev *dev; + struct ecore_chain pbl; + struct ecore_sb_info *sb; + char name[32]; + u64 n_comp; + __le16 *hw_cons_ptr; + u8 index; + int irq_rid; + struct resource *irq; + void *irq_handle; +}; + +struct qlnxr_device_attr { + /* Vendor specific information */ + u32 vendor_id; + u32 vendor_part_id; + u32 hw_ver; + u64 fw_ver; + + u64 node_guid; /* node GUID */ + u64 sys_image_guid; /* System image GUID */ + + u8 max_cnq; + u8 max_sge; /* Maximum # of scatter/gather entries + * per Work Request supported + */ + u16 max_inline; + u32 max_sqe; /* Maximum number of send outstanding send work + * requests on any Work Queue supported + */ + u32 max_rqe; /* Maximum number of receive outstanding receive + * work requests on any Work Queue supported + */ + u8 max_qp_resp_rd_atomic_resc; /* Maximum number of RDMA Reads + * & atomic operation that can + * be outstanding per QP + */ + + u8 max_qp_req_rd_atomic_resc; /* The maximum depth per QP for + * initiation of RDMA Read + * & atomic operations + */ + u64 max_dev_resp_rd_atomic_resc; + u32 max_cq; + u32 max_qp; + u32 max_mr; /* Maximum # of MRs supported */ + u64 max_mr_size; /* Size (in bytes) of largest contiguous memory + * block that can be registered by this device + */ + u32 max_cqe; + u32 max_mw; /* Maximum # of memory windows supported */ + u32 max_fmr; + u32 max_mr_mw_fmr_pbl; + u64 max_mr_mw_fmr_size; + u32 max_pd; /* Maximum # of protection domains supported */ + u32 max_ah; + u8 max_pkey; + u32 max_srq; /* Maximum number of SRQs */ + u32 max_srq_wr; /* Maximum number of WRs per SRQ */ + u8 max_srq_sge; /* Maximum number of SGE per WQE */ + u8 max_stats_queues; /* Maximum number of statistics queues */ + u32 dev_caps; + + /* Abilty to support RNR-NAK generation */ + +#define QLNXR_ROCE_DEV_CAP_RNR_NAK_MASK 0x1 +#define QLNXR_ROCE_DEV_CAP_RNR_NAK_SHIFT 0 + /* Abilty to support shutdown port */ +#define QLNXR_ROCE_DEV_CAP_SHUTDOWN_PORT_MASK 0x1 +#define QLNXR_ROCE_DEV_CAP_SHUTDOWN_PORT_SHIFT 1 + /* Abilty to support port active event */ +#define QLNXR_ROCE_DEV_CAP_PORT_ACTIVE_EVENT_MASK 0x1 +#define QLNXR_ROCE_DEV_CAP_PORT_ACTIVE_EVENT_SHIFT 2 + /* Abilty to support port change event */ +#define QLNXR_ROCE_DEV_CAP_PORT_CHANGE_EVENT_MASK 0x1 +#define QLNXR_ROCE_DEV_CAP_PORT_CHANGE_EVENT_SHIFT 3 + /* Abilty to support system image GUID */ +#define QLNXR_ROCE_DEV_CAP_SYS_IMAGE_MASK 0x1 +#define QLNXR_ROCE_DEV_CAP_SYS_IMAGE_SHIFT 4 + /* Abilty to support bad P_Key counter support */ +#define QLNXR_ROCE_DEV_CAP_BAD_PKEY_CNT_MASK 0x1 +#define QLNXR_ROCE_DEV_CAP_BAD_PKEY_CNT_SHIFT 5 + /* Abilty to support atomic operations */ +#define QLNXR_ROCE_DEV_CAP_ATOMIC_OP_MASK 0x1 +#define QLNXR_ROCE_DEV_CAP_ATOMIC_OP_SHIFT 6 +#define QLNXR_ROCE_DEV_CAP_RESIZE_CQ_MASK 0x1 +#define QLNXR_ROCE_DEV_CAP_RESIZE_CQ_SHIFT 7 + /* Abilty to support modifying the maximum number of + * outstanding work requests per QP + */ +#define QLNXR_ROCE_DEV_CAP_RESIZE_MAX_WR_MASK 0x1 +#define QLNXR_ROCE_DEV_CAP_RESIZE_MAX_WR_SHIFT 8 + + /* Abilty to support automatic path migration */ +#define QLNXR_ROCE_DEV_CAP_AUTO_PATH_MIG_MASK 0x1 +#define QLNXR_ROCE_DEV_CAP_AUTO_PATH_MIG_SHIFT 9 + /* Abilty to support the base memory management extensions */ +#define QLNXR_ROCE_DEV_CAP_BASE_MEMORY_EXT_MASK 0x1 +#define QLNXR_ROCE_DEV_CAP_BASE_MEMORY_EXT_SHIFT 10 +#define QLNXR_ROCE_DEV_CAP_BASE_QUEUE_EXT_MASK 0x1 +#define QLNXR_ROCE_DEV_CAP_BASE_QUEUE_EXT_SHIFT 11 + /* Abilty to support multipile page sizes per memory region */ +#define QLNXR_ROCE_DEV_CAP_MULTI_PAGE_PER_MR_EXT_MASK 0x1 +#define QLNXR_ROCE_DEV_CAP_MULTI_PAGE_PER_MR_EXT_SHIFT 12 + /* Abilty to support block list physical buffer list */ +#define QLNXR_ROCE_DEV_CAP_BLOCK_MODE_MASK 0x1 +#define QLNXR_ROCE_DEV_CAP_BLOCK_MODE_SHIFT 13 + /* Abilty to support zero based virtual addresses */ +#define QLNXR_ROCE_DEV_CAP_ZBVA_MASK 0x1 +#define QLNXR_ROCE_DEV_CAP_ZBVA_SHIFT 14 + /* Abilty to support local invalidate fencing */ +#define QLNXR_ROCE_DEV_CAP_LOCAL_INV_FENCE_MASK 0x1 +#define QLNXR_ROCE_DEV_CAP_LOCAL_INV_FENCE_SHIFT 15 + /* Abilty to support Loopback on QP */ +#define QLNXR_ROCE_DEV_CAP_LB_INDICATOR_MASK 0x1 +#define QLNXR_ROCE_DEV_CAP_LB_INDICATOR_SHIFT 16 + u64 page_size_caps; + u8 dev_ack_delay; + u32 reserved_lkey; /* Value of reserved L_key */ + u32 bad_pkey_counter;/* Bad P_key counter support + * indicator + */ + struct ecore_rdma_events events; +}; + +struct qlnxr_dev { + struct ib_device ibdev; + qlnx_host_t *ha; + struct ecore_dev *cdev; + + /* Added to extend Applications Support */ + struct pci_dev *pdev; + uint32_t dp_module; + uint8_t dp_level; + + void *rdma_ctx; + + struct mtx idr_lock; + struct idr qpidr; + + uint32_t wq_multiplier; + int num_cnq; + + struct ecore_sb_info sb_array[QLNXR_MAX_MSIX]; + struct qlnxr_cnq cnq_array[QLNXR_MAX_MSIX]; + + int sb_start; + + int gsi_qp_created; + struct qlnxr_cq *gsi_sqcq; + struct qlnxr_cq *gsi_rqcq; + struct qlnxr_qp *gsi_qp; + + /* TBD: we'll need an array of these probablly per DPI... */ + void __iomem *db_addr; + uint64_t db_phys_addr; + uint32_t db_size; + uint16_t dpi; + + uint64_t guid; + enum ib_atomic_cap atomic_cap; + + union ib_gid sgid_tbl[QLNXR_MAX_SGID]; + struct mtx sgid_lock; + struct notifier_block nb_inet; + struct notifier_block nb_inet6; + + uint8_t mr_key; + struct list_head entry; + + struct dentry *dbgfs; + + uint8_t gsi_ll2_mac_address[ETH_ALEN]; + uint8_t gsi_ll2_handle; + + unsigned long enet_state; + + struct workqueue_struct *iwarp_wq; + + volatile uint32_t pd_count; + struct qlnxr_device_attr attr; + uint8_t user_dpm_enabled; +}; + +typedef struct qlnxr_dev qlnxr_dev_t; + + +struct qlnxr_pd { + struct ib_pd ibpd; + u32 pd_id; + struct qlnxr_ucontext *uctx; +}; + +struct qlnxr_ucontext { + struct ib_ucontext ibucontext; + struct qlnxr_dev *dev; + struct qlnxr_pd *pd; + u64 dpi_addr; + u64 dpi_phys_addr; + u32 dpi_size; + u16 dpi; + + struct list_head mm_head; + struct mutex mm_list_lock; +}; + + + +struct qlnxr_dev_attr { + struct ib_device_attr ib_attr; +}; + +struct qlnxr_dma_mem { + void *va; + dma_addr_t pa; + u32 size; +}; + +struct qlnxr_pbl { + struct list_head list_entry; + void *va; + dma_addr_t pa; +}; + +struct qlnxr_queue_info { + void *va; + dma_addr_t dma; + u32 size; + u16 len; + u16 entry_size; /* Size of an element in the queue */ + u16 id; /* qid, where to ring the doorbell. */ + u16 head, tail; + bool created; +}; + +struct qlnxr_eq { + struct qlnxr_queue_info q; + u32 vector; + int cq_cnt; + struct qlnxr_dev *dev; + char irq_name[32]; +}; + +struct qlnxr_mq { + struct qlnxr_queue_info sq; + struct qlnxr_queue_info cq; + bool rearm_cq; +}; + +struct phy_info { + u16 auto_speeds_supported; + u16 fixed_speeds_supported; + u16 phy_type; + u16 interface_type; +}; + +union db_prod64 { + struct rdma_pwm_val32_data data; + u64 raw; +}; + +enum qlnxr_cq_type { + QLNXR_CQ_TYPE_GSI, + QLNXR_CQ_TYPE_KERNEL, + QLNXR_CQ_TYPE_USER +}; + +struct qlnxr_pbl_info { + u32 num_pbls; + u32 num_pbes; + u32 pbl_size; + u32 pbe_size; + bool two_layered; +}; + +struct qlnxr_userq { + struct ib_umem *umem; + struct qlnxr_pbl_info pbl_info; + struct qlnxr_pbl *pbl_tbl; + u64 buf_addr; + size_t buf_len; +}; + +struct qlnxr_cq { + struct ib_cq ibcq; /* must be first */ + + enum qlnxr_cq_type cq_type; + uint32_t sig; + uint16_t icid; + + /* relevant to cqs created from kernel space only (ULPs) */ + spinlock_t cq_lock; + uint8_t arm_flags; + struct ecore_chain pbl; + + void __iomem *db_addr; /* db address for cons update*/ + union db_prod64 db; + + uint8_t pbl_toggle; + union rdma_cqe *latest_cqe; + union rdma_cqe *toggle_cqe; + + /* TODO: remove since it is redundant with 32 bit chains */ + uint32_t cq_cons; + + /* relevant to cqs created from user space only (applications) */ + struct qlnxr_userq q; + + /* destroy-IRQ handler race prevention */ + uint8_t destroyed; + uint16_t cnq_notif; +}; + + +struct qlnxr_ah { + struct ib_ah ibah; + struct ib_ah_attr attr; +}; + +union db_prod32 { + struct rdma_pwm_val16_data data; + u32 raw; +}; + +struct qlnxr_qp_hwq_info { + /* WQE Elements*/ + struct ecore_chain pbl; + u64 p_phys_addr_tbl; + u32 max_sges; + + /* WQE */ + u16 prod; /* WQE prod index for SW ring */ + u16 cons; /* WQE cons index for SW ring */ + u16 wqe_cons; + u16 gsi_cons; /* filled in by GSI implementation */ + u16 max_wr; + + /* DB */ + void __iomem *db; /* Doorbell address */ + union db_prod32 db_data; /* Doorbell data */ + + /* Required for iwarp_only */ + void __iomem *iwarp_db2; /* Doorbell address */ + union db_prod32 iwarp_db2_data; /* Doorbell data */ +}; + +#define QLNXR_INC_SW_IDX(p_info, index) \ + do { \ + p_info->index = (p_info->index + 1) & \ + ecore_chain_get_capacity(p_info->pbl) \ + } while (0) + +struct qlnxr_srq_hwq_info { + u32 max_sges; + u32 max_wr; + struct ecore_chain pbl; + u64 p_phys_addr_tbl; + u32 wqe_prod; /* WQE prod index in HW ring */ + u32 sge_prod; /* SGE prod index in HW ring */ + u32 wr_prod_cnt; /* wr producer count */ + u32 wr_cons_cnt; /* wr consumer count */ + u32 num_elems; + + u32 *virt_prod_pair_addr; /* producer pair virtual address */ + dma_addr_t phy_prod_pair_addr; /* producer pair physical address */ +}; + +struct qlnxr_srq { + struct ib_srq ibsrq; + struct qlnxr_dev *dev; + /* relevant to cqs created from user space only (applications) */ + struct qlnxr_userq usrq; + struct qlnxr_srq_hwq_info hw_srq; + struct ib_umem *prod_umem; + u16 srq_id; + /* lock to protect srq recv post */ + spinlock_t lock; +}; + +enum qlnxr_qp_err_bitmap { + QLNXR_QP_ERR_SQ_FULL = 1 << 0, + QLNXR_QP_ERR_RQ_FULL = 1 << 1, + QLNXR_QP_ERR_BAD_SR = 1 << 2, + QLNXR_QP_ERR_BAD_RR = 1 << 3, + QLNXR_QP_ERR_SQ_PBL_FULL = 1 << 4, + QLNXR_QP_ERR_RQ_PBL_FULL = 1 << 5, +}; + +struct mr_info { + struct qlnxr_pbl *pbl_table; + struct qlnxr_pbl_info pbl_info; + struct list_head free_pbl_list; + struct list_head inuse_pbl_list; + u32 completed; + u32 completed_handled; +}; + +#if __FreeBSD_version < 1102000 +#define DEFINE_IB_FAST_REG +#else +#define DEFINE_ALLOC_MR +#endif + +#ifdef DEFINE_IB_FAST_REG +struct qlnxr_fast_reg_page_list { + struct ib_fast_reg_page_list ibfrpl; + struct qlnxr_dev *dev; + struct mr_info info; +}; +#endif +struct qlnxr_qp { + struct ib_qp ibqp; /* must be first */ + struct qlnxr_dev *dev; + struct qlnxr_iw_ep *ep; + struct qlnxr_qp_hwq_info sq; + struct qlnxr_qp_hwq_info rq; + + u32 max_inline_data; + +#if __FreeBSD_version >= 1100000 + spinlock_t q_lock ____cacheline_aligned; +#else + spinlock_t q_lock; +#endif + + struct qlnxr_cq *sq_cq; + struct qlnxr_cq *rq_cq; + struct qlnxr_srq *srq; + enum ecore_roce_qp_state state; /* QP state */ + u32 id; + struct qlnxr_pd *pd; + enum ib_qp_type qp_type; + struct ecore_rdma_qp *ecore_qp; + u32 qp_id; + u16 icid; + u16 mtu; + int sgid_idx; + u32 rq_psn; + u32 sq_psn; + u32 qkey; + u32 dest_qp_num; + u32 sig; /* unique siganture to identify valid QP */ + + /* relevant to qps created from kernel space only (ULPs) */ + u8 prev_wqe_size; + u16 wqe_cons; + u32 err_bitmap; + bool signaled; + /* SQ shadow */ + struct { + u64 wr_id; + enum ib_wc_opcode opcode; + u32 bytes_len; + u8 wqe_size; + bool signaled; + dma_addr_t icrc_mapping; + u32 *icrc; +#ifdef DEFINE_IB_FAST_REG + struct qlnxr_fast_reg_page_list *frmr; +#endif + struct qlnxr_mr *mr; + } *wqe_wr_id; + + /* RQ shadow */ + struct { + u64 wr_id; + struct ib_sge sg_list[RDMA_MAX_SGE_PER_RQ_WQE]; + uint8_t wqe_size; + + /* for GSI only */ + u8 smac[ETH_ALEN]; + u16 vlan_id; + int rc; + } *rqe_wr_id; + + /* relevant to qps created from user space only (applications) */ + struct qlnxr_userq usq; + struct qlnxr_userq urq; + atomic_t refcnt; + bool destroyed; +}; + +enum qlnxr_mr_type { + QLNXR_MR_USER, + QLNXR_MR_KERNEL, + QLNXR_MR_DMA, + QLNXR_MR_FRMR +}; + + +struct qlnxr_mr { + struct ib_mr ibmr; + struct ib_umem *umem; + + struct ecore_rdma_register_tid_in_params hw_mr; + enum qlnxr_mr_type type; + + struct qlnxr_dev *dev; + struct mr_info info; + + u64 *pages; + u32 npages; + + u64 *iova_start; /* valid only for kernel_mr */ +}; + + +struct qlnxr_mm { + struct { + u64 phy_addr; + unsigned long len; + } key; + struct list_head entry; +}; + +struct qlnxr_iw_listener { + struct qlnxr_dev *dev; + struct iw_cm_id *cm_id; + int backlog; + void *ecore_handle; +}; + +struct qlnxr_iw_ep { + struct qlnxr_dev *dev; + struct iw_cm_id *cm_id; + struct qlnxr_qp *qp; + void *ecore_context; + u8 during_connect; +}; + +static inline void +qlnxr_inc_sw_cons(struct qlnxr_qp_hwq_info *info) +{ + info->cons = (info->cons + 1) % info->max_wr; + info->wqe_cons++; +} + +static inline void +qlnxr_inc_sw_prod(struct qlnxr_qp_hwq_info *info) +{ + info->prod = (info->prod + 1) % info->max_wr; +} + +static inline struct qlnxr_dev * +get_qlnxr_dev(struct ib_device *ibdev) +{ + return container_of(ibdev, struct qlnxr_dev, ibdev); +} + +static inline struct qlnxr_ucontext * +get_qlnxr_ucontext(struct ib_ucontext *ibucontext) +{ + return container_of(ibucontext, struct qlnxr_ucontext, ibucontext); +} + +static inline struct qlnxr_pd * +get_qlnxr_pd(struct ib_pd *ibpd) +{ + return container_of(ibpd, struct qlnxr_pd, ibpd); +} + +static inline struct qlnxr_cq * +get_qlnxr_cq(struct ib_cq *ibcq) +{ + return container_of(ibcq, struct qlnxr_cq, ibcq); +} + +static inline struct qlnxr_qp * +get_qlnxr_qp(struct ib_qp *ibqp) +{ + return container_of(ibqp, struct qlnxr_qp, ibqp); +} + +static inline struct qlnxr_mr * +get_qlnxr_mr(struct ib_mr *ibmr) +{ + return container_of(ibmr, struct qlnxr_mr, ibmr); +} + +static inline struct qlnxr_ah * +get_qlnxr_ah(struct ib_ah *ibah) +{ + return container_of(ibah, struct qlnxr_ah, ibah); +} + +static inline struct qlnxr_srq * +get_qlnxr_srq(struct ib_srq *ibsrq) +{ + return container_of(ibsrq, struct qlnxr_srq, ibsrq); +} + +static inline bool qlnxr_qp_has_srq(struct qlnxr_qp *qp) +{ + return !!qp->srq; +} + +static inline bool qlnxr_qp_has_sq(struct qlnxr_qp *qp) +{ + if (qp->qp_type == IB_QPT_GSI) + return 0; + + return 1; +} + +static inline bool qlnxr_qp_has_rq(struct qlnxr_qp *qp) +{ + if (qp->qp_type == IB_QPT_GSI || qlnxr_qp_has_srq(qp)) + return 0; + + return 1; +} + + +#ifdef DEFINE_IB_FAST_REG +static inline struct qlnxr_fast_reg_page_list *get_qlnxr_frmr_list( + struct ib_fast_reg_page_list *ifrpl) +{ + return container_of(ifrpl, struct qlnxr_fast_reg_page_list, ibfrpl); +} +#endif + +#define SET_FIELD2(value, name, flag) \ + do { \ + (value) |= ((flag) << (name ## _SHIFT)); \ + } while (0) + +#define QLNXR_RESP_IMM (RDMA_CQE_RESPONDER_IMM_FLG_MASK << \ + RDMA_CQE_RESPONDER_IMM_FLG_SHIFT) +#define QLNXR_RESP_RDMA (RDMA_CQE_RESPONDER_RDMA_FLG_MASK << \ + RDMA_CQE_RESPONDER_RDMA_FLG_SHIFT) +#define QLNXR_RESP_INV (RDMA_CQE_RESPONDER_INV_FLG_MASK << \ + RDMA_CQE_RESPONDER_INV_FLG_SHIFT) + +#define QLNXR_RESP_RDMA_IMM (QLNXR_RESP_IMM | QLNXR_RESP_RDMA) + +static inline int +qlnxr_get_dmac(struct qlnxr_dev *dev, struct ib_ah_attr *ah_attr, u8 *mac_addr) +{ +#ifdef DEFINE_NO_IP_BASED_GIDS + u8 *guid = &ah_attr->grh.dgid.raw[8]; /* GID's 64 MSBs are the GUID */ +#endif + union ib_gid zero_sgid = { { 0 } }; + struct in6_addr in6; + + if (!memcmp(&ah_attr->grh.dgid, &zero_sgid, sizeof(union ib_gid))) { + memset(mac_addr, 0x00, ETH_ALEN); + return -EINVAL; + } + + memcpy(&in6, ah_attr->grh.dgid.raw, sizeof(in6)); + +#ifdef DEFINE_NO_IP_BASED_GIDS + /* get the MAC address from the GUID i.e. EUI-64 to MAC address */ + mac_addr[0] = guid[0] ^ 2; /* toggle the local/universal bit to local */ + mac_addr[1] = guid[1]; + mac_addr[2] = guid[2]; + mac_addr[3] = guid[5]; + mac_addr[4] = guid[6]; + mac_addr[5] = guid[7]; +#else + memcpy(mac_addr, ah_attr->dmac, ETH_ALEN); +#endif + return 0; +} + +extern int qlnx_rdma_ll2_set_mac_filter(void *rdma_ctx, uint8_t *old_mac_address, + uint8_t *new_mac_address); + + +#define QLNXR_ROCE_PKEY_MAX 1 +#define QLNXR_ROCE_PKEY_TABLE_LEN 1 +#define QLNXR_ROCE_PKEY_DEFAULT 0xffff + +#if __FreeBSD_version < 1100000 +#define DEFINE_IB_AH_ATTR_WITH_DMAC (0) +#define DEFINE_IB_UMEM_WITH_CHUNK (1) +#else +#define DEFINE_IB_AH_ATTR_WITH_DMAC (1) +#endif + +#define QLNX_IS_IWARP(rdev) IS_IWARP(ECORE_LEADING_HWFN(rdev->cdev)) +#define QLNX_IS_ROCE(rdev) IS_ROCE(ECORE_LEADING_HWFN(rdev->cdev)) + +#define MAX_RXMIT_CONNS 16 + +#endif /* #ifndef __QLNX_DEF_H_ */ Property changes on: head/sys/dev/qlnx/qlnxr/qlnxr_def.h ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Added: svn:mime-type ## -0,0 +1 ## +text/plain \ No newline at end of property Index: head/sys/dev/qlnx/qlnxr/qlnxr_os.c =================================================================== --- head/sys/dev/qlnx/qlnxr/qlnxr_os.c (nonexistent) +++ head/sys/dev/qlnx/qlnxr/qlnxr_os.c (revision 343598) @@ -0,0 +1,1366 @@ +/* + * Copyright (c) 2018-2019 Cavium, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + + +/* + * File: qlnxr_os.c + */ +#include +__FBSDID("$FreeBSD$"); + +#include "qlnxr_def.h" + +SYSCTL_NODE(_dev, OID_AUTO, qnxr, CTLFLAG_RW, 0, "Qlogic RDMA module"); + +uint32_t delayed_ack = 0; +SYSCTL_UINT(_dev_qnxr, OID_AUTO, delayed_ack, CTLFLAG_RW, &delayed_ack, 1, + "iWARP: Delayed Ack: 0 - Disabled 1 - Enabled. Default: Disabled"); + +uint32_t timestamp = 1; +SYSCTL_UINT(_dev_qnxr, OID_AUTO, timestamp, CTLFLAG_RW, ×tamp, 1, + "iWARP: Timestamp: 0 - Disabled 1 - Enabled. Default:Enabled"); + +uint32_t rcv_wnd_size = 0; +SYSCTL_UINT(_dev_qnxr, OID_AUTO, rcv_wnd_size, CTLFLAG_RW, &rcv_wnd_size, 1, + "iWARP: Receive Window Size in K. Default 1M"); + +uint32_t crc_needed = 1; +SYSCTL_UINT(_dev_qnxr, OID_AUTO, crc_needed, CTLFLAG_RW, &crc_needed, 1, + "iWARP: CRC needed 0 - Disabled 1 - Enabled. Default:Enabled"); + +uint32_t peer2peer = 1; +SYSCTL_UINT(_dev_qnxr, OID_AUTO, peer2peer, CTLFLAG_RW, &peer2peer, 1, + "iWARP: Support peer2peer ULPs 0 - Disabled 1 - Enabled. Default:Enabled"); + +uint32_t mpa_enhanced = 1; +SYSCTL_UINT(_dev_qnxr, OID_AUTO, mpa_enhanced, CTLFLAG_RW, &mpa_enhanced, 1, + "iWARP: MPA Enhanced mode. Default:1"); + +uint32_t rtr_type = 7; +SYSCTL_UINT(_dev_qnxr, OID_AUTO, rtr_type, CTLFLAG_RW, &rtr_type, 1, + "iWARP: RDMAP opcode to use for the RTR message: BITMAP 1: RDMA_SEND 2: RDMA_WRITE 4: RDMA_READ. Default: 7"); + + +#define QNXR_WQ_MULTIPLIER_MIN (1) +#define QNXR_WQ_MULTIPLIER_MAX (7) +#define QNXR_WQ_MULTIPLIER_DFT (3) + +uint32_t wq_multiplier= QNXR_WQ_MULTIPLIER_DFT; +SYSCTL_UINT(_dev_qnxr, OID_AUTO, wq_multiplier, CTLFLAG_RW, &wq_multiplier, 1, + " When creating a WQ the actual number of WQE created will" + " be multiplied by this number (default is 3)."); +static ssize_t +show_rev(struct device *device, struct device_attribute *attr, + char *buf) +{ + struct qlnxr_dev *dev = dev_get_drvdata(device); + + return sprintf(buf, "0x%x\n", dev->cdev->vendor_id); +} + +static ssize_t +show_hca_type(struct device *device, + struct device_attribute *attr, char *buf) +{ + struct qlnxr_dev *dev = dev_get_drvdata(device); + return sprintf(buf, "QLogic0x%x\n", dev->cdev->device_id); +} + +static ssize_t +show_fw_ver(struct device *device, + struct device_attribute *attr, char *buf) +{ + struct qlnxr_dev *dev = dev_get_drvdata(device); + uint32_t fw_ver = (uint32_t) dev->attr.fw_ver; + + return sprintf(buf, "%d.%d.%d\n", + (fw_ver >> 24) & 0xff, (fw_ver >> 16) & 0xff, + (fw_ver >> 8) & 0xff); +} +static ssize_t +show_board(struct device *device, + struct device_attribute *attr, char *buf) +{ + struct qlnxr_dev *dev = dev_get_drvdata(device); + return sprintf(buf, "%x\n", dev->cdev->device_id); +} + +static DEVICE_ATTR(hw_rev, S_IRUGO, show_rev, NULL); +static DEVICE_ATTR(hca_type, S_IRUGO, show_hca_type, NULL); +static DEVICE_ATTR(fw_ver, S_IRUGO, show_fw_ver, NULL); +static DEVICE_ATTR(board_id, S_IRUGO, show_board, NULL); + +static struct device_attribute *qlnxr_class_attributes[] = { + &dev_attr_hw_rev, + &dev_attr_hca_type, + &dev_attr_fw_ver, + &dev_attr_board_id +}; + +static void +qlnxr_ib_dispatch_event(qlnxr_dev_t *dev, uint8_t port_num, + enum ib_event_type type) +{ + struct ib_event ibev; + + QL_DPRINT12(dev->ha, "enter\n"); + + ibev.device = &dev->ibdev; + ibev.element.port_num = port_num; + ibev.event = type; + + ib_dispatch_event(&ibev); + + QL_DPRINT12(dev->ha, "exit\n"); +} + +static int +__qlnxr_iw_destroy_listen(struct iw_cm_id *cm_id) +{ + qlnxr_iw_destroy_listen(cm_id); + + return (0); +} + +static int +qlnxr_register_device(qlnxr_dev_t *dev) +{ + struct ib_device *ibdev; + struct iw_cm_verbs *iwcm; + int ret; + + QL_DPRINT12(dev->ha, "enter\n"); + + ibdev = &dev->ibdev; + + strlcpy(ibdev->name, "qlnxr%d", IB_DEVICE_NAME_MAX); + + memset(&ibdev->node_guid, 0, sizeof(ibdev->node_guid)); + memcpy(&ibdev->node_guid, dev->ha->primary_mac, ETHER_ADDR_LEN); + + memcpy(ibdev->node_desc, QLNXR_NODE_DESC, sizeof(QLNXR_NODE_DESC)); + + ibdev->owner = THIS_MODULE; + ibdev->uverbs_abi_ver = 7; + ibdev->local_dma_lkey = 0; + + ibdev->uverbs_cmd_mask = + (1ull << IB_USER_VERBS_CMD_GET_CONTEXT) | + (1ull << IB_USER_VERBS_CMD_QUERY_DEVICE) | + (1ull << IB_USER_VERBS_CMD_QUERY_PORT) | + (1ull << IB_USER_VERBS_CMD_ALLOC_PD) | + (1ull << IB_USER_VERBS_CMD_DEALLOC_PD) | + (1ull << IB_USER_VERBS_CMD_REG_MR) | + (1ull << IB_USER_VERBS_CMD_DEREG_MR) | + (1ull << IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL) | + (1ull << IB_USER_VERBS_CMD_CREATE_CQ) | + (1ull << IB_USER_VERBS_CMD_DESTROY_CQ) | + (1ull << IB_USER_VERBS_CMD_REQ_NOTIFY_CQ) | + (1ull << IB_USER_VERBS_CMD_CREATE_QP) | + (1ull << IB_USER_VERBS_CMD_MODIFY_QP) | + (1ull << IB_USER_VERBS_CMD_QUERY_QP) | + (1ull << IB_USER_VERBS_CMD_DESTROY_QP) | + (1ull << IB_USER_VERBS_CMD_POLL_CQ) | + (1ull << IB_USER_VERBS_CMD_POST_SEND) | + (1ull << IB_USER_VERBS_CMD_POST_RECV); + + if (QLNX_IS_IWARP(dev)) { + ibdev->node_type = RDMA_NODE_RNIC; + ibdev->query_gid = qlnxr_iw_query_gid; + } else { + ibdev->node_type = RDMA_NODE_IB_CA; + ibdev->query_gid = qlnxr_query_gid; + ibdev->uverbs_cmd_mask |= + (1ull << IB_USER_VERBS_CMD_CREATE_SRQ) | + (1ull << IB_USER_VERBS_CMD_DESTROY_SRQ) | + (1ull << IB_USER_VERBS_CMD_QUERY_SRQ) | + (1ull << IB_USER_VERBS_CMD_MODIFY_SRQ) | + (1ull << IB_USER_VERBS_CMD_POST_SRQ_RECV); + ibdev->create_srq = qlnxr_create_srq; + ibdev->destroy_srq = qlnxr_destroy_srq; + ibdev->modify_srq = qlnxr_modify_srq; + ibdev->query_srq = qlnxr_query_srq; + ibdev->post_srq_recv = qlnxr_post_srq_recv; + } + + ibdev->phys_port_cnt = 1; + ibdev->num_comp_vectors = dev->num_cnq; + + /* mandatory verbs. */ + ibdev->query_device = qlnxr_query_device; + ibdev->query_port = qlnxr_query_port; + ibdev->modify_port = qlnxr_modify_port; + + ibdev->alloc_ucontext = qlnxr_alloc_ucontext; + ibdev->dealloc_ucontext = qlnxr_dealloc_ucontext; + /* mandatory to support user space verbs consumer. */ + ibdev->mmap = qlnxr_mmap; + + ibdev->alloc_pd = qlnxr_alloc_pd; + ibdev->dealloc_pd = qlnxr_dealloc_pd; + + ibdev->create_cq = qlnxr_create_cq; + ibdev->destroy_cq = qlnxr_destroy_cq; + ibdev->resize_cq = qlnxr_resize_cq; + ibdev->req_notify_cq = qlnxr_arm_cq; + + ibdev->create_qp = qlnxr_create_qp; + ibdev->modify_qp = qlnxr_modify_qp; + ibdev->query_qp = qlnxr_query_qp; + ibdev->destroy_qp = qlnxr_destroy_qp; + + ibdev->query_pkey = qlnxr_query_pkey; + ibdev->create_ah = qlnxr_create_ah; + ibdev->destroy_ah = qlnxr_destroy_ah; + ibdev->query_ah = qlnxr_query_ah; + ibdev->modify_ah = qlnxr_modify_ah; + ibdev->get_dma_mr = qlnxr_get_dma_mr; + ibdev->dereg_mr = qlnxr_dereg_mr; + ibdev->reg_user_mr = qlnxr_reg_user_mr; + +#if __FreeBSD_version >= 1102000 + ibdev->alloc_mr = qlnxr_alloc_mr; + ibdev->map_mr_sg = qlnxr_map_mr_sg; + ibdev->get_port_immutable = qlnxr_get_port_immutable; +#else + ibdev->reg_phys_mr = qlnxr_reg_kernel_mr; + ibdev->alloc_fast_reg_mr = qlnxr_alloc_frmr; + ibdev->alloc_fast_reg_page_list = qlnxr_alloc_frmr_page_list; + ibdev->free_fast_reg_page_list = qlnxr_free_frmr_page_list; +#endif /* #if __FreeBSD_version >= 1102000 */ + + ibdev->poll_cq = qlnxr_poll_cq; + ibdev->post_send = qlnxr_post_send; + ibdev->post_recv = qlnxr_post_recv; + ibdev->process_mad = qlnxr_process_mad; + + + + ibdev->dma_device = &dev->pdev->dev; + + ibdev->get_link_layer = qlnxr_link_layer; + + if (QLNX_IS_IWARP(dev)) { + iwcm = kmalloc(sizeof(*iwcm), GFP_KERNEL); + + device_printf(dev->ha->pci_dev, "device is IWARP\n"); + if (iwcm == NULL) + return (-ENOMEM); + + ibdev->iwcm = iwcm; + + iwcm->connect = qlnxr_iw_connect; + iwcm->accept = qlnxr_iw_accept; + iwcm->reject = qlnxr_iw_reject; + +#if (__FreeBSD_version >= 1004000) && (__FreeBSD_version < 1102000) + + iwcm->create_listen_ep = qlnxr_iw_create_listen; + iwcm->destroy_listen_ep = qlnxr_iw_destroy_listen; +#else + iwcm->create_listen = qlnxr_iw_create_listen; + iwcm->destroy_listen = __qlnxr_iw_destroy_listen; +#endif + iwcm->add_ref = qlnxr_iw_qp_add_ref; + iwcm->rem_ref = qlnxr_iw_qp_rem_ref; + iwcm->get_qp = qlnxr_iw_get_qp; + } + + ret = ib_register_device(ibdev, NULL); + if (ret) { + kfree(iwcm); + } + + QL_DPRINT12(dev->ha, "exit\n"); + return ret; +} + +#define HILO_U64(hi, lo) ((((u64)(hi)) << 32) + (lo)) + +static void +qlnxr_intr(void *handle) +{ + struct qlnxr_cnq *cnq = handle; + struct qlnxr_cq *cq; + struct regpair *cq_handle; + u16 hw_comp_cons, sw_comp_cons; + qlnx_host_t *ha; + + ha = cnq->dev->ha; + + QL_DPRINT12(ha, "enter cnq = %p\n", handle); + + ecore_sb_ack(cnq->sb, IGU_INT_DISABLE, 0 /*do not update*/); + + ecore_sb_update_sb_idx(cnq->sb); + + hw_comp_cons = le16_to_cpu(*cnq->hw_cons_ptr); + sw_comp_cons = ecore_chain_get_cons_idx(&cnq->pbl); + + rmb(); + + QL_DPRINT12(ha, "enter cnq = %p hw_comp_cons = 0x%x sw_comp_cons = 0x%x\n", + handle, hw_comp_cons, sw_comp_cons); + + while (sw_comp_cons != hw_comp_cons) { + cq_handle = (struct regpair *)ecore_chain_consume(&cnq->pbl); + cq = (struct qlnxr_cq *)(uintptr_t)HILO_U64(cq_handle->hi, + cq_handle->lo); + + if (cq == NULL) { + QL_DPRINT11(ha, "cq == NULL\n"); + break; + } + + if (cq->sig != QLNXR_CQ_MAGIC_NUMBER) { + QL_DPRINT11(ha, + "cq->sig = 0x%x QLNXR_CQ_MAGIC_NUMBER = 0x%x\n", + cq->sig, QLNXR_CQ_MAGIC_NUMBER); + break; + } + cq->arm_flags = 0; + + if (!cq->destroyed && cq->ibcq.comp_handler) { + QL_DPRINT11(ha, "calling comp_handler = %p " + "ibcq = %p cq_context = 0x%x\n", + &cq->ibcq, cq->ibcq.cq_context); + + (*cq->ibcq.comp_handler) (&cq->ibcq, cq->ibcq.cq_context); + } + cq->cnq_notif++; + + sw_comp_cons = ecore_chain_get_cons_idx(&cnq->pbl); + + cnq->n_comp++; + } + + ecore_rdma_cnq_prod_update(cnq->dev->rdma_ctx, cnq->index, sw_comp_cons); + + ecore_sb_ack(cnq->sb, IGU_INT_ENABLE, 1 /*update*/); + + QL_DPRINT12(ha, "exit cnq = %p\n", handle); + return; +} + +static void +qlnxr_release_irqs(struct qlnxr_dev *dev) +{ + int i; + qlnx_host_t *ha; + + ha = dev->ha; + + QL_DPRINT12(ha, "enter\n"); + + for (i = 0; i < dev->num_cnq; i++) { + if (dev->cnq_array[i].irq_handle) + (void)bus_teardown_intr(dev->ha->pci_dev, + dev->cnq_array[i].irq, + dev->cnq_array[i].irq_handle); + + if (dev->cnq_array[i].irq) + (void) bus_release_resource(dev->ha->pci_dev, + SYS_RES_IRQ, + dev->cnq_array[i].irq_rid, + dev->cnq_array[i].irq); + } + QL_DPRINT12(ha, "exit\n"); + return; +} + +static int +qlnxr_setup_irqs(struct qlnxr_dev *dev) +{ + int start_irq_rid; + int i; + qlnx_host_t *ha; + + ha = dev->ha; + + start_irq_rid = dev->sb_start + 2; + + QL_DPRINT12(ha, "enter start_irq_rid = %d num_rss = %d\n", + start_irq_rid, dev->ha->num_rss); + + + for (i = 0; i < dev->num_cnq; i++) { + + dev->cnq_array[i].irq_rid = start_irq_rid + i; + + dev->cnq_array[i].irq = bus_alloc_resource_any(dev->ha->pci_dev, + SYS_RES_IRQ, + &dev->cnq_array[i].irq_rid, + (RF_ACTIVE | RF_SHAREABLE)); + + if (dev->cnq_array[i].irq == NULL) { + + QL_DPRINT11(ha, + "bus_alloc_resource_any failed irq_rid = %d\n", + dev->cnq_array[i].irq_rid); + + goto qlnxr_setup_irqs_err; + } + + if (bus_setup_intr(dev->ha->pci_dev, + dev->cnq_array[i].irq, + (INTR_TYPE_NET | INTR_MPSAFE), + NULL, qlnxr_intr, &dev->cnq_array[i], + &dev->cnq_array[i].irq_handle)) { + + QL_DPRINT11(ha, "bus_setup_intr failed\n"); + goto qlnxr_setup_irqs_err; + } + QL_DPRINT12(ha, "irq_rid = %d irq = %p irq_handle = %p\n", + dev->cnq_array[i].irq_rid, dev->cnq_array[i].irq, + dev->cnq_array[i].irq_handle); + } + + QL_DPRINT12(ha, "exit\n"); + return (0); + +qlnxr_setup_irqs_err: + qlnxr_release_irqs(dev); + + QL_DPRINT12(ha, "exit -1\n"); + return (-1); +} + +static void +qlnxr_free_resources(struct qlnxr_dev *dev) +{ + int i; + qlnx_host_t *ha; + + ha = dev->ha; + + QL_DPRINT12(ha, "enter dev->num_cnq = %d\n", dev->num_cnq); + + if (QLNX_IS_IWARP(dev)) { + if (dev->iwarp_wq != NULL) + destroy_workqueue(dev->iwarp_wq); + } + + for (i = 0; i < dev->num_cnq; i++) { + qlnx_free_mem_sb(dev->ha, &dev->sb_array[i]); + ecore_chain_free(&dev->ha->cdev, &dev->cnq_array[i].pbl); + } + + bzero(dev->cnq_array, (sizeof(struct qlnxr_cnq) * QLNXR_MAX_MSIX)); + bzero(dev->sb_array, (sizeof(struct ecore_sb_info) * QLNXR_MAX_MSIX)); + bzero(dev->sgid_tbl, (sizeof(union ib_gid) * QLNXR_MAX_SGID)); + + if (mtx_initialized(&dev->idr_lock)) + mtx_destroy(&dev->idr_lock); + + if (mtx_initialized(&dev->sgid_lock)) + mtx_destroy(&dev->sgid_lock); + + QL_DPRINT12(ha, "exit\n"); + return; +} + + +static int +qlnxr_alloc_resources(struct qlnxr_dev *dev) +{ + uint16_t n_entries; + int i, rc; + qlnx_host_t *ha; + + ha = dev->ha; + + QL_DPRINT12(ha, "enter\n"); + + bzero(dev->sgid_tbl, (sizeof (union ib_gid) * QLNXR_MAX_SGID)); + + mtx_init(&dev->idr_lock, "idr_lock", NULL, MTX_DEF); + mtx_init(&dev->sgid_lock, "sgid_lock", NULL, MTX_DEF); + + idr_init(&dev->qpidr); + + bzero(dev->sb_array, (sizeof (struct ecore_sb_info) * QLNXR_MAX_MSIX)); + bzero(dev->cnq_array, (sizeof (struct qlnxr_cnq) * QLNXR_MAX_MSIX)); + + dev->sb_start = ecore_rdma_get_sb_id(dev->rdma_ctx, 0); + + QL_DPRINT12(ha, "dev->sb_start = 0x%x\n", dev->sb_start); + + /* Allocate CNQ PBLs */ + + n_entries = min_t(u32, ECORE_RDMA_MAX_CNQ_SIZE, QLNXR_ROCE_MAX_CNQ_SIZE); + + for (i = 0; i < dev->num_cnq; i++) { + rc = qlnx_alloc_mem_sb(dev->ha, &dev->sb_array[i], + dev->sb_start + i); + if (rc) + goto qlnxr_alloc_resources_exit; + + rc = ecore_chain_alloc(&dev->ha->cdev, + ECORE_CHAIN_USE_TO_CONSUME_PRODUCE, + ECORE_CHAIN_MODE_PBL, + ECORE_CHAIN_CNT_TYPE_U16, + n_entries, + sizeof(struct regpair *), + &dev->cnq_array[i].pbl, + NULL); + + /* configure cnq, except name since ibdev.name is still NULL */ + dev->cnq_array[i].dev = dev; + dev->cnq_array[i].sb = &dev->sb_array[i]; + dev->cnq_array[i].hw_cons_ptr = + &(dev->sb_array[i].sb_virt->pi_array[ECORE_ROCE_PROTOCOL_INDEX]); + dev->cnq_array[i].index = i; + sprintf(dev->cnq_array[i].name, "qlnxr%d@pci:%d", + i, (dev->ha->pci_func)); + + } + + QL_DPRINT12(ha, "exit\n"); + return 0; + +qlnxr_alloc_resources_exit: + + qlnxr_free_resources(dev); + + QL_DPRINT12(ha, "exit -ENOMEM\n"); + return -ENOMEM; +} + +void +qlnxr_affiliated_event(void *context, u8 e_code, void *fw_handle) +{ +#define EVENT_TYPE_NOT_DEFINED 0 +#define EVENT_TYPE_CQ 1 +#define EVENT_TYPE_QP 2 +#define EVENT_TYPE_GENERAL 3 + + struct qlnxr_dev *dev = (struct qlnxr_dev *)context; + struct regpair *async_handle = (struct regpair *)fw_handle; + u64 roceHandle64 = ((u64)async_handle->hi << 32) + async_handle->lo; + struct qlnxr_cq *cq = (struct qlnxr_cq *)(uintptr_t)roceHandle64; + struct qlnxr_qp *qp = (struct qlnxr_qp *)(uintptr_t)roceHandle64; + u8 event_type = EVENT_TYPE_NOT_DEFINED; + struct ib_event event; + qlnx_host_t *ha; + + ha = dev->ha; + + QL_DPRINT12(ha, "enter context = %p e_code = 0x%x fw_handle = %p\n", + context, e_code, fw_handle); + + if (QLNX_IS_IWARP(dev)) { + switch (e_code) { + + case ECORE_IWARP_EVENT_CQ_OVERFLOW: + event.event = IB_EVENT_CQ_ERR; + event_type = EVENT_TYPE_CQ; + break; + + default: + QL_DPRINT12(ha, + "unsupported event %d on handle=%llx\n", + e_code, roceHandle64); + break; + } + } else { + switch (e_code) { + + case ROCE_ASYNC_EVENT_CQ_OVERFLOW_ERR: + event.event = IB_EVENT_CQ_ERR; + event_type = EVENT_TYPE_CQ; + break; + + case ROCE_ASYNC_EVENT_SQ_DRAINED: + event.event = IB_EVENT_SQ_DRAINED; + event_type = EVENT_TYPE_QP; + break; + + case ROCE_ASYNC_EVENT_QP_CATASTROPHIC_ERR: + event.event = IB_EVENT_QP_FATAL; + event_type = EVENT_TYPE_QP; + break; + + case ROCE_ASYNC_EVENT_LOCAL_INVALID_REQUEST_ERR: + event.event = IB_EVENT_QP_REQ_ERR; + event_type = EVENT_TYPE_QP; + break; + + case ROCE_ASYNC_EVENT_LOCAL_ACCESS_ERR: + event.event = IB_EVENT_QP_ACCESS_ERR; + event_type = EVENT_TYPE_QP; + break; + + /* NOTE the following are not implemented in FW + * ROCE_ASYNC_EVENT_CQ_ERR + * ROCE_ASYNC_EVENT_COMM_EST + */ + /* TODO associate the following events - + * ROCE_ASYNC_EVENT_SRQ_LIMIT + * ROCE_ASYNC_EVENT_LAST_WQE_REACHED + * ROCE_ASYNC_EVENT_LOCAL_CATASTROPHIC_ERR (un-affiliated) + */ + default: + QL_DPRINT12(ha, + "unsupported event 0x%x on fw_handle = %p\n", + e_code, fw_handle); + break; + } + } + + switch (event_type) { + + case EVENT_TYPE_CQ: + if (cq && cq->sig == QLNXR_CQ_MAGIC_NUMBER) { + struct ib_cq *ibcq = &cq->ibcq; + + if (ibcq->event_handler) { + event.device = ibcq->device; + event.element.cq = ibcq; + ibcq->event_handler(&event, ibcq->cq_context); + } + } else { + QL_DPRINT11(ha, + "CQ event with invalid CQ pointer" + " Handle = %llx\n", roceHandle64); + } + QL_DPRINT12(ha, + "CQ event 0x%x on handle = %p\n", e_code, cq); + break; + + case EVENT_TYPE_QP: + if (qp && qp->sig == QLNXR_QP_MAGIC_NUMBER) { + struct ib_qp *ibqp = &qp->ibqp; + + if (ibqp->event_handler) { + event.device = ibqp->device; + event.element.qp = ibqp; + ibqp->event_handler(&event, ibqp->qp_context); + } + } else { + QL_DPRINT11(ha, + "QP event 0x%x with invalid QP pointer" + " qp handle = %p\n", + e_code, roceHandle64); + } + QL_DPRINT12(ha, "QP event 0x%x on qp handle = %p\n", + e_code, qp); + break; + + case EVENT_TYPE_GENERAL: + break; + + default: + break; + + } + + QL_DPRINT12(ha, "exit\n"); + + return; +} + +void +qlnxr_unaffiliated_event(void *context, u8 e_code) +{ + struct qlnxr_dev *dev = (struct qlnxr_dev *)context; + qlnx_host_t *ha; + + ha = dev->ha; + + QL_DPRINT12(ha, "enter/exit \n"); + return; +} + + +static int +qlnxr_set_device_attr(struct qlnxr_dev *dev) +{ + struct ecore_rdma_device *ecore_attr; + struct qlnxr_device_attr *attr; + u32 page_size; + + ecore_attr = ecore_rdma_query_device(dev->rdma_ctx); + + page_size = ~dev->attr.page_size_caps + 1; + if(page_size > PAGE_SIZE) { + QL_DPRINT12(dev->ha, "Kernel page size : %ld is smaller than" + " minimum page size : %ld required by qlnxr\n", + PAGE_SIZE, page_size); + return -ENODEV; + } + attr = &dev->attr; + attr->vendor_id = ecore_attr->vendor_id; + attr->vendor_part_id = ecore_attr->vendor_part_id; + + QL_DPRINT12(dev->ha, "in qlnxr_set_device_attr, vendor : %x device : %x\n", + attr->vendor_id, attr->vendor_part_id); + + attr->hw_ver = ecore_attr->hw_ver; + attr->fw_ver = ecore_attr->fw_ver; + attr->node_guid = ecore_attr->node_guid; + attr->sys_image_guid = ecore_attr->sys_image_guid; + attr->max_cnq = ecore_attr->max_cnq; + attr->max_sge = ecore_attr->max_sge; + attr->max_inline = ecore_attr->max_inline; + attr->max_sqe = min_t(u32, ecore_attr->max_wqe, QLNXR_MAX_SQE); + attr->max_rqe = min_t(u32, ecore_attr->max_wqe, QLNXR_MAX_RQE); + attr->max_qp_resp_rd_atomic_resc = ecore_attr->max_qp_resp_rd_atomic_resc; + attr->max_qp_req_rd_atomic_resc = ecore_attr->max_qp_req_rd_atomic_resc; + attr->max_dev_resp_rd_atomic_resc = + ecore_attr->max_dev_resp_rd_atomic_resc; + attr->max_cq = ecore_attr->max_cq; + attr->max_qp = ecore_attr->max_qp; + attr->max_mr = ecore_attr->max_mr; + attr->max_mr_size = ecore_attr->max_mr_size; + attr->max_cqe = min_t(u64, ecore_attr->max_cqe, QLNXR_MAX_CQES); + attr->max_mw = ecore_attr->max_mw; + attr->max_fmr = ecore_attr->max_fmr; + attr->max_mr_mw_fmr_pbl = ecore_attr->max_mr_mw_fmr_pbl; + attr->max_mr_mw_fmr_size = ecore_attr->max_mr_mw_fmr_size; + attr->max_pd = ecore_attr->max_pd; + attr->max_ah = ecore_attr->max_ah; + attr->max_pkey = ecore_attr->max_pkey; + attr->max_srq = ecore_attr->max_srq; + attr->max_srq_wr = ecore_attr->max_srq_wr; + //attr->dev_caps = ecore_attr->dev_caps; + attr->page_size_caps = ecore_attr->page_size_caps; + attr->dev_ack_delay = ecore_attr->dev_ack_delay; + attr->reserved_lkey = ecore_attr->reserved_lkey; + attr->bad_pkey_counter = ecore_attr->bad_pkey_counter; + attr->max_stats_queues = ecore_attr->max_stats_queues; + + return 0; +} + + +static int +qlnxr_init_hw(struct qlnxr_dev *dev) +{ + struct ecore_rdma_events events; + struct ecore_rdma_add_user_out_params out_params; + struct ecore_rdma_cnq_params *cur_pbl; + struct ecore_rdma_start_in_params *in_params; + dma_addr_t p_phys_table; + u32 page_cnt; + int rc = 0; + int i; + qlnx_host_t *ha; + + ha = dev->ha; + + QL_DPRINT12(ha, "enter\n"); + + in_params = kzalloc(sizeof(*in_params), GFP_KERNEL); + if (!in_params) { + rc = -ENOMEM; + goto out; + } + + bzero(&out_params, sizeof(struct ecore_rdma_add_user_out_params)); + bzero(&events, sizeof(struct ecore_rdma_events)); + + in_params->desired_cnq = dev->num_cnq; + + for (i = 0; i < dev->num_cnq; i++) { + cur_pbl = &in_params->cnq_pbl_list[i]; + + page_cnt = ecore_chain_get_page_cnt(&dev->cnq_array[i].pbl); + cur_pbl->num_pbl_pages = page_cnt; + + p_phys_table = ecore_chain_get_pbl_phys(&dev->cnq_array[i].pbl); + cur_pbl->pbl_ptr = (u64)p_phys_table; + } + + events.affiliated_event = qlnxr_affiliated_event; + events.unaffiliated_event = qlnxr_unaffiliated_event; + events.context = dev; + + in_params->events = &events; + in_params->roce.cq_mode = ECORE_RDMA_CQ_MODE_32_BITS; + in_params->max_mtu = dev->ha->max_frame_size; + + + if (QLNX_IS_IWARP(dev)) { + if (delayed_ack) + in_params->iwarp.flags |= ECORE_IWARP_DA_EN; + + if (timestamp) + in_params->iwarp.flags |= ECORE_IWARP_TS_EN; + + in_params->iwarp.rcv_wnd_size = rcv_wnd_size*1024; + in_params->iwarp.crc_needed = crc_needed; + in_params->iwarp.ooo_num_rx_bufs = + (MAX_RXMIT_CONNS * in_params->iwarp.rcv_wnd_size) / + in_params->max_mtu; + + in_params->iwarp.mpa_peer2peer = peer2peer; + in_params->iwarp.mpa_rev = + mpa_enhanced ? ECORE_MPA_REV2 : ECORE_MPA_REV1; + in_params->iwarp.mpa_rtr = rtr_type; + } + + memcpy(&in_params->mac_addr[0], dev->ha->primary_mac, ETH_ALEN); + + rc = ecore_rdma_start(dev->rdma_ctx, in_params); + if (rc) + goto out; + + rc = ecore_rdma_add_user(dev->rdma_ctx, &out_params); + if (rc) + goto out; + + dev->db_addr = (void *)(uintptr_t)out_params.dpi_addr; + dev->db_phys_addr = out_params.dpi_phys_addr; + dev->db_size = out_params.dpi_size; + dev->dpi = out_params.dpi; + + qlnxr_set_device_attr(dev); + + QL_DPRINT12(ha, + "cdev->doorbells = %p, db_phys_addr = %p db_size = 0x%x\n", + (void *)ha->cdev.doorbells, + (void *)ha->cdev.db_phys_addr, ha->cdev.db_size); + + QL_DPRINT12(ha, + "db_addr = %p db_phys_addr = %p db_size = 0x%x dpi = 0x%x\n", + (void *)dev->db_addr, (void *)dev->db_phys_addr, + dev->db_size, dev->dpi); +out: + kfree(in_params); + + QL_DPRINT12(ha, "exit\n"); + return rc; +} + +static void +qlnxr_build_sgid_mac(union ib_gid *sgid, unsigned char *mac_addr, + bool is_vlan, u16 vlan_id) +{ + sgid->global.subnet_prefix = OSAL_CPU_TO_BE64(0xfe80000000000000LL); + sgid->raw[8] = mac_addr[0] ^ 2; + sgid->raw[9] = mac_addr[1]; + sgid->raw[10] = mac_addr[2]; + if (is_vlan) { + sgid->raw[11] = vlan_id >> 8; + sgid->raw[12] = vlan_id & 0xff; + } else { + sgid->raw[11] = 0xff; + sgid->raw[12] = 0xfe; + } + sgid->raw[13] = mac_addr[3]; + sgid->raw[14] = mac_addr[4]; + sgid->raw[15] = mac_addr[5]; +} +static bool +qlnxr_add_sgid(struct qlnxr_dev *dev, union ib_gid *new_sgid); + +static void +qlnxr_add_ip_based_gid(struct qlnxr_dev *dev, struct ifnet *ifp) +{ + struct ifaddr *ifa; + union ib_gid gid; + + CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { + if (ifa->ifa_addr && ifa->ifa_addr->sa_family == AF_INET) { + + QL_DPRINT12(dev->ha, "IP address : %x\n", ((struct sockaddr_in *) ifa->ifa_addr)->sin_addr.s_addr); + ipv6_addr_set_v4mapped( + ((struct sockaddr_in *) ifa->ifa_addr)->sin_addr.s_addr, + (struct in6_addr *)&gid); + QL_DPRINT12(dev->ha, "gid generated : %llx\n", gid); + + qlnxr_add_sgid(dev, &gid); + } + } + for (int i = 0; i < 16; i++) { + QL_DPRINT12(dev->ha, "gid generated : %x\n", gid.raw[i]); + } +} + +static bool +qlnxr_add_sgid(struct qlnxr_dev *dev, union ib_gid *new_sgid) +{ + union ib_gid zero_sgid = { { 0 } }; + int i; + //unsigned long flags; + mtx_lock(&dev->sgid_lock); + for (i = 0; i < QLNXR_MAX_SGID; i++) { + if (!memcmp(&dev->sgid_tbl[i], &zero_sgid, + sizeof(union ib_gid))) { + /* found free entry */ + memcpy(&dev->sgid_tbl[i], new_sgid, + sizeof(union ib_gid)); + QL_DPRINT12(dev->ha, "copying sgid : %llx\n", + *new_sgid); + mtx_unlock(&dev->sgid_lock); + //TODO ib_dispatch event here? + return true; + } else if (!memcmp(&dev->sgid_tbl[i], new_sgid, + sizeof(union ib_gid))) { + /* entry already present, no addition required */ + mtx_unlock(&dev->sgid_lock); + QL_DPRINT12(dev->ha, "sgid present : %llx\n", + *new_sgid); + return false; + } + } + if (i == QLNXR_MAX_SGID) { + QL_DPRINT12(dev->ha, "didn't find an empty entry in sgid_tbl\n"); + } + mtx_unlock(&dev->sgid_lock); + return false; +} + +static bool qlnxr_del_sgid(struct qlnxr_dev *dev, union ib_gid *gid) +{ + int found = false; + int i; + //unsigned long flags; + + QL_DPRINT12(dev->ha, "removing gid %llx %llx\n", + gid->global.interface_id, + gid->global.subnet_prefix); + mtx_lock(&dev->sgid_lock); + /* first is the default sgid which cannot be deleted */ + for (i = 1; i < QLNXR_MAX_SGID; i++) { + if (!memcmp(&dev->sgid_tbl[i], gid, sizeof(union ib_gid))) { + /* found matching entry */ + memset(&dev->sgid_tbl[i], 0, sizeof(union ib_gid)); + found = true; + break; + } + } + mtx_unlock(&dev->sgid_lock); + + return found; +} + +#if __FreeBSD_version < 1100000 + +static inline int +is_vlan_dev(struct ifnet *ifp) +{ + return (ifp->if_type == IFT_L2VLAN); +} + +static inline uint16_t +vlan_dev_vlan_id(struct ifnet *ifp) +{ + uint16_t vtag; + + if (VLAN_TAG(ifp, &vtag) == 0) + return (vtag); + + return (0); +} + +#endif /* #if __FreeBSD_version < 1100000 */ + +static void +qlnxr_add_sgids(struct qlnxr_dev *dev) +{ + qlnx_host_t *ha = dev->ha; + u16 vlan_id; + bool is_vlan; + union ib_gid vgid; + + qlnxr_add_ip_based_gid(dev, ha->ifp); + /* MAC/VLAN base GIDs */ + is_vlan = is_vlan_dev(ha->ifp); + vlan_id = (is_vlan) ? vlan_dev_vlan_id(ha->ifp) : 0; + qlnxr_build_sgid_mac(&vgid, ha->primary_mac, is_vlan, vlan_id); + qlnxr_add_sgid(dev, &vgid); +} + +static int +qlnxr_add_default_sgid(struct qlnxr_dev *dev) +{ + /* GID Index 0 - Invariant manufacturer-assigned EUI-64 */ + union ib_gid *sgid = &dev->sgid_tbl[0]; + struct ecore_rdma_device *qattr; + qlnx_host_t *ha; + ha = dev->ha; + + qattr = ecore_rdma_query_device(dev->rdma_ctx); + if(sgid == NULL) + QL_DPRINT12(ha, "sgid = NULL?\n"); + + sgid->global.subnet_prefix = OSAL_CPU_TO_BE64(0xfe80000000000000LL); + QL_DPRINT12(ha, "node_guid = %llx", dev->attr.node_guid); + memcpy(&sgid->raw[8], &qattr->node_guid, + sizeof(qattr->node_guid)); + //memcpy(&sgid->raw[8], &dev->attr.node_guid, + // sizeof(dev->attr.node_guid)); + QL_DPRINT12(ha, "DEFAULT sgid=[%x][%x][%x][%x][%x][%x][%x][%x][%x][%x][%x][%x][%x][%x][%x][%x]\n", + sgid->raw[0], sgid->raw[1], sgid->raw[2], sgid->raw[3], sgid->raw[4], sgid->raw[5], + sgid->raw[6], sgid->raw[7], sgid->raw[8], sgid->raw[9], sgid->raw[10], sgid->raw[11], + sgid->raw[12], sgid->raw[13], sgid->raw[14], sgid->raw[15]); + return 0; +} + +static int qlnxr_addr_event (struct qlnxr_dev *dev, + unsigned long event, + struct ifnet *ifp, + union ib_gid *gid) +{ + bool is_vlan = false; + union ib_gid vgid; + u16 vlan_id = 0xffff; + + QL_DPRINT12(dev->ha, "Link event occured\n"); + is_vlan = is_vlan_dev(dev->ha->ifp); + vlan_id = (is_vlan) ? vlan_dev_vlan_id(dev->ha->ifp) : 0; + + switch (event) { + case NETDEV_UP : + qlnxr_add_sgid(dev, gid); + if (is_vlan) { + qlnxr_build_sgid_mac(&vgid, dev->ha->primary_mac, is_vlan, vlan_id); + qlnxr_add_sgid(dev, &vgid); + } + break; + case NETDEV_DOWN : + qlnxr_del_sgid(dev, gid); + if (is_vlan) { + qlnxr_build_sgid_mac(&vgid, dev->ha->primary_mac, is_vlan, vlan_id); + qlnxr_del_sgid(dev, &vgid); + } + break; + default : + break; + } + return 1; +} + +static int qlnxr_inetaddr_event(struct notifier_block *notifier, + unsigned long event, void *ptr) +{ + struct ifaddr *ifa = ptr; + union ib_gid gid; + struct qlnxr_dev *dev = container_of(notifier, struct qlnxr_dev, nb_inet); + qlnx_host_t *ha = dev->ha; + + ipv6_addr_set_v4mapped( + ((struct sockaddr_in *) ifa->ifa_addr)->sin_addr.s_addr, + (struct in6_addr *)&gid); + return qlnxr_addr_event(dev, event, ha->ifp, &gid); +} + +static int +qlnxr_register_inet(struct qlnxr_dev *dev) +{ + int ret; + dev->nb_inet.notifier_call = qlnxr_inetaddr_event; + ret = register_inetaddr_notifier(&dev->nb_inet); + if (ret) { + QL_DPRINT12(dev->ha, "Failed to register inetaddr\n"); + return ret; + } + /* TODO : add for CONFIG_IPV6) */ + return 0; +} + +static int +qlnxr_build_sgid_tbl(struct qlnxr_dev *dev) +{ + qlnxr_add_default_sgid(dev); + qlnxr_add_sgids(dev); + return 0; +} + +static struct qlnx_rdma_if qlnxr_drv; + +static void * +qlnxr_add(void *eth_dev) +{ + struct qlnxr_dev *dev; + int ret; + //device_t pci_dev; + qlnx_host_t *ha; + + ha = eth_dev; + + QL_DPRINT12(ha, "enter [ha = %p]\n", ha); + + dev = (struct qlnxr_dev *)ib_alloc_device(sizeof(struct qlnxr_dev)); + + if (dev == NULL) + return (NULL); + + dev->ha = eth_dev; + dev->cdev = &ha->cdev; + /* Added to extend Application support */ + dev->pdev = kzalloc(sizeof(struct pci_dev), GFP_KERNEL); + + dev->pdev->dev = *(dev->ha->pci_dev); + dev->pdev->device = pci_get_device(dev->ha->pci_dev); + dev->pdev->vendor = pci_get_vendor(dev->ha->pci_dev); + + dev->rdma_ctx = &ha->cdev.hwfns[0]; + dev->wq_multiplier = wq_multiplier; + dev->num_cnq = QLNX_NUM_CNQ; + + QL_DPRINT12(ha, + "ha = %p dev = %p ha->cdev = %p\n", + ha, dev, &ha->cdev); + QL_DPRINT12(ha, + "dev->cdev = %p dev->rdma_ctx = %p\n", + dev->cdev, dev->rdma_ctx); + + ret = qlnxr_alloc_resources(dev); + + if (ret) + goto qlnxr_add_err; + + ret = qlnxr_setup_irqs(dev); + + if (ret) { + qlnxr_free_resources(dev); + goto qlnxr_add_err; + } + + ret = qlnxr_init_hw(dev); + + if (ret) { + qlnxr_release_irqs(dev); + qlnxr_free_resources(dev); + goto qlnxr_add_err; + } + + qlnxr_register_device(dev); + for (int i = 0; i < ARRAY_SIZE(qlnxr_class_attributes); ++i) { + if (device_create_file(&dev->ibdev.dev, qlnxr_class_attributes[i])) + goto sysfs_err; + } + qlnxr_build_sgid_tbl(dev); + //ret = qlnxr_register_inet(dev); + QL_DPRINT12(ha, "exit\n"); + if (!test_and_set_bit(QLNXR_ENET_STATE_BIT, &dev->enet_state)) { + QL_DPRINT12(ha, "dispatching IB_PORT_ACITVE event\n"); + qlnxr_ib_dispatch_event(dev, QLNXR_PORT, + IB_EVENT_PORT_ACTIVE); + } + + return (dev); +sysfs_err: + for (int i = 0; i < ARRAY_SIZE(qlnxr_class_attributes); ++i) { + device_remove_file(&dev->ibdev.dev, qlnxr_class_attributes[i]); + } + ib_unregister_device(&dev->ibdev); + +qlnxr_add_err: + ib_dealloc_device(&dev->ibdev); + + QL_DPRINT12(ha, "exit failed\n"); + return (NULL); +} + +static void +qlnxr_remove_sysfiles(struct qlnxr_dev *dev) +{ + int i; + for (i = 0; i < ARRAY_SIZE(qlnxr_class_attributes); ++i) + device_remove_file(&dev->ibdev.dev, qlnxr_class_attributes[i]); +} + +static int +qlnxr_remove(void *eth_dev, void *qlnx_rdma_dev) +{ + struct qlnxr_dev *dev; + qlnx_host_t *ha; + + dev = qlnx_rdma_dev; + ha = eth_dev; + + if ((ha == NULL) || (dev == NULL)) + return (0); + + QL_DPRINT12(ha, "enter ha = %p qlnx_rdma_dev = %p pd_count = %d\n", + ha, qlnx_rdma_dev, dev->pd_count); + + qlnxr_ib_dispatch_event(dev, QLNXR_PORT, + IB_EVENT_PORT_ERR); + + if (QLNX_IS_IWARP(dev)) { + if (dev->pd_count) + return (EBUSY); + } + + ib_unregister_device(&dev->ibdev); + + if (QLNX_IS_ROCE(dev)) { + if (dev->pd_count) + return (EBUSY); + } + + ecore_rdma_remove_user(dev->rdma_ctx, dev->dpi); + ecore_rdma_stop(dev->rdma_ctx); + + qlnxr_release_irqs(dev); + + qlnxr_free_resources(dev); + + qlnxr_remove_sysfiles(dev); + ib_dealloc_device(&dev->ibdev); + + QL_DPRINT12(ha, "exit ha = %p qlnx_rdma_dev = %p\n", ha, qlnx_rdma_dev); + return (0); +} + +int +qlnx_rdma_ll2_set_mac_filter(void *rdma_ctx, uint8_t *old_mac_address, + uint8_t *new_mac_address) +{ + struct ecore_hwfn *p_hwfn = rdma_ctx; + struct qlnx_host *ha; + int ret = 0; + + ha = (struct qlnx_host *)(p_hwfn->p_dev); + QL_DPRINT2(ha, "enter rdma_ctx (%p)\n", rdma_ctx); + + if (old_mac_address) + ecore_llh_remove_mac_filter(p_hwfn->p_dev, 0, old_mac_address); + + if (new_mac_address) + ret = ecore_llh_add_mac_filter(p_hwfn->p_dev, 0, new_mac_address); + + QL_DPRINT2(ha, "exit rdma_ctx (%p)\n", rdma_ctx); + return (ret); +} + +static void +qlnxr_mac_address_change(struct qlnxr_dev *dev) +{ + qlnx_host_t *ha; + + ha = dev->ha; + + QL_DPRINT12(ha, "enter/exit\n"); + + return; +} + +static void +qlnxr_notify(void *eth_dev, void *qlnx_rdma_dev, enum qlnx_rdma_event event) +{ + struct qlnxr_dev *dev; + qlnx_host_t *ha; + + dev = qlnx_rdma_dev; + + if (dev == NULL) + return; + + ha = dev->ha; + + QL_DPRINT12(ha, "enter (%p, %d)\n", qlnx_rdma_dev, event); + + switch (event) { + + case QLNX_ETHDEV_UP: + if (!test_and_set_bit(QLNXR_ENET_STATE_BIT, &dev->enet_state)) + qlnxr_ib_dispatch_event(dev, QLNXR_PORT, + IB_EVENT_PORT_ACTIVE); + break; + + case QLNX_ETHDEV_CHANGE_ADDR: + qlnxr_mac_address_change(dev); + break; + + case QLNX_ETHDEV_DOWN: + if (test_and_set_bit(QLNXR_ENET_STATE_BIT, &dev->enet_state)) + qlnxr_ib_dispatch_event(dev, QLNXR_PORT, + IB_EVENT_PORT_ERR); + break; + } + + QL_DPRINT12(ha, "exit (%p, %d)\n", qlnx_rdma_dev, event); + return; +} + +static int +qlnxr_mod_load(void) +{ + int ret; + + + qlnxr_drv.add = qlnxr_add; + qlnxr_drv.remove = qlnxr_remove; + qlnxr_drv.notify = qlnxr_notify; + + ret = qlnx_rdma_register_if(&qlnxr_drv); + + return (0); +} + +static int +qlnxr_mod_unload(void) +{ + int ret; + + ret = qlnx_rdma_deregister_if(&qlnxr_drv); + return (ret); +} + +static int +qlnxr_event_handler(module_t mod, int event, void *arg) +{ + + int ret = 0; + + switch (event) { + + case MOD_LOAD: + ret = qlnxr_mod_load(); + break; + + case MOD_UNLOAD: + ret = qlnxr_mod_unload(); + break; + + default: + break; + } + + return (ret); +} + +static moduledata_t qlnxr_mod_info = { + .name = "qlnxr", + .evhand = qlnxr_event_handler, +}; + +MODULE_VERSION(qlnxr, 1); +MODULE_DEPEND(qlnxr, if_qlnxe, 1, 1, 1); +MODULE_DEPEND(qlnxr, ibcore, 1, 1, 1); + +#if __FreeBSD_version >= 1100000 +MODULE_DEPEND(qlnxr, linuxkpi, 1, 1, 1); +#endif /* #if __FreeBSD_version >= 1100000 */ + +DECLARE_MODULE(qlnxr, qlnxr_mod_info, SI_SUB_LAST, SI_ORDER_ANY); + Property changes on: head/sys/dev/qlnx/qlnxr/qlnxr_os.c ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Added: svn:mime-type ## -0,0 +1 ## +text/plain \ No newline at end of property Index: head/sys/dev/qlnx/qlnxr/qlnxr_roce.h =================================================================== --- head/sys/dev/qlnx/qlnxr/qlnxr_roce.h (nonexistent) +++ head/sys/dev/qlnx/qlnxr/qlnxr_roce.h (revision 343598) @@ -0,0 +1,675 @@ +/* + * Copyright (c) 2018-2019 Cavium, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * + * $FreeBSD$ + * + */ + +#ifndef __QLNXR_ROCE_H__ +#define __QLNXR_ROCE_H__ + + +/* + * roce completion notification queue element + */ +struct roce_cnqe { + struct regpair cq_handle; +}; + + +struct roce_cqe_responder { + struct regpair srq_wr_id; + struct regpair qp_handle; + __le32 imm_data_or_inv_r_Key; + __le32 length; + __le32 reserved0; + __le16 rq_cons; + u8 flags; +#define ROCE_CQE_RESPONDER_TOGGLE_BIT_MASK 0x1 +#define ROCE_CQE_RESPONDER_TOGGLE_BIT_SHIFT 0 +#define ROCE_CQE_RESPONDER_TYPE_MASK 0x3 +#define ROCE_CQE_RESPONDER_TYPE_SHIFT 1 +#define ROCE_CQE_RESPONDER_INV_FLG_MASK 0x1 +#define ROCE_CQE_RESPONDER_INV_FLG_SHIFT 3 +#define ROCE_CQE_RESPONDER_IMM_FLG_MASK 0x1 +#define ROCE_CQE_RESPONDER_IMM_FLG_SHIFT 4 +#define ROCE_CQE_RESPONDER_RDMA_FLG_MASK 0x1 +#define ROCE_CQE_RESPONDER_RDMA_FLG_SHIFT 5 +#define ROCE_CQE_RESPONDER_RESERVED2_MASK 0x3 +#define ROCE_CQE_RESPONDER_RESERVED2_SHIFT 6 + u8 status; +}; + +struct roce_cqe_requester { + __le16 sq_cons; + __le16 reserved0; + __le32 reserved1; + struct regpair qp_handle; + struct regpair reserved2; + __le32 reserved3; + __le16 reserved4; + u8 flags; +#define ROCE_CQE_REQUESTER_TOGGLE_BIT_MASK 0x1 +#define ROCE_CQE_REQUESTER_TOGGLE_BIT_SHIFT 0 +#define ROCE_CQE_REQUESTER_TYPE_MASK 0x3 +#define ROCE_CQE_REQUESTER_TYPE_SHIFT 1 +#define ROCE_CQE_REQUESTER_RESERVED5_MASK 0x1F +#define ROCE_CQE_REQUESTER_RESERVED5_SHIFT 3 + u8 status; +}; + +struct roce_cqe_common { + struct regpair reserved0; + struct regpair qp_handle; + __le16 reserved1[7]; + u8 flags; +#define ROCE_CQE_COMMON_TOGGLE_BIT_MASK 0x1 +#define ROCE_CQE_COMMON_TOGGLE_BIT_SHIFT 0 +#define ROCE_CQE_COMMON_TYPE_MASK 0x3 +#define ROCE_CQE_COMMON_TYPE_SHIFT 1 +#define ROCE_CQE_COMMON_RESERVED2_MASK 0x1F +#define ROCE_CQE_COMMON_RESERVED2_SHIFT 3 + u8 status; +}; + +/* + * roce completion queue element + */ +union roce_cqe { + struct roce_cqe_responder resp; + struct roce_cqe_requester req; + struct roce_cqe_common cmn; +}; + + + + +/* + * CQE requester status enumeration + */ +enum roce_cqe_requester_status_enum { + ROCE_CQE_REQ_STS_OK, + ROCE_CQE_REQ_STS_BAD_RESPONSE_ERR, + ROCE_CQE_REQ_STS_LOCAL_LENGTH_ERR, + ROCE_CQE_REQ_STS_LOCAL_QP_OPERATION_ERR, + ROCE_CQE_REQ_STS_LOCAL_PROTECTION_ERR, + ROCE_CQE_REQ_STS_MEMORY_MGT_OPERATION_ERR, + ROCE_CQE_REQ_STS_REMOTE_INVALID_REQUEST_ERR, + ROCE_CQE_REQ_STS_REMOTE_ACCESS_ERR, + ROCE_CQE_REQ_STS_REMOTE_OPERATION_ERR, + ROCE_CQE_REQ_STS_RNR_NAK_RETRY_CNT_ERR, + ROCE_CQE_REQ_STS_TRANSPORT_RETRY_CNT_ERR, + ROCE_CQE_REQ_STS_WORK_REQUEST_FLUSHED_ERR, + MAX_ROCE_CQE_REQUESTER_STATUS_ENUM +}; + + + +/* + * CQE responder status enumeration + */ +enum roce_cqe_responder_status_enum { + ROCE_CQE_RESP_STS_OK, + ROCE_CQE_RESP_STS_LOCAL_ACCESS_ERR, + ROCE_CQE_RESP_STS_LOCAL_LENGTH_ERR, + ROCE_CQE_RESP_STS_LOCAL_QP_OPERATION_ERR, + ROCE_CQE_RESP_STS_LOCAL_PROTECTION_ERR, + ROCE_CQE_RESP_STS_MEMORY_MGT_OPERATION_ERR, + ROCE_CQE_RESP_STS_REMOTE_INVALID_REQUEST_ERR, + ROCE_CQE_RESP_STS_WORK_REQUEST_FLUSHED_ERR, + MAX_ROCE_CQE_RESPONDER_STATUS_ENUM +}; + + +/* + * CQE type enumeration + */ +enum roce_cqe_type { + ROCE_CQE_TYPE_REQUESTER, + ROCE_CQE_TYPE_RESPONDER_RQ, + ROCE_CQE_TYPE_RESPONDER_SRQ, + ROCE_CQE_TYPE_INVALID, + MAX_ROCE_CQE_TYPE +}; + + +/* + * memory window type enumeration + */ +enum roce_mw_type { + ROCE_MW_TYPE_1, + ROCE_MW_TYPE_2A, + MAX_ROCE_MW_TYPE +}; + + +struct roce_rq_sge { + struct regpair addr; + __le32 length; + __le32 flags; +#define ROCE_RQ_SGE_L_KEY_MASK 0x3FFFFFF +#define ROCE_RQ_SGE_L_KEY_SHIFT 0 +#define ROCE_RQ_SGE_NUM_SGES_MASK 0x7 +#define ROCE_RQ_SGE_NUM_SGES_SHIFT 26 +#define ROCE_RQ_SGE_RESERVED0_MASK 0x7 +#define ROCE_RQ_SGE_RESERVED0_SHIFT 29 +}; + + +struct roce_sq_atomic_wqe { + struct regpair remote_va; + __le32 xrc_srq; + u8 req_type; + u8 flags; +#define ROCE_SQ_ATOMIC_WQE_COMP_FLG_MASK 0x1 +#define ROCE_SQ_ATOMIC_WQE_COMP_FLG_SHIFT 0 +#define ROCE_SQ_ATOMIC_WQE_RD_FENCE_FLG_MASK 0x1 +#define ROCE_SQ_ATOMIC_WQE_RD_FENCE_FLG_SHIFT 1 +#define ROCE_SQ_ATOMIC_WQE_INV_FENCE_FLG_MASK 0x1 +#define ROCE_SQ_ATOMIC_WQE_INV_FENCE_FLG_SHIFT 2 +#define ROCE_SQ_ATOMIC_WQE_SE_FLG_MASK 0x1 +#define ROCE_SQ_ATOMIC_WQE_SE_FLG_SHIFT 3 +#define ROCE_SQ_ATOMIC_WQE_INLINE_FLG_MASK 0x1 +#define ROCE_SQ_ATOMIC_WQE_INLINE_FLG_SHIFT 4 +#define ROCE_SQ_ATOMIC_WQE_RESERVED0_MASK 0x7 +#define ROCE_SQ_ATOMIC_WQE_RESERVED0_SHIFT 5 + u8 reserved1; + u8 prev_wqe_size; + struct regpair swap_data; + __le32 r_key; + __le32 reserved2; + struct regpair cmp_data; + struct regpair reserved3; +}; + + +/* + * First element (16 bytes) of atomic wqe + */ +struct roce_sq_atomic_wqe_1st { + struct regpair remote_va; + __le32 xrc_srq; + u8 req_type; + u8 flags; +#define ROCE_SQ_ATOMIC_WQE_1ST_COMP_FLG_MASK 0x1 +#define ROCE_SQ_ATOMIC_WQE_1ST_COMP_FLG_SHIFT 0 +#define ROCE_SQ_ATOMIC_WQE_1ST_RD_FENCE_FLG_MASK 0x1 +#define ROCE_SQ_ATOMIC_WQE_1ST_RD_FENCE_FLG_SHIFT 1 +#define ROCE_SQ_ATOMIC_WQE_1ST_INV_FENCE_FLG_MASK 0x1 +#define ROCE_SQ_ATOMIC_WQE_1ST_INV_FENCE_FLG_SHIFT 2 +#define ROCE_SQ_ATOMIC_WQE_1ST_SE_FLG_MASK 0x1 +#define ROCE_SQ_ATOMIC_WQE_1ST_SE_FLG_SHIFT 3 +#define ROCE_SQ_ATOMIC_WQE_1ST_INLINE_FLG_MASK 0x1 +#define ROCE_SQ_ATOMIC_WQE_1ST_INLINE_FLG_SHIFT 4 +#define ROCE_SQ_ATOMIC_WQE_1ST_RESERVED0_MASK 0x7 +#define ROCE_SQ_ATOMIC_WQE_1ST_RESERVED0_SHIFT 5 + u8 reserved1; + u8 prev_wqe_size; +}; + + +/* + * Second element (16 bytes) of atomic wqe + */ +struct roce_sq_atomic_wqe_2nd { + struct regpair swap_data; + __le32 r_key; + __le32 reserved2; +}; + + +/* + * Third element (16 bytes) of atomic wqe + */ +struct roce_sq_atomic_wqe_3rd { + struct regpair cmp_data; + struct regpair reserved3; +}; + + +struct roce_sq_bind_wqe { + struct regpair addr; + __le32 l_key; + u8 req_type; + u8 flags; +#define ROCE_SQ_BIND_WQE_COMP_FLG_MASK 0x1 +#define ROCE_SQ_BIND_WQE_COMP_FLG_SHIFT 0 +#define ROCE_SQ_BIND_WQE_RD_FENCE_FLG_MASK 0x1 +#define ROCE_SQ_BIND_WQE_RD_FENCE_FLG_SHIFT 1 +#define ROCE_SQ_BIND_WQE_INV_FENCE_FLG_MASK 0x1 +#define ROCE_SQ_BIND_WQE_INV_FENCE_FLG_SHIFT 2 +#define ROCE_SQ_BIND_WQE_SE_FLG_MASK 0x1 +#define ROCE_SQ_BIND_WQE_SE_FLG_SHIFT 3 +#define ROCE_SQ_BIND_WQE_INLINE_FLG_MASK 0x1 +#define ROCE_SQ_BIND_WQE_INLINE_FLG_SHIFT 4 +#define ROCE_SQ_BIND_WQE_RESERVED0_MASK 0x7 +#define ROCE_SQ_BIND_WQE_RESERVED0_SHIFT 5 + u8 access_ctrl; +#define ROCE_SQ_BIND_WQE_REMOTE_READ_MASK 0x1 +#define ROCE_SQ_BIND_WQE_REMOTE_READ_SHIFT 0 +#define ROCE_SQ_BIND_WQE_REMOTE_WRITE_MASK 0x1 +#define ROCE_SQ_BIND_WQE_REMOTE_WRITE_SHIFT 1 +#define ROCE_SQ_BIND_WQE_ENABLE_ATOMIC_MASK 0x1 +#define ROCE_SQ_BIND_WQE_ENABLE_ATOMIC_SHIFT 2 +#define ROCE_SQ_BIND_WQE_LOCAL_READ_MASK 0x1 +#define ROCE_SQ_BIND_WQE_LOCAL_READ_SHIFT 3 +#define ROCE_SQ_BIND_WQE_LOCAL_WRITE_MASK 0x1 +#define ROCE_SQ_BIND_WQE_LOCAL_WRITE_SHIFT 4 +#define ROCE_SQ_BIND_WQE_RESERVED1_MASK 0x7 +#define ROCE_SQ_BIND_WQE_RESERVED1_SHIFT 5 + u8 prev_wqe_size; + u8 bind_ctrl; +#define ROCE_SQ_BIND_WQE_ZERO_BASED_MASK 0x1 +#define ROCE_SQ_BIND_WQE_ZERO_BASED_SHIFT 0 +#define ROCE_SQ_BIND_WQE_MW_TYPE_MASK 0x1 +#define ROCE_SQ_BIND_WQE_MW_TYPE_SHIFT 1 +#define ROCE_SQ_BIND_WQE_RESERVED2_MASK 0x3F +#define ROCE_SQ_BIND_WQE_RESERVED2_SHIFT 2 + u8 reserved3[2]; + u8 length_hi; + __le32 length_lo; + __le32 parent_l_key; + __le32 reserved6; +}; + + +/* + * First element (16 bytes) of bind wqe + */ +struct roce_sq_bind_wqe_1st { + struct regpair addr; + __le32 l_key; + u8 req_type; + u8 flags; +#define ROCE_SQ_BIND_WQE_1ST_COMP_FLG_MASK 0x1 +#define ROCE_SQ_BIND_WQE_1ST_COMP_FLG_SHIFT 0 +#define ROCE_SQ_BIND_WQE_1ST_RD_FENCE_FLG_MASK 0x1 +#define ROCE_SQ_BIND_WQE_1ST_RD_FENCE_FLG_SHIFT 1 +#define ROCE_SQ_BIND_WQE_1ST_INV_FENCE_FLG_MASK 0x1 +#define ROCE_SQ_BIND_WQE_1ST_INV_FENCE_FLG_SHIFT 2 +#define ROCE_SQ_BIND_WQE_1ST_SE_FLG_MASK 0x1 +#define ROCE_SQ_BIND_WQE_1ST_SE_FLG_SHIFT 3 +#define ROCE_SQ_BIND_WQE_1ST_INLINE_FLG_MASK 0x1 +#define ROCE_SQ_BIND_WQE_1ST_INLINE_FLG_SHIFT 4 +#define ROCE_SQ_BIND_WQE_1ST_RESERVED0_MASK 0x7 +#define ROCE_SQ_BIND_WQE_1ST_RESERVED0_SHIFT 5 + u8 access_ctrl; +#define ROCE_SQ_BIND_WQE_1ST_REMOTE_READ_MASK 0x1 +#define ROCE_SQ_BIND_WQE_1ST_REMOTE_READ_SHIFT 0 +#define ROCE_SQ_BIND_WQE_1ST_REMOTE_WRITE_MASK 0x1 +#define ROCE_SQ_BIND_WQE_1ST_REMOTE_WRITE_SHIFT 1 +#define ROCE_SQ_BIND_WQE_1ST_ENABLE_ATOMIC_MASK 0x1 +#define ROCE_SQ_BIND_WQE_1ST_ENABLE_ATOMIC_SHIFT 2 +#define ROCE_SQ_BIND_WQE_1ST_LOCAL_READ_MASK 0x1 +#define ROCE_SQ_BIND_WQE_1ST_LOCAL_READ_SHIFT 3 +#define ROCE_SQ_BIND_WQE_1ST_LOCAL_WRITE_MASK 0x1 +#define ROCE_SQ_BIND_WQE_1ST_LOCAL_WRITE_SHIFT 4 +#define ROCE_SQ_BIND_WQE_1ST_RESERVED1_MASK 0x7 +#define ROCE_SQ_BIND_WQE_1ST_RESERVED1_SHIFT 5 + u8 prev_wqe_size; +}; + + +/* + * Second element (16 bytes) of bind wqe + */ +struct roce_sq_bind_wqe_2nd { + u8 bind_ctrl; +#define ROCE_SQ_BIND_WQE_2ND_ZERO_BASED_MASK 0x1 +#define ROCE_SQ_BIND_WQE_2ND_ZERO_BASED_SHIFT 0 +#define ROCE_SQ_BIND_WQE_2ND_MW_TYPE_MASK 0x1 +#define ROCE_SQ_BIND_WQE_2ND_MW_TYPE_SHIFT 1 +#define ROCE_SQ_BIND_WQE_2ND_RESERVED2_MASK 0x3F +#define ROCE_SQ_BIND_WQE_2ND_RESERVED2_SHIFT 2 + u8 reserved3[2]; + u8 length_hi; + __le32 length_lo; + __le32 parent_l_key; + __le32 reserved6; +}; + + +/* + * Structure with only the SQ WQE common fields. Size is of one SQ element (16B) + */ +struct roce_sq_common_wqe { + __le32 reserved1[3]; + u8 req_type; + u8 flags; +#define ROCE_SQ_COMMON_WQE_COMP_FLG_MASK 0x1 +#define ROCE_SQ_COMMON_WQE_COMP_FLG_SHIFT 0 +#define ROCE_SQ_COMMON_WQE_RD_FENCE_FLG_MASK 0x1 +#define ROCE_SQ_COMMON_WQE_RD_FENCE_FLG_SHIFT 1 +#define ROCE_SQ_COMMON_WQE_INV_FENCE_FLG_MASK 0x1 +#define ROCE_SQ_COMMON_WQE_INV_FENCE_FLG_SHIFT 2 +#define ROCE_SQ_COMMON_WQE_SE_FLG_MASK 0x1 +#define ROCE_SQ_COMMON_WQE_SE_FLG_SHIFT 3 +#define ROCE_SQ_COMMON_WQE_INLINE_FLG_MASK 0x1 +#define ROCE_SQ_COMMON_WQE_INLINE_FLG_SHIFT 4 +#define ROCE_SQ_COMMON_WQE_RESERVED0_MASK 0x7 +#define ROCE_SQ_COMMON_WQE_RESERVED0_SHIFT 5 + u8 reserved2; + u8 prev_wqe_size; +}; + + +struct roce_sq_fmr_wqe { + struct regpair addr; + __le32 l_key; + u8 req_type; + u8 flags; +#define ROCE_SQ_FMR_WQE_COMP_FLG_MASK 0x1 +#define ROCE_SQ_FMR_WQE_COMP_FLG_SHIFT 0 +#define ROCE_SQ_FMR_WQE_RD_FENCE_FLG_MASK 0x1 +#define ROCE_SQ_FMR_WQE_RD_FENCE_FLG_SHIFT 1 +#define ROCE_SQ_FMR_WQE_INV_FENCE_FLG_MASK 0x1 +#define ROCE_SQ_FMR_WQE_INV_FENCE_FLG_SHIFT 2 +#define ROCE_SQ_FMR_WQE_SE_FLG_MASK 0x1 +#define ROCE_SQ_FMR_WQE_SE_FLG_SHIFT 3 +#define ROCE_SQ_FMR_WQE_INLINE_FLG_MASK 0x1 +#define ROCE_SQ_FMR_WQE_INLINE_FLG_SHIFT 4 +#define ROCE_SQ_FMR_WQE_RESERVED0_MASK 0x7 +#define ROCE_SQ_FMR_WQE_RESERVED0_SHIFT 5 + u8 access_ctrl; +#define ROCE_SQ_FMR_WQE_REMOTE_READ_MASK 0x1 +#define ROCE_SQ_FMR_WQE_REMOTE_READ_SHIFT 0 +#define ROCE_SQ_FMR_WQE_REMOTE_WRITE_MASK 0x1 +#define ROCE_SQ_FMR_WQE_REMOTE_WRITE_SHIFT 1 +#define ROCE_SQ_FMR_WQE_ENABLE_ATOMIC_MASK 0x1 +#define ROCE_SQ_FMR_WQE_ENABLE_ATOMIC_SHIFT 2 +#define ROCE_SQ_FMR_WQE_LOCAL_READ_MASK 0x1 +#define ROCE_SQ_FMR_WQE_LOCAL_READ_SHIFT 3 +#define ROCE_SQ_FMR_WQE_LOCAL_WRITE_MASK 0x1 +#define ROCE_SQ_FMR_WQE_LOCAL_WRITE_SHIFT 4 +#define ROCE_SQ_FMR_WQE_RESERVED1_MASK 0x7 +#define ROCE_SQ_FMR_WQE_RESERVED1_SHIFT 5 + u8 prev_wqe_size; + u8 fmr_ctrl; +#define ROCE_SQ_FMR_WQE_PAGE_SIZE_LOG_MASK 0x1F +#define ROCE_SQ_FMR_WQE_PAGE_SIZE_LOG_SHIFT 0 +#define ROCE_SQ_FMR_WQE_ZERO_BASED_MASK 0x1 +#define ROCE_SQ_FMR_WQE_ZERO_BASED_SHIFT 5 +#define ROCE_SQ_FMR_WQE_BIND_EN_MASK 0x1 +#define ROCE_SQ_FMR_WQE_BIND_EN_SHIFT 6 +#define ROCE_SQ_FMR_WQE_RESERVED2_MASK 0x1 +#define ROCE_SQ_FMR_WQE_RESERVED2_SHIFT 7 + u8 reserved3[2]; + u8 length_hi; + __le32 length_lo; + struct regpair pbl_addr; +}; + + +/* + * First element (16 bytes) of fmr wqe + */ +struct roce_sq_fmr_wqe_1st { + struct regpair addr; + __le32 l_key; + u8 req_type; + u8 flags; +#define ROCE_SQ_FMR_WQE_1ST_COMP_FLG_MASK 0x1 +#define ROCE_SQ_FMR_WQE_1ST_COMP_FLG_SHIFT 0 +#define ROCE_SQ_FMR_WQE_1ST_RD_FENCE_FLG_MASK 0x1 +#define ROCE_SQ_FMR_WQE_1ST_RD_FENCE_FLG_SHIFT 1 +#define ROCE_SQ_FMR_WQE_1ST_INV_FENCE_FLG_MASK 0x1 +#define ROCE_SQ_FMR_WQE_1ST_INV_FENCE_FLG_SHIFT 2 +#define ROCE_SQ_FMR_WQE_1ST_SE_FLG_MASK 0x1 +#define ROCE_SQ_FMR_WQE_1ST_SE_FLG_SHIFT 3 +#define ROCE_SQ_FMR_WQE_1ST_INLINE_FLG_MASK 0x1 +#define ROCE_SQ_FMR_WQE_1ST_INLINE_FLG_SHIFT 4 +#define ROCE_SQ_FMR_WQE_1ST_RESERVED0_MASK 0x7 +#define ROCE_SQ_FMR_WQE_1ST_RESERVED0_SHIFT 5 + u8 access_ctrl; +#define ROCE_SQ_FMR_WQE_1ST_REMOTE_READ_MASK 0x1 +#define ROCE_SQ_FMR_WQE_1ST_REMOTE_READ_SHIFT 0 +#define ROCE_SQ_FMR_WQE_1ST_REMOTE_WRITE_MASK 0x1 +#define ROCE_SQ_FMR_WQE_1ST_REMOTE_WRITE_SHIFT 1 +#define ROCE_SQ_FMR_WQE_1ST_ENABLE_ATOMIC_MASK 0x1 +#define ROCE_SQ_FMR_WQE_1ST_ENABLE_ATOMIC_SHIFT 2 +#define ROCE_SQ_FMR_WQE_1ST_LOCAL_READ_MASK 0x1 +#define ROCE_SQ_FMR_WQE_1ST_LOCAL_READ_SHIFT 3 +#define ROCE_SQ_FMR_WQE_1ST_LOCAL_WRITE_MASK 0x1 +#define ROCE_SQ_FMR_WQE_1ST_LOCAL_WRITE_SHIFT 4 +#define ROCE_SQ_FMR_WQE_1ST_RESERVED1_MASK 0x7 +#define ROCE_SQ_FMR_WQE_1ST_RESERVED1_SHIFT 5 + u8 prev_wqe_size; +}; + + +/* + * Second element (16 bytes) of fmr wqe + */ +struct roce_sq_fmr_wqe_2nd { + u8 fmr_ctrl; +#define ROCE_SQ_FMR_WQE_2ND_PAGE_SIZE_LOG_MASK 0x1F +#define ROCE_SQ_FMR_WQE_2ND_PAGE_SIZE_LOG_SHIFT 0 +#define ROCE_SQ_FMR_WQE_2ND_ZERO_BASED_MASK 0x1 +#define ROCE_SQ_FMR_WQE_2ND_ZERO_BASED_SHIFT 5 +#define ROCE_SQ_FMR_WQE_2ND_BIND_EN_MASK 0x1 +#define ROCE_SQ_FMR_WQE_2ND_BIND_EN_SHIFT 6 +#define ROCE_SQ_FMR_WQE_2ND_RESERVED2_MASK 0x1 +#define ROCE_SQ_FMR_WQE_2ND_RESERVED2_SHIFT 7 + u8 reserved3[2]; + u8 length_hi; + __le32 length_lo; + struct regpair pbl_addr; +}; + + +struct roce_sq_local_inv_wqe { + struct regpair reserved; + __le32 inv_l_key; + u8 req_type; + u8 flags; +#define ROCE_SQ_LOCAL_INV_WQE_COMP_FLG_MASK 0x1 +#define ROCE_SQ_LOCAL_INV_WQE_COMP_FLG_SHIFT 0 +#define ROCE_SQ_LOCAL_INV_WQE_RD_FENCE_FLG_MASK 0x1 +#define ROCE_SQ_LOCAL_INV_WQE_RD_FENCE_FLG_SHIFT 1 +#define ROCE_SQ_LOCAL_INV_WQE_INV_FENCE_FLG_MASK 0x1 +#define ROCE_SQ_LOCAL_INV_WQE_INV_FENCE_FLG_SHIFT 2 +#define ROCE_SQ_LOCAL_INV_WQE_SE_FLG_MASK 0x1 +#define ROCE_SQ_LOCAL_INV_WQE_SE_FLG_SHIFT 3 +#define ROCE_SQ_LOCAL_INV_WQE_INLINE_FLG_MASK 0x1 +#define ROCE_SQ_LOCAL_INV_WQE_INLINE_FLG_SHIFT 4 +#define ROCE_SQ_LOCAL_INV_WQE_RESERVED0_MASK 0x7 +#define ROCE_SQ_LOCAL_INV_WQE_RESERVED0_SHIFT 5 + u8 reserved1; + u8 prev_wqe_size; +}; + + +struct roce_sq_rdma_wqe { + __le32 imm_data; + __le32 length; + __le32 xrc_srq; + u8 req_type; + u8 flags; +#define ROCE_SQ_RDMA_WQE_COMP_FLG_MASK 0x1 +#define ROCE_SQ_RDMA_WQE_COMP_FLG_SHIFT 0 +#define ROCE_SQ_RDMA_WQE_RD_FENCE_FLG_MASK 0x1 +#define ROCE_SQ_RDMA_WQE_RD_FENCE_FLG_SHIFT 1 +#define ROCE_SQ_RDMA_WQE_INV_FENCE_FLG_MASK 0x1 +#define ROCE_SQ_RDMA_WQE_INV_FENCE_FLG_SHIFT 2 +#define ROCE_SQ_RDMA_WQE_SE_FLG_MASK 0x1 +#define ROCE_SQ_RDMA_WQE_SE_FLG_SHIFT 3 +#define ROCE_SQ_RDMA_WQE_INLINE_FLG_MASK 0x1 +#define ROCE_SQ_RDMA_WQE_INLINE_FLG_SHIFT 4 +#define ROCE_SQ_RDMA_WQE_RESERVED0_MASK 0x7 +#define ROCE_SQ_RDMA_WQE_RESERVED0_SHIFT 5 + u8 wqe_size; + u8 prev_wqe_size; + struct regpair remote_va; + __le32 r_key; + __le32 reserved1; +}; + + +/* + * First element (16 bytes) of rdma wqe + */ +struct roce_sq_rdma_wqe_1st { + __le32 imm_data; + __le32 length; + __le32 xrc_srq; + u8 req_type; + u8 flags; +#define ROCE_SQ_RDMA_WQE_1ST_COMP_FLG_MASK 0x1 +#define ROCE_SQ_RDMA_WQE_1ST_COMP_FLG_SHIFT 0 +#define ROCE_SQ_RDMA_WQE_1ST_RD_FENCE_FLG_MASK 0x1 +#define ROCE_SQ_RDMA_WQE_1ST_RD_FENCE_FLG_SHIFT 1 +#define ROCE_SQ_RDMA_WQE_1ST_INV_FENCE_FLG_MASK 0x1 +#define ROCE_SQ_RDMA_WQE_1ST_INV_FENCE_FLG_SHIFT 2 +#define ROCE_SQ_RDMA_WQE_1ST_SE_FLG_MASK 0x1 +#define ROCE_SQ_RDMA_WQE_1ST_SE_FLG_SHIFT 3 +#define ROCE_SQ_RDMA_WQE_1ST_INLINE_FLG_MASK 0x1 +#define ROCE_SQ_RDMA_WQE_1ST_INLINE_FLG_SHIFT 4 +#define ROCE_SQ_RDMA_WQE_1ST_RESERVED0_MASK 0x7 +#define ROCE_SQ_RDMA_WQE_1ST_RESERVED0_SHIFT 5 + u8 wqe_size; + u8 prev_wqe_size; +}; + + +/* + * Second element (16 bytes) of rdma wqe + */ +struct roce_sq_rdma_wqe_2nd { + struct regpair remote_va; + __le32 r_key; + __le32 reserved1; +}; + + +/* + * SQ WQE req type enumeration + */ +enum roce_sq_req_type { + ROCE_SQ_REQ_TYPE_SEND, + ROCE_SQ_REQ_TYPE_SEND_WITH_IMM, + ROCE_SQ_REQ_TYPE_SEND_WITH_INVALIDATE, + ROCE_SQ_REQ_TYPE_RDMA_WR, + ROCE_SQ_REQ_TYPE_RDMA_WR_WITH_IMM, + ROCE_SQ_REQ_TYPE_RDMA_RD, + ROCE_SQ_REQ_TYPE_ATOMIC_CMP_AND_SWAP, + ROCE_SQ_REQ_TYPE_ATOMIC_ADD, + ROCE_SQ_REQ_TYPE_LOCAL_INVALIDATE, + ROCE_SQ_REQ_TYPE_FAST_MR, + ROCE_SQ_REQ_TYPE_BIND, + ROCE_SQ_REQ_TYPE_INVALID, + MAX_ROCE_SQ_REQ_TYPE +}; + + +struct roce_sq_send_wqe { + __le32 inv_key_or_imm_data; + __le32 length; + __le32 xrc_srq; + u8 req_type; + u8 flags; +#define ROCE_SQ_SEND_WQE_COMP_FLG_MASK 0x1 +#define ROCE_SQ_SEND_WQE_COMP_FLG_SHIFT 0 +#define ROCE_SQ_SEND_WQE_RD_FENCE_FLG_MASK 0x1 +#define ROCE_SQ_SEND_WQE_RD_FENCE_FLG_SHIFT 1 +#define ROCE_SQ_SEND_WQE_INV_FENCE_FLG_MASK 0x1 +#define ROCE_SQ_SEND_WQE_INV_FENCE_FLG_SHIFT 2 +#define ROCE_SQ_SEND_WQE_SE_FLG_MASK 0x1 +#define ROCE_SQ_SEND_WQE_SE_FLG_SHIFT 3 +#define ROCE_SQ_SEND_WQE_INLINE_FLG_MASK 0x1 +#define ROCE_SQ_SEND_WQE_INLINE_FLG_SHIFT 4 +#define ROCE_SQ_SEND_WQE_RESERVED0_MASK 0x7 +#define ROCE_SQ_SEND_WQE_RESERVED0_SHIFT 5 + u8 wqe_size; + u8 prev_wqe_size; +}; + + +struct roce_sq_sge { + __le32 length; + struct regpair addr; + __le32 l_key; +}; + + +struct roce_srq_prod { + __le16 prod; +}; + + +struct roce_srq_sge { + struct regpair addr; + __le32 length; + __le32 l_key; + struct regpair wr_id; + u8 flags; +#define ROCE_SRQ_SGE_NUM_SGES_MASK 0x3 +#define ROCE_SRQ_SGE_NUM_SGES_SHIFT 0 +#define ROCE_SRQ_SGE_RESERVED0_MASK 0x3F +#define ROCE_SRQ_SGE_RESERVED0_SHIFT 2 + u8 reserved1; + __le16 reserved2; + __le32 reserved3; +}; + + +/* + * RoCE doorbell data for SQ and RQ + */ +struct roce_pwm_val16_data { + __le16 icid; + __le16 prod_val; +}; + + +union roce_pwm_val16_data_union { + struct roce_pwm_val16_data as_struct; + __le32 as_dword; +}; + + +/* + * RoCE doorbell data for CQ + */ +struct roce_pwm_val32_data { + __le16 icid; + u8 agg_flags; + u8 params; +#define ROCE_PWM_VAL32_DATA_AGG_CMD_MASK 0x3 +#define ROCE_PWM_VAL32_DATA_AGG_CMD_SHIFT 0 +#define ROCE_PWM_VAL32_DATA_BYPASS_EN_MASK 0x1 +#define ROCE_PWM_VAL32_DATA_BYPASS_EN_SHIFT 2 +#define ROCE_PWM_VAL32_DATA_RESERVED_MASK 0x1F +#define ROCE_PWM_VAL32_DATA_RESERVED_SHIFT 3 + __le32 cq_cons_val; +}; + + +union roce_pwm_val32_data_union { + struct roce_pwm_val32_data as_struct; + struct regpair as_repair; +}; + +#endif /* __QLNXR_ROCE_H__ */ Property changes on: head/sys/dev/qlnx/qlnxr/qlnxr_roce.h ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Added: svn:mime-type ## -0,0 +1 ## +text/plain \ No newline at end of property Index: head/sys/dev/qlnx/qlnxr/qlnxr_user.h =================================================================== --- head/sys/dev/qlnx/qlnxr/qlnxr_user.h (nonexistent) +++ head/sys/dev/qlnx/qlnxr/qlnxr_user.h (revision 343598) @@ -0,0 +1,112 @@ +/* + * Copyright (c) 2018-2019 Cavium, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * + * $FreeBSD$ + * + */ + +#ifndef __QLNXR_USER_H__ +#define __QLNXR_USER_H__ + +#define QLNXR_ABI_VERSION (7) +#define QLNXR_BE_ROCE_ABI_VERSION (1) + +/* user kernel communication data structures. */ + +struct qlnxr_alloc_ucontext_resp { + u64 db_pa; + u32 db_size; + + uint32_t max_send_wr; + uint32_t max_recv_wr; + uint32_t max_srq_wr; + uint32_t sges_per_send_wr; + uint32_t sges_per_recv_wr; + uint32_t sges_per_srq_wr; + int max_cqes; + uint8_t dpm_enabled; + uint8_t wids_enabled; + uint16_t wid_count; +}; + +struct qlnxr_alloc_pd_ureq { + u64 rsvd1; +}; + +struct qlnxr_alloc_pd_uresp { + u32 pd_id; +}; + +struct qlnxr_create_cq_ureq { + uint64_t addr; /* user space virtual address of CQ buffer */ + size_t len; /* size of CQ buffer */ +}; + +struct qlnxr_create_cq_uresp { + u32 db_offset; + u16 icid; +}; + +struct qlnxr_create_qp_ureq { + u32 qp_handle_hi; + u32 qp_handle_lo; + + /* SQ */ + uint64_t sq_addr; /* user space virtual address of SQ buffer */ + size_t sq_len; /* length of SQ buffer */ + + /* RQ */ + uint64_t rq_addr; /* user space virtual address of RQ buffer */ + size_t rq_len; /* length of RQ buffer */ +}; + +struct qlnxr_create_qp_uresp { + u32 qp_id; + int atomic_supported; + + /* SQ*/ + u32 sq_db_offset; + u16 sq_icid; + + /* RQ */ + u32 rq_db_offset; + u16 rq_icid; + + u32 rq_db2_offset; +}; + +struct qlnxr_create_srq_ureq { + /* user space virtual address of producer pair */ + uint64_t prod_pair_addr; + uint64_t srq_addr; /* user space virtual address of SQ buffer */ + size_t srq_len; /* length of SQ buffer */ +}; + +struct qlnxr_create_srq_uresp { + u16 srq_id; +}; + +#endif /* #ifndef __QLNXR_USER_H__ */ Property changes on: head/sys/dev/qlnx/qlnxr/qlnxr_user.h ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Added: svn:mime-type ## -0,0 +1 ## +text/plain \ No newline at end of property Index: head/sys/dev/qlnx/qlnxr/qlnxr_verbs.c =================================================================== --- head/sys/dev/qlnx/qlnxr/qlnxr_verbs.c (nonexistent) +++ head/sys/dev/qlnx/qlnxr/qlnxr_verbs.c (revision 343598) @@ -0,0 +1,7306 @@ +/* + * Copyright (c) 2018-2019 Cavium, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + + +/* + * File: qlnxr_verbs.c + */ +#include +__FBSDID("$FreeBSD$"); + +#include "qlnxr_def.h" +#include "rdma_common.h" +#include "qlnxr_roce.h" +#include "qlnxr_cm.h" + +#define upper_32_bits(x) (uint32_t)(x >> 32) +#define lower_32_bits(x) (uint32_t)(x) +#define HILO_U64(hi, lo) ((((u64)(hi)) << 32) + (lo)) + +#define TYPEPTR_ADDR_SET(type_ptr, field, vaddr) \ + do { \ + (type_ptr)->field.hi = cpu_to_le32(upper_32_bits(vaddr));\ + (type_ptr)->field.lo = cpu_to_le32(lower_32_bits(vaddr));\ + } while (0) + + +#define RQ_SGE_SET(sge, vaddr, vlength, vflags) \ + do { \ + TYPEPTR_ADDR_SET(sge, addr, vaddr); \ + (sge)->length = cpu_to_le32(vlength); \ + (sge)->flags = cpu_to_le32(vflags); \ + } while (0) + +#define SRQ_HDR_SET(hdr, vwr_id, num_sge) \ + do { \ + TYPEPTR_ADDR_SET(hdr, wr_id, vwr_id); \ + (hdr)->num_sges = num_sge; \ + } while (0) + +#define SRQ_SGE_SET(sge, vaddr, vlength, vlkey) \ + do { \ + TYPEPTR_ADDR_SET(sge, addr, vaddr); \ + (sge)->length = cpu_to_le32(vlength); \ + (sge)->l_key = cpu_to_le32(vlkey); \ + } while (0) + +#define NIPQUAD(addr) \ + ((unsigned char *)&addr)[0], \ + ((unsigned char *)&addr)[1], \ + ((unsigned char *)&addr)[2], \ + ((unsigned char *)&addr)[3] + +struct ib_srq *qlnxr_create_srq(struct ib_pd *, + struct ib_srq_init_attr *, + struct ib_udata *); + +int qlnxr_destroy_srq(struct ib_srq *); + +int qlnxr_modify_srq(struct ib_srq *, + struct ib_srq_attr *, + enum ib_srq_attr_mask, + struct ib_udata *); +static int +qlnxr_check_srq_params(struct ib_pd *ibpd, + struct qlnxr_dev *dev, + struct ib_srq_init_attr *attrs); + +static int +qlnxr_init_srq_user_params(struct ib_ucontext *ib_ctx, + struct qlnxr_srq *srq, + struct qlnxr_create_srq_ureq *ureq, + int access, int dmasync); + +static int +qlnxr_alloc_srq_kernel_params(struct qlnxr_srq *srq, + struct qlnxr_dev *dev, + struct ib_srq_init_attr *init_attr); + +extern enum _ecore_status_t +ecore_rdma_modify_srq(void *rdma_cxt, + struct ecore_rdma_modify_srq_in_params *in_params); + +extern enum _ecore_status_t +ecore_rdma_destroy_srq(void *rdma_cxt, + struct ecore_rdma_destroy_srq_in_params *in_params); + +extern enum _ecore_status_t +ecore_rdma_create_srq(void *rdma_cxt, + struct ecore_rdma_create_srq_in_params *in_params, + struct ecore_rdma_create_srq_out_params *out_params); + + +static int +qlnxr_copy_srq_uresp(struct qlnxr_dev *dev, + struct qlnxr_srq *srq, + struct ib_udata *udata); + +static void +qlnxr_free_srq_user_params(struct qlnxr_srq *srq); + +static void +qlnxr_free_srq_kernel_params(struct qlnxr_srq *srq); + + +static u32 +qlnxr_srq_elem_left(struct qlnxr_srq_hwq_info *hw_srq); + +int +qlnxr_iw_query_gid(struct ib_device *ibdev, u8 port, int index, + union ib_gid *sgid) +{ + struct qlnxr_dev *dev; + qlnx_host_t *ha; + + dev = get_qlnxr_dev(ibdev); + ha = dev->ha; + + QL_DPRINT12(ha, "enter\n"); + + memset(sgid->raw, 0, sizeof(sgid->raw)); + + memcpy(sgid->raw, dev->ha->primary_mac, sizeof (dev->ha->primary_mac)); + + QL_DPRINT12(ha, "exit\n"); + + return 0; +} + +int +qlnxr_query_gid(struct ib_device *ibdev, u8 port, int index, + union ib_gid *sgid) +{ + struct qlnxr_dev *dev; + qlnx_host_t *ha; + + dev = get_qlnxr_dev(ibdev); + ha = dev->ha; + QL_DPRINT12(ha, "enter index: %d\n", index); +#if 0 + int ret = 0; + /* @@@: if DEFINE_ROCE_GID_TABLE to be used here */ + //if (!rdma_cap_roce_gid_table(ibdev, port)) { + if (!(rdma_protocol_roce(ibdev, port) && + ibdev->add_gid && ibdev->del_gid)) { + QL_DPRINT11(ha, "acquire gid failed\n"); + return -ENODEV; + } + + ret = ib_get_cached_gid(ibdev, port, index, sgid, NULL); + if (ret == -EAGAIN) { + memcpy(sgid, &zgid, sizeof(*sgid)); + return 0; + } +#endif + if ((index >= QLNXR_MAX_SGID) || (index < 0)) { + QL_DPRINT12(ha, "invalid gid index %d\n", index); + memset(sgid, 0, sizeof(*sgid)); + return -EINVAL; + } + memcpy(sgid, &dev->sgid_tbl[index], sizeof(*sgid)); + + QL_DPRINT12(ha, "exit : %p\n", sgid); + + return 0; +} + +struct ib_srq * +qlnxr_create_srq(struct ib_pd *ibpd, struct ib_srq_init_attr *init_attr, + struct ib_udata *udata) +{ + struct qlnxr_dev *dev; + qlnx_host_t *ha; + struct ecore_rdma_destroy_srq_in_params destroy_in_params; + struct ecore_rdma_create_srq_out_params out_params; + struct ecore_rdma_create_srq_in_params in_params; + u64 pbl_base_addr, phy_prod_pair_addr; + struct qlnxr_pd *pd = get_qlnxr_pd(ibpd); + struct ib_ucontext *ib_ctx = NULL; + struct qlnxr_srq_hwq_info *hw_srq; + struct qlnxr_ucontext *ctx = NULL; + struct qlnxr_create_srq_ureq ureq; + u32 page_cnt, page_size; + struct qlnxr_srq *srq; + int ret = 0; + + dev = get_qlnxr_dev((ibpd->device)); + ha = dev->ha; + + QL_DPRINT12(ha, "enter\n"); + + ret = qlnxr_check_srq_params(ibpd, dev, init_attr); + + srq = kzalloc(sizeof(*srq), GFP_KERNEL); + if (!srq) { + QL_DPRINT11(ha, "cannot allocate memory for srq\n"); + return NULL; //@@@ : TODO what to return here? + } + + srq->dev = dev; + hw_srq = &srq->hw_srq; + spin_lock_init(&srq->lock); + memset(&in_params, 0, sizeof(in_params)); + + if (udata && ibpd->uobject && ibpd->uobject->context) { + ib_ctx = ibpd->uobject->context; + ctx = get_qlnxr_ucontext(ib_ctx); + + memset(&ureq, 0, sizeof(ureq)); + if (ib_copy_from_udata(&ureq, udata, min(sizeof(ureq), + udata->inlen))) { + QL_DPRINT11(ha, "problem" + " copying data from user space\n"); + goto err0; + } + + ret = qlnxr_init_srq_user_params(ib_ctx, srq, &ureq, 0, 0); + if (ret) + goto err0; + + page_cnt = srq->usrq.pbl_info.num_pbes; + pbl_base_addr = srq->usrq.pbl_tbl->pa; + phy_prod_pair_addr = hw_srq->phy_prod_pair_addr; + // @@@ : if DEFINE_IB_UMEM_PAGE_SHIFT + // page_size = BIT(srq->usrq.umem->page_shift); + // else + page_size = srq->usrq.umem->page_size; + } else { + struct ecore_chain *pbl; + ret = qlnxr_alloc_srq_kernel_params(srq, dev, init_attr); + if (ret) + goto err0; + pbl = &hw_srq->pbl; + + page_cnt = ecore_chain_get_page_cnt(pbl); + pbl_base_addr = ecore_chain_get_pbl_phys(pbl); + phy_prod_pair_addr = hw_srq->phy_prod_pair_addr; + page_size = pbl->elem_per_page << 4; + } + + in_params.pd_id = pd->pd_id; + in_params.pbl_base_addr = pbl_base_addr; + in_params.prod_pair_addr = phy_prod_pair_addr; + in_params.num_pages = page_cnt; + in_params.page_size = page_size; + + ret = ecore_rdma_create_srq(dev->rdma_ctx, &in_params, &out_params); + if (ret) + goto err1; + + srq->srq_id = out_params.srq_id; + + if (udata) { + ret = qlnxr_copy_srq_uresp(dev, srq, udata); + if (ret) + goto err2; + } + + QL_DPRINT12(ha, "created srq with srq_id = 0x%0x\n", srq->srq_id); + return &srq->ibsrq; +err2: + memset(&in_params, 0, sizeof(in_params)); + destroy_in_params.srq_id = srq->srq_id; + ecore_rdma_destroy_srq(dev->rdma_ctx, &destroy_in_params); + +err1: + if (udata) + qlnxr_free_srq_user_params(srq); + else + qlnxr_free_srq_kernel_params(srq); + +err0: + kfree(srq); + return ERR_PTR(-EFAULT); +} + +int +qlnxr_destroy_srq(struct ib_srq *ibsrq) +{ + struct qlnxr_dev *dev; + struct qlnxr_srq *srq; + qlnx_host_t *ha; + struct ecore_rdma_destroy_srq_in_params in_params; + + srq = get_qlnxr_srq(ibsrq); + dev = srq->dev; + ha = dev->ha; + + memset(&in_params, 0, sizeof(in_params)); + in_params.srq_id = srq->srq_id; + + ecore_rdma_destroy_srq(dev->rdma_ctx, &in_params); + + if (ibsrq->pd->uobject && ibsrq->pd->uobject->context) + qlnxr_free_srq_user_params(srq); + else + qlnxr_free_srq_kernel_params(srq); + + QL_DPRINT12(ha, "destroyed srq_id=0x%0x\n", srq->srq_id); + kfree(srq); + return 0; +} + +int +qlnxr_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr, + enum ib_srq_attr_mask attr_mask, struct ib_udata *udata) +{ + struct qlnxr_dev *dev; + struct qlnxr_srq *srq; + qlnx_host_t *ha; + struct ecore_rdma_modify_srq_in_params in_params; + int ret = 0; + + srq = get_qlnxr_srq(ibsrq); + dev = srq->dev; + ha = dev->ha; + + QL_DPRINT12(ha, "enter\n"); + if (attr_mask & IB_SRQ_MAX_WR) { + QL_DPRINT12(ha, "invalid attribute mask=0x%x" + " specified for %p\n", attr_mask, srq); + return -EINVAL; + } + + if (attr_mask & IB_SRQ_LIMIT) { + if (attr->srq_limit >= srq->hw_srq.max_wr) { + QL_DPRINT12(ha, "invalid srq_limit=0x%x" + " (max_srq_limit = 0x%x)\n", + attr->srq_limit, srq->hw_srq.max_wr); + return -EINVAL; + } + memset(&in_params, 0, sizeof(in_params)); + in_params.srq_id = srq->srq_id; + in_params.wqe_limit = attr->srq_limit; + ret = ecore_rdma_modify_srq(dev->rdma_ctx, &in_params); + if (ret) + return ret; + } + + QL_DPRINT12(ha, "modified srq with srq_id = 0x%0x\n", srq->srq_id); + return 0; +} + +int +qlnxr_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *srq_attr) +{ + struct qlnxr_dev *dev; + struct qlnxr_srq *srq; + qlnx_host_t *ha; + struct ecore_rdma_device *qattr; + srq = get_qlnxr_srq(ibsrq); + dev = srq->dev; + ha = dev->ha; + //qattr = &dev->attr; + qattr = ecore_rdma_query_device(dev->rdma_ctx); + QL_DPRINT12(ha, "enter\n"); + + if (!dev->rdma_ctx) { + QL_DPRINT12(ha, "called with invalid params" + " rdma_ctx is NULL\n"); + return -EINVAL; + } + + srq_attr->srq_limit = qattr->max_srq; + srq_attr->max_wr = qattr->max_srq_wr; + srq_attr->max_sge = qattr->max_sge; + + QL_DPRINT12(ha, "exit\n"); + return 0; +} + +/* Increment srq wr producer by one */ +static +void qlnxr_inc_srq_wr_prod (struct qlnxr_srq_hwq_info *info) +{ + info->wr_prod_cnt++; +} + +/* Increment srq wr consumer by one */ +static +void qlnxr_inc_srq_wr_cons(struct qlnxr_srq_hwq_info *info) +{ + info->wr_cons_cnt++; +} + +/* get_port_immutable verb is not available in FreeBSD */ +#if 0 +int +qlnxr_roce_port_immutable(struct ib_device *ibdev, u8 port_num, + struct ib_port_immutable *immutable) +{ + struct qlnxr_dev *dev; + qlnx_host_t *ha; + dev = get_qlnxr_dev(ibdev); + ha = dev->ha; + + QL_DPRINT12(ha, "entered but not implemented!!!\n"); +} +#endif + +int +qlnxr_post_srq_recv(struct ib_srq *ibsrq, struct ib_recv_wr *wr, + struct ib_recv_wr **bad_wr) +{ + struct qlnxr_dev *dev; + struct qlnxr_srq *srq; + qlnx_host_t *ha; + struct qlnxr_srq_hwq_info *hw_srq; + struct ecore_chain *pbl; + unsigned long flags; + int status = 0; + u32 num_sge, offset; + + srq = get_qlnxr_srq(ibsrq); + dev = srq->dev; + ha = dev->ha; + hw_srq = &srq->hw_srq; + + QL_DPRINT12(ha, "enter\n"); + spin_lock_irqsave(&srq->lock, flags); + + pbl = &srq->hw_srq.pbl; + while (wr) { + struct rdma_srq_wqe_header *hdr; + int i; + + if (!qlnxr_srq_elem_left(hw_srq) || + wr->num_sge > srq->hw_srq.max_sges) { + QL_DPRINT11(ha, "WR cannot be posted" + " (%d, %d) || (%d > %d)\n", + hw_srq->wr_prod_cnt, hw_srq->wr_cons_cnt, + wr->num_sge, srq->hw_srq.max_sges); + status = -ENOMEM; + *bad_wr = wr; + break; + } + + hdr = ecore_chain_produce(pbl); + num_sge = wr->num_sge; + /* Set number of sge and WR id in header */ + SRQ_HDR_SET(hdr, wr->wr_id, num_sge); + + /* PBL is maintained in case of WR granularity. + * So increment WR producer in case we post a WR. + */ + qlnxr_inc_srq_wr_prod(hw_srq); + hw_srq->wqe_prod++; + hw_srq->sge_prod++; + + QL_DPRINT12(ha, "SRQ WR : SGEs: %d with wr_id[%d] = %llx\n", + wr->num_sge, hw_srq->wqe_prod, wr->wr_id); + + for (i = 0; i < wr->num_sge; i++) { + struct rdma_srq_sge *srq_sge = + ecore_chain_produce(pbl); + /* Set SGE length, lkey and address */ + SRQ_SGE_SET(srq_sge, wr->sg_list[i].addr, + wr->sg_list[i].length, wr->sg_list[i].lkey); + + QL_DPRINT12(ha, "[%d]: len %d, key %x, addr %x:%x\n", + i, srq_sge->length, srq_sge->l_key, + srq_sge->addr.hi, srq_sge->addr.lo); + hw_srq->sge_prod++; + } + wmb(); + /* + * SRQ prod is 8 bytes. Need to update SGE prod in index + * in first 4 bytes and need to update WQE prod in next + * 4 bytes. + */ + *(srq->hw_srq.virt_prod_pair_addr) = hw_srq->sge_prod; + offset = offsetof(struct rdma_srq_producers, wqe_prod); + *((u8 *)srq->hw_srq.virt_prod_pair_addr + offset) = + hw_srq->wqe_prod; + /* Flush prod after updating it */ + wmb(); + wr = wr->next; + } + + QL_DPRINT12(ha, "Elements in SRQ: %d\n", + ecore_chain_get_elem_left(pbl)); + + spin_unlock_irqrestore(&srq->lock, flags); + QL_DPRINT12(ha, "exit\n"); + return status; +} + +int +#if __FreeBSD_version < 1102000 +qlnxr_query_device(struct ib_device *ibdev, struct ib_device_attr *attr) +#else +qlnxr_query_device(struct ib_device *ibdev, struct ib_device_attr *attr, + struct ib_udata *udata) +#endif /* #if __FreeBSD_version < 1102000 */ + +{ + struct qlnxr_dev *dev; + struct ecore_rdma_device *qattr; + qlnx_host_t *ha; + + dev = get_qlnxr_dev(ibdev); + ha = dev->ha; + + QL_DPRINT12(ha, "enter\n"); + +#if __FreeBSD_version > 1102000 + if (udata->inlen || udata->outlen) + return -EINVAL; +#endif /* #if __FreeBSD_version > 1102000 */ + + if (dev->rdma_ctx == NULL) { + return -EINVAL; + } + + qattr = ecore_rdma_query_device(dev->rdma_ctx); + + memset(attr, 0, sizeof *attr); + + attr->fw_ver = qattr->fw_ver; + attr->sys_image_guid = qattr->sys_image_guid; + attr->max_mr_size = qattr->max_mr_size; + attr->page_size_cap = qattr->page_size_caps; + attr->vendor_id = qattr->vendor_id; + attr->vendor_part_id = qattr->vendor_part_id; + attr->hw_ver = qattr->hw_ver; + attr->max_qp = qattr->max_qp; + attr->device_cap_flags = IB_DEVICE_CURR_QP_STATE_MOD | + IB_DEVICE_RC_RNR_NAK_GEN | + IB_DEVICE_LOCAL_DMA_LKEY | + IB_DEVICE_MEM_MGT_EXTENSIONS; + + attr->max_sge = qattr->max_sge; + attr->max_sge_rd = qattr->max_sge; + attr->max_cq = qattr->max_cq; + attr->max_cqe = qattr->max_cqe; + attr->max_mr = qattr->max_mr; + attr->max_mw = qattr->max_mw; + attr->max_pd = qattr->max_pd; + attr->atomic_cap = dev->atomic_cap; + attr->max_fmr = qattr->max_fmr; + attr->max_map_per_fmr = 16; /* TBD: FMR */ + + /* There is an implicit assumption in some of the ib_xxx apps that the + * qp_rd_atom is smaller than the qp_init_rd_atom. Specifically, in + * communication the qp_rd_atom is passed to the other side and used as + * init_rd_atom without check device capabilities for init_rd_atom. + * for this reason, we set the qp_rd_atom to be the minimum between the + * two...There is an additional assumption in mlx4 driver that the + * values are power of two, fls is performed on the value - 1, which + * in fact gives a larger power of two for values which are not a power + * of two. This should be fixed in mlx4 driver, but until then -> + * we provide a value that is a power of two in our code. + */ + attr->max_qp_init_rd_atom = + 1 << (fls(qattr->max_qp_req_rd_atomic_resc) - 1); + attr->max_qp_rd_atom = + min(1 << (fls(qattr->max_qp_resp_rd_atomic_resc) - 1), + attr->max_qp_init_rd_atom); + + attr->max_srq = qattr->max_srq; + attr->max_srq_sge = qattr->max_srq_sge; + attr->max_srq_wr = qattr->max_srq_wr; + + /* TODO: R&D to more properly configure the following */ + attr->local_ca_ack_delay = qattr->dev_ack_delay; + attr->max_fast_reg_page_list_len = qattr->max_mr/8; + attr->max_pkeys = QLNXR_ROCE_PKEY_MAX; + attr->max_ah = qattr->max_ah; + + QL_DPRINT12(ha, "exit\n"); + return 0; +} + +static inline void +get_link_speed_and_width(int speed, uint8_t *ib_speed, uint8_t *ib_width) +{ + switch (speed) { + case 1000: + *ib_speed = IB_SPEED_SDR; + *ib_width = IB_WIDTH_1X; + break; + case 10000: + *ib_speed = IB_SPEED_QDR; + *ib_width = IB_WIDTH_1X; + break; + + case 20000: + *ib_speed = IB_SPEED_DDR; + *ib_width = IB_WIDTH_4X; + break; + + case 25000: + *ib_speed = IB_SPEED_EDR; + *ib_width = IB_WIDTH_1X; + break; + + case 40000: + *ib_speed = IB_SPEED_QDR; + *ib_width = IB_WIDTH_4X; + break; + + case 50000: + *ib_speed = IB_SPEED_QDR; + *ib_width = IB_WIDTH_4X; // TODO doesn't add up to 50... + break; + + case 100000: + *ib_speed = IB_SPEED_EDR; + *ib_width = IB_WIDTH_4X; + break; + + default: + /* Unsupported */ + *ib_speed = IB_SPEED_SDR; + *ib_width = IB_WIDTH_1X; + } + return; +} + +int +qlnxr_query_port(struct ib_device *ibdev, uint8_t port, + struct ib_port_attr *attr) +{ + struct qlnxr_dev *dev; + struct ecore_rdma_port *rdma_port; + qlnx_host_t *ha; + + dev = get_qlnxr_dev(ibdev); + ha = dev->ha; + + QL_DPRINT12(ha, "enter\n"); + + if (port > 1) { + QL_DPRINT12(ha, "port [%d] > 1 \n", port); + return -EINVAL; + } + + if (dev->rdma_ctx == NULL) { + QL_DPRINT12(ha, "rdma_ctx == NULL\n"); + return -EINVAL; + } + + rdma_port = ecore_rdma_query_port(dev->rdma_ctx); + memset(attr, 0, sizeof *attr); + + if (rdma_port->port_state == ECORE_RDMA_PORT_UP) { + attr->state = IB_PORT_ACTIVE; + attr->phys_state = 5; + } else { + attr->state = IB_PORT_DOWN; + attr->phys_state = 3; + } + + attr->max_mtu = IB_MTU_4096; + attr->active_mtu = iboe_get_mtu(dev->ha->ifp->if_mtu); + attr->lid = 0; + attr->lmc = 0; + attr->sm_lid = 0; + attr->sm_sl = 0; + attr->port_cap_flags = 0; + + if (QLNX_IS_IWARP(dev)) { + attr->gid_tbl_len = 1; + attr->pkey_tbl_len = 1; + } else { + attr->gid_tbl_len = QLNXR_MAX_SGID; + attr->pkey_tbl_len = QLNXR_ROCE_PKEY_TABLE_LEN; + } + + attr->bad_pkey_cntr = rdma_port->pkey_bad_counter; + attr->qkey_viol_cntr = 0; + + get_link_speed_and_width(rdma_port->link_speed, + &attr->active_speed, &attr->active_width); + + attr->max_msg_sz = rdma_port->max_msg_size; + attr->max_vl_num = 4; /* TODO -> figure this one out... */ + + QL_DPRINT12(ha, "state = %d phys_state = %d " + " link_speed = %d active_speed = %d active_width = %d" + " attr->gid_tbl_len = %d attr->pkey_tbl_len = %d" + " max_msg_sz = 0x%x max_vl_num = 0x%x \n", + attr->state, attr->phys_state, + rdma_port->link_speed, attr->active_speed, + attr->active_width, attr->gid_tbl_len, attr->pkey_tbl_len, + attr->max_msg_sz, attr->max_vl_num); + + QL_DPRINT12(ha, "exit\n"); + return 0; +} + +int +qlnxr_modify_port(struct ib_device *ibdev, uint8_t port, int mask, + struct ib_port_modify *props) +{ + struct qlnxr_dev *dev; + qlnx_host_t *ha; + + dev = get_qlnxr_dev(ibdev); + ha = dev->ha; + + QL_DPRINT12(ha, "enter\n"); + + if (port > 1) { + QL_DPRINT12(ha, "port (%d) > 1\n", port); + return -EINVAL; + } + + QL_DPRINT12(ha, "exit\n"); + return 0; +} + +enum rdma_link_layer +qlnxr_link_layer(struct ib_device *ibdev, uint8_t port_num) +{ + struct qlnxr_dev *dev; + qlnx_host_t *ha; + + dev = get_qlnxr_dev(ibdev); + ha = dev->ha; + + QL_DPRINT12(ha, "ibdev = %p port_num = 0x%x\n", ibdev, port_num); + + return IB_LINK_LAYER_ETHERNET; +} + +struct ib_pd * +qlnxr_alloc_pd(struct ib_device *ibdev, struct ib_ucontext *context, + struct ib_udata *udata) +{ + struct qlnxr_pd *pd = NULL; + u16 pd_id; + int rc; + struct qlnxr_dev *dev; + qlnx_host_t *ha; + + dev = get_qlnxr_dev(ibdev); + ha = dev->ha; + + QL_DPRINT12(ha, "ibdev = %p context = %p" + " udata = %p enter\n", ibdev, context, udata); + + if (dev->rdma_ctx == NULL) { + QL_DPRINT11(ha, "dev->rdma_ctx = NULL\n"); + rc = -1; + goto err; + } + + pd = kzalloc(sizeof(*pd), GFP_KERNEL); + if (!pd) { + rc = -ENOMEM; + QL_DPRINT11(ha, "kzalloc(pd) = NULL\n"); + goto err; + } + + rc = ecore_rdma_alloc_pd(dev->rdma_ctx, &pd_id); + if (rc) { + QL_DPRINT11(ha, "ecore_rdma_alloc_pd failed\n"); + goto err; + } + + pd->pd_id = pd_id; + + if (udata && context) { + + rc = ib_copy_to_udata(udata, &pd->pd_id, sizeof(pd->pd_id)); + if (rc) { + QL_DPRINT11(ha, "ib_copy_to_udata failed\n"); + ecore_rdma_free_pd(dev->rdma_ctx, pd_id); + goto err; + } + + pd->uctx = get_qlnxr_ucontext(context); + pd->uctx->pd = pd; + } + + atomic_add_rel_32(&dev->pd_count, 1); + QL_DPRINT12(ha, "exit [pd, pd_id, pd_count] = [%p, 0x%x, %d]\n", + pd, pd_id, dev->pd_count); + + return &pd->ibpd; + +err: + kfree(pd); + QL_DPRINT12(ha, "exit -1\n"); + return ERR_PTR(rc); +} + +int +qlnxr_dealloc_pd(struct ib_pd *ibpd) +{ + struct qlnxr_pd *pd; + struct qlnxr_dev *dev; + qlnx_host_t *ha; + + pd = get_qlnxr_pd(ibpd); + dev = get_qlnxr_dev((ibpd->device)); + ha = dev->ha; + + QL_DPRINT12(ha, "enter\n"); + + if (pd == NULL) { + QL_DPRINT11(ha, "pd = NULL\n"); + } else { + ecore_rdma_free_pd(dev->rdma_ctx, pd->pd_id); + kfree(pd); + atomic_subtract_rel_32(&dev->pd_count, 1); + QL_DPRINT12(ha, "exit [pd, pd_id, pd_count] = [%p, 0x%x, %d]\n", + pd, pd->pd_id, dev->pd_count); + } + + QL_DPRINT12(ha, "exit\n"); + return 0; +} + +#define ROCE_WQE_ELEM_SIZE sizeof(struct rdma_sq_sge) +#define RDMA_MAX_SGE_PER_SRQ (4) /* Should be part of HSI */ +/* Should be part of HSI */ +#define RDMA_MAX_SRQ_WQE_SIZE (RDMA_MAX_SGE_PER_SRQ + 1) /* +1 for header */ +#define DB_ADDR_SHIFT(addr) ((addr) << DB_PWM_ADDR_OFFSET_SHIFT) + +static void qlnxr_cleanup_user(struct qlnxr_dev *, struct qlnxr_qp *); +static void qlnxr_cleanup_kernel(struct qlnxr_dev *, struct qlnxr_qp *); + +int +qlnxr_query_pkey(struct ib_device *ibdev, u8 port, u16 index, u16 *pkey) +{ + struct qlnxr_dev *dev; + qlnx_host_t *ha; + + dev = get_qlnxr_dev(ibdev); + ha = dev->ha; + + QL_DPRINT12(ha, "enter index = 0x%x\n", index); + + if (index > QLNXR_ROCE_PKEY_TABLE_LEN) + return -EINVAL; + + *pkey = QLNXR_ROCE_PKEY_DEFAULT; + + QL_DPRINT12(ha, "exit\n"); + return 0; +} + + +static inline bool +qlnxr_get_vlan_id_qp(qlnx_host_t *ha, struct ib_qp_attr *attr, int attr_mask, + u16 *vlan_id) +{ + bool ret = false; + + QL_DPRINT12(ha, "enter \n"); + + *vlan_id = 0; + +#if __FreeBSD_version >= 1100000 + u16 tmp_vlan_id; + +#if __FreeBSD_version >= 1102000 + union ib_gid *dgid; + + dgid = &attr->ah_attr.grh.dgid; + tmp_vlan_id = (dgid->raw[11] << 8) | dgid->raw[12]; + + if (!(tmp_vlan_id & ~EVL_VLID_MASK)) { + *vlan_id = tmp_vlan_id; + ret = true; + } +#else + tmp_vlan_id = attr->vlan_id; + + if ((attr_mask & IB_QP_VID) && (!(tmp_vlan_id & ~EVL_VLID_MASK))) { + *vlan_id = tmp_vlan_id; + ret = true; + } + +#endif /* #if __FreeBSD_version > 1102000 */ + +#else + ret = true; + +#endif /* #if __FreeBSD_version >= 1100000 */ + + QL_DPRINT12(ha, "exit vlan_id = 0x%x ret = %d \n", *vlan_id, ret); + + return (ret); +} + +static inline void +get_gid_info(struct ib_qp *ibqp, struct ib_qp_attr *attr, + int attr_mask, + struct qlnxr_dev *dev, + struct qlnxr_qp *qp, + struct ecore_rdma_modify_qp_in_params *qp_params) +{ + int i; + qlnx_host_t *ha; + + ha = dev->ha; + + QL_DPRINT12(ha, "enter\n"); + + memcpy(&qp_params->sgid.bytes[0], + &dev->sgid_tbl[qp->sgid_idx].raw[0], + sizeof(qp_params->sgid.bytes)); + memcpy(&qp_params->dgid.bytes[0], + &attr->ah_attr.grh.dgid.raw[0], + sizeof(qp_params->dgid)); + + qlnxr_get_vlan_id_qp(ha, attr, attr_mask, &qp_params->vlan_id); + + for (i = 0; i < (sizeof(qp_params->sgid.dwords)/sizeof(uint32_t)); i++) { + qp_params->sgid.dwords[i] = ntohl(qp_params->sgid.dwords[i]); + qp_params->dgid.dwords[i] = ntohl(qp_params->dgid.dwords[i]); + } + + QL_DPRINT12(ha, "exit\n"); + return; +} + + + +static int +qlnxr_add_mmap(struct qlnxr_ucontext *uctx, u64 phy_addr, unsigned long len) +{ + struct qlnxr_mm *mm; + qlnx_host_t *ha; + + ha = uctx->dev->ha; + + QL_DPRINT12(ha, "enter\n"); + + mm = kzalloc(sizeof(*mm), GFP_KERNEL); + if (mm == NULL) { + QL_DPRINT11(ha, "mm = NULL\n"); + return -ENOMEM; + } + + mm->key.phy_addr = phy_addr; + + /* This function might be called with a length which is not a multiple + * of PAGE_SIZE, while the mapping is PAGE_SIZE grained and the kernel + * forces this granularity by increasing the requested size if needed. + * When qedr_mmap is called, it will search the list with the updated + * length as a key. To prevent search failures, the length is rounded up + * in advance to PAGE_SIZE. + */ + mm->key.len = roundup(len, PAGE_SIZE); + INIT_LIST_HEAD(&mm->entry); + + mutex_lock(&uctx->mm_list_lock); + list_add(&mm->entry, &uctx->mm_head); + mutex_unlock(&uctx->mm_list_lock); + + QL_DPRINT12(ha, "added (addr=0x%llx,len=0x%lx) for ctx=%p\n", + (unsigned long long)mm->key.phy_addr, + (unsigned long)mm->key.len, uctx); + + return 0; +} + +static bool +qlnxr_search_mmap(struct qlnxr_ucontext *uctx, u64 phy_addr, unsigned long len) +{ + bool found = false; + struct qlnxr_mm *mm; + qlnx_host_t *ha; + + ha = uctx->dev->ha; + + QL_DPRINT12(ha, "enter\n"); + + mutex_lock(&uctx->mm_list_lock); + list_for_each_entry(mm, &uctx->mm_head, entry) { + if (len != mm->key.len || phy_addr != mm->key.phy_addr) + continue; + + found = true; + break; + } + mutex_unlock(&uctx->mm_list_lock); + + QL_DPRINT12(ha, + "searched for (addr=0x%llx,len=0x%lx) for ctx=%p, found=%d\n", + mm->key.phy_addr, mm->key.len, uctx, found); + + return found; +} + +struct +ib_ucontext *qlnxr_alloc_ucontext(struct ib_device *ibdev, + struct ib_udata *udata) +{ + int rc; + struct qlnxr_ucontext *ctx; + struct qlnxr_alloc_ucontext_resp uresp; + struct qlnxr_dev *dev = get_qlnxr_dev(ibdev); + qlnx_host_t *ha = dev->ha; + struct ecore_rdma_add_user_out_params oparams; + + if (!udata) { + return ERR_PTR(-EFAULT); + } + + ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); + if (!ctx) + return ERR_PTR(-ENOMEM); + + rc = ecore_rdma_add_user(dev->rdma_ctx, &oparams); + if (rc) { + QL_DPRINT12(ha, + "Failed to allocate a DPI for a new RoCE application " + ",rc = %d. To overcome this, consider to increase " + "the number of DPIs, increase the doorbell BAR size " + "or just close unnecessary RoCE applications. In " + "order to increase the number of DPIs consult the " + "README\n", rc); + goto err; + } + + ctx->dpi = oparams.dpi; + ctx->dpi_addr = oparams.dpi_addr; + ctx->dpi_phys_addr = oparams.dpi_phys_addr; + ctx->dpi_size = oparams.dpi_size; + INIT_LIST_HEAD(&ctx->mm_head); + mutex_init(&ctx->mm_list_lock); + + memset(&uresp, 0, sizeof(uresp)); + uresp.dpm_enabled = offsetof(struct qlnxr_alloc_ucontext_resp, dpm_enabled) + < udata->outlen ? dev->user_dpm_enabled : 0; //TODO: figure this out + uresp.wids_enabled = offsetof(struct qlnxr_alloc_ucontext_resp, wids_enabled) + < udata->outlen ? 1 : 0; //TODO: figure this out + uresp.wid_count = offsetof(struct qlnxr_alloc_ucontext_resp, wid_count) + < udata->outlen ? oparams.wid_count : 0; //TODO: figure this out + uresp.db_pa = ctx->dpi_phys_addr; + uresp.db_size = ctx->dpi_size; + uresp.max_send_wr = dev->attr.max_sqe; + uresp.max_recv_wr = dev->attr.max_rqe; + uresp.max_srq_wr = dev->attr.max_srq_wr; + uresp.sges_per_send_wr = QLNXR_MAX_SQE_ELEMENTS_PER_SQE; + uresp.sges_per_recv_wr = QLNXR_MAX_RQE_ELEMENTS_PER_RQE; + uresp.sges_per_srq_wr = dev->attr.max_srq_sge; + uresp.max_cqes = QLNXR_MAX_CQES; + + rc = ib_copy_to_udata(udata, &uresp, sizeof(uresp)); + if (rc) + goto err; + + ctx->dev = dev; + + rc = qlnxr_add_mmap(ctx, ctx->dpi_phys_addr, ctx->dpi_size); + if (rc) + goto err; + QL_DPRINT12(ha, "Allocated user context %p\n", + &ctx->ibucontext); + + return &ctx->ibucontext; +err: + kfree(ctx); + return ERR_PTR(rc); +} + +int +qlnxr_dealloc_ucontext(struct ib_ucontext *ibctx) +{ + struct qlnxr_ucontext *uctx = get_qlnxr_ucontext(ibctx); + struct qlnxr_dev *dev = uctx->dev; + qlnx_host_t *ha = dev->ha; + struct qlnxr_mm *mm, *tmp; + int status = 0; + + QL_DPRINT12(ha, "Deallocating user context %p\n", + uctx); + + if (dev) { + ecore_rdma_remove_user(uctx->dev->rdma_ctx, uctx->dpi); + } + + list_for_each_entry_safe(mm, tmp, &uctx->mm_head, entry) { + QL_DPRINT12(ha, "deleted addr= 0x%llx, len = 0x%lx for" + " ctx=%p\n", + mm->key.phy_addr, mm->key.len, uctx); + list_del(&mm->entry); + kfree(mm); + } + kfree(uctx); + return status; +} + +int +qlnxr_mmap(struct ib_ucontext *context, struct vm_area_struct *vma) +{ + struct qlnxr_ucontext *ucontext = get_qlnxr_ucontext(context); + struct qlnxr_dev *dev = get_qlnxr_dev((context->device)); + unsigned long vm_page = vma->vm_pgoff << PAGE_SHIFT; + u64 unmapped_db; + unsigned long len = (vma->vm_end - vma->vm_start); + int rc = 0; + bool found; + qlnx_host_t *ha; + + ha = dev->ha; + +#if __FreeBSD_version > 1102000 + unmapped_db = dev->db_phys_addr + (ucontext->dpi * ucontext->dpi_size); +#else + unmapped_db = dev->db_phys_addr; +#endif /* #if __FreeBSD_version > 1102000 */ + + QL_DPRINT12(ha, "qedr_mmap enter vm_page=0x%lx" + " vm_pgoff=0x%lx unmapped_db=0x%llx db_size=%x, len=%lx\n", + vm_page, vma->vm_pgoff, unmapped_db, + dev->db_size, len); + + if ((vma->vm_start & (PAGE_SIZE - 1)) || (len & (PAGE_SIZE - 1))) { + QL_DPRINT11(ha, "Vma_start not page aligned " + "vm_start = %ld vma_end = %ld\n", vma->vm_start, + vma->vm_end); + return -EINVAL; + } + + found = qlnxr_search_mmap(ucontext, vm_page, len); + if (!found) { + QL_DPRINT11(ha, "Vma_pgoff not found in mapped array = %ld\n", + vma->vm_pgoff); + return -EINVAL; + } + + QL_DPRINT12(ha, "Mapping doorbell bar\n"); + +#if __FreeBSD_version > 1102000 + + if ((vm_page < unmapped_db) || + ((vm_page + len) > (unmapped_db + ucontext->dpi_size))) { + QL_DPRINT11(ha, "failed pages are outside of dpi;" + "page address=0x%lx, unmapped_db=0x%lx, dpi_size=0x%x\n", + vm_page, unmapped_db, ucontext->dpi_size); + return -EINVAL; + } + + if (vma->vm_flags & VM_READ) { + QL_DPRINT11(ha, "failed mmap, cannot map doorbell bar for read\n"); + return -EINVAL; + } + + vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot); + rc = io_remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff, len, + vma->vm_page_prot); + +#else + + if ((vm_page >= unmapped_db) && (vm_page <= (unmapped_db + + dev->db_size))) { + + QL_DPRINT12(ha, "Mapping doorbell bar\n"); + + vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot); + + rc = io_remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff, + PAGE_SIZE, vma->vm_page_prot); + } else { + QL_DPRINT12(ha, "Mapping chains\n"); + rc = io_remap_pfn_range(vma, vma->vm_start, + vma->vm_pgoff, len, vma->vm_page_prot); + } + +#endif /* #if __FreeBSD_version > 1102000 */ + + QL_DPRINT12(ha, "exit [%d]\n", rc); + return rc; +} + +struct ib_mr * +qlnxr_get_dma_mr(struct ib_pd *ibpd, int acc) +{ + struct qlnxr_mr *mr; + struct qlnxr_dev *dev = get_qlnxr_dev((ibpd->device)); + struct qlnxr_pd *pd = get_qlnxr_pd(ibpd); + int rc; + qlnx_host_t *ha; + + ha = dev->ha; + + QL_DPRINT12(ha, "enter\n"); + + if (acc & IB_ACCESS_MW_BIND) { + QL_DPRINT12(ha, "Unsupported access flags received for dma mr\n"); + } + + mr = kzalloc(sizeof(*mr), GFP_KERNEL); + if (!mr) { + rc = -ENOMEM; + QL_DPRINT12(ha, "kzalloc(mr) failed %d\n", rc); + goto err0; + } + + mr->type = QLNXR_MR_DMA; + + rc = ecore_rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid); + if (rc) { + QL_DPRINT12(ha, "ecore_rdma_alloc_tid failed %d\n", rc); + goto err1; + } + + /* index only, 18 bit long, lkey = itid << 8 | key */ + mr->hw_mr.tid_type = ECORE_RDMA_TID_REGISTERED_MR; + mr->hw_mr.pd = pd->pd_id; + mr->hw_mr.local_read = 1; + mr->hw_mr.local_write = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0; + mr->hw_mr.remote_read = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0; + mr->hw_mr.remote_write = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0; + mr->hw_mr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0; + mr->hw_mr.dma_mr = true; + + rc = ecore_rdma_register_tid(dev->rdma_ctx, &mr->hw_mr); + if (rc) { + QL_DPRINT12(ha, "ecore_rdma_register_tid failed %d\n", rc); + goto err2; + } + + mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key; + + if (mr->hw_mr.remote_write || mr->hw_mr.remote_read || + mr->hw_mr.remote_atomic) { + mr->ibmr.rkey = mr->hw_mr.itid << 8 | mr->hw_mr.key; + } + + QL_DPRINT12(ha, "lkey = %x\n", mr->ibmr.lkey); + + return &mr->ibmr; + +err2: + ecore_rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid); +err1: + kfree(mr); +err0: + QL_DPRINT12(ha, "exit [%d]\n", rc); + + return ERR_PTR(rc); +} + +static void +qlnxr_free_pbl(struct qlnxr_dev *dev, struct qlnxr_pbl_info *pbl_info, + struct qlnxr_pbl *pbl) +{ + int i; + qlnx_host_t *ha; + + ha = dev->ha; + + QL_DPRINT12(ha, "enter\n"); + + for (i = 0; i < pbl_info->num_pbls; i++) { + if (!pbl[i].va) + continue; + qlnx_dma_free_coherent(&dev->ha->cdev, pbl[i].va, pbl[i].pa, + pbl_info->pbl_size); + } + kfree(pbl); + + QL_DPRINT12(ha, "exit\n"); + return; +} + +#define MIN_FW_PBL_PAGE_SIZE (4*1024) +#define MAX_FW_PBL_PAGE_SIZE (64*1024) + +#define NUM_PBES_ON_PAGE(_page_size) (_page_size / sizeof(u64)) +#define MAX_PBES_ON_PAGE NUM_PBES_ON_PAGE(MAX_FW_PBL_PAGE_SIZE) +#define MAX_PBES_TWO_LAYER (MAX_PBES_ON_PAGE*MAX_PBES_ON_PAGE) + +static struct qlnxr_pbl * +qlnxr_alloc_pbl_tbl(struct qlnxr_dev *dev, + struct qlnxr_pbl_info *pbl_info, gfp_t flags) +{ + void *va; + dma_addr_t pa; + dma_addr_t *pbl_main_tbl; + struct qlnxr_pbl *pbl_table; + int i, rc = 0; + qlnx_host_t *ha; + + ha = dev->ha; + + QL_DPRINT12(ha, "enter\n"); + + pbl_table = kzalloc(sizeof(*pbl_table) * pbl_info->num_pbls, flags); + + if (!pbl_table) { + QL_DPRINT12(ha, "pbl_table = NULL\n"); + return NULL; + } + + for (i = 0; i < pbl_info->num_pbls; i++) { + va = qlnx_dma_alloc_coherent(&dev->ha->cdev, &pa, pbl_info->pbl_size); + if (!va) { + QL_DPRINT11(ha, "Failed to allocate pbl#%d\n", i); + rc = -ENOMEM; + goto err; + } + memset(va, 0, pbl_info->pbl_size); + pbl_table[i].va = va; + pbl_table[i].pa = pa; + } + + /* Two-Layer PBLs, if we have more than one pbl we need to initialize + * the first one with physical pointers to all of the rest + */ + pbl_main_tbl = (dma_addr_t *)pbl_table[0].va; + for (i = 0; i < pbl_info->num_pbls - 1; i++) + pbl_main_tbl[i] = pbl_table[i + 1].pa; + + QL_DPRINT12(ha, "exit\n"); + return pbl_table; + +err: + qlnxr_free_pbl(dev, pbl_info, pbl_table); + + QL_DPRINT12(ha, "exit with error\n"); + return NULL; +} + +static int +qlnxr_prepare_pbl_tbl(struct qlnxr_dev *dev, + struct qlnxr_pbl_info *pbl_info, + u32 num_pbes, + int two_layer_capable) +{ + u32 pbl_capacity; + u32 pbl_size; + u32 num_pbls; + qlnx_host_t *ha; + + ha = dev->ha; + + QL_DPRINT12(ha, "enter\n"); + + if ((num_pbes > MAX_PBES_ON_PAGE) && two_layer_capable) { + if (num_pbes > MAX_PBES_TWO_LAYER) { + QL_DPRINT11(ha, "prepare pbl table: too many pages %d\n", + num_pbes); + return -EINVAL; + } + + /* calculate required pbl page size */ + pbl_size = MIN_FW_PBL_PAGE_SIZE; + pbl_capacity = NUM_PBES_ON_PAGE(pbl_size) * + NUM_PBES_ON_PAGE(pbl_size); + + while (pbl_capacity < num_pbes) { + pbl_size *= 2; + pbl_capacity = pbl_size / sizeof(u64); + pbl_capacity = pbl_capacity * pbl_capacity; + } + + num_pbls = DIV_ROUND_UP(num_pbes, NUM_PBES_ON_PAGE(pbl_size)); + num_pbls++; /* One for the layer0 ( points to the pbls) */ + pbl_info->two_layered = true; + } else { + /* One layered PBL */ + num_pbls = 1; + pbl_size = max_t(u32, MIN_FW_PBL_PAGE_SIZE, \ + roundup_pow_of_two((num_pbes * sizeof(u64)))); + pbl_info->two_layered = false; + } + + pbl_info->num_pbls = num_pbls; + pbl_info->pbl_size = pbl_size; + pbl_info->num_pbes = num_pbes; + + QL_DPRINT12(ha, "prepare pbl table: num_pbes=%d, num_pbls=%d pbl_size=%d\n", + pbl_info->num_pbes, pbl_info->num_pbls, pbl_info->pbl_size); + + return 0; +} + +#define upper_32_bits(x) (uint32_t)(x >> 32) +#define lower_32_bits(x) (uint32_t)(x) + +static void +qlnxr_populate_pbls(struct qlnxr_dev *dev, struct ib_umem *umem, + struct qlnxr_pbl *pbl, struct qlnxr_pbl_info *pbl_info) +{ + struct regpair *pbe; + struct qlnxr_pbl *pbl_tbl; + struct scatterlist *sg; + int shift, pg_cnt, pages, pbe_cnt, total_num_pbes = 0; + qlnx_host_t *ha; + +#ifdef DEFINE_IB_UMEM_WITH_CHUNK + int i; + struct ib_umem_chunk *chunk = NULL; +#else + int entry; +#endif + + + ha = dev->ha; + + QL_DPRINT12(ha, "enter\n"); + + if (!pbl_info) { + QL_DPRINT11(ha, "PBL_INFO not initialized\n"); + return; + } + + if (!pbl_info->num_pbes) { + QL_DPRINT11(ha, "pbl_info->num_pbes == 0\n"); + return; + } + + /* If we have a two layered pbl, the first pbl points to the rest + * of the pbls and the first entry lays on the second pbl in the table + */ + if (pbl_info->two_layered) + pbl_tbl = &pbl[1]; + else + pbl_tbl = pbl; + + pbe = (struct regpair *)pbl_tbl->va; + if (!pbe) { + QL_DPRINT12(ha, "pbe is NULL\n"); + return; + } + + pbe_cnt = 0; + + shift = ilog2(umem->page_size); + +#ifndef DEFINE_IB_UMEM_WITH_CHUNK + + for_each_sg(umem->sg_head.sgl, sg, umem->nmap, entry) { + +#else + list_for_each_entry(chunk, &umem->chunk_list, list) { + /* get all the dma regions from the chunk. */ + for (i = 0; i < chunk->nmap; i++) { + sg = &chunk->page_list[i]; +#endif + pages = sg_dma_len(sg) >> shift; + for (pg_cnt = 0; pg_cnt < pages; pg_cnt++) { + /* store the page address in pbe */ + pbe->lo = + cpu_to_le32(sg_dma_address(sg) + + (umem->page_size * pg_cnt)); + pbe->hi = + cpu_to_le32(upper_32_bits + ((sg_dma_address(sg) + + umem->page_size * pg_cnt))); + + QL_DPRINT12(ha, + "Populate pbl table:" + " pbe->addr=0x%x:0x%x " + " pbe_cnt = %d total_num_pbes=%d" + " pbe=%p\n", pbe->lo, pbe->hi, pbe_cnt, + total_num_pbes, pbe); + + pbe_cnt ++; + total_num_pbes ++; + pbe++; + + if (total_num_pbes == pbl_info->num_pbes) + return; + + /* if the given pbl is full storing the pbes, + * move to next pbl. + */ + if (pbe_cnt == + (pbl_info->pbl_size / sizeof(u64))) { + pbl_tbl++; + pbe = (struct regpair *)pbl_tbl->va; + pbe_cnt = 0; + } + } +#ifdef DEFINE_IB_UMEM_WITH_CHUNK + } +#endif + } + QL_DPRINT12(ha, "exit\n"); + return; +} + +static void +free_mr_info(struct qlnxr_dev *dev, struct mr_info *info) +{ + struct qlnxr_pbl *pbl, *tmp; + qlnx_host_t *ha; + + ha = dev->ha; + + QL_DPRINT12(ha, "enter\n"); + + if (info->pbl_table) + list_add_tail(&info->pbl_table->list_entry, + &info->free_pbl_list); + + if (!list_empty(&info->inuse_pbl_list)) + list_splice(&info->inuse_pbl_list, &info->free_pbl_list); + + list_for_each_entry_safe(pbl, tmp, &info->free_pbl_list, list_entry) { + list_del(&pbl->list_entry); + qlnxr_free_pbl(dev, &info->pbl_info, pbl); + } + QL_DPRINT12(ha, "exit\n"); + + return; +} + +static int +qlnxr_init_mr_info(struct qlnxr_dev *dev, struct mr_info *info, + size_t page_list_len, bool two_layered) +{ + int rc; + struct qlnxr_pbl *tmp; + qlnx_host_t *ha; + + ha = dev->ha; + + QL_DPRINT12(ha, "enter\n"); + + INIT_LIST_HEAD(&info->free_pbl_list); + INIT_LIST_HEAD(&info->inuse_pbl_list); + + rc = qlnxr_prepare_pbl_tbl(dev, &info->pbl_info, + page_list_len, two_layered); + if (rc) { + QL_DPRINT11(ha, "qlnxr_prepare_pbl_tbl [%d]\n", rc); + goto done; + } + + info->pbl_table = qlnxr_alloc_pbl_tbl(dev, &info->pbl_info, GFP_KERNEL); + + if (!info->pbl_table) { + rc = -ENOMEM; + QL_DPRINT11(ha, "qlnxr_alloc_pbl_tbl returned NULL\n"); + goto done; + } + + QL_DPRINT12(ha, "pbl_table_pa = %pa\n", &info->pbl_table->pa); + + /* in usual case we use 2 PBLs, so we add one to free + * list and allocating another one + */ + tmp = qlnxr_alloc_pbl_tbl(dev, &info->pbl_info, GFP_KERNEL); + + if (!tmp) { + QL_DPRINT11(ha, "Extra PBL is not allocated\n"); + goto done; /* it's OK if second allocation fails, so rc = 0*/ + } + + list_add_tail(&tmp->list_entry, &info->free_pbl_list); + + QL_DPRINT12(ha, "extra pbl_table_pa = %pa\n", &tmp->pa); + +done: + if (rc) + free_mr_info(dev, info); + + QL_DPRINT12(ha, "exit [%d]\n", rc); + + return rc; +} + + +struct ib_mr * +#if __FreeBSD_version >= 1102000 +qlnxr_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len, + u64 usr_addr, int acc, struct ib_udata *udata) +#else +qlnxr_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len, + u64 usr_addr, int acc, struct ib_udata *udata, int mr_id) +#endif /* #if __FreeBSD_version >= 1102000 */ +{ + int rc = -ENOMEM; + struct qlnxr_dev *dev = get_qlnxr_dev((ibpd->device)); + struct qlnxr_mr *mr; + struct qlnxr_pd *pd; + qlnx_host_t *ha; + + ha = dev->ha; + + QL_DPRINT12(ha, "enter\n"); + + pd = get_qlnxr_pd(ibpd); + + QL_DPRINT12(ha, "qedr_register user mr pd = %d" + " start = %lld, len = %lld, usr_addr = %lld, acc = %d\n", + pd->pd_id, start, len, usr_addr, acc); + + if (acc & IB_ACCESS_REMOTE_WRITE && !(acc & IB_ACCESS_LOCAL_WRITE)) { + QL_DPRINT11(ha, + "(acc & IB_ACCESS_REMOTE_WRITE &&" + " !(acc & IB_ACCESS_LOCAL_WRITE))\n"); + return ERR_PTR(-EINVAL); + } + + mr = kzalloc(sizeof(*mr), GFP_KERNEL); + if (!mr) { + QL_DPRINT11(ha, "kzalloc(mr) failed\n"); + return ERR_PTR(rc); + } + + mr->type = QLNXR_MR_USER; + + mr->umem = ib_umem_get(ibpd->uobject->context, start, len, acc, 0); + if (IS_ERR(mr->umem)) { + rc = -EFAULT; + QL_DPRINT11(ha, "ib_umem_get failed [%p]\n", mr->umem); + goto err0; + } + + rc = qlnxr_init_mr_info(dev, &mr->info, ib_umem_page_count(mr->umem), 1); + if (rc) { + QL_DPRINT11(ha, + "qlnxr_init_mr_info failed [%d]\n", rc); + goto err1; + } + + qlnxr_populate_pbls(dev, mr->umem, mr->info.pbl_table, + &mr->info.pbl_info); + + rc = ecore_rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid); + + if (rc) { + QL_DPRINT11(ha, "roce alloc tid returned an error %d\n", rc); + goto err1; + } + + /* index only, 18 bit long, lkey = itid << 8 | key */ + mr->hw_mr.tid_type = ECORE_RDMA_TID_REGISTERED_MR; + mr->hw_mr.key = 0; + mr->hw_mr.pd = pd->pd_id; + mr->hw_mr.local_read = 1; + mr->hw_mr.local_write = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0; + mr->hw_mr.remote_read = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0; + mr->hw_mr.remote_write = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0; + mr->hw_mr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0; + mr->hw_mr.mw_bind = false; /* TBD MW BIND */ + mr->hw_mr.pbl_ptr = mr->info.pbl_table[0].pa; + mr->hw_mr.pbl_two_level = mr->info.pbl_info.two_layered; + mr->hw_mr.pbl_page_size_log = ilog2(mr->info.pbl_info.pbl_size); + mr->hw_mr.page_size_log = ilog2(mr->umem->page_size); /* for the MR pages */ + +#if __FreeBSD_version >= 1102000 + mr->hw_mr.fbo = ib_umem_offset(mr->umem); +#else + mr->hw_mr.fbo = mr->umem->offset; +#endif + mr->hw_mr.length = len; + mr->hw_mr.vaddr = usr_addr; + mr->hw_mr.zbva = false; /* TBD figure when this should be true */ + mr->hw_mr.phy_mr = false; /* Fast MR - True, Regular Register False */ + mr->hw_mr.dma_mr = false; + + rc = ecore_rdma_register_tid(dev->rdma_ctx, &mr->hw_mr); + if (rc) { + QL_DPRINT11(ha, "roce register tid returned an error %d\n", rc); + goto err2; + } + + mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key; + if (mr->hw_mr.remote_write || mr->hw_mr.remote_read || + mr->hw_mr.remote_atomic) + mr->ibmr.rkey = mr->hw_mr.itid << 8 | mr->hw_mr.key; + + QL_DPRINT12(ha, "register user mr lkey: %x\n", mr->ibmr.lkey); + + return (&mr->ibmr); + +err2: + ecore_rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid); +err1: + qlnxr_free_pbl(dev, &mr->info.pbl_info, mr->info.pbl_table); +err0: + kfree(mr); + + QL_DPRINT12(ha, "exit [%d]\n", rc); + return (ERR_PTR(rc)); +} + +int +qlnxr_dereg_mr(struct ib_mr *ib_mr) +{ + struct qlnxr_mr *mr = get_qlnxr_mr(ib_mr); + struct qlnxr_dev *dev = get_qlnxr_dev((ib_mr->device)); + int rc = 0; + qlnx_host_t *ha; + + ha = dev->ha; + + QL_DPRINT12(ha, "enter\n"); + + if ((mr->type != QLNXR_MR_DMA) && (mr->type != QLNXR_MR_FRMR)) + qlnxr_free_pbl(dev, &mr->info.pbl_info, mr->info.pbl_table); + + /* it could be user registered memory. */ + if (mr->umem) + ib_umem_release(mr->umem); + + kfree(mr->pages); + + kfree(mr); + + QL_DPRINT12(ha, "exit\n"); + return rc; +} + +static int +qlnxr_copy_cq_uresp(struct qlnxr_dev *dev, + struct qlnxr_cq *cq, struct ib_udata *udata) +{ + struct qlnxr_create_cq_uresp uresp; + int rc; + qlnx_host_t *ha; + + ha = dev->ha; + + QL_DPRINT12(ha, "enter\n"); + + memset(&uresp, 0, sizeof(uresp)); + + uresp.db_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_UCM_RDMA_CQ_CONS_32BIT); + uresp.icid = cq->icid; + + rc = ib_copy_to_udata(udata, &uresp, sizeof(uresp)); + + if (rc) { + QL_DPRINT12(ha, "ib_copy_to_udata error cqid=0x%x[%d]\n", + cq->icid, rc); + } + + QL_DPRINT12(ha, "exit [%d]\n", rc); + return rc; +} + +static void +consume_cqe(struct qlnxr_cq *cq) +{ + + if (cq->latest_cqe == cq->toggle_cqe) + cq->pbl_toggle ^= RDMA_RESIZE_CQ_RAMROD_DATA_TOGGLE_BIT_MASK; + + cq->latest_cqe = ecore_chain_consume(&cq->pbl); +} + +static inline int +qlnxr_align_cq_entries(int entries) +{ + u64 size, aligned_size; + + /* We allocate an extra entry that we don't report to the FW. + * Why? + * The CQE size is 32 bytes but the FW writes in chunks of 64 bytes + * (for performance purposes). Allocating an extra entry and telling + * the FW we have less prevents overwriting the first entry in case of + * a wrap i.e. when the FW writes the last entry and the application + * hasn't read the first one. + */ + size = (entries + 1) * QLNXR_CQE_SIZE; + + /* We align to PAGE_SIZE. + * Why? + * Since the CQ is going to be mapped and the mapping is anyhow in whole + * kernel pages we benefit from the possibly extra CQEs. + */ + aligned_size = ALIGN(size, PAGE_SIZE); + + /* note: for CQs created in user space the result of this function + * should match the size mapped in user space + */ + return (aligned_size / QLNXR_CQE_SIZE); +} + +static inline int +qlnxr_init_user_queue(struct ib_ucontext *ib_ctx, struct qlnxr_dev *dev, + struct qlnxr_userq *q, u64 buf_addr, size_t buf_len, + int access, int dmasync, int alloc_and_init) +{ + int page_cnt; + int rc; + qlnx_host_t *ha; + + ha = dev->ha; + + QL_DPRINT12(ha, "enter\n"); + + q->buf_addr = buf_addr; + q->buf_len = buf_len; + + QL_DPRINT12(ha, "buf_addr : %llx, buf_len : %x, access : %x" + " dmasync : %x\n", q->buf_addr, q->buf_len, + access, dmasync); + + q->umem = ib_umem_get(ib_ctx, q->buf_addr, q->buf_len, access, dmasync); + + if (IS_ERR(q->umem)) { + QL_DPRINT11(ha, "ib_umem_get failed [%lx]\n", PTR_ERR(q->umem)); + return PTR_ERR(q->umem); + } + + page_cnt = ib_umem_page_count(q->umem); + rc = qlnxr_prepare_pbl_tbl(dev, &q->pbl_info, page_cnt, + 0 /* SQ and RQ don't support dual layer pbl. + * CQ may, but this is yet uncoded. + */); + if (rc) { + QL_DPRINT11(ha, "qlnxr_prepare_pbl_tbl failed [%d]\n", rc); + goto err; + } + + if (alloc_and_init) { + q->pbl_tbl = qlnxr_alloc_pbl_tbl(dev, &q->pbl_info, GFP_KERNEL); + + if (!q->pbl_tbl) { + QL_DPRINT11(ha, "qlnxr_alloc_pbl_tbl failed\n"); + rc = -ENOMEM; + goto err; + } + + qlnxr_populate_pbls(dev, q->umem, q->pbl_tbl, &q->pbl_info); + } else { + q->pbl_tbl = kzalloc(sizeof(*q->pbl_tbl), GFP_KERNEL); + + if (!q->pbl_tbl) { + QL_DPRINT11(ha, "qlnxr_alloc_pbl_tbl failed\n"); + rc = -ENOMEM; + goto err; + } + } + + QL_DPRINT12(ha, "exit\n"); + return 0; + +err: + ib_umem_release(q->umem); + q->umem = NULL; + + QL_DPRINT12(ha, "exit [%d]\n", rc); + return rc; +} + +#if __FreeBSD_version >= 1102000 + +struct ib_cq * +qlnxr_create_cq(struct ib_device *ibdev, + const struct ib_cq_init_attr *attr, + struct ib_ucontext *ib_ctx, + struct ib_udata *udata) + +#else + +#if __FreeBSD_version >= 1100000 + +struct ib_cq * +qlnxr_create_cq(struct ib_device *ibdev, + struct ib_cq_init_attr *attr, + struct ib_ucontext *ib_ctx, + struct ib_udata *udata) + +#else + +struct ib_cq * +qlnxr_create_cq(struct ib_device *ibdev, + int entries, + int vector, + struct ib_ucontext *ib_ctx, + struct ib_udata *udata) +#endif /* #if __FreeBSD_version >= 1100000 */ + +#endif /* #if __FreeBSD_version >= 1102000 */ +{ + struct qlnxr_ucontext *ctx; + struct ecore_rdma_destroy_cq_out_params destroy_oparams; + struct ecore_rdma_destroy_cq_in_params destroy_iparams; + struct qlnxr_dev *dev; + struct ecore_rdma_create_cq_in_params params; + struct qlnxr_create_cq_ureq ureq; + +#if __FreeBSD_version >= 1100000 + int vector = attr->comp_vector; + int entries = attr->cqe; +#endif + struct qlnxr_cq *cq; + int chain_entries, rc, page_cnt; + u64 pbl_ptr; + u16 icid; + qlnx_host_t *ha; + + dev = get_qlnxr_dev(ibdev); + ha = dev->ha; + + QL_DPRINT12(ha, "called from %s. entries = %d, " + "vector = %d\n", + (udata ? "User Lib" : "Kernel"), entries, vector); + + memset(¶ms, 0, sizeof(struct ecore_rdma_create_cq_in_params)); + memset(&destroy_iparams, 0, sizeof(struct ecore_rdma_destroy_cq_in_params)); + memset(&destroy_oparams, 0, sizeof(struct ecore_rdma_destroy_cq_out_params)); + + if (entries > QLNXR_MAX_CQES) { + QL_DPRINT11(ha, + "the number of entries %d is too high. " + "Must be equal or below %d.\n", + entries, QLNXR_MAX_CQES); + return ERR_PTR(-EINVAL); + } + chain_entries = qlnxr_align_cq_entries(entries); + chain_entries = min_t(int, chain_entries, QLNXR_MAX_CQES); + + cq = qlnx_zalloc((sizeof(struct qlnxr_cq))); + + if (!cq) + return ERR_PTR(-ENOMEM); + + if (udata) { + memset(&ureq, 0, sizeof(ureq)); + + if (ib_copy_from_udata(&ureq, udata, + min(sizeof(ureq), udata->inlen))) { + QL_DPRINT11(ha, "ib_copy_from_udata failed\n"); + goto err0; + } + + if (!ureq.len) { + QL_DPRINT11(ha, "ureq.len == 0\n"); + goto err0; + } + + cq->cq_type = QLNXR_CQ_TYPE_USER; + + qlnxr_init_user_queue(ib_ctx, dev, &cq->q, ureq.addr, ureq.len, + IB_ACCESS_LOCAL_WRITE, 1, 1); + + pbl_ptr = cq->q.pbl_tbl->pa; + page_cnt = cq->q.pbl_info.num_pbes; + cq->ibcq.cqe = chain_entries; + } else { + cq->cq_type = QLNXR_CQ_TYPE_KERNEL; + + rc = ecore_chain_alloc(&dev->ha->cdev, + ECORE_CHAIN_USE_TO_CONSUME, + ECORE_CHAIN_MODE_PBL, + ECORE_CHAIN_CNT_TYPE_U32, + chain_entries, + sizeof(union roce_cqe), + &cq->pbl, NULL); + + if (rc) + goto err1; + + page_cnt = ecore_chain_get_page_cnt(&cq->pbl); + pbl_ptr = ecore_chain_get_pbl_phys(&cq->pbl); + cq->ibcq.cqe = cq->pbl.capacity; + } + + params.cq_handle_hi = upper_32_bits((uintptr_t)cq); + params.cq_handle_lo = lower_32_bits((uintptr_t)cq); + params.cnq_id = vector; + params.cq_size = chain_entries - 1; + params.pbl_num_pages = page_cnt; + params.pbl_ptr = pbl_ptr; + params.pbl_two_level = 0; + + if (ib_ctx != NULL) { + ctx = get_qlnxr_ucontext(ib_ctx); + params.dpi = ctx->dpi; + } else { + params.dpi = dev->dpi; + } + + rc = ecore_rdma_create_cq(dev->rdma_ctx, ¶ms, &icid); + if (rc) + goto err2; + + cq->icid = icid; + cq->sig = QLNXR_CQ_MAGIC_NUMBER; + spin_lock_init(&cq->cq_lock); + + if (ib_ctx) { + rc = qlnxr_copy_cq_uresp(dev, cq, udata); + if (rc) + goto err3; + } else { + /* Generate doorbell address. + * Configure bits 3-9 with DQ_PWM_OFFSET_UCM_RDMA_CQ_CONS_32BIT. + * TODO: consider moving to device scope as it is a function of + * the device. + * TODO: add ifdef if plan to support 16 bit. + */ + cq->db_addr = dev->db_addr + + DB_ADDR_SHIFT(DQ_PWM_OFFSET_UCM_RDMA_CQ_CONS_32BIT); + cq->db.data.icid = cq->icid; + cq->db.data.params = DB_AGG_CMD_SET << + RDMA_PWM_VAL32_DATA_AGG_CMD_SHIFT; + + /* point to the very last element, passing it we will toggle */ + cq->toggle_cqe = ecore_chain_get_last_elem(&cq->pbl); + cq->pbl_toggle = RDMA_RESIZE_CQ_RAMROD_DATA_TOGGLE_BIT_MASK; + + /* must be different from pbl_toggle */ + cq->latest_cqe = NULL; + consume_cqe(cq); + cq->cq_cons = ecore_chain_get_cons_idx_u32(&cq->pbl); + } + + QL_DPRINT12(ha, "exit icid = 0x%0x, addr = %p," + " number of entries = 0x%x\n", + cq->icid, cq, params.cq_size); + QL_DPRINT12(ha,"cq_addr = %p\n", cq); + return &cq->ibcq; + +err3: + destroy_iparams.icid = cq->icid; + ecore_rdma_destroy_cq(dev->rdma_ctx, &destroy_iparams, &destroy_oparams); +err2: + if (udata) + qlnxr_free_pbl(dev, &cq->q.pbl_info, cq->q.pbl_tbl); + else + ecore_chain_free(&dev->ha->cdev, &cq->pbl); +err1: + if (udata) + ib_umem_release(cq->q.umem); +err0: + kfree(cq); + + QL_DPRINT12(ha, "exit error\n"); + + return ERR_PTR(-EINVAL); +} + +int qlnxr_resize_cq(struct ib_cq *ibcq, int new_cnt, struct ib_udata *udata) +{ + int status = 0; + struct qlnxr_dev *dev = get_qlnxr_dev((ibcq->device)); + qlnx_host_t *ha; + + ha = dev->ha; + + QL_DPRINT12(ha, "enter/exit\n"); + + return status; +} + +int +qlnxr_destroy_cq(struct ib_cq *ibcq) +{ + struct qlnxr_dev *dev = get_qlnxr_dev((ibcq->device)); + struct ecore_rdma_destroy_cq_out_params oparams; + struct ecore_rdma_destroy_cq_in_params iparams; + struct qlnxr_cq *cq = get_qlnxr_cq(ibcq); + int rc = 0; + qlnx_host_t *ha; + + ha = dev->ha; + + QL_DPRINT12(ha, "enter cq_id = %d\n", cq->icid); + + cq->destroyed = 1; + + /* TODO: Syncronize irq of the CNQ the CQ belongs to for validation + * that all completions with notification are dealt with. The rest + * of the completions are not interesting + */ + + /* GSIs CQs are handled by driver, so they don't exist in the FW */ + + if (cq->cq_type != QLNXR_CQ_TYPE_GSI) { + + iparams.icid = cq->icid; + + rc = ecore_rdma_destroy_cq(dev->rdma_ctx, &iparams, &oparams); + + if (rc) { + QL_DPRINT12(ha, "ecore_rdma_destroy_cq failed cq_id = %d\n", + cq->icid); + return rc; + } + + QL_DPRINT12(ha, "free cq->pbl cq_id = %d\n", cq->icid); + ecore_chain_free(&dev->ha->cdev, &cq->pbl); + } + + if (ibcq->uobject && ibcq->uobject->context) { + qlnxr_free_pbl(dev, &cq->q.pbl_info, cq->q.pbl_tbl); + ib_umem_release(cq->q.umem); + } + + cq->sig = ~cq->sig; + + kfree(cq); + + QL_DPRINT12(ha, "exit cq_id = %d\n", cq->icid); + + return rc; +} + +static int +qlnxr_check_qp_attrs(struct ib_pd *ibpd, + struct qlnxr_dev *dev, + struct ib_qp_init_attr *attrs, + struct ib_udata *udata) +{ + struct ecore_rdma_device *qattr; + qlnx_host_t *ha; + + qattr = ecore_rdma_query_device(dev->rdma_ctx); + ha = dev->ha; + + QL_DPRINT12(ha, "enter\n"); + + QL_DPRINT12(ha, "attrs->sq_sig_type = %d\n", attrs->sq_sig_type); + QL_DPRINT12(ha, "attrs->qp_type = %d\n", attrs->qp_type); + QL_DPRINT12(ha, "attrs->create_flags = %d\n", attrs->create_flags); + +#if __FreeBSD_version < 1102000 + QL_DPRINT12(ha, "attrs->qpg_type = %d\n", attrs->qpg_type); +#endif + + QL_DPRINT12(ha, "attrs->port_num = %d\n", attrs->port_num); + QL_DPRINT12(ha, "attrs->cap.max_send_wr = 0x%x\n", attrs->cap.max_send_wr); + QL_DPRINT12(ha, "attrs->cap.max_recv_wr = 0x%x\n", attrs->cap.max_recv_wr); + QL_DPRINT12(ha, "attrs->cap.max_send_sge = 0x%x\n", attrs->cap.max_send_sge); + QL_DPRINT12(ha, "attrs->cap.max_recv_sge = 0x%x\n", attrs->cap.max_recv_sge); + QL_DPRINT12(ha, "attrs->cap.max_inline_data = 0x%x\n", + attrs->cap.max_inline_data); + +#if __FreeBSD_version < 1102000 + QL_DPRINT12(ha, "attrs->cap.qpg_tss_mask_sz = 0x%x\n", + attrs->cap.qpg_tss_mask_sz); +#endif + + QL_DPRINT12(ha, "\n\nqattr->vendor_id = 0x%x\n", qattr->vendor_id); + QL_DPRINT12(ha, "qattr->vendor_part_id = 0x%x\n", qattr->vendor_part_id); + QL_DPRINT12(ha, "qattr->hw_ver = 0x%x\n", qattr->hw_ver); + QL_DPRINT12(ha, "qattr->fw_ver = %p\n", (void *)qattr->fw_ver); + QL_DPRINT12(ha, "qattr->node_guid = %p\n", (void *)qattr->node_guid); + QL_DPRINT12(ha, "qattr->sys_image_guid = %p\n", + (void *)qattr->sys_image_guid); + QL_DPRINT12(ha, "qattr->max_cnq = 0x%x\n", qattr->max_cnq); + QL_DPRINT12(ha, "qattr->max_sge = 0x%x\n", qattr->max_sge); + QL_DPRINT12(ha, "qattr->max_srq_sge = 0x%x\n", qattr->max_srq_sge); + QL_DPRINT12(ha, "qattr->max_inline = 0x%x\n", qattr->max_inline); + QL_DPRINT12(ha, "qattr->max_wqe = 0x%x\n", qattr->max_wqe); + QL_DPRINT12(ha, "qattr->max_srq_wqe = 0x%x\n", qattr->max_srq_wqe); + QL_DPRINT12(ha, "qattr->max_qp_resp_rd_atomic_resc = 0x%x\n", + qattr->max_qp_resp_rd_atomic_resc); + QL_DPRINT12(ha, "qattr->max_qp_req_rd_atomic_resc = 0x%x\n", + qattr->max_qp_req_rd_atomic_resc); + QL_DPRINT12(ha, "qattr->max_dev_resp_rd_atomic_resc = 0x%x\n", + qattr->max_dev_resp_rd_atomic_resc); + QL_DPRINT12(ha, "qattr->max_cq = 0x%x\n", qattr->max_cq); + QL_DPRINT12(ha, "qattr->max_qp = 0x%x\n", qattr->max_qp); + QL_DPRINT12(ha, "qattr->max_srq = 0x%x\n", qattr->max_srq); + QL_DPRINT12(ha, "qattr->max_mr = 0x%x\n", qattr->max_mr); + QL_DPRINT12(ha, "qattr->max_mr_size = %p\n", (void *)qattr->max_mr_size); + QL_DPRINT12(ha, "qattr->max_cqe = 0x%x\n", qattr->max_cqe); + QL_DPRINT12(ha, "qattr->max_mw = 0x%x\n", qattr->max_mw); + QL_DPRINT12(ha, "qattr->max_fmr = 0x%x\n", qattr->max_fmr); + QL_DPRINT12(ha, "qattr->max_mr_mw_fmr_pbl = 0x%x\n", + qattr->max_mr_mw_fmr_pbl); + QL_DPRINT12(ha, "qattr->max_mr_mw_fmr_size = %p\n", + (void *)qattr->max_mr_mw_fmr_size); + QL_DPRINT12(ha, "qattr->max_pd = 0x%x\n", qattr->max_pd); + QL_DPRINT12(ha, "qattr->max_ah = 0x%x\n", qattr->max_ah); + QL_DPRINT12(ha, "qattr->max_pkey = 0x%x\n", qattr->max_pkey); + QL_DPRINT12(ha, "qattr->max_srq_wr = 0x%x\n", qattr->max_srq_wr); + QL_DPRINT12(ha, "qattr->max_stats_queues = 0x%x\n", + qattr->max_stats_queues); + //QL_DPRINT12(ha, "qattr->dev_caps = 0x%x\n", qattr->dev_caps); + QL_DPRINT12(ha, "qattr->page_size_caps = %p\n", + (void *)qattr->page_size_caps); + QL_DPRINT12(ha, "qattr->dev_ack_delay = 0x%x\n", qattr->dev_ack_delay); + QL_DPRINT12(ha, "qattr->reserved_lkey = 0x%x\n", qattr->reserved_lkey); + QL_DPRINT12(ha, "qattr->bad_pkey_counter = 0x%x\n", + qattr->bad_pkey_counter); + + if ((attrs->qp_type == IB_QPT_GSI) && udata) { + QL_DPRINT12(ha, "unexpected udata when creating GSI QP\n"); + return -EINVAL; + } + + if (udata && !(ibpd->uobject && ibpd->uobject->context)) { + QL_DPRINT12(ha, "called from user without context\n"); + return -EINVAL; + } + + /* QP0... attrs->qp_type == IB_QPT_GSI */ + if (attrs->qp_type != IB_QPT_RC && attrs->qp_type != IB_QPT_GSI) { + QL_DPRINT12(ha, "unsupported qp type=0x%x requested\n", + attrs->qp_type); + return -EINVAL; + } + if (attrs->qp_type == IB_QPT_GSI && attrs->srq) { + QL_DPRINT12(ha, "cannot create GSI qp with SRQ\n"); + return -EINVAL; + } + /* Skip the check for QP1 to support CM size of 128 */ + if (attrs->cap.max_send_wr > qattr->max_wqe) { + QL_DPRINT12(ha, "cannot create a SQ with %d elements " + " (max_send_wr=0x%x)\n", + attrs->cap.max_send_wr, qattr->max_wqe); + return -EINVAL; + } + if (!attrs->srq && (attrs->cap.max_recv_wr > qattr->max_wqe)) { + QL_DPRINT12(ha, "cannot create a RQ with %d elements" + " (max_recv_wr=0x%x)\n", + attrs->cap.max_recv_wr, qattr->max_wqe); + return -EINVAL; + } + if (attrs->cap.max_inline_data > qattr->max_inline) { + QL_DPRINT12(ha, + "unsupported inline data size=0x%x " + "requested (max_inline=0x%x)\n", + attrs->cap.max_inline_data, qattr->max_inline); + return -EINVAL; + } + if (attrs->cap.max_send_sge > qattr->max_sge) { + QL_DPRINT12(ha, + "unsupported send_sge=0x%x " + "requested (max_send_sge=0x%x)\n", + attrs->cap.max_send_sge, qattr->max_sge); + return -EINVAL; + } + if (attrs->cap.max_recv_sge > qattr->max_sge) { + QL_DPRINT12(ha, + "unsupported recv_sge=0x%x requested " + " (max_recv_sge=0x%x)\n", + attrs->cap.max_recv_sge, qattr->max_sge); + return -EINVAL; + } + /* unprivileged user space cannot create special QP */ + if (ibpd->uobject && attrs->qp_type == IB_QPT_GSI) { + QL_DPRINT12(ha, + "userspace can't create special QPs of type=0x%x\n", + attrs->qp_type); + return -EINVAL; + } + /* allow creating only one GSI type of QP */ + if (attrs->qp_type == IB_QPT_GSI && dev->gsi_qp_created) { + QL_DPRINT12(ha, + "create qp: GSI special QPs already created.\n"); + return -EINVAL; + } + + /* verify consumer QPs are not trying to use GSI QP's CQ */ + if ((attrs->qp_type != IB_QPT_GSI) && (dev->gsi_qp_created)) { + struct qlnxr_cq *send_cq = get_qlnxr_cq(attrs->send_cq); + struct qlnxr_cq *recv_cq = get_qlnxr_cq(attrs->recv_cq); + + if ((send_cq->cq_type == QLNXR_CQ_TYPE_GSI) || + (recv_cq->cq_type == QLNXR_CQ_TYPE_GSI)) { + QL_DPRINT11(ha, "consumer QP cannot use GSI CQs.\n"); + return -EINVAL; + } + } + QL_DPRINT12(ha, "exit\n"); + return 0; +} + +static int +qlnxr_copy_srq_uresp(struct qlnxr_dev *dev, + struct qlnxr_srq *srq, + struct ib_udata *udata) +{ + struct qlnxr_create_srq_uresp uresp; + qlnx_host_t *ha; + int rc; + + ha = dev->ha; + + QL_DPRINT12(ha, "enter\n"); + + memset(&uresp, 0, sizeof(uresp)); + + uresp.srq_id = srq->srq_id; + + rc = ib_copy_to_udata(udata, &uresp, sizeof(uresp)); + + QL_DPRINT12(ha, "exit [%d]\n", rc); + return rc; +} + +static void +qlnxr_copy_rq_uresp(struct qlnxr_dev *dev, + struct qlnxr_create_qp_uresp *uresp, + struct qlnxr_qp *qp) +{ + qlnx_host_t *ha; + + ha = dev->ha; + + /* Return if QP is associated with SRQ instead of RQ */ + QL_DPRINT12(ha, "enter qp->srq = %p\n", qp->srq); + + if (qp->srq) + return; + + /* iWARP requires two doorbells per RQ. */ + if (QLNX_IS_IWARP(dev)) { + + uresp->rq_db_offset = + DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_IWARP_RQ_PROD); + uresp->rq_db2_offset = + DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_FLAGS); + + QL_DPRINT12(ha, "uresp->rq_db_offset = 0x%x " + "uresp->rq_db2_offset = 0x%x\n", + uresp->rq_db_offset, uresp->rq_db2_offset); + } else { + uresp->rq_db_offset = + DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_ROCE_RQ_PROD); + } + uresp->rq_icid = qp->icid; + + QL_DPRINT12(ha, "exit\n"); + return; +} + +static void +qlnxr_copy_sq_uresp(struct qlnxr_dev *dev, + struct qlnxr_create_qp_uresp *uresp, + struct qlnxr_qp *qp) +{ + qlnx_host_t *ha; + + ha = dev->ha; + + QL_DPRINT12(ha, "enter\n"); + + uresp->sq_db_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD); + + /* iWARP uses the same cid for rq and sq*/ + if (QLNX_IS_IWARP(dev)) { + uresp->sq_icid = qp->icid; + QL_DPRINT12(ha, "uresp->sq_icid = 0x%x\n", uresp->sq_icid); + } else + uresp->sq_icid = qp->icid + 1; + + QL_DPRINT12(ha, "exit\n"); + return; +} + +static int +qlnxr_copy_qp_uresp(struct qlnxr_dev *dev, + struct qlnxr_qp *qp, + struct ib_udata *udata) +{ + int rc; + struct qlnxr_create_qp_uresp uresp; + qlnx_host_t *ha; + + ha = dev->ha; + + QL_DPRINT12(ha, "enter qp->icid =0x%x\n", qp->icid); + + memset(&uresp, 0, sizeof(uresp)); + qlnxr_copy_sq_uresp(dev, &uresp, qp); + qlnxr_copy_rq_uresp(dev, &uresp, qp); + + uresp.atomic_supported = dev->atomic_cap != IB_ATOMIC_NONE; + uresp.qp_id = qp->qp_id; + + rc = ib_copy_to_udata(udata, &uresp, sizeof(uresp)); + + QL_DPRINT12(ha, "exit [%d]\n", rc); + return rc; +} + + +static void +qlnxr_set_common_qp_params(struct qlnxr_dev *dev, + struct qlnxr_qp *qp, + struct qlnxr_pd *pd, + struct ib_qp_init_attr *attrs) +{ + qlnx_host_t *ha; + + ha = dev->ha; + + QL_DPRINT12(ha, "enter\n"); + + spin_lock_init(&qp->q_lock); + + atomic_set(&qp->refcnt, 1); + qp->pd = pd; + qp->sig = QLNXR_QP_MAGIC_NUMBER; + qp->qp_type = attrs->qp_type; + qp->max_inline_data = ROCE_REQ_MAX_INLINE_DATA_SIZE; + qp->sq.max_sges = attrs->cap.max_send_sge; + qp->state = ECORE_ROCE_QP_STATE_RESET; + qp->signaled = (attrs->sq_sig_type == IB_SIGNAL_ALL_WR) ? true : false; + qp->sq_cq = get_qlnxr_cq(attrs->send_cq); + qp->rq_cq = get_qlnxr_cq(attrs->recv_cq); + qp->dev = dev; + + if (!attrs->srq) { + /* QP is associated with RQ instead of SRQ */ + qp->rq.max_sges = attrs->cap.max_recv_sge; + QL_DPRINT12(ha, "RQ params:\trq_max_sges = %d, rq_cq_id = %d\n", + qp->rq.max_sges, qp->rq_cq->icid); + } else { + qp->srq = get_qlnxr_srq(attrs->srq); + } + + QL_DPRINT12(ha, + "QP params:\tpd = %d, qp_type = %d, max_inline_data = %d," + " state = %d, signaled = %d, use_srq=%d\n", + pd->pd_id, qp->qp_type, qp->max_inline_data, + qp->state, qp->signaled, ((attrs->srq) ? 1 : 0)); + QL_DPRINT12(ha, "SQ params:\tsq_max_sges = %d, sq_cq_id = %d\n", + qp->sq.max_sges, qp->sq_cq->icid); + return; +} + +static int +qlnxr_check_srq_params(struct ib_pd *ibpd, + struct qlnxr_dev *dev, + struct ib_srq_init_attr *attrs) +{ + struct ecore_rdma_device *qattr; + qlnx_host_t *ha; + + ha = dev->ha; + qattr = ecore_rdma_query_device(dev->rdma_ctx); + + QL_DPRINT12(ha, "enter\n"); + + if (attrs->attr.max_wr > qattr->max_srq_wqe) { + QL_DPRINT12(ha, "unsupported srq_wr=0x%x" + " requested (max_srq_wr=0x%x)\n", + attrs->attr.max_wr, qattr->max_srq_wr); + return -EINVAL; + } + + if (attrs->attr.max_sge > qattr->max_sge) { + QL_DPRINT12(ha, + "unsupported sge=0x%x requested (max_srq_sge=0x%x)\n", + attrs->attr.max_sge, qattr->max_sge); + return -EINVAL; + } + + if (attrs->attr.srq_limit > attrs->attr.max_wr) { + QL_DPRINT12(ha, + "unsupported srq_limit=0x%x requested" + " (max_srq_limit=0x%x)\n", + attrs->attr.srq_limit, attrs->attr.srq_limit); + return -EINVAL; + } + + QL_DPRINT12(ha, "exit\n"); + return 0; +} + + +static void +qlnxr_free_srq_user_params(struct qlnxr_srq *srq) +{ + struct qlnxr_dev *dev = srq->dev; + qlnx_host_t *ha; + + ha = dev->ha; + + QL_DPRINT12(ha, "enter\n"); + + qlnxr_free_pbl(srq->dev, &srq->usrq.pbl_info, srq->usrq.pbl_tbl); + ib_umem_release(srq->usrq.umem); + ib_umem_release(srq->prod_umem); + + QL_DPRINT12(ha, "exit\n"); + return; +} + +static void +qlnxr_free_srq_kernel_params(struct qlnxr_srq *srq) +{ + struct qlnxr_srq_hwq_info *hw_srq = &srq->hw_srq; + struct qlnxr_dev *dev = srq->dev; + qlnx_host_t *ha; + + ha = dev->ha; + + QL_DPRINT12(ha, "enter\n"); + + ecore_chain_free(dev->cdev, &hw_srq->pbl); + + qlnx_dma_free_coherent(&dev->cdev, + hw_srq->virt_prod_pair_addr, + hw_srq->phy_prod_pair_addr, + sizeof(struct rdma_srq_producers)); + + QL_DPRINT12(ha, "exit\n"); + + return; +} + +static int +qlnxr_init_srq_user_params(struct ib_ucontext *ib_ctx, + struct qlnxr_srq *srq, + struct qlnxr_create_srq_ureq *ureq, + int access, int dmasync) +{ +#ifdef DEFINE_IB_UMEM_WITH_CHUNK + struct ib_umem_chunk *chunk; +#endif + struct scatterlist *sg; + int rc; + struct qlnxr_dev *dev = srq->dev; + qlnx_host_t *ha; + + ha = dev->ha; + + QL_DPRINT12(ha, "enter\n"); + + rc = qlnxr_init_user_queue(ib_ctx, srq->dev, &srq->usrq, ureq->srq_addr, + ureq->srq_len, access, dmasync, 1); + if (rc) + return rc; + + srq->prod_umem = ib_umem_get(ib_ctx, ureq->prod_pair_addr, + sizeof(struct rdma_srq_producers), + access, dmasync); + if (IS_ERR(srq->prod_umem)) { + + qlnxr_free_pbl(srq->dev, &srq->usrq.pbl_info, srq->usrq.pbl_tbl); + ib_umem_release(srq->usrq.umem); + + QL_DPRINT12(ha, "ib_umem_get failed for producer [%p]\n", + PTR_ERR(srq->prod_umem)); + + return PTR_ERR(srq->prod_umem); + } + +#ifdef DEFINE_IB_UMEM_WITH_CHUNK + chunk = container_of((&srq->prod_umem->chunk_list)->next, + typeof(*chunk), list); + sg = &chunk->page_list[0]; +#else + sg = srq->prod_umem->sg_head.sgl; +#endif + srq->hw_srq.phy_prod_pair_addr = sg_dma_address(sg); + + QL_DPRINT12(ha, "exit\n"); + return 0; +} + + +static int +qlnxr_alloc_srq_kernel_params(struct qlnxr_srq *srq, + struct qlnxr_dev *dev, + struct ib_srq_init_attr *init_attr) +{ + struct qlnxr_srq_hwq_info *hw_srq = &srq->hw_srq; + dma_addr_t phy_prod_pair_addr; + u32 num_elems, max_wr; + void *va; + int rc; + qlnx_host_t *ha; + + ha = dev->ha; + + QL_DPRINT12(ha, "enter\n"); + + va = qlnx_dma_alloc_coherent(&dev->cdev, + &phy_prod_pair_addr, + sizeof(struct rdma_srq_producers)); + if (!va) { + QL_DPRINT11(ha, "qlnx_dma_alloc_coherent failed for produceer\n"); + return -ENOMEM; + } + + hw_srq->phy_prod_pair_addr = phy_prod_pair_addr; + hw_srq->virt_prod_pair_addr = va; + + max_wr = init_attr->attr.max_wr; + + num_elems = max_wr * RDMA_MAX_SRQ_WQE_SIZE; + + rc = ecore_chain_alloc(dev->cdev, + ECORE_CHAIN_USE_TO_CONSUME_PRODUCE, + ECORE_CHAIN_MODE_PBL, + ECORE_CHAIN_CNT_TYPE_U32, + num_elems, + ECORE_RDMA_SRQ_WQE_ELEM_SIZE, + &hw_srq->pbl, NULL); + + if (rc) { + QL_DPRINT11(ha, "ecore_chain_alloc failed [%d]\n", rc); + goto err0; + } + + hw_srq->max_wr = max_wr; + hw_srq->num_elems = num_elems; + hw_srq->max_sges = RDMA_MAX_SGE_PER_SRQ; + + QL_DPRINT12(ha, "exit\n"); + return 0; + +err0: + qlnx_dma_free_coherent(&dev->cdev, va, phy_prod_pair_addr, + sizeof(struct rdma_srq_producers)); + + QL_DPRINT12(ha, "exit [%d]\n", rc); + return rc; +} + +static inline void +qlnxr_init_common_qp_in_params(struct qlnxr_dev *dev, + struct qlnxr_pd *pd, + struct qlnxr_qp *qp, + struct ib_qp_init_attr *attrs, + bool fmr_and_reserved_lkey, + struct ecore_rdma_create_qp_in_params *params) +{ + qlnx_host_t *ha; + + ha = dev->ha; + + QL_DPRINT12(ha, "enter\n"); + + /* QP handle to be written in an async event */ + params->qp_handle_async_lo = lower_32_bits((uintptr_t)qp); + params->qp_handle_async_hi = upper_32_bits((uintptr_t)qp); + + params->signal_all = (attrs->sq_sig_type == IB_SIGNAL_ALL_WR); + params->fmr_and_reserved_lkey = fmr_and_reserved_lkey; + params->pd = pd->pd_id; + params->dpi = pd->uctx ? pd->uctx->dpi : dev->dpi; + params->sq_cq_id = get_qlnxr_cq(attrs->send_cq)->icid; + params->stats_queue = 0; + + params->rq_cq_id = get_qlnxr_cq(attrs->recv_cq)->icid; + + if (qp->srq) { + /* QP is associated with SRQ instead of RQ */ + params->srq_id = qp->srq->srq_id; + params->use_srq = true; + QL_DPRINT11(ha, "exit srq_id = 0x%x use_srq = 0x%x\n", + params->srq_id, params->use_srq); + return; + } + + params->srq_id = 0; + params->use_srq = false; + + QL_DPRINT12(ha, "exit\n"); + return; +} + + +static inline void +qlnxr_qp_user_print( struct qlnxr_dev *dev, + struct qlnxr_qp *qp) +{ + QL_DPRINT12((dev->ha), "qp=%p. sq_addr=0x%llx, sq_len=%zd, " + "rq_addr=0x%llx, rq_len=%zd\n", + qp, qp->usq.buf_addr, qp->usq.buf_len, qp->urq.buf_addr, + qp->urq.buf_len); + return; +} + +static int +qlnxr_idr_add(struct qlnxr_dev *dev, void *ptr, u32 id) +{ + u32 newid; + int rc; + qlnx_host_t *ha; + + ha = dev->ha; + + QL_DPRINT12(ha, "enter\n"); + + if (!QLNX_IS_IWARP(dev)) + return 0; + + do { + if (!idr_pre_get(&dev->qpidr, GFP_KERNEL)) { + QL_DPRINT11(ha, "idr_pre_get failed\n"); + return -ENOMEM; + } + + mtx_lock(&dev->idr_lock); + + rc = idr_get_new_above(&dev->qpidr, ptr, id, &newid); + + mtx_unlock(&dev->idr_lock); + + } while (rc == -EAGAIN); + + QL_DPRINT12(ha, "exit [%d]\n", rc); + + return rc; +} + +static void +qlnxr_idr_remove(struct qlnxr_dev *dev, u32 id) +{ + qlnx_host_t *ha; + + ha = dev->ha; + + QL_DPRINT12(ha, "enter\n"); + + if (!QLNX_IS_IWARP(dev)) + return; + + mtx_lock(&dev->idr_lock); + idr_remove(&dev->qpidr, id); + mtx_unlock(&dev->idr_lock); + + QL_DPRINT12(ha, "exit \n"); + + return; +} + +static inline void +qlnxr_iwarp_populate_user_qp(struct qlnxr_dev *dev, + struct qlnxr_qp *qp, + struct ecore_rdma_create_qp_out_params *out_params) +{ + qlnx_host_t *ha; + + ha = dev->ha; + + QL_DPRINT12(ha, "enter\n"); + + qp->usq.pbl_tbl->va = out_params->sq_pbl_virt; + qp->usq.pbl_tbl->pa = out_params->sq_pbl_phys; + + qlnxr_populate_pbls(dev, qp->usq.umem, qp->usq.pbl_tbl, + &qp->usq.pbl_info); + + if (qp->srq) { + QL_DPRINT11(ha, "qp->srq = %p\n", qp->srq); + return; + } + + qp->urq.pbl_tbl->va = out_params->rq_pbl_virt; + qp->urq.pbl_tbl->pa = out_params->rq_pbl_phys; + + qlnxr_populate_pbls(dev, qp->urq.umem, qp->urq.pbl_tbl, + &qp->urq.pbl_info); + + QL_DPRINT12(ha, "exit\n"); + return; +} + +static int +qlnxr_create_user_qp(struct qlnxr_dev *dev, + struct qlnxr_qp *qp, + struct ib_pd *ibpd, + struct ib_udata *udata, + struct ib_qp_init_attr *attrs) +{ + struct ecore_rdma_destroy_qp_out_params d_out_params; + struct ecore_rdma_create_qp_in_params in_params; + struct ecore_rdma_create_qp_out_params out_params; + struct qlnxr_pd *pd = get_qlnxr_pd(ibpd); + struct ib_ucontext *ib_ctx = NULL; + struct qlnxr_ucontext *ctx = NULL; + struct qlnxr_create_qp_ureq ureq; + int alloc_and_init = QLNX_IS_ROCE(dev); + int rc = -EINVAL; + qlnx_host_t *ha; + + ha = dev->ha; + + QL_DPRINT12(ha, "enter\n"); + + ib_ctx = ibpd->uobject->context; + ctx = get_qlnxr_ucontext(ib_ctx); + + memset(&ureq, 0, sizeof(ureq)); + rc = ib_copy_from_udata(&ureq, udata, sizeof(ureq)); + + if (rc) { + QL_DPRINT11(ha, "ib_copy_from_udata failed [%d]\n", rc); + return rc; + } + + /* SQ - read access only (0), dma sync not required (0) */ + rc = qlnxr_init_user_queue(ib_ctx, dev, &qp->usq, ureq.sq_addr, + ureq.sq_len, 0, 0, + alloc_and_init); + if (rc) { + QL_DPRINT11(ha, "qlnxr_init_user_queue failed [%d]\n", rc); + return rc; + } + + if (!qp->srq) { + /* RQ - read access only (0), dma sync not required (0) */ + rc = qlnxr_init_user_queue(ib_ctx, dev, &qp->urq, ureq.rq_addr, + ureq.rq_len, 0, 0, + alloc_and_init); + + if (rc) { + QL_DPRINT11(ha, "qlnxr_init_user_queue failed [%d]\n", rc); + return rc; + } + } + + memset(&in_params, 0, sizeof(in_params)); + qlnxr_init_common_qp_in_params(dev, pd, qp, attrs, false, &in_params); + in_params.qp_handle_lo = ureq.qp_handle_lo; + in_params.qp_handle_hi = ureq.qp_handle_hi; + in_params.sq_num_pages = qp->usq.pbl_info.num_pbes; + in_params.sq_pbl_ptr = qp->usq.pbl_tbl->pa; + + if (!qp->srq) { + in_params.rq_num_pages = qp->urq.pbl_info.num_pbes; + in_params.rq_pbl_ptr = qp->urq.pbl_tbl->pa; + } + + qp->ecore_qp = ecore_rdma_create_qp(dev->rdma_ctx, &in_params, &out_params); + + if (!qp->ecore_qp) { + rc = -ENOMEM; + QL_DPRINT11(ha, "ecore_rdma_create_qp failed\n"); + goto err1; + } + + if (QLNX_IS_IWARP(dev)) + qlnxr_iwarp_populate_user_qp(dev, qp, &out_params); + + qp->qp_id = out_params.qp_id; + qp->icid = out_params.icid; + + rc = qlnxr_copy_qp_uresp(dev, qp, udata); + + if (rc) { + QL_DPRINT11(ha, "qlnxr_copy_qp_uresp failed\n"); + goto err; + } + + qlnxr_qp_user_print(dev, qp); + + QL_DPRINT12(ha, "exit\n"); + return 0; +err: + rc = ecore_rdma_destroy_qp(dev->rdma_ctx, qp->ecore_qp, &d_out_params); + + if (rc) + QL_DPRINT12(ha, "fatal fault\n"); + +err1: + qlnxr_cleanup_user(dev, qp); + + QL_DPRINT12(ha, "exit[%d]\n", rc); + return rc; +} + +static void +qlnxr_set_roce_db_info(struct qlnxr_dev *dev, + struct qlnxr_qp *qp) +{ + qlnx_host_t *ha; + + ha = dev->ha; + + QL_DPRINT12(ha, "enter qp = %p qp->srq %p\n", qp, qp->srq); + + qp->sq.db = dev->db_addr + + DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD); + qp->sq.db_data.data.icid = qp->icid + 1; + + if (!qp->srq) { + qp->rq.db = dev->db_addr + + DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_ROCE_RQ_PROD); + qp->rq.db_data.data.icid = qp->icid; + } + + QL_DPRINT12(ha, "exit\n"); + return; +} + +static void +qlnxr_set_iwarp_db_info(struct qlnxr_dev *dev, + struct qlnxr_qp *qp) + +{ + qlnx_host_t *ha; + + ha = dev->ha; + + QL_DPRINT12(ha, "enter qp = %p qp->srq %p\n", qp, qp->srq); + + qp->sq.db = dev->db_addr + + DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD); + qp->sq.db_data.data.icid = qp->icid; + + if (!qp->srq) { + qp->rq.db = dev->db_addr + + DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_IWARP_RQ_PROD); + qp->rq.db_data.data.icid = qp->icid; + + qp->rq.iwarp_db2 = dev->db_addr + + DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_FLAGS); + qp->rq.iwarp_db2_data.data.icid = qp->icid; + qp->rq.iwarp_db2_data.data.value = DQ_TCM_IWARP_POST_RQ_CF_CMD; + } + + QL_DPRINT12(ha, + "qp->sq.db = %p qp->sq.db_data.data.icid =0x%x\n" + "\t\t\tqp->rq.db = %p qp->rq.db_data.data.icid =0x%x\n" + "\t\t\tqp->rq.iwarp_db2 = %p qp->rq.iwarp_db2.data.icid =0x%x" + " qp->rq.iwarp_db2.data.prod_val =0x%x\n", + qp->sq.db, qp->sq.db_data.data.icid, + qp->rq.db, qp->rq.db_data.data.icid, + qp->rq.iwarp_db2, qp->rq.iwarp_db2_data.data.icid, + qp->rq.iwarp_db2_data.data.value); + + QL_DPRINT12(ha, "exit\n"); + return; +} + +static int +qlnxr_roce_create_kernel_qp(struct qlnxr_dev *dev, + struct qlnxr_qp *qp, + struct ecore_rdma_create_qp_in_params *in_params, + u32 n_sq_elems, + u32 n_rq_elems) +{ + struct ecore_rdma_create_qp_out_params out_params; + int rc; + qlnx_host_t *ha; + + ha = dev->ha; + + QL_DPRINT12(ha, "enter\n"); + + rc = ecore_chain_alloc( + dev->cdev, + ECORE_CHAIN_USE_TO_PRODUCE, + ECORE_CHAIN_MODE_PBL, + ECORE_CHAIN_CNT_TYPE_U32, + n_sq_elems, + QLNXR_SQE_ELEMENT_SIZE, + &qp->sq.pbl, + NULL); + + if (rc) { + QL_DPRINT11(ha, "ecore_chain_alloc qp->sq.pbl failed[%d]\n", rc); + return rc; + } + + in_params->sq_num_pages = ecore_chain_get_page_cnt(&qp->sq.pbl); + in_params->sq_pbl_ptr = ecore_chain_get_pbl_phys(&qp->sq.pbl); + + if (!qp->srq) { + + rc = ecore_chain_alloc( + dev->cdev, + ECORE_CHAIN_USE_TO_CONSUME_PRODUCE, + ECORE_CHAIN_MODE_PBL, + ECORE_CHAIN_CNT_TYPE_U32, + n_rq_elems, + QLNXR_RQE_ELEMENT_SIZE, + &qp->rq.pbl, + NULL); + + if (rc) { + QL_DPRINT11(ha, + "ecore_chain_alloc qp->rq.pbl failed[%d]\n", rc); + return rc; + } + + in_params->rq_num_pages = ecore_chain_get_page_cnt(&qp->rq.pbl); + in_params->rq_pbl_ptr = ecore_chain_get_pbl_phys(&qp->rq.pbl); + } + + qp->ecore_qp = ecore_rdma_create_qp(dev->rdma_ctx, in_params, &out_params); + + if (!qp->ecore_qp) { + QL_DPRINT11(ha, "qp->ecore_qp == NULL\n"); + return -EINVAL; + } + + qp->qp_id = out_params.qp_id; + qp->icid = out_params.icid; + + qlnxr_set_roce_db_info(dev, qp); + + QL_DPRINT12(ha, "exit\n"); + return 0; +} + +static int +qlnxr_iwarp_create_kernel_qp(struct qlnxr_dev *dev, + struct qlnxr_qp *qp, + struct ecore_rdma_create_qp_in_params *in_params, + u32 n_sq_elems, + u32 n_rq_elems) +{ + struct ecore_rdma_destroy_qp_out_params d_out_params; + struct ecore_rdma_create_qp_out_params out_params; + struct ecore_chain_ext_pbl ext_pbl; + int rc; + qlnx_host_t *ha; + + ha = dev->ha; + + QL_DPRINT12(ha, "enter\n"); + + in_params->sq_num_pages = ECORE_CHAIN_PAGE_CNT(n_sq_elems, + QLNXR_SQE_ELEMENT_SIZE, + ECORE_CHAIN_MODE_PBL); + in_params->rq_num_pages = ECORE_CHAIN_PAGE_CNT(n_rq_elems, + QLNXR_RQE_ELEMENT_SIZE, + ECORE_CHAIN_MODE_PBL); + + QL_DPRINT12(ha, "n_sq_elems = 0x%x" + " n_rq_elems = 0x%x in_params\n" + "\t\t\tqp_handle_lo\t\t= 0x%08x\n" + "\t\t\tqp_handle_hi\t\t= 0x%08x\n" + "\t\t\tqp_handle_async_lo\t\t= 0x%08x\n" + "\t\t\tqp_handle_async_hi\t\t= 0x%08x\n" + "\t\t\tuse_srq\t\t\t= 0x%x\n" + "\t\t\tsignal_all\t\t= 0x%x\n" + "\t\t\tfmr_and_reserved_lkey\t= 0x%x\n" + "\t\t\tpd\t\t\t= 0x%x\n" + "\t\t\tdpi\t\t\t= 0x%x\n" + "\t\t\tsq_cq_id\t\t\t= 0x%x\n" + "\t\t\tsq_num_pages\t\t= 0x%x\n" + "\t\t\tsq_pbl_ptr\t\t= %p\n" + "\t\t\tmax_sq_sges\t\t= 0x%x\n" + "\t\t\trq_cq_id\t\t\t= 0x%x\n" + "\t\t\trq_num_pages\t\t= 0x%x\n" + "\t\t\trq_pbl_ptr\t\t= %p\n" + "\t\t\tsrq_id\t\t\t= 0x%x\n" + "\t\t\tstats_queue\t\t= 0x%x\n", + n_sq_elems, n_rq_elems, + in_params->qp_handle_lo, + in_params->qp_handle_hi, + in_params->qp_handle_async_lo, + in_params->qp_handle_async_hi, + in_params->use_srq, + in_params->signal_all, + in_params->fmr_and_reserved_lkey, + in_params->pd, + in_params->dpi, + in_params->sq_cq_id, + in_params->sq_num_pages, + (void *)in_params->sq_pbl_ptr, + in_params->max_sq_sges, + in_params->rq_cq_id, + in_params->rq_num_pages, + (void *)in_params->rq_pbl_ptr, + in_params->srq_id, + in_params->stats_queue ); + + memset(&out_params, 0, sizeof (struct ecore_rdma_create_qp_out_params)); + memset(&ext_pbl, 0, sizeof (struct ecore_chain_ext_pbl)); + + qp->ecore_qp = ecore_rdma_create_qp(dev->rdma_ctx, in_params, &out_params); + + if (!qp->ecore_qp) { + QL_DPRINT11(ha, "ecore_rdma_create_qp failed\n"); + return -EINVAL; + } + + /* Now we allocate the chain */ + ext_pbl.p_pbl_virt = out_params.sq_pbl_virt; + ext_pbl.p_pbl_phys = out_params.sq_pbl_phys; + + QL_DPRINT12(ha, "ext_pbl.p_pbl_virt = %p " + "ext_pbl.p_pbl_phys = %p\n", + ext_pbl.p_pbl_virt, ext_pbl.p_pbl_phys); + + rc = ecore_chain_alloc( + dev->cdev, + ECORE_CHAIN_USE_TO_PRODUCE, + ECORE_CHAIN_MODE_PBL, + ECORE_CHAIN_CNT_TYPE_U32, + n_sq_elems, + QLNXR_SQE_ELEMENT_SIZE, + &qp->sq.pbl, + &ext_pbl); + + if (rc) { + QL_DPRINT11(ha, + "ecore_chain_alloc qp->sq.pbl failed rc = %d\n", rc); + goto err; + } + + ext_pbl.p_pbl_virt = out_params.rq_pbl_virt; + ext_pbl.p_pbl_phys = out_params.rq_pbl_phys; + + QL_DPRINT12(ha, "ext_pbl.p_pbl_virt = %p " + "ext_pbl.p_pbl_phys = %p\n", + ext_pbl.p_pbl_virt, ext_pbl.p_pbl_phys); + + if (!qp->srq) { + + rc = ecore_chain_alloc( + dev->cdev, + ECORE_CHAIN_USE_TO_CONSUME_PRODUCE, + ECORE_CHAIN_MODE_PBL, + ECORE_CHAIN_CNT_TYPE_U32, + n_rq_elems, + QLNXR_RQE_ELEMENT_SIZE, + &qp->rq.pbl, + &ext_pbl); + + if (rc) { + QL_DPRINT11(ha,, "ecore_chain_alloc qp->rq.pbl" + " failed rc = %d\n", rc); + goto err; + } + } + + QL_DPRINT12(ha, "qp_id = 0x%x icid =0x%x\n", + out_params.qp_id, out_params.icid); + + qp->qp_id = out_params.qp_id; + qp->icid = out_params.icid; + + qlnxr_set_iwarp_db_info(dev, qp); + + QL_DPRINT12(ha, "exit\n"); + return 0; + +err: + ecore_rdma_destroy_qp(dev->rdma_ctx, qp->ecore_qp, &d_out_params); + + QL_DPRINT12(ha, "exit rc = %d\n", rc); + return rc; +} + +static int +qlnxr_create_kernel_qp(struct qlnxr_dev *dev, + struct qlnxr_qp *qp, + struct ib_pd *ibpd, + struct ib_qp_init_attr *attrs) +{ + struct ecore_rdma_create_qp_in_params in_params; + struct qlnxr_pd *pd = get_qlnxr_pd(ibpd); + int rc = -EINVAL; + u32 n_rq_elems; + u32 n_sq_elems; + u32 n_sq_entries; + struct ecore_rdma_device *qattr = ecore_rdma_query_device(dev->rdma_ctx); + qlnx_host_t *ha; + + ha = dev->ha; + + QL_DPRINT12(ha, "enter\n"); + + memset(&in_params, 0, sizeof(in_params)); + + /* A single work request may take up to MAX_SQ_WQE_SIZE elements in + * the ring. The ring should allow at least a single WR, even if the + * user requested none, due to allocation issues. + * We should add an extra WR since the prod and cons indices of + * wqe_wr_id are managed in such a way that the WQ is considered full + * when (prod+1)%max_wr==cons. We currently don't do that because we + * double the number of entries due an iSER issue that pushes far more + * WRs than indicated. If we decline its ib_post_send() then we get + * error prints in the dmesg we'd like to avoid. + */ + qp->sq.max_wr = min_t(u32, attrs->cap.max_send_wr * dev->wq_multiplier, + qattr->max_wqe); + + qp->wqe_wr_id = kzalloc(qp->sq.max_wr * sizeof(*qp->wqe_wr_id), + GFP_KERNEL); + if (!qp->wqe_wr_id) { + QL_DPRINT11(ha, "failed SQ shadow memory allocation\n"); + return -ENOMEM; + } + + /* QP handle to be written in CQE */ + in_params.qp_handle_lo = lower_32_bits((uintptr_t)qp); + in_params.qp_handle_hi = upper_32_bits((uintptr_t)qp); + + /* A single work request may take up to MAX_RQ_WQE_SIZE elements in + * the ring. There ring should allow at least a single WR, even if the + * user requested none, due to allocation issues. + */ + qp->rq.max_wr = (u16)max_t(u32, attrs->cap.max_recv_wr, 1); + + /* Allocate driver internal RQ array */ + if (!qp->srq) { + qp->rqe_wr_id = kzalloc(qp->rq.max_wr * sizeof(*qp->rqe_wr_id), + GFP_KERNEL); + if (!qp->rqe_wr_id) { + QL_DPRINT11(ha, "failed RQ shadow memory allocation\n"); + kfree(qp->wqe_wr_id); + return -ENOMEM; + } + } + + //qlnxr_init_common_qp_in_params(dev, pd, qp, attrs, true, &in_params); + + in_params.qp_handle_async_lo = lower_32_bits((uintptr_t)qp); + in_params.qp_handle_async_hi = upper_32_bits((uintptr_t)qp); + + in_params.signal_all = (attrs->sq_sig_type == IB_SIGNAL_ALL_WR); + in_params.fmr_and_reserved_lkey = true; + in_params.pd = pd->pd_id; + in_params.dpi = pd->uctx ? pd->uctx->dpi : dev->dpi; + in_params.sq_cq_id = get_qlnxr_cq(attrs->send_cq)->icid; + in_params.stats_queue = 0; + + in_params.rq_cq_id = get_qlnxr_cq(attrs->recv_cq)->icid; + + if (qp->srq) { + /* QP is associated with SRQ instead of RQ */ + in_params.srq_id = qp->srq->srq_id; + in_params.use_srq = true; + QL_DPRINT11(ha, "exit srq_id = 0x%x use_srq = 0x%x\n", + in_params.srq_id, in_params.use_srq); + } else { + in_params.srq_id = 0; + in_params.use_srq = false; + } + + n_sq_entries = attrs->cap.max_send_wr; + n_sq_entries = min_t(u32, n_sq_entries, qattr->max_wqe); + n_sq_entries = max_t(u32, n_sq_entries, 1); + n_sq_elems = n_sq_entries * QLNXR_MAX_SQE_ELEMENTS_PER_SQE; + + n_rq_elems = qp->rq.max_wr * QLNXR_MAX_RQE_ELEMENTS_PER_RQE; + + if (QLNX_IS_ROCE(dev)) { + rc = qlnxr_roce_create_kernel_qp(dev, qp, &in_params, + n_sq_elems, n_rq_elems); + } else { + rc = qlnxr_iwarp_create_kernel_qp(dev, qp, &in_params, + n_sq_elems, n_rq_elems); + } + + if (rc) + qlnxr_cleanup_kernel(dev, qp); + + QL_DPRINT12(ha, "exit [%d]\n", rc); + return rc; +} + +struct ib_qp * +qlnxr_create_qp(struct ib_pd *ibpd, + struct ib_qp_init_attr *attrs, + struct ib_udata *udata) +{ + struct qlnxr_dev *dev = get_qlnxr_dev(ibpd->device); + struct qlnxr_pd *pd = get_qlnxr_pd(ibpd); + struct qlnxr_qp *qp; + int rc = 0; + qlnx_host_t *ha; + + ha = dev->ha; + + QL_DPRINT12(ha, "enter\n"); + + rc = qlnxr_check_qp_attrs(ibpd, dev, attrs, udata); + if (rc) { + QL_DPRINT11(ha, "qlnxr_check_qp_attrs failed [%d]\n", rc); + return ERR_PTR(rc); + } + + QL_DPRINT12(ha, "called from %s, event_handle=%p," + " eepd=%p sq_cq=%p, sq_icid=%d, rq_cq=%p, rq_icid=%d\n", + (udata ? "user library" : "kernel"), + attrs->event_handler, pd, + get_qlnxr_cq(attrs->send_cq), + get_qlnxr_cq(attrs->send_cq)->icid, + get_qlnxr_cq(attrs->recv_cq), + get_qlnxr_cq(attrs->recv_cq)->icid); + + qp = qlnx_zalloc(sizeof(struct qlnxr_qp)); + + if (!qp) { + QL_DPRINT11(ha, "kzalloc(qp) failed\n"); + return ERR_PTR(-ENOMEM); + } + + qlnxr_set_common_qp_params(dev, qp, pd, attrs); + + if (attrs->qp_type == IB_QPT_GSI) { + QL_DPRINT11(ha, "calling qlnxr_create_gsi_qp\n"); + return qlnxr_create_gsi_qp(dev, attrs, qp); + } + + if (udata) { + rc = qlnxr_create_user_qp(dev, qp, ibpd, udata, attrs); + + if (rc) { + QL_DPRINT11(ha, "qlnxr_create_user_qp failed\n"); + goto err; + } + } else { + rc = qlnxr_create_kernel_qp(dev, qp, ibpd, attrs); + + if (rc) { + QL_DPRINT11(ha, "qlnxr_create_kernel_qp failed\n"); + goto err; + } + } + + qp->ibqp.qp_num = qp->qp_id; + + rc = qlnxr_idr_add(dev, qp, qp->qp_id); + + if (rc) { + QL_DPRINT11(ha, "qlnxr_idr_add failed\n"); + goto err; + } + + QL_DPRINT12(ha, "exit [%p]\n", &qp->ibqp); + + return &qp->ibqp; +err: + kfree(qp); + + QL_DPRINT12(ha, "failed exit\n"); + return ERR_PTR(-EFAULT); +} + + +static enum ib_qp_state +qlnxr_get_ibqp_state(enum ecore_roce_qp_state qp_state) +{ + enum ib_qp_state state = IB_QPS_ERR; + + switch (qp_state) { + case ECORE_ROCE_QP_STATE_RESET: + state = IB_QPS_RESET; + break; + + case ECORE_ROCE_QP_STATE_INIT: + state = IB_QPS_INIT; + break; + + case ECORE_ROCE_QP_STATE_RTR: + state = IB_QPS_RTR; + break; + + case ECORE_ROCE_QP_STATE_RTS: + state = IB_QPS_RTS; + break; + + case ECORE_ROCE_QP_STATE_SQD: + state = IB_QPS_SQD; + break; + + case ECORE_ROCE_QP_STATE_ERR: + state = IB_QPS_ERR; + break; + + case ECORE_ROCE_QP_STATE_SQE: + state = IB_QPS_SQE; + break; + } + return state; +} + +static enum ecore_roce_qp_state +qlnxr_get_state_from_ibqp( enum ib_qp_state qp_state) +{ + enum ecore_roce_qp_state ecore_qp_state; + + ecore_qp_state = ECORE_ROCE_QP_STATE_ERR; + + switch (qp_state) { + case IB_QPS_RESET: + ecore_qp_state = ECORE_ROCE_QP_STATE_RESET; + break; + + case IB_QPS_INIT: + ecore_qp_state = ECORE_ROCE_QP_STATE_INIT; + break; + + case IB_QPS_RTR: + ecore_qp_state = ECORE_ROCE_QP_STATE_RTR; + break; + + case IB_QPS_RTS: + ecore_qp_state = ECORE_ROCE_QP_STATE_RTS; + break; + + case IB_QPS_SQD: + ecore_qp_state = ECORE_ROCE_QP_STATE_SQD; + break; + + case IB_QPS_ERR: + ecore_qp_state = ECORE_ROCE_QP_STATE_ERR; + break; + + default: + ecore_qp_state = ECORE_ROCE_QP_STATE_ERR; + break; + } + + return (ecore_qp_state); +} + +static void +qlnxr_reset_qp_hwq_info(struct qlnxr_qp_hwq_info *qph) +{ + ecore_chain_reset(&qph->pbl); + qph->prod = qph->cons = 0; + qph->wqe_cons = 0; + qph->db_data.data.value = cpu_to_le16(0); + + return; +} + +static int +qlnxr_update_qp_state(struct qlnxr_dev *dev, + struct qlnxr_qp *qp, + enum ecore_roce_qp_state new_state) +{ + int status = 0; + uint32_t reg_addr; + struct ecore_dev *cdev; + qlnx_host_t *ha; + + ha = dev->ha; + cdev = &ha->cdev; + + QL_DPRINT12(ha, "enter qp = %p new_state = 0x%x qp->state = 0x%x\n", + qp, new_state, qp->state); + + if (new_state == qp->state) { + return 0; + } + + switch (qp->state) { + case ECORE_ROCE_QP_STATE_RESET: + switch (new_state) { + case ECORE_ROCE_QP_STATE_INIT: + qp->prev_wqe_size = 0; + qlnxr_reset_qp_hwq_info(&qp->sq); + if (!(qp->srq)) + qlnxr_reset_qp_hwq_info(&qp->rq); + break; + default: + status = -EINVAL; + break; + }; + break; + case ECORE_ROCE_QP_STATE_INIT: + /* INIT->XXX */ + switch (new_state) { + case ECORE_ROCE_QP_STATE_RTR: + /* Update doorbell (in case post_recv was done before move to RTR) */ + if (qp->srq) + break; + wmb(); + //writel(qp->rq.db_data.raw, qp->rq.db); + //if (QLNX_IS_IWARP(dev)) + // writel(qp->rq.iwarp_db2_data.raw, + // qp->rq.iwarp_db2); + + reg_addr = (uint32_t)((uint8_t *)qp->rq.db - + (uint8_t *)cdev->doorbells); + + bus_write_4(ha->pci_dbells, reg_addr, qp->rq.db_data.raw); + bus_barrier(ha->pci_dbells, 0, 0, BUS_SPACE_BARRIER_READ); + + if (QLNX_IS_IWARP(dev)) { + reg_addr = (uint32_t)((uint8_t *)qp->rq.iwarp_db2 - + (uint8_t *)cdev->doorbells); + bus_write_4(ha->pci_dbells, reg_addr,\ + qp->rq.iwarp_db2_data.raw); + bus_barrier(ha->pci_dbells, 0, 0,\ + BUS_SPACE_BARRIER_READ); + } + + + mmiowb(); + break; + case ECORE_ROCE_QP_STATE_ERR: + /* TBD:flush qps... */ + break; + default: + /* invalid state change. */ + status = -EINVAL; + break; + }; + break; + case ECORE_ROCE_QP_STATE_RTR: + /* RTR->XXX */ + switch (new_state) { + case ECORE_ROCE_QP_STATE_RTS: + break; + case ECORE_ROCE_QP_STATE_ERR: + break; + default: + /* invalid state change. */ + status = -EINVAL; + break; + }; + break; + case ECORE_ROCE_QP_STATE_RTS: + /* RTS->XXX */ + switch (new_state) { + case ECORE_ROCE_QP_STATE_SQD: + break; + case ECORE_ROCE_QP_STATE_ERR: + break; + default: + /* invalid state change. */ + status = -EINVAL; + break; + }; + break; + case ECORE_ROCE_QP_STATE_SQD: + /* SQD->XXX */ + switch (new_state) { + case ECORE_ROCE_QP_STATE_RTS: + case ECORE_ROCE_QP_STATE_ERR: + break; + default: + /* invalid state change. */ + status = -EINVAL; + break; + }; + break; + case ECORE_ROCE_QP_STATE_ERR: + /* ERR->XXX */ + switch (new_state) { + case ECORE_ROCE_QP_STATE_RESET: + if ((qp->rq.prod != qp->rq.cons) || + (qp->sq.prod != qp->sq.cons)) { + QL_DPRINT11(ha, + "Error->Reset with rq/sq " + "not empty rq.prod=0x%x rq.cons=0x%x" + " sq.prod=0x%x sq.cons=0x%x\n", + qp->rq.prod, qp->rq.cons, + qp->sq.prod, qp->sq.cons); + status = -EINVAL; + } + break; + default: + status = -EINVAL; + break; + }; + break; + default: + status = -EINVAL; + break; + }; + + QL_DPRINT12(ha, "exit\n"); + return status; +} + +int +qlnxr_modify_qp(struct ib_qp *ibqp, + struct ib_qp_attr *attr, + int attr_mask, + struct ib_udata *udata) +{ + int rc = 0; + struct qlnxr_qp *qp = get_qlnxr_qp(ibqp); + struct qlnxr_dev *dev = get_qlnxr_dev(&qp->dev->ibdev); + struct ecore_rdma_modify_qp_in_params qp_params = { 0 }; + enum ib_qp_state old_qp_state, new_qp_state; + struct ecore_rdma_device *qattr = ecore_rdma_query_device(dev->rdma_ctx); + qlnx_host_t *ha; + + ha = dev->ha; + + QL_DPRINT12(ha, + "enter qp = %p attr_mask = 0x%x, state = %d udata = %p\n", + qp, attr_mask, attr->qp_state, udata); + + old_qp_state = qlnxr_get_ibqp_state(qp->state); + if (attr_mask & IB_QP_STATE) + new_qp_state = attr->qp_state; + else + new_qp_state = old_qp_state; + + if (QLNX_IS_ROCE(dev)) { +#if __FreeBSD_version >= 1100000 + if (!ib_modify_qp_is_ok(old_qp_state, + new_qp_state, + ibqp->qp_type, + attr_mask, + IB_LINK_LAYER_ETHERNET)) { + QL_DPRINT12(ha, + "invalid attribute mask=0x%x" + " specified for qpn=0x%x of type=0x%x \n" + " old_qp_state=0x%x, new_qp_state=0x%x\n", + attr_mask, qp->qp_id, ibqp->qp_type, + old_qp_state, new_qp_state); + rc = -EINVAL; + goto err; + } +#else + if (!ib_modify_qp_is_ok(old_qp_state, + new_qp_state, + ibqp->qp_type, + attr_mask )) { + QL_DPRINT12(ha, + "invalid attribute mask=0x%x" + " specified for qpn=0x%x of type=0x%x \n" + " old_qp_state=0x%x, new_qp_state=0x%x\n", + attr_mask, qp->qp_id, ibqp->qp_type, + old_qp_state, new_qp_state); + rc = -EINVAL; + goto err; + } + +#endif /* #if __FreeBSD_version >= 1100000 */ + } + /* translate the masks... */ + if (attr_mask & IB_QP_STATE) { + SET_FIELD(qp_params.modify_flags, + ECORE_RDMA_MODIFY_QP_VALID_NEW_STATE, 1); + qp_params.new_state = qlnxr_get_state_from_ibqp(attr->qp_state); + } + + // TBD consider changing ecore to be a flag as well... + if (attr_mask & IB_QP_EN_SQD_ASYNC_NOTIFY) + qp_params.sqd_async = true; + + if (attr_mask & IB_QP_PKEY_INDEX) { + SET_FIELD(qp_params.modify_flags, + ECORE_ROCE_MODIFY_QP_VALID_PKEY, + 1); + if (attr->pkey_index >= QLNXR_ROCE_PKEY_TABLE_LEN) { + rc = -EINVAL; + goto err; + } + + qp_params.pkey = QLNXR_ROCE_PKEY_DEFAULT; + } + + if (attr_mask & IB_QP_QKEY) { + qp->qkey = attr->qkey; + } + + /* tbd consider splitting in ecore.. */ + if (attr_mask & IB_QP_ACCESS_FLAGS) { + SET_FIELD(qp_params.modify_flags, + ECORE_RDMA_MODIFY_QP_VALID_RDMA_OPS_EN, 1); + qp_params.incoming_rdma_read_en = + attr->qp_access_flags & IB_ACCESS_REMOTE_READ; + qp_params.incoming_rdma_write_en = + attr->qp_access_flags & IB_ACCESS_REMOTE_WRITE; + qp_params.incoming_atomic_en = + attr->qp_access_flags & IB_ACCESS_REMOTE_ATOMIC; + } + + if (attr_mask & (IB_QP_AV | IB_QP_PATH_MTU)) { + if (attr_mask & IB_QP_PATH_MTU) { + if (attr->path_mtu < IB_MTU_256 || + attr->path_mtu > IB_MTU_4096) { + + QL_DPRINT12(ha, + "Only MTU sizes of 256, 512, 1024," + " 2048 and 4096 are supported " + " attr->path_mtu = [%d]\n", + attr->path_mtu); + + rc = -EINVAL; + goto err; + } + qp->mtu = min(ib_mtu_enum_to_int(attr->path_mtu), + ib_mtu_enum_to_int( + iboe_get_mtu(dev->ha->ifp->if_mtu))); + } + + if (qp->mtu == 0) { + qp->mtu = ib_mtu_enum_to_int( + iboe_get_mtu(dev->ha->ifp->if_mtu)); + QL_DPRINT12(ha, "fixing zetoed MTU to qp->mtu = %d\n", + qp->mtu); + } + + SET_FIELD(qp_params.modify_flags, + ECORE_ROCE_MODIFY_QP_VALID_ADDRESS_VECTOR, + 1); + + qp_params.traffic_class_tos = attr->ah_attr.grh.traffic_class; + qp_params.flow_label = attr->ah_attr.grh.flow_label; + qp_params.hop_limit_ttl = attr->ah_attr.grh.hop_limit; + + qp->sgid_idx = attr->ah_attr.grh.sgid_index; + + get_gid_info(ibqp, attr, attr_mask, dev, qp, &qp_params); + + rc = qlnxr_get_dmac(dev, &attr->ah_attr, qp_params.remote_mac_addr); + if (rc) + return rc; + + qp_params.use_local_mac = true; + memcpy(qp_params.local_mac_addr, dev->ha->primary_mac, ETH_ALEN); + + QL_DPRINT12(ha, "dgid=0x%x:0x%x:0x%x:0x%x\n", + qp_params.dgid.dwords[0], qp_params.dgid.dwords[1], + qp_params.dgid.dwords[2], qp_params.dgid.dwords[3]); + QL_DPRINT12(ha, "sgid=0x%x:0x%x:0x%x:0x%x\n", + qp_params.sgid.dwords[0], qp_params.sgid.dwords[1], + qp_params.sgid.dwords[2], qp_params.sgid.dwords[3]); + QL_DPRINT12(ha, + "remote_mac=[0x%x:0x%x:0x%x:0x%x:0x%x:0x%x]\n", + qp_params.remote_mac_addr[0], + qp_params.remote_mac_addr[1], + qp_params.remote_mac_addr[2], + qp_params.remote_mac_addr[3], + qp_params.remote_mac_addr[4], + qp_params.remote_mac_addr[5]); + + qp_params.mtu = qp->mtu; + } + + if (qp_params.mtu == 0) { + /* stay with current MTU */ + if (qp->mtu) { + qp_params.mtu = qp->mtu; + } else { + qp_params.mtu = ib_mtu_enum_to_int( + iboe_get_mtu(dev->ha->ifp->if_mtu)); + } + } + + if (attr_mask & IB_QP_TIMEOUT) { + SET_FIELD(qp_params.modify_flags, \ + ECORE_ROCE_MODIFY_QP_VALID_ACK_TIMEOUT, 1); + + qp_params.ack_timeout = attr->timeout; + if (attr->timeout) { + u32 temp; + + /* 12.7.34 LOCAL ACK TIMEOUT + * Value representing the transport (ACK) timeout for + * use by the remote, expressed as (4.096 μS*2Local ACK + * Timeout) + */ + /* We use 1UL since the temporal value may be overflow + * 32 bits + */ + temp = 4096 * (1UL << attr->timeout) / 1000 / 1000; + qp_params.ack_timeout = temp; /* FW requires [msec] */ + } + else + qp_params.ack_timeout = 0; /* infinite */ + } + if (attr_mask & IB_QP_RETRY_CNT) { + SET_FIELD(qp_params.modify_flags,\ + ECORE_ROCE_MODIFY_QP_VALID_RETRY_CNT, 1); + qp_params.retry_cnt = attr->retry_cnt; + } + + if (attr_mask & IB_QP_RNR_RETRY) { + SET_FIELD(qp_params.modify_flags, + ECORE_ROCE_MODIFY_QP_VALID_RNR_RETRY_CNT, + 1); + qp_params.rnr_retry_cnt = attr->rnr_retry; + } + + if (attr_mask & IB_QP_RQ_PSN) { + SET_FIELD(qp_params.modify_flags, + ECORE_ROCE_MODIFY_QP_VALID_RQ_PSN, + 1); + qp_params.rq_psn = attr->rq_psn; + qp->rq_psn = attr->rq_psn; + } + + if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC) { + if (attr->max_rd_atomic > qattr->max_qp_req_rd_atomic_resc) { + rc = -EINVAL; + QL_DPRINT12(ha, + "unsupported max_rd_atomic=%d, supported=%d\n", + attr->max_rd_atomic, + qattr->max_qp_req_rd_atomic_resc); + goto err; + } + + SET_FIELD(qp_params.modify_flags, + ECORE_RDMA_MODIFY_QP_VALID_MAX_RD_ATOMIC_REQ, + 1); + qp_params.max_rd_atomic_req = attr->max_rd_atomic; + } + + if (attr_mask & IB_QP_MIN_RNR_TIMER) { + SET_FIELD(qp_params.modify_flags, + ECORE_ROCE_MODIFY_QP_VALID_MIN_RNR_NAK_TIMER, + 1); + qp_params.min_rnr_nak_timer = attr->min_rnr_timer; + } + + if (attr_mask & IB_QP_SQ_PSN) { + SET_FIELD(qp_params.modify_flags, + ECORE_ROCE_MODIFY_QP_VALID_SQ_PSN, + 1); + qp_params.sq_psn = attr->sq_psn; + qp->sq_psn = attr->sq_psn; + } + + if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) { + if (attr->max_dest_rd_atomic > + qattr->max_qp_resp_rd_atomic_resc) { + QL_DPRINT12(ha, + "unsupported max_dest_rd_atomic=%d, " + "supported=%d\n", + attr->max_dest_rd_atomic, + qattr->max_qp_resp_rd_atomic_resc); + + rc = -EINVAL; + goto err; + } + + SET_FIELD(qp_params.modify_flags, + ECORE_RDMA_MODIFY_QP_VALID_MAX_RD_ATOMIC_RESP, + 1); + qp_params.max_rd_atomic_resp = attr->max_dest_rd_atomic; + } + + if (attr_mask & IB_QP_DEST_QPN) { + SET_FIELD(qp_params.modify_flags, + ECORE_ROCE_MODIFY_QP_VALID_DEST_QP, + 1); + + qp_params.dest_qp = attr->dest_qp_num; + qp->dest_qp_num = attr->dest_qp_num; + } + + /* + * Update the QP state before the actual ramrod to prevent a race with + * fast path. Modifying the QP state to error will cause the device to + * flush the CQEs and while polling the flushed CQEs will considered as + * a potential issue if the QP isn't in error state. + */ + if ((attr_mask & IB_QP_STATE) && (qp->qp_type != IB_QPT_GSI) && + (!udata) && (qp_params.new_state == ECORE_ROCE_QP_STATE_ERR)) + qp->state = ECORE_ROCE_QP_STATE_ERR; + + if (qp->qp_type != IB_QPT_GSI) + rc = ecore_rdma_modify_qp(dev->rdma_ctx, qp->ecore_qp, &qp_params); + + if (attr_mask & IB_QP_STATE) { + if ((qp->qp_type != IB_QPT_GSI) && (!udata)) + rc = qlnxr_update_qp_state(dev, qp, qp_params.new_state); + qp->state = qp_params.new_state; + } + +err: + QL_DPRINT12(ha, "exit\n"); + return rc; +} + +static int +qlnxr_to_ib_qp_acc_flags(struct ecore_rdma_query_qp_out_params *params) +{ + int ib_qp_acc_flags = 0; + + if (params->incoming_rdma_write_en) + ib_qp_acc_flags |= IB_ACCESS_REMOTE_WRITE; + if (params->incoming_rdma_read_en) + ib_qp_acc_flags |= IB_ACCESS_REMOTE_READ; + if (params->incoming_atomic_en) + ib_qp_acc_flags |= IB_ACCESS_REMOTE_ATOMIC; + if (true) /* FIXME -> local write ?? */ + ib_qp_acc_flags |= IB_ACCESS_LOCAL_WRITE; + + return ib_qp_acc_flags; +} + +static enum ib_mtu +qlnxr_mtu_int_to_enum(u16 mtu) +{ + enum ib_mtu ib_mtu_size; + + switch (mtu) { + case 256: + ib_mtu_size = IB_MTU_256; + break; + + case 512: + ib_mtu_size = IB_MTU_512; + break; + + case 1024: + ib_mtu_size = IB_MTU_1024; + break; + + case 2048: + ib_mtu_size = IB_MTU_2048; + break; + + case 4096: + ib_mtu_size = IB_MTU_4096; + break; + + default: + ib_mtu_size = IB_MTU_1024; + break; + } + return (ib_mtu_size); +} + +int +qlnxr_query_qp(struct ib_qp *ibqp, + struct ib_qp_attr *qp_attr, + int attr_mask, + struct ib_qp_init_attr *qp_init_attr) +{ + int rc = 0; + struct ecore_rdma_query_qp_out_params params; + struct qlnxr_qp *qp = get_qlnxr_qp(ibqp); + struct qlnxr_dev *dev = qp->dev; + qlnx_host_t *ha; + + ha = dev->ha; + + QL_DPRINT12(ha, "enter\n"); + + memset(¶ms, 0, sizeof(params)); + + rc = ecore_rdma_query_qp(dev->rdma_ctx, qp->ecore_qp, ¶ms); + if (rc) + goto err; + + memset(qp_attr, 0, sizeof(*qp_attr)); + memset(qp_init_attr, 0, sizeof(*qp_init_attr)); + + qp_attr->qp_state = qlnxr_get_ibqp_state(params.state); + qp_attr->cur_qp_state = qlnxr_get_ibqp_state(params.state); + + /* In some cases in iWARP qelr will ask for the state only */ + if (QLNX_IS_IWARP(dev) && (attr_mask == IB_QP_STATE)) { + QL_DPRINT11(ha, "only state requested\n"); + return 0; + } + + qp_attr->path_mtu = qlnxr_mtu_int_to_enum(params.mtu); + qp_attr->path_mig_state = IB_MIG_MIGRATED; + qp_attr->rq_psn = params.rq_psn; + qp_attr->sq_psn = params.sq_psn; + qp_attr->dest_qp_num = params.dest_qp; + + qp_attr->qp_access_flags = qlnxr_to_ib_qp_acc_flags(¶ms); + + QL_DPRINT12(ha, "qp_state = 0x%x cur_qp_state = 0x%x " + "path_mtu = %d qp_access_flags = 0x%x\n", + qp_attr->qp_state, qp_attr->cur_qp_state, qp_attr->path_mtu, + qp_attr->qp_access_flags); + + qp_attr->cap.max_send_wr = qp->sq.max_wr; + qp_attr->cap.max_recv_wr = qp->rq.max_wr; + qp_attr->cap.max_send_sge = qp->sq.max_sges; + qp_attr->cap.max_recv_sge = qp->rq.max_sges; + qp_attr->cap.max_inline_data = qp->max_inline_data; + qp_init_attr->cap = qp_attr->cap; + + memcpy(&qp_attr->ah_attr.grh.dgid.raw[0], ¶ms.dgid.bytes[0], + sizeof(qp_attr->ah_attr.grh.dgid.raw)); + + qp_attr->ah_attr.grh.flow_label = params.flow_label; + qp_attr->ah_attr.grh.sgid_index = qp->sgid_idx; + qp_attr->ah_attr.grh.hop_limit = params.hop_limit_ttl; + qp_attr->ah_attr.grh.traffic_class = params.traffic_class_tos; + + qp_attr->ah_attr.ah_flags = IB_AH_GRH; + qp_attr->ah_attr.port_num = 1; /* FIXME -> check this */ + qp_attr->ah_attr.sl = 0;/* FIXME -> check this */ + qp_attr->timeout = params.timeout; + qp_attr->rnr_retry = params.rnr_retry; + qp_attr->retry_cnt = params.retry_cnt; + qp_attr->min_rnr_timer = params.min_rnr_nak_timer; + qp_attr->pkey_index = params.pkey_index; + qp_attr->port_num = 1; /* FIXME -> check this */ + qp_attr->ah_attr.src_path_bits = 0; + qp_attr->ah_attr.static_rate = 0; + qp_attr->alt_pkey_index = 0; + qp_attr->alt_port_num = 0; + qp_attr->alt_timeout = 0; + memset(&qp_attr->alt_ah_attr, 0, sizeof(qp_attr->alt_ah_attr)); + + qp_attr->sq_draining = (params.state == ECORE_ROCE_QP_STATE_SQD) ? 1 : 0; + qp_attr->max_dest_rd_atomic = params.max_dest_rd_atomic; + qp_attr->max_rd_atomic = params.max_rd_atomic; + qp_attr->en_sqd_async_notify = (params.sqd_async)? 1 : 0; + + QL_DPRINT12(ha, "max_inline_data=%d\n", + qp_attr->cap.max_inline_data); + +err: + QL_DPRINT12(ha, "exit\n"); + return rc; +} + + +static void +qlnxr_cleanup_user(struct qlnxr_dev *dev, struct qlnxr_qp *qp) +{ + qlnx_host_t *ha; + + ha = dev->ha; + + QL_DPRINT12(ha, "enter\n"); + + if (qp->usq.umem) + ib_umem_release(qp->usq.umem); + + qp->usq.umem = NULL; + + if (qp->urq.umem) + ib_umem_release(qp->urq.umem); + + qp->urq.umem = NULL; + + QL_DPRINT12(ha, "exit\n"); + return; +} + +static void +qlnxr_cleanup_kernel(struct qlnxr_dev *dev, struct qlnxr_qp *qp) +{ + qlnx_host_t *ha; + + ha = dev->ha; + + QL_DPRINT12(ha, "enter\n"); + + if (qlnxr_qp_has_sq(qp)) { + QL_DPRINT12(ha, "freeing SQ\n"); + ha->qlnxr_debug = 1; +// ecore_chain_free(dev->cdev, &qp->sq.pbl); + ha->qlnxr_debug = 0; + kfree(qp->wqe_wr_id); + } + + if (qlnxr_qp_has_rq(qp)) { + QL_DPRINT12(ha, "freeing RQ\n"); + ha->qlnxr_debug = 1; + // ecore_chain_free(dev->cdev, &qp->rq.pbl); + ha->qlnxr_debug = 0; + kfree(qp->rqe_wr_id); + } + + QL_DPRINT12(ha, "exit\n"); + return; +} + +int +qlnxr_free_qp_resources(struct qlnxr_dev *dev, + struct qlnxr_qp *qp) +{ + int rc = 0; + qlnx_host_t *ha; + struct ecore_rdma_destroy_qp_out_params d_out_params; + + ha = dev->ha; + + QL_DPRINT12(ha, "enter\n"); + +#if 0 + if (qp->qp_type != IB_QPT_GSI) { + rc = ecore_rdma_destroy_qp(dev->rdma_ctx, qp->ecore_qp, + &d_out_params); + if (rc) + return rc; + } + + if (qp->ibqp.uobject && qp->ibqp.uobject->context) + qlnxr_cleanup_user(dev, qp); + else + qlnxr_cleanup_kernel(dev, qp); +#endif + + if (qp->ibqp.uobject && qp->ibqp.uobject->context) + qlnxr_cleanup_user(dev, qp); + else + qlnxr_cleanup_kernel(dev, qp); + + if (qp->qp_type != IB_QPT_GSI) { + rc = ecore_rdma_destroy_qp(dev->rdma_ctx, qp->ecore_qp, + &d_out_params); + if (rc) + return rc; + } + + QL_DPRINT12(ha, "exit\n"); + return 0; +} + +int +qlnxr_destroy_qp(struct ib_qp *ibqp) +{ + struct qlnxr_qp *qp = get_qlnxr_qp(ibqp); + struct qlnxr_dev *dev = qp->dev; + int rc = 0; + struct ib_qp_attr attr; + int attr_mask = 0; + qlnx_host_t *ha; + + ha = dev->ha; + + QL_DPRINT12(ha, "enter qp = %p, qp_type=%d\n", qp, qp->qp_type); + + qp->destroyed = 1; + + if (QLNX_IS_ROCE(dev) && (qp->state != (ECORE_ROCE_QP_STATE_RESET | + ECORE_ROCE_QP_STATE_ERR | + ECORE_ROCE_QP_STATE_INIT))) { + + attr.qp_state = IB_QPS_ERR; + attr_mask |= IB_QP_STATE; + + /* change the QP state to ERROR */ + qlnxr_modify_qp(ibqp, &attr, attr_mask, NULL); + } + + if (qp->qp_type == IB_QPT_GSI) + qlnxr_destroy_gsi_qp(dev); + + qp->sig = ~qp->sig; + + qlnxr_free_qp_resources(dev, qp); + + if (atomic_dec_and_test(&qp->refcnt)) { + /* TODO: only for iWARP? */ + qlnxr_idr_remove(dev, qp->qp_id); + kfree(qp); + } + + QL_DPRINT12(ha, "exit\n"); + return rc; +} + +static inline int +qlnxr_wq_is_full(struct qlnxr_qp_hwq_info *wq) +{ + return (((wq->prod + 1) % wq->max_wr) == wq->cons); +} + +static int +sge_data_len(struct ib_sge *sg_list, int num_sge) +{ + int i, len = 0; + for (i = 0; i < num_sge; i++) + len += sg_list[i].length; + return len; +} + +static void +swap_wqe_data64(u64 *p) +{ + int i; + + for (i = 0; i < QLNXR_SQE_ELEMENT_SIZE / sizeof(u64); i++, p++) + *p = cpu_to_be64(cpu_to_le64(*p)); +} + + +static u32 +qlnxr_prepare_sq_inline_data(struct qlnxr_dev *dev, + struct qlnxr_qp *qp, + u8 *wqe_size, + struct ib_send_wr *wr, + struct ib_send_wr **bad_wr, + u8 *bits, + u8 bit) +{ + int i, seg_siz; + char *seg_prt, *wqe; + u32 data_size = sge_data_len(wr->sg_list, wr->num_sge); + qlnx_host_t *ha; + + ha = dev->ha; + + QL_DPRINT12(ha, "enter[%d]\n", data_size); + + if (data_size > ROCE_REQ_MAX_INLINE_DATA_SIZE) { + QL_DPRINT12(ha, + "Too much inline data in WR:[%d, %d]\n", + data_size, ROCE_REQ_MAX_INLINE_DATA_SIZE); + *bad_wr = wr; + return 0; + } + + if (!data_size) + return data_size; + + /* set the bit */ + *bits |= bit; + + seg_prt = wqe = NULL; + seg_siz = 0; + + /* copy data inline */ + for (i = 0; i < wr->num_sge; i++) { + u32 len = wr->sg_list[i].length; + void *src = (void *)(uintptr_t)wr->sg_list[i].addr; + + while (len > 0) { + u32 cur; + + /* new segment required */ + if (!seg_siz) { + wqe = (char *)ecore_chain_produce(&qp->sq.pbl); + seg_prt = wqe; + seg_siz = sizeof(struct rdma_sq_common_wqe); + (*wqe_size)++; + } + + /* calculate currently allowed length */ + cur = MIN(len, seg_siz); + + memcpy(seg_prt, src, cur); + + /* update segment variables */ + seg_prt += cur; + seg_siz -= cur; + /* update sge variables */ + src += cur; + len -= cur; + + /* swap fully-completed segments */ + if (!seg_siz) + swap_wqe_data64((u64 *)wqe); + } + } + + /* swap last not completed segment */ + if (seg_siz) + swap_wqe_data64((u64 *)wqe); + + QL_DPRINT12(ha, "exit\n"); + return data_size; +} + +static u32 +qlnxr_prepare_sq_sges(struct qlnxr_dev *dev, struct qlnxr_qp *qp, + u8 *wqe_size, struct ib_send_wr *wr) +{ + int i; + u32 data_size = 0; + qlnx_host_t *ha; + + ha = dev->ha; + + QL_DPRINT12(ha, "enter wr->num_sge = %d \n", wr->num_sge); + + for (i = 0; i < wr->num_sge; i++) { + struct rdma_sq_sge *sge = ecore_chain_produce(&qp->sq.pbl); + + TYPEPTR_ADDR_SET(sge, addr, wr->sg_list[i].addr); + sge->l_key = cpu_to_le32(wr->sg_list[i].lkey); + sge->length = cpu_to_le32(wr->sg_list[i].length); + data_size += wr->sg_list[i].length; + } + + if (wqe_size) + *wqe_size += wr->num_sge; + + QL_DPRINT12(ha, "exit data_size = %d\n", data_size); + return data_size; +} + +static u32 +qlnxr_prepare_sq_rdma_data(struct qlnxr_dev *dev, + struct qlnxr_qp *qp, + struct rdma_sq_rdma_wqe_1st *rwqe, + struct rdma_sq_rdma_wqe_2nd *rwqe2, + struct ib_send_wr *wr, + struct ib_send_wr **bad_wr) +{ + qlnx_host_t *ha; + u32 ret = 0; + + ha = dev->ha; + + QL_DPRINT12(ha, "enter\n"); + + rwqe2->r_key = cpu_to_le32(rdma_wr(wr)->rkey); + TYPEPTR_ADDR_SET(rwqe2, remote_va, rdma_wr(wr)->remote_addr); + + if (wr->send_flags & IB_SEND_INLINE) { + u8 flags = 0; + SET_FIELD2(flags, RDMA_SQ_RDMA_WQE_1ST_INLINE_FLG, 1); + return qlnxr_prepare_sq_inline_data(dev, qp, &rwqe->wqe_size, + wr, bad_wr, &rwqe->flags, flags); + } + + ret = qlnxr_prepare_sq_sges(dev, qp, &rwqe->wqe_size, wr); + + QL_DPRINT12(ha, "exit ret = 0x%x\n", ret); + + return (ret); +} + +static u32 +qlnxr_prepare_sq_send_data(struct qlnxr_dev *dev, + struct qlnxr_qp *qp, + struct rdma_sq_send_wqe *swqe, + struct rdma_sq_send_wqe *swqe2, + struct ib_send_wr *wr, + struct ib_send_wr **bad_wr) +{ + qlnx_host_t *ha; + u32 ret = 0; + + ha = dev->ha; + + QL_DPRINT12(ha, "enter\n"); + + memset(swqe2, 0, sizeof(*swqe2)); + + if (wr->send_flags & IB_SEND_INLINE) { + u8 flags = 0; + SET_FIELD2(flags, RDMA_SQ_SEND_WQE_INLINE_FLG, 1); + return qlnxr_prepare_sq_inline_data(dev, qp, &swqe->wqe_size, + wr, bad_wr, &swqe->flags, flags); + } + + ret = qlnxr_prepare_sq_sges(dev, qp, &swqe->wqe_size, wr); + + QL_DPRINT12(ha, "exit ret = 0x%x\n", ret); + + return (ret); +} + +static void +qlnx_handle_completed_mrs(struct qlnxr_dev *dev, struct mr_info *info) +{ + qlnx_host_t *ha; + + ha = dev->ha; + + int work = info->completed - info->completed_handled - 1; + + QL_DPRINT12(ha, "enter [%d]\n", work); + + while (work-- > 0 && !list_empty(&info->inuse_pbl_list)) { + struct qlnxr_pbl *pbl; + + /* Free all the page list that are possible to be freed + * (all the ones that were invalidated), under the assumption + * that if an FMR was completed successfully that means that + * if there was an invalidate operation before it also ended + */ + pbl = list_first_entry(&info->inuse_pbl_list, + struct qlnxr_pbl, + list_entry); + list_del(&pbl->list_entry); + list_add_tail(&pbl->list_entry, &info->free_pbl_list); + info->completed_handled++; + } + + QL_DPRINT12(ha, "exit\n"); + return; +} + +#if __FreeBSD_version >= 1102000 + +static int qlnxr_prepare_reg(struct qlnxr_qp *qp, + struct rdma_sq_fmr_wqe_1st *fwqe1, + struct ib_reg_wr *wr) +{ + struct qlnxr_mr *mr = get_qlnxr_mr(wr->mr); + struct rdma_sq_fmr_wqe_2nd *fwqe2; + + fwqe2 = (struct rdma_sq_fmr_wqe_2nd *)ecore_chain_produce(&qp->sq.pbl); + fwqe1->addr.hi = upper_32_bits(mr->ibmr.iova); + fwqe1->addr.lo = lower_32_bits(mr->ibmr.iova); + fwqe1->l_key = wr->key; + + fwqe2->access_ctrl = 0; + + SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_REMOTE_READ, + !!(wr->access & IB_ACCESS_REMOTE_READ)); + SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_REMOTE_WRITE, + !!(wr->access & IB_ACCESS_REMOTE_WRITE)); + SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_ENABLE_ATOMIC, + !!(wr->access & IB_ACCESS_REMOTE_ATOMIC)); + SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_LOCAL_READ, 1); + SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_LOCAL_WRITE, + !!(wr->access & IB_ACCESS_LOCAL_WRITE)); + fwqe2->fmr_ctrl = 0; + + SET_FIELD2(fwqe2->fmr_ctrl, RDMA_SQ_FMR_WQE_2ND_PAGE_SIZE_LOG, + ilog2(mr->ibmr.page_size) - 12); + + fwqe2->length_hi = 0; /* TODO - figure out why length is only 32bit.. */ + fwqe2->length_lo = mr->ibmr.length; + fwqe2->pbl_addr.hi = upper_32_bits(mr->info.pbl_table->pa); + fwqe2->pbl_addr.lo = lower_32_bits(mr->info.pbl_table->pa); + + qp->wqe_wr_id[qp->sq.prod].mr = mr; + + return 0; +} + +#else + +static void +build_frmr_pbes(struct qlnxr_dev *dev, struct ib_send_wr *wr, + struct mr_info *info) +{ + int i; + u64 buf_addr = 0; + int num_pbes, total_num_pbes = 0; + struct regpair *pbe; + struct qlnxr_pbl *pbl_tbl = info->pbl_table; + struct qlnxr_pbl_info *pbl_info = &info->pbl_info; + qlnx_host_t *ha; + + ha = dev->ha; + + QL_DPRINT12(ha, "enter\n"); + + pbe = (struct regpair *)pbl_tbl->va; + num_pbes = 0; + + for (i = 0; i < wr->wr.fast_reg.page_list_len; i++) { + buf_addr = wr->wr.fast_reg.page_list->page_list[i]; + pbe->lo = cpu_to_le32((u32)buf_addr); + pbe->hi = cpu_to_le32((u32)upper_32_bits(buf_addr)); + + num_pbes += 1; + pbe++; + total_num_pbes++; + + if (total_num_pbes == pbl_info->num_pbes) + return; + + /* if the given pbl is full storing the pbes, + * move to next pbl. + */ + if (num_pbes == + (pbl_info->pbl_size / sizeof(u64))) { + pbl_tbl++; + pbe = (struct regpair *)pbl_tbl->va; + num_pbes = 0; + } + } + QL_DPRINT12(ha, "exit\n"); + + return; +} + +static int +qlnxr_prepare_safe_pbl(struct qlnxr_dev *dev, struct mr_info *info) +{ + int rc = 0; + qlnx_host_t *ha; + + ha = dev->ha; + + QL_DPRINT12(ha, "enter\n"); + + if (info->completed == 0) { + //DP_VERBOSE(dev, QLNXR_MSG_MR, "First FMR\n"); + /* first fmr */ + return 0; + } + + qlnx_handle_completed_mrs(dev, info); + + list_add_tail(&info->pbl_table->list_entry, &info->inuse_pbl_list); + + if (list_empty(&info->free_pbl_list)) { + info->pbl_table = qlnxr_alloc_pbl_tbl(dev, &info->pbl_info, + GFP_ATOMIC); + } else { + info->pbl_table = list_first_entry(&info->free_pbl_list, + struct qlnxr_pbl, + list_entry); + list_del(&info->pbl_table->list_entry); + } + + if (!info->pbl_table) + rc = -ENOMEM; + + QL_DPRINT12(ha, "exit\n"); + return rc; +} + +static inline int +qlnxr_prepare_fmr(struct qlnxr_qp *qp, + struct rdma_sq_fmr_wqe_1st *fwqe1, + struct ib_send_wr *wr) +{ + struct qlnxr_dev *dev = qp->dev; + u64 fbo; + struct qlnxr_fast_reg_page_list *frmr_list = + get_qlnxr_frmr_list(wr->wr.fast_reg.page_list); + struct rdma_sq_fmr_wqe *fwqe2 = + (struct rdma_sq_fmr_wqe *)ecore_chain_produce(&qp->sq.pbl); + int rc = 0; + qlnx_host_t *ha; + + ha = dev->ha; + + QL_DPRINT12(ha, "enter\n"); + + if (wr->wr.fast_reg.page_list_len == 0) + BUG(); + + rc = qlnxr_prepare_safe_pbl(dev, &frmr_list->info); + if (rc) + return rc; + + fwqe1->addr.hi = upper_32_bits(wr->wr.fast_reg.iova_start); + fwqe1->addr.lo = lower_32_bits(wr->wr.fast_reg.iova_start); + fwqe1->l_key = wr->wr.fast_reg.rkey; + + SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_REMOTE_READ, + !!(wr->wr.fast_reg.access_flags & IB_ACCESS_REMOTE_READ)); + SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_REMOTE_WRITE, + !!(wr->wr.fast_reg.access_flags & IB_ACCESS_REMOTE_WRITE)); + SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_ENABLE_ATOMIC, + !!(wr->wr.fast_reg.access_flags & IB_ACCESS_REMOTE_ATOMIC)); + SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_LOCAL_READ, 1); + SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_LOCAL_WRITE, + !!(wr->wr.fast_reg.access_flags & IB_ACCESS_LOCAL_WRITE)); + + fwqe2->fmr_ctrl = 0; + + SET_FIELD2(fwqe2->fmr_ctrl, RDMA_SQ_FMR_WQE_2ND_PAGE_SIZE_LOG, + ilog2(1 << wr->wr.fast_reg.page_shift) - 12); + SET_FIELD2(fwqe2->fmr_ctrl, RDMA_SQ_FMR_WQE_2ND_ZERO_BASED, 0); + + fwqe2->length_hi = 0; /* Todo - figure this out... why length is only 32bit.. */ + fwqe2->length_lo = wr->wr.fast_reg.length; + fwqe2->pbl_addr.hi = upper_32_bits(frmr_list->info.pbl_table->pa); + fwqe2->pbl_addr.lo = lower_32_bits(frmr_list->info.pbl_table->pa); + + /* produce another wqe for fwqe3 */ + ecore_chain_produce(&qp->sq.pbl); + + fbo = wr->wr.fast_reg.iova_start - + (wr->wr.fast_reg.page_list->page_list[0] & PAGE_MASK); + + QL_DPRINT12(ha, "wr.fast_reg.iova_start = %p rkey=%x addr=%x:%x" + " length = %x pbl_addr %x:%x\n", + wr->wr.fast_reg.iova_start, wr->wr.fast_reg.rkey, + fwqe1->addr.hi, fwqe1->addr.lo, fwqe2->length_lo, + fwqe2->pbl_addr.hi, fwqe2->pbl_addr.lo); + + build_frmr_pbes(dev, wr, &frmr_list->info); + + qp->wqe_wr_id[qp->sq.prod].frmr = frmr_list; + + QL_DPRINT12(ha, "exit\n"); + return 0; +} + +#endif /* #if __FreeBSD_version >= 1102000 */ + +static enum ib_wc_opcode +qlnxr_ib_to_wc_opcode(enum ib_wr_opcode opcode) +{ + switch (opcode) { + case IB_WR_RDMA_WRITE: + case IB_WR_RDMA_WRITE_WITH_IMM: + return IB_WC_RDMA_WRITE; + case IB_WR_SEND_WITH_IMM: + case IB_WR_SEND: + case IB_WR_SEND_WITH_INV: + return IB_WC_SEND; + case IB_WR_RDMA_READ: + return IB_WC_RDMA_READ; + case IB_WR_ATOMIC_CMP_AND_SWP: + return IB_WC_COMP_SWAP; + case IB_WR_ATOMIC_FETCH_AND_ADD: + return IB_WC_FETCH_ADD; + +#if __FreeBSD_version >= 1102000 + case IB_WR_REG_MR: + return IB_WC_REG_MR; +#else + case IB_WR_FAST_REG_MR: + return IB_WC_FAST_REG_MR; +#endif /* #if __FreeBSD_version >= 1102000 */ + + case IB_WR_LOCAL_INV: + return IB_WC_LOCAL_INV; + default: + return IB_WC_SEND; + } +} +static inline bool +qlnxr_can_post_send(struct qlnxr_qp *qp, struct ib_send_wr *wr) +{ + int wq_is_full, err_wr, pbl_is_full; + struct qlnxr_dev *dev = qp->dev; + qlnx_host_t *ha; + + ha = dev->ha; + + QL_DPRINT12(ha, "enter[qp, wr] = [%p,%p]\n", qp, wr); + + /* prevent SQ overflow and/or processing of a bad WR */ + err_wr = wr->num_sge > qp->sq.max_sges; + wq_is_full = qlnxr_wq_is_full(&qp->sq); + pbl_is_full = ecore_chain_get_elem_left_u32(&qp->sq.pbl) < + QLNXR_MAX_SQE_ELEMENTS_PER_SQE; + if (wq_is_full || err_wr || pbl_is_full) { + if (wq_is_full && + !(qp->err_bitmap & QLNXR_QP_ERR_SQ_FULL)) { + + qp->err_bitmap |= QLNXR_QP_ERR_SQ_FULL; + + QL_DPRINT12(ha, + "error: WQ is full. Post send on QP failed" + " (this error appears only once) " + "[qp, wr, qp->err_bitmap]=[%p, %p, 0x%x]\n", + qp, wr, qp->err_bitmap); + } + + if (err_wr && + !(qp->err_bitmap & QLNXR_QP_ERR_BAD_SR)) { + + qp->err_bitmap |= QLNXR_QP_ERR_BAD_SR; + + QL_DPRINT12(ha, + "error: WQ is bad. Post send on QP failed" + " (this error appears only once) " + "[qp, wr, qp->err_bitmap]=[%p, %p, 0x%x]\n", + qp, wr, qp->err_bitmap); + } + + if (pbl_is_full && + !(qp->err_bitmap & QLNXR_QP_ERR_SQ_PBL_FULL)) { + + qp->err_bitmap |= QLNXR_QP_ERR_SQ_PBL_FULL; + + QL_DPRINT12(ha, + "error: WQ PBL is full. Post send on QP failed" + " (this error appears only once) " + "[qp, wr, qp->err_bitmap]=[%p, %p, 0x%x]\n", + qp, wr, qp->err_bitmap); + } + return false; + } + QL_DPRINT12(ha, "exit[qp, wr] = [%p,%p]\n", qp, wr); + return true; +} + +int +qlnxr_post_send(struct ib_qp *ibqp, + struct ib_send_wr *wr, + struct ib_send_wr **bad_wr) +{ + struct qlnxr_dev *dev = get_qlnxr_dev(ibqp->device); + struct qlnxr_qp *qp = get_qlnxr_qp(ibqp); + unsigned long flags; + int status = 0, rc = 0; + bool comp; + qlnx_host_t *ha; + uint32_t reg_addr; + + *bad_wr = NULL; + ha = dev->ha; + + QL_DPRINT12(ha, "exit[ibqp, wr, bad_wr] = [%p, %p, %p]\n", + ibqp, wr, bad_wr); + + if (!(ha->ifp->if_drv_flags & IFF_DRV_RUNNING)) + return -EINVAL; + + if (qp->qp_type == IB_QPT_GSI) + return qlnxr_gsi_post_send(ibqp, wr, bad_wr); + + spin_lock_irqsave(&qp->q_lock, flags); + + if (QLNX_IS_ROCE(dev) && (qp->state != ECORE_ROCE_QP_STATE_RTS) && + (qp->state != ECORE_ROCE_QP_STATE_ERR) && + (qp->state != ECORE_ROCE_QP_STATE_SQD)) { + spin_unlock_irqrestore(&qp->q_lock, flags); + *bad_wr = wr; + QL_DPRINT11(ha, "QP in wrong state! QP icid=0x%x state %d\n", + qp->icid, qp->state); + return -EINVAL; + } + + if (!wr) { + QL_DPRINT11(ha, "Got an empty post send???\n"); + } + + while (wr) { + struct rdma_sq_common_wqe *wqe; + struct rdma_sq_send_wqe *swqe; + struct rdma_sq_send_wqe *swqe2; + struct rdma_sq_rdma_wqe_1st *rwqe; + struct rdma_sq_rdma_wqe_2nd *rwqe2; + struct rdma_sq_local_inv_wqe *iwqe; + struct rdma_sq_atomic_wqe *awqe1; + struct rdma_sq_atomic_wqe *awqe2; + struct rdma_sq_atomic_wqe *awqe3; + struct rdma_sq_fmr_wqe_1st *fwqe1; + + if (!qlnxr_can_post_send(qp, wr)) { + status = -ENOMEM; + *bad_wr = wr; + break; + } + + wqe = ecore_chain_produce(&qp->sq.pbl); + + qp->wqe_wr_id[qp->sq.prod].signaled = + !!(wr->send_flags & IB_SEND_SIGNALED) || qp->signaled; + + /* common fields */ + wqe->flags = 0; + wqe->flags |= (RDMA_SQ_SEND_WQE_COMP_FLG_MASK << + RDMA_SQ_SEND_WQE_COMP_FLG_SHIFT); + + SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_SE_FLG, \ + !!(wr->send_flags & IB_SEND_SOLICITED)); + + comp = (!!(wr->send_flags & IB_SEND_SIGNALED)) || + (qp->signaled); + + SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_COMP_FLG, comp); + SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_RD_FENCE_FLG, \ + !!(wr->send_flags & IB_SEND_FENCE)); + + wqe->prev_wqe_size = qp->prev_wqe_size; + + qp->wqe_wr_id[qp->sq.prod].opcode = qlnxr_ib_to_wc_opcode(wr->opcode); + + + switch (wr->opcode) { + + case IB_WR_SEND_WITH_IMM: + + wqe->req_type = RDMA_SQ_REQ_TYPE_SEND_WITH_IMM; + swqe = (struct rdma_sq_send_wqe *)wqe; + swqe->wqe_size = 2; + swqe2 = (struct rdma_sq_send_wqe *) + ecore_chain_produce(&qp->sq.pbl); + swqe->inv_key_or_imm_data = + cpu_to_le32(wr->ex.imm_data); + swqe->length = cpu_to_le32( + qlnxr_prepare_sq_send_data(dev, + qp, swqe, swqe2, wr, + bad_wr)); + + qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size; + qp->prev_wqe_size = swqe->wqe_size; + qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length; + + QL_DPRINT12(ha, "SEND w/ IMM length = %d imm data=%x\n", + swqe->length, wr->ex.imm_data); + + break; + + case IB_WR_SEND: + + wqe->req_type = RDMA_SQ_REQ_TYPE_SEND; + swqe = (struct rdma_sq_send_wqe *)wqe; + + swqe->wqe_size = 2; + swqe2 = (struct rdma_sq_send_wqe *) + ecore_chain_produce(&qp->sq.pbl); + swqe->length = cpu_to_le32( + qlnxr_prepare_sq_send_data(dev, + qp, swqe, swqe2, wr, + bad_wr)); + qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size; + qp->prev_wqe_size = swqe->wqe_size; + qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length; + + QL_DPRINT12(ha, "SEND w/o IMM length = %d\n", + swqe->length); + + break; + + case IB_WR_SEND_WITH_INV: + + wqe->req_type = RDMA_SQ_REQ_TYPE_SEND_WITH_INVALIDATE; + swqe = (struct rdma_sq_send_wqe *)wqe; + swqe2 = (struct rdma_sq_send_wqe *) + ecore_chain_produce(&qp->sq.pbl); + swqe->wqe_size = 2; + swqe->inv_key_or_imm_data = + cpu_to_le32(wr->ex.invalidate_rkey); + swqe->length = cpu_to_le32(qlnxr_prepare_sq_send_data(dev, + qp, swqe, swqe2, wr, bad_wr)); + qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size; + qp->prev_wqe_size = swqe->wqe_size; + qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length; + + QL_DPRINT12(ha, "SEND w INVALIDATE length = %d\n", + swqe->length); + break; + + case IB_WR_RDMA_WRITE_WITH_IMM: + + wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_WR_WITH_IMM; + rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe; + + rwqe->wqe_size = 2; + rwqe->imm_data = htonl(cpu_to_le32(wr->ex.imm_data)); + rwqe2 = (struct rdma_sq_rdma_wqe_2nd *) + ecore_chain_produce(&qp->sq.pbl); + rwqe->length = cpu_to_le32(qlnxr_prepare_sq_rdma_data(dev, + qp, rwqe, rwqe2, wr, bad_wr)); + qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size; + qp->prev_wqe_size = rwqe->wqe_size; + qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length; + + QL_DPRINT12(ha, + "RDMA WRITE w/ IMM length = %d imm data=%x\n", + rwqe->length, rwqe->imm_data); + + break; + + case IB_WR_RDMA_WRITE: + + wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_WR; + rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe; + + rwqe->wqe_size = 2; + rwqe2 = (struct rdma_sq_rdma_wqe_2nd *) + ecore_chain_produce(&qp->sq.pbl); + rwqe->length = cpu_to_le32(qlnxr_prepare_sq_rdma_data(dev, + qp, rwqe, rwqe2, wr, bad_wr)); + qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size; + qp->prev_wqe_size = rwqe->wqe_size; + qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length; + + QL_DPRINT12(ha, + "RDMA WRITE w/o IMM length = %d\n", + rwqe->length); + + break; + + case IB_WR_RDMA_READ_WITH_INV: + + QL_DPRINT12(ha, + "RDMA READ WITH INVALIDATE not supported\n"); + + *bad_wr = wr; + rc = -EINVAL; + + break; + + case IB_WR_RDMA_READ: + + wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_RD; + rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe; + + rwqe->wqe_size = 2; + rwqe2 = (struct rdma_sq_rdma_wqe_2nd *) + ecore_chain_produce(&qp->sq.pbl); + rwqe->length = cpu_to_le32(qlnxr_prepare_sq_rdma_data(dev, + qp, rwqe, rwqe2, wr, bad_wr)); + + qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size; + qp->prev_wqe_size = rwqe->wqe_size; + qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length; + + QL_DPRINT12(ha, "RDMA READ length = %d\n", + rwqe->length); + + break; + + case IB_WR_ATOMIC_CMP_AND_SWP: + case IB_WR_ATOMIC_FETCH_AND_ADD: + + QL_DPRINT12(ha, + "ATOMIC operation = %s\n", + ((wr->opcode == IB_WR_ATOMIC_CMP_AND_SWP) ? + "IB_WR_ATOMIC_CMP_AND_SWP" : + "IB_WR_ATOMIC_FETCH_AND_ADD")); + + awqe1 = (struct rdma_sq_atomic_wqe *)wqe; + awqe1->prev_wqe_size = 4; + + awqe2 = (struct rdma_sq_atomic_wqe *) + ecore_chain_produce(&qp->sq.pbl); + + TYPEPTR_ADDR_SET(awqe2, remote_va, \ + atomic_wr(wr)->remote_addr); + + awqe2->r_key = cpu_to_le32(atomic_wr(wr)->rkey); + + awqe3 = (struct rdma_sq_atomic_wqe *) + ecore_chain_produce(&qp->sq.pbl); + + if (wr->opcode == IB_WR_ATOMIC_FETCH_AND_ADD) { + wqe->req_type = RDMA_SQ_REQ_TYPE_ATOMIC_ADD; + TYPEPTR_ADDR_SET(awqe3, swap_data, + atomic_wr(wr)->compare_add); + } else { + wqe->req_type = RDMA_SQ_REQ_TYPE_ATOMIC_CMP_AND_SWAP; + TYPEPTR_ADDR_SET(awqe3, swap_data, + atomic_wr(wr)->swap); + TYPEPTR_ADDR_SET(awqe3, cmp_data, + atomic_wr(wr)->compare_add); + } + + qlnxr_prepare_sq_sges(dev, qp, NULL, wr); + + qp->wqe_wr_id[qp->sq.prod].wqe_size = awqe1->prev_wqe_size; + qp->prev_wqe_size = awqe1->prev_wqe_size; + + break; + + case IB_WR_LOCAL_INV: + + QL_DPRINT12(ha, + "INVALIDATE length (IB_WR_LOCAL_INV)\n"); + + iwqe = (struct rdma_sq_local_inv_wqe *)wqe; + iwqe->prev_wqe_size = 1; + + iwqe->req_type = RDMA_SQ_REQ_TYPE_LOCAL_INVALIDATE; + iwqe->inv_l_key = wr->ex.invalidate_rkey; + qp->wqe_wr_id[qp->sq.prod].wqe_size = iwqe->prev_wqe_size; + qp->prev_wqe_size = iwqe->prev_wqe_size; + + break; + +#if __FreeBSD_version >= 1102000 + + case IB_WR_REG_MR: + + QL_DPRINT12(ha, "IB_WR_REG_MR\n"); + + wqe->req_type = RDMA_SQ_REQ_TYPE_FAST_MR; + fwqe1 = (struct rdma_sq_fmr_wqe_1st *)wqe; + fwqe1->wqe_size = 2; + + rc = qlnxr_prepare_reg(qp, fwqe1, reg_wr(wr)); + if (rc) { + QL_DPRINT11(ha, "IB_WR_REG_MR failed rc=%d\n", rc); + *bad_wr = wr; + break; + } + + qp->wqe_wr_id[qp->sq.prod].wqe_size = fwqe1->wqe_size; + qp->prev_wqe_size = fwqe1->wqe_size; + + break; +#else + case IB_WR_FAST_REG_MR: + + QL_DPRINT12(ha, "FAST_MR (IB_WR_FAST_REG_MR)\n"); + + wqe->req_type = RDMA_SQ_REQ_TYPE_FAST_MR; + fwqe1 = (struct rdma_sq_fmr_wqe_1st *)wqe; + fwqe1->prev_wqe_size = 3; + + rc = qlnxr_prepare_fmr(qp, fwqe1, wr); + + if (rc) { + QL_DPRINT12(ha, + "FAST_MR (IB_WR_FAST_REG_MR) failed" + " rc = %d\n", rc); + *bad_wr = wr; + break; + } + + qp->wqe_wr_id[qp->sq.prod].wqe_size = fwqe1->prev_wqe_size; + qp->prev_wqe_size = fwqe1->prev_wqe_size; + + break; +#endif /* #if __FreeBSD_version >= 1102000 */ + + default: + + QL_DPRINT12(ha, "Invalid Opcode 0x%x!\n", wr->opcode); + + rc = -EINVAL; + *bad_wr = wr; + break; + } + + if (*bad_wr) { + /* + * restore prod to its position before this WR was processed + */ + ecore_chain_set_prod(&qp->sq.pbl, + le16_to_cpu(qp->sq.db_data.data.value), + wqe); + /* restore prev_wqe_size */ + qp->prev_wqe_size = wqe->prev_wqe_size; + status = rc; + + QL_DPRINT12(ha, "failed *bad_wr = %p\n", *bad_wr); + break; /* out of the loop */ + } + + qp->wqe_wr_id[qp->sq.prod].wr_id = wr->wr_id; + + qlnxr_inc_sw_prod(&qp->sq); + + qp->sq.db_data.data.value++; + + wr = wr->next; + } + + /* Trigger doorbell + * If there was a failure in the first WR then it will be triggered in + * vane. However this is not harmful (as long as the producer value is + * unchanged). For performance reasons we avoid checking for this + * redundant doorbell. + */ + wmb(); + //writel(qp->sq.db_data.raw, qp->sq.db); + + reg_addr = (uint32_t)((uint8_t *)qp->sq.db - (uint8_t *)ha->cdev.doorbells); + bus_write_4(ha->pci_dbells, reg_addr, qp->sq.db_data.raw); + bus_barrier(ha->pci_dbells, 0, 0, BUS_SPACE_BARRIER_READ); + + mmiowb(); + + spin_unlock_irqrestore(&qp->q_lock, flags); + + QL_DPRINT12(ha, "exit[ibqp, wr, bad_wr] = [%p, %p, %p]\n", + ibqp, wr, bad_wr); + + return status; +} + +static u32 +qlnxr_srq_elem_left(struct qlnxr_srq_hwq_info *hw_srq) +{ + u32 used; + + /* Calculate number of elements used based on producer + * count and consumer count and subtract it from max + * work request supported so that we get elements left. + */ + used = hw_srq->wr_prod_cnt - hw_srq->wr_cons_cnt; + + return hw_srq->max_wr - used; +} + + +int +qlnxr_post_recv(struct ib_qp *ibqp, + struct ib_recv_wr *wr, + struct ib_recv_wr **bad_wr) +{ + struct qlnxr_qp *qp = get_qlnxr_qp(ibqp); + struct qlnxr_dev *dev = qp->dev; + unsigned long flags; + int status = 0; + qlnx_host_t *ha; + uint32_t reg_addr; + + ha = dev->ha; + + if (!(ha->ifp->if_drv_flags & IFF_DRV_RUNNING)) + return -EINVAL; + + QL_DPRINT12(ha, "enter\n"); + + if (qp->qp_type == IB_QPT_GSI) { + QL_DPRINT12(ha, "(qp->qp_type = IB_QPT_GSI)\n"); + return qlnxr_gsi_post_recv(ibqp, wr, bad_wr); + } + + if (qp->srq) { + QL_DPRINT11(ha, "qp->srq [%p]" + " QP is associated with SRQ, cannot post RQ buffers\n", + qp->srq); + return -EINVAL; + } + + spin_lock_irqsave(&qp->q_lock, flags); + + if (qp->state == ECORE_ROCE_QP_STATE_RESET) { + spin_unlock_irqrestore(&qp->q_lock, flags); + *bad_wr = wr; + + QL_DPRINT11(ha, "qp->qp_type = ECORE_ROCE_QP_STATE_RESET\n"); + + return -EINVAL; + } + + while (wr) { + int i; + + if ((ecore_chain_get_elem_left_u32(&qp->rq.pbl) < + QLNXR_MAX_RQE_ELEMENTS_PER_RQE) || + (wr->num_sge > qp->rq.max_sges)) { + status = -ENOMEM; + *bad_wr = wr; + break; + } + for (i = 0; i < wr->num_sge; i++) { + u32 flags = 0; + struct rdma_rq_sge *rqe = ecore_chain_produce(&qp->rq.pbl); + + /* first one must include the number of SGE in the list */ + if (!i) + SET_FIELD(flags, RDMA_RQ_SGE_NUM_SGES, wr->num_sge); + + SET_FIELD(flags, RDMA_RQ_SGE_L_KEY, wr->sg_list[i].lkey); + + RQ_SGE_SET(rqe, wr->sg_list[i].addr, \ + wr->sg_list[i].length, flags); + } + /* Special case of no sges. FW requires between 1-4 sges... + * in this case we need to post 1 sge with length zero. this is + * because rdma write with immediate consumes an RQ. */ + if (!wr->num_sge) { + u32 flags = 0; + struct rdma_rq_sge *rqe = ecore_chain_produce(&qp->rq.pbl); + + /* first one must include the number of SGE in the list */ + SET_FIELD(flags, RDMA_RQ_SGE_L_KEY, 0); + SET_FIELD(flags, RDMA_RQ_SGE_NUM_SGES, 1); + + //RQ_SGE_SET(rqe, 0, 0, flags); + rqe->addr.hi = 0; + rqe->addr.lo = 0; + + rqe->length = 0; + rqe->flags = cpu_to_le32(flags); + + i = 1; + } + + qp->rqe_wr_id[qp->rq.prod].wr_id = wr->wr_id; + qp->rqe_wr_id[qp->rq.prod].wqe_size = i; + + qlnxr_inc_sw_prod(&qp->rq); + + wmb(); + + qp->rq.db_data.data.value++; + + // writel(qp->rq.db_data.raw, qp->rq.db); + mmiowb(); + // if (QLNX_IS_IWARP(dev)) { + // writel(qp->rq.iwarp_db2_data.raw, qp->rq.iwarp_db2); + // mmiowb(); /* for second doorbell */ + // } + + reg_addr = (uint32_t)((uint8_t *)qp->rq.db - + (uint8_t *)ha->cdev.doorbells); + + bus_write_4(ha->pci_dbells, reg_addr, qp->rq.db_data.raw); + bus_barrier(ha->pci_dbells, 0, 0, BUS_SPACE_BARRIER_READ); + + if (QLNX_IS_IWARP(dev)) { + reg_addr = (uint32_t)((uint8_t *)qp->rq.iwarp_db2 - + (uint8_t *)ha->cdev.doorbells); + bus_write_4(ha->pci_dbells, reg_addr, \ + qp->rq.iwarp_db2_data.raw); + bus_barrier(ha->pci_dbells, 0, 0, \ + BUS_SPACE_BARRIER_READ); + } + + wr = wr->next; + } + + spin_unlock_irqrestore(&qp->q_lock, flags); + + QL_DPRINT12(ha, "exit status = 0x%x\n", status); + + return status; +} + +/* In fmr we need to increase the number of fmr completed counter for the fmr + * algorithm determining whether we can free a pbl or not. + * we need to perform this whether the work request was signaled or not. for + * this purpose we call this function from the condition that checks if a wr + * should be skipped, to make sure we don't miss it ( possibly this fmr + * operation was not signalted) + */ +static inline void +qlnxr_chk_if_fmr(struct qlnxr_qp *qp) +{ +#if __FreeBSD_version >= 1102000 + + if (qp->wqe_wr_id[qp->sq.cons].opcode == IB_WC_REG_MR) + qp->wqe_wr_id[qp->sq.cons].mr->info.completed++; +#else + if (qp->wqe_wr_id[qp->sq.cons].opcode == IB_WC_FAST_REG_MR) + qp->wqe_wr_id[qp->sq.cons].frmr->info.completed++; + +#endif /* #if __FreeBSD_version >= 1102000 */ +} + +static int +process_req(struct qlnxr_dev *dev, + struct qlnxr_qp *qp, + struct qlnxr_cq *cq, + int num_entries, + struct ib_wc *wc, + u16 hw_cons, + enum ib_wc_status status, + int force) +{ + u16 cnt = 0; + qlnx_host_t *ha = dev->ha; + + QL_DPRINT12(ha, "enter\n"); + + while (num_entries && qp->sq.wqe_cons != hw_cons) { + if (!qp->wqe_wr_id[qp->sq.cons].signaled && !force) { + qlnxr_chk_if_fmr(qp); + /* skip WC */ + goto next_cqe; + } + + /* fill WC */ + wc->status = status; + wc->vendor_err = 0; + wc->wc_flags = 0; + wc->src_qp = qp->id; + wc->qp = &qp->ibqp; + + // common section + wc->wr_id = qp->wqe_wr_id[qp->sq.cons].wr_id; + wc->opcode = qp->wqe_wr_id[qp->sq.cons].opcode; + + switch (wc->opcode) { + + case IB_WC_RDMA_WRITE: + + wc->byte_len = qp->wqe_wr_id[qp->sq.cons].bytes_len; + + QL_DPRINT12(ha, + "opcode = IB_WC_RDMA_WRITE bytes = %d\n", + qp->wqe_wr_id[qp->sq.cons].bytes_len); + break; + + case IB_WC_COMP_SWAP: + case IB_WC_FETCH_ADD: + wc->byte_len = 8; + break; + +#if __FreeBSD_version >= 1102000 + case IB_WC_REG_MR: + qp->wqe_wr_id[qp->sq.cons].mr->info.completed++; + break; +#else + case IB_WC_FAST_REG_MR: + qp->wqe_wr_id[qp->sq.cons].frmr->info.completed++; + break; +#endif /* #if __FreeBSD_version >= 1102000 */ + + case IB_WC_RDMA_READ: + case IB_WC_SEND: + + QL_DPRINT12(ha, "opcode = 0x%x \n", wc->opcode); + break; + default: + ;//DP_ERR("TBD ERROR"); + } + + num_entries--; + wc++; + cnt++; +next_cqe: + while (qp->wqe_wr_id[qp->sq.cons].wqe_size--) + ecore_chain_consume(&qp->sq.pbl); + qlnxr_inc_sw_cons(&qp->sq); + } + + QL_DPRINT12(ha, "exit cnt = 0x%x\n", cnt); + return cnt; +} + +static int +qlnxr_poll_cq_req(struct qlnxr_dev *dev, + struct qlnxr_qp *qp, + struct qlnxr_cq *cq, + int num_entries, + struct ib_wc *wc, + struct rdma_cqe_requester *req) +{ + int cnt = 0; + qlnx_host_t *ha = dev->ha; + + QL_DPRINT12(ha, "enter req->status = 0x%x\n", req->status); + + switch (req->status) { + + case RDMA_CQE_REQ_STS_OK: + + cnt = process_req(dev, qp, cq, num_entries, wc, req->sq_cons, + IB_WC_SUCCESS, 0); + break; + + case RDMA_CQE_REQ_STS_WORK_REQUEST_FLUSHED_ERR: + + if (qp->state != ECORE_ROCE_QP_STATE_ERR) + cnt = process_req(dev, qp, cq, num_entries, wc, req->sq_cons, + IB_WC_WR_FLUSH_ERR, 1); + break; + + default: /* other errors case */ + + /* process all WQE before the cosumer */ + qp->state = ECORE_ROCE_QP_STATE_ERR; + cnt = process_req(dev, qp, cq, num_entries, wc, + req->sq_cons - 1, IB_WC_SUCCESS, 0); + wc += cnt; + /* if we have extra WC fill it with actual error info */ + + if (cnt < num_entries) { + enum ib_wc_status wc_status; + + switch (req->status) { + case RDMA_CQE_REQ_STS_BAD_RESPONSE_ERR: + wc_status = IB_WC_BAD_RESP_ERR; + break; + case RDMA_CQE_REQ_STS_LOCAL_LENGTH_ERR: + wc_status = IB_WC_LOC_LEN_ERR; + break; + case RDMA_CQE_REQ_STS_LOCAL_QP_OPERATION_ERR: + wc_status = IB_WC_LOC_QP_OP_ERR; + break; + case RDMA_CQE_REQ_STS_LOCAL_PROTECTION_ERR: + wc_status = IB_WC_LOC_PROT_ERR; + break; + case RDMA_CQE_REQ_STS_MEMORY_MGT_OPERATION_ERR: + wc_status = IB_WC_MW_BIND_ERR; + break; + case RDMA_CQE_REQ_STS_REMOTE_INVALID_REQUEST_ERR: + wc_status = IB_WC_REM_INV_REQ_ERR; + break; + case RDMA_CQE_REQ_STS_REMOTE_ACCESS_ERR: + wc_status = IB_WC_REM_ACCESS_ERR; + break; + case RDMA_CQE_REQ_STS_REMOTE_OPERATION_ERR: + wc_status = IB_WC_REM_OP_ERR; + break; + case RDMA_CQE_REQ_STS_RNR_NAK_RETRY_CNT_ERR: + wc_status = IB_WC_RNR_RETRY_EXC_ERR; + break; + case RDMA_CQE_REQ_STS_TRANSPORT_RETRY_CNT_ERR: + wc_status = IB_WC_RETRY_EXC_ERR; + break; + default: + wc_status = IB_WC_GENERAL_ERR; + } + + cnt += process_req(dev, qp, cq, 1, wc, req->sq_cons, + wc_status, 1 /* force use of WC */); + } + } + + QL_DPRINT12(ha, "exit cnt = %d\n", cnt); + return cnt; +} + +static void +__process_resp_one(struct qlnxr_dev *dev, + struct qlnxr_qp *qp, + struct qlnxr_cq *cq, + struct ib_wc *wc, + struct rdma_cqe_responder *resp, + u64 wr_id) +{ + enum ib_wc_status wc_status = IB_WC_SUCCESS; +#if __FreeBSD_version < 1102000 + u8 flags; +#endif + qlnx_host_t *ha = dev->ha; + + QL_DPRINT12(ha, "enter qp = %p resp->status = 0x%x\n", + qp, resp->status); + + wc->opcode = IB_WC_RECV; + wc->wc_flags = 0; + + switch (resp->status) { + + case RDMA_CQE_RESP_STS_LOCAL_ACCESS_ERR: + wc_status = IB_WC_LOC_ACCESS_ERR; + break; + + case RDMA_CQE_RESP_STS_LOCAL_LENGTH_ERR: + wc_status = IB_WC_LOC_LEN_ERR; + break; + + case RDMA_CQE_RESP_STS_LOCAL_QP_OPERATION_ERR: + wc_status = IB_WC_LOC_QP_OP_ERR; + break; + + case RDMA_CQE_RESP_STS_LOCAL_PROTECTION_ERR: + wc_status = IB_WC_LOC_PROT_ERR; + break; + + case RDMA_CQE_RESP_STS_MEMORY_MGT_OPERATION_ERR: + wc_status = IB_WC_MW_BIND_ERR; + break; + + case RDMA_CQE_RESP_STS_REMOTE_INVALID_REQUEST_ERR: + wc_status = IB_WC_REM_INV_RD_REQ_ERR; + break; + + case RDMA_CQE_RESP_STS_OK: + +#if __FreeBSD_version >= 1102000 + if (resp->flags & QLNXR_RESP_IMM) { + wc->ex.imm_data = + le32_to_cpu(resp->imm_data_or_inv_r_Key); + wc->wc_flags |= IB_WC_WITH_IMM; + + if (resp->flags & QLNXR_RESP_RDMA) + wc->opcode = IB_WC_RECV_RDMA_WITH_IMM; + + if (resp->flags & QLNXR_RESP_INV) { + QL_DPRINT11(ha, + "Invalid flags QLNXR_RESP_INV [0x%x]" + "qp = %p qp->id = 0x%x cq = %p" + " cq->icid = 0x%x\n", + resp->flags, qp, qp->id, cq, cq->icid ); + } + } else if (resp->flags & QLNXR_RESP_INV) { + wc->ex.imm_data = + le32_to_cpu(resp->imm_data_or_inv_r_Key); + wc->wc_flags |= IB_WC_WITH_INVALIDATE; + + if (resp->flags & QLNXR_RESP_RDMA) { + QL_DPRINT11(ha, + "Invalid flags QLNXR_RESP_RDMA [0x%x]" + "qp = %p qp->id = 0x%x cq = %p" + " cq->icid = 0x%x\n", + resp->flags, qp, qp->id, cq, cq->icid ); + } + } else if (resp->flags & QLNXR_RESP_RDMA) { + QL_DPRINT11(ha, "Invalid flags QLNXR_RESP_RDMA [0x%x]" + "qp = %p qp->id = 0x%x cq = %p cq->icid = 0x%x\n", + resp->flags, qp, qp->id, cq, cq->icid ); + } +#else + wc_status = IB_WC_SUCCESS; + wc->byte_len = le32_to_cpu(resp->length); + + flags = resp->flags & QLNXR_RESP_RDMA_IMM; + + switch (flags) { + + case QLNXR_RESP_RDMA_IMM: + /* update opcode */ + wc->opcode = IB_WC_RECV_RDMA_WITH_IMM; + /* fall to set imm data */ + case QLNXR_RESP_IMM: + wc->ex.imm_data = + le32_to_cpu(resp->imm_data_or_inv_r_Key); + wc->wc_flags |= IB_WC_WITH_IMM; + break; + case QLNXR_RESP_RDMA: + QL_DPRINT11(ha, "Invalid flags QLNXR_RESP_RDMA [0x%x]" + "qp = %p qp->id = 0x%x cq = %p cq->icid = 0x%x\n", + resp->flags, qp, qp->id, cq, cq->icid ); + break; + default: + /* valid configuration, but nothing todo here */ + ; + } +#endif /* #if __FreeBSD_version >= 1102000 */ + + break; + default: + wc_status = IB_WC_GENERAL_ERR; + } + + /* fill WC */ + wc->status = wc_status; + wc->vendor_err = 0; + wc->src_qp = qp->id; + wc->qp = &qp->ibqp; + wc->wr_id = wr_id; + + QL_DPRINT12(ha, "exit status = 0x%x\n", wc_status); + + return; +} + +static int +process_resp_one_srq(struct qlnxr_dev *dev, + struct qlnxr_qp *qp, + struct qlnxr_cq *cq, + struct ib_wc *wc, + struct rdma_cqe_responder *resp) +{ + struct qlnxr_srq *srq = qp->srq; + u64 wr_id; + qlnx_host_t *ha = dev->ha; + + QL_DPRINT12(ha, "enter\n"); + + wr_id = HILO_U64(resp->srq_wr_id.hi, resp->srq_wr_id.lo); + + if (resp->status == RDMA_CQE_RESP_STS_WORK_REQUEST_FLUSHED_ERR) { + wc->status = IB_WC_WR_FLUSH_ERR; + wc->vendor_err = 0; + wc->wr_id = wr_id; + wc->byte_len = 0; + wc->src_qp = qp->id; + wc->qp = &qp->ibqp; + wc->wr_id = wr_id; + } else { + __process_resp_one(dev, qp, cq, wc, resp, wr_id); + } + + /* PBL is maintained in case of WR granularity. + * So increment WR consumer after consuming WR + */ + srq->hw_srq.wr_cons_cnt++; + + QL_DPRINT12(ha, "exit\n"); + return 1; +} + +static int +process_resp_one(struct qlnxr_dev *dev, + struct qlnxr_qp *qp, + struct qlnxr_cq *cq, + struct ib_wc *wc, + struct rdma_cqe_responder *resp) +{ + qlnx_host_t *ha = dev->ha; + u64 wr_id = qp->rqe_wr_id[qp->rq.cons].wr_id; + + QL_DPRINT12(ha, "enter\n"); + + __process_resp_one(dev, qp, cq, wc, resp, wr_id); + + while (qp->rqe_wr_id[qp->rq.cons].wqe_size--) + ecore_chain_consume(&qp->rq.pbl); + qlnxr_inc_sw_cons(&qp->rq); + + QL_DPRINT12(ha, "exit\n"); + return 1; +} + +static int +process_resp_flush(struct qlnxr_qp *qp, + int num_entries, + struct ib_wc *wc, + u16 hw_cons) +{ + u16 cnt = 0; + qlnx_host_t *ha = qp->dev->ha; + + QL_DPRINT12(ha, "enter\n"); + + while (num_entries && qp->rq.wqe_cons != hw_cons) { + /* fill WC */ + wc->status = IB_WC_WR_FLUSH_ERR; + wc->vendor_err = 0; + wc->wc_flags = 0; + wc->src_qp = qp->id; + wc->byte_len = 0; + wc->wr_id = qp->rqe_wr_id[qp->rq.cons].wr_id; + wc->qp = &qp->ibqp; + num_entries--; + wc++; + cnt++; + while (qp->rqe_wr_id[qp->rq.cons].wqe_size--) + ecore_chain_consume(&qp->rq.pbl); + qlnxr_inc_sw_cons(&qp->rq); + } + + QL_DPRINT12(ha, "exit cnt = 0x%x\n", cnt); + return cnt; +} + +static void +try_consume_resp_cqe(struct qlnxr_cq *cq, + struct qlnxr_qp *qp, + struct rdma_cqe_responder *resp, + int *update) +{ + if (le16_to_cpu(resp->rq_cons) == qp->rq.wqe_cons) { + consume_cqe(cq); + *update |= 1; + } +} + +static int +qlnxr_poll_cq_resp_srq(struct qlnxr_dev *dev, + struct qlnxr_qp *qp, + struct qlnxr_cq *cq, + int num_entries, + struct ib_wc *wc, + struct rdma_cqe_responder *resp, + int *update) +{ + int cnt; + qlnx_host_t *ha = dev->ha; + + QL_DPRINT12(ha, "enter\n"); + + cnt = process_resp_one_srq(dev, qp, cq, wc, resp); + consume_cqe(cq); + *update |= 1; + + QL_DPRINT12(ha, "exit cnt = 0x%x\n", cnt); + return cnt; +} + +static int +qlnxr_poll_cq_resp(struct qlnxr_dev *dev, + struct qlnxr_qp *qp, + struct qlnxr_cq *cq, + int num_entries, + struct ib_wc *wc, + struct rdma_cqe_responder *resp, + int *update) +{ + int cnt; + qlnx_host_t *ha = dev->ha; + + QL_DPRINT12(ha, "enter\n"); + + if (resp->status == RDMA_CQE_RESP_STS_WORK_REQUEST_FLUSHED_ERR) { + cnt = process_resp_flush(qp, num_entries, wc, + resp->rq_cons); + try_consume_resp_cqe(cq, qp, resp, update); + } else { + cnt = process_resp_one(dev, qp, cq, wc, resp); + consume_cqe(cq); + *update |= 1; + } + + QL_DPRINT12(ha, "exit cnt = 0x%x\n", cnt); + return cnt; +} + +static void +try_consume_req_cqe(struct qlnxr_cq *cq, struct qlnxr_qp *qp, + struct rdma_cqe_requester *req, int *update) +{ + if (le16_to_cpu(req->sq_cons) == qp->sq.wqe_cons) { + consume_cqe(cq); + *update |= 1; + } +} + +static void +doorbell_cq(struct qlnxr_dev *dev, struct qlnxr_cq *cq, u32 cons, u8 flags) +{ + uint64_t reg_addr; + qlnx_host_t *ha = dev->ha; + + QL_DPRINT12(ha, "enter\n"); + + wmb(); + cq->db.data.agg_flags = flags; + cq->db.data.value = cpu_to_le32(cons); + + reg_addr = (uint64_t)((uint8_t *)cq->db_addr - + (uint8_t *)(ha->cdev.doorbells)); + + bus_write_8(ha->pci_dbells, reg_addr, cq->db.raw); + bus_barrier(ha->pci_dbells, 0, 0, BUS_SPACE_BARRIER_READ); + + QL_DPRINT12(ha, "exit\n"); + return; + +//#ifdef __LP64__ +// writeq(cq->db.raw, cq->db_addr); +//#else + /* Note that since the FW allows 64 bit write only, in 32bit systems + * the value of db_addr must be low enough. This is currently not + * enforced. + */ +// writel(cq->db.raw & 0xffffffff, cq->db_addr); +// mmiowb(); +//#endif +} + + +static int +is_valid_cqe(struct qlnxr_cq *cq, union rdma_cqe *cqe) +{ + struct rdma_cqe_requester *resp_cqe = &cqe->req; + return (resp_cqe->flags & RDMA_RESIZE_CQ_RAMROD_DATA_TOGGLE_BIT_MASK) == + cq->pbl_toggle; +} + +int +qlnxr_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc) +{ + struct qlnxr_cq *cq = get_qlnxr_cq(ibcq); + struct qlnxr_dev *dev = get_qlnxr_dev((ibcq->device)); + int done = 0; + union rdma_cqe *cqe = cq->latest_cqe; + int update = 0; + u32 old_cons, new_cons; + unsigned long flags; + qlnx_host_t *ha = dev->ha; + + QL_DPRINT12(ha, "enter\n"); + + if (!(ha->ifp->if_drv_flags & IFF_DRV_RUNNING)) + return -EINVAL; + + if (cq->destroyed) { + QL_DPRINT11(ha, "called after destroy for cq %p (icid=%d)\n", + cq, cq->icid); + return 0; + } + + if (cq->cq_type == QLNXR_CQ_TYPE_GSI) + return qlnxr_gsi_poll_cq(ibcq, num_entries, wc); + + spin_lock_irqsave(&cq->cq_lock, flags); + + old_cons = ecore_chain_get_cons_idx_u32(&cq->pbl); + + while (num_entries && is_valid_cqe(cq, cqe)) { + int cnt = 0; + struct qlnxr_qp *qp; + struct rdma_cqe_requester *resp_cqe; + enum rdma_cqe_type cqe_type; + + /* prevent speculative reads of any field of CQE */ + rmb(); + + resp_cqe = &cqe->req; + qp = (struct qlnxr_qp *)(uintptr_t)HILO_U64(resp_cqe->qp_handle.hi, + resp_cqe->qp_handle.lo); + + if (!qp) { + QL_DPRINT11(ha, "qp = NULL\n"); + break; + } + + wc->qp = &qp->ibqp; + + cqe_type = GET_FIELD(resp_cqe->flags, RDMA_CQE_REQUESTER_TYPE); + + switch (cqe_type) { + case RDMA_CQE_TYPE_REQUESTER: + cnt = qlnxr_poll_cq_req(dev, qp, cq, num_entries, + wc, &cqe->req); + try_consume_req_cqe(cq, qp, &cqe->req, &update); + break; + case RDMA_CQE_TYPE_RESPONDER_RQ: + cnt = qlnxr_poll_cq_resp(dev, qp, cq, num_entries, + wc, &cqe->resp, &update); + break; + case RDMA_CQE_TYPE_RESPONDER_SRQ: + cnt = qlnxr_poll_cq_resp_srq(dev, qp, cq, num_entries, + wc, &cqe->resp, &update); + break; + case RDMA_CQE_TYPE_INVALID: + default: + QL_DPRINT11(ha, "cqe type [0x%x] invalid\n", cqe_type); + break; + } + num_entries -= cnt; + wc += cnt; + done += cnt; + + cqe = cq->latest_cqe; + } + new_cons = ecore_chain_get_cons_idx_u32(&cq->pbl); + + cq->cq_cons += new_cons - old_cons; + + if (update) { + /* doorbell notifies abount latest VALID entry, + * but chain already point to the next INVALID one + */ + doorbell_cq(dev, cq, cq->cq_cons - 1, cq->arm_flags); + QL_DPRINT12(ha, "cq = %p cons = 0x%x " + "arm_flags = 0x%x db.icid = 0x%x\n", cq, + (cq->cq_cons - 1), cq->arm_flags, cq->db.data.icid); + } + + spin_unlock_irqrestore(&cq->cq_lock, flags); + + QL_DPRINT12(ha, "exit\n"); + + return done; +} + + +int +qlnxr_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags) +{ + struct qlnxr_cq *cq = get_qlnxr_cq(ibcq); + unsigned long sflags; + struct qlnxr_dev *dev; + qlnx_host_t *ha; + + dev = get_qlnxr_dev((ibcq->device)); + ha = dev->ha; + + QL_DPRINT12(ha, "enter ibcq = %p flags = 0x%x " + "cp = %p cons = 0x%x cq_type = 0x%x\n", ibcq, + flags, cq, cq->cq_cons, cq->cq_type); + + if (!(ha->ifp->if_drv_flags & IFF_DRV_RUNNING)) + return -EINVAL; + + if (cq->destroyed) { + QL_DPRINT11(ha, "cq was already destroyed cq = %p icid=%d\n", + cq, cq->icid); + return -EINVAL; + } + + if (cq->cq_type == QLNXR_CQ_TYPE_GSI) { + return 0; + } + + spin_lock_irqsave(&cq->cq_lock, sflags); + + cq->arm_flags = 0; + + if (flags & IB_CQ_SOLICITED) { + cq->arm_flags |= DQ_UCM_ROCE_CQ_ARM_SE_CF_CMD; + } + if (flags & IB_CQ_NEXT_COMP) { + cq->arm_flags |= DQ_UCM_ROCE_CQ_ARM_CF_CMD; + } + + doorbell_cq(dev, cq, (cq->cq_cons - 1), cq->arm_flags); + + spin_unlock_irqrestore(&cq->cq_lock, sflags); + + QL_DPRINT12(ha, "exit ibcq = %p flags = 0x%x\n", ibcq, flags); + return 0; +} + + +static struct qlnxr_mr * +__qlnxr_alloc_mr(struct ib_pd *ibpd, int max_page_list_len) +{ + struct qlnxr_pd *pd = get_qlnxr_pd(ibpd); + struct qlnxr_dev *dev = get_qlnxr_dev((ibpd->device)); + struct qlnxr_mr *mr; + int rc = -ENOMEM; + qlnx_host_t *ha; + + ha = dev->ha; + + QL_DPRINT12(ha, "enter ibpd = %p pd = %p " + " pd_id = %d max_page_list_len = %d\n", + ibpd, pd, pd->pd_id, max_page_list_len); + + mr = kzalloc(sizeof(*mr), GFP_KERNEL); + if (!mr) { + QL_DPRINT11(ha, "kzalloc(mr) failed\n"); + return ERR_PTR(rc); + } + + mr->dev = dev; + mr->type = QLNXR_MR_FRMR; + + rc = qlnxr_init_mr_info(dev, &mr->info, max_page_list_len, + 1 /* allow dual layer pbl */); + if (rc) { + QL_DPRINT11(ha, "qlnxr_init_mr_info failed\n"); + goto err0; + } + + rc = ecore_rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid); + if (rc) { + QL_DPRINT11(ha, "ecore_rdma_alloc_tid failed\n"); + goto err0; + } + + /* index only, 18 bit long, lkey = itid << 8 | key */ + mr->hw_mr.tid_type = ECORE_RDMA_TID_FMR; + mr->hw_mr.key = 0; + mr->hw_mr.pd = pd->pd_id; + mr->hw_mr.local_read = 1; + mr->hw_mr.local_write = 0; + mr->hw_mr.remote_read = 0; + mr->hw_mr.remote_write = 0; + mr->hw_mr.remote_atomic = 0; + mr->hw_mr.mw_bind = false; /* TBD MW BIND */ + mr->hw_mr.pbl_ptr = 0; /* Will be supplied during post */ + mr->hw_mr.pbl_two_level = mr->info.pbl_info.two_layered; + mr->hw_mr.pbl_page_size_log = ilog2(mr->info.pbl_info.pbl_size); + mr->hw_mr.fbo = 0; + mr->hw_mr.length = 0; + mr->hw_mr.vaddr = 0; + mr->hw_mr.zbva = false; /* TBD figure when this should be true */ + mr->hw_mr.phy_mr = true; /* Fast MR - True, Regular Register False */ + mr->hw_mr.dma_mr = false; + + rc = ecore_rdma_register_tid(dev->rdma_ctx, &mr->hw_mr); + if (rc) { + QL_DPRINT11(ha, "ecore_rdma_register_tid failed\n"); + goto err1; + } + + mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key; + mr->ibmr.rkey = mr->ibmr.lkey; + + QL_DPRINT12(ha, "exit mr = %p mr->ibmr.lkey = 0x%x\n", + mr, mr->ibmr.lkey); + + return mr; + +err1: + ecore_rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid); +err0: + kfree(mr); + + QL_DPRINT12(ha, "exit\n"); + + return ERR_PTR(rc); +} + +#if __FreeBSD_version >= 1102000 + +struct ib_mr * +qlnxr_alloc_mr(struct ib_pd *ibpd, enum ib_mr_type mr_type, u32 max_num_sg) +{ + struct qlnxr_dev *dev; + struct qlnxr_mr *mr; + qlnx_host_t *ha; + + dev = get_qlnxr_dev(ibpd->device); + ha = dev->ha; + + QL_DPRINT12(ha, "enter\n"); + + if (mr_type != IB_MR_TYPE_MEM_REG) + return ERR_PTR(-EINVAL); + + mr = __qlnxr_alloc_mr(ibpd, max_num_sg); + + if (IS_ERR(mr)) + return ERR_PTR(-EINVAL); + + QL_DPRINT12(ha, "exit mr = %p &mr->ibmr = %p\n", mr, &mr->ibmr); + + return &mr->ibmr; +} + +static int +qlnxr_set_page(struct ib_mr *ibmr, u64 addr) +{ + struct qlnxr_mr *mr = get_qlnxr_mr(ibmr); + struct qlnxr_pbl *pbl_table; + struct regpair *pbe; + struct qlnxr_dev *dev; + qlnx_host_t *ha; + u32 pbes_in_page; + + dev = mr->dev; + ha = dev->ha; + + if (unlikely(mr->npages == mr->info.pbl_info.num_pbes)) { + QL_DPRINT12(ha, "fails mr->npages %d\n", mr->npages); + return -ENOMEM; + } + + QL_DPRINT12(ha, "mr->npages %d addr = %p enter\n", mr->npages, + ((void *)addr)); + + pbes_in_page = mr->info.pbl_info.pbl_size / sizeof(u64); + pbl_table = mr->info.pbl_table + (mr->npages / pbes_in_page); + pbe = (struct regpair *)pbl_table->va; + pbe += mr->npages % pbes_in_page; + pbe->lo = cpu_to_le32((u32)addr); + pbe->hi = cpu_to_le32((u32)upper_32_bits(addr)); + + mr->npages++; + + QL_DPRINT12(ha, "mr->npages %d addr = %p exit \n", mr->npages, + ((void *)addr)); + return 0; +} + +int +qlnxr_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, + int sg_nents, unsigned int *sg_offset) +{ + int ret; + struct qlnxr_mr *mr = get_qlnxr_mr(ibmr); + qlnx_host_t *ha; + + if (mr == NULL) + return (-1); + + if (mr->dev == NULL) + return (-1); + + ha = mr->dev->ha; + + QL_DPRINT12(ha, "enter\n"); + + mr->npages = 0; + qlnx_handle_completed_mrs(mr->dev, &mr->info); + + ret = ib_sg_to_pages(ibmr, sg, sg_nents, NULL, qlnxr_set_page); + + QL_DPRINT12(ha, "exit ret = %d\n", ret); + + return (ret); +} + +#else + +struct ib_mr * +qlnxr_alloc_frmr(struct ib_pd *ibpd, int max_page_list_len) +{ + struct qlnxr_dev *dev; + struct qlnxr_mr *mr; + qlnx_host_t *ha; + struct ib_mr *ibmr = NULL; + + dev = get_qlnxr_dev((ibpd->device)); + ha = dev->ha; + + QL_DPRINT12(ha, "enter\n"); + + mr = __qlnxr_alloc_mr(ibpd, max_page_list_len); + + if (IS_ERR(mr)) { + ibmr = ERR_PTR(-EINVAL); + } else { + ibmr = &mr->ibmr; + } + + QL_DPRINT12(ha, "exit %p\n", ibmr); + return (ibmr); +} + +void +qlnxr_free_frmr_page_list(struct ib_fast_reg_page_list *page_list) +{ + struct qlnxr_fast_reg_page_list *frmr_list; + + frmr_list = get_qlnxr_frmr_list(page_list); + + free_mr_info(frmr_list->dev, &frmr_list->info); + + kfree(frmr_list->ibfrpl.page_list); + kfree(frmr_list); + + return; +} + +struct ib_fast_reg_page_list * +qlnxr_alloc_frmr_page_list(struct ib_device *ibdev, int page_list_len) +{ + struct qlnxr_fast_reg_page_list *frmr_list = NULL; + struct qlnxr_dev *dev; + int size = page_list_len * sizeof(u64); + int rc = -ENOMEM; + qlnx_host_t *ha; + + dev = get_qlnxr_dev(ibdev); + ha = dev->ha; + + QL_DPRINT12(ha, "enter\n"); + + frmr_list = kzalloc(sizeof(*frmr_list), GFP_KERNEL); + if (!frmr_list) { + QL_DPRINT11(ha, "kzalloc(frmr_list) failed\n"); + goto err; + } + + frmr_list->dev = dev; + frmr_list->ibfrpl.page_list = kzalloc(size, GFP_KERNEL); + if (!frmr_list->ibfrpl.page_list) { + QL_DPRINT11(ha, "frmr_list->ibfrpl.page_list = NULL failed\n"); + goto err0; + } + + rc = qlnxr_init_mr_info(dev, &frmr_list->info, page_list_len, + 1 /* allow dual layer pbl */); + if (rc) + goto err1; + + QL_DPRINT12(ha, "exit %p\n", &frmr_list->ibfrpl); + + return &frmr_list->ibfrpl; + +err1: + kfree(frmr_list->ibfrpl.page_list); +err0: + kfree(frmr_list); +err: + QL_DPRINT12(ha, "exit with error\n"); + + return ERR_PTR(rc); +} + +static int +qlnxr_validate_phys_buf_list(qlnx_host_t *ha, struct ib_phys_buf *buf_list, + int buf_cnt, uint64_t *total_size) +{ + u64 size = 0; + + *total_size = 0; + + if (!buf_cnt || buf_list == NULL) { + QL_DPRINT11(ha, + "failed buf_list = %p buf_cnt = %d\n", buf_list, buf_cnt); + return (-1); + } + + size = buf_list->size; + + if (!size) { + QL_DPRINT11(ha, + "failed buf_list = %p buf_cnt = %d" + " buf_list->size = 0\n", buf_list, buf_cnt); + return (-1); + } + + while (buf_cnt) { + + *total_size += buf_list->size; + + if (buf_list->size != size) { + QL_DPRINT11(ha, + "failed buf_list = %p buf_cnt = %d" + " all buffers should have same size\n", + buf_list, buf_cnt); + return (-1); + } + + buf_list++; + buf_cnt--; + } + return (0); +} + +static size_t +qlnxr_get_num_pages(qlnx_host_t *ha, struct ib_phys_buf *buf_list, + int buf_cnt) +{ + int i; + size_t num_pages = 0; + u64 size; + + for (i = 0; i < buf_cnt; i++) { + + size = 0; + while (size < buf_list->size) { + size += PAGE_SIZE; + num_pages++; + } + buf_list++; + } + return (num_pages); +} + +static void +qlnxr_populate_phys_mem_pbls(struct qlnxr_dev *dev, + struct ib_phys_buf *buf_list, int buf_cnt, + struct qlnxr_pbl *pbl, struct qlnxr_pbl_info *pbl_info) +{ + struct regpair *pbe; + struct qlnxr_pbl *pbl_tbl; + int pg_cnt, pages, pbe_cnt, total_num_pbes = 0; + qlnx_host_t *ha; + int i; + u64 pbe_addr; + + ha = dev->ha; + + QL_DPRINT12(ha, "enter\n"); + + if (!pbl_info) { + QL_DPRINT11(ha, "PBL_INFO not initialized\n"); + return; + } + + if (!pbl_info->num_pbes) { + QL_DPRINT11(ha, "pbl_info->num_pbes == 0\n"); + return; + } + + /* If we have a two layered pbl, the first pbl points to the rest + * of the pbls and the first entry lays on the second pbl in the table + */ + if (pbl_info->two_layered) + pbl_tbl = &pbl[1]; + else + pbl_tbl = pbl; + + pbe = (struct regpair *)pbl_tbl->va; + if (!pbe) { + QL_DPRINT12(ha, "pbe is NULL\n"); + return; + } + + pbe_cnt = 0; + + for (i = 0; i < buf_cnt; i++) { + + pages = buf_list->size >> PAGE_SHIFT; + + for (pg_cnt = 0; pg_cnt < pages; pg_cnt++) { + /* store the page address in pbe */ + + pbe_addr = buf_list->addr + (PAGE_SIZE * pg_cnt); + + pbe->lo = cpu_to_le32((u32)pbe_addr); + pbe->hi = cpu_to_le32(((u32)(pbe_addr >> 32))); + + QL_DPRINT12(ha, "Populate pbl table:" + " pbe->addr=0x%x:0x%x " + " pbe_cnt = %d total_num_pbes=%d" + " pbe=%p\n", pbe->lo, pbe->hi, pbe_cnt, + total_num_pbes, pbe); + + pbe_cnt ++; + total_num_pbes ++; + pbe++; + + if (total_num_pbes == pbl_info->num_pbes) + return; + + /* if the given pbl is full storing the pbes, + * move to next pbl. */ + + if (pbe_cnt == (pbl_info->pbl_size / sizeof(u64))) { + pbl_tbl++; + pbe = (struct regpair *)pbl_tbl->va; + pbe_cnt = 0; + } + } + buf_list++; + } + QL_DPRINT12(ha, "exit\n"); + return; +} + +struct ib_mr * +qlnxr_reg_kernel_mr(struct ib_pd *ibpd, + struct ib_phys_buf *buf_list, + int buf_cnt, int acc, u64 *iova_start) +{ + int rc = -ENOMEM; + struct qlnxr_dev *dev = get_qlnxr_dev((ibpd->device)); + struct qlnxr_mr *mr; + struct qlnxr_pd *pd; + qlnx_host_t *ha; + size_t num_pages = 0; + uint64_t length; + + ha = dev->ha; + + QL_DPRINT12(ha, "enter\n"); + + pd = get_qlnxr_pd(ibpd); + + QL_DPRINT12(ha, "pd = %d buf_list = %p, buf_cnt = %d," + " iova_start = %p, acc = %d\n", + pd->pd_id, buf_list, buf_cnt, iova_start, acc); + + //if (acc & IB_ACCESS_REMOTE_WRITE && !(acc & IB_ACCESS_LOCAL_WRITE)) { + // QL_DPRINT11(ha, "(acc & IB_ACCESS_REMOTE_WRITE &&" + // " !(acc & IB_ACCESS_LOCAL_WRITE))\n"); + // return ERR_PTR(-EINVAL); + //} + + mr = kzalloc(sizeof(*mr), GFP_KERNEL); + if (!mr) { + QL_DPRINT11(ha, "kzalloc(mr) failed\n"); + return ERR_PTR(rc); + } + + mr->type = QLNXR_MR_KERNEL; + mr->iova_start = iova_start; + + rc = qlnxr_validate_phys_buf_list(ha, buf_list, buf_cnt, &length); + if (rc) + goto err0; + + num_pages = qlnxr_get_num_pages(ha, buf_list, buf_cnt); + if (!num_pages) + goto err0; + + rc = qlnxr_init_mr_info(dev, &mr->info, num_pages, 1); + if (rc) { + QL_DPRINT11(ha, + "qlnxr_init_mr_info failed [%d]\n", rc); + goto err1; + } + + qlnxr_populate_phys_mem_pbls(dev, buf_list, buf_cnt, mr->info.pbl_table, + &mr->info.pbl_info); + + rc = ecore_rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid); + + if (rc) { + QL_DPRINT11(ha, "roce alloc tid returned an error %d\n", rc); + goto err1; + } + + /* index only, 18 bit long, lkey = itid << 8 | key */ + mr->hw_mr.tid_type = ECORE_RDMA_TID_REGISTERED_MR; + mr->hw_mr.key = 0; + mr->hw_mr.pd = pd->pd_id; + mr->hw_mr.local_read = 1; + mr->hw_mr.local_write = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0; + mr->hw_mr.remote_read = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0; + mr->hw_mr.remote_write = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0; + mr->hw_mr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0; + mr->hw_mr.mw_bind = false; /* TBD MW BIND */ + mr->hw_mr.pbl_ptr = mr->info.pbl_table[0].pa; + mr->hw_mr.pbl_two_level = mr->info.pbl_info.two_layered; + mr->hw_mr.pbl_page_size_log = ilog2(mr->info.pbl_info.pbl_size); + mr->hw_mr.page_size_log = ilog2(PAGE_SIZE); /* for the MR pages */ + + mr->hw_mr.fbo = 0; + + mr->hw_mr.length = length; + mr->hw_mr.vaddr = (uint64_t)iova_start; + mr->hw_mr.zbva = false; /* TBD figure when this should be true */ + mr->hw_mr.phy_mr = false; /* Fast MR - True, Regular Register False */ + mr->hw_mr.dma_mr = false; + + rc = ecore_rdma_register_tid(dev->rdma_ctx, &mr->hw_mr); + if (rc) { + QL_DPRINT11(ha, "roce register tid returned an error %d\n", rc); + goto err2; + } + + mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key; + if (mr->hw_mr.remote_write || mr->hw_mr.remote_read || + mr->hw_mr.remote_atomic) + mr->ibmr.rkey = mr->hw_mr.itid << 8 | mr->hw_mr.key; + + QL_DPRINT12(ha, "lkey: %x\n", mr->ibmr.lkey); + + return (&mr->ibmr); + +err2: + ecore_rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid); +err1: + qlnxr_free_pbl(dev, &mr->info.pbl_info, mr->info.pbl_table); +err0: + kfree(mr); + + QL_DPRINT12(ha, "exit [%d]\n", rc); + return (ERR_PTR(rc)); +} + +#endif /* #if __FreeBSD_version >= 1102000 */ + +struct ib_ah * +#if __FreeBSD_version >= 1102000 +qlnxr_create_ah(struct ib_pd *ibpd, struct ib_ah_attr *attr, + struct ib_udata *udata) +#else +qlnxr_create_ah(struct ib_pd *ibpd, struct ib_ah_attr *attr) +#endif /* #if __FreeBSD_version >= 1102000 */ +{ + struct qlnxr_dev *dev; + qlnx_host_t *ha; + struct qlnxr_ah *ah; + + dev = get_qlnxr_dev((ibpd->device)); + ha = dev->ha; + + QL_DPRINT12(ha, "in create_ah\n"); + + ah = kzalloc(sizeof(*ah), GFP_ATOMIC); + if (!ah) { + QL_DPRINT12(ha, "no address handle can be allocated\n"); + return ERR_PTR(-ENOMEM); + } + + ah->attr = *attr; + + return &ah->ibah; +} + +int +qlnxr_destroy_ah(struct ib_ah *ibah) +{ + struct qlnxr_dev *dev; + qlnx_host_t *ha; + struct qlnxr_ah *ah = get_qlnxr_ah(ibah); + + dev = get_qlnxr_dev((ibah->device)); + ha = dev->ha; + + QL_DPRINT12(ha, "in destroy_ah\n"); + + kfree(ah); + return 0; +} + +int +qlnxr_query_ah(struct ib_ah *ibah, struct ib_ah_attr *attr) +{ + struct qlnxr_dev *dev; + qlnx_host_t *ha; + + dev = get_qlnxr_dev((ibah->device)); + ha = dev->ha; + QL_DPRINT12(ha, "Query AH not supported\n"); + return -EINVAL; +} + +int +qlnxr_modify_ah(struct ib_ah *ibah, struct ib_ah_attr *attr) +{ + struct qlnxr_dev *dev; + qlnx_host_t *ha; + + dev = get_qlnxr_dev((ibah->device)); + ha = dev->ha; + QL_DPRINT12(ha, "Modify AH not supported\n"); + return -ENOSYS; +} + +#if __FreeBSD_version >= 1102000 +int +qlnxr_process_mad(struct ib_device *ibdev, + int process_mad_flags, + u8 port_num, + const struct ib_wc *in_wc, + const struct ib_grh *in_grh, + const struct ib_mad_hdr *mad_hdr, + size_t in_mad_size, + struct ib_mad_hdr *out_mad, + size_t *out_mad_size, + u16 *out_mad_pkey_index) + +#else + +int +qlnxr_process_mad(struct ib_device *ibdev, + int process_mad_flags, + u8 port_num, + struct ib_wc *in_wc, + struct ib_grh *in_grh, + struct ib_mad *in_mad, + struct ib_mad *out_mad) + +#endif /* #if __FreeBSD_version >= 1102000 */ +{ + struct qlnxr_dev *dev; + qlnx_host_t *ha; + + dev = get_qlnxr_dev(ibdev); + ha = dev->ha; + QL_DPRINT12(ha, "process mad not supported\n"); + + return -ENOSYS; +// QL_DPRINT12(ha, "qlnxr_process_mad in_mad %x %x %x %x %x %x %x %x\n", +// in_mad->mad_hdr.attr_id, in_mad->mad_hdr.base_version, +// in_mad->mad_hdr.attr_mod, in_mad->mad_hdr.class_specific, +// in_mad->mad_hdr.class_version, in_mad->mad_hdr.method, +// in_mad->mad_hdr.mgmt_class, in_mad->mad_hdr.status); + +// return IB_MAD_RESULT_SUCCESS; +} + + +#if __FreeBSD_version >= 1102000 +int +qlnxr_get_port_immutable(struct ib_device *ibdev, u8 port_num, + struct ib_port_immutable *immutable) +{ + struct qlnxr_dev *dev; + qlnx_host_t *ha; + struct ib_port_attr attr; + int err; + + dev = get_qlnxr_dev(ibdev); + ha = dev->ha; + + QL_DPRINT12(ha, "enter\n"); + + err = qlnxr_query_port(ibdev, port_num, &attr); + if (err) + return err; + + if (QLNX_IS_IWARP(dev)) { + immutable->pkey_tbl_len = 1; + immutable->gid_tbl_len = 1; + immutable->core_cap_flags = RDMA_CORE_PORT_IWARP; + immutable->max_mad_size = 0; + } else { + immutable->pkey_tbl_len = attr.pkey_tbl_len; + immutable->gid_tbl_len = attr.gid_tbl_len; + immutable->core_cap_flags = RDMA_CORE_PORT_IBA_ROCE; + immutable->max_mad_size = IB_MGMT_MAD_SIZE; + } + + QL_DPRINT12(ha, "exit\n"); + return 0; +} +#endif /* #if __FreeBSD_version > 1102000 */ + + +/***** iWARP related functions *************/ + + +static void +qlnxr_iw_mpa_request(void *context, + struct ecore_iwarp_cm_event_params *params) +{ + struct qlnxr_iw_listener *listener = (struct qlnxr_iw_listener *)context; + struct qlnxr_dev *dev = listener->dev; + struct qlnxr_iw_ep *ep; + struct iw_cm_event event; + struct sockaddr_in *laddr; + struct sockaddr_in *raddr; + qlnx_host_t *ha; + + ha = dev->ha; + + QL_DPRINT12(ha, "enter\n"); + + if (params->cm_info->ip_version != ECORE_TCP_IPV4) { + QL_DPRINT11(ha, "only IPv4 supported [0x%x]\n", + params->cm_info->ip_version); + return; + } + + ep = kzalloc(sizeof(*ep), GFP_ATOMIC); + + if (!ep) { + QL_DPRINT11(ha, "kzalloc{ep) failed\n"); + return; + } + + ep->dev = dev; + ep->ecore_context = params->ep_context; + + memset(&event, 0, sizeof(event)); + + event.event = IW_CM_EVENT_CONNECT_REQUEST; + event.status = params->status; + + laddr = (struct sockaddr_in *)&event.local_addr; + raddr = (struct sockaddr_in *)&event.remote_addr; + + laddr->sin_family = AF_INET; + raddr->sin_family = AF_INET; + + laddr->sin_port = htons(params->cm_info->local_port); + raddr->sin_port = htons(params->cm_info->remote_port); + + laddr->sin_addr.s_addr = htonl(params->cm_info->local_ip[0]); + raddr->sin_addr.s_addr = htonl(params->cm_info->remote_ip[0]); + + event.provider_data = (void *)ep; + event.private_data = (void *)params->cm_info->private_data; + event.private_data_len = (u8)params->cm_info->private_data_len; + +#if __FreeBSD_version >= 1100000 + event.ord = params->cm_info->ord; + event.ird = params->cm_info->ird; +#endif /* #if __FreeBSD_version >= 1100000 */ + + listener->cm_id->event_handler(listener->cm_id, &event); + + QL_DPRINT12(ha, "exit\n"); + + return; +} + +static void +qlnxr_iw_issue_event(void *context, + struct ecore_iwarp_cm_event_params *params, + enum iw_cm_event_type event_type, + char *str) +{ + struct qlnxr_iw_ep *ep = (struct qlnxr_iw_ep *)context; + struct qlnxr_dev *dev = ep->dev; + struct iw_cm_event event; + qlnx_host_t *ha; + + ha = dev->ha; + + QL_DPRINT12(ha, "enter\n"); + + memset(&event, 0, sizeof(event)); + event.status = params->status; + event.event = event_type; + + if (params->cm_info != NULL) { +#if __FreeBSD_version >= 1100000 + event.ird = params->cm_info->ird; + event.ord = params->cm_info->ord; + QL_DPRINT12(ha, "ord=[%d] \n", event.ord); + QL_DPRINT12(ha, "ird=[%d] \n", event.ird); +#endif /* #if __FreeBSD_version >= 1100000 */ + + event.private_data_len = params->cm_info->private_data_len; + event.private_data = (void *)params->cm_info->private_data; + QL_DPRINT12(ha, "private_data_len=[%d] \n", + event.private_data_len); + } + + QL_DPRINT12(ha, "event=[%d] %s\n", event.event, str); + QL_DPRINT12(ha, "status=[%d] \n", event.status); + + if (ep) { + if (ep->cm_id) + ep->cm_id->event_handler(ep->cm_id, &event); + else + QL_DPRINT11(ha, "ep->cm_id == NULL \n"); + } else { + QL_DPRINT11(ha, "ep == NULL \n"); + } + + QL_DPRINT12(ha, "exit\n"); + + return; +} + +static void +qlnxr_iw_close_event(void *context, + struct ecore_iwarp_cm_event_params *params) +{ + struct qlnxr_iw_ep *ep = (struct qlnxr_iw_ep *)context; + struct qlnxr_dev *dev = ep->dev; + qlnx_host_t *ha; + + ha = dev->ha; + + QL_DPRINT12(ha, "enter\n"); + + if (ep->cm_id) { + qlnxr_iw_issue_event(context, + params, + IW_CM_EVENT_CLOSE, + "IW_CM_EVENT_EVENT_CLOSE"); + ep->cm_id->rem_ref(ep->cm_id); + ep->cm_id = NULL; + } + + QL_DPRINT12(ha, "exit\n"); + + return; +} + +#if __FreeBSD_version >= 1102000 + +static void +qlnxr_iw_passive_complete(void *context, + struct ecore_iwarp_cm_event_params *params) +{ + struct qlnxr_iw_ep *ep = (struct qlnxr_iw_ep *)context; + struct qlnxr_dev *dev = ep->dev; + qlnx_host_t *ha; + + ha = dev->ha; + + /* We will only reach the following state if MPA_REJECT was called on + * passive. In this case there will be no associated QP. + */ + if ((params->status == -ECONNREFUSED) && (ep->qp == NULL)) { + QL_DPRINT11(ha, "PASSIVE connection refused releasing ep...\n"); + kfree(ep); + return; + } + + /* We always issue an established event, however, ofed does not look + * at event code for established. So if there was a failure, we follow + * with close... + */ + qlnxr_iw_issue_event(context, + params, + IW_CM_EVENT_ESTABLISHED, + "IW_CM_EVENT_ESTABLISHED"); + + if (params->status < 0) { + qlnxr_iw_close_event(context, params); + } + + return; +} + +struct qlnxr_discon_work { + struct work_struct work; + struct qlnxr_iw_ep *ep; + enum ecore_iwarp_event_type event; + int status; +}; + +static void +qlnxr_iw_disconnect_worker(struct work_struct *work) +{ + struct qlnxr_discon_work *dwork = + container_of(work, struct qlnxr_discon_work, work); + struct ecore_rdma_modify_qp_in_params qp_params = { 0 }; + struct qlnxr_iw_ep *ep = dwork->ep; + struct qlnxr_dev *dev = ep->dev; + struct qlnxr_qp *qp = ep->qp; + struct iw_cm_event event; + + if (qp->destroyed) { + kfree(dwork); + qlnxr_iw_qp_rem_ref(&qp->ibqp); + return; + } + + memset(&event, 0, sizeof(event)); + event.status = dwork->status; + event.event = IW_CM_EVENT_DISCONNECT; + + /* Success means graceful disconnect was requested. modifying + * to SQD is translated to graceful disconnect. O/w reset is sent + */ + if (dwork->status) + qp_params.new_state = ECORE_ROCE_QP_STATE_ERR; + else + qp_params.new_state = ECORE_ROCE_QP_STATE_SQD; + + kfree(dwork); + + if (ep->cm_id) + ep->cm_id->event_handler(ep->cm_id, &event); + + SET_FIELD(qp_params.modify_flags, + ECORE_RDMA_MODIFY_QP_VALID_NEW_STATE, 1); + + ecore_rdma_modify_qp(dev->rdma_ctx, qp->ecore_qp, &qp_params); + + qlnxr_iw_qp_rem_ref(&qp->ibqp); + + return; +} + +void +qlnxr_iw_disconnect_event(void *context, + struct ecore_iwarp_cm_event_params *params) +{ + struct qlnxr_discon_work *work; + struct qlnxr_iw_ep *ep = (struct qlnxr_iw_ep *)context; + struct qlnxr_dev *dev = ep->dev; + struct qlnxr_qp *qp = ep->qp; + + work = kzalloc(sizeof(*work), GFP_ATOMIC); + if (!work) + return; + + qlnxr_iw_qp_add_ref(&qp->ibqp); + work->ep = ep; + work->event = params->event; + work->status = params->status; + + INIT_WORK(&work->work, qlnxr_iw_disconnect_worker); + queue_work(dev->iwarp_wq, &work->work); + + return; +} + +#endif /* #if __FreeBSD_version >= 1102000 */ + +static int +qlnxr_iw_mpa_reply(void *context, + struct ecore_iwarp_cm_event_params *params) +{ + struct qlnxr_iw_ep *ep = (struct qlnxr_iw_ep *)context; + struct qlnxr_dev *dev = ep->dev; + struct ecore_iwarp_send_rtr_in rtr_in; + int rc; + qlnx_host_t *ha; + + ha = dev->ha; + + QL_DPRINT12(ha, "enter\n"); + + if (!(ha->ifp->if_drv_flags & IFF_DRV_RUNNING)) + return -EINVAL; + + bzero(&rtr_in, sizeof(struct ecore_iwarp_send_rtr_in)); + rtr_in.ep_context = params->ep_context; + + rc = ecore_iwarp_send_rtr(dev->rdma_ctx, &rtr_in); + + QL_DPRINT12(ha, "exit rc = %d\n", rc); + return rc; +} + + +void +qlnxr_iw_qp_event(void *context, + struct ecore_iwarp_cm_event_params *params, + enum ib_event_type ib_event, + char *str) +{ + struct qlnxr_iw_ep *ep = (struct qlnxr_iw_ep *)context; + struct qlnxr_dev *dev = ep->dev; + struct ib_qp *ibqp = &(ep->qp->ibqp); + struct ib_event event; + qlnx_host_t *ha; + + ha = dev->ha; + + QL_DPRINT12(ha, + "[context, event, event_handler] = [%p, 0x%x, %s, %p] enter\n", + context, params->event, str, ibqp->event_handler); + + if (ibqp->event_handler) { + event.event = ib_event; + event.device = ibqp->device; + event.element.qp = ibqp; + ibqp->event_handler(&event, ibqp->qp_context); + } + + return; +} + +int +qlnxr_iw_event_handler(void *context, + struct ecore_iwarp_cm_event_params *params) +{ + struct qlnxr_iw_ep *ep = (struct qlnxr_iw_ep *)context; + struct qlnxr_dev *dev = ep->dev; + qlnx_host_t *ha; + + ha = dev->ha; + + QL_DPRINT12(ha, "[context, event] = [%p, 0x%x] " + "enter\n", context, params->event); + + switch (params->event) { + + /* Passive side request received */ + case ECORE_IWARP_EVENT_MPA_REQUEST: + qlnxr_iw_mpa_request(context, params); + break; + + case ECORE_IWARP_EVENT_ACTIVE_MPA_REPLY: + qlnxr_iw_mpa_reply(context, params); + break; + + /* Passive side established ( ack on mpa response ) */ + case ECORE_IWARP_EVENT_PASSIVE_COMPLETE: + +#if __FreeBSD_version >= 1102000 + + ep->during_connect = 0; + qlnxr_iw_passive_complete(context, params); + +#else + qlnxr_iw_issue_event(context, + params, + IW_CM_EVENT_ESTABLISHED, + "IW_CM_EVENT_ESTABLISHED"); +#endif /* #if __FreeBSD_version >= 1102000 */ + break; + + /* Active side reply received */ + case ECORE_IWARP_EVENT_ACTIVE_COMPLETE: + ep->during_connect = 0; + qlnxr_iw_issue_event(context, + params, + IW_CM_EVENT_CONNECT_REPLY, + "IW_CM_EVENT_CONNECT_REPLY"); + if (params->status < 0) { + struct qlnxr_iw_ep *ep = (struct qlnxr_iw_ep *)context; + + ep->cm_id->rem_ref(ep->cm_id); + ep->cm_id = NULL; + } + break; + + case ECORE_IWARP_EVENT_DISCONNECT: + +#if __FreeBSD_version >= 1102000 + qlnxr_iw_disconnect_event(context, params); +#else + qlnxr_iw_issue_event(context, + params, + IW_CM_EVENT_DISCONNECT, + "IW_CM_EVENT_DISCONNECT"); + qlnxr_iw_close_event(context, params); +#endif /* #if __FreeBSD_version >= 1102000 */ + break; + + case ECORE_IWARP_EVENT_CLOSE: + ep->during_connect = 0; + qlnxr_iw_close_event(context, params); + break; + + case ECORE_IWARP_EVENT_RQ_EMPTY: + qlnxr_iw_qp_event(context, params, IB_EVENT_QP_FATAL, + "IWARP_EVENT_RQ_EMPTY"); + break; + + case ECORE_IWARP_EVENT_IRQ_FULL: + qlnxr_iw_qp_event(context, params, IB_EVENT_QP_FATAL, + "IWARP_EVENT_IRQ_FULL"); + break; + + case ECORE_IWARP_EVENT_LLP_TIMEOUT: + qlnxr_iw_qp_event(context, params, IB_EVENT_QP_FATAL, + "IWARP_EVENT_LLP_TIMEOUT"); + break; + + case ECORE_IWARP_EVENT_REMOTE_PROTECTION_ERROR: + qlnxr_iw_qp_event(context, params, IB_EVENT_QP_ACCESS_ERR, + "IWARP_EVENT_REMOTE_PROTECTION_ERROR"); + break; + + case ECORE_IWARP_EVENT_CQ_OVERFLOW: + qlnxr_iw_qp_event(context, params, IB_EVENT_QP_FATAL, + "QED_IWARP_EVENT_CQ_OVERFLOW"); + break; + + case ECORE_IWARP_EVENT_QP_CATASTROPHIC: + qlnxr_iw_qp_event(context, params, IB_EVENT_QP_FATAL, + "QED_IWARP_EVENT_QP_CATASTROPHIC"); + break; + + case ECORE_IWARP_EVENT_LOCAL_ACCESS_ERROR: + qlnxr_iw_qp_event(context, params, IB_EVENT_QP_ACCESS_ERR, + "IWARP_EVENT_LOCAL_ACCESS_ERROR"); + break; + + case ECORE_IWARP_EVENT_REMOTE_OPERATION_ERROR: + qlnxr_iw_qp_event(context, params, IB_EVENT_QP_FATAL, + "IWARP_EVENT_REMOTE_OPERATION_ERROR"); + break; + + case ECORE_IWARP_EVENT_TERMINATE_RECEIVED: + QL_DPRINT12(ha, "Got terminate message" + " ECORE_IWARP_EVENT_TERMINATE_RECEIVED\n"); + break; + + default: + QL_DPRINT12(ha, + "Unknown event [0x%x] received \n", params->event); + break; + }; + + QL_DPRINT12(ha, "[context, event] = [%p, 0x%x] " + "exit\n", context, params->event); + return 0; +} + +static int +qlnxr_addr4_resolve(struct qlnxr_dev *dev, + struct sockaddr_in *src_in, + struct sockaddr_in *dst_in, + u8 *dst_mac) +{ + int rc; + +#if __FreeBSD_version >= 1100000 + rc = arpresolve(dev->ha->ifp, 0, NULL, (struct sockaddr *)dst_in, + dst_mac, NULL, NULL); +#else + struct llentry *lle; + + rc = arpresolve(dev->ha->ifp, NULL, NULL, (struct sockaddr *)dst_in, + dst_mac, &lle); +#endif + + QL_DPRINT12(dev->ha, "rc = %d " + "sa_len = 0x%x sa_family = 0x%x IP Address = %d.%d.%d.%d " + "Dest MAC %02x:%02x:%02x:%02x:%02x:%02x\n", rc, + dst_in->sin_len, dst_in->sin_family, + NIPQUAD((dst_in->sin_addr.s_addr)), + dst_mac[0], dst_mac[1], dst_mac[2], + dst_mac[3], dst_mac[4], dst_mac[5]); + + return rc; +} + +int +qlnxr_iw_connect(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param) +{ + struct qlnxr_dev *dev; + struct ecore_iwarp_connect_out out_params; + struct ecore_iwarp_connect_in in_params; + struct qlnxr_iw_ep *ep; + struct qlnxr_qp *qp; + struct sockaddr_in *laddr; + struct sockaddr_in *raddr; + int rc = 0; + qlnx_host_t *ha; + + dev = get_qlnxr_dev((cm_id->device)); + ha = dev->ha; + + QL_DPRINT12(ha, "[cm_id, conn_param] = [%p, %p] " + "enter \n", cm_id, conn_param); + + if (!(ha->ifp->if_drv_flags & IFF_DRV_RUNNING)) + return -EINVAL; + + qp = idr_find(&dev->qpidr, conn_param->qpn); + + laddr = (struct sockaddr_in *)&cm_id->local_addr; + raddr = (struct sockaddr_in *)&cm_id->remote_addr; + + QL_DPRINT12(ha, + "local = [%d.%d.%d.%d, %d] remote = [%d.%d.%d.%d, %d]\n", + NIPQUAD((laddr->sin_addr.s_addr)), laddr->sin_port, + NIPQUAD((raddr->sin_addr.s_addr)), raddr->sin_port); + + ep = kzalloc(sizeof(*ep), GFP_KERNEL); + if (!ep) { + QL_DPRINT11(ha, "struct qlnxr_iw_ep " + "alloc memory failed\n"); + return -ENOMEM; + } + + ep->dev = dev; + ep->qp = qp; + cm_id->add_ref(cm_id); + ep->cm_id = cm_id; + + memset(&in_params, 0, sizeof (struct ecore_iwarp_connect_in)); + memset(&out_params, 0, sizeof (struct ecore_iwarp_connect_out)); + + in_params.event_cb = qlnxr_iw_event_handler; + in_params.cb_context = ep; + + in_params.cm_info.ip_version = ECORE_TCP_IPV4; + + in_params.cm_info.remote_ip[0] = ntohl(raddr->sin_addr.s_addr); + in_params.cm_info.local_ip[0] = ntohl(laddr->sin_addr.s_addr); + in_params.cm_info.remote_port = ntohs(raddr->sin_port); + in_params.cm_info.local_port = ntohs(laddr->sin_port); + in_params.cm_info.vlan = 0; + in_params.mss = dev->ha->ifp->if_mtu - 40; + + QL_DPRINT12(ha, "remote_ip = [%d.%d.%d.%d] " + "local_ip = [%d.%d.%d.%d] remote_port = %d local_port = %d " + "vlan = %d\n", + NIPQUAD((in_params.cm_info.remote_ip[0])), + NIPQUAD((in_params.cm_info.local_ip[0])), + in_params.cm_info.remote_port, in_params.cm_info.local_port, + in_params.cm_info.vlan); + + rc = qlnxr_addr4_resolve(dev, laddr, raddr, (u8 *)in_params.remote_mac_addr); + + if (rc) { + QL_DPRINT11(ha, "qlnxr_addr4_resolve failed\n"); + goto err; + } + + QL_DPRINT12(ha, "ord = %d ird=%d private_data=%p" + " private_data_len=%d rq_psn=%d\n", + conn_param->ord, conn_param->ird, conn_param->private_data, + conn_param->private_data_len, qp->rq_psn); + + in_params.cm_info.ord = conn_param->ord; + in_params.cm_info.ird = conn_param->ird; + in_params.cm_info.private_data = conn_param->private_data; + in_params.cm_info.private_data_len = conn_param->private_data_len; + in_params.qp = qp->ecore_qp; + + memcpy(in_params.local_mac_addr, dev->ha->primary_mac, ETH_ALEN); + + rc = ecore_iwarp_connect(dev->rdma_ctx, &in_params, &out_params); + + if (rc) { + QL_DPRINT12(ha, "ecore_iwarp_connect failed\n"); + goto err; + } + + QL_DPRINT12(ha, "exit\n"); + + return rc; + +err: + cm_id->rem_ref(cm_id); + kfree(ep); + + QL_DPRINT12(ha, "exit [%d]\n", rc); + return rc; +} + +int +qlnxr_iw_create_listen(struct iw_cm_id *cm_id, int backlog) +{ + struct qlnxr_dev *dev; + struct qlnxr_iw_listener *listener; + struct ecore_iwarp_listen_in iparams; + struct ecore_iwarp_listen_out oparams; + struct sockaddr_in *laddr; + qlnx_host_t *ha; + int rc; + + dev = get_qlnxr_dev((cm_id->device)); + ha = dev->ha; + + QL_DPRINT12(ha, "enter\n"); + + if (!(ha->ifp->if_drv_flags & IFF_DRV_RUNNING)) + return -EINVAL; + + laddr = (struct sockaddr_in *)&cm_id->local_addr; + + listener = kzalloc(sizeof(*listener), GFP_KERNEL); + + if (listener == NULL) { + QL_DPRINT11(ha, "listener memory alloc failed\n"); + return -ENOMEM; + } + + listener->dev = dev; + cm_id->add_ref(cm_id); + listener->cm_id = cm_id; + listener->backlog = backlog; + + memset(&iparams, 0, sizeof (struct ecore_iwarp_listen_in)); + memset(&oparams, 0, sizeof (struct ecore_iwarp_listen_out)); + + iparams.cb_context = listener; + iparams.event_cb = qlnxr_iw_event_handler; + iparams.max_backlog = backlog; + + iparams.ip_version = ECORE_TCP_IPV4; + + iparams.ip_addr[0] = ntohl(laddr->sin_addr.s_addr); + iparams.port = ntohs(laddr->sin_port); + iparams.vlan = 0; + + QL_DPRINT12(ha, "[%d.%d.%d.%d, %d] iparamsport=%d\n", + NIPQUAD((laddr->sin_addr.s_addr)), + laddr->sin_port, iparams.port); + + rc = ecore_iwarp_create_listen(dev->rdma_ctx, &iparams, &oparams); + if (rc) { + QL_DPRINT11(ha, + "ecore_iwarp_create_listen failed rc = %d\n", rc); + goto err; + } + + listener->ecore_handle = oparams.handle; + cm_id->provider_data = listener; + + QL_DPRINT12(ha, "exit\n"); + return rc; + +err: + cm_id->rem_ref(cm_id); + kfree(listener); + + QL_DPRINT12(ha, "exit [%d]\n", rc); + return rc; +} + +void +qlnxr_iw_destroy_listen(struct iw_cm_id *cm_id) +{ + struct qlnxr_iw_listener *listener = cm_id->provider_data; + struct qlnxr_dev *dev = get_qlnxr_dev((cm_id->device)); + int rc = 0; + qlnx_host_t *ha; + + ha = dev->ha; + + QL_DPRINT12(ha, "enter\n"); + + if (listener->ecore_handle) + rc = ecore_iwarp_destroy_listen(dev->rdma_ctx, + listener->ecore_handle); + + cm_id->rem_ref(cm_id); + + QL_DPRINT12(ha, "exit [%d]\n", rc); + return; +} + +int +qlnxr_iw_accept(struct iw_cm_id *cm_id, + struct iw_cm_conn_param *conn_param) +{ + struct qlnxr_iw_ep *ep = (struct qlnxr_iw_ep *)cm_id->provider_data; + struct qlnxr_dev *dev = ep->dev; + struct qlnxr_qp *qp; + struct ecore_iwarp_accept_in params; + int rc; + qlnx_host_t *ha; + + ha = dev->ha; + + QL_DPRINT12(ha, "enter qpid=%d\n", conn_param->qpn); + + if (!(ha->ifp->if_drv_flags & IFF_DRV_RUNNING)) + return -EINVAL; + + qp = idr_find(&dev->qpidr, conn_param->qpn); + if (!qp) { + QL_DPRINT11(ha, "idr_find failed invalid qpn = %d\n", + conn_param->qpn); + return -EINVAL; + } + ep->qp = qp; + qp->ep = ep; + cm_id->add_ref(cm_id); + ep->cm_id = cm_id; + + params.ep_context = ep->ecore_context; + params.cb_context = ep; + params.qp = ep->qp->ecore_qp; + params.private_data = conn_param->private_data; + params.private_data_len = conn_param->private_data_len; + params.ird = conn_param->ird; + params.ord = conn_param->ord; + + rc = ecore_iwarp_accept(dev->rdma_ctx, ¶ms); + if (rc) { + QL_DPRINT11(ha, "ecore_iwarp_accept failed %d\n", rc); + goto err; + } + + QL_DPRINT12(ha, "exit\n"); + return 0; +err: + cm_id->rem_ref(cm_id); + QL_DPRINT12(ha, "exit rc = %d\n", rc); + return rc; +} + +int +qlnxr_iw_reject(struct iw_cm_id *cm_id, const void *pdata, u8 pdata_len) +{ +#if __FreeBSD_version >= 1102000 + + struct qlnxr_iw_ep *ep = (struct qlnxr_iw_ep *)cm_id->provider_data; + struct qlnxr_dev *dev = ep->dev; + struct ecore_iwarp_reject_in params; + int rc; + + params.ep_context = ep->ecore_context; + params.cb_context = ep; + params.private_data = pdata; + params.private_data_len = pdata_len; + ep->qp = NULL; + + rc = ecore_iwarp_reject(dev->rdma_ctx, ¶ms); + + return rc; + +#else + + printf("iWARP reject_cr not implemented\n"); + return -EINVAL; + +#endif /* #if __FreeBSD_version >= 1102000 */ +} + +void +qlnxr_iw_qp_add_ref(struct ib_qp *ibqp) +{ + struct qlnxr_qp *qp = get_qlnxr_qp(ibqp); + qlnx_host_t *ha; + + ha = qp->dev->ha; + + QL_DPRINT12(ha, "enter ibqp = %p\n", ibqp); + + atomic_inc(&qp->refcnt); + + QL_DPRINT12(ha, "exit \n"); + return; +} + +void +qlnxr_iw_qp_rem_ref(struct ib_qp *ibqp) +{ + struct qlnxr_qp *qp = get_qlnxr_qp(ibqp); + qlnx_host_t *ha; + + ha = qp->dev->ha; + + QL_DPRINT12(ha, "enter ibqp = %p qp = %p\n", ibqp, qp); + + if (atomic_dec_and_test(&qp->refcnt)) { + qlnxr_idr_remove(qp->dev, qp->qp_id); + kfree(qp); + } + + QL_DPRINT12(ha, "exit \n"); + return; +} + +struct ib_qp * +qlnxr_iw_get_qp(struct ib_device *ibdev, int qpn) +{ + struct qlnxr_dev *dev = get_qlnxr_dev(ibdev); + struct ib_qp *qp; + qlnx_host_t *ha; + + ha = dev->ha; + + QL_DPRINT12(ha, "enter dev = %p ibdev = %p qpn = %d\n", dev, ibdev, qpn); + + qp = idr_find(&dev->qpidr, qpn); + + QL_DPRINT12(ha, "exit qp = %p\n", qp); + + return (qp); +} Property changes on: head/sys/dev/qlnx/qlnxr/qlnxr_verbs.c ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Added: svn:mime-type ## -0,0 +1 ## +text/plain \ No newline at end of property Index: head/sys/dev/qlnx/qlnxr/qlnxr_verbs.h =================================================================== --- head/sys/dev/qlnx/qlnxr/qlnxr_verbs.h (nonexistent) +++ head/sys/dev/qlnx/qlnxr/qlnxr_verbs.h (revision 343598) @@ -0,0 +1,267 @@ +/* + * Copyright (c) 2018-2019 Cavium, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * + * $FreeBSD$ + */ + + +#ifndef __QLNXR_VERBS_H__ +#define __QLNXR_VERBS_H__ + +extern int qlnxr_iw_query_gid(struct ib_device *, + uint8_t port, + int index, + union ib_gid *gid); + +extern int qlnxr_query_gid(struct ib_device *, + u8 port, + int index, + union ib_gid *gid); + +extern struct ib_srq *qlnxr_create_srq(struct ib_pd *, + struct ib_srq_init_attr *, + struct ib_udata *); + +extern int qlnxr_destroy_srq(struct ib_srq *); + + +extern int qlnxr_modify_srq(struct ib_srq *, + struct ib_srq_attr *, + enum ib_srq_attr_mask, + struct ib_udata *); + +extern int qlnxr_query_srq(struct ib_srq *, + struct ib_srq_attr *); + +extern int qlnxr_post_srq_recv(struct ib_srq *, + struct ib_recv_wr *, + struct ib_recv_wr **bad_recv_wr); + +#if __FreeBSD_version < 1102000 +extern int qlnxr_query_device(struct ib_device *, struct ib_device_attr *); +#else +extern int qlnxr_query_device(struct ib_device *, struct ib_device_attr *, + struct ib_udata *); +extern int qlnxr_get_port_immutable(struct ib_device *ibdev, u8 port_num, + struct ib_port_immutable *immutable); +#endif + +extern int qlnxr_query_port(struct ib_device *, + u8 port, + struct ib_port_attr *props); + +extern int qlnxr_modify_port(struct ib_device *, + u8 port, + int mask, + struct ib_port_modify *props); + +extern enum rdma_link_layer qlnxr_link_layer(struct ib_device *device, + uint8_t port_num); + +struct ib_pd *qlnxr_alloc_pd(struct ib_device *, + struct ib_ucontext *, + struct ib_udata *); + +extern int qlnxr_dealloc_pd(struct ib_pd *pd); + +#if __FreeBSD_version >= 1102000 +extern struct ib_cq *qlnxr_create_cq(struct ib_device *ibdev, + const struct ib_cq_init_attr *attr, + struct ib_ucontext *ib_ctx, + struct ib_udata *udata); +#else +#if __FreeBSD_version >= 1100000 +extern struct ib_cq *qlnxr_create_cq(struct ib_device *ibdev, + struct ib_cq_init_attr *attr, + struct ib_ucontext *ib_ctx, + struct ib_udata *udata); +#else +extern struct ib_cq *qlnxr_create_cq(struct ib_device *ibdev, + int cqe, + int comp_vector, + struct ib_ucontext *ib_ctx, + struct ib_udata *udata); +#endif +#endif /* #if __FreeBSD_version >= 1102000 */ + +extern int qlnxr_destroy_cq(struct ib_cq *); + +extern int qlnxr_resize_cq(struct ib_cq *, + int cqe, + struct ib_udata *); + +extern int qlnxr_poll_cq(struct ib_cq *, + int num_entries, + struct ib_wc *wc); + + +extern struct ib_qp *qlnxr_create_qp(struct ib_pd *, + struct ib_qp_init_attr *attrs, + struct ib_udata *); + +extern int qlnxr_modify_qp(struct ib_qp *, + struct ib_qp_attr *attr, + int attr_mask, + struct ib_udata *udata); + +extern int qlnxr_query_qp(struct ib_qp *, + struct ib_qp_attr *qp_attr, + int qp_attr_mask, + struct ib_qp_init_attr *); + +extern int qlnxr_destroy_qp(struct ib_qp *); + +extern int qlnxr_query_pkey(struct ib_device *, + u8 port, + u16 index, + u16 *pkey); + +#if __FreeBSD_version >= 1102000 +extern struct ib_ah *qlnxr_create_ah(struct ib_pd *ibpd, + struct ib_ah_attr *attr, struct ib_udata *udata); +#else +extern struct ib_ah *qlnxr_create_ah(struct ib_pd *ibpd, + struct ib_ah_attr *attr); +#endif /* #if __FreeBSD_version >= 1102000 */ + +extern int qlnxr_destroy_ah(struct ib_ah *ibah); + +extern int qlnxr_query_ah(struct ib_ah *ibah, + struct ib_ah_attr *attr); + +extern int qlnxr_modify_ah(struct ib_ah *ibah, + struct ib_ah_attr *attr); + +#if __FreeBSD_version >= 1102000 +extern int qlnxr_process_mad(struct ib_device *ibdev, + int process_mad_flags, + u8 port_num, + const struct ib_wc *in_wc, + const struct ib_grh *in_grh, + const struct ib_mad_hdr *mad_hdr, + size_t in_mad_size, + struct ib_mad_hdr *out_mad, + size_t *out_mad_size, + u16 *out_mad_pkey_index); +#else +extern int qlnxr_process_mad(struct ib_device *ibdev, + int process_mad_flags, + u8 port_num, + struct ib_wc *in_wc, + struct ib_grh *in_grh, + struct ib_mad *in_mad, + struct ib_mad *out_mad); +#endif /* #if __FreeBSD_version >= 1102000 */ + +extern int qlnxr_post_send(struct ib_qp *, + struct ib_send_wr *, + struct ib_send_wr **bad_wr); + +extern int qlnxr_post_recv(struct ib_qp *, + struct ib_recv_wr *, + struct ib_recv_wr **bad_wr); + +extern int qlnxr_arm_cq(struct ib_cq *, + enum ib_cq_notify_flags flags); + +extern struct ib_mr *qlnxr_get_dma_mr(struct ib_pd *, + int acc); + +#if __FreeBSD_version < 1102000 +extern struct ib_mr *qlnxr_reg_kernel_mr(struct ib_pd *, + struct ib_phys_buf *buffer_list, + int num_phys_buf, + int acc, + u64 *iova_start); +#endif /* #if __FreeBSD_version < 1102000 */ + +extern int qlnxr_dereg_mr(struct ib_mr *); + +#if __FreeBSD_version >= 1102000 +extern struct ib_mr *qlnxr_reg_user_mr(struct ib_pd *, + u64 start, + u64 length, + u64 virt, + int acc, + struct ib_udata *); +#else +extern struct ib_mr *qlnxr_reg_user_mr(struct ib_pd *, + u64 start, + u64 length, + u64 virt, + int acc, + struct ib_udata *, + int mr_id); +#endif /* #if __FreeBSD_version >= 1102000 */ + +#if __FreeBSD_version >= 1102000 + +extern struct ib_mr *qlnxr_alloc_mr(struct ib_pd *pd, + enum ib_mr_type mr_type, u32 max_num_sg); +extern int qlnxr_map_mr_sg(struct ib_mr *mr, struct scatterlist *sg, + int sg_nents, unsigned int *sg_offset); +#else + +extern struct ib_mr *qlnxr_alloc_frmr(struct ib_pd *pd, + int max_page_list_len); + + +extern struct ib_fast_reg_page_list *qlnxr_alloc_frmr_page_list( + struct ib_device *ibdev, + int page_list_len); + +extern void qlnxr_free_frmr_page_list(struct ib_fast_reg_page_list *page_list); + +#endif /* #if __FreeBSD_version >= 1102000 */ + +extern struct ib_ucontext *qlnxr_alloc_ucontext(struct ib_device *ibdev, + struct ib_udata *udata); + +extern int qlnxr_dealloc_ucontext(struct ib_ucontext *ibctx); + + +extern int qlnxr_mmap(struct ib_ucontext *, struct vm_area_struct *vma); + +extern int qlnxr_iw_connect(struct iw_cm_id *cm_id, + struct iw_cm_conn_param *conn_param); + +extern int qlnxr_iw_create_listen(struct iw_cm_id *cm_id, int backlog); + +void qlnxr_iw_destroy_listen(struct iw_cm_id *cm_id); + +extern int qlnxr_iw_accept(struct iw_cm_id *cm_id, + struct iw_cm_conn_param *conn_param); + +extern int qlnxr_iw_reject(struct iw_cm_id *cm_id, const void *pdata, u8 pdata_len); + +extern void qlnxr_iw_qp_add_ref(struct ib_qp *qp); + +extern void qlnxr_iw_qp_rem_ref(struct ib_qp *qp); + +extern struct ib_qp *qlnxr_iw_get_qp(struct ib_device *dev, int qpn); + +#endif /* #ifndef __QLNXR_VERBS_H__ */ + Property changes on: head/sys/dev/qlnx/qlnxr/qlnxr_verbs.h ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Added: svn:mime-type ## -0,0 +1 ## +text/plain \ No newline at end of property Index: head/sys/modules/qlnx/Makefile =================================================================== --- head/sys/modules/qlnx/Makefile (revision 343597) +++ head/sys/modules/qlnx/Makefile (revision 343598) @@ -1,41 +1,42 @@ #/* # * Copyright (c) 2017-2018 Cavium, Inc. # * All rights reserved. # * # * Redistribution and use in source and binary forms, with or without # * modification, are permitted provided that the following conditions # * are met: # * # * 1. Redistributions of source code must retain the above copyright # * notice, this list of conditions and the following disclaimer. # * 2. Redistributions in binary form must reproduce the above copyright # * notice, this list of conditions and the following disclaimer in the # * documentation and/or other materials provided with the distribution. # * # * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" # * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE # * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE # * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR # * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF # * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS # * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN # * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) # * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # * POSSIBILITY OF SUCH DAMAGE. # */ #/* # * File : Makefile # * Author : David C Somayajulu, Cavium, Inc., San Jose, CA 95131. # */ # # $FreeBSD$ # SYSDIR?=${SRCTOP}/sys .include "${SYSDIR}/conf/kern.opts.mk" SUBDIR=qlnxe SUBDIR+=qlnxev +SUBDIR+=qlnxr .include Index: head/sys/modules/qlnx/qlnxe/Makefile =================================================================== --- head/sys/modules/qlnx/qlnxe/Makefile (revision 343597) +++ head/sys/modules/qlnx/qlnxe/Makefile (revision 343598) @@ -1,78 +1,98 @@ #/* # * Copyright (c) 2017-2018 Cavium, Inc. # * All rights reserved. # * # * Redistribution and use in source and binary forms, with or without # * modification, are permitted provided that the following conditions # * are met: # * # * 1. Redistributions of source code must retain the above copyright # * notice, this list of conditions and the following disclaimer. # * 2. Redistributions in binary form must reproduce the above copyright # * notice, this list of conditions and the following disclaimer in the # * documentation and/or other materials provided with the distribution. # * # * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" # * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE # * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE # * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR # * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF # * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS # * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN # * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) # * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # * POSSIBILITY OF SUCH DAMAGE. # */ #/* # * File : Makefile # * Author : David C Somayajulu, Cavium, Inc., San Jose, CA 95131. # */ # # $FreeBSD$ # .PATH: ${SRCTOP}/sys/dev/qlnx/qlnxe #.PATH: ${.CURDIR} KMOD=if_qlnxe SRCS=ecore_cxt.c ecore_dcbx.c ecore_dev.c ecore_hw.c SRCS+=ecore_init_fw_funcs.c ecore_int.c ecore_mcp.c SRCS+=ecore_sp_commands.c ecore_spq.c ecore_l2.c SRCS+=ecore_init_ops.c ecore_dbg_fw_funcs.c SRCS+=ecore_mng_tlv.c SRCS+=ecore_sriov.c SRCS+=ecore_vf.c +#roce/iwarp files. Compilation can be turned off roce/iwarp are not required. +# In other words if you don't need RDMA please comment out SRCS adds for +# ecore_rdma.c ecore_roce.c ecore_iwarp.c ecore_ooo.c ecore_ll2.c qlnx_rdma.c +SRCS+=ecore_rdma.c +SRCS+=ecore_roce.c +SRCS+=ecore_iwarp.c +SRCS+=ecore_ooo.c +SRCS+=ecore_ll2.c +SRCS+=qlnx_rdma.c + + SRCS+=qlnx_ioctl.c SRCS+=qlnx_os.c SRCS+= device_if.h SRCS+= bus_if.h SRCS+= pci_if.h SRCS+= pci_iov_if.h .include #CFLAGS += -DQLNX_DEBUG CFLAGS += -DECORE_PACKAGE CFLAGS += -DCONFIG_ECORE_L2 CFLAGS += -DECORE_CONFIG_DIRECT_HWFN CFLAGS+= -I${SRCTOP}/sys/compat/linuxkpi/common/include #CFLAGS += -g #CFLAGS += -fno-inline #CFLAGS += -DQLNX_SOFT_LRO #CFLAGS += -DQLNX_QSORT_LRO #CFLAGS += -DQLNX_MAX_COALESCE #CFLAGS += -DQLNX_USER_LLDP CFLAGS += -DCONFIG_ECORE_SRIOV + +# For roce/iwarp files. Compilation can be turned off if roce/iwarp are not required. +# In other words if you don't need RDMA please comment out the CFLAGS which define +# CONFIG_ECORE_LL2 CONFIG_ECORE_ROCE CONFIG_ECORE_IWARP QLNX_ENABLE_IWARP +CFLAGS += -DCONFIG_ECORE_LL2 +CFLAGS += -DCONFIG_ECORE_ROCE +CFLAGS += -DCONFIG_ECORE_IWARP +CFLAGS += -DCONFIG_ECORE_RDMA +CFLAGS += -DQLNX_ENABLE_IWARP CWARNFLAGS+= -Wno-cast-qual Index: head/sys/modules/qlnx/qlnxr/Makefile =================================================================== --- head/sys/modules/qlnx/qlnxr/Makefile (nonexistent) +++ head/sys/modules/qlnx/qlnxr/Makefile (revision 343598) @@ -0,0 +1,85 @@ +#/* +# * Copyright (c) 2017-2018 Cavium, Inc. +# * All rights reserved. +# * +# * Redistribution and use in source and binary forms, with or without +# * modification, are permitted provided that the following conditions +# * are met: +# * +# * 1. Redistributions of source code must retain the above copyright +# * notice, this list of conditions and the following disclaimer. +# * 2. Redistributions in binary form must reproduce the above copyright +# * notice, this list of conditions and the following disclaimer in the +# * documentation and/or other materials provided with the distribution. +# * +# * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +# * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# * POSSIBILITY OF SUCH DAMAGE. +# */ +#/* +# * File : Makefile +# * Author : David C Somayajulu, Cavium, Inc., San Jose, CA 95131. +# */ +# +# $FreeBSD$ +# + +#.PATH: ${.CURDIR} +#OFEDDIR= /usr/src/sys +#ETHDRVR=${.CURDIR}/../qlnxe + +.PATH: ${SRCTOP}/sys/dev/qlnx/qlnxr +OFEDDIR=${SRCTOP}/sys +ETHDRVR=${SRCTOP}/sys/dev/qlnx/qlnxe + +KMOD= qlnxr +SRCS= device_if.h bus_if.h vnode_if.h pci_if.h \ + opt_inet.h opt_inet6.h \ + qlnxr_os.c\ + qlnxr_cm.c\ + qlnxr_verbs.c + +.include + +CFLAGS+= -I${.CURDIR} +CFLAGS+= -I${ETHDRVR} +CFLAGS+= -I${OFEDDIR}/ofed/include +CFLAGS+= -I${OFEDDIR}/ofed/include/uapi +CFLAGS+= -I${OFEDDIR}/compat/linuxkpi/common/include + +CFLAGS+= -DLINUX_TYPES_DEFINED +CFLAGS+= -DCONFIG_INFINIBAND_USER_MEM +CFLAGS+= -DINET6 -DINET +#CFLAGS+= -DDEFINE_NO_IP_BASED_GIDS + +CWARNEXTRA += -Wno-cast-qual +CWARNEXTRA += -Wno-unused-function +CWARNEXTRA += -Wno-gnu-variable-sized-type-not-at-end +CWARNEXTRA += -Wno-missing-prototypes +CWARNEXTRA += -Wno-constant-conversion +CWARNEXTRA += -Wno-format + +CWARNEXTRA += -Wno-shift-sign-overflow +CWARNEXTRA += -Wno-empty-body + +CFLAGS += -DQLNX_DEBUG +CFLAGS += -DECORE_PACKAGE +CFLAGS += -DCONFIG_ECORE_L2 +CFLAGS += -DCONFIG_ECORE_LL2 +CFLAGS += -DCONFIG_ECORE_ROCE +CFLAGS += -DCONFIG_ECORE_IWARP +CFLAGS += -DCONFIG_ECORE_RDMA +CFLAGS += -DECORE_CONFIG_DIRECT_HWFN +CFLAGS += -g -fno-inline +CFLAGS += -DQLNX_RDMA + +CFLAGS+= -Wno-cast-qual -Wno-pointer-arith + Property changes on: head/sys/modules/qlnx/qlnxr/Makefile ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Added: svn:mime-type ## -0,0 +1 ## +text/plain \ No newline at end of property