diff --git a/sys/dev/qlnx/qlnxe/qlnx_def.h b/sys/dev/qlnx/qlnxe/qlnx_def.h index 87ef7985647d..cc8fb3f5f7eb 100644 --- a/sys/dev/qlnx/qlnxe/qlnx_def.h +++ b/sys/dev/qlnx/qlnxe/qlnx_def.h @@ -1,758 +1,758 @@ /* * Copyright (c) 2017-2018 Cavium, Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. * * $FreeBSD$ * */ /* * File: qlnx_def.h * Author : David C Somayajulu, Cavium Inc., San Jose, CA 95131. */ #ifndef _QLNX_DEF_H_ #define _QLNX_DEF_H_ #define VER_SIZE 16 struct qlnx_ivec { uint32_t rss_idx; void *ha; struct resource *irq; void *handle; int irq_rid; }; typedef struct qlnx_ivec qlnx_ivec_t; //#define QLNX_MAX_RSS 30 #define QLNX_MAX_VF_RSS 4 #define QLNX_MAX_RSS 36 #define QLNX_DEFAULT_RSS 16 #define QLNX_MAX_TC 1 enum QLNX_STATE { QLNX_STATE_CLOSED, QLNX_STATE_OPEN, }; #define HILO_U64(hi, lo) ((((u64)(hi)) << 32) + (lo)) #define MAX_NUM_TC 8 #define MAX_NUM_PRI 8 #ifndef BITS_PER_BYTE #define BITS_PER_BYTE 8 #endif /* #ifndef BITS_PER_BYTE */ /* * RX ring buffer contains pointer to kmalloc() data only, */ struct sw_rx_data { void *data; bus_dmamap_t map; dma_addr_t dma_addr; }; enum qlnx_agg_state { QLNX_AGG_STATE_NONE = 0, QLNX_AGG_STATE_START = 1, QLNX_AGG_STATE_ERROR = 2 }; struct qlnx_agg_info { /* rx_buf is a data buffer that can be placed /consumed from rx bd * chain. It has two purposes: We will preallocate the data buffer * for each aggregation when we open the interface and will place this * buffer on the rx-bd-ring when we receive TPA_START. We don't want * to be in a state where allocation fails, as we can't reuse the * consumer buffer in the rx-chain since FW may still be writing to it * (since header needs to be modified for TPA. * The second purpose is to keep a pointer to the bd buffer during * aggregation. */ struct sw_rx_data rx_buf; enum qlnx_agg_state agg_state; uint16_t placement_offset; struct mbuf *mpf; /* first mbuf in chain */ struct mbuf *mpl; /* last mbuf in chain */ }; #define RX_RING_SIZE_POW 13 #define RX_RING_SIZE (1 << RX_RING_SIZE_POW) #define TX_RING_SIZE_POW 14 #define TX_RING_SIZE (1 << TX_RING_SIZE_POW) struct qlnx_rx_queue { volatile __le16 *hw_cons_ptr; struct sw_rx_data sw_rx_ring[RX_RING_SIZE]; uint16_t sw_rx_cons; uint16_t sw_rx_prod; struct ecore_chain rx_bd_ring; struct ecore_chain rx_comp_ring; void __iomem *hw_rxq_prod_addr; void *handle; /* LRO */ struct qlnx_agg_info tpa_info[ETH_TPA_MAX_AGGS_NUM]; uint32_t rx_buf_size; uint16_t num_rx_buffers; uint16_t rxq_id; #ifdef QLNX_SOFT_LRO struct lro_ctrl lro; #endif }; union db_prod { struct eth_db_data data; uint32_t raw; }; struct sw_tx_bd { struct mbuf *mp; bus_dmamap_t map; uint8_t flags; int nsegs; /* Set on the first BD descriptor when there is a split BD */ #define QLNX_TSO_SPLIT_BD (1<<0) }; #define QLNX_MAX_SEGMENTS 255 struct qlnx_tx_queue { int index; /* Queue index */ volatile __le16 *hw_cons_ptr; struct sw_tx_bd sw_tx_ring[TX_RING_SIZE]; uint16_t sw_tx_cons; uint16_t sw_tx_prod; struct ecore_chain tx_pbl; void __iomem *doorbell_addr; void *handle; union db_prod tx_db; bus_dma_segment_t segs[QLNX_MAX_SEGMENTS]; uint16_t num_tx_buffers; }; #define BD_UNMAP_ADDR(bd) HILO_U64(le32toh((bd)->addr.hi), \ le32toh((bd)->addr.lo)) #define BD_UNMAP_LEN(bd) (le16toh((bd)->nbytes)) #define BD_SET_UNMAP_ADDR_LEN(bd, maddr, len) \ do { \ (bd)->addr.hi = htole32(U64_HI(maddr)); \ (bd)->addr.lo = htole32(U64_LO(maddr)); \ (bd)->nbytes = htole16(len); \ } while (0); #define QLNX_FP_MAX_SEGS 24 struct qlnx_fastpath { void *edev; uint8_t rss_id; struct ecore_sb_info *sb_info; struct qlnx_rx_queue *rxq; struct qlnx_tx_queue *txq[MAX_NUM_TC]; char name[64]; struct mtx tx_mtx; char tx_mtx_name[32]; struct buf_ring *tx_br; uint32_t tx_ring_full; struct task fp_task; struct taskqueue *fp_taskqueue; /* transmit statistics */ uint64_t tx_pkts_processed; uint64_t tx_pkts_freed; uint64_t tx_pkts_transmitted; uint64_t tx_pkts_completed; uint64_t tx_tso_pkts; uint64_t tx_non_tso_pkts; #ifdef QLNX_TRACE_PERF_DATA uint64_t tx_pkts_trans_ctx; uint64_t tx_pkts_compl_ctx; uint64_t tx_pkts_trans_fp; uint64_t tx_pkts_compl_fp; uint64_t tx_pkts_compl_intr; #endif uint64_t tx_lso_wnd_min_len; uint64_t tx_defrag; uint64_t tx_nsegs_gt_elem_left; uint32_t tx_tso_max_nsegs; uint32_t tx_tso_min_nsegs; uint32_t tx_tso_max_pkt_len; uint32_t tx_tso_min_pkt_len; uint64_t tx_pkts[QLNX_FP_MAX_SEGS]; #ifdef QLNX_TRACE_PERF_DATA uint64_t tx_pkts_hist[QLNX_FP_MAX_SEGS]; uint64_t tx_comInt[QLNX_FP_MAX_SEGS]; uint64_t tx_pkts_q[QLNX_FP_MAX_SEGS]; #endif uint64_t err_tx_nsegs_gt_elem_left; uint64_t err_tx_dmamap_create; uint64_t err_tx_defrag_dmamap_load; uint64_t err_tx_non_tso_max_seg; uint64_t err_tx_dmamap_load; uint64_t err_tx_defrag; uint64_t err_tx_free_pkt_null; uint64_t err_tx_cons_idx_conflict; uint64_t lro_cnt_64; uint64_t lro_cnt_128; uint64_t lro_cnt_256; uint64_t lro_cnt_512; uint64_t lro_cnt_1024; /* receive statistics */ uint64_t rx_pkts; uint64_t tpa_start; uint64_t tpa_cont; uint64_t tpa_end; uint64_t err_m_getcl; uint64_t err_m_getjcl; uint64_t err_rx_hw_errors; uint64_t err_rx_alloc_errors; uint64_t err_rx_jumbo_chain_pkts; uint64_t err_rx_mp_null; uint64_t err_rx_tpa_invalid_agg_num; }; struct qlnx_update_vport_params { uint8_t vport_id; uint8_t update_vport_active_rx_flg; uint8_t vport_active_rx_flg; uint8_t update_vport_active_tx_flg; uint8_t vport_active_tx_flg; uint8_t update_inner_vlan_removal_flg; uint8_t inner_vlan_removal_flg; struct ecore_rss_params *rss_params; struct ecore_sge_tpa_params *sge_tpa_params; }; /* * link related */ struct qlnx_link_output { bool link_up; uint32_t supported_caps; uint32_t advertised_caps; uint32_t link_partner_caps; uint32_t speed; /* In Mb/s */ bool autoneg; uint32_t media_type; uint32_t duplex; }; typedef struct qlnx_link_output qlnx_link_output_t; #define QLNX_LINK_DUPLEX 0x0001 #define QLNX_LINK_CAP_FIBRE 0x0001 #define QLNX_LINK_CAP_Autoneg 0x0002 #define QLNX_LINK_CAP_Pause 0x0004 #define QLNX_LINK_CAP_Asym_Pause 0x0008 #define QLNX_LINK_CAP_1000baseT_Half 0x0010 #define QLNX_LINK_CAP_1000baseT_Full 0x0020 #define QLNX_LINK_CAP_10000baseKR_Full 0x0040 #define QLNX_LINK_CAP_25000baseKR_Full 0x0080 #define QLNX_LINK_CAP_40000baseLR4_Full 0x0100 #define QLNX_LINK_CAP_50000baseKR2_Full 0x0200 #define QLNX_LINK_CAP_100000baseKR4_Full 0x0400 /* Functions definition */ #define XMIT_PLAIN 0 #define XMIT_L4_CSUM (1 << 0) #define XMIT_LSO (1 << 1) #define CQE_FLAGS_ERR (PARSING_AND_ERR_FLAGS_IPHDRERROR_MASK << \ PARSING_AND_ERR_FLAGS_IPHDRERROR_SHIFT | \ PARSING_AND_ERR_FLAGS_L4CHKSMERROR_MASK << \ PARSING_AND_ERR_FLAGS_L4CHKSMERROR_SHIFT | \ PARSING_AND_ERR_FLAGS_TUNNELIPHDRERROR_MASK << \ PARSING_AND_ERR_FLAGS_TUNNELIPHDRERROR_SHIFT | \ PARSING_AND_ERR_FLAGS_TUNNELL4CHKSMERROR_MASK << \ PARSING_AND_ERR_FLAGS_TUNNELL4CHKSMERROR_SHIFT) #define RX_COPY_THRESH 92 #define ETH_MAX_PACKET_SIZE 1500 #define QLNX_MFW_VERSION_LENGTH 32 #define QLNX_STORMFW_VERSION_LENGTH 32 #define QLNX_TX_ELEM_RESERVE 2 #define QLNX_TX_ELEM_THRESH 128 #define QLNX_TX_ELEM_MAX_THRESH 512 #define QLNX_TX_ELEM_MIN_THRESH 32 #define QLNX_TX_COMPL_THRESH 32 #define QLNX_TPA_MAX_AGG_BUFFERS (20) #define QLNX_MAX_NUM_MULTICAST_ADDRS ECORE_MAX_MC_ADDRS typedef struct _qlnx_mcast { uint16_t rsrvd; uint8_t addr[6]; } __packed qlnx_mcast_t; typedef struct _qlnx_vf_attr { uint8_t mac_addr[ETHER_ADDR_LEN]; uint32_t num_rings; } qlnx_vf_attr_t; typedef struct _qlnx_sriov_task { struct task pf_task; struct taskqueue *pf_taskqueue; #define QLNX_SRIOV_TASK_FLAGS_VF_PF_MSG 0x01 #define QLNX_SRIOV_TASK_FLAGS_VF_FLR_UPDATE 0x02 #define QLNX_SRIOV_TASK_FLAGS_BULLETIN_UPDATE 0x04 volatile uint32_t flags; } qlnx_sriov_task_t; /* * Adapter structure contains the hardware independent information of the * pci function. */ struct qlnx_host { /* interface to ecore */ struct ecore_dev cdev; uint32_t state; /* some flags */ volatile struct { volatile uint32_t hw_init :1, callout_init :1, slowpath_start :1, parent_tag :1, lock_init :1; } flags; /* interface to o.s */ device_t pci_dev; uint8_t pci_func; uint8_t dev_unit; uint16_t device_id; - struct ifnet *ifp; + if_t ifp; int if_flags; volatile int link_up; struct ifmedia media; uint16_t max_frame_size; struct cdev *ioctl_dev; /* resources */ struct resource *pci_reg; int reg_rid; struct resource *pci_dbells; int dbells_rid; uint64_t dbells_phys_addr; uint32_t dbells_size; struct resource *msix_bar; int msix_rid; int msix_count; struct mtx hw_lock; /* debug */ uint32_t dbg_level; uint32_t dbg_trace_lro_cnt; uint32_t dbg_trace_tso_pkt_len; uint32_t dp_level; uint32_t dp_module; /* misc */ uint8_t mfw_ver[QLNX_MFW_VERSION_LENGTH]; uint8_t stormfw_ver[QLNX_STORMFW_VERSION_LENGTH]; uint32_t flash_size; /* dma related */ bus_dma_tag_t parent_tag; bus_dma_tag_t tx_tag; bus_dma_tag_t rx_tag; struct ecore_sb_info sb_array[QLNX_MAX_RSS]; struct qlnx_rx_queue rxq_array[QLNX_MAX_RSS]; struct qlnx_tx_queue txq_array[(QLNX_MAX_RSS * MAX_NUM_TC)]; struct qlnx_fastpath fp_array[QLNX_MAX_RSS]; /* tx related */ struct callout tx_callout; uint32_t txr_idx; /* rx related */ uint32_t rx_pkt_threshold; uint32_t rx_jumbo_buf_eq_mtu; /* slow path related */ struct resource *sp_irq[MAX_HWFNS_PER_DEVICE]; void *sp_handle[MAX_HWFNS_PER_DEVICE]; int sp_irq_rid[MAX_HWFNS_PER_DEVICE]; struct task sp_task[MAX_HWFNS_PER_DEVICE]; struct taskqueue *sp_taskqueue[MAX_HWFNS_PER_DEVICE]; struct callout qlnx_callout; /* fast path related */ int num_rss; int num_tc; #define QLNX_MAX_TSS_CNT(ha) ((ha->num_rss) * (ha->num_tc)) qlnx_ivec_t irq_vec[QLNX_MAX_RSS]; uint8_t filter; uint32_t nmcast; qlnx_mcast_t mcast[QLNX_MAX_NUM_MULTICAST_ADDRS]; struct ecore_filter_mcast ecore_mcast; uint8_t primary_mac[ETH_ALEN]; uint8_t prio_to_tc[MAX_NUM_PRI]; struct ecore_eth_stats hw_stats; struct ecore_rss_params rss_params; uint32_t rx_buf_size; bool rx_csum_offload; uint32_t rx_coalesce_usecs; uint32_t tx_coalesce_usecs; /* link related */ qlnx_link_output_t if_link; /* global counters */ uint64_t sp_interrupts; uint64_t err_illegal_intr; uint64_t err_fp_null; uint64_t err_get_proto_invalid_type; /* error recovery related */ uint32_t error_recovery; struct task err_task; struct taskqueue *err_taskqueue; /* grcdump related */ uint32_t err_inject; uint32_t grcdump_taken; uint32_t grcdump_dwords[QLNX_MAX_HW_FUNCS]; uint32_t grcdump_size[QLNX_MAX_HW_FUNCS]; void *grcdump[QLNX_MAX_HW_FUNCS]; uint32_t idle_chk_taken; uint32_t idle_chk_dwords[QLNX_MAX_HW_FUNCS]; uint32_t idle_chk_size[QLNX_MAX_HW_FUNCS]; void *idle_chk[QLNX_MAX_HW_FUNCS]; /* storm stats related */ #define QLNX_STORM_STATS_TOTAL \ (QLNX_MAX_HW_FUNCS * QLNX_STORM_STATS_SAMPLES_PER_HWFN) qlnx_storm_stats_t storm_stats[QLNX_STORM_STATS_TOTAL]; uint32_t storm_stats_index; uint32_t storm_stats_enable; uint32_t storm_stats_gather; uint32_t personality; uint16_t sriov_initialized; uint16_t num_vfs; qlnx_vf_attr_t *vf_attr; qlnx_sriov_task_t sriov_task[MAX_HWFNS_PER_DEVICE]; uint32_t curr_vf; void *next; void *qlnx_rdma; volatile int qlnxr_debug; }; typedef struct qlnx_host qlnx_host_t; /* note that align has to be a power of 2 */ #define QL_ALIGN(size, align) (((size) + ((align) - 1)) & (~((align) - 1))); #define QL_MIN(x, y) ((x < y) ? x : y) #define QL_RUNNING(ifp) \ - ((ifp->if_drv_flags & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) == \ + ((if_getdrvflags(ifp) & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) == \ IFF_DRV_RUNNING) #define QLNX_MAX_MTU 9000 #define QLNX_MAX_SEGMENTS_NON_TSO (ETH_TX_MAX_BDS_PER_NON_LSO_PACKET - 1) //#define QLNX_MAX_TSO_FRAME_SIZE ((64 * 1024 - 1) + 22) #define QLNX_MAX_TSO_FRAME_SIZE 65536 #define QLNX_MAX_TX_MBUF_SIZE 65536 /* bytes - bd_len = 16bits */ #define QL_MAC_CMP(mac1, mac2) \ ((((*(uint32_t *) mac1) == (*(uint32_t *) mac2) && \ (*(uint16_t *)(mac1 + 4)) == (*(uint16_t *)(mac2 + 4)))) ? 0 : 1) #define for_each_rss(i) for (i = 0; i < ha->num_rss; i++) /* * Debug Related */ #ifdef QLNX_DEBUG #define QL_DPRINT1(ha, x, ...) \ do { \ if ((ha)->dbg_level & 0x0001) { \ device_printf ((ha)->pci_dev, \ "[%s:%d]" x, \ __func__, __LINE__, \ ## __VA_ARGS__); \ } \ } while (0) #define QL_DPRINT2(ha, x, ...) \ do { \ if ((ha)->dbg_level & 0x0002) { \ device_printf ((ha)->pci_dev, \ "[%s:%d]" x, \ __func__, __LINE__, \ ## __VA_ARGS__); \ } \ } while (0) #define QL_DPRINT3(ha, x, ...) \ do { \ if ((ha)->dbg_level & 0x0004) { \ device_printf ((ha)->pci_dev, \ "[%s:%d]" x, \ __func__, __LINE__, \ ## __VA_ARGS__); \ } \ } while (0) #define QL_DPRINT4(ha, x, ...) \ do { \ if ((ha)->dbg_level & 0x0008) { \ device_printf ((ha)->pci_dev, \ "[%s:%d]" x, \ __func__, __LINE__, \ ## __VA_ARGS__); \ } \ } while (0) #define QL_DPRINT5(ha, x, ...) \ do { \ if ((ha)->dbg_level & 0x0010) { \ device_printf ((ha)->pci_dev, \ "[%s:%d]" x, \ __func__, __LINE__, \ ## __VA_ARGS__); \ } \ } while (0) #define QL_DPRINT6(ha, x, ...) \ do { \ if ((ha)->dbg_level & 0x0020) { \ device_printf ((ha)->pci_dev, \ "[%s:%d]" x, \ __func__, __LINE__, \ ## __VA_ARGS__); \ } \ } while (0) #define QL_DPRINT7(ha, x, ...) \ do { \ if ((ha)->dbg_level & 0x0040) { \ device_printf ((ha)->pci_dev, \ "[%s:%d]" x, \ __func__, __LINE__, \ ## __VA_ARGS__); \ } \ } while (0) #define QL_DPRINT8(ha, x, ...) \ do { \ if ((ha)->dbg_level & 0x0080) { \ device_printf ((ha)->pci_dev, \ "[%s:%d]" x, \ __func__, __LINE__, \ ## __VA_ARGS__); \ } \ } while (0) #define QL_DPRINT9(ha, x, ...) \ do { \ if ((ha)->dbg_level & 0x0100) { \ device_printf ((ha)->pci_dev, \ "[%s:%d]" x, \ __func__, __LINE__, \ ## __VA_ARGS__); \ } \ } while (0) #define QL_DPRINT11(ha, x, ...) \ do { \ if ((ha)->dbg_level & 0x0400) { \ device_printf ((ha)->pci_dev, \ "[%s:%d]" x, \ __func__, __LINE__, \ ## __VA_ARGS__); \ } \ } while (0) #define QL_DPRINT12(ha, x, ...) \ do { \ if ((ha)->dbg_level & 0x0800) { \ device_printf ((ha)->pci_dev, \ "[%s:%d]" x, \ __func__, __LINE__, \ ## __VA_ARGS__); \ } \ } while (0) #define QL_DPRINT13(ha, x, ...) \ do { \ if ((ha)->dbg_level & 0x1000) { \ device_printf ((ha)->pci_dev, \ "[%s:%d]" x, \ __func__, __LINE__, \ ## __VA_ARGS__); \ } \ } while (0) #else #define QL_DPRINT1(ha, x, ...) #define QL_DPRINT2(ha, x, ...) #define QL_DPRINT3(ha, x, ...) #define QL_DPRINT4(ha, x, ...) #define QL_DPRINT5(ha, x, ...) #define QL_DPRINT6(ha, x, ...) #define QL_DPRINT7(ha, x, ...) #define QL_DPRINT8(ha, x, ...) #define QL_DPRINT9(ha, x, ...) #define QL_DPRINT11(ha, x, ...) #define QL_DPRINT12(ha, x, ...) #define QL_DPRINT13(ha, x, ...) #endif /* #ifdef QLNX_DEBUG */ #define QL_ASSERT(ha, x, y) if (!x) panic y #define QL_ERR_INJECT(ha, val) (ha->err_inject == val) #define QL_RESET_ERR_INJECT(ha, val) {if (ha->err_inject == val) ha->err_inject = 0;} #define QL_ERR_INJCT_TX_INT_DIFF 0x0001 #define QL_ERR_INJCT_TX_INT_MBUF_NULL 0x0002 /* * exported functions */ extern int qlnx_make_cdev(qlnx_host_t *ha); extern void qlnx_del_cdev(qlnx_host_t *ha); extern int qlnx_grc_dump(qlnx_host_t *ha, uint32_t *num_dumped_dwords, int hwfn_index); extern int qlnx_idle_chk(qlnx_host_t *ha, uint32_t *num_dumped_dwords, int hwfn_index); extern uint8_t *qlnx_get_mac_addr(qlnx_host_t *ha); extern void qlnx_fill_link(qlnx_host_t *ha, struct ecore_hwfn *hwfn, struct qlnx_link_output *if_link); extern int qlnx_set_lldp_tlvx(qlnx_host_t *ha, qlnx_lldp_sys_tlvs_t *lldp_tlvs); extern int qlnx_vf_device(qlnx_host_t *ha); extern void qlnx_free_mem_sb(qlnx_host_t *ha, struct ecore_sb_info *sb_info); extern int qlnx_alloc_mem_sb(qlnx_host_t *ha, struct ecore_sb_info *sb_info, u16 sb_id); /* * Some OS specific stuff */ #if (defined IFM_100G_SR4) #define QLNX_IFM_100G_SR4 IFM_100G_SR4 #define QLNX_IFM_100G_LR4 IFM_100G_LR4 #define QLNX_IFM_100G_CR4 IFM_100G_CR4 #else #define QLNX_IFM_100G_SR4 IFM_UNKNOWN #define QLNX_IFM_100G_LR4 IFM_UNKNOWN #endif /* #if (defined IFM_100G_SR4) */ #if (defined IFM_25G_SR) #define QLNX_IFM_25G_SR IFM_25G_SR #define QLNX_IFM_25G_CR IFM_25G_CR #else #define QLNX_IFM_25G_SR IFM_UNKNOWN #define QLNX_IFM_25G_CR IFM_UNKNOWN #endif /* #if (defined IFM_25G_SR) */ #define QLNX_INC_IERRORS(ifp) if_inc_counter(ifp, IFCOUNTER_IERRORS, 1) #define QLNX_INC_IQDROPS(ifp) if_inc_counter(ifp, IFCOUNTER_IQDROPS, 1) #define QLNX_INC_IPACKETS(ifp) if_inc_counter(ifp, IFCOUNTER_IPACKETS, 1) #define QLNX_INC_OPACKETS(ifp) if_inc_counter(ifp, IFCOUNTER_OPACKETS, 1) #define QLNX_INC_OBYTES(ifp, len) \ if_inc_counter(ifp, IFCOUNTER_OBYTES, len) #define QLNX_INC_IBYTES(ifp, len) \ if_inc_counter(ha->ifp, IFCOUNTER_IBYTES, len) #define CQE_L3_PACKET(flags) \ ((((flags) & PARSING_AND_ERR_FLAGS_L3TYPE_MASK) == e_l3_type_ipv4) || \ (((flags) & PARSING_AND_ERR_FLAGS_L3TYPE_MASK) == e_l3_type_ipv6)) #define CQE_IP_HDR_ERR(flags) \ ((flags) & (PARSING_AND_ERR_FLAGS_IPHDRERROR_MASK \ << PARSING_AND_ERR_FLAGS_IPHDRERROR_SHIFT)) #define CQE_L4_HAS_CSUM(flags) \ ((flags) & (PARSING_AND_ERR_FLAGS_L4CHKSMWASCALCULATED_MASK \ << PARSING_AND_ERR_FLAGS_L4CHKSMWASCALCULATED_SHIFT)) #define CQE_HAS_VLAN(flags) \ ((flags) & (PARSING_AND_ERR_FLAGS_TAG8021QEXIST_MASK \ << PARSING_AND_ERR_FLAGS_TAG8021QEXIST_SHIFT)) #ifndef QLNX_RDMA #if defined(__i386__) || defined(__amd64__) static __inline void prefetch(void *x) { __asm volatile("prefetcht0 %0" :: "m" (*(unsigned long *)x)); } #else #define prefetch(x) #endif #endif #endif /* #ifndef _QLNX_DEF_H_ */ diff --git a/sys/dev/qlnx/qlnxe/qlnx_ioctl.c b/sys/dev/qlnx/qlnxe/qlnx_ioctl.c index f87cb8b49921..aa382a1046e3 100644 --- a/sys/dev/qlnx/qlnxe/qlnx_ioctl.c +++ b/sys/dev/qlnx/qlnxe/qlnx_ioctl.c @@ -1,1079 +1,1079 @@ /* * Copyright (c) 2017-2018 Cavium, Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ /* * File: qlnx_ioctl.c * Author : David C Somayajulu, Qlogic Corporation, Aliso Viejo, CA 92656. */ #include __FBSDID("$FreeBSD$"); #include "qlnx_os.h" #include "bcm_osal.h" #include "reg_addr.h" #include "ecore_gtt_reg_addr.h" #include "ecore.h" #include "ecore_chain.h" #include "ecore_status.h" #include "ecore_hw.h" #include "ecore_rt_defs.h" #include "ecore_init_ops.h" #include "ecore_int.h" #include "ecore_cxt.h" #include "ecore_spq.h" #include "ecore_init_fw_funcs.h" #include "ecore_sp_commands.h" #include "ecore_dev_api.h" #include "ecore_l2_api.h" #include "ecore_mcp.h" #include "ecore_hw_defs.h" #include "mcp_public.h" #include "ecore_iro.h" #include "nvm_cfg.h" #include "ecore_dev_api.h" #include "ecore_dbg_fw_funcs.h" #include "ecore_dcbx_api.h" #include "qlnx_ioctl.h" #include "qlnx_def.h" #include "qlnx_ver.h" #include static int qlnx_eioctl(struct cdev *dev, u_long cmd, caddr_t data, int fflag, struct thread *td); static struct cdevsw qlnx_cdevsw = { .d_version = D_VERSION, .d_ioctl = qlnx_eioctl, .d_name = "qlnxioctl", }; int qlnx_make_cdev(qlnx_host_t *ha) { ha->ioctl_dev = make_dev(&qlnx_cdevsw, - ha->ifp->if_dunit, + if_getdunit(ha->ifp), UID_ROOT, GID_WHEEL, 0600, "%s", if_name(ha->ifp)); if (ha->ioctl_dev == NULL) return (-1); ha->ioctl_dev->si_drv1 = ha; return (0); } void qlnx_del_cdev(qlnx_host_t *ha) { if (ha->ioctl_dev != NULL) destroy_dev(ha->ioctl_dev); return; } int qlnx_grc_dump(qlnx_host_t *ha, uint32_t *num_dumped_dwords, int hwfn_index) { int rval = EINVAL; struct ecore_hwfn *p_hwfn; struct ecore_ptt *p_ptt; if (ha->grcdump_dwords[hwfn_index]) { /* the grcdump is already available */ *num_dumped_dwords = ha->grcdump_dwords[hwfn_index]; return (0); } ecore_dbg_set_app_ver(ecore_dbg_get_fw_func_ver()); p_hwfn = &ha->cdev.hwfns[hwfn_index]; p_ptt = ecore_ptt_acquire(p_hwfn); if (!p_ptt) { QL_DPRINT1(ha,"ecore_ptt_acquire failed\n"); return (rval); } if ((rval = ecore_dbg_grc_dump(p_hwfn, p_ptt, ha->grcdump[hwfn_index], (ha->grcdump_size[hwfn_index] >> 2), num_dumped_dwords)) == DBG_STATUS_OK) { rval = 0; ha->grcdump_taken = 1; } else QL_DPRINT1(ha,"ecore_dbg_grc_dump failed [%d, 0x%x]\n", hwfn_index, rval); ecore_ptt_release(p_hwfn, p_ptt); return (rval); } static void qlnx_get_grc_dump_size(qlnx_host_t *ha, qlnx_grcdump_t *grcdump) { int i; grcdump->pci_func = ha->pci_func; for (i = 0; i < ha->cdev.num_hwfns; i++) grcdump->grcdump_size[i] = ha->grcdump_size[i]; return; } static int qlnx_get_grc_dump(qlnx_host_t *ha, qlnx_grcdump_t *grcdump) { int i; int rval = 0; uint32_t dwords = 0; grcdump->pci_func = ha->pci_func; for (i = 0; i < ha->cdev.num_hwfns; i++) { if ((ha->grcdump[i] == NULL) || (grcdump->grcdump[i] == NULL) || (grcdump->grcdump_size[i] < ha->grcdump_size[i])) return (EINVAL); rval = qlnx_grc_dump(ha, &dwords, i); if (rval) break; grcdump->grcdump_dwords[i] = dwords; QL_DPRINT1(ha,"grcdump_dwords[%d] = 0x%x\n", i, dwords); rval = copyout(ha->grcdump[i], grcdump->grcdump[i], ha->grcdump_size[i]); if (rval) break; ha->grcdump_dwords[i] = 0; } ha->grcdump_taken = 0; return (rval); } int qlnx_idle_chk(qlnx_host_t *ha, uint32_t *num_dumped_dwords, int hwfn_index) { int rval = EINVAL; struct ecore_hwfn *p_hwfn; struct ecore_ptt *p_ptt; if (ha->idle_chk_dwords[hwfn_index]) { /* the idle check is already available */ *num_dumped_dwords = ha->idle_chk_dwords[hwfn_index]; return (0); } ecore_dbg_set_app_ver(ecore_dbg_get_fw_func_ver()); p_hwfn = &ha->cdev.hwfns[hwfn_index]; p_ptt = ecore_ptt_acquire(p_hwfn); if (!p_ptt) { QL_DPRINT1(ha,"ecore_ptt_acquire failed\n"); return (rval); } if ((rval = ecore_dbg_idle_chk_dump(p_hwfn, p_ptt, ha->idle_chk[hwfn_index], (ha->idle_chk_size[hwfn_index] >> 2), num_dumped_dwords)) == DBG_STATUS_OK) { rval = 0; ha->idle_chk_taken = 1; } else QL_DPRINT1(ha,"ecore_dbg_idle_chk_dump failed [%d, 0x%x]\n", hwfn_index, rval); ecore_ptt_release(p_hwfn, p_ptt); return (rval); } static void qlnx_get_idle_chk_size(qlnx_host_t *ha, qlnx_idle_chk_t *idle_chk) { int i; idle_chk->pci_func = ha->pci_func; for (i = 0; i < ha->cdev.num_hwfns; i++) idle_chk->idle_chk_size[i] = ha->idle_chk_size[i]; return; } static int qlnx_get_idle_chk(qlnx_host_t *ha, qlnx_idle_chk_t *idle_chk) { int i; int rval = 0; uint32_t dwords = 0; idle_chk->pci_func = ha->pci_func; for (i = 0; i < ha->cdev.num_hwfns; i++) { if ((ha->idle_chk[i] == NULL) || (idle_chk->idle_chk[i] == NULL) || (idle_chk->idle_chk_size[i] < ha->idle_chk_size[i])) return (EINVAL); rval = qlnx_idle_chk(ha, &dwords, i); if (rval) break; idle_chk->idle_chk_dwords[i] = dwords; QL_DPRINT1(ha,"idle_chk_dwords[%d] = 0x%x\n", i, dwords); rval = copyout(ha->idle_chk[i], idle_chk->idle_chk[i], ha->idle_chk_size[i]); if (rval) break; ha->idle_chk_dwords[i] = 0; } ha->idle_chk_taken = 0; return (rval); } static uint32_t qlnx_get_trace_cmd_size(qlnx_host_t *ha, int hwfn_index, uint16_t cmd) { int rval = -1; struct ecore_hwfn *p_hwfn; struct ecore_ptt *p_ptt; uint32_t num_dwords = 0; p_hwfn = &ha->cdev.hwfns[hwfn_index]; p_ptt = ecore_ptt_acquire(p_hwfn); if (!p_ptt) { QL_DPRINT1(ha, "ecore_ptt_acquire [%d, 0x%x]failed\n", hwfn_index, cmd); return (0); } switch (cmd) { case QLNX_MCP_TRACE: rval = ecore_dbg_mcp_trace_get_dump_buf_size(p_hwfn, p_ptt, &num_dwords); break; case QLNX_REG_FIFO: rval = ecore_dbg_reg_fifo_get_dump_buf_size(p_hwfn, p_ptt, &num_dwords); break; case QLNX_IGU_FIFO: rval = ecore_dbg_igu_fifo_get_dump_buf_size(p_hwfn, p_ptt, &num_dwords); break; case QLNX_PROTECTION_OVERRIDE: rval = ecore_dbg_protection_override_get_dump_buf_size(p_hwfn, p_ptt, &num_dwords); break; case QLNX_FW_ASSERTS: rval = ecore_dbg_fw_asserts_get_dump_buf_size(p_hwfn, p_ptt, &num_dwords); break; } if (rval != DBG_STATUS_OK) { QL_DPRINT1(ha,"cmd = 0x%x failed [0x%x]\n", cmd, rval); num_dwords = 0; } ecore_ptt_release(p_hwfn, p_ptt); return ((num_dwords * sizeof (uint32_t))); } static void qlnx_get_trace_size(qlnx_host_t *ha, qlnx_trace_t *trace) { int i; trace->pci_func = ha->pci_func; for (i = 0; i < ha->cdev.num_hwfns; i++) { trace->size[i] = qlnx_get_trace_cmd_size(ha, i, trace->cmd); } return; } static int qlnx_get_trace(qlnx_host_t *ha, int hwfn_index, qlnx_trace_t *trace) { int rval = -1; struct ecore_hwfn *p_hwfn; struct ecore_ptt *p_ptt; uint32_t num_dwords = 0; void *buffer; buffer = qlnx_zalloc(trace->size[hwfn_index]); if (buffer == NULL) { QL_DPRINT1(ha,"qlnx_zalloc [%d, 0x%x]failed\n", hwfn_index, trace->cmd); return (ENXIO); } ecore_dbg_set_app_ver(ecore_dbg_get_fw_func_ver()); p_hwfn = &ha->cdev.hwfns[hwfn_index]; p_ptt = ecore_ptt_acquire(p_hwfn); if (!p_ptt) { QL_DPRINT1(ha, "ecore_ptt_acquire [%d, 0x%x]failed\n", hwfn_index, trace->cmd); return (ENXIO); } switch (trace->cmd) { case QLNX_MCP_TRACE: rval = ecore_dbg_mcp_trace_dump(p_hwfn, p_ptt, buffer, (trace->size[hwfn_index] >> 2), &num_dwords); break; case QLNX_REG_FIFO: rval = ecore_dbg_reg_fifo_dump(p_hwfn, p_ptt, buffer, (trace->size[hwfn_index] >> 2), &num_dwords); break; case QLNX_IGU_FIFO: rval = ecore_dbg_igu_fifo_dump(p_hwfn, p_ptt, buffer, (trace->size[hwfn_index] >> 2), &num_dwords); break; case QLNX_PROTECTION_OVERRIDE: rval = ecore_dbg_protection_override_dump(p_hwfn, p_ptt, buffer, (trace->size[hwfn_index] >> 2), &num_dwords); break; case QLNX_FW_ASSERTS: rval = ecore_dbg_fw_asserts_dump(p_hwfn, p_ptt, buffer, (trace->size[hwfn_index] >> 2), &num_dwords); break; } if (rval != DBG_STATUS_OK) { QL_DPRINT1(ha,"cmd = 0x%x failed [0x%x]\n", trace->cmd, rval); num_dwords = 0; } ecore_ptt_release(p_hwfn, p_ptt); trace->dwords[hwfn_index] = num_dwords; if (num_dwords) { rval = copyout(buffer, trace->buffer[hwfn_index], (num_dwords << 2)); } return (rval); } static int qlnx_reg_rd_wr(qlnx_host_t *ha, qlnx_reg_rd_wr_t *reg_rd_wr) { int rval = 0; struct ecore_hwfn *p_hwfn; if (reg_rd_wr->hwfn_index >= QLNX_MAX_HW_FUNCS) { return (EINVAL); } p_hwfn = &ha->cdev.hwfns[reg_rd_wr->hwfn_index]; switch (reg_rd_wr->cmd) { case QLNX_REG_READ_CMD: if (reg_rd_wr->access_type == QLNX_REG_ACCESS_DIRECT) { reg_rd_wr->val = qlnx_reg_rd32(p_hwfn, reg_rd_wr->addr); } break; case QLNX_REG_WRITE_CMD: if (reg_rd_wr->access_type == QLNX_REG_ACCESS_DIRECT) { qlnx_reg_wr32(p_hwfn, reg_rd_wr->addr, reg_rd_wr->val); } break; default: rval = EINVAL; break; } return (rval); } static int qlnx_rd_wr_pci_config(qlnx_host_t *ha, qlnx_pcicfg_rd_wr_t *pci_cfg_rd_wr) { int rval = 0; switch (pci_cfg_rd_wr->cmd) { case QLNX_PCICFG_READ: pci_cfg_rd_wr->val = pci_read_config(ha->pci_dev, pci_cfg_rd_wr->reg, pci_cfg_rd_wr->width); break; case QLNX_PCICFG_WRITE: pci_write_config(ha->pci_dev, pci_cfg_rd_wr->reg, pci_cfg_rd_wr->val, pci_cfg_rd_wr->width); break; default: rval = EINVAL; break; } return (rval); } static void qlnx_mac_addr(qlnx_host_t *ha, qlnx_perm_mac_addr_t *mac_addr) { bzero(mac_addr->addr, sizeof(mac_addr->addr)); snprintf(mac_addr->addr, sizeof(mac_addr->addr), "%02x:%02x:%02x:%02x:%02x:%02x", ha->primary_mac[0], ha->primary_mac[1], ha->primary_mac[2], ha->primary_mac[3], ha->primary_mac[4], ha->primary_mac[5]); return; } static int qlnx_get_regs(qlnx_host_t *ha, qlnx_get_regs_t *regs) { int i; int rval = 0; uint32_t dwords = 0; uint8_t *outb; regs->reg_buf_len = 0; outb = regs->reg_buf; for (i = 0; i < ha->cdev.num_hwfns; i++) { rval = qlnx_grc_dump(ha, &dwords, i); if (rval) break; regs->reg_buf_len += (dwords << 2); rval = copyout(ha->grcdump[i], outb, ha->grcdump_size[i]); if (rval) break; ha->grcdump_dwords[i] = 0; outb += regs->reg_buf_len; } ha->grcdump_taken = 0; return (rval); } extern char qlnx_name_str[]; extern char qlnx_ver_str[]; static int qlnx_drv_info(qlnx_host_t *ha, qlnx_drvinfo_t *drv_info) { int i; bzero(drv_info, sizeof(qlnx_drvinfo_t)); snprintf(drv_info->drv_name, sizeof(drv_info->drv_name), "%s", qlnx_name_str); snprintf(drv_info->drv_version, sizeof(drv_info->drv_version), "%s", qlnx_ver_str); snprintf(drv_info->mfw_version, sizeof(drv_info->mfw_version), "%s", ha->mfw_ver); snprintf(drv_info->stormfw_version, sizeof(drv_info->stormfw_version), "%s", ha->stormfw_ver); drv_info->eeprom_dump_len = ha->flash_size; for (i = 0; i < ha->cdev.num_hwfns; i++) { drv_info->reg_dump_len += ha->grcdump_size[i]; } snprintf(drv_info->bus_info, sizeof(drv_info->bus_info), "%d:%d:%d", pci_get_bus(ha->pci_dev), pci_get_slot(ha->pci_dev), ha->pci_func); return (0); } static int qlnx_dev_settings(qlnx_host_t *ha, qlnx_dev_setting_t *dev_info) { struct ecore_hwfn *p_hwfn; struct qlnx_link_output if_link; p_hwfn = &ha->cdev.hwfns[0]; qlnx_fill_link(ha, p_hwfn, &if_link); dev_info->supported = if_link.supported_caps; dev_info->advertising = if_link.advertised_caps; dev_info->speed = if_link.speed; dev_info->duplex = if_link.duplex; dev_info->port = ha->pci_func & 0x1; dev_info->autoneg = if_link.autoneg; return (0); } static int qlnx_write_nvram(qlnx_host_t *ha, qlnx_nvram_t *nvram, uint32_t cmd) { uint8_t *buf; int ret = 0; if ((nvram->data == NULL) || (nvram->data_len == 0)) return (EINVAL); buf = qlnx_zalloc(nvram->data_len); ret = copyin(nvram->data, buf, nvram->data_len); QL_DPRINT9(ha, "issue cmd = 0x%x data = %p \ data_len = 0x%x ret = 0x%x exit\n", cmd, nvram->data, nvram->data_len, ret); if (ret == 0) { ret = ecore_mcp_nvm_write(&ha->cdev, cmd, nvram->offset, buf, nvram->data_len); } QL_DPRINT9(ha, "cmd = 0x%x data = %p \ data_len = 0x%x resp = 0x%x ret = 0x%x exit\n", cmd, nvram->data, nvram->data_len, ha->cdev.mcp_nvm_resp, ret); free(buf, M_QLNXBUF); return (ret); } static int qlnx_read_nvram(qlnx_host_t *ha, qlnx_nvram_t *nvram) { uint8_t *buf; int ret = 0; if ((nvram->data == NULL) || (nvram->data_len == 0)) return (EINVAL); buf = qlnx_zalloc(nvram->data_len); ret = ecore_mcp_nvm_read(&ha->cdev, nvram->offset, buf, nvram->data_len); QL_DPRINT9(ha, " data = %p data_len = 0x%x \ resp = 0x%x ret = 0x%x exit\n", nvram->data, nvram->data_len, ha->cdev.mcp_nvm_resp, ret); if (ret == 0) { ret = copyout(buf, nvram->data, nvram->data_len); } free(buf, M_QLNXBUF); return (ret); } static int qlnx_get_nvram_resp(qlnx_host_t *ha, qlnx_nvram_t *nvram) { uint8_t *buf; int ret = 0; if ((nvram->data == NULL) || (nvram->data_len == 0)) return (EINVAL); buf = qlnx_zalloc(nvram->data_len); ret = ecore_mcp_nvm_resp(&ha->cdev, buf); QL_DPRINT9(ha, "data = %p data_len = 0x%x \ resp = 0x%x ret = 0x%x exit\n", nvram->data, nvram->data_len, ha->cdev.mcp_nvm_resp, ret); if (ret == 0) { ret = copyout(buf, nvram->data, nvram->data_len); } free(buf, M_QLNXBUF); return (ret); } static int qlnx_nvram(qlnx_host_t *ha, qlnx_nvram_t *nvram) { int ret = 0; switch (nvram->cmd) { case QLNX_NVRAM_CMD_WRITE_NVRAM: ret = qlnx_write_nvram(ha, nvram, ECORE_NVM_WRITE_NVRAM); break; case QLNX_NVRAM_CMD_PUT_FILE_DATA: ret = qlnx_write_nvram(ha, nvram, ECORE_PUT_FILE_DATA); break; case QLNX_NVRAM_CMD_READ_NVRAM: ret = qlnx_read_nvram(ha, nvram); break; case QLNX_NVRAM_CMD_SET_SECURE_MODE: ret = ecore_mcp_nvm_set_secure_mode(&ha->cdev, nvram->offset); QL_DPRINT9(ha, "QLNX_NVRAM_CMD_SET_SECURE_MODE \ resp = 0x%x ret = 0x%x exit\n", ha->cdev.mcp_nvm_resp, ret); break; case QLNX_NVRAM_CMD_DEL_FILE: ret = ecore_mcp_nvm_del_file(&ha->cdev, nvram->offset); QL_DPRINT9(ha, "QLNX_NVRAM_CMD_DEL_FILE \ resp = 0x%x ret = 0x%x exit\n", ha->cdev.mcp_nvm_resp, ret); break; case QLNX_NVRAM_CMD_PUT_FILE_BEGIN: ret = ecore_mcp_nvm_put_file_begin(&ha->cdev, nvram->offset); QL_DPRINT9(ha, "QLNX_NVRAM_CMD_PUT_FILE_BEGIN \ resp = 0x%x ret = 0x%x exit\n", ha->cdev.mcp_nvm_resp, ret); break; case QLNX_NVRAM_CMD_GET_NVRAM_RESP: ret = qlnx_get_nvram_resp(ha, nvram); break; default: ret = EINVAL; break; } return (ret); } static void qlnx_storm_stats(qlnx_host_t *ha, qlnx_storm_stats_dump_t *s_stats) { int i; int index; int ret; int stats_copied = 0; s_stats->num_hwfns = ha->cdev.num_hwfns; // if (ha->storm_stats_index < QLNX_STORM_STATS_SAMPLES_PER_HWFN) // return; s_stats->num_samples = ha->storm_stats_index; for (i = 0; i < ha->cdev.num_hwfns; i++) { index = (QLNX_STORM_STATS_SAMPLES_PER_HWFN * i); if (s_stats->buffer[i]) { ret = copyout(&ha->storm_stats[index], s_stats->buffer[i], QLNX_STORM_STATS_BYTES_PER_HWFN); if (ret) { printf("%s [%d]: failed\n", __func__, i); } if (s_stats->num_samples == QLNX_STORM_STATS_SAMPLES_PER_HWFN) { bzero((void *)&ha->storm_stats[i], QLNX_STORM_STATS_BYTES_PER_HWFN); stats_copied = 1; } } } if (stats_copied) ha->storm_stats_index = 0; return; } #ifdef QLNX_USER_LLDP static int qlnx_lldp_configure(qlnx_host_t *ha, struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt, uint32_t enable) { int ret = 0; uint8_t lldp_mac[6] = {0}; struct ecore_lldp_config_params lldp_params; struct ecore_lldp_sys_tlvs tlv_params; ret = ecore_mcp_get_lldp_mac(p_hwfn, p_ptt, lldp_mac); if (ret != ECORE_SUCCESS) { device_printf(ha->pci_dev, "%s: ecore_mcp_get_lldp_mac failed\n", __func__); return (-1); } bzero(&lldp_params, sizeof(struct ecore_lldp_config_params)); bzero(&tlv_params, sizeof(struct ecore_lldp_sys_tlvs)); lldp_params.agent = ECORE_LLDP_NEAREST_BRIDGE; lldp_params.tx_interval = 30; //Default value used as suggested by MFW lldp_params.tx_hold = 4; //Default value used as suggested by MFW lldp_params.tx_credit = 5; //Default value used as suggested by MFW lldp_params.rx_enable = enable ? 1 : 0; lldp_params.tx_enable = enable ? 1 : 0; lldp_params.chassis_id_tlv[0] = 0; lldp_params.chassis_id_tlv[0] |= (QLNX_LLDP_TYPE_CHASSIS_ID << 1); lldp_params.chassis_id_tlv[0] |= ((QLNX_LLDP_CHASSIS_ID_SUBTYPE_OCTETS + QLNX_LLDP_CHASSIS_ID_MAC_ADDR_LEN) << 8); lldp_params.chassis_id_tlv[0] |= (QLNX_LLDP_CHASSIS_ID_SUBTYPE_MAC << 16); lldp_params.chassis_id_tlv[0] |= lldp_mac[0] << 24; lldp_params.chassis_id_tlv[1] = lldp_mac[1] | (lldp_mac[2] << 8) | (lldp_mac[3] << 16) | (lldp_mac[4] << 24); lldp_params.chassis_id_tlv[2] = lldp_mac[5]; lldp_params.port_id_tlv[0] = 0; lldp_params.port_id_tlv[0] |= (QLNX_LLDP_TYPE_PORT_ID << 1); lldp_params.port_id_tlv[0] |= ((QLNX_LLDP_PORT_ID_SUBTYPE_OCTETS + QLNX_LLDP_PORT_ID_MAC_ADDR_LEN) << 8); lldp_params.port_id_tlv[0] |= (QLNX_LLDP_PORT_ID_SUBTYPE_MAC << 16); lldp_params.port_id_tlv[0] |= lldp_mac[0] << 24; lldp_params.port_id_tlv[1] = lldp_mac[1] | (lldp_mac[2] << 8) | (lldp_mac[3] << 16) | (lldp_mac[4] << 24); lldp_params.port_id_tlv[2] = lldp_mac[5]; ret = ecore_lldp_set_params(p_hwfn, p_ptt, &lldp_params); if (ret != ECORE_SUCCESS) { device_printf(ha->pci_dev, "%s: ecore_lldp_set_params failed\n", __func__); return (-1); } //If LLDP is disable then disable discard_mandatory_tlv flag if (!enable) { tlv_params.discard_mandatory_tlv = false; tlv_params.buf_size = 0; ret = ecore_lldp_set_system_tlvs(p_hwfn, p_ptt, &tlv_params); } if (ret != ECORE_SUCCESS) { device_printf(ha->pci_dev, "%s: ecore_lldp_set_system_tlvs failed\n", __func__); } return (ret); } static int qlnx_register_default_lldp_tlvs(qlnx_host_t *ha, struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt) { int ret = 0; ret = ecore_lldp_register_tlv(p_hwfn, p_ptt, ECORE_LLDP_NEAREST_BRIDGE, QLNX_LLDP_TYPE_CHASSIS_ID); if (ret != ECORE_SUCCESS) { device_printf(ha->pci_dev, "%s: QLNX_LLDP_TYPE_CHASSIS_ID failed\n", __func__); goto qlnx_register_default_lldp_tlvs_exit; } //register Port ID TLV ret = ecore_lldp_register_tlv(p_hwfn, p_ptt, ECORE_LLDP_NEAREST_BRIDGE, QLNX_LLDP_TYPE_PORT_ID); if (ret != ECORE_SUCCESS) { device_printf(ha->pci_dev, "%s: QLNX_LLDP_TYPE_PORT_ID failed\n", __func__); goto qlnx_register_default_lldp_tlvs_exit; } //register TTL TLV ret = ecore_lldp_register_tlv(p_hwfn, p_ptt, ECORE_LLDP_NEAREST_BRIDGE, QLNX_LLDP_TYPE_TTL); if (ret != ECORE_SUCCESS) { device_printf(ha->pci_dev, "%s: QLNX_LLDP_TYPE_TTL failed\n", __func__); goto qlnx_register_default_lldp_tlvs_exit; } //register Port Description TLV ret = ecore_lldp_register_tlv(p_hwfn, p_ptt, ECORE_LLDP_NEAREST_BRIDGE, QLNX_LLDP_TYPE_PORT_DESC); if (ret != ECORE_SUCCESS) { device_printf(ha->pci_dev, "%s: QLNX_LLDP_TYPE_PORT_DESC failed\n", __func__); goto qlnx_register_default_lldp_tlvs_exit; } //register System Name TLV ret = ecore_lldp_register_tlv(p_hwfn, p_ptt, ECORE_LLDP_NEAREST_BRIDGE, QLNX_LLDP_TYPE_SYS_NAME); if (ret != ECORE_SUCCESS) { device_printf(ha->pci_dev, "%s: QLNX_LLDP_TYPE_SYS_NAME failed\n", __func__); goto qlnx_register_default_lldp_tlvs_exit; } //register System Description TLV ret = ecore_lldp_register_tlv(p_hwfn, p_ptt, ECORE_LLDP_NEAREST_BRIDGE, QLNX_LLDP_TYPE_SYS_DESC); if (ret != ECORE_SUCCESS) { device_printf(ha->pci_dev, "%s: QLNX_LLDP_TYPE_SYS_DESC failed\n", __func__); goto qlnx_register_default_lldp_tlvs_exit; } //register System Capabilities TLV ret = ecore_lldp_register_tlv(p_hwfn, p_ptt, ECORE_LLDP_NEAREST_BRIDGE, QLNX_LLDP_TYPE_SYS_CAPS); if (ret != ECORE_SUCCESS) { device_printf(ha->pci_dev, "%s: QLNX_LLDP_TYPE_SYS_CAPS failed\n", __func__); goto qlnx_register_default_lldp_tlvs_exit; } //register Management Address TLV ret = ecore_lldp_register_tlv(p_hwfn, p_ptt, ECORE_LLDP_NEAREST_BRIDGE, QLNX_LLDP_TYPE_MGMT_ADDR); if (ret != ECORE_SUCCESS) { device_printf(ha->pci_dev, "%s: QLNX_LLDP_TYPE_MGMT_ADDR failed\n", __func__); goto qlnx_register_default_lldp_tlvs_exit; } //register Organizationally Specific TLVs ret = ecore_lldp_register_tlv(p_hwfn, p_ptt, ECORE_LLDP_NEAREST_BRIDGE, QLNX_LLDP_TYPE_ORG_SPECIFIC); if (ret != ECORE_SUCCESS) { device_printf(ha->pci_dev, "%s: QLNX_LLDP_TYPE_ORG_SPECIFIC failed\n", __func__); } qlnx_register_default_lldp_tlvs_exit: return (ret); } int qlnx_set_lldp_tlvx(qlnx_host_t *ha, qlnx_lldp_sys_tlvs_t *lldp_tlvs) { int ret = 0; struct ecore_hwfn *p_hwfn; struct ecore_ptt *p_ptt; struct ecore_lldp_sys_tlvs tlv_params; p_hwfn = &ha->cdev.hwfns[0]; p_ptt = ecore_ptt_acquire(p_hwfn); if (!p_ptt) { device_printf(ha->pci_dev, "%s: ecore_ptt_acquire failed\n", __func__); return (ENXIO); } ret = qlnx_lldp_configure(ha, p_hwfn, p_ptt, 0); if (ret) { device_printf(ha->pci_dev, "%s: qlnx_lldp_configure disable failed\n", __func__); goto qlnx_set_lldp_tlvx_exit; } ret = qlnx_register_default_lldp_tlvs(ha, p_hwfn, p_ptt); if (ret) { device_printf(ha->pci_dev, "%s: qlnx_register_default_lldp_tlvs failed\n", __func__); goto qlnx_set_lldp_tlvx_exit; } ret = qlnx_lldp_configure(ha, p_hwfn, p_ptt, 1); if (ret) { device_printf(ha->pci_dev, "%s: qlnx_lldp_configure enable failed\n", __func__); goto qlnx_set_lldp_tlvx_exit; } if (lldp_tlvs != NULL) { bzero(&tlv_params, sizeof(struct ecore_lldp_sys_tlvs)); tlv_params.discard_mandatory_tlv = (lldp_tlvs->discard_mandatory_tlv ? true: false); tlv_params.buf_size = lldp_tlvs->buf_size; memcpy(tlv_params.buf, lldp_tlvs->buf, lldp_tlvs->buf_size); ret = ecore_lldp_set_system_tlvs(p_hwfn, p_ptt, &tlv_params); if (ret) { device_printf(ha->pci_dev, "%s: ecore_lldp_set_system_tlvs failed\n", __func__); } } qlnx_set_lldp_tlvx_exit: ecore_ptt_release(p_hwfn, p_ptt); return (ret); } #endif /* #ifdef QLNX_USER_LLDP */ static int qlnx_eioctl(struct cdev *dev, u_long cmd, caddr_t data, int fflag, struct thread *td) { qlnx_host_t *ha; int rval = 0; qlnx_trace_t *trace; int i; if ((ha = (qlnx_host_t *)dev->si_drv1) == NULL) return ENXIO; switch (cmd) { case QLNX_GRC_DUMP_SIZE: qlnx_get_grc_dump_size(ha, (qlnx_grcdump_t *)data); break; case QLNX_GRC_DUMP: rval = qlnx_get_grc_dump(ha, (qlnx_grcdump_t *)data); break; case QLNX_IDLE_CHK_SIZE: qlnx_get_idle_chk_size(ha, (qlnx_idle_chk_t *)data); break; case QLNX_IDLE_CHK: rval = qlnx_get_idle_chk(ha, (qlnx_idle_chk_t *)data); break; case QLNX_DRV_INFO: rval = qlnx_drv_info(ha, (qlnx_drvinfo_t *)data); break; case QLNX_DEV_SETTING: rval = qlnx_dev_settings(ha, (qlnx_dev_setting_t *)data); break; case QLNX_GET_REGS: rval = qlnx_get_regs(ha, (qlnx_get_regs_t *)data); break; case QLNX_NVRAM: rval = qlnx_nvram(ha, (qlnx_nvram_t *)data); break; case QLNX_RD_WR_REG: rval = qlnx_reg_rd_wr(ha, (qlnx_reg_rd_wr_t *)data); break; case QLNX_RD_WR_PCICFG: rval = qlnx_rd_wr_pci_config(ha, (qlnx_pcicfg_rd_wr_t *)data); break; case QLNX_MAC_ADDR: qlnx_mac_addr(ha, (qlnx_perm_mac_addr_t *)data); break; case QLNX_STORM_STATS: qlnx_storm_stats(ha, (qlnx_storm_stats_dump_t *)data); break; case QLNX_TRACE_SIZE: qlnx_get_trace_size(ha, (qlnx_trace_t *)data); break; case QLNX_TRACE: trace = (qlnx_trace_t *)data; for (i = 0; i < ha->cdev.num_hwfns; i++) { if (trace->size[i] && trace->cmd && trace->buffer[i]) rval = qlnx_get_trace(ha, i, trace); if (rval) break; } break; #ifdef QLNX_USER_LLDP case QLNX_SET_LLDP_TLVS: rval = qlnx_set_lldp_tlvx(ha, (qlnx_lldp_sys_tlvs_t *)data); break; #endif /* #ifdef QLNX_USER_LLDP */ default: rval = EINVAL; break; } return (rval); } diff --git a/sys/dev/qlnx/qlnxe/qlnx_os.c b/sys/dev/qlnx/qlnxe/qlnx_os.c index 9d31024c29c1..bbd4c0fe7e8c 100644 --- a/sys/dev/qlnx/qlnxe/qlnx_os.c +++ b/sys/dev/qlnx/qlnxe/qlnx_os.c @@ -1,8348 +1,8347 @@ /* * Copyright (c) 2017-2018 Cavium, Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ /* * File: qlnx_os.c * Author : David C Somayajulu, Cavium, Inc., San Jose, CA 95131. */ #include __FBSDID("$FreeBSD$"); #include "qlnx_os.h" #include "bcm_osal.h" #include "reg_addr.h" #include "ecore_gtt_reg_addr.h" #include "ecore.h" #include "ecore_chain.h" #include "ecore_status.h" #include "ecore_hw.h" #include "ecore_rt_defs.h" #include "ecore_init_ops.h" #include "ecore_int.h" #include "ecore_cxt.h" #include "ecore_spq.h" #include "ecore_init_fw_funcs.h" #include "ecore_sp_commands.h" #include "ecore_dev_api.h" #include "ecore_l2_api.h" #include "ecore_mcp.h" #include "ecore_hw_defs.h" #include "mcp_public.h" #include "ecore_iro.h" #include "nvm_cfg.h" #include "ecore_dev_api.h" #include "ecore_dbg_fw_funcs.h" #include "ecore_iov_api.h" #include "ecore_vf_api.h" #include "qlnx_ioctl.h" #include "qlnx_def.h" #include "qlnx_ver.h" #ifdef QLNX_ENABLE_IWARP #include "qlnx_rdma.h" #endif /* #ifdef QLNX_ENABLE_IWARP */ #include /* * static functions */ /* * ioctl related functions */ static void qlnx_add_sysctls(qlnx_host_t *ha); /* * main driver */ static void qlnx_release(qlnx_host_t *ha); static void qlnx_fp_isr(void *arg); static void qlnx_init_ifnet(device_t dev, qlnx_host_t *ha); static void qlnx_init(void *arg); static void qlnx_init_locked(qlnx_host_t *ha); static int qlnx_set_multi(qlnx_host_t *ha, uint32_t add_multi); static int qlnx_set_promisc(qlnx_host_t *ha); static int qlnx_set_allmulti(qlnx_host_t *ha); -static int qlnx_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data); -static int qlnx_media_change(struct ifnet *ifp); -static void qlnx_media_status(struct ifnet *ifp, struct ifmediareq *ifmr); +static int qlnx_ioctl(if_t ifp, u_long cmd, caddr_t data); +static int qlnx_media_change(if_t ifp); +static void qlnx_media_status(if_t ifp, struct ifmediareq *ifmr); static void qlnx_stop(qlnx_host_t *ha); static int qlnx_send(qlnx_host_t *ha, struct qlnx_fastpath *fp, struct mbuf **m_headp); static int qlnx_get_ifq_snd_maxlen(qlnx_host_t *ha); static uint32_t qlnx_get_optics(qlnx_host_t *ha, struct qlnx_link_output *if_link); -static int qlnx_transmit(struct ifnet *ifp, struct mbuf *mp); -static int qlnx_transmit_locked(struct ifnet *ifp, struct qlnx_fastpath *fp, +static int qlnx_transmit(if_t ifp, struct mbuf *mp); +static int qlnx_transmit_locked(if_t ifp, struct qlnx_fastpath *fp, struct mbuf *mp); -static void qlnx_qflush(struct ifnet *ifp); +static void qlnx_qflush(if_t ifp); static int qlnx_alloc_parent_dma_tag(qlnx_host_t *ha); static void qlnx_free_parent_dma_tag(qlnx_host_t *ha); static int qlnx_alloc_tx_dma_tag(qlnx_host_t *ha); static void qlnx_free_tx_dma_tag(qlnx_host_t *ha); static int qlnx_alloc_rx_dma_tag(qlnx_host_t *ha); static void qlnx_free_rx_dma_tag(qlnx_host_t *ha); static int qlnx_get_mfw_version(qlnx_host_t *ha, uint32_t *mfw_ver); static int qlnx_get_flash_size(qlnx_host_t *ha, uint32_t *flash_size); static int qlnx_nic_setup(struct ecore_dev *cdev, struct ecore_pf_params *func_params); static int qlnx_nic_start(struct ecore_dev *cdev); static int qlnx_slowpath_start(qlnx_host_t *ha); static int qlnx_slowpath_stop(qlnx_host_t *ha); static int qlnx_init_hw(qlnx_host_t *ha); static void qlnx_set_id(struct ecore_dev *cdev, char name[NAME_SIZE], char ver_str[VER_SIZE]); static void qlnx_unload(qlnx_host_t *ha); static int qlnx_load(qlnx_host_t *ha); static void qlnx_hw_set_multi(qlnx_host_t *ha, uint8_t *mta, uint32_t mcnt, uint32_t add_mac); static void qlnx_dump_buf8(qlnx_host_t *ha, const char *msg, void *dbuf, uint32_t len); static int qlnx_alloc_rx_buffer(qlnx_host_t *ha, struct qlnx_rx_queue *rxq); static void qlnx_reuse_rx_data(struct qlnx_rx_queue *rxq); static void qlnx_update_rx_prod(struct ecore_hwfn *p_hwfn, struct qlnx_rx_queue *rxq); static int qlnx_set_rx_accept_filter(qlnx_host_t *ha, uint8_t filter); static int qlnx_grc_dumpsize(qlnx_host_t *ha, uint32_t *num_dwords, int hwfn_index); static int qlnx_idle_chk_size(qlnx_host_t *ha, uint32_t *num_dwords, int hwfn_index); static void qlnx_timer(void *arg); static int qlnx_alloc_tx_br(qlnx_host_t *ha, struct qlnx_fastpath *fp); static void qlnx_free_tx_br(qlnx_host_t *ha, struct qlnx_fastpath *fp); static void qlnx_trigger_dump(qlnx_host_t *ha); static uint16_t qlnx_num_tx_compl(qlnx_host_t *ha, struct qlnx_fastpath *fp, struct qlnx_tx_queue *txq); static void qlnx_tx_int(qlnx_host_t *ha, struct qlnx_fastpath *fp, struct qlnx_tx_queue *txq); static int qlnx_rx_int(qlnx_host_t *ha, struct qlnx_fastpath *fp, int budget, int lro_enable); static void qlnx_fp_taskqueue(void *context, int pending); static void qlnx_sample_storm_stats(qlnx_host_t *ha); static int qlnx_alloc_tpa_mbuf(qlnx_host_t *ha, uint16_t rx_buf_size, struct qlnx_agg_info *tpa); static void qlnx_free_tpa_mbuf(qlnx_host_t *ha, struct qlnx_agg_info *tpa); static uint64_t qlnx_get_counter(if_t ifp, ift_counter cnt); /* * Hooks to the Operating Systems */ static int qlnx_pci_probe (device_t); static int qlnx_pci_attach (device_t); static int qlnx_pci_detach (device_t); #ifndef QLNX_VF #ifdef CONFIG_ECORE_SRIOV static int qlnx_iov_init(device_t dev, uint16_t num_vfs, const nvlist_t *params); static void qlnx_iov_uninit(device_t dev); static int qlnx_iov_add_vf(device_t dev, uint16_t vfnum, const nvlist_t *params); static void qlnx_initialize_sriov(qlnx_host_t *ha); static void qlnx_pf_taskqueue(void *context, int pending); static int qlnx_create_pf_taskqueues(qlnx_host_t *ha); static void qlnx_destroy_pf_taskqueues(qlnx_host_t *ha); static void qlnx_inform_vf_link_state(struct ecore_hwfn *p_hwfn, qlnx_host_t *ha); #endif /* #ifdef CONFIG_ECORE_SRIOV */ static device_method_t qlnx_pci_methods[] = { /* Device interface */ DEVMETHOD(device_probe, qlnx_pci_probe), DEVMETHOD(device_attach, qlnx_pci_attach), DEVMETHOD(device_detach, qlnx_pci_detach), #ifdef CONFIG_ECORE_SRIOV DEVMETHOD(pci_iov_init, qlnx_iov_init), DEVMETHOD(pci_iov_uninit, qlnx_iov_uninit), DEVMETHOD(pci_iov_add_vf, qlnx_iov_add_vf), #endif /* #ifdef CONFIG_ECORE_SRIOV */ { 0, 0 } }; static driver_t qlnx_pci_driver = { "ql", qlnx_pci_methods, sizeof (qlnx_host_t), }; MODULE_VERSION(if_qlnxe,1); DRIVER_MODULE(if_qlnxe, pci, qlnx_pci_driver, 0, 0); MODULE_DEPEND(if_qlnxe, pci, 1, 1, 1); MODULE_DEPEND(if_qlnxe, ether, 1, 1, 1); #else static device_method_t qlnxv_pci_methods[] = { /* Device interface */ DEVMETHOD(device_probe, qlnx_pci_probe), DEVMETHOD(device_attach, qlnx_pci_attach), DEVMETHOD(device_detach, qlnx_pci_detach), { 0, 0 } }; static driver_t qlnxv_pci_driver = { "ql", qlnxv_pci_methods, sizeof (qlnx_host_t), }; MODULE_VERSION(if_qlnxev,1); DRIVER_MODULE(if_qlnxev, pci, qlnxv_pci_driver, 0, 0); MODULE_DEPEND(if_qlnxev, pci, 1, 1, 1); MODULE_DEPEND(if_qlnxev, ether, 1, 1, 1); #endif /* #ifdef QLNX_VF */ MALLOC_DEFINE(M_QLNXBUF, "qlnxbuf", "Buffers for qlnx driver"); char qlnx_dev_str[128]; char qlnx_ver_str[VER_SIZE]; char qlnx_name_str[NAME_SIZE]; /* * Some PCI Configuration Space Related Defines */ #ifndef PCI_VENDOR_QLOGIC #define PCI_VENDOR_QLOGIC 0x1077 #endif /* 40G Adapter QLE45xxx*/ #ifndef QLOGIC_PCI_DEVICE_ID_1634 #define QLOGIC_PCI_DEVICE_ID_1634 0x1634 #endif /* 100G Adapter QLE45xxx*/ #ifndef QLOGIC_PCI_DEVICE_ID_1644 #define QLOGIC_PCI_DEVICE_ID_1644 0x1644 #endif /* 25G Adapter QLE45xxx*/ #ifndef QLOGIC_PCI_DEVICE_ID_1656 #define QLOGIC_PCI_DEVICE_ID_1656 0x1656 #endif /* 50G Adapter QLE45xxx*/ #ifndef QLOGIC_PCI_DEVICE_ID_1654 #define QLOGIC_PCI_DEVICE_ID_1654 0x1654 #endif /* 10G/25G/40G Adapter QLE41xxx*/ #ifndef QLOGIC_PCI_DEVICE_ID_8070 #define QLOGIC_PCI_DEVICE_ID_8070 0x8070 #endif /* SRIOV Device (All Speeds) Adapter QLE41xxx*/ #ifndef QLOGIC_PCI_DEVICE_ID_8090 #define QLOGIC_PCI_DEVICE_ID_8090 0x8090 #endif SYSCTL_NODE(_hw, OID_AUTO, qlnxe, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, "qlnxe driver parameters"); /* Number of Queues: 0 (Auto) or 1 to 32 (fixed queue number) */ static int qlnxe_queue_count = QLNX_DEFAULT_RSS; SYSCTL_INT(_hw_qlnxe, OID_AUTO, queue_count, CTLFLAG_RDTUN, &qlnxe_queue_count, 0, "Multi-Queue queue count"); /* * Note on RDMA personality setting * * Read the personality configured in NVRAM * If the personality is ETH_ONLY, ETH_IWARP or ETH_ROCE and * the configured personality in sysctl is QLNX_PERSONALITY_DEFAULT * use the personality in NVRAM. * Otherwise use t the personality configured in sysctl. * */ #define QLNX_PERSONALITY_DEFAULT 0x0 /* use personality in NVRAM */ #define QLNX_PERSONALITY_ETH_ONLY 0x1 /* Override with ETH_ONLY */ #define QLNX_PERSONALITY_ETH_IWARP 0x2 /* Override with ETH_IWARP */ #define QLNX_PERSONALITY_ETH_ROCE 0x3 /* Override with ETH_ROCE */ #define QLNX_PERSONALITY_BITS_PER_FUNC 4 #define QLNX_PERSONALIY_MASK 0xF /* RDMA configuration; 64bit field allows setting for 16 physical functions*/ static uint64_t qlnxe_rdma_configuration = 0x22222222; SYSCTL_U64(_hw_qlnxe, OID_AUTO, rdma_configuration, CTLFLAG_RDTUN, &qlnxe_rdma_configuration, 0, "RDMA Configuration"); int qlnx_vf_device(qlnx_host_t *ha) { uint16_t device_id; device_id = ha->device_id; if (device_id == QLOGIC_PCI_DEVICE_ID_8090) return 0; return -1; } static int qlnx_valid_device(qlnx_host_t *ha) { uint16_t device_id; device_id = ha->device_id; #ifndef QLNX_VF if ((device_id == QLOGIC_PCI_DEVICE_ID_1634) || (device_id == QLOGIC_PCI_DEVICE_ID_1644) || (device_id == QLOGIC_PCI_DEVICE_ID_1656) || (device_id == QLOGIC_PCI_DEVICE_ID_1654) || (device_id == QLOGIC_PCI_DEVICE_ID_8070)) return 0; #else if (device_id == QLOGIC_PCI_DEVICE_ID_8090) return 0; #endif /* #ifndef QLNX_VF */ return -1; } #ifdef QLNX_ENABLE_IWARP static int qlnx_rdma_supported(struct qlnx_host *ha) { uint16_t device_id; device_id = pci_get_device(ha->pci_dev); if ((device_id == QLOGIC_PCI_DEVICE_ID_1634) || (device_id == QLOGIC_PCI_DEVICE_ID_1656) || (device_id == QLOGIC_PCI_DEVICE_ID_1654) || (device_id == QLOGIC_PCI_DEVICE_ID_8070)) return (0); return (-1); } #endif /* #ifdef QLNX_ENABLE_IWARP */ /* * Name: qlnx_pci_probe * Function: Validate the PCI device to be a QLA80XX device */ static int qlnx_pci_probe(device_t dev) { snprintf(qlnx_ver_str, sizeof(qlnx_ver_str), "v%d.%d.%d", QLNX_VERSION_MAJOR, QLNX_VERSION_MINOR, QLNX_VERSION_BUILD); snprintf(qlnx_name_str, sizeof(qlnx_name_str), "qlnx"); if (pci_get_vendor(dev) != PCI_VENDOR_QLOGIC) { return (ENXIO); } switch (pci_get_device(dev)) { #ifndef QLNX_VF case QLOGIC_PCI_DEVICE_ID_1644: snprintf(qlnx_dev_str, sizeof(qlnx_dev_str), "%s v%d.%d.%d", "Qlogic 100GbE PCI CNA Adapter-Ethernet Function", QLNX_VERSION_MAJOR, QLNX_VERSION_MINOR, QLNX_VERSION_BUILD); device_set_desc_copy(dev, qlnx_dev_str); break; case QLOGIC_PCI_DEVICE_ID_1634: snprintf(qlnx_dev_str, sizeof(qlnx_dev_str), "%s v%d.%d.%d", "Qlogic 40GbE PCI CNA Adapter-Ethernet Function", QLNX_VERSION_MAJOR, QLNX_VERSION_MINOR, QLNX_VERSION_BUILD); device_set_desc_copy(dev, qlnx_dev_str); break; case QLOGIC_PCI_DEVICE_ID_1656: snprintf(qlnx_dev_str, sizeof(qlnx_dev_str), "%s v%d.%d.%d", "Qlogic 25GbE PCI CNA Adapter-Ethernet Function", QLNX_VERSION_MAJOR, QLNX_VERSION_MINOR, QLNX_VERSION_BUILD); device_set_desc_copy(dev, qlnx_dev_str); break; case QLOGIC_PCI_DEVICE_ID_1654: snprintf(qlnx_dev_str, sizeof(qlnx_dev_str), "%s v%d.%d.%d", "Qlogic 50GbE PCI CNA Adapter-Ethernet Function", QLNX_VERSION_MAJOR, QLNX_VERSION_MINOR, QLNX_VERSION_BUILD); device_set_desc_copy(dev, qlnx_dev_str); break; case QLOGIC_PCI_DEVICE_ID_8070: snprintf(qlnx_dev_str, sizeof(qlnx_dev_str), "%s v%d.%d.%d", "Qlogic 10GbE/25GbE/40GbE PCI CNA (AH)" " Adapter-Ethernet Function", QLNX_VERSION_MAJOR, QLNX_VERSION_MINOR, QLNX_VERSION_BUILD); device_set_desc_copy(dev, qlnx_dev_str); break; #else case QLOGIC_PCI_DEVICE_ID_8090: snprintf(qlnx_dev_str, sizeof(qlnx_dev_str), "%s v%d.%d.%d", "Qlogic SRIOV PCI CNA (AH) " "Adapter-Ethernet Function", QLNX_VERSION_MAJOR, QLNX_VERSION_MINOR, QLNX_VERSION_BUILD); device_set_desc_copy(dev, qlnx_dev_str); break; #endif /* #ifndef QLNX_VF */ default: return (ENXIO); } #ifdef QLNX_ENABLE_IWARP qlnx_rdma_init(); #endif /* #ifdef QLNX_ENABLE_IWARP */ return (BUS_PROBE_DEFAULT); } static uint16_t qlnx_num_tx_compl(qlnx_host_t *ha, struct qlnx_fastpath *fp, struct qlnx_tx_queue *txq) { u16 hw_bd_cons; u16 ecore_cons_idx; uint16_t diff; hw_bd_cons = le16toh(*txq->hw_cons_ptr); ecore_cons_idx = ecore_chain_get_cons_idx(&txq->tx_pbl); if (hw_bd_cons < ecore_cons_idx) { diff = (1 << 16) - (ecore_cons_idx - hw_bd_cons); } else { diff = hw_bd_cons - ecore_cons_idx; } return diff; } static void qlnx_sp_intr(void *arg) { struct ecore_hwfn *p_hwfn; qlnx_host_t *ha; int i; p_hwfn = arg; if (p_hwfn == NULL) { printf("%s: spurious slowpath intr\n", __func__); return; } ha = (qlnx_host_t *)p_hwfn->p_dev; QL_DPRINT2(ha, "enter\n"); for (i = 0; i < ha->cdev.num_hwfns; i++) { if (&ha->cdev.hwfns[i] == p_hwfn) { taskqueue_enqueue(ha->sp_taskqueue[i], &ha->sp_task[i]); break; } } QL_DPRINT2(ha, "exit\n"); return; } static void qlnx_sp_taskqueue(void *context, int pending) { struct ecore_hwfn *p_hwfn; p_hwfn = context; if (p_hwfn != NULL) { qlnx_sp_isr(p_hwfn); } return; } static int qlnx_create_sp_taskqueues(qlnx_host_t *ha) { int i; uint8_t tq_name[32]; for (i = 0; i < ha->cdev.num_hwfns; i++) { struct ecore_hwfn *p_hwfn = &ha->cdev.hwfns[i]; bzero(tq_name, sizeof (tq_name)); snprintf(tq_name, sizeof (tq_name), "ql_sp_tq_%d", i); TASK_INIT(&ha->sp_task[i], 0, qlnx_sp_taskqueue, p_hwfn); ha->sp_taskqueue[i] = taskqueue_create(tq_name, M_NOWAIT, taskqueue_thread_enqueue, &ha->sp_taskqueue[i]); if (ha->sp_taskqueue[i] == NULL) return (-1); taskqueue_start_threads(&ha->sp_taskqueue[i], 1, PI_NET, "%s", tq_name); QL_DPRINT1(ha, "%p\n", ha->sp_taskqueue[i]); } return (0); } static void qlnx_destroy_sp_taskqueues(qlnx_host_t *ha) { int i; for (i = 0; i < ha->cdev.num_hwfns; i++) { if (ha->sp_taskqueue[i] != NULL) { taskqueue_drain(ha->sp_taskqueue[i], &ha->sp_task[i]); taskqueue_free(ha->sp_taskqueue[i]); } } return; } static void qlnx_fp_taskqueue(void *context, int pending) { struct qlnx_fastpath *fp; qlnx_host_t *ha; - struct ifnet *ifp; + if_t ifp; fp = context; if (fp == NULL) return; ha = (qlnx_host_t *)fp->edev; ifp = ha->ifp; - if(ifp->if_drv_flags & IFF_DRV_RUNNING) { + if(if_getdrvflags(ifp) & IFF_DRV_RUNNING) { if (!drbr_empty(ifp, fp->tx_br)) { if(mtx_trylock(&fp->tx_mtx)) { #ifdef QLNX_TRACE_PERF_DATA tx_pkts = fp->tx_pkts_transmitted; tx_compl = fp->tx_pkts_completed; #endif qlnx_transmit_locked(ifp, fp, NULL); #ifdef QLNX_TRACE_PERF_DATA fp->tx_pkts_trans_fp += (fp->tx_pkts_transmitted - tx_pkts); fp->tx_pkts_compl_fp += (fp->tx_pkts_completed - tx_compl); #endif mtx_unlock(&fp->tx_mtx); } } } QL_DPRINT2(ha, "exit \n"); return; } static int qlnx_create_fp_taskqueues(qlnx_host_t *ha) { int i; uint8_t tq_name[32]; struct qlnx_fastpath *fp; for (i = 0; i < ha->num_rss; i++) { fp = &ha->fp_array[i]; bzero(tq_name, sizeof (tq_name)); snprintf(tq_name, sizeof (tq_name), "ql_fp_tq_%d", i); TASK_INIT(&fp->fp_task, 0, qlnx_fp_taskqueue, fp); fp->fp_taskqueue = taskqueue_create(tq_name, M_NOWAIT, taskqueue_thread_enqueue, &fp->fp_taskqueue); if (fp->fp_taskqueue == NULL) return (-1); taskqueue_start_threads(&fp->fp_taskqueue, 1, PI_NET, "%s", tq_name); QL_DPRINT1(ha, "%p\n",fp->fp_taskqueue); } return (0); } static void qlnx_destroy_fp_taskqueues(qlnx_host_t *ha) { int i; struct qlnx_fastpath *fp; for (i = 0; i < ha->num_rss; i++) { fp = &ha->fp_array[i]; if (fp->fp_taskqueue != NULL) { taskqueue_drain(fp->fp_taskqueue, &fp->fp_task); taskqueue_free(fp->fp_taskqueue); fp->fp_taskqueue = NULL; } } return; } static void qlnx_drain_fp_taskqueues(qlnx_host_t *ha) { int i; struct qlnx_fastpath *fp; for (i = 0; i < ha->num_rss; i++) { fp = &ha->fp_array[i]; if (fp->fp_taskqueue != NULL) { QLNX_UNLOCK(ha); taskqueue_drain(fp->fp_taskqueue, &fp->fp_task); QLNX_LOCK(ha); } } return; } static void qlnx_get_params(qlnx_host_t *ha) { if ((qlnxe_queue_count < 0) || (qlnxe_queue_count > QLNX_MAX_RSS)) { device_printf(ha->pci_dev, "invalid queue_count value (%d)\n", qlnxe_queue_count); qlnxe_queue_count = 0; } return; } static void qlnx_error_recovery_taskqueue(void *context, int pending) { qlnx_host_t *ha; ha = context; QL_DPRINT2(ha, "enter\n"); QLNX_LOCK(ha); qlnx_stop(ha); QLNX_UNLOCK(ha); #ifdef QLNX_ENABLE_IWARP qlnx_rdma_dev_remove(ha); #endif /* #ifdef QLNX_ENABLE_IWARP */ qlnx_slowpath_stop(ha); qlnx_slowpath_start(ha); #ifdef QLNX_ENABLE_IWARP qlnx_rdma_dev_add(ha); #endif /* #ifdef QLNX_ENABLE_IWARP */ qlnx_init(ha); callout_reset(&ha->qlnx_callout, hz, qlnx_timer, ha); QL_DPRINT2(ha, "exit\n"); return; } static int qlnx_create_error_recovery_taskqueue(qlnx_host_t *ha) { uint8_t tq_name[32]; bzero(tq_name, sizeof (tq_name)); snprintf(tq_name, sizeof (tq_name), "ql_err_tq"); TASK_INIT(&ha->err_task, 0, qlnx_error_recovery_taskqueue, ha); ha->err_taskqueue = taskqueue_create(tq_name, M_NOWAIT, taskqueue_thread_enqueue, &ha->err_taskqueue); if (ha->err_taskqueue == NULL) return (-1); taskqueue_start_threads(&ha->err_taskqueue, 1, PI_NET, "%s", tq_name); QL_DPRINT1(ha, "%p\n",ha->err_taskqueue); return (0); } static void qlnx_destroy_error_recovery_taskqueue(qlnx_host_t *ha) { if (ha->err_taskqueue != NULL) { taskqueue_drain(ha->err_taskqueue, &ha->err_task); taskqueue_free(ha->err_taskqueue); } ha->err_taskqueue = NULL; return; } /* * Name: qlnx_pci_attach * Function: attaches the device to the operating system */ static int qlnx_pci_attach(device_t dev) { qlnx_host_t *ha = NULL; uint32_t rsrc_len_reg __unused = 0; uint32_t rsrc_len_dbells = 0; uint32_t rsrc_len_msix __unused = 0; int i; uint32_t mfw_ver; uint32_t num_sp_msix = 0; uint32_t num_rdma_irqs = 0; if ((ha = device_get_softc(dev)) == NULL) { device_printf(dev, "cannot get softc\n"); return (ENOMEM); } memset(ha, 0, sizeof (qlnx_host_t)); ha->device_id = pci_get_device(dev); if (qlnx_valid_device(ha) != 0) { device_printf(dev, "device is not valid device\n"); return (ENXIO); } ha->pci_func = pci_get_function(dev); ha->pci_dev = dev; mtx_init(&ha->hw_lock, "qlnx_hw_lock", MTX_NETWORK_LOCK, MTX_DEF); ha->flags.lock_init = 1; pci_enable_busmaster(dev); /* * map the PCI BARs */ ha->reg_rid = PCIR_BAR(0); ha->pci_reg = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &ha->reg_rid, RF_ACTIVE); if (ha->pci_reg == NULL) { device_printf(dev, "unable to map BAR0\n"); goto qlnx_pci_attach_err; } rsrc_len_reg = (uint32_t) bus_get_resource_count(dev, SYS_RES_MEMORY, ha->reg_rid); ha->dbells_rid = PCIR_BAR(2); rsrc_len_dbells = (uint32_t) bus_get_resource_count(dev, SYS_RES_MEMORY, ha->dbells_rid); if (rsrc_len_dbells) { ha->pci_dbells = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &ha->dbells_rid, RF_ACTIVE); if (ha->pci_dbells == NULL) { device_printf(dev, "unable to map BAR1\n"); goto qlnx_pci_attach_err; } ha->dbells_phys_addr = (uint64_t) bus_get_resource_start(dev, SYS_RES_MEMORY, ha->dbells_rid); ha->dbells_size = rsrc_len_dbells; } else { if (qlnx_vf_device(ha) != 0) { device_printf(dev, " BAR1 size is zero\n"); goto qlnx_pci_attach_err; } } ha->msix_rid = PCIR_BAR(4); ha->msix_bar = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &ha->msix_rid, RF_ACTIVE); if (ha->msix_bar == NULL) { device_printf(dev, "unable to map BAR2\n"); goto qlnx_pci_attach_err; } rsrc_len_msix = (uint32_t) bus_get_resource_count(dev, SYS_RES_MEMORY, ha->msix_rid); ha->dbg_level = 0x0000; QL_DPRINT1(ha, "\n\t\t\t" "pci_dev = %p pci_reg = %p, reg_len = 0x%08x reg_rid = 0x%08x" "\n\t\t\tdbells = %p, dbells_len = 0x%08x dbells_rid = 0x%08x" "\n\t\t\tmsix = %p, msix_len = 0x%08x msix_rid = 0x%08x" " msix_avail = 0x%x " "\n\t\t\t[ncpus = %d]\n", ha->pci_dev, ha->pci_reg, rsrc_len_reg, ha->reg_rid, ha->pci_dbells, rsrc_len_dbells, ha->dbells_rid, ha->msix_bar, rsrc_len_msix, ha->msix_rid, pci_msix_count(dev), mp_ncpus); /* * allocate dma tags */ if (qlnx_alloc_parent_dma_tag(ha)) goto qlnx_pci_attach_err; if (qlnx_alloc_tx_dma_tag(ha)) goto qlnx_pci_attach_err; if (qlnx_alloc_rx_dma_tag(ha)) goto qlnx_pci_attach_err; if (qlnx_init_hw(ha) != 0) goto qlnx_pci_attach_err; ha->flags.hw_init = 1; qlnx_get_params(ha); if((pci_get_device(dev) == QLOGIC_PCI_DEVICE_ID_1644) && (qlnxe_queue_count == QLNX_DEFAULT_RSS)) { qlnxe_queue_count = QLNX_MAX_RSS; } /* * Allocate MSI-x vectors */ if (qlnx_vf_device(ha) != 0) { if (qlnxe_queue_count == 0) ha->num_rss = QLNX_DEFAULT_RSS; else ha->num_rss = qlnxe_queue_count; num_sp_msix = ha->cdev.num_hwfns; } else { uint8_t max_rxq; uint8_t max_txq; ecore_vf_get_num_rxqs(&ha->cdev.hwfns[0], &max_rxq); ecore_vf_get_num_rxqs(&ha->cdev.hwfns[0], &max_txq); if (max_rxq < max_txq) ha->num_rss = max_rxq; else ha->num_rss = max_txq; if (ha->num_rss > QLNX_MAX_VF_RSS) ha->num_rss = QLNX_MAX_VF_RSS; num_sp_msix = 0; } if (ha->num_rss > mp_ncpus) ha->num_rss = mp_ncpus; ha->num_tc = QLNX_MAX_TC; ha->msix_count = pci_msix_count(dev); #ifdef QLNX_ENABLE_IWARP num_rdma_irqs = qlnx_rdma_get_num_irqs(ha); #endif /* #ifdef QLNX_ENABLE_IWARP */ if (!ha->msix_count || (ha->msix_count < (num_sp_msix + 1 + num_rdma_irqs))) { device_printf(dev, "%s: msix_count[%d] not enough\n", __func__, ha->msix_count); goto qlnx_pci_attach_err; } if (ha->msix_count > (ha->num_rss + num_sp_msix + num_rdma_irqs)) ha->msix_count = ha->num_rss + num_sp_msix + num_rdma_irqs; else ha->num_rss = ha->msix_count - (num_sp_msix + num_rdma_irqs); QL_DPRINT1(ha, "\n\t\t\t" "pci_reg = %p, reg_len = 0x%08x reg_rid = 0x%08x" "\n\t\t\tdbells = %p, dbells_len = 0x%08x dbells_rid = 0x%08x" "\n\t\t\tmsix = %p, msix_len = 0x%08x msix_rid = 0x%08x" " msix_avail = 0x%x msix_alloc = 0x%x" "\n\t\t\t[ncpus = %d][num_rss = 0x%x] [num_tc = 0x%x]\n", ha->pci_reg, rsrc_len_reg, ha->reg_rid, ha->pci_dbells, rsrc_len_dbells, ha->dbells_rid, ha->msix_bar, rsrc_len_msix, ha->msix_rid, pci_msix_count(dev), ha->msix_count, mp_ncpus, ha->num_rss, ha->num_tc); if (pci_alloc_msix(dev, &ha->msix_count)) { device_printf(dev, "%s: pci_alloc_msix[%d] failed\n", __func__, ha->msix_count); ha->msix_count = 0; goto qlnx_pci_attach_err; } /* * Initialize slow path interrupt and task queue */ if (num_sp_msix) { if (qlnx_create_sp_taskqueues(ha) != 0) goto qlnx_pci_attach_err; for (i = 0; i < ha->cdev.num_hwfns; i++) { struct ecore_hwfn *p_hwfn = &ha->cdev.hwfns[i]; ha->sp_irq_rid[i] = i + 1; ha->sp_irq[i] = bus_alloc_resource_any(dev, SYS_RES_IRQ, &ha->sp_irq_rid[i], (RF_ACTIVE | RF_SHAREABLE)); if (ha->sp_irq[i] == NULL) { device_printf(dev, "could not allocate mbx interrupt\n"); goto qlnx_pci_attach_err; } if (bus_setup_intr(dev, ha->sp_irq[i], (INTR_TYPE_NET | INTR_MPSAFE), NULL, qlnx_sp_intr, p_hwfn, &ha->sp_handle[i])) { device_printf(dev, "could not setup slow path interrupt\n"); goto qlnx_pci_attach_err; } QL_DPRINT1(ha, "p_hwfn [%p] sp_irq_rid %d" " sp_irq %p sp_handle %p\n", p_hwfn, ha->sp_irq_rid[i], ha->sp_irq[i], ha->sp_handle[i]); } } /* * initialize fast path interrupt */ if (qlnx_create_fp_taskqueues(ha) != 0) goto qlnx_pci_attach_err; for (i = 0; i < ha->num_rss; i++) { ha->irq_vec[i].rss_idx = i; ha->irq_vec[i].ha = ha; ha->irq_vec[i].irq_rid = (1 + num_sp_msix) + i; ha->irq_vec[i].irq = bus_alloc_resource_any(dev, SYS_RES_IRQ, &ha->irq_vec[i].irq_rid, (RF_ACTIVE | RF_SHAREABLE)); if (ha->irq_vec[i].irq == NULL) { device_printf(dev, "could not allocate interrupt[%d] irq_rid = %d\n", i, ha->irq_vec[i].irq_rid); goto qlnx_pci_attach_err; } if (qlnx_alloc_tx_br(ha, &ha->fp_array[i])) { device_printf(dev, "could not allocate tx_br[%d]\n", i); goto qlnx_pci_attach_err; } } if (qlnx_vf_device(ha) != 0) { callout_init(&ha->qlnx_callout, 1); ha->flags.callout_init = 1; for (i = 0; i < ha->cdev.num_hwfns; i++) { if (qlnx_grc_dumpsize(ha, &ha->grcdump_size[i], i) != 0) goto qlnx_pci_attach_err; if (ha->grcdump_size[i] == 0) goto qlnx_pci_attach_err; ha->grcdump_size[i] = ha->grcdump_size[i] << 2; QL_DPRINT1(ha, "grcdump_size[%d] = 0x%08x\n", i, ha->grcdump_size[i]); ha->grcdump[i] = qlnx_zalloc(ha->grcdump_size[i]); if (ha->grcdump[i] == NULL) { device_printf(dev, "grcdump alloc[%d] failed\n", i); goto qlnx_pci_attach_err; } if (qlnx_idle_chk_size(ha, &ha->idle_chk_size[i], i) != 0) goto qlnx_pci_attach_err; if (ha->idle_chk_size[i] == 0) goto qlnx_pci_attach_err; ha->idle_chk_size[i] = ha->idle_chk_size[i] << 2; QL_DPRINT1(ha, "idle_chk_size[%d] = 0x%08x\n", i, ha->idle_chk_size[i]); ha->idle_chk[i] = qlnx_zalloc(ha->idle_chk_size[i]); if (ha->idle_chk[i] == NULL) { device_printf(dev, "idle_chk alloc failed\n"); goto qlnx_pci_attach_err; } } if (qlnx_create_error_recovery_taskqueue(ha) != 0) goto qlnx_pci_attach_err; } if (qlnx_slowpath_start(ha) != 0) goto qlnx_pci_attach_err; else ha->flags.slowpath_start = 1; if (qlnx_vf_device(ha) != 0) { if (qlnx_get_flash_size(ha, &ha->flash_size) != 0) { qlnx_mdelay(__func__, 1000); qlnx_trigger_dump(ha); goto qlnx_pci_attach_err0; } if (qlnx_get_mfw_version(ha, &mfw_ver) != 0) { qlnx_mdelay(__func__, 1000); qlnx_trigger_dump(ha); goto qlnx_pci_attach_err0; } } else { struct ecore_hwfn *p_hwfn = &ha->cdev.hwfns[0]; ecore_mcp_get_mfw_ver(p_hwfn, NULL, &mfw_ver, NULL); } snprintf(ha->mfw_ver, sizeof(ha->mfw_ver), "%d.%d.%d.%d", ((mfw_ver >> 24) & 0xFF), ((mfw_ver >> 16) & 0xFF), ((mfw_ver >> 8) & 0xFF), (mfw_ver & 0xFF)); snprintf(ha->stormfw_ver, sizeof(ha->stormfw_ver), "%d.%d.%d.%d", FW_MAJOR_VERSION, FW_MINOR_VERSION, FW_REVISION_VERSION, FW_ENGINEERING_VERSION); QL_DPRINT1(ha, "STORM_FW version %s MFW version %s\n", ha->stormfw_ver, ha->mfw_ver); qlnx_init_ifnet(dev, ha); /* * add sysctls */ qlnx_add_sysctls(ha); qlnx_pci_attach_err0: /* * create ioctl device interface */ if (qlnx_vf_device(ha) != 0) { if (qlnx_make_cdev(ha)) { device_printf(dev, "%s: ql_make_cdev failed\n", __func__); goto qlnx_pci_attach_err; } #ifdef QLNX_ENABLE_IWARP qlnx_rdma_dev_add(ha); #endif /* #ifdef QLNX_ENABLE_IWARP */ } #ifndef QLNX_VF #ifdef CONFIG_ECORE_SRIOV if (qlnx_vf_device(ha) != 0) qlnx_initialize_sriov(ha); #endif /* #ifdef CONFIG_ECORE_SRIOV */ #endif /* #ifdef QLNX_VF */ QL_DPRINT2(ha, "success\n"); return (0); qlnx_pci_attach_err: qlnx_release(ha); return (ENXIO); } /* * Name: qlnx_pci_detach * Function: Unhooks the device from the operating system */ static int qlnx_pci_detach(device_t dev) { qlnx_host_t *ha = NULL; if ((ha = device_get_softc(dev)) == NULL) { device_printf(dev, "%s: cannot get softc\n", __func__); return (ENOMEM); } if (qlnx_vf_device(ha) != 0) { #ifdef CONFIG_ECORE_SRIOV int ret; ret = pci_iov_detach(dev); if (ret) { device_printf(dev, "%s: SRIOV in use\n", __func__); return (ret); } #endif /* #ifdef CONFIG_ECORE_SRIOV */ #ifdef QLNX_ENABLE_IWARP if (qlnx_rdma_dev_remove(ha) != 0) return (EBUSY); #endif /* #ifdef QLNX_ENABLE_IWARP */ } QLNX_LOCK(ha); qlnx_stop(ha); QLNX_UNLOCK(ha); qlnx_release(ha); return (0); } #ifdef QLNX_ENABLE_IWARP static uint8_t qlnx_get_personality(uint8_t pci_func) { uint8_t personality; personality = (qlnxe_rdma_configuration >> (pci_func * QLNX_PERSONALITY_BITS_PER_FUNC)) & QLNX_PERSONALIY_MASK; return (personality); } static void qlnx_set_personality(qlnx_host_t *ha) { uint8_t personality; personality = qlnx_get_personality(ha->pci_func); switch (personality) { case QLNX_PERSONALITY_DEFAULT: device_printf(ha->pci_dev, "%s: DEFAULT\n", __func__); ha->personality = ECORE_PCI_DEFAULT; break; case QLNX_PERSONALITY_ETH_ONLY: device_printf(ha->pci_dev, "%s: ETH_ONLY\n", __func__); ha->personality = ECORE_PCI_ETH; break; case QLNX_PERSONALITY_ETH_IWARP: device_printf(ha->pci_dev, "%s: ETH_IWARP\n", __func__); ha->personality = ECORE_PCI_ETH_IWARP; break; case QLNX_PERSONALITY_ETH_ROCE: device_printf(ha->pci_dev, "%s: ETH_ROCE\n", __func__); ha->personality = ECORE_PCI_ETH_ROCE; break; } return; } #endif /* #ifdef QLNX_ENABLE_IWARP */ static int qlnx_init_hw(qlnx_host_t *ha) { int rval = 0; struct ecore_hw_prepare_params params; ecore_init_struct(&ha->cdev); /* ha->dp_module = ECORE_MSG_PROBE | ECORE_MSG_INTR | ECORE_MSG_SP | ECORE_MSG_LINK | ECORE_MSG_SPQ | ECORE_MSG_RDMA; ha->dp_level = ECORE_LEVEL_VERBOSE;*/ //ha->dp_module = ECORE_MSG_RDMA | ECORE_MSG_INTR | ECORE_MSG_LL2; ha->dp_level = ECORE_LEVEL_NOTICE; //ha->dp_level = ECORE_LEVEL_VERBOSE; ecore_init_dp(&ha->cdev, ha->dp_module, ha->dp_level, ha->pci_dev); ha->cdev.regview = ha->pci_reg; ha->personality = ECORE_PCI_DEFAULT; if (qlnx_vf_device(ha) == 0) { ha->cdev.b_is_vf = true; if (ha->pci_dbells != NULL) { ha->cdev.doorbells = ha->pci_dbells; ha->cdev.db_phys_addr = ha->dbells_phys_addr; ha->cdev.db_size = ha->dbells_size; } else { ha->pci_dbells = ha->pci_reg; } } else { ha->cdev.doorbells = ha->pci_dbells; ha->cdev.db_phys_addr = ha->dbells_phys_addr; ha->cdev.db_size = ha->dbells_size; #ifdef QLNX_ENABLE_IWARP if (qlnx_rdma_supported(ha) == 0) qlnx_set_personality(ha); #endif /* #ifdef QLNX_ENABLE_IWARP */ } QL_DPRINT2(ha, "%s: %s\n", __func__, (ha->personality == ECORE_PCI_ETH_IWARP ? "iwarp": "ethernet")); bzero(¶ms, sizeof (struct ecore_hw_prepare_params)); params.personality = ha->personality; params.drv_resc_alloc = false; params.chk_reg_fifo = false; params.initiate_pf_flr = true; params.epoch = 0; ecore_hw_prepare(&ha->cdev, ¶ms); qlnx_set_id(&ha->cdev, qlnx_name_str, qlnx_ver_str); QL_DPRINT1(ha, "ha = %p cdev = %p p_hwfn = %p\n", ha, &ha->cdev, &ha->cdev.hwfns[0]); return (rval); } static void qlnx_release(qlnx_host_t *ha) { device_t dev; int i; dev = ha->pci_dev; QL_DPRINT2(ha, "enter\n"); for (i = 0; i < QLNX_MAX_HW_FUNCS; i++) { if (ha->idle_chk[i] != NULL) { free(ha->idle_chk[i], M_QLNXBUF); ha->idle_chk[i] = NULL; } if (ha->grcdump[i] != NULL) { free(ha->grcdump[i], M_QLNXBUF); ha->grcdump[i] = NULL; } } if (ha->flags.callout_init) callout_drain(&ha->qlnx_callout); if (ha->flags.slowpath_start) { qlnx_slowpath_stop(ha); } if (ha->flags.hw_init) ecore_hw_remove(&ha->cdev); qlnx_del_cdev(ha); if (ha->ifp != NULL) ether_ifdetach(ha->ifp); qlnx_free_tx_dma_tag(ha); qlnx_free_rx_dma_tag(ha); qlnx_free_parent_dma_tag(ha); if (qlnx_vf_device(ha) != 0) { qlnx_destroy_error_recovery_taskqueue(ha); } for (i = 0; i < ha->num_rss; i++) { struct qlnx_fastpath *fp = &ha->fp_array[i]; if (ha->irq_vec[i].handle) { (void)bus_teardown_intr(dev, ha->irq_vec[i].irq, ha->irq_vec[i].handle); } if (ha->irq_vec[i].irq) { (void)bus_release_resource(dev, SYS_RES_IRQ, ha->irq_vec[i].irq_rid, ha->irq_vec[i].irq); } qlnx_free_tx_br(ha, fp); } qlnx_destroy_fp_taskqueues(ha); for (i = 0; i < ha->cdev.num_hwfns; i++) { if (ha->sp_handle[i]) (void)bus_teardown_intr(dev, ha->sp_irq[i], ha->sp_handle[i]); if (ha->sp_irq[i]) (void) bus_release_resource(dev, SYS_RES_IRQ, ha->sp_irq_rid[i], ha->sp_irq[i]); } qlnx_destroy_sp_taskqueues(ha); if (ha->msix_count) pci_release_msi(dev); if (ha->flags.lock_init) { mtx_destroy(&ha->hw_lock); } if (ha->pci_reg) (void) bus_release_resource(dev, SYS_RES_MEMORY, ha->reg_rid, ha->pci_reg); if (ha->dbells_size && ha->pci_dbells) (void) bus_release_resource(dev, SYS_RES_MEMORY, ha->dbells_rid, ha->pci_dbells); if (ha->msix_bar) (void) bus_release_resource(dev, SYS_RES_MEMORY, ha->msix_rid, ha->msix_bar); QL_DPRINT2(ha, "exit\n"); return; } static void qlnx_trigger_dump(qlnx_host_t *ha) { int i; if (ha->ifp != NULL) - ha->ifp->if_drv_flags &= ~(IFF_DRV_OACTIVE | IFF_DRV_RUNNING); + if_setdrvflagbits(ha->ifp, 0, (IFF_DRV_OACTIVE | IFF_DRV_RUNNING)); QL_DPRINT2(ha, "enter\n"); if (qlnx_vf_device(ha) == 0) return; ha->error_recovery = 1; for (i = 0; i < ha->cdev.num_hwfns; i++) { qlnx_grc_dump(ha, &ha->grcdump_dwords[i], i); qlnx_idle_chk(ha, &ha->idle_chk_dwords[i], i); } QL_DPRINT2(ha, "exit\n"); return; } static int qlnx_trigger_dump_sysctl(SYSCTL_HANDLER_ARGS) { int err, ret = 0; qlnx_host_t *ha; err = sysctl_handle_int(oidp, &ret, 0, req); if (err || !req->newptr) return (err); if (ret == 1) { ha = (qlnx_host_t *)arg1; qlnx_trigger_dump(ha); } return (err); } static int qlnx_set_tx_coalesce(SYSCTL_HANDLER_ARGS) { int err, i, ret = 0, usecs = 0; qlnx_host_t *ha; struct ecore_hwfn *p_hwfn; struct qlnx_fastpath *fp; err = sysctl_handle_int(oidp, &usecs, 0, req); if (err || !req->newptr || !usecs || (usecs > 255)) return (err); ha = (qlnx_host_t *)arg1; if (qlnx_vf_device(ha) == 0) return (-1); for (i = 0; i < ha->num_rss; i++) { p_hwfn = &ha->cdev.hwfns[(i % ha->cdev.num_hwfns)]; fp = &ha->fp_array[i]; if (fp->txq[0]->handle != NULL) { ret = ecore_set_queue_coalesce(p_hwfn, 0, (uint16_t)usecs, fp->txq[0]->handle); } } if (!ret) ha->tx_coalesce_usecs = (uint8_t)usecs; return (err); } static int qlnx_set_rx_coalesce(SYSCTL_HANDLER_ARGS) { int err, i, ret = 0, usecs = 0; qlnx_host_t *ha; struct ecore_hwfn *p_hwfn; struct qlnx_fastpath *fp; err = sysctl_handle_int(oidp, &usecs, 0, req); if (err || !req->newptr || !usecs || (usecs > 255)) return (err); ha = (qlnx_host_t *)arg1; if (qlnx_vf_device(ha) == 0) return (-1); for (i = 0; i < ha->num_rss; i++) { p_hwfn = &ha->cdev.hwfns[(i % ha->cdev.num_hwfns)]; fp = &ha->fp_array[i]; if (fp->rxq->handle != NULL) { ret = ecore_set_queue_coalesce(p_hwfn, (uint16_t)usecs, 0, fp->rxq->handle); } } if (!ret) ha->rx_coalesce_usecs = (uint8_t)usecs; return (err); } static void qlnx_add_sp_stats_sysctls(qlnx_host_t *ha) { struct sysctl_ctx_list *ctx; struct sysctl_oid_list *children; struct sysctl_oid *ctx_oid; ctx = device_get_sysctl_ctx(ha->pci_dev); children = SYSCTL_CHILDREN(device_get_sysctl_tree(ha->pci_dev)); ctx_oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "spstat", CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "spstat"); children = SYSCTL_CHILDREN(ctx_oid); SYSCTL_ADD_QUAD(ctx, children, OID_AUTO, "sp_interrupts", CTLFLAG_RD, &ha->sp_interrupts, "No. of slowpath interrupts"); return; } static void qlnx_add_fp_stats_sysctls(qlnx_host_t *ha) { struct sysctl_ctx_list *ctx; struct sysctl_oid_list *children; struct sysctl_oid_list *node_children; struct sysctl_oid *ctx_oid; int i, j; uint8_t name_str[16]; ctx = device_get_sysctl_ctx(ha->pci_dev); children = SYSCTL_CHILDREN(device_get_sysctl_tree(ha->pci_dev)); ctx_oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "fpstat", CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "fpstat"); children = SYSCTL_CHILDREN(ctx_oid); for (i = 0; i < ha->num_rss; i++) { bzero(name_str, (sizeof(uint8_t) * sizeof(name_str))); snprintf(name_str, sizeof(name_str), "%d", i); ctx_oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, name_str, CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, name_str); node_children = SYSCTL_CHILDREN(ctx_oid); /* Tx Related */ SYSCTL_ADD_QUAD(ctx, node_children, OID_AUTO, "tx_pkts_processed", CTLFLAG_RD, &ha->fp_array[i].tx_pkts_processed, "No. of packets processed for transmission"); SYSCTL_ADD_QUAD(ctx, node_children, OID_AUTO, "tx_pkts_freed", CTLFLAG_RD, &ha->fp_array[i].tx_pkts_freed, "No. of freed packets"); SYSCTL_ADD_QUAD(ctx, node_children, OID_AUTO, "tx_pkts_transmitted", CTLFLAG_RD, &ha->fp_array[i].tx_pkts_transmitted, "No. of transmitted packets"); SYSCTL_ADD_QUAD(ctx, node_children, OID_AUTO, "tx_pkts_completed", CTLFLAG_RD, &ha->fp_array[i].tx_pkts_completed, "No. of transmit completions"); SYSCTL_ADD_QUAD(ctx, node_children, OID_AUTO, "tx_non_tso_pkts", CTLFLAG_RD, &ha->fp_array[i].tx_non_tso_pkts, "No. of non LSO transmited packets"); #ifdef QLNX_TRACE_PERF_DATA SYSCTL_ADD_QUAD(ctx, node_children, OID_AUTO, "tx_pkts_trans_ctx", CTLFLAG_RD, &ha->fp_array[i].tx_pkts_trans_ctx, "No. of transmitted packets in transmit context"); SYSCTL_ADD_QUAD(ctx, node_children, OID_AUTO, "tx_pkts_compl_ctx", CTLFLAG_RD, &ha->fp_array[i].tx_pkts_compl_ctx, "No. of transmit completions in transmit context"); SYSCTL_ADD_QUAD(ctx, node_children, OID_AUTO, "tx_pkts_trans_fp", CTLFLAG_RD, &ha->fp_array[i].tx_pkts_trans_fp, "No. of transmitted packets in taskqueue"); SYSCTL_ADD_QUAD(ctx, node_children, OID_AUTO, "tx_pkts_compl_fp", CTLFLAG_RD, &ha->fp_array[i].tx_pkts_compl_fp, "No. of transmit completions in taskqueue"); SYSCTL_ADD_QUAD(ctx, node_children, OID_AUTO, "tx_pkts_compl_intr", CTLFLAG_RD, &ha->fp_array[i].tx_pkts_compl_intr, "No. of transmit completions in interrupt ctx"); #endif SYSCTL_ADD_QUAD(ctx, node_children, OID_AUTO, "tx_tso_pkts", CTLFLAG_RD, &ha->fp_array[i].tx_tso_pkts, "No. of LSO transmited packets"); SYSCTL_ADD_QUAD(ctx, node_children, OID_AUTO, "tx_lso_wnd_min_len", CTLFLAG_RD, &ha->fp_array[i].tx_lso_wnd_min_len, "tx_lso_wnd_min_len"); SYSCTL_ADD_QUAD(ctx, node_children, OID_AUTO, "tx_defrag", CTLFLAG_RD, &ha->fp_array[i].tx_defrag, "tx_defrag"); SYSCTL_ADD_QUAD(ctx, node_children, OID_AUTO, "tx_nsegs_gt_elem_left", CTLFLAG_RD, &ha->fp_array[i].tx_nsegs_gt_elem_left, "tx_nsegs_gt_elem_left"); SYSCTL_ADD_UINT(ctx, node_children, OID_AUTO, "tx_tso_max_nsegs", CTLFLAG_RD, &ha->fp_array[i].tx_tso_max_nsegs, ha->fp_array[i].tx_tso_max_nsegs, "tx_tso_max_nsegs"); SYSCTL_ADD_UINT(ctx, node_children, OID_AUTO, "tx_tso_min_nsegs", CTLFLAG_RD, &ha->fp_array[i].tx_tso_min_nsegs, ha->fp_array[i].tx_tso_min_nsegs, "tx_tso_min_nsegs"); SYSCTL_ADD_UINT(ctx, node_children, OID_AUTO, "tx_tso_max_pkt_len", CTLFLAG_RD, &ha->fp_array[i].tx_tso_max_pkt_len, ha->fp_array[i].tx_tso_max_pkt_len, "tx_tso_max_pkt_len"); SYSCTL_ADD_UINT(ctx, node_children, OID_AUTO, "tx_tso_min_pkt_len", CTLFLAG_RD, &ha->fp_array[i].tx_tso_min_pkt_len, ha->fp_array[i].tx_tso_min_pkt_len, "tx_tso_min_pkt_len"); for (j = 0; j < QLNX_FP_MAX_SEGS; j++) { bzero(name_str, (sizeof(uint8_t) * sizeof(name_str))); snprintf(name_str, sizeof(name_str), "tx_pkts_nseg_%02d", (j+1)); SYSCTL_ADD_QUAD(ctx, node_children, OID_AUTO, name_str, CTLFLAG_RD, &ha->fp_array[i].tx_pkts[j], name_str); } #ifdef QLNX_TRACE_PERF_DATA for (j = 0; j < 18; j++) { bzero(name_str, (sizeof(uint8_t) * sizeof(name_str))); snprintf(name_str, sizeof(name_str), "tx_pkts_hist_%02d", (j+1)); SYSCTL_ADD_QUAD(ctx, node_children, OID_AUTO, name_str, CTLFLAG_RD, &ha->fp_array[i].tx_pkts_hist[j], name_str); } for (j = 0; j < 5; j++) { bzero(name_str, (sizeof(uint8_t) * sizeof(name_str))); snprintf(name_str, sizeof(name_str), "tx_comInt_%02d", (j+1)); SYSCTL_ADD_QUAD(ctx, node_children, OID_AUTO, name_str, CTLFLAG_RD, &ha->fp_array[i].tx_comInt[j], name_str); } for (j = 0; j < 18; j++) { bzero(name_str, (sizeof(uint8_t) * sizeof(name_str))); snprintf(name_str, sizeof(name_str), "tx_pkts_q_%02d", (j+1)); SYSCTL_ADD_QUAD(ctx, node_children, OID_AUTO, name_str, CTLFLAG_RD, &ha->fp_array[i].tx_pkts_q[j], name_str); } #endif SYSCTL_ADD_QUAD(ctx, node_children, OID_AUTO, "err_tx_nsegs_gt_elem_left", CTLFLAG_RD, &ha->fp_array[i].err_tx_nsegs_gt_elem_left, "err_tx_nsegs_gt_elem_left"); SYSCTL_ADD_QUAD(ctx, node_children, OID_AUTO, "err_tx_dmamap_create", CTLFLAG_RD, &ha->fp_array[i].err_tx_dmamap_create, "err_tx_dmamap_create"); SYSCTL_ADD_QUAD(ctx, node_children, OID_AUTO, "err_tx_defrag_dmamap_load", CTLFLAG_RD, &ha->fp_array[i].err_tx_defrag_dmamap_load, "err_tx_defrag_dmamap_load"); SYSCTL_ADD_QUAD(ctx, node_children, OID_AUTO, "err_tx_non_tso_max_seg", CTLFLAG_RD, &ha->fp_array[i].err_tx_non_tso_max_seg, "err_tx_non_tso_max_seg"); SYSCTL_ADD_QUAD(ctx, node_children, OID_AUTO, "err_tx_dmamap_load", CTLFLAG_RD, &ha->fp_array[i].err_tx_dmamap_load, "err_tx_dmamap_load"); SYSCTL_ADD_QUAD(ctx, node_children, OID_AUTO, "err_tx_defrag", CTLFLAG_RD, &ha->fp_array[i].err_tx_defrag, "err_tx_defrag"); SYSCTL_ADD_QUAD(ctx, node_children, OID_AUTO, "err_tx_free_pkt_null", CTLFLAG_RD, &ha->fp_array[i].err_tx_free_pkt_null, "err_tx_free_pkt_null"); SYSCTL_ADD_QUAD(ctx, node_children, OID_AUTO, "err_tx_cons_idx_conflict", CTLFLAG_RD, &ha->fp_array[i].err_tx_cons_idx_conflict, "err_tx_cons_idx_conflict"); SYSCTL_ADD_QUAD(ctx, node_children, OID_AUTO, "lro_cnt_64", CTLFLAG_RD, &ha->fp_array[i].lro_cnt_64, "lro_cnt_64"); SYSCTL_ADD_QUAD(ctx, node_children, OID_AUTO, "lro_cnt_128", CTLFLAG_RD, &ha->fp_array[i].lro_cnt_128, "lro_cnt_128"); SYSCTL_ADD_QUAD(ctx, node_children, OID_AUTO, "lro_cnt_256", CTLFLAG_RD, &ha->fp_array[i].lro_cnt_256, "lro_cnt_256"); SYSCTL_ADD_QUAD(ctx, node_children, OID_AUTO, "lro_cnt_512", CTLFLAG_RD, &ha->fp_array[i].lro_cnt_512, "lro_cnt_512"); SYSCTL_ADD_QUAD(ctx, node_children, OID_AUTO, "lro_cnt_1024", CTLFLAG_RD, &ha->fp_array[i].lro_cnt_1024, "lro_cnt_1024"); /* Rx Related */ SYSCTL_ADD_QUAD(ctx, node_children, OID_AUTO, "rx_pkts", CTLFLAG_RD, &ha->fp_array[i].rx_pkts, "No. of received packets"); SYSCTL_ADD_QUAD(ctx, node_children, OID_AUTO, "tpa_start", CTLFLAG_RD, &ha->fp_array[i].tpa_start, "No. of tpa_start packets"); SYSCTL_ADD_QUAD(ctx, node_children, OID_AUTO, "tpa_cont", CTLFLAG_RD, &ha->fp_array[i].tpa_cont, "No. of tpa_cont packets"); SYSCTL_ADD_QUAD(ctx, node_children, OID_AUTO, "tpa_end", CTLFLAG_RD, &ha->fp_array[i].tpa_end, "No. of tpa_end packets"); SYSCTL_ADD_QUAD(ctx, node_children, OID_AUTO, "err_m_getcl", CTLFLAG_RD, &ha->fp_array[i].err_m_getcl, "err_m_getcl"); SYSCTL_ADD_QUAD(ctx, node_children, OID_AUTO, "err_m_getjcl", CTLFLAG_RD, &ha->fp_array[i].err_m_getjcl, "err_m_getjcl"); SYSCTL_ADD_QUAD(ctx, node_children, OID_AUTO, "err_rx_hw_errors", CTLFLAG_RD, &ha->fp_array[i].err_rx_hw_errors, "err_rx_hw_errors"); SYSCTL_ADD_QUAD(ctx, node_children, OID_AUTO, "err_rx_alloc_errors", CTLFLAG_RD, &ha->fp_array[i].err_rx_alloc_errors, "err_rx_alloc_errors"); } return; } static void qlnx_add_hw_stats_sysctls(qlnx_host_t *ha) { struct sysctl_ctx_list *ctx; struct sysctl_oid_list *children; struct sysctl_oid *ctx_oid; ctx = device_get_sysctl_ctx(ha->pci_dev); children = SYSCTL_CHILDREN(device_get_sysctl_tree(ha->pci_dev)); ctx_oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "hwstat", CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "hwstat"); children = SYSCTL_CHILDREN(ctx_oid); SYSCTL_ADD_QUAD(ctx, children, OID_AUTO, "no_buff_discards", CTLFLAG_RD, &ha->hw_stats.common.no_buff_discards, "No. of packets discarded due to lack of buffer"); SYSCTL_ADD_QUAD(ctx, children, OID_AUTO, "packet_too_big_discard", CTLFLAG_RD, &ha->hw_stats.common.packet_too_big_discard, "No. of packets discarded because packet was too big"); SYSCTL_ADD_QUAD(ctx, children, OID_AUTO, "ttl0_discard", CTLFLAG_RD, &ha->hw_stats.common.ttl0_discard, "ttl0_discard"); SYSCTL_ADD_QUAD(ctx, children, OID_AUTO, "rx_ucast_bytes", CTLFLAG_RD, &ha->hw_stats.common.rx_ucast_bytes, "rx_ucast_bytes"); SYSCTL_ADD_QUAD(ctx, children, OID_AUTO, "rx_mcast_bytes", CTLFLAG_RD, &ha->hw_stats.common.rx_mcast_bytes, "rx_mcast_bytes"); SYSCTL_ADD_QUAD(ctx, children, OID_AUTO, "rx_bcast_bytes", CTLFLAG_RD, &ha->hw_stats.common.rx_bcast_bytes, "rx_bcast_bytes"); SYSCTL_ADD_QUAD(ctx, children, OID_AUTO, "rx_ucast_pkts", CTLFLAG_RD, &ha->hw_stats.common.rx_ucast_pkts, "rx_ucast_pkts"); SYSCTL_ADD_QUAD(ctx, children, OID_AUTO, "rx_mcast_pkts", CTLFLAG_RD, &ha->hw_stats.common.rx_mcast_pkts, "rx_mcast_pkts"); SYSCTL_ADD_QUAD(ctx, children, OID_AUTO, "rx_bcast_pkts", CTLFLAG_RD, &ha->hw_stats.common.rx_bcast_pkts, "rx_bcast_pkts"); SYSCTL_ADD_QUAD(ctx, children, OID_AUTO, "mftag_filter_discards", CTLFLAG_RD, &ha->hw_stats.common.mftag_filter_discards, "mftag_filter_discards"); SYSCTL_ADD_QUAD(ctx, children, OID_AUTO, "mac_filter_discards", CTLFLAG_RD, &ha->hw_stats.common.mac_filter_discards, "mac_filter_discards"); SYSCTL_ADD_QUAD(ctx, children, OID_AUTO, "tx_ucast_bytes", CTLFLAG_RD, &ha->hw_stats.common.tx_ucast_bytes, "tx_ucast_bytes"); SYSCTL_ADD_QUAD(ctx, children, OID_AUTO, "tx_mcast_bytes", CTLFLAG_RD, &ha->hw_stats.common.tx_mcast_bytes, "tx_mcast_bytes"); SYSCTL_ADD_QUAD(ctx, children, OID_AUTO, "tx_bcast_bytes", CTLFLAG_RD, &ha->hw_stats.common.tx_bcast_bytes, "tx_bcast_bytes"); SYSCTL_ADD_QUAD(ctx, children, OID_AUTO, "tx_ucast_pkts", CTLFLAG_RD, &ha->hw_stats.common.tx_ucast_pkts, "tx_ucast_pkts"); SYSCTL_ADD_QUAD(ctx, children, OID_AUTO, "tx_mcast_pkts", CTLFLAG_RD, &ha->hw_stats.common.tx_mcast_pkts, "tx_mcast_pkts"); SYSCTL_ADD_QUAD(ctx, children, OID_AUTO, "tx_bcast_pkts", CTLFLAG_RD, &ha->hw_stats.common.tx_bcast_pkts, "tx_bcast_pkts"); SYSCTL_ADD_QUAD(ctx, children, OID_AUTO, "tx_err_drop_pkts", CTLFLAG_RD, &ha->hw_stats.common.tx_err_drop_pkts, "tx_err_drop_pkts"); SYSCTL_ADD_QUAD(ctx, children, OID_AUTO, "tpa_coalesced_pkts", CTLFLAG_RD, &ha->hw_stats.common.tpa_coalesced_pkts, "tpa_coalesced_pkts"); SYSCTL_ADD_QUAD(ctx, children, OID_AUTO, "tpa_coalesced_events", CTLFLAG_RD, &ha->hw_stats.common.tpa_coalesced_events, "tpa_coalesced_events"); SYSCTL_ADD_QUAD(ctx, children, OID_AUTO, "tpa_aborts_num", CTLFLAG_RD, &ha->hw_stats.common.tpa_aborts_num, "tpa_aborts_num"); SYSCTL_ADD_QUAD(ctx, children, OID_AUTO, "tpa_not_coalesced_pkts", CTLFLAG_RD, &ha->hw_stats.common.tpa_not_coalesced_pkts, "tpa_not_coalesced_pkts"); SYSCTL_ADD_QUAD(ctx, children, OID_AUTO, "tpa_coalesced_bytes", CTLFLAG_RD, &ha->hw_stats.common.tpa_coalesced_bytes, "tpa_coalesced_bytes"); SYSCTL_ADD_QUAD(ctx, children, OID_AUTO, "rx_64_byte_packets", CTLFLAG_RD, &ha->hw_stats.common.rx_64_byte_packets, "rx_64_byte_packets"); SYSCTL_ADD_QUAD(ctx, children, OID_AUTO, "rx_65_to_127_byte_packets", CTLFLAG_RD, &ha->hw_stats.common.rx_65_to_127_byte_packets, "rx_65_to_127_byte_packets"); SYSCTL_ADD_QUAD(ctx, children, OID_AUTO, "rx_128_to_255_byte_packets", CTLFLAG_RD, &ha->hw_stats.common.rx_128_to_255_byte_packets, "rx_128_to_255_byte_packets"); SYSCTL_ADD_QUAD(ctx, children, OID_AUTO, "rx_256_to_511_byte_packets", CTLFLAG_RD, &ha->hw_stats.common.rx_256_to_511_byte_packets, "rx_256_to_511_byte_packets"); SYSCTL_ADD_QUAD(ctx, children, OID_AUTO, "rx_512_to_1023_byte_packets", CTLFLAG_RD, &ha->hw_stats.common.rx_512_to_1023_byte_packets, "rx_512_to_1023_byte_packets"); SYSCTL_ADD_QUAD(ctx, children, OID_AUTO, "rx_1024_to_1518_byte_packets", CTLFLAG_RD, &ha->hw_stats.common.rx_1024_to_1518_byte_packets, "rx_1024_to_1518_byte_packets"); SYSCTL_ADD_QUAD(ctx, children, OID_AUTO, "rx_1519_to_1522_byte_packets", CTLFLAG_RD, &ha->hw_stats.bb.rx_1519_to_1522_byte_packets, "rx_1519_to_1522_byte_packets"); SYSCTL_ADD_QUAD(ctx, children, OID_AUTO, "rx_1523_to_2047_byte_packets", CTLFLAG_RD, &ha->hw_stats.bb.rx_1519_to_2047_byte_packets, "rx_1523_to_2047_byte_packets"); SYSCTL_ADD_QUAD(ctx, children, OID_AUTO, "rx_2048_to_4095_byte_packets", CTLFLAG_RD, &ha->hw_stats.bb.rx_2048_to_4095_byte_packets, "rx_2048_to_4095_byte_packets"); SYSCTL_ADD_QUAD(ctx, children, OID_AUTO, "rx_4096_to_9216_byte_packets", CTLFLAG_RD, &ha->hw_stats.bb.rx_4096_to_9216_byte_packets, "rx_4096_to_9216_byte_packets"); SYSCTL_ADD_QUAD(ctx, children, OID_AUTO, "rx_9217_to_16383_byte_packets", CTLFLAG_RD, &ha->hw_stats.bb.rx_9217_to_16383_byte_packets, "rx_9217_to_16383_byte_packets"); SYSCTL_ADD_QUAD(ctx, children, OID_AUTO, "rx_crc_errors", CTLFLAG_RD, &ha->hw_stats.common.rx_crc_errors, "rx_crc_errors"); SYSCTL_ADD_QUAD(ctx, children, OID_AUTO, "rx_mac_crtl_frames", CTLFLAG_RD, &ha->hw_stats.common.rx_mac_crtl_frames, "rx_mac_crtl_frames"); SYSCTL_ADD_QUAD(ctx, children, OID_AUTO, "rx_pause_frames", CTLFLAG_RD, &ha->hw_stats.common.rx_pause_frames, "rx_pause_frames"); SYSCTL_ADD_QUAD(ctx, children, OID_AUTO, "rx_pfc_frames", CTLFLAG_RD, &ha->hw_stats.common.rx_pfc_frames, "rx_pfc_frames"); SYSCTL_ADD_QUAD(ctx, children, OID_AUTO, "rx_align_errors", CTLFLAG_RD, &ha->hw_stats.common.rx_align_errors, "rx_align_errors"); SYSCTL_ADD_QUAD(ctx, children, OID_AUTO, "rx_carrier_errors", CTLFLAG_RD, &ha->hw_stats.common.rx_carrier_errors, "rx_carrier_errors"); SYSCTL_ADD_QUAD(ctx, children, OID_AUTO, "rx_oversize_packets", CTLFLAG_RD, &ha->hw_stats.common.rx_oversize_packets, "rx_oversize_packets"); SYSCTL_ADD_QUAD(ctx, children, OID_AUTO, "rx_jabbers", CTLFLAG_RD, &ha->hw_stats.common.rx_jabbers, "rx_jabbers"); SYSCTL_ADD_QUAD(ctx, children, OID_AUTO, "rx_undersize_packets", CTLFLAG_RD, &ha->hw_stats.common.rx_undersize_packets, "rx_undersize_packets"); SYSCTL_ADD_QUAD(ctx, children, OID_AUTO, "rx_fragments", CTLFLAG_RD, &ha->hw_stats.common.rx_fragments, "rx_fragments"); SYSCTL_ADD_QUAD(ctx, children, OID_AUTO, "tx_64_byte_packets", CTLFLAG_RD, &ha->hw_stats.common.tx_64_byte_packets, "tx_64_byte_packets"); SYSCTL_ADD_QUAD(ctx, children, OID_AUTO, "tx_65_to_127_byte_packets", CTLFLAG_RD, &ha->hw_stats.common.tx_65_to_127_byte_packets, "tx_65_to_127_byte_packets"); SYSCTL_ADD_QUAD(ctx, children, OID_AUTO, "tx_128_to_255_byte_packets", CTLFLAG_RD, &ha->hw_stats.common.tx_128_to_255_byte_packets, "tx_128_to_255_byte_packets"); SYSCTL_ADD_QUAD(ctx, children, OID_AUTO, "tx_256_to_511_byte_packets", CTLFLAG_RD, &ha->hw_stats.common.tx_256_to_511_byte_packets, "tx_256_to_511_byte_packets"); SYSCTL_ADD_QUAD(ctx, children, OID_AUTO, "tx_512_to_1023_byte_packets", CTLFLAG_RD, &ha->hw_stats.common.tx_512_to_1023_byte_packets, "tx_512_to_1023_byte_packets"); SYSCTL_ADD_QUAD(ctx, children, OID_AUTO, "tx_1024_to_1518_byte_packets", CTLFLAG_RD, &ha->hw_stats.common.tx_1024_to_1518_byte_packets, "tx_1024_to_1518_byte_packets"); SYSCTL_ADD_QUAD(ctx, children, OID_AUTO, "tx_1519_to_2047_byte_packets", CTLFLAG_RD, &ha->hw_stats.bb.tx_1519_to_2047_byte_packets, "tx_1519_to_2047_byte_packets"); SYSCTL_ADD_QUAD(ctx, children, OID_AUTO, "tx_2048_to_4095_byte_packets", CTLFLAG_RD, &ha->hw_stats.bb.tx_2048_to_4095_byte_packets, "tx_2048_to_4095_byte_packets"); SYSCTL_ADD_QUAD(ctx, children, OID_AUTO, "tx_4096_to_9216_byte_packets", CTLFLAG_RD, &ha->hw_stats.bb.tx_4096_to_9216_byte_packets, "tx_4096_to_9216_byte_packets"); SYSCTL_ADD_QUAD(ctx, children, OID_AUTO, "tx_9217_to_16383_byte_packets", CTLFLAG_RD, &ha->hw_stats.bb.tx_9217_to_16383_byte_packets, "tx_9217_to_16383_byte_packets"); SYSCTL_ADD_QUAD(ctx, children, OID_AUTO, "tx_pause_frames", CTLFLAG_RD, &ha->hw_stats.common.tx_pause_frames, "tx_pause_frames"); SYSCTL_ADD_QUAD(ctx, children, OID_AUTO, "tx_pfc_frames", CTLFLAG_RD, &ha->hw_stats.common.tx_pfc_frames, "tx_pfc_frames"); SYSCTL_ADD_QUAD(ctx, children, OID_AUTO, "tx_lpi_entry_count", CTLFLAG_RD, &ha->hw_stats.bb.tx_lpi_entry_count, "tx_lpi_entry_count"); SYSCTL_ADD_QUAD(ctx, children, OID_AUTO, "tx_total_collisions", CTLFLAG_RD, &ha->hw_stats.bb.tx_total_collisions, "tx_total_collisions"); SYSCTL_ADD_QUAD(ctx, children, OID_AUTO, "brb_truncates", CTLFLAG_RD, &ha->hw_stats.common.brb_truncates, "brb_truncates"); SYSCTL_ADD_QUAD(ctx, children, OID_AUTO, "brb_discards", CTLFLAG_RD, &ha->hw_stats.common.brb_discards, "brb_discards"); SYSCTL_ADD_QUAD(ctx, children, OID_AUTO, "rx_mac_bytes", CTLFLAG_RD, &ha->hw_stats.common.rx_mac_bytes, "rx_mac_bytes"); SYSCTL_ADD_QUAD(ctx, children, OID_AUTO, "rx_mac_uc_packets", CTLFLAG_RD, &ha->hw_stats.common.rx_mac_uc_packets, "rx_mac_uc_packets"); SYSCTL_ADD_QUAD(ctx, children, OID_AUTO, "rx_mac_mc_packets", CTLFLAG_RD, &ha->hw_stats.common.rx_mac_mc_packets, "rx_mac_mc_packets"); SYSCTL_ADD_QUAD(ctx, children, OID_AUTO, "rx_mac_bc_packets", CTLFLAG_RD, &ha->hw_stats.common.rx_mac_bc_packets, "rx_mac_bc_packets"); SYSCTL_ADD_QUAD(ctx, children, OID_AUTO, "rx_mac_frames_ok", CTLFLAG_RD, &ha->hw_stats.common.rx_mac_frames_ok, "rx_mac_frames_ok"); SYSCTL_ADD_QUAD(ctx, children, OID_AUTO, "tx_mac_bytes", CTLFLAG_RD, &ha->hw_stats.common.tx_mac_bytes, "tx_mac_bytes"); SYSCTL_ADD_QUAD(ctx, children, OID_AUTO, "tx_mac_uc_packets", CTLFLAG_RD, &ha->hw_stats.common.tx_mac_uc_packets, "tx_mac_uc_packets"); SYSCTL_ADD_QUAD(ctx, children, OID_AUTO, "tx_mac_mc_packets", CTLFLAG_RD, &ha->hw_stats.common.tx_mac_mc_packets, "tx_mac_mc_packets"); SYSCTL_ADD_QUAD(ctx, children, OID_AUTO, "tx_mac_bc_packets", CTLFLAG_RD, &ha->hw_stats.common.tx_mac_bc_packets, "tx_mac_bc_packets"); SYSCTL_ADD_QUAD(ctx, children, OID_AUTO, "tx_mac_ctrl_frames", CTLFLAG_RD, &ha->hw_stats.common.tx_mac_ctrl_frames, "tx_mac_ctrl_frames"); return; } static void qlnx_add_sysctls(qlnx_host_t *ha) { device_t dev = ha->pci_dev; struct sysctl_ctx_list *ctx; struct sysctl_oid_list *children; ctx = device_get_sysctl_ctx(dev); children = SYSCTL_CHILDREN(device_get_sysctl_tree(dev)); qlnx_add_fp_stats_sysctls(ha); qlnx_add_sp_stats_sysctls(ha); if (qlnx_vf_device(ha) != 0) qlnx_add_hw_stats_sysctls(ha); SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "Driver_Version", CTLFLAG_RD, qlnx_ver_str, 0, "Driver Version"); SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "STORMFW_Version", CTLFLAG_RD, ha->stormfw_ver, 0, "STORM Firmware Version"); SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "MFW_Version", CTLFLAG_RD, ha->mfw_ver, 0, "Management Firmware Version"); SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "personality", CTLFLAG_RD, &ha->personality, ha->personality, "\tpersonality = 0 => Ethernet Only\n" "\tpersonality = 3 => Ethernet and RoCE\n" "\tpersonality = 4 => Ethernet and iWARP\n" "\tpersonality = 6 => Default in Shared Memory\n"); ha->dbg_level = 0; SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "debug", CTLFLAG_RW, &ha->dbg_level, ha->dbg_level, "Debug Level"); ha->dp_level = 0x01; SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "dp_level", CTLFLAG_RW, &ha->dp_level, ha->dp_level, "DP Level"); ha->dbg_trace_lro_cnt = 0; SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "dbg_trace_lro_cnt", CTLFLAG_RW, &ha->dbg_trace_lro_cnt, ha->dbg_trace_lro_cnt, "Trace LRO Counts"); ha->dbg_trace_tso_pkt_len = 0; SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "dbg_trace_tso_pkt_len", CTLFLAG_RW, &ha->dbg_trace_tso_pkt_len, ha->dbg_trace_tso_pkt_len, "Trace TSO packet lengths"); ha->dp_module = 0; SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "dp_module", CTLFLAG_RW, &ha->dp_module, ha->dp_module, "DP Module"); ha->err_inject = 0; SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "err_inject", CTLFLAG_RW, &ha->err_inject, ha->err_inject, "Error Inject"); ha->storm_stats_enable = 0; SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "storm_stats_enable", CTLFLAG_RW, &ha->storm_stats_enable, ha->storm_stats_enable, "Enable Storm Statistics Gathering"); ha->storm_stats_index = 0; SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "storm_stats_index", CTLFLAG_RD, &ha->storm_stats_index, ha->storm_stats_index, "Enable Storm Statistics Gathering Current Index"); ha->grcdump_taken = 0; SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "grcdump_taken", CTLFLAG_RD, &ha->grcdump_taken, ha->grcdump_taken, "grcdump_taken"); ha->idle_chk_taken = 0; SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "idle_chk_taken", CTLFLAG_RD, &ha->idle_chk_taken, ha->idle_chk_taken, "idle_chk_taken"); SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "rx_coalesce_usecs", CTLFLAG_RD, &ha->rx_coalesce_usecs, ha->rx_coalesce_usecs, "rx_coalesce_usecs"); SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "tx_coalesce_usecs", CTLFLAG_RD, &ha->tx_coalesce_usecs, ha->tx_coalesce_usecs, "tx_coalesce_usecs"); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "trigger_dump", CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_NEEDGIANT, (void *)ha, 0, qlnx_trigger_dump_sysctl, "I", "trigger_dump"); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "set_rx_coalesce_usecs", CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_NEEDGIANT, (void *)ha, 0, qlnx_set_rx_coalesce, "I", "rx interrupt coalesce period microseconds"); SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "set_tx_coalesce_usecs", CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_NEEDGIANT, (void *)ha, 0, qlnx_set_tx_coalesce, "I", "tx interrupt coalesce period microseconds"); ha->rx_pkt_threshold = 128; SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "rx_pkt_threshold", CTLFLAG_RW, &ha->rx_pkt_threshold, ha->rx_pkt_threshold, "No. of Rx Pkts to process at a time"); ha->rx_jumbo_buf_eq_mtu = 0; SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "rx_jumbo_buf_eq_mtu", CTLFLAG_RW, &ha->rx_jumbo_buf_eq_mtu, ha->rx_jumbo_buf_eq_mtu, "== 0 => Rx Jumbo buffers are capped to 4Kbytes\n" "otherwise Rx Jumbo buffers are set to >= MTU size\n"); SYSCTL_ADD_QUAD(ctx, children, OID_AUTO, "err_illegal_intr", CTLFLAG_RD, &ha->err_illegal_intr, "err_illegal_intr"); SYSCTL_ADD_QUAD(ctx, children, OID_AUTO, "err_fp_null", CTLFLAG_RD, &ha->err_fp_null, "err_fp_null"); SYSCTL_ADD_QUAD(ctx, children, OID_AUTO, "err_get_proto_invalid_type", CTLFLAG_RD, &ha->err_get_proto_invalid_type, "err_get_proto_invalid_type"); return; } /***************************************************************************** * Operating System Network Interface Functions *****************************************************************************/ static void qlnx_init_ifnet(device_t dev, qlnx_host_t *ha) { uint16_t device_id; - struct ifnet *ifp; + if_t ifp; ifp = ha->ifp = if_alloc(IFT_ETHER); if (ifp == NULL) panic("%s: cannot if_alloc()\n", device_get_nameunit(dev)); if_initname(ifp, device_get_name(dev), device_get_unit(dev)); device_id = pci_get_device(ha->pci_dev); if (device_id == QLOGIC_PCI_DEVICE_ID_1634) - ifp->if_baudrate = IF_Gbps(40); + if_setbaudrate(ifp, IF_Gbps(40)); else if ((device_id == QLOGIC_PCI_DEVICE_ID_1656) || (device_id == QLOGIC_PCI_DEVICE_ID_8070)) - ifp->if_baudrate = IF_Gbps(25); + if_setbaudrate(ifp, IF_Gbps(25)); else if (device_id == QLOGIC_PCI_DEVICE_ID_1654) - ifp->if_baudrate = IF_Gbps(50); + if_setbaudrate(ifp, IF_Gbps(50)); else if (device_id == QLOGIC_PCI_DEVICE_ID_1644) - ifp->if_baudrate = IF_Gbps(100); + if_setbaudrate(ifp, IF_Gbps(100)); - ifp->if_capabilities = IFCAP_LINKSTATE; + if_setcapabilities(ifp, IFCAP_LINKSTATE); - ifp->if_init = qlnx_init; - ifp->if_softc = ha; - ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; - ifp->if_ioctl = qlnx_ioctl; - ifp->if_transmit = qlnx_transmit; - ifp->if_qflush = qlnx_qflush; + if_setinitfn(ifp, qlnx_init); + if_setsoftc(ifp, ha); + if_setflags(ifp, IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST); + if_setioctlfn(ifp, qlnx_ioctl); + if_settransmitfn(ifp, qlnx_transmit); + if_setqflushfn(ifp, qlnx_qflush); - IFQ_SET_MAXLEN(&ifp->if_snd, qlnx_get_ifq_snd_maxlen(ha)); - ifp->if_snd.ifq_drv_maxlen = qlnx_get_ifq_snd_maxlen(ha); - IFQ_SET_READY(&ifp->if_snd); + if_setsendqlen(ifp, qlnx_get_ifq_snd_maxlen(ha)); + if_setsendqready(ifp); if_setgetcounterfn(ifp, qlnx_get_counter); - ha->max_frame_size = ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN; + ha->max_frame_size = if_getmtu(ifp) + ETHER_HDR_LEN + ETHER_CRC_LEN; memcpy(ha->primary_mac, qlnx_get_mac_addr(ha), ETH_ALEN); if (!ha->primary_mac[0] && !ha->primary_mac[1] && !ha->primary_mac[2] && !ha->primary_mac[3] && !ha->primary_mac[4] && !ha->primary_mac[5]) { uint32_t rnd; rnd = arc4random(); ha->primary_mac[0] = 0x00; ha->primary_mac[1] = 0x0e; ha->primary_mac[2] = 0x1e; ha->primary_mac[3] = rnd & 0xFF; ha->primary_mac[4] = (rnd >> 8) & 0xFF; ha->primary_mac[5] = (rnd >> 16) & 0xFF; } ether_ifattach(ifp, ha->primary_mac); - bcopy(IF_LLADDR(ha->ifp), ha->primary_mac, ETHER_ADDR_LEN); + bcopy(if_getlladdr(ha->ifp), ha->primary_mac, ETHER_ADDR_LEN); - ifp->if_capabilities = IFCAP_HWCSUM; - ifp->if_capabilities |= IFCAP_JUMBO_MTU; + if_setcapabilities(ifp, IFCAP_HWCSUM); + if_setcapabilitiesbit(ifp, IFCAP_JUMBO_MTU, 0); - ifp->if_capabilities |= IFCAP_VLAN_MTU; - ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING; - ifp->if_capabilities |= IFCAP_VLAN_HWFILTER; - ifp->if_capabilities |= IFCAP_VLAN_HWCSUM; - ifp->if_capabilities |= IFCAP_VLAN_HWTSO; - ifp->if_capabilities |= IFCAP_TSO4; - ifp->if_capabilities |= IFCAP_TSO6; - ifp->if_capabilities |= IFCAP_LRO; + if_setcapabilitiesbit(ifp, IFCAP_VLAN_MTU, 0); + if_setcapabilitiesbit(ifp, IFCAP_VLAN_HWTAGGING, 0); + if_setcapabilitiesbit(ifp, IFCAP_VLAN_HWFILTER, 0); + if_setcapabilitiesbit(ifp, IFCAP_VLAN_HWCSUM, 0); + if_setcapabilitiesbit(ifp, IFCAP_VLAN_HWTSO, 0); + if_setcapabilitiesbit(ifp, IFCAP_TSO4, 0); + if_setcapabilitiesbit(ifp, IFCAP_TSO6, 0); + if_setcapabilitiesbit(ifp, IFCAP_LRO, 0); - ifp->if_hw_tsomax = QLNX_MAX_TSO_FRAME_SIZE - - (ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN); - ifp->if_hw_tsomaxsegcount = QLNX_MAX_SEGMENTS - 1 /* hdr */; - ifp->if_hw_tsomaxsegsize = QLNX_MAX_TX_MBUF_SIZE; + if_sethwtsomax(ifp, QLNX_MAX_TSO_FRAME_SIZE - + (ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN)); + if_sethwtsomaxsegcount(ifp, QLNX_MAX_SEGMENTS - 1); /* hdr */ + if_sethwtsomaxsegsize(ifp, QLNX_MAX_TX_MBUF_SIZE); - ifp->if_capenable = ifp->if_capabilities; + if_setcapenable(ifp, if_getcapabilities(ifp)); - ifp->if_hwassist = CSUM_IP; - ifp->if_hwassist |= CSUM_TCP | CSUM_UDP; - ifp->if_hwassist |= CSUM_TCP_IPV6 | CSUM_UDP_IPV6; - ifp->if_hwassist |= CSUM_TSO; + if_sethwassist(ifp, CSUM_IP); + if_sethwassistbits(ifp, CSUM_TCP | CSUM_UDP, 0); + if_sethwassistbits(ifp, CSUM_TCP_IPV6 | CSUM_UDP_IPV6, 0); + if_sethwassistbits(ifp, CSUM_TSO, 0); - ifp->if_hdrlen = sizeof(struct ether_vlan_header); + if_setifheaderlen(ifp, sizeof(struct ether_vlan_header)); ifmedia_init(&ha->media, IFM_IMASK, qlnx_media_change,\ qlnx_media_status); if (device_id == QLOGIC_PCI_DEVICE_ID_1634) { ifmedia_add(&ha->media, (IFM_ETHER | IFM_40G_LR4), 0, NULL); ifmedia_add(&ha->media, (IFM_ETHER | IFM_40G_SR4), 0, NULL); ifmedia_add(&ha->media, (IFM_ETHER | IFM_40G_CR4), 0, NULL); } else if ((device_id == QLOGIC_PCI_DEVICE_ID_1656) || (device_id == QLOGIC_PCI_DEVICE_ID_8070)) { ifmedia_add(&ha->media, (IFM_ETHER | QLNX_IFM_25G_SR), 0, NULL); ifmedia_add(&ha->media, (IFM_ETHER | QLNX_IFM_25G_CR), 0, NULL); } else if (device_id == QLOGIC_PCI_DEVICE_ID_1654) { ifmedia_add(&ha->media, (IFM_ETHER | IFM_50G_KR2), 0, NULL); ifmedia_add(&ha->media, (IFM_ETHER | IFM_50G_CR2), 0, NULL); } else if (device_id == QLOGIC_PCI_DEVICE_ID_1644) { ifmedia_add(&ha->media, (IFM_ETHER | QLNX_IFM_100G_LR4), 0, NULL); ifmedia_add(&ha->media, (IFM_ETHER | QLNX_IFM_100G_SR4), 0, NULL); ifmedia_add(&ha->media, (IFM_ETHER | QLNX_IFM_100G_CR4), 0, NULL); } ifmedia_add(&ha->media, (IFM_ETHER | IFM_FDX), 0, NULL); ifmedia_add(&ha->media, (IFM_ETHER | IFM_AUTO), 0, NULL); ifmedia_set(&ha->media, (IFM_ETHER | IFM_AUTO)); QL_DPRINT2(ha, "exit\n"); return; } static void qlnx_init_locked(qlnx_host_t *ha) { - struct ifnet *ifp = ha->ifp; + if_t ifp = ha->ifp; QL_DPRINT1(ha, "Driver Initialization start \n"); qlnx_stop(ha); if (qlnx_load(ha) == 0) { - ifp->if_drv_flags |= IFF_DRV_RUNNING; - ifp->if_drv_flags &= ~IFF_DRV_OACTIVE; + if_setdrvflagbits(ifp, IFF_DRV_RUNNING, 0); + if_setdrvflagbits(ifp, 0, IFF_DRV_OACTIVE); #ifdef QLNX_ENABLE_IWARP if (qlnx_vf_device(ha) != 0) { qlnx_rdma_dev_open(ha); } #endif /* #ifdef QLNX_ENABLE_IWARP */ } return; } static void qlnx_init(void *arg) { qlnx_host_t *ha; ha = (qlnx_host_t *)arg; QL_DPRINT2(ha, "enter\n"); QLNX_LOCK(ha); qlnx_init_locked(ha); QLNX_UNLOCK(ha); QL_DPRINT2(ha, "exit\n"); return; } static int qlnx_config_mcast_mac_addr(qlnx_host_t *ha, uint8_t *mac_addr, uint32_t add_mac) { struct ecore_filter_mcast *mcast; struct ecore_dev *cdev; int rc; cdev = &ha->cdev; mcast = &ha->ecore_mcast; bzero(mcast, sizeof(struct ecore_filter_mcast)); if (add_mac) mcast->opcode = ECORE_FILTER_ADD; else mcast->opcode = ECORE_FILTER_REMOVE; mcast->num_mc_addrs = 1; memcpy(mcast->mac, mac_addr, ETH_ALEN); rc = ecore_filter_mcast_cmd(cdev, mcast, ECORE_SPQ_MODE_CB, NULL); return (rc); } static int qlnx_hw_add_mcast(qlnx_host_t *ha, uint8_t *mta) { int i; for (i = 0; i < QLNX_MAX_NUM_MULTICAST_ADDRS; i++) { if (QL_MAC_CMP(ha->mcast[i].addr, mta) == 0) return 0; /* its been already added */ } for (i = 0; i < QLNX_MAX_NUM_MULTICAST_ADDRS; i++) { if ((ha->mcast[i].addr[0] == 0) && (ha->mcast[i].addr[1] == 0) && (ha->mcast[i].addr[2] == 0) && (ha->mcast[i].addr[3] == 0) && (ha->mcast[i].addr[4] == 0) && (ha->mcast[i].addr[5] == 0)) { if (qlnx_config_mcast_mac_addr(ha, mta, 1)) return (-1); bcopy(mta, ha->mcast[i].addr, ETH_ALEN); ha->nmcast++; return 0; } } return 0; } static int qlnx_hw_del_mcast(qlnx_host_t *ha, uint8_t *mta) { int i; for (i = 0; i < QLNX_MAX_NUM_MULTICAST_ADDRS; i++) { if (QL_MAC_CMP(ha->mcast[i].addr, mta) == 0) { if (qlnx_config_mcast_mac_addr(ha, mta, 0)) return (-1); ha->mcast[i].addr[0] = 0; ha->mcast[i].addr[1] = 0; ha->mcast[i].addr[2] = 0; ha->mcast[i].addr[3] = 0; ha->mcast[i].addr[4] = 0; ha->mcast[i].addr[5] = 0; ha->nmcast--; return 0; } } return 0; } /* * Name: qls_hw_set_multi * Function: Sets the Multicast Addresses provided the host O.S into the * hardware (for the given interface) */ static void qlnx_hw_set_multi(qlnx_host_t *ha, uint8_t *mta, uint32_t mcnt, uint32_t add_mac) { int i; for (i = 0; i < mcnt; i++) { if (add_mac) { if (qlnx_hw_add_mcast(ha, mta)) break; } else { if (qlnx_hw_del_mcast(ha, mta)) break; } mta += ETHER_HDR_LEN; } return; } static u_int qlnx_copy_maddr(void *arg, struct sockaddr_dl *sdl, u_int mcnt) { uint8_t *mta = arg; if (mcnt == QLNX_MAX_NUM_MULTICAST_ADDRS) return (0); bcopy(LLADDR(sdl), &mta[mcnt * ETHER_HDR_LEN], ETHER_HDR_LEN); return (1); } static int qlnx_set_multi(qlnx_host_t *ha, uint32_t add_multi) { uint8_t mta[QLNX_MAX_NUM_MULTICAST_ADDRS * ETHER_HDR_LEN]; - struct ifnet *ifp = ha->ifp; + if_t ifp = ha->ifp; u_int mcnt; if (qlnx_vf_device(ha) == 0) return (0); mcnt = if_foreach_llmaddr(ifp, qlnx_copy_maddr, mta); QLNX_LOCK(ha); qlnx_hw_set_multi(ha, mta, mcnt, add_multi); QLNX_UNLOCK(ha); return (0); } static int qlnx_set_promisc(qlnx_host_t *ha) { int rc = 0; uint8_t filter; if (qlnx_vf_device(ha) == 0) return (0); filter = ha->filter; filter |= ECORE_ACCEPT_MCAST_UNMATCHED; filter |= ECORE_ACCEPT_UCAST_UNMATCHED; rc = qlnx_set_rx_accept_filter(ha, filter); return (rc); } static int qlnx_set_allmulti(qlnx_host_t *ha) { int rc = 0; uint8_t filter; if (qlnx_vf_device(ha) == 0) return (0); filter = ha->filter; filter |= ECORE_ACCEPT_MCAST_UNMATCHED; rc = qlnx_set_rx_accept_filter(ha, filter); return (rc); } static int -qlnx_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data) +qlnx_ioctl(if_t ifp, u_long cmd, caddr_t data) { int ret = 0, mask; struct ifreq *ifr = (struct ifreq *)data; struct ifaddr *ifa = (struct ifaddr *)data; qlnx_host_t *ha; - ha = (qlnx_host_t *)ifp->if_softc; + ha = (qlnx_host_t *)if_getsoftc(ifp); switch (cmd) { case SIOCSIFADDR: QL_DPRINT4(ha, "SIOCSIFADDR (0x%lx)\n", cmd); if (ifa->ifa_addr->sa_family == AF_INET) { - ifp->if_flags |= IFF_UP; - if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) { + if_setflagbits(ifp, IFF_UP, 0); + if (!(if_getdrvflags(ifp) & IFF_DRV_RUNNING)) { QLNX_LOCK(ha); qlnx_init_locked(ha); QLNX_UNLOCK(ha); } QL_DPRINT4(ha, "SIOCSIFADDR (0x%lx) ipv4 [0x%08x]\n", cmd, ntohl(IA_SIN(ifa)->sin_addr.s_addr)); arp_ifinit(ifp, ifa); } else { ether_ioctl(ifp, cmd, data); } break; case SIOCSIFMTU: QL_DPRINT4(ha, "SIOCSIFMTU (0x%lx)\n", cmd); if (ifr->ifr_mtu > QLNX_MAX_MTU) { ret = EINVAL; } else { QLNX_LOCK(ha); - ifp->if_mtu = ifr->ifr_mtu; + if_setmtu(ifp, ifr->ifr_mtu); ha->max_frame_size = - ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN; - if (ifp->if_drv_flags & IFF_DRV_RUNNING) { + if_getmtu(ifp) + ETHER_HDR_LEN + ETHER_CRC_LEN; + if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) { qlnx_init_locked(ha); } QLNX_UNLOCK(ha); } break; case SIOCSIFFLAGS: QL_DPRINT4(ha, "SIOCSIFFLAGS (0x%lx)\n", cmd); QLNX_LOCK(ha); - if (ifp->if_flags & IFF_UP) { - if (ifp->if_drv_flags & IFF_DRV_RUNNING) { - if ((ifp->if_flags ^ ha->if_flags) & + if (if_getflags(ifp) & IFF_UP) { + if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) { + if ((if_getflags(ifp) ^ ha->if_flags) & IFF_PROMISC) { ret = qlnx_set_promisc(ha); - } else if ((ifp->if_flags ^ ha->if_flags) & + } else if ((if_getflags(ifp) ^ ha->if_flags) & IFF_ALLMULTI) { ret = qlnx_set_allmulti(ha); } } else { - ha->max_frame_size = ifp->if_mtu + + ha->max_frame_size = if_getmtu(ifp) + ETHER_HDR_LEN + ETHER_CRC_LEN; qlnx_init_locked(ha); } } else { - if (ifp->if_drv_flags & IFF_DRV_RUNNING) + if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) qlnx_stop(ha); - ha->if_flags = ifp->if_flags; + ha->if_flags = if_getflags(ifp); } QLNX_UNLOCK(ha); break; case SIOCADDMULTI: QL_DPRINT4(ha, "%s (0x%lx)\n", "SIOCADDMULTI", cmd); - if (ifp->if_drv_flags & IFF_DRV_RUNNING) { + if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) { if (qlnx_set_multi(ha, 1)) ret = EINVAL; } break; case SIOCDELMULTI: QL_DPRINT4(ha, "%s (0x%lx)\n", "SIOCDELMULTI", cmd); - if (ifp->if_drv_flags & IFF_DRV_RUNNING) { + if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) { if (qlnx_set_multi(ha, 0)) ret = EINVAL; } break; case SIOCSIFMEDIA: case SIOCGIFMEDIA: QL_DPRINT4(ha, "SIOCSIFMEDIA/SIOCGIFMEDIA (0x%lx)\n", cmd); ret = ifmedia_ioctl(ifp, ifr, &ha->media, cmd); break; case SIOCSIFCAP: - mask = ifr->ifr_reqcap ^ ifp->if_capenable; + mask = ifr->ifr_reqcap ^ if_getcapenable(ifp); QL_DPRINT4(ha, "SIOCSIFCAP (0x%lx)\n", cmd); if (mask & IFCAP_HWCSUM) - ifp->if_capenable ^= IFCAP_HWCSUM; + if_togglecapenable(ifp, IFCAP_HWCSUM); if (mask & IFCAP_TSO4) - ifp->if_capenable ^= IFCAP_TSO4; + if_togglecapenable(ifp, IFCAP_TSO4); if (mask & IFCAP_TSO6) - ifp->if_capenable ^= IFCAP_TSO6; + if_togglecapenable(ifp, IFCAP_TSO6); if (mask & IFCAP_VLAN_HWTAGGING) - ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING; + if_togglecapenable(ifp, IFCAP_VLAN_HWTAGGING); if (mask & IFCAP_VLAN_HWTSO) - ifp->if_capenable ^= IFCAP_VLAN_HWTSO; + if_togglecapenable(ifp, IFCAP_VLAN_HWTSO); if (mask & IFCAP_LRO) - ifp->if_capenable ^= IFCAP_LRO; + if_togglecapenable(ifp, IFCAP_LRO); QLNX_LOCK(ha); - if (ifp->if_drv_flags & IFF_DRV_RUNNING) + if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) qlnx_init_locked(ha); QLNX_UNLOCK(ha); VLAN_CAPABILITIES(ifp); break; case SIOCGI2C: { struct ifi2creq i2c; struct ecore_hwfn *p_hwfn = &ha->cdev.hwfns[0]; struct ecore_ptt *p_ptt; ret = copyin(ifr_data_get_ptr(ifr), &i2c, sizeof(i2c)); if (ret) break; if ((i2c.len > sizeof (i2c.data)) || (i2c.dev_addr != 0xA0 && i2c.dev_addr != 0xA2)) { ret = EINVAL; break; } p_ptt = ecore_ptt_acquire(p_hwfn); if (!p_ptt) { QL_DPRINT1(ha, "ecore_ptt_acquire failed\n"); ret = -1; break; } ret = ecore_mcp_phy_sfp_read(p_hwfn, p_ptt, (ha->pci_func & 0x1), i2c.dev_addr, i2c.offset, i2c.len, &i2c.data[0]); ecore_ptt_release(p_hwfn, p_ptt); if (ret) { ret = -1; break; } ret = copyout(&i2c, ifr_data_get_ptr(ifr), sizeof(i2c)); QL_DPRINT8(ha, "SIOCGI2C copyout ret = %d \ len = %d addr = 0x%02x offset = 0x%04x \ data[0..7]=0x%02x 0x%02x 0x%02x 0x%02x 0x%02x \ 0x%02x 0x%02x 0x%02x\n", ret, i2c.len, i2c.dev_addr, i2c.offset, i2c.data[0], i2c.data[1], i2c.data[2], i2c.data[3], i2c.data[4], i2c.data[5], i2c.data[6], i2c.data[7]); break; } default: QL_DPRINT4(ha, "default (0x%lx)\n", cmd); ret = ether_ioctl(ifp, cmd, data); break; } return (ret); } static int -qlnx_media_change(struct ifnet *ifp) +qlnx_media_change(if_t ifp) { qlnx_host_t *ha; struct ifmedia *ifm; int ret = 0; - ha = (qlnx_host_t *)ifp->if_softc; + ha = (qlnx_host_t *)if_getsoftc(ifp); QL_DPRINT2(ha, "enter\n"); ifm = &ha->media; if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER) ret = EINVAL; QL_DPRINT2(ha, "exit\n"); return (ret); } static void -qlnx_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) +qlnx_media_status(if_t ifp, struct ifmediareq *ifmr) { qlnx_host_t *ha; - ha = (qlnx_host_t *)ifp->if_softc; + ha = (qlnx_host_t *)if_getsoftc(ifp); QL_DPRINT2(ha, "enter\n"); ifmr->ifm_status = IFM_AVALID; ifmr->ifm_active = IFM_ETHER; if (ha->link_up) { ifmr->ifm_status |= IFM_ACTIVE; ifmr->ifm_active |= (IFM_FDX | qlnx_get_optics(ha, &ha->if_link)); if (ha->if_link.link_partner_caps & (QLNX_LINK_CAP_Pause | QLNX_LINK_CAP_Asym_Pause)) ifmr->ifm_active |= (IFM_ETH_RXPAUSE | IFM_ETH_TXPAUSE); } QL_DPRINT2(ha, "exit (%s)\n", (ha->link_up ? "link_up" : "link_down")); return; } static void qlnx_free_tx_pkt(qlnx_host_t *ha, struct qlnx_fastpath *fp, struct qlnx_tx_queue *txq) { u16 idx; struct mbuf *mp; bus_dmamap_t map; int i; // struct eth_tx_bd *tx_data_bd; struct eth_tx_1st_bd *first_bd; int nbds = 0; idx = txq->sw_tx_cons; mp = txq->sw_tx_ring[idx].mp; map = txq->sw_tx_ring[idx].map; if ((mp == NULL) || QL_ERR_INJECT(ha, QL_ERR_INJCT_TX_INT_MBUF_NULL)){ QL_RESET_ERR_INJECT(ha, QL_ERR_INJCT_TX_INT_MBUF_NULL); QL_DPRINT1(ha, "(mp == NULL) " " tx_idx = 0x%x" " ecore_prod_idx = 0x%x" " ecore_cons_idx = 0x%x" " hw_bd_cons = 0x%x" " txq_db_last = 0x%x" " elem_left = 0x%x\n", fp->rss_id, ecore_chain_get_prod_idx(&txq->tx_pbl), ecore_chain_get_cons_idx(&txq->tx_pbl), le16toh(*txq->hw_cons_ptr), txq->tx_db.raw, ecore_chain_get_elem_left(&txq->tx_pbl)); fp->err_tx_free_pkt_null++; //DEBUG qlnx_trigger_dump(ha); return; } else { QLNX_INC_OPACKETS((ha->ifp)); QLNX_INC_OBYTES((ha->ifp), (mp->m_pkthdr.len)); bus_dmamap_sync(ha->tx_tag, map, BUS_DMASYNC_POSTWRITE); bus_dmamap_unload(ha->tx_tag, map); fp->tx_pkts_freed++; fp->tx_pkts_completed++; m_freem(mp); } first_bd = (struct eth_tx_1st_bd *)ecore_chain_consume(&txq->tx_pbl); nbds = first_bd->data.nbds; // BD_SET_UNMAP_ADDR_LEN(first_bd, 0, 0); for (i = 1; i < nbds; i++) { /* tx_data_bd = */ ecore_chain_consume(&txq->tx_pbl); // BD_SET_UNMAP_ADDR_LEN(tx_data_bd, 0, 0); } txq->sw_tx_ring[idx].flags = 0; txq->sw_tx_ring[idx].mp = NULL; txq->sw_tx_ring[idx].map = (bus_dmamap_t)0; return; } static void qlnx_tx_int(qlnx_host_t *ha, struct qlnx_fastpath *fp, struct qlnx_tx_queue *txq) { u16 hw_bd_cons; u16 ecore_cons_idx; uint16_t diff; uint16_t idx, idx2; hw_bd_cons = le16toh(*txq->hw_cons_ptr); while (hw_bd_cons != (ecore_cons_idx = ecore_chain_get_cons_idx(&txq->tx_pbl))) { if (hw_bd_cons < ecore_cons_idx) { diff = (1 << 16) - (ecore_cons_idx - hw_bd_cons); } else { diff = hw_bd_cons - ecore_cons_idx; } if ((diff > TX_RING_SIZE) || QL_ERR_INJECT(ha, QL_ERR_INJCT_TX_INT_DIFF)){ QL_RESET_ERR_INJECT(ha, QL_ERR_INJCT_TX_INT_DIFF); QL_DPRINT1(ha, "(diff = 0x%x) " " tx_idx = 0x%x" " ecore_prod_idx = 0x%x" " ecore_cons_idx = 0x%x" " hw_bd_cons = 0x%x" " txq_db_last = 0x%x" " elem_left = 0x%x\n", diff, fp->rss_id, ecore_chain_get_prod_idx(&txq->tx_pbl), ecore_chain_get_cons_idx(&txq->tx_pbl), le16toh(*txq->hw_cons_ptr), txq->tx_db.raw, ecore_chain_get_elem_left(&txq->tx_pbl)); fp->err_tx_cons_idx_conflict++; //DEBUG qlnx_trigger_dump(ha); } idx = (txq->sw_tx_cons + 1) & (TX_RING_SIZE - 1); idx2 = (txq->sw_tx_cons + 2) & (TX_RING_SIZE - 1); prefetch(txq->sw_tx_ring[idx].mp); prefetch(txq->sw_tx_ring[idx2].mp); qlnx_free_tx_pkt(ha, fp, txq); txq->sw_tx_cons = (txq->sw_tx_cons + 1) & (TX_RING_SIZE - 1); } return; } static int -qlnx_transmit_locked(struct ifnet *ifp,struct qlnx_fastpath *fp, struct mbuf *mp) +qlnx_transmit_locked(if_t ifp, struct qlnx_fastpath *fp, struct mbuf *mp) { int ret = 0; struct qlnx_tx_queue *txq; qlnx_host_t * ha; uint16_t elem_left; txq = fp->txq[0]; ha = (qlnx_host_t *)fp->edev; - if ((!(ifp->if_drv_flags & IFF_DRV_RUNNING)) || (!ha->link_up)) { + if ((!(if_getdrvflags(ifp) & IFF_DRV_RUNNING)) || (!ha->link_up)) { if(mp != NULL) ret = drbr_enqueue(ifp, fp->tx_br, mp); return (ret); } if(mp != NULL) ret = drbr_enqueue(ifp, fp->tx_br, mp); mp = drbr_peek(ifp, fp->tx_br); while (mp != NULL) { if (qlnx_send(ha, fp, &mp)) { if (mp != NULL) { drbr_putback(ifp, fp->tx_br, mp); } else { fp->tx_pkts_processed++; drbr_advance(ifp, fp->tx_br); } goto qlnx_transmit_locked_exit; } else { drbr_advance(ifp, fp->tx_br); fp->tx_pkts_transmitted++; fp->tx_pkts_processed++; } mp = drbr_peek(ifp, fp->tx_br); } qlnx_transmit_locked_exit: if((qlnx_num_tx_compl(ha,fp, fp->txq[0]) > QLNX_TX_COMPL_THRESH) || ((int)(elem_left = ecore_chain_get_elem_left(&txq->tx_pbl)) < QLNX_TX_ELEM_MAX_THRESH)) (void)qlnx_tx_int(ha, fp, fp->txq[0]); QL_DPRINT2(ha, "%s: exit ret = %d\n", __func__, ret); return ret; } static int -qlnx_transmit(struct ifnet *ifp, struct mbuf *mp) +qlnx_transmit(if_t ifp, struct mbuf *mp) { - qlnx_host_t *ha = (qlnx_host_t *)ifp->if_softc; + qlnx_host_t *ha = (qlnx_host_t *)if_getsoftc(ifp); struct qlnx_fastpath *fp; int rss_id = 0, ret = 0; #ifdef QLNX_TRACEPERF_DATA uint64_t tx_pkts = 0, tx_compl = 0; #endif QL_DPRINT2(ha, "enter\n"); if (M_HASHTYPE_GET(mp) != M_HASHTYPE_NONE) rss_id = (mp->m_pkthdr.flowid % ECORE_RSS_IND_TABLE_SIZE) % ha->num_rss; fp = &ha->fp_array[rss_id]; if (fp->tx_br == NULL) { ret = EINVAL; goto qlnx_transmit_exit; } if (mtx_trylock(&fp->tx_mtx)) { #ifdef QLNX_TRACEPERF_DATA tx_pkts = fp->tx_pkts_transmitted; tx_compl = fp->tx_pkts_completed; #endif ret = qlnx_transmit_locked(ifp, fp, mp); #ifdef QLNX_TRACEPERF_DATA fp->tx_pkts_trans_ctx += (fp->tx_pkts_transmitted - tx_pkts); fp->tx_pkts_compl_ctx += (fp->tx_pkts_completed - tx_compl); #endif mtx_unlock(&fp->tx_mtx); } else { if (mp != NULL && (fp->fp_taskqueue != NULL)) { ret = drbr_enqueue(ifp, fp->tx_br, mp); taskqueue_enqueue(fp->fp_taskqueue, &fp->fp_task); } } qlnx_transmit_exit: QL_DPRINT2(ha, "exit ret = %d\n", ret); return ret; } static void -qlnx_qflush(struct ifnet *ifp) +qlnx_qflush(if_t ifp) { int rss_id; struct qlnx_fastpath *fp; struct mbuf *mp; qlnx_host_t *ha; - ha = (qlnx_host_t *)ifp->if_softc; + ha = (qlnx_host_t *)if_getsoftc(ifp); QL_DPRINT2(ha, "enter\n"); for (rss_id = 0; rss_id < ha->num_rss; rss_id++) { fp = &ha->fp_array[rss_id]; if (fp == NULL) continue; if (fp->tx_br) { mtx_lock(&fp->tx_mtx); while ((mp = drbr_dequeue(ifp, fp->tx_br)) != NULL) { fp->tx_pkts_freed++; m_freem(mp); } mtx_unlock(&fp->tx_mtx); } } QL_DPRINT2(ha, "exit\n"); return; } static void qlnx_txq_doorbell_wr32(qlnx_host_t *ha, void *reg_addr, uint32_t value) { uint32_t offset; offset = (uint32_t)((uint8_t *)reg_addr - (uint8_t *)ha->pci_dbells); bus_write_4(ha->pci_dbells, offset, value); bus_barrier(ha->pci_reg, 0, 0, BUS_SPACE_BARRIER_READ); bus_barrier(ha->pci_dbells, 0, 0, BUS_SPACE_BARRIER_READ); return; } static uint32_t qlnx_tcp_offset(qlnx_host_t *ha, struct mbuf *mp) { struct ether_vlan_header *eh = NULL; struct ip *ip = NULL; struct ip6_hdr *ip6 = NULL; struct tcphdr *th = NULL; uint32_t ehdrlen = 0, ip_hlen = 0, offset = 0; uint16_t etype = 0; uint8_t buf[sizeof(struct ip6_hdr)]; eh = mtod(mp, struct ether_vlan_header *); if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) { ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; etype = ntohs(eh->evl_proto); } else { ehdrlen = ETHER_HDR_LEN; etype = ntohs(eh->evl_encap_proto); } switch (etype) { case ETHERTYPE_IP: ip = (struct ip *)(mp->m_data + ehdrlen); ip_hlen = sizeof (struct ip); if (mp->m_len < (ehdrlen + ip_hlen)) { m_copydata(mp, ehdrlen, sizeof(struct ip), buf); ip = (struct ip *)buf; } th = (struct tcphdr *)(ip + 1); offset = ip_hlen + ehdrlen + (th->th_off << 2); break; case ETHERTYPE_IPV6: ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen); ip_hlen = sizeof(struct ip6_hdr); if (mp->m_len < (ehdrlen + ip_hlen)) { m_copydata(mp, ehdrlen, sizeof (struct ip6_hdr), buf); ip6 = (struct ip6_hdr *)buf; } th = (struct tcphdr *)(ip6 + 1); offset = ip_hlen + ehdrlen + (th->th_off << 2); break; default: break; } return (offset); } static __inline int qlnx_tso_check(struct qlnx_fastpath *fp, bus_dma_segment_t *segs, int nsegs, uint32_t offset) { int i; uint32_t sum, nbds_in_hdr = 1; uint32_t window; bus_dma_segment_t *s_seg; /* If the header spans multiple segments, skip those segments */ if (nsegs < ETH_TX_LSO_WINDOW_BDS_NUM) return (0); i = 0; while ((i < nsegs) && (offset >= segs->ds_len)) { offset = offset - segs->ds_len; segs++; i++; nbds_in_hdr++; } window = ETH_TX_LSO_WINDOW_BDS_NUM - nbds_in_hdr; nsegs = nsegs - i; while (nsegs >= window) { sum = 0; s_seg = segs; for (i = 0; i < window; i++){ sum += s_seg->ds_len; s_seg++; } if (sum < ETH_TX_LSO_WINDOW_MIN_LEN) { fp->tx_lso_wnd_min_len++; return (-1); } nsegs = nsegs - 1; segs++; } return (0); } static int qlnx_send(qlnx_host_t *ha, struct qlnx_fastpath *fp, struct mbuf **m_headp) { bus_dma_segment_t *segs; bus_dmamap_t map = 0; uint32_t nsegs = 0; int ret = -1; struct mbuf *m_head = *m_headp; uint16_t idx = 0; uint16_t elem_left; uint8_t nbd = 0; struct qlnx_tx_queue *txq; struct eth_tx_1st_bd *first_bd; struct eth_tx_2nd_bd *second_bd; struct eth_tx_3rd_bd *third_bd; struct eth_tx_bd *tx_data_bd; int seg_idx = 0; uint32_t nbds_in_hdr = 0; uint32_t offset = 0; #ifdef QLNX_TRACE_PERF_DATA uint16_t bd_used; #endif QL_DPRINT8(ha, "enter[%d]\n", fp->rss_id); if (!ha->link_up) return (-1); first_bd = NULL; second_bd = NULL; third_bd = NULL; tx_data_bd = NULL; txq = fp->txq[0]; if ((int)(elem_left = ecore_chain_get_elem_left(&txq->tx_pbl)) < QLNX_TX_ELEM_MIN_THRESH) { fp->tx_nsegs_gt_elem_left++; fp->err_tx_nsegs_gt_elem_left++; return (ENOBUFS); } idx = txq->sw_tx_prod; map = txq->sw_tx_ring[idx].map; segs = txq->segs; ret = bus_dmamap_load_mbuf_sg(ha->tx_tag, map, m_head, segs, &nsegs, BUS_DMA_NOWAIT); if (ha->dbg_trace_tso_pkt_len) { if (m_head->m_pkthdr.csum_flags & CSUM_TSO) { if (!fp->tx_tso_min_pkt_len) { fp->tx_tso_min_pkt_len = m_head->m_pkthdr.len; fp->tx_tso_min_pkt_len = m_head->m_pkthdr.len; } else { if (fp->tx_tso_min_pkt_len > m_head->m_pkthdr.len) fp->tx_tso_min_pkt_len = m_head->m_pkthdr.len; if (fp->tx_tso_max_pkt_len < m_head->m_pkthdr.len) fp->tx_tso_max_pkt_len = m_head->m_pkthdr.len; } } } if (m_head->m_pkthdr.csum_flags & CSUM_TSO) offset = qlnx_tcp_offset(ha, m_head); if ((ret == EFBIG) || ((nsegs > QLNX_MAX_SEGMENTS_NON_TSO) && ( (!(m_head->m_pkthdr.csum_flags & CSUM_TSO)) || ((m_head->m_pkthdr.csum_flags & CSUM_TSO) && qlnx_tso_check(fp, segs, nsegs, offset))))) { struct mbuf *m; QL_DPRINT8(ha, "EFBIG [%d]\n", m_head->m_pkthdr.len); fp->tx_defrag++; m = m_defrag(m_head, M_NOWAIT); if (m == NULL) { fp->err_tx_defrag++; fp->tx_pkts_freed++; m_freem(m_head); *m_headp = NULL; QL_DPRINT1(ha, "m_defrag() = NULL [%d]\n", ret); return (ENOBUFS); } m_head = m; *m_headp = m_head; if ((ret = bus_dmamap_load_mbuf_sg(ha->tx_tag, map, m_head, segs, &nsegs, BUS_DMA_NOWAIT))) { fp->err_tx_defrag_dmamap_load++; QL_DPRINT1(ha, "bus_dmamap_load_mbuf_sg failed0 [%d, %d]\n", ret, m_head->m_pkthdr.len); fp->tx_pkts_freed++; m_freem(m_head); *m_headp = NULL; return (ret); } if ((nsegs > QLNX_MAX_SEGMENTS_NON_TSO) && !(m_head->m_pkthdr.csum_flags & CSUM_TSO)) { fp->err_tx_non_tso_max_seg++; QL_DPRINT1(ha, "(%d) nsegs too many for non-TSO [%d, %d]\n", ret, nsegs, m_head->m_pkthdr.len); fp->tx_pkts_freed++; m_freem(m_head); *m_headp = NULL; return (ret); } if (m_head->m_pkthdr.csum_flags & CSUM_TSO) offset = qlnx_tcp_offset(ha, m_head); } else if (ret) { fp->err_tx_dmamap_load++; QL_DPRINT1(ha, "bus_dmamap_load_mbuf_sg failed1 [%d, %d]\n", ret, m_head->m_pkthdr.len); fp->tx_pkts_freed++; m_freem(m_head); *m_headp = NULL; return (ret); } QL_ASSERT(ha, (nsegs != 0), ("qlnx_send: empty packet")); if (ha->dbg_trace_tso_pkt_len) { if (nsegs < QLNX_FP_MAX_SEGS) fp->tx_pkts[(nsegs - 1)]++; else fp->tx_pkts[(QLNX_FP_MAX_SEGS - 1)]++; } #ifdef QLNX_TRACE_PERF_DATA if (m_head->m_pkthdr.csum_flags & CSUM_TSO) { if(m_head->m_pkthdr.len <= 2048) fp->tx_pkts_hist[0]++; else if((m_head->m_pkthdr.len > 2048) && (m_head->m_pkthdr.len <= 4096)) fp->tx_pkts_hist[1]++; else if((m_head->m_pkthdr.len > 4096) && (m_head->m_pkthdr.len <= 8192)) fp->tx_pkts_hist[2]++; else if((m_head->m_pkthdr.len > 8192) && (m_head->m_pkthdr.len <= 12288 )) fp->tx_pkts_hist[3]++; else if((m_head->m_pkthdr.len > 11288) && (m_head->m_pkthdr.len <= 16394)) fp->tx_pkts_hist[4]++; else if((m_head->m_pkthdr.len > 16384) && (m_head->m_pkthdr.len <= 20480)) fp->tx_pkts_hist[5]++; else if((m_head->m_pkthdr.len > 20480) && (m_head->m_pkthdr.len <= 24576)) fp->tx_pkts_hist[6]++; else if((m_head->m_pkthdr.len > 24576) && (m_head->m_pkthdr.len <= 28672)) fp->tx_pkts_hist[7]++; else if((m_head->m_pkthdr.len > 28762) && (m_head->m_pkthdr.len <= 32768)) fp->tx_pkts_hist[8]++; else if((m_head->m_pkthdr.len > 32768) && (m_head->m_pkthdr.len <= 36864)) fp->tx_pkts_hist[9]++; else if((m_head->m_pkthdr.len > 36864) && (m_head->m_pkthdr.len <= 40960)) fp->tx_pkts_hist[10]++; else if((m_head->m_pkthdr.len > 40960) && (m_head->m_pkthdr.len <= 45056)) fp->tx_pkts_hist[11]++; else if((m_head->m_pkthdr.len > 45056) && (m_head->m_pkthdr.len <= 49152)) fp->tx_pkts_hist[12]++; else if((m_head->m_pkthdr.len > 49512) && m_head->m_pkthdr.len <= 53248)) fp->tx_pkts_hist[13]++; else if((m_head->m_pkthdr.len > 53248) && (m_head->m_pkthdr.len <= 57344)) fp->tx_pkts_hist[14]++; else if((m_head->m_pkthdr.len > 53248) && (m_head->m_pkthdr.len <= 57344)) fp->tx_pkts_hist[15]++; else if((m_head->m_pkthdr.len > 57344) && (m_head->m_pkthdr.len <= 61440)) fp->tx_pkts_hist[16]++; else fp->tx_pkts_hist[17]++; } if (m_head->m_pkthdr.csum_flags & CSUM_TSO) { elem_left = ecore_chain_get_elem_left(&txq->tx_pbl); bd_used = TX_RING_SIZE - elem_left; if(bd_used <= 100) fp->tx_pkts_q[0]++; else if((bd_used > 100) && (bd_used <= 500)) fp->tx_pkts_q[1]++; else if((bd_used > 500) && (bd_used <= 1000)) fp->tx_pkts_q[2]++; else if((bd_used > 1000) && (bd_used <= 2000)) fp->tx_pkts_q[3]++; else if((bd_used > 3000) && (bd_used <= 4000)) fp->tx_pkts_q[4]++; else if((bd_used > 4000) && (bd_used <= 5000)) fp->tx_pkts_q[5]++; else if((bd_used > 6000) && (bd_used <= 7000)) fp->tx_pkts_q[6]++; else if((bd_used > 7000) && (bd_used <= 8000)) fp->tx_pkts_q[7]++; else if((bd_used > 8000) && (bd_used <= 9000)) fp->tx_pkts_q[8]++; else if((bd_used > 9000) && (bd_used <= 10000)) fp->tx_pkts_q[9]++; else if((bd_used > 10000) && (bd_used <= 11000)) fp->tx_pkts_q[10]++; else if((bd_used > 11000) && (bd_used <= 12000)) fp->tx_pkts_q[11]++; else if((bd_used > 12000) && (bd_used <= 13000)) fp->tx_pkts_q[12]++; else if((bd_used > 13000) && (bd_used <= 14000)) fp->tx_pkts_q[13]++; else if((bd_used > 14000) && (bd_used <= 15000)) fp->tx_pkts_q[14]++; else if((bd_used > 15000) && (bd_used <= 16000)) fp->tx_pkts_q[15]++; else fp->tx_pkts_q[16]++; } #endif /* end of QLNX_TRACE_PERF_DATA */ if ((nsegs + QLNX_TX_ELEM_RESERVE) > (int)(elem_left = ecore_chain_get_elem_left(&txq->tx_pbl))) { QL_DPRINT1(ha, "(%d, 0x%x) insuffient BDs" " in chain[%d] trying to free packets\n", nsegs, elem_left, fp->rss_id); fp->tx_nsegs_gt_elem_left++; (void)qlnx_tx_int(ha, fp, txq); if ((nsegs + QLNX_TX_ELEM_RESERVE) > (int)(elem_left = ecore_chain_get_elem_left(&txq->tx_pbl))) { QL_DPRINT1(ha, "(%d, 0x%x) insuffient BDs in chain[%d]\n", nsegs, elem_left, fp->rss_id); fp->err_tx_nsegs_gt_elem_left++; fp->tx_ring_full = 1; if (ha->storm_stats_enable) ha->storm_stats_gather = 1; return (ENOBUFS); } } bus_dmamap_sync(ha->tx_tag, map, BUS_DMASYNC_PREWRITE); txq->sw_tx_ring[idx].mp = m_head; first_bd = (struct eth_tx_1st_bd *)ecore_chain_produce(&txq->tx_pbl); memset(first_bd, 0, sizeof(*first_bd)); first_bd->data.bd_flags.bitfields = 1 << ETH_TX_1ST_BD_FLAGS_START_BD_SHIFT; BD_SET_UNMAP_ADDR_LEN(first_bd, segs->ds_addr, segs->ds_len); nbd++; if (m_head->m_pkthdr.csum_flags & CSUM_IP) { first_bd->data.bd_flags.bitfields |= (1 << ETH_TX_1ST_BD_FLAGS_IP_CSUM_SHIFT); } if (m_head->m_pkthdr.csum_flags & (CSUM_UDP | CSUM_TCP | CSUM_TCP_IPV6 | CSUM_UDP_IPV6)) { first_bd->data.bd_flags.bitfields |= (1 << ETH_TX_1ST_BD_FLAGS_L4_CSUM_SHIFT); } if (m_head->m_flags & M_VLANTAG) { first_bd->data.vlan = m_head->m_pkthdr.ether_vtag; first_bd->data.bd_flags.bitfields |= (1 << ETH_TX_1ST_BD_FLAGS_VLAN_INSERTION_SHIFT); } if (m_head->m_pkthdr.csum_flags & CSUM_TSO) { first_bd->data.bd_flags.bitfields |= (1 << ETH_TX_1ST_BD_FLAGS_LSO_SHIFT); first_bd->data.bd_flags.bitfields |= (1 << ETH_TX_1ST_BD_FLAGS_IP_CSUM_SHIFT); nbds_in_hdr = 1; if (offset == segs->ds_len) { BD_SET_UNMAP_ADDR_LEN(first_bd, segs->ds_addr, offset); segs++; seg_idx++; second_bd = (struct eth_tx_2nd_bd *) ecore_chain_produce(&txq->tx_pbl); memset(second_bd, 0, sizeof(*second_bd)); nbd++; if (seg_idx < nsegs) { BD_SET_UNMAP_ADDR_LEN(second_bd, \ (segs->ds_addr), (segs->ds_len)); segs++; seg_idx++; } third_bd = (struct eth_tx_3rd_bd *) ecore_chain_produce(&txq->tx_pbl); memset(third_bd, 0, sizeof(*third_bd)); third_bd->data.lso_mss = m_head->m_pkthdr.tso_segsz; third_bd->data.bitfields |= (nbds_in_hdr<ds_addr), (segs->ds_len)); segs++; seg_idx++; } for (; seg_idx < nsegs; seg_idx++) { tx_data_bd = (struct eth_tx_bd *) ecore_chain_produce(&txq->tx_pbl); memset(tx_data_bd, 0, sizeof(*tx_data_bd)); BD_SET_UNMAP_ADDR_LEN(tx_data_bd, \ segs->ds_addr,\ segs->ds_len); segs++; nbd++; } } else if (offset < segs->ds_len) { BD_SET_UNMAP_ADDR_LEN(first_bd, segs->ds_addr, offset); second_bd = (struct eth_tx_2nd_bd *) ecore_chain_produce(&txq->tx_pbl); memset(second_bd, 0, sizeof(*second_bd)); BD_SET_UNMAP_ADDR_LEN(second_bd, \ (segs->ds_addr + offset),\ (segs->ds_len - offset)); nbd++; segs++; third_bd = (struct eth_tx_3rd_bd *) ecore_chain_produce(&txq->tx_pbl); memset(third_bd, 0, sizeof(*third_bd)); BD_SET_UNMAP_ADDR_LEN(third_bd, \ segs->ds_addr,\ segs->ds_len); third_bd->data.lso_mss = m_head->m_pkthdr.tso_segsz; third_bd->data.bitfields |= (nbds_in_hdr<tx_pbl); memset(tx_data_bd, 0, sizeof(*tx_data_bd)); BD_SET_UNMAP_ADDR_LEN(tx_data_bd, \ segs->ds_addr,\ segs->ds_len); segs++; nbd++; } } else { offset = offset - segs->ds_len; segs++; for (seg_idx = 1; seg_idx < nsegs; seg_idx++) { if (offset) nbds_in_hdr++; tx_data_bd = (struct eth_tx_bd *) ecore_chain_produce(&txq->tx_pbl); memset(tx_data_bd, 0, sizeof(*tx_data_bd)); if (second_bd == NULL) { second_bd = (struct eth_tx_2nd_bd *) tx_data_bd; } else if (third_bd == NULL) { third_bd = (struct eth_tx_3rd_bd *) tx_data_bd; } if (offset && (offset < segs->ds_len)) { BD_SET_UNMAP_ADDR_LEN(tx_data_bd,\ segs->ds_addr, offset); tx_data_bd = (struct eth_tx_bd *) ecore_chain_produce(&txq->tx_pbl); memset(tx_data_bd, 0, sizeof(*tx_data_bd)); if (second_bd == NULL) { second_bd = (struct eth_tx_2nd_bd *)tx_data_bd; } else if (third_bd == NULL) { third_bd = (struct eth_tx_3rd_bd *)tx_data_bd; } BD_SET_UNMAP_ADDR_LEN(tx_data_bd,\ (segs->ds_addr + offset), \ (segs->ds_len - offset)); nbd++; offset = 0; } else { if (offset) offset = offset - segs->ds_len; BD_SET_UNMAP_ADDR_LEN(tx_data_bd,\ segs->ds_addr, segs->ds_len); } segs++; nbd++; } if (third_bd == NULL) { third_bd = (struct eth_tx_3rd_bd *) ecore_chain_produce(&txq->tx_pbl); memset(third_bd, 0, sizeof(*third_bd)); } third_bd->data.lso_mss = m_head->m_pkthdr.tso_segsz; third_bd->data.bitfields |= (nbds_in_hdr<tx_tso_pkts++; } else { segs++; for (seg_idx = 1; seg_idx < nsegs; seg_idx++) { tx_data_bd = (struct eth_tx_bd *) ecore_chain_produce(&txq->tx_pbl); memset(tx_data_bd, 0, sizeof(*tx_data_bd)); BD_SET_UNMAP_ADDR_LEN(tx_data_bd, segs->ds_addr,\ segs->ds_len); segs++; nbd++; } first_bd->data.bitfields = (m_head->m_pkthdr.len & ETH_TX_DATA_1ST_BD_PKT_LEN_MASK) << ETH_TX_DATA_1ST_BD_PKT_LEN_SHIFT; first_bd->data.bitfields = htole16(first_bd->data.bitfields); fp->tx_non_tso_pkts++; } first_bd->data.nbds = nbd; if (ha->dbg_trace_tso_pkt_len) { if (fp->tx_tso_max_nsegs < nsegs) fp->tx_tso_max_nsegs = nsegs; if ((nsegs < fp->tx_tso_min_nsegs) || (!fp->tx_tso_min_nsegs)) fp->tx_tso_min_nsegs = nsegs; } txq->sw_tx_ring[idx].nsegs = nsegs; txq->sw_tx_prod = (txq->sw_tx_prod + 1) & (TX_RING_SIZE - 1); txq->tx_db.data.bd_prod = htole16(ecore_chain_get_prod_idx(&txq->tx_pbl)); qlnx_txq_doorbell_wr32(ha, txq->doorbell_addr, txq->tx_db.raw); QL_DPRINT8(ha, "exit[%d]\n", fp->rss_id); return (0); } static void qlnx_stop(qlnx_host_t *ha) { - struct ifnet *ifp = ha->ifp; + if_t ifp = ha->ifp; int i; - ifp->if_drv_flags &= ~(IFF_DRV_OACTIVE | IFF_DRV_RUNNING); + if_setdrvflagbits(ifp, 0, (IFF_DRV_OACTIVE | IFF_DRV_RUNNING)); /* * We simply lock and unlock each fp->tx_mtx to * propagate the if_drv_flags * state to each tx thread */ QL_DPRINT1(ha, "QLNX STATE = %d\n",ha->state); if (ha->state == QLNX_STATE_OPEN) { for (i = 0; i < ha->num_rss; i++) { struct qlnx_fastpath *fp = &ha->fp_array[i]; mtx_lock(&fp->tx_mtx); mtx_unlock(&fp->tx_mtx); if (fp->fp_taskqueue != NULL) taskqueue_enqueue(fp->fp_taskqueue, &fp->fp_task); } } #ifdef QLNX_ENABLE_IWARP if (qlnx_vf_device(ha) != 0) { qlnx_rdma_dev_close(ha); } #endif /* #ifdef QLNX_ENABLE_IWARP */ qlnx_unload(ha); return; } static int qlnx_get_ifq_snd_maxlen(qlnx_host_t *ha) { return(TX_RING_SIZE - 1); } uint8_t * qlnx_get_mac_addr(qlnx_host_t *ha) { struct ecore_hwfn *p_hwfn; unsigned char mac[ETHER_ADDR_LEN]; uint8_t p_is_forced; p_hwfn = &ha->cdev.hwfns[0]; if (qlnx_vf_device(ha) != 0) return (p_hwfn->hw_info.hw_mac_addr); ecore_vf_read_bulletin(p_hwfn, &p_is_forced); if (ecore_vf_bulletin_get_forced_mac(p_hwfn, mac, &p_is_forced) == true) { device_printf(ha->pci_dev, "%s: p_is_forced = %d" " mac_addr = %02x:%02x:%02x:%02x:%02x:%02x\n", __func__, p_is_forced, mac[0], mac[1], mac[2], mac[3], mac[4], mac[5]); memcpy(ha->primary_mac, mac, ETH_ALEN); } return (ha->primary_mac); } static uint32_t qlnx_get_optics(qlnx_host_t *ha, struct qlnx_link_output *if_link) { uint32_t ifm_type = 0; switch (if_link->media_type) { case MEDIA_MODULE_FIBER: case MEDIA_UNSPECIFIED: if (if_link->speed == (100 * 1000)) ifm_type = QLNX_IFM_100G_SR4; else if (if_link->speed == (40 * 1000)) ifm_type = IFM_40G_SR4; else if (if_link->speed == (25 * 1000)) ifm_type = QLNX_IFM_25G_SR; else if (if_link->speed == (10 * 1000)) ifm_type = (IFM_10G_LR | IFM_10G_SR); else if (if_link->speed == (1 * 1000)) ifm_type = (IFM_1000_SX | IFM_1000_LX); break; case MEDIA_DA_TWINAX: if (if_link->speed == (100 * 1000)) ifm_type = QLNX_IFM_100G_CR4; else if (if_link->speed == (40 * 1000)) ifm_type = IFM_40G_CR4; else if (if_link->speed == (25 * 1000)) ifm_type = QLNX_IFM_25G_CR; else if (if_link->speed == (10 * 1000)) ifm_type = IFM_10G_TWINAX; break; default : ifm_type = IFM_UNKNOWN; break; } return (ifm_type); } /***************************************************************************** * Interrupt Service Functions *****************************************************************************/ static int qlnx_rx_jumbo_chain(qlnx_host_t *ha, struct qlnx_fastpath *fp, struct mbuf *mp_head, uint16_t len) { struct mbuf *mp, *mpf, *mpl; struct sw_rx_data *sw_rx_data; struct qlnx_rx_queue *rxq; uint16_t len_in_buffer; rxq = fp->rxq; mpf = mpl = mp = NULL; while (len) { rxq->sw_rx_cons = (rxq->sw_rx_cons + 1) & (RX_RING_SIZE - 1); sw_rx_data = &rxq->sw_rx_ring[rxq->sw_rx_cons]; mp = sw_rx_data->data; if (mp == NULL) { QL_DPRINT1(ha, "mp = NULL\n"); fp->err_rx_mp_null++; rxq->sw_rx_cons = (rxq->sw_rx_cons + 1) & (RX_RING_SIZE - 1); if (mpf != NULL) m_freem(mpf); return (-1); } bus_dmamap_sync(ha->rx_tag, sw_rx_data->map, BUS_DMASYNC_POSTREAD); if (qlnx_alloc_rx_buffer(ha, rxq) != 0) { QL_DPRINT1(ha, "New buffer allocation failed, dropping" " incoming packet and reusing its buffer\n"); qlnx_reuse_rx_data(rxq); fp->err_rx_alloc_errors++; if (mpf != NULL) m_freem(mpf); return (-1); } ecore_chain_consume(&rxq->rx_bd_ring); if (len > rxq->rx_buf_size) len_in_buffer = rxq->rx_buf_size; else len_in_buffer = len; len = len - len_in_buffer; mp->m_flags &= ~M_PKTHDR; mp->m_next = NULL; mp->m_len = len_in_buffer; if (mpf == NULL) mpf = mpl = mp; else { mpl->m_next = mp; mpl = mp; } } if (mpf != NULL) mp_head->m_next = mpf; return (0); } static void qlnx_tpa_start(qlnx_host_t *ha, struct qlnx_fastpath *fp, struct qlnx_rx_queue *rxq, struct eth_fast_path_rx_tpa_start_cqe *cqe) { uint32_t agg_index; - struct ifnet *ifp = ha->ifp; + if_t ifp = ha->ifp; struct mbuf *mp; struct mbuf *mpf = NULL, *mpl = NULL, *mpc = NULL; struct sw_rx_data *sw_rx_data; dma_addr_t addr; bus_dmamap_t map; struct eth_rx_bd *rx_bd; int i; uint8_t hash_type; agg_index = cqe->tpa_agg_index; QL_DPRINT7(ha, "[rss_id = %d]: enter\n \ \t type = 0x%x\n \ \t bitfields = 0x%x\n \ \t seg_len = 0x%x\n \ \t pars_flags = 0x%x\n \ \t vlan_tag = 0x%x\n \ \t rss_hash = 0x%x\n \ \t len_on_first_bd = 0x%x\n \ \t placement_offset = 0x%x\n \ \t tpa_agg_index = 0x%x\n \ \t header_len = 0x%x\n \ \t ext_bd_len_list[0] = 0x%x\n \ \t ext_bd_len_list[1] = 0x%x\n \ \t ext_bd_len_list[2] = 0x%x\n \ \t ext_bd_len_list[3] = 0x%x\n \ \t ext_bd_len_list[4] = 0x%x\n", fp->rss_id, cqe->type, cqe->bitfields, cqe->seg_len, cqe->pars_flags.flags, cqe->vlan_tag, cqe->rss_hash, cqe->len_on_first_bd, cqe->placement_offset, cqe->tpa_agg_index, cqe->header_len, cqe->ext_bd_len_list[0], cqe->ext_bd_len_list[1], cqe->ext_bd_len_list[2], cqe->ext_bd_len_list[3], cqe->ext_bd_len_list[4]); if (agg_index >= ETH_TPA_MAX_AGGS_NUM) { fp->err_rx_tpa_invalid_agg_num++; return; } sw_rx_data = &rxq->sw_rx_ring[rxq->sw_rx_cons]; bus_dmamap_sync(ha->rx_tag, sw_rx_data->map, BUS_DMASYNC_POSTREAD); mp = sw_rx_data->data; QL_DPRINT7(ha, "[rss_id = %d]: mp = %p \n ", fp->rss_id, mp); if (mp == NULL) { QL_DPRINT7(ha, "[%d]: mp = NULL\n", fp->rss_id); fp->err_rx_mp_null++; rxq->sw_rx_cons = (rxq->sw_rx_cons + 1) & (RX_RING_SIZE - 1); return; } if ((le16toh(cqe->pars_flags.flags)) & CQE_FLAGS_ERR) { QL_DPRINT7(ha, "[%d]: CQE in CONS = %u has error," " flags = %x, dropping incoming packet\n", fp->rss_id, rxq->sw_rx_cons, le16toh(cqe->pars_flags.flags)); fp->err_rx_hw_errors++; qlnx_reuse_rx_data(rxq); QLNX_INC_IERRORS(ifp); return; } if (qlnx_alloc_rx_buffer(ha, rxq) != 0) { QL_DPRINT7(ha, "[%d]: New buffer allocation failed," " dropping incoming packet and reusing its buffer\n", fp->rss_id); fp->err_rx_alloc_errors++; QLNX_INC_IQDROPS(ifp); /* * Load the tpa mbuf into the rx ring and save the * posted mbuf */ map = sw_rx_data->map; addr = sw_rx_data->dma_addr; sw_rx_data = &rxq->sw_rx_ring[rxq->sw_rx_prod]; sw_rx_data->data = rxq->tpa_info[agg_index].rx_buf.data; sw_rx_data->dma_addr = rxq->tpa_info[agg_index].rx_buf.dma_addr; sw_rx_data->map = rxq->tpa_info[agg_index].rx_buf.map; rxq->tpa_info[agg_index].rx_buf.data = mp; rxq->tpa_info[agg_index].rx_buf.dma_addr = addr; rxq->tpa_info[agg_index].rx_buf.map = map; rx_bd = (struct eth_rx_bd *) ecore_chain_produce(&rxq->rx_bd_ring); rx_bd->addr.hi = htole32(U64_HI(sw_rx_data->dma_addr)); rx_bd->addr.lo = htole32(U64_LO(sw_rx_data->dma_addr)); bus_dmamap_sync(ha->rx_tag, sw_rx_data->map, BUS_DMASYNC_PREREAD); rxq->sw_rx_prod = (rxq->sw_rx_prod + 1) & (RX_RING_SIZE - 1); rxq->sw_rx_cons = (rxq->sw_rx_cons + 1) & (RX_RING_SIZE - 1); ecore_chain_consume(&rxq->rx_bd_ring); /* Now reuse any buffers posted in ext_bd_len_list */ for (i = 0; i < ETH_TPA_CQE_START_LEN_LIST_SIZE; i++) { if (cqe->ext_bd_len_list[i] == 0) break; qlnx_reuse_rx_data(rxq); } rxq->tpa_info[agg_index].agg_state = QLNX_AGG_STATE_ERROR; return; } if (rxq->tpa_info[agg_index].agg_state != QLNX_AGG_STATE_NONE) { QL_DPRINT7(ha, "[%d]: invalid aggregation state," " dropping incoming packet and reusing its buffer\n", fp->rss_id); QLNX_INC_IQDROPS(ifp); /* if we already have mbuf head in aggregation free it */ if (rxq->tpa_info[agg_index].mpf) { m_freem(rxq->tpa_info[agg_index].mpf); rxq->tpa_info[agg_index].mpl = NULL; } rxq->tpa_info[agg_index].mpf = mp; rxq->tpa_info[agg_index].mpl = NULL; rxq->sw_rx_cons = (rxq->sw_rx_cons + 1) & (RX_RING_SIZE - 1); ecore_chain_consume(&rxq->rx_bd_ring); /* Now reuse any buffers posted in ext_bd_len_list */ for (i = 0; i < ETH_TPA_CQE_START_LEN_LIST_SIZE; i++) { if (cqe->ext_bd_len_list[i] == 0) break; qlnx_reuse_rx_data(rxq); } rxq->tpa_info[agg_index].agg_state = QLNX_AGG_STATE_ERROR; return; } /* * first process the ext_bd_len_list * if this fails then we simply drop the packet */ ecore_chain_consume(&rxq->rx_bd_ring); rxq->sw_rx_cons = (rxq->sw_rx_cons + 1) & (RX_RING_SIZE - 1); for (i = 0; i < ETH_TPA_CQE_START_LEN_LIST_SIZE; i++) { QL_DPRINT7(ha, "[%d]: 4\n ", fp->rss_id); if (cqe->ext_bd_len_list[i] == 0) break; sw_rx_data = &rxq->sw_rx_ring[rxq->sw_rx_cons]; bus_dmamap_sync(ha->rx_tag, sw_rx_data->map, BUS_DMASYNC_POSTREAD); mpc = sw_rx_data->data; if (mpc == NULL) { QL_DPRINT7(ha, "[%d]: mpc = NULL\n", fp->rss_id); fp->err_rx_mp_null++; if (mpf != NULL) m_freem(mpf); mpf = mpl = NULL; rxq->tpa_info[agg_index].agg_state = QLNX_AGG_STATE_ERROR; ecore_chain_consume(&rxq->rx_bd_ring); rxq->sw_rx_cons = (rxq->sw_rx_cons + 1) & (RX_RING_SIZE - 1); continue; } if (qlnx_alloc_rx_buffer(ha, rxq) != 0) { QL_DPRINT7(ha, "[%d]: New buffer allocation failed," " dropping incoming packet and reusing its" " buffer\n", fp->rss_id); qlnx_reuse_rx_data(rxq); if (mpf != NULL) m_freem(mpf); mpf = mpl = NULL; rxq->tpa_info[agg_index].agg_state = QLNX_AGG_STATE_ERROR; ecore_chain_consume(&rxq->rx_bd_ring); rxq->sw_rx_cons = (rxq->sw_rx_cons + 1) & (RX_RING_SIZE - 1); continue; } mpc->m_flags &= ~M_PKTHDR; mpc->m_next = NULL; mpc->m_len = cqe->ext_bd_len_list[i]; if (mpf == NULL) { mpf = mpl = mpc; } else { mpl->m_len = ha->rx_buf_size; mpl->m_next = mpc; mpl = mpc; } ecore_chain_consume(&rxq->rx_bd_ring); rxq->sw_rx_cons = (rxq->sw_rx_cons + 1) & (RX_RING_SIZE - 1); } if (rxq->tpa_info[agg_index].agg_state != QLNX_AGG_STATE_NONE) { QL_DPRINT7(ha, "[%d]: invalid aggregation state, dropping" " incoming packet and reusing its buffer\n", fp->rss_id); QLNX_INC_IQDROPS(ifp); rxq->tpa_info[agg_index].mpf = mp; rxq->tpa_info[agg_index].mpl = NULL; return; } rxq->tpa_info[agg_index].placement_offset = cqe->placement_offset; if (mpf != NULL) { mp->m_len = ha->rx_buf_size; mp->m_next = mpf; rxq->tpa_info[agg_index].mpf = mp; rxq->tpa_info[agg_index].mpl = mpl; } else { mp->m_len = cqe->len_on_first_bd + cqe->placement_offset; rxq->tpa_info[agg_index].mpf = mp; rxq->tpa_info[agg_index].mpl = mp; mp->m_next = NULL; } mp->m_flags |= M_PKTHDR; /* assign packet to this interface interface */ mp->m_pkthdr.rcvif = ifp; /* assume no hardware checksum has complated */ mp->m_pkthdr.csum_flags = 0; //mp->m_pkthdr.flowid = fp->rss_id; mp->m_pkthdr.flowid = cqe->rss_hash; hash_type = cqe->bitfields & (ETH_FAST_PATH_RX_REG_CQE_RSS_HASH_TYPE_MASK << ETH_FAST_PATH_RX_REG_CQE_RSS_HASH_TYPE_SHIFT); switch (hash_type) { case RSS_HASH_TYPE_IPV4: M_HASHTYPE_SET(mp, M_HASHTYPE_RSS_IPV4); break; case RSS_HASH_TYPE_TCP_IPV4: M_HASHTYPE_SET(mp, M_HASHTYPE_RSS_TCP_IPV4); break; case RSS_HASH_TYPE_IPV6: M_HASHTYPE_SET(mp, M_HASHTYPE_RSS_IPV6); break; case RSS_HASH_TYPE_TCP_IPV6: M_HASHTYPE_SET(mp, M_HASHTYPE_RSS_TCP_IPV6); break; default: M_HASHTYPE_SET(mp, M_HASHTYPE_OPAQUE); break; } mp->m_pkthdr.csum_flags |= (CSUM_IP_CHECKED | CSUM_IP_VALID | CSUM_DATA_VALID | CSUM_PSEUDO_HDR); mp->m_pkthdr.csum_data = 0xFFFF; if (CQE_HAS_VLAN(cqe->pars_flags.flags)) { mp->m_pkthdr.ether_vtag = le16toh(cqe->vlan_tag); mp->m_flags |= M_VLANTAG; } rxq->tpa_info[agg_index].agg_state = QLNX_AGG_STATE_START; QL_DPRINT7(ha, "[%d]: 5\n\tagg_state = %d\n\t mpf = %p mpl = %p\n", fp->rss_id, rxq->tpa_info[agg_index].agg_state, rxq->tpa_info[agg_index].mpf, rxq->tpa_info[agg_index].mpl); return; } static void qlnx_tpa_cont(qlnx_host_t *ha, struct qlnx_fastpath *fp, struct qlnx_rx_queue *rxq, struct eth_fast_path_rx_tpa_cont_cqe *cqe) { struct sw_rx_data *sw_rx_data; int i; struct mbuf *mpf = NULL, *mpl = NULL, *mpc = NULL; struct mbuf *mp; uint32_t agg_index; QL_DPRINT7(ha, "[%d]: enter\n \ \t type = 0x%x\n \ \t tpa_agg_index = 0x%x\n \ \t len_list[0] = 0x%x\n \ \t len_list[1] = 0x%x\n \ \t len_list[2] = 0x%x\n \ \t len_list[3] = 0x%x\n \ \t len_list[4] = 0x%x\n \ \t len_list[5] = 0x%x\n", fp->rss_id, cqe->type, cqe->tpa_agg_index, cqe->len_list[0], cqe->len_list[1], cqe->len_list[2], cqe->len_list[3], cqe->len_list[4], cqe->len_list[5]); agg_index = cqe->tpa_agg_index; if (agg_index >= ETH_TPA_MAX_AGGS_NUM) { QL_DPRINT7(ha, "[%d]: 0\n ", fp->rss_id); fp->err_rx_tpa_invalid_agg_num++; return; } for (i = 0; i < ETH_TPA_CQE_CONT_LEN_LIST_SIZE; i++) { QL_DPRINT7(ha, "[%d]: 1\n ", fp->rss_id); if (cqe->len_list[i] == 0) break; if (rxq->tpa_info[agg_index].agg_state != QLNX_AGG_STATE_START) { qlnx_reuse_rx_data(rxq); continue; } sw_rx_data = &rxq->sw_rx_ring[rxq->sw_rx_cons]; bus_dmamap_sync(ha->rx_tag, sw_rx_data->map, BUS_DMASYNC_POSTREAD); mpc = sw_rx_data->data; if (mpc == NULL) { QL_DPRINT7(ha, "[%d]: mpc = NULL\n", fp->rss_id); fp->err_rx_mp_null++; if (mpf != NULL) m_freem(mpf); mpf = mpl = NULL; rxq->tpa_info[agg_index].agg_state = QLNX_AGG_STATE_ERROR; ecore_chain_consume(&rxq->rx_bd_ring); rxq->sw_rx_cons = (rxq->sw_rx_cons + 1) & (RX_RING_SIZE - 1); continue; } if (qlnx_alloc_rx_buffer(ha, rxq) != 0) { QL_DPRINT7(ha, "[%d]: New buffer allocation failed," " dropping incoming packet and reusing its" " buffer\n", fp->rss_id); qlnx_reuse_rx_data(rxq); if (mpf != NULL) m_freem(mpf); mpf = mpl = NULL; rxq->tpa_info[agg_index].agg_state = QLNX_AGG_STATE_ERROR; ecore_chain_consume(&rxq->rx_bd_ring); rxq->sw_rx_cons = (rxq->sw_rx_cons + 1) & (RX_RING_SIZE - 1); continue; } mpc->m_flags &= ~M_PKTHDR; mpc->m_next = NULL; mpc->m_len = cqe->len_list[i]; if (mpf == NULL) { mpf = mpl = mpc; } else { mpl->m_len = ha->rx_buf_size; mpl->m_next = mpc; mpl = mpc; } ecore_chain_consume(&rxq->rx_bd_ring); rxq->sw_rx_cons = (rxq->sw_rx_cons + 1) & (RX_RING_SIZE - 1); } QL_DPRINT7(ha, "[%d]: 2\n" "\tmpf = %p mpl = %p\n", fp->rss_id, mpf, mpl); if (mpf != NULL) { mp = rxq->tpa_info[agg_index].mpl; mp->m_len = ha->rx_buf_size; mp->m_next = mpf; rxq->tpa_info[agg_index].mpl = mpl; } return; } static int qlnx_tpa_end(qlnx_host_t *ha, struct qlnx_fastpath *fp, struct qlnx_rx_queue *rxq, struct eth_fast_path_rx_tpa_end_cqe *cqe) { struct sw_rx_data *sw_rx_data; int i; struct mbuf *mpf = NULL, *mpl = NULL, *mpc = NULL; struct mbuf *mp; uint32_t agg_index; uint32_t len = 0; - struct ifnet *ifp = ha->ifp; + if_t ifp = ha->ifp; QL_DPRINT7(ha, "[%d]: enter\n \ \t type = 0x%x\n \ \t tpa_agg_index = 0x%x\n \ \t total_packet_len = 0x%x\n \ \t num_of_bds = 0x%x\n \ \t end_reason = 0x%x\n \ \t num_of_coalesced_segs = 0x%x\n \ \t ts_delta = 0x%x\n \ \t len_list[0] = 0x%x\n \ \t len_list[1] = 0x%x\n \ \t len_list[2] = 0x%x\n \ \t len_list[3] = 0x%x\n", fp->rss_id, cqe->type, cqe->tpa_agg_index, cqe->total_packet_len, cqe->num_of_bds, cqe->end_reason, cqe->num_of_coalesced_segs, cqe->ts_delta, cqe->len_list[0], cqe->len_list[1], cqe->len_list[2], cqe->len_list[3]); agg_index = cqe->tpa_agg_index; if (agg_index >= ETH_TPA_MAX_AGGS_NUM) { QL_DPRINT7(ha, "[%d]: 0\n ", fp->rss_id); fp->err_rx_tpa_invalid_agg_num++; return (0); } for (i = 0; i < ETH_TPA_CQE_END_LEN_LIST_SIZE; i++) { QL_DPRINT7(ha, "[%d]: 1\n ", fp->rss_id); if (cqe->len_list[i] == 0) break; if (rxq->tpa_info[agg_index].agg_state != QLNX_AGG_STATE_START) { QL_DPRINT7(ha, "[%d]: 2\n ", fp->rss_id); qlnx_reuse_rx_data(rxq); continue; } sw_rx_data = &rxq->sw_rx_ring[rxq->sw_rx_cons]; bus_dmamap_sync(ha->rx_tag, sw_rx_data->map, BUS_DMASYNC_POSTREAD); mpc = sw_rx_data->data; if (mpc == NULL) { QL_DPRINT7(ha, "[%d]: mpc = NULL\n", fp->rss_id); fp->err_rx_mp_null++; if (mpf != NULL) m_freem(mpf); mpf = mpl = NULL; rxq->tpa_info[agg_index].agg_state = QLNX_AGG_STATE_ERROR; ecore_chain_consume(&rxq->rx_bd_ring); rxq->sw_rx_cons = (rxq->sw_rx_cons + 1) & (RX_RING_SIZE - 1); continue; } if (qlnx_alloc_rx_buffer(ha, rxq) != 0) { QL_DPRINT7(ha, "[%d]: New buffer allocation failed," " dropping incoming packet and reusing its" " buffer\n", fp->rss_id); qlnx_reuse_rx_data(rxq); if (mpf != NULL) m_freem(mpf); mpf = mpl = NULL; rxq->tpa_info[agg_index].agg_state = QLNX_AGG_STATE_ERROR; ecore_chain_consume(&rxq->rx_bd_ring); rxq->sw_rx_cons = (rxq->sw_rx_cons + 1) & (RX_RING_SIZE - 1); continue; } mpc->m_flags &= ~M_PKTHDR; mpc->m_next = NULL; mpc->m_len = cqe->len_list[i]; if (mpf == NULL) { mpf = mpl = mpc; } else { mpl->m_len = ha->rx_buf_size; mpl->m_next = mpc; mpl = mpc; } ecore_chain_consume(&rxq->rx_bd_ring); rxq->sw_rx_cons = (rxq->sw_rx_cons + 1) & (RX_RING_SIZE - 1); } QL_DPRINT7(ha, "[%d]: 5\n ", fp->rss_id); if (mpf != NULL) { QL_DPRINT7(ha, "[%d]: 6\n ", fp->rss_id); mp = rxq->tpa_info[agg_index].mpl; mp->m_len = ha->rx_buf_size; mp->m_next = mpf; } if (rxq->tpa_info[agg_index].agg_state != QLNX_AGG_STATE_START) { QL_DPRINT7(ha, "[%d]: 7\n ", fp->rss_id); if (rxq->tpa_info[agg_index].mpf != NULL) m_freem(rxq->tpa_info[agg_index].mpf); rxq->tpa_info[agg_index].mpf = NULL; rxq->tpa_info[agg_index].mpl = NULL; rxq->tpa_info[agg_index].agg_state = QLNX_AGG_STATE_NONE; return (0); } mp = rxq->tpa_info[agg_index].mpf; m_adj(mp, rxq->tpa_info[agg_index].placement_offset); mp->m_pkthdr.len = cqe->total_packet_len; if (mp->m_next == NULL) mp->m_len = mp->m_pkthdr.len; else { /* compute the total packet length */ mpf = mp; while (mpf != NULL) { len += mpf->m_len; mpf = mpf->m_next; } if (cqe->total_packet_len > len) { mpl = rxq->tpa_info[agg_index].mpl; mpl->m_len += (cqe->total_packet_len - len); } } QLNX_INC_IPACKETS(ifp); QLNX_INC_IBYTES(ifp, (cqe->total_packet_len)); QL_DPRINT7(ha, "[%d]: 8 csum_data = 0x%x csum_flags = 0x%" PRIu64 "\n \ m_len = 0x%x m_pkthdr_len = 0x%x\n", fp->rss_id, mp->m_pkthdr.csum_data, (uint64_t)mp->m_pkthdr.csum_flags, mp->m_len, mp->m_pkthdr.len); - (*ifp->if_input)(ifp, mp); + if_input(ifp, mp); rxq->tpa_info[agg_index].mpf = NULL; rxq->tpa_info[agg_index].mpl = NULL; rxq->tpa_info[agg_index].agg_state = QLNX_AGG_STATE_NONE; return (cqe->num_of_coalesced_segs); } static int qlnx_rx_int(qlnx_host_t *ha, struct qlnx_fastpath *fp, int budget, int lro_enable) { uint16_t hw_comp_cons, sw_comp_cons; int rx_pkt = 0; struct qlnx_rx_queue *rxq = fp->rxq; - struct ifnet *ifp = ha->ifp; + if_t ifp = ha->ifp; struct ecore_dev *cdev = &ha->cdev; struct ecore_hwfn *p_hwfn; #ifdef QLNX_SOFT_LRO struct lro_ctrl *lro; lro = &rxq->lro; #endif /* #ifdef QLNX_SOFT_LRO */ hw_comp_cons = le16toh(*rxq->hw_cons_ptr); sw_comp_cons = ecore_chain_get_cons_idx(&rxq->rx_comp_ring); p_hwfn = &ha->cdev.hwfns[(fp->rss_id % cdev->num_hwfns)]; /* Memory barrier to prevent the CPU from doing speculative reads of CQE * / BD in the while-loop before reading hw_comp_cons. If the CQE is * read before it is written by FW, then FW writes CQE and SB, and then * the CPU reads the hw_comp_cons, it will use an old CQE. */ /* Loop to complete all indicated BDs */ while (sw_comp_cons != hw_comp_cons) { union eth_rx_cqe *cqe; struct eth_fast_path_rx_reg_cqe *fp_cqe; struct sw_rx_data *sw_rx_data; register struct mbuf *mp; enum eth_rx_cqe_type cqe_type; uint16_t len, pad, len_on_first_bd; uint8_t *data; uint8_t hash_type; /* Get the CQE from the completion ring */ cqe = (union eth_rx_cqe *) ecore_chain_consume(&rxq->rx_comp_ring); cqe_type = cqe->fast_path_regular.type; if (cqe_type == ETH_RX_CQE_TYPE_SLOW_PATH) { QL_DPRINT3(ha, "Got a slowath CQE\n"); ecore_eth_cqe_completion(p_hwfn, (struct eth_slow_path_rx_cqe *)cqe); goto next_cqe; } if (cqe_type != ETH_RX_CQE_TYPE_REGULAR) { switch (cqe_type) { case ETH_RX_CQE_TYPE_TPA_START: qlnx_tpa_start(ha, fp, rxq, &cqe->fast_path_tpa_start); fp->tpa_start++; break; case ETH_RX_CQE_TYPE_TPA_CONT: qlnx_tpa_cont(ha, fp, rxq, &cqe->fast_path_tpa_cont); fp->tpa_cont++; break; case ETH_RX_CQE_TYPE_TPA_END: rx_pkt += qlnx_tpa_end(ha, fp, rxq, &cqe->fast_path_tpa_end); fp->tpa_end++; break; default: break; } goto next_cqe; } /* Get the data from the SW ring */ sw_rx_data = &rxq->sw_rx_ring[rxq->sw_rx_cons]; mp = sw_rx_data->data; if (mp == NULL) { QL_DPRINT1(ha, "mp = NULL\n"); fp->err_rx_mp_null++; rxq->sw_rx_cons = (rxq->sw_rx_cons + 1) & (RX_RING_SIZE - 1); goto next_cqe; } bus_dmamap_sync(ha->rx_tag, sw_rx_data->map, BUS_DMASYNC_POSTREAD); /* non GRO */ fp_cqe = &cqe->fast_path_regular;/* MK CR TPA check assembly */ len = le16toh(fp_cqe->pkt_len); pad = fp_cqe->placement_offset; #if 0 QL_DPRINT3(ha, "CQE type = %x, flags = %x, vlan = %x," " len %u, parsing flags = %d pad = %d\n", cqe_type, fp_cqe->bitfields, le16toh(fp_cqe->vlan_tag), len, le16toh(fp_cqe->pars_flags.flags), pad); #endif data = mtod(mp, uint8_t *); data = data + pad; if (0) qlnx_dump_buf8(ha, __func__, data, len); /* For every Rx BD consumed, we allocate a new BD so the BD ring * is always with a fixed size. If allocation fails, we take the * consumed BD and return it to the ring in the PROD position. * The packet that was received on that BD will be dropped (and * not passed to the upper stack). */ /* If this is an error packet then drop it */ if ((le16toh(cqe->fast_path_regular.pars_flags.flags)) & CQE_FLAGS_ERR) { QL_DPRINT1(ha, "CQE in CONS = %u has error, flags = %x," " dropping incoming packet\n", sw_comp_cons, le16toh(cqe->fast_path_regular.pars_flags.flags)); fp->err_rx_hw_errors++; qlnx_reuse_rx_data(rxq); QLNX_INC_IERRORS(ifp); goto next_cqe; } if (qlnx_alloc_rx_buffer(ha, rxq) != 0) { QL_DPRINT1(ha, "New buffer allocation failed, dropping" " incoming packet and reusing its buffer\n"); qlnx_reuse_rx_data(rxq); fp->err_rx_alloc_errors++; QLNX_INC_IQDROPS(ifp); goto next_cqe; } ecore_chain_consume(&rxq->rx_bd_ring); len_on_first_bd = fp_cqe->len_on_first_bd; m_adj(mp, pad); mp->m_pkthdr.len = len; if ((len > 60 ) && (len > len_on_first_bd)) { mp->m_len = len_on_first_bd; if (qlnx_rx_jumbo_chain(ha, fp, mp, (len - len_on_first_bd)) != 0) { m_freem(mp); QLNX_INC_IQDROPS(ifp); goto next_cqe; } } else if (len_on_first_bd < len) { fp->err_rx_jumbo_chain_pkts++; } else { mp->m_len = len; } mp->m_flags |= M_PKTHDR; /* assign packet to this interface interface */ mp->m_pkthdr.rcvif = ifp; /* assume no hardware checksum has complated */ mp->m_pkthdr.csum_flags = 0; mp->m_pkthdr.flowid = fp_cqe->rss_hash; hash_type = fp_cqe->bitfields & (ETH_FAST_PATH_RX_REG_CQE_RSS_HASH_TYPE_MASK << ETH_FAST_PATH_RX_REG_CQE_RSS_HASH_TYPE_SHIFT); switch (hash_type) { case RSS_HASH_TYPE_IPV4: M_HASHTYPE_SET(mp, M_HASHTYPE_RSS_IPV4); break; case RSS_HASH_TYPE_TCP_IPV4: M_HASHTYPE_SET(mp, M_HASHTYPE_RSS_TCP_IPV4); break; case RSS_HASH_TYPE_IPV6: M_HASHTYPE_SET(mp, M_HASHTYPE_RSS_IPV6); break; case RSS_HASH_TYPE_TCP_IPV6: M_HASHTYPE_SET(mp, M_HASHTYPE_RSS_TCP_IPV6); break; default: M_HASHTYPE_SET(mp, M_HASHTYPE_OPAQUE); break; } if (CQE_L3_PACKET(fp_cqe->pars_flags.flags)) { mp->m_pkthdr.csum_flags |= CSUM_IP_CHECKED; } if (!(CQE_IP_HDR_ERR(fp_cqe->pars_flags.flags))) { mp->m_pkthdr.csum_flags |= CSUM_IP_VALID; } if (CQE_L4_HAS_CSUM(fp_cqe->pars_flags.flags)) { mp->m_pkthdr.csum_data = 0xFFFF; mp->m_pkthdr.csum_flags |= (CSUM_DATA_VALID | CSUM_PSEUDO_HDR); } if (CQE_HAS_VLAN(fp_cqe->pars_flags.flags)) { mp->m_pkthdr.ether_vtag = le16toh(fp_cqe->vlan_tag); mp->m_flags |= M_VLANTAG; } QLNX_INC_IPACKETS(ifp); QLNX_INC_IBYTES(ifp, len); #ifdef QLNX_SOFT_LRO if (lro_enable) tcp_lro_queue_mbuf(lro, mp); else - (*ifp->if_input)(ifp, mp); + if_input(ifp, mp); #else - (*ifp->if_input)(ifp, mp); + if_input(ifp, mp); #endif /* #ifdef QLNX_SOFT_LRO */ rx_pkt++; rxq->sw_rx_cons = (rxq->sw_rx_cons + 1) & (RX_RING_SIZE - 1); next_cqe: /* don't consume bd rx buffer */ ecore_chain_recycle_consumed(&rxq->rx_comp_ring); sw_comp_cons = ecore_chain_get_cons_idx(&rxq->rx_comp_ring); /* CR TPA - revisit how to handle budget in TPA perhaps increase on "end" */ if (rx_pkt == budget) break; } /* repeat while sw_comp_cons != hw_comp_cons... */ /* Update producers */ qlnx_update_rx_prod(p_hwfn, rxq); return rx_pkt; } /* * fast path interrupt */ static void qlnx_fp_isr(void *arg) { qlnx_ivec_t *ivec = arg; qlnx_host_t *ha; struct qlnx_fastpath *fp = NULL; int idx; ha = ivec->ha; if (ha->state != QLNX_STATE_OPEN) { return; } idx = ivec->rss_idx; if ((idx = ivec->rss_idx) >= ha->num_rss) { QL_DPRINT1(ha, "illegal interrupt[%d]\n", idx); ha->err_illegal_intr++; return; } fp = &ha->fp_array[idx]; if (fp == NULL) { ha->err_fp_null++; } else { int rx_int = 0; #ifdef QLNX_SOFT_LRO int total_rx_count = 0; #endif int lro_enable, tc; struct qlnx_tx_queue *txq; uint16_t elem_left; - lro_enable = ha->ifp->if_capenable & IFCAP_LRO; + lro_enable = if_getcapenable(ha->ifp) & IFCAP_LRO; ecore_sb_ack(fp->sb_info, IGU_INT_DISABLE, 0); do { for (tc = 0; tc < ha->num_tc; tc++) { txq = fp->txq[tc]; if((int)(elem_left = ecore_chain_get_elem_left(&txq->tx_pbl)) < QLNX_TX_ELEM_THRESH) { if (mtx_trylock(&fp->tx_mtx)) { #ifdef QLNX_TRACE_PERF_DATA tx_compl = fp->tx_pkts_completed; #endif qlnx_tx_int(ha, fp, fp->txq[tc]); #ifdef QLNX_TRACE_PERF_DATA fp->tx_pkts_compl_intr += (fp->tx_pkts_completed - tx_compl); if ((fp->tx_pkts_completed - tx_compl) <= 32) fp->tx_comInt[0]++; else if (((fp->tx_pkts_completed - tx_compl) > 32) && ((fp->tx_pkts_completed - tx_compl) <= 64)) fp->tx_comInt[1]++; else if(((fp->tx_pkts_completed - tx_compl) > 64) && ((fp->tx_pkts_completed - tx_compl) <= 128)) fp->tx_comInt[2]++; else if(((fp->tx_pkts_completed - tx_compl) > 128)) fp->tx_comInt[3]++; #endif mtx_unlock(&fp->tx_mtx); } } } rx_int = qlnx_rx_int(ha, fp, ha->rx_pkt_threshold, lro_enable); if (rx_int) { fp->rx_pkts += rx_int; #ifdef QLNX_SOFT_LRO total_rx_count += rx_int; #endif } } while (rx_int); #ifdef QLNX_SOFT_LRO { struct lro_ctrl *lro; lro = &fp->rxq->lro; if (lro_enable && total_rx_count) { #ifdef QLNX_TRACE_LRO_CNT if (lro->lro_mbuf_count & ~1023) fp->lro_cnt_1024++; else if (lro->lro_mbuf_count & ~511) fp->lro_cnt_512++; else if (lro->lro_mbuf_count & ~255) fp->lro_cnt_256++; else if (lro->lro_mbuf_count & ~127) fp->lro_cnt_128++; else if (lro->lro_mbuf_count & ~63) fp->lro_cnt_64++; #endif /* #ifdef QLNX_TRACE_LRO_CNT */ tcp_lro_flush_all(lro); } } #endif /* #ifdef QLNX_SOFT_LRO */ ecore_sb_update_sb_idx(fp->sb_info); rmb(); ecore_sb_ack(fp->sb_info, IGU_INT_ENABLE, 1); } return; } /* * slow path interrupt processing function * can be invoked in polled mode or in interrupt mode via taskqueue. */ void qlnx_sp_isr(void *arg) { struct ecore_hwfn *p_hwfn; qlnx_host_t *ha; p_hwfn = arg; ha = (qlnx_host_t *)p_hwfn->p_dev; ha->sp_interrupts++; QL_DPRINT2(ha, "enter\n"); ecore_int_sp_dpc(p_hwfn); QL_DPRINT2(ha, "exit\n"); return; } /***************************************************************************** * Support Functions for DMA'able Memory *****************************************************************************/ static void qlnx_dmamap_callback(void *arg, bus_dma_segment_t *segs, int nsegs, int error) { *((bus_addr_t *)arg) = 0; if (error) { printf("%s: bus_dmamap_load failed (%d)\n", __func__, error); return; } *((bus_addr_t *)arg) = segs[0].ds_addr; return; } static int qlnx_alloc_dmabuf(qlnx_host_t *ha, qlnx_dma_t *dma_buf) { int ret = 0; bus_addr_t b_addr; ret = bus_dma_tag_create( ha->parent_tag,/* parent */ dma_buf->alignment, ((bus_size_t)(1ULL << 32)),/* boundary */ BUS_SPACE_MAXADDR, /* lowaddr */ BUS_SPACE_MAXADDR, /* highaddr */ NULL, NULL, /* filter, filterarg */ dma_buf->size, /* maxsize */ 1, /* nsegments */ dma_buf->size, /* maxsegsize */ 0, /* flags */ NULL, NULL, /* lockfunc, lockarg */ &dma_buf->dma_tag); if (ret) { QL_DPRINT1(ha, "could not create dma tag\n"); goto qlnx_alloc_dmabuf_exit; } ret = bus_dmamem_alloc(dma_buf->dma_tag, (void **)&dma_buf->dma_b, (BUS_DMA_ZERO | BUS_DMA_COHERENT | BUS_DMA_NOWAIT), &dma_buf->dma_map); if (ret) { bus_dma_tag_destroy(dma_buf->dma_tag); QL_DPRINT1(ha, "bus_dmamem_alloc failed\n"); goto qlnx_alloc_dmabuf_exit; } ret = bus_dmamap_load(dma_buf->dma_tag, dma_buf->dma_map, dma_buf->dma_b, dma_buf->size, qlnx_dmamap_callback, &b_addr, BUS_DMA_NOWAIT); if (ret || !b_addr) { bus_dma_tag_destroy(dma_buf->dma_tag); bus_dmamem_free(dma_buf->dma_tag, dma_buf->dma_b, dma_buf->dma_map); ret = -1; goto qlnx_alloc_dmabuf_exit; } dma_buf->dma_addr = b_addr; qlnx_alloc_dmabuf_exit: return ret; } static void qlnx_free_dmabuf(qlnx_host_t *ha, qlnx_dma_t *dma_buf) { bus_dmamap_unload(dma_buf->dma_tag, dma_buf->dma_map); bus_dmamem_free(dma_buf->dma_tag, dma_buf->dma_b, dma_buf->dma_map); bus_dma_tag_destroy(dma_buf->dma_tag); return; } void * qlnx_dma_alloc_coherent(void *ecore_dev, bus_addr_t *phys, uint32_t size) { qlnx_dma_t dma_buf; qlnx_dma_t *dma_p; qlnx_host_t *ha __unused; ha = (qlnx_host_t *)ecore_dev; size = (size + (PAGE_SIZE - 1)) & ~(PAGE_SIZE - 1); memset(&dma_buf, 0, sizeof (qlnx_dma_t)); dma_buf.size = size + PAGE_SIZE; dma_buf.alignment = 8; if (qlnx_alloc_dmabuf((qlnx_host_t *)ecore_dev, &dma_buf) != 0) return (NULL); bzero((uint8_t *)dma_buf.dma_b, dma_buf.size); *phys = dma_buf.dma_addr; dma_p = (qlnx_dma_t *)((uint8_t *)dma_buf.dma_b + size); memcpy(dma_p, &dma_buf, sizeof(qlnx_dma_t)); QL_DPRINT5(ha, "[%p %p %p %p 0x%08x ]\n", (void *)dma_buf.dma_map, (void *)dma_buf.dma_tag, dma_buf.dma_b, (void *)dma_buf.dma_addr, size); return (dma_buf.dma_b); } void qlnx_dma_free_coherent(void *ecore_dev, void *v_addr, bus_addr_t phys, uint32_t size) { qlnx_dma_t dma_buf, *dma_p; qlnx_host_t *ha; ha = (qlnx_host_t *)ecore_dev; if (v_addr == NULL) return; size = (size + (PAGE_SIZE - 1)) & ~(PAGE_SIZE - 1); dma_p = (qlnx_dma_t *)((uint8_t *)v_addr + size); QL_DPRINT5(ha, "[%p %p %p %p 0x%08x ]\n", (void *)dma_p->dma_map, (void *)dma_p->dma_tag, dma_p->dma_b, (void *)dma_p->dma_addr, size); dma_buf = *dma_p; if (!ha->qlnxr_debug) qlnx_free_dmabuf((qlnx_host_t *)ecore_dev, &dma_buf); return; } static int qlnx_alloc_parent_dma_tag(qlnx_host_t *ha) { int ret; device_t dev; dev = ha->pci_dev; /* * Allocate parent DMA Tag */ ret = bus_dma_tag_create( bus_get_dma_tag(dev), /* parent */ 1,((bus_size_t)(1ULL << 32)),/* alignment, boundary */ BUS_SPACE_MAXADDR, /* lowaddr */ BUS_SPACE_MAXADDR, /* highaddr */ NULL, NULL, /* filter, filterarg */ BUS_SPACE_MAXSIZE_32BIT,/* maxsize */ 0, /* nsegments */ BUS_SPACE_MAXSIZE_32BIT,/* maxsegsize */ 0, /* flags */ NULL, NULL, /* lockfunc, lockarg */ &ha->parent_tag); if (ret) { QL_DPRINT1(ha, "could not create parent dma tag\n"); return (-1); } ha->flags.parent_tag = 1; return (0); } static void qlnx_free_parent_dma_tag(qlnx_host_t *ha) { if (ha->parent_tag != NULL) { bus_dma_tag_destroy(ha->parent_tag); ha->parent_tag = NULL; } return; } static int qlnx_alloc_tx_dma_tag(qlnx_host_t *ha) { if (bus_dma_tag_create(NULL, /* parent */ 1, 0, /* alignment, bounds */ BUS_SPACE_MAXADDR, /* lowaddr */ BUS_SPACE_MAXADDR, /* highaddr */ NULL, NULL, /* filter, filterarg */ QLNX_MAX_TSO_FRAME_SIZE, /* maxsize */ QLNX_MAX_SEGMENTS, /* nsegments */ QLNX_MAX_TX_MBUF_SIZE, /* maxsegsize */ 0, /* flags */ NULL, /* lockfunc */ NULL, /* lockfuncarg */ &ha->tx_tag)) { QL_DPRINT1(ha, "tx_tag alloc failed\n"); return (-1); } return (0); } static void qlnx_free_tx_dma_tag(qlnx_host_t *ha) { if (ha->tx_tag != NULL) { bus_dma_tag_destroy(ha->tx_tag); ha->tx_tag = NULL; } return; } static int qlnx_alloc_rx_dma_tag(qlnx_host_t *ha) { if (bus_dma_tag_create(NULL, /* parent */ 1, 0, /* alignment, bounds */ BUS_SPACE_MAXADDR, /* lowaddr */ BUS_SPACE_MAXADDR, /* highaddr */ NULL, NULL, /* filter, filterarg */ MJUM9BYTES, /* maxsize */ 1, /* nsegments */ MJUM9BYTES, /* maxsegsize */ 0, /* flags */ NULL, /* lockfunc */ NULL, /* lockfuncarg */ &ha->rx_tag)) { QL_DPRINT1(ha, " rx_tag alloc failed\n"); return (-1); } return (0); } static void qlnx_free_rx_dma_tag(qlnx_host_t *ha) { if (ha->rx_tag != NULL) { bus_dma_tag_destroy(ha->rx_tag); ha->rx_tag = NULL; } return; } /********************************* * Exported functions *********************************/ uint32_t qlnx_pci_bus_get_bar_size(void *ecore_dev, uint8_t bar_id) { uint32_t bar_size; bar_id = bar_id * 2; bar_size = bus_get_resource_count(((qlnx_host_t *)ecore_dev)->pci_dev, SYS_RES_MEMORY, PCIR_BAR(bar_id)); return (bar_size); } uint32_t qlnx_pci_read_config_byte(void *ecore_dev, uint32_t pci_reg, uint8_t *reg_value) { *reg_value = pci_read_config(((qlnx_host_t *)ecore_dev)->pci_dev, pci_reg, 1); return 0; } uint32_t qlnx_pci_read_config_word(void *ecore_dev, uint32_t pci_reg, uint16_t *reg_value) { *reg_value = pci_read_config(((qlnx_host_t *)ecore_dev)->pci_dev, pci_reg, 2); return 0; } uint32_t qlnx_pci_read_config_dword(void *ecore_dev, uint32_t pci_reg, uint32_t *reg_value) { *reg_value = pci_read_config(((qlnx_host_t *)ecore_dev)->pci_dev, pci_reg, 4); return 0; } void qlnx_pci_write_config_byte(void *ecore_dev, uint32_t pci_reg, uint8_t reg_value) { pci_write_config(((qlnx_host_t *)ecore_dev)->pci_dev, pci_reg, reg_value, 1); return; } void qlnx_pci_write_config_word(void *ecore_dev, uint32_t pci_reg, uint16_t reg_value) { pci_write_config(((qlnx_host_t *)ecore_dev)->pci_dev, pci_reg, reg_value, 2); return; } void qlnx_pci_write_config_dword(void *ecore_dev, uint32_t pci_reg, uint32_t reg_value) { pci_write_config(((qlnx_host_t *)ecore_dev)->pci_dev, pci_reg, reg_value, 4); return; } int qlnx_pci_find_capability(void *ecore_dev, int cap) { int reg; qlnx_host_t *ha; ha = ecore_dev; if (pci_find_cap(ha->pci_dev, PCIY_EXPRESS, ®) == 0) return reg; else { QL_DPRINT1(ha, "failed\n"); return 0; } } int qlnx_pci_find_ext_capability(void *ecore_dev, int ext_cap) { int reg; qlnx_host_t *ha; ha = ecore_dev; if (pci_find_extcap(ha->pci_dev, ext_cap, ®) == 0) return reg; else { QL_DPRINT1(ha, "failed\n"); return 0; } } uint32_t qlnx_reg_rd32(void *hwfn, uint32_t reg_addr) { uint32_t data32; struct ecore_hwfn *p_hwfn; p_hwfn = hwfn; data32 = bus_read_4(((qlnx_host_t *)p_hwfn->p_dev)->pci_reg, \ (bus_size_t)(p_hwfn->reg_offset + reg_addr)); return (data32); } void qlnx_reg_wr32(void *hwfn, uint32_t reg_addr, uint32_t value) { struct ecore_hwfn *p_hwfn = hwfn; bus_write_4(((qlnx_host_t *)p_hwfn->p_dev)->pci_reg, \ (bus_size_t)(p_hwfn->reg_offset + reg_addr), value); return; } void qlnx_reg_wr16(void *hwfn, uint32_t reg_addr, uint16_t value) { struct ecore_hwfn *p_hwfn = hwfn; bus_write_2(((qlnx_host_t *)p_hwfn->p_dev)->pci_reg, \ (bus_size_t)(p_hwfn->reg_offset + reg_addr), value); return; } void qlnx_dbell_wr32_db(void *hwfn, void *reg_addr, uint32_t value) { struct ecore_dev *cdev; struct ecore_hwfn *p_hwfn; uint32_t offset; p_hwfn = hwfn; cdev = p_hwfn->p_dev; offset = (uint32_t)((uint8_t *)reg_addr - (uint8_t *)(p_hwfn->doorbells)); bus_write_4(((qlnx_host_t *)cdev)->pci_dbells, offset, value); return; } void qlnx_dbell_wr32(void *hwfn, uint32_t reg_addr, uint32_t value) { struct ecore_hwfn *p_hwfn = hwfn; bus_write_4(((qlnx_host_t *)p_hwfn->p_dev)->pci_dbells, \ (bus_size_t)(p_hwfn->db_offset + reg_addr), value); return; } uint32_t qlnx_direct_reg_rd32(void *p_hwfn, uint32_t *reg_addr) { uint32_t data32; bus_size_t offset; struct ecore_dev *cdev; cdev = ((struct ecore_hwfn *)p_hwfn)->p_dev; offset = (bus_size_t)((uint8_t *)reg_addr - (uint8_t *)(cdev->regview)); data32 = bus_read_4(((qlnx_host_t *)cdev)->pci_reg, offset); return (data32); } void qlnx_direct_reg_wr32(void *p_hwfn, void *reg_addr, uint32_t value) { bus_size_t offset; struct ecore_dev *cdev; cdev = ((struct ecore_hwfn *)p_hwfn)->p_dev; offset = (bus_size_t)((uint8_t *)reg_addr - (uint8_t *)(cdev->regview)); bus_write_4(((qlnx_host_t *)cdev)->pci_reg, offset, value); return; } void qlnx_direct_reg_wr64(void *p_hwfn, void *reg_addr, uint64_t value) { bus_size_t offset; struct ecore_dev *cdev; cdev = ((struct ecore_hwfn *)p_hwfn)->p_dev; offset = (bus_size_t)((uint8_t *)reg_addr - (uint8_t *)(cdev->regview)); bus_write_8(((qlnx_host_t *)cdev)->pci_reg, offset, value); return; } void * qlnx_zalloc(uint32_t size) { caddr_t va; va = malloc((unsigned long)size, M_QLNXBUF, M_NOWAIT); bzero(va, size); return ((void *)va); } void qlnx_barrier(void *p_hwfn) { qlnx_host_t *ha; ha = (qlnx_host_t *)((struct ecore_hwfn *)p_hwfn)->p_dev; bus_barrier(ha->pci_reg, 0, 0, BUS_SPACE_BARRIER_WRITE); } void qlnx_link_update(void *p_hwfn) { qlnx_host_t *ha; int prev_link_state; ha = (qlnx_host_t *)((struct ecore_hwfn *)p_hwfn)->p_dev; qlnx_fill_link(ha, p_hwfn, &ha->if_link); prev_link_state = ha->link_up; ha->link_up = ha->if_link.link_up; if (prev_link_state != ha->link_up) { if (ha->link_up) { if_link_state_change(ha->ifp, LINK_STATE_UP); } else { if_link_state_change(ha->ifp, LINK_STATE_DOWN); } } #ifndef QLNX_VF #ifdef CONFIG_ECORE_SRIOV if (qlnx_vf_device(ha) != 0) { if (ha->sriov_initialized) qlnx_inform_vf_link_state(p_hwfn, ha); } #endif /* #ifdef CONFIG_ECORE_SRIOV */ #endif /* #ifdef QLNX_VF */ return; } static void __qlnx_osal_vf_fill_acquire_resc_req(struct ecore_hwfn *p_hwfn, struct ecore_vf_acquire_sw_info *p_sw_info) { p_sw_info->driver_version = (QLNX_VERSION_MAJOR << 24) | (QLNX_VERSION_MINOR << 16) | QLNX_VERSION_BUILD; p_sw_info->os_type = VFPF_ACQUIRE_OS_FREEBSD; return; } void qlnx_osal_vf_fill_acquire_resc_req(void *p_hwfn, void *p_resc_req, void *p_sw_info) { __qlnx_osal_vf_fill_acquire_resc_req(p_hwfn, p_sw_info); return; } void qlnx_fill_link(qlnx_host_t *ha, struct ecore_hwfn *hwfn, struct qlnx_link_output *if_link) { struct ecore_mcp_link_params link_params; struct ecore_mcp_link_state link_state; uint8_t p_change; struct ecore_ptt *p_ptt = NULL; memset(if_link, 0, sizeof(*if_link)); memset(&link_params, 0, sizeof(struct ecore_mcp_link_params)); memset(&link_state, 0, sizeof(struct ecore_mcp_link_state)); ha = (qlnx_host_t *)hwfn->p_dev; /* Prepare source inputs */ /* we only deal with physical functions */ if (qlnx_vf_device(ha) != 0) { p_ptt = ecore_ptt_acquire(hwfn); if (p_ptt == NULL) { QL_DPRINT1(ha, "ecore_ptt_acquire failed\n"); return; } ecore_mcp_get_media_type(hwfn, p_ptt, &if_link->media_type); ecore_ptt_release(hwfn, p_ptt); memcpy(&link_params, ecore_mcp_get_link_params(hwfn), sizeof(link_params)); memcpy(&link_state, ecore_mcp_get_link_state(hwfn), sizeof(link_state)); } else { ecore_mcp_get_media_type(hwfn, NULL, &if_link->media_type); ecore_vf_read_bulletin(hwfn, &p_change); ecore_vf_get_link_params(hwfn, &link_params); ecore_vf_get_link_state(hwfn, &link_state); } /* Set the link parameters to pass to protocol driver */ if (link_state.link_up) { if_link->link_up = true; if_link->speed = link_state.speed; } if_link->supported_caps = QLNX_LINK_CAP_FIBRE; if (link_params.speed.autoneg) if_link->supported_caps |= QLNX_LINK_CAP_Autoneg; if (link_params.pause.autoneg || (link_params.pause.forced_rx && link_params.pause.forced_tx)) if_link->supported_caps |= QLNX_LINK_CAP_Asym_Pause; if (link_params.pause.autoneg || link_params.pause.forced_rx || link_params.pause.forced_tx) if_link->supported_caps |= QLNX_LINK_CAP_Pause; if (link_params.speed.advertised_speeds & NVM_CFG1_PORT_DRV_SPEED_CAPABILITY_MASK_1G) if_link->supported_caps |= QLNX_LINK_CAP_1000baseT_Half | QLNX_LINK_CAP_1000baseT_Full; if (link_params.speed.advertised_speeds & NVM_CFG1_PORT_DRV_SPEED_CAPABILITY_MASK_10G) if_link->supported_caps |= QLNX_LINK_CAP_10000baseKR_Full; if (link_params.speed.advertised_speeds & NVM_CFG1_PORT_DRV_SPEED_CAPABILITY_MASK_25G) if_link->supported_caps |= QLNX_LINK_CAP_25000baseKR_Full; if (link_params.speed.advertised_speeds & NVM_CFG1_PORT_DRV_LINK_SPEED_40G) if_link->supported_caps |= QLNX_LINK_CAP_40000baseLR4_Full; if (link_params.speed.advertised_speeds & NVM_CFG1_PORT_DRV_SPEED_CAPABILITY_MASK_50G) if_link->supported_caps |= QLNX_LINK_CAP_50000baseKR2_Full; if (link_params.speed.advertised_speeds & NVM_CFG1_PORT_DRV_SPEED_CAPABILITY_MASK_BB_100G) if_link->supported_caps |= QLNX_LINK_CAP_100000baseKR4_Full; if_link->advertised_caps = if_link->supported_caps; if_link->autoneg = link_params.speed.autoneg; if_link->duplex = QLNX_LINK_DUPLEX; /* Link partner capabilities */ if (link_state.partner_adv_speed & ECORE_LINK_PARTNER_SPEED_1G_HD) if_link->link_partner_caps |= QLNX_LINK_CAP_1000baseT_Half; if (link_state.partner_adv_speed & ECORE_LINK_PARTNER_SPEED_1G_FD) if_link->link_partner_caps |= QLNX_LINK_CAP_1000baseT_Full; if (link_state.partner_adv_speed & ECORE_LINK_PARTNER_SPEED_10G) if_link->link_partner_caps |= QLNX_LINK_CAP_10000baseKR_Full; if (link_state.partner_adv_speed & ECORE_LINK_PARTNER_SPEED_25G) if_link->link_partner_caps |= QLNX_LINK_CAP_25000baseKR_Full; if (link_state.partner_adv_speed & ECORE_LINK_PARTNER_SPEED_40G) if_link->link_partner_caps |= QLNX_LINK_CAP_40000baseLR4_Full; if (link_state.partner_adv_speed & ECORE_LINK_PARTNER_SPEED_50G) if_link->link_partner_caps |= QLNX_LINK_CAP_50000baseKR2_Full; if (link_state.partner_adv_speed & ECORE_LINK_PARTNER_SPEED_100G) if_link->link_partner_caps |= QLNX_LINK_CAP_100000baseKR4_Full; if (link_state.an_complete) if_link->link_partner_caps |= QLNX_LINK_CAP_Autoneg; if (link_state.partner_adv_pause) if_link->link_partner_caps |= QLNX_LINK_CAP_Pause; if ((link_state.partner_adv_pause == ECORE_LINK_PARTNER_ASYMMETRIC_PAUSE) || (link_state.partner_adv_pause == ECORE_LINK_PARTNER_BOTH_PAUSE)) if_link->link_partner_caps |= QLNX_LINK_CAP_Asym_Pause; return; } void qlnx_schedule_recovery(void *p_hwfn) { qlnx_host_t *ha; ha = (qlnx_host_t *)((struct ecore_hwfn *)p_hwfn)->p_dev; if (qlnx_vf_device(ha) != 0) { taskqueue_enqueue(ha->err_taskqueue, &ha->err_task); } return; } static int qlnx_nic_setup(struct ecore_dev *cdev, struct ecore_pf_params *func_params) { int rc, i; for (i = 0; i < cdev->num_hwfns; i++) { struct ecore_hwfn *p_hwfn = &cdev->hwfns[i]; p_hwfn->pf_params = *func_params; #ifdef QLNX_ENABLE_IWARP if (qlnx_vf_device((qlnx_host_t *)cdev) != 0) { p_hwfn->using_ll2 = true; } #endif /* #ifdef QLNX_ENABLE_IWARP */ } rc = ecore_resc_alloc(cdev); if (rc) goto qlnx_nic_setup_exit; ecore_resc_setup(cdev); qlnx_nic_setup_exit: return rc; } static int qlnx_nic_start(struct ecore_dev *cdev) { int rc; struct ecore_hw_init_params params; bzero(¶ms, sizeof (struct ecore_hw_init_params)); params.p_tunn = NULL; params.b_hw_start = true; params.int_mode = cdev->int_mode; params.allow_npar_tx_switch = true; params.bin_fw_data = NULL; rc = ecore_hw_init(cdev, ¶ms); if (rc) { ecore_resc_free(cdev); return rc; } return 0; } static int qlnx_slowpath_start(qlnx_host_t *ha) { struct ecore_dev *cdev; struct ecore_pf_params pf_params; int rc; memset(&pf_params, 0, sizeof(struct ecore_pf_params)); pf_params.eth_pf_params.num_cons = (ha->num_rss) * (ha->num_tc + 1); #ifdef QLNX_ENABLE_IWARP if (qlnx_vf_device(ha) != 0) { if(ha->personality == ECORE_PCI_ETH_IWARP) { device_printf(ha->pci_dev, "setting parameters required by iWARP dev\n"); pf_params.rdma_pf_params.num_qps = 1024; pf_params.rdma_pf_params.num_srqs = 1024; pf_params.rdma_pf_params.gl_pi = ECORE_ROCE_PROTOCOL_INDEX; pf_params.rdma_pf_params.rdma_protocol = ECORE_RDMA_PROTOCOL_IWARP; } else if(ha->personality == ECORE_PCI_ETH_ROCE) { device_printf(ha->pci_dev, "setting parameters required by RoCE dev\n"); pf_params.rdma_pf_params.num_qps = 8192; pf_params.rdma_pf_params.num_srqs = 8192; //pf_params.rdma_pf_params.min_dpis = 0; pf_params.rdma_pf_params.min_dpis = 8; pf_params.rdma_pf_params.roce_edpm_mode = 0; pf_params.rdma_pf_params.gl_pi = ECORE_ROCE_PROTOCOL_INDEX; pf_params.rdma_pf_params.rdma_protocol = ECORE_RDMA_PROTOCOL_ROCE; } } #endif /* #ifdef QLNX_ENABLE_IWARP */ cdev = &ha->cdev; rc = qlnx_nic_setup(cdev, &pf_params); if (rc) goto qlnx_slowpath_start_exit; cdev->int_mode = ECORE_INT_MODE_MSIX; cdev->int_coalescing_mode = ECORE_COAL_MODE_ENABLE; #ifdef QLNX_MAX_COALESCE cdev->rx_coalesce_usecs = 255; cdev->tx_coalesce_usecs = 255; #endif rc = qlnx_nic_start(cdev); ha->rx_coalesce_usecs = cdev->rx_coalesce_usecs; ha->tx_coalesce_usecs = cdev->tx_coalesce_usecs; #ifdef QLNX_USER_LLDP (void)qlnx_set_lldp_tlvx(ha, NULL); #endif /* #ifdef QLNX_USER_LLDP */ qlnx_slowpath_start_exit: return (rc); } static int qlnx_slowpath_stop(qlnx_host_t *ha) { struct ecore_dev *cdev; device_t dev = ha->pci_dev; int i; cdev = &ha->cdev; ecore_hw_stop(cdev); for (i = 0; i < ha->cdev.num_hwfns; i++) { if (ha->sp_handle[i]) (void)bus_teardown_intr(dev, ha->sp_irq[i], ha->sp_handle[i]); ha->sp_handle[i] = NULL; if (ha->sp_irq[i]) (void) bus_release_resource(dev, SYS_RES_IRQ, ha->sp_irq_rid[i], ha->sp_irq[i]); ha->sp_irq[i] = NULL; } ecore_resc_free(cdev); return 0; } static void qlnx_set_id(struct ecore_dev *cdev, char name[NAME_SIZE], char ver_str[VER_SIZE]) { int i; memcpy(cdev->name, name, NAME_SIZE); for_each_hwfn(cdev, i) { snprintf(cdev->hwfns[i].name, NAME_SIZE, "%s-%d", name, i); } cdev->drv_type = DRV_ID_DRV_TYPE_FREEBSD; return ; } void qlnx_get_protocol_stats(void *cdev, int proto_type, void *proto_stats) { enum ecore_mcp_protocol_type type; union ecore_mcp_protocol_stats *stats; struct ecore_eth_stats eth_stats; qlnx_host_t *ha; ha = cdev; stats = proto_stats; type = proto_type; switch (type) { case ECORE_MCP_LAN_STATS: ecore_get_vport_stats((struct ecore_dev *)cdev, ð_stats); stats->lan_stats.ucast_rx_pkts = eth_stats.common.rx_ucast_pkts; stats->lan_stats.ucast_tx_pkts = eth_stats.common.tx_ucast_pkts; stats->lan_stats.fcs_err = -1; break; default: ha->err_get_proto_invalid_type++; QL_DPRINT1(ha, "invalid protocol type 0x%x\n", type); break; } return; } static int qlnx_get_mfw_version(qlnx_host_t *ha, uint32_t *mfw_ver) { struct ecore_hwfn *p_hwfn; struct ecore_ptt *p_ptt; p_hwfn = &ha->cdev.hwfns[0]; p_ptt = ecore_ptt_acquire(p_hwfn); if (p_ptt == NULL) { QL_DPRINT1(ha, "ecore_ptt_acquire failed\n"); return (-1); } ecore_mcp_get_mfw_ver(p_hwfn, p_ptt, mfw_ver, NULL); ecore_ptt_release(p_hwfn, p_ptt); return (0); } static int qlnx_get_flash_size(qlnx_host_t *ha, uint32_t *flash_size) { struct ecore_hwfn *p_hwfn; struct ecore_ptt *p_ptt; p_hwfn = &ha->cdev.hwfns[0]; p_ptt = ecore_ptt_acquire(p_hwfn); if (p_ptt == NULL) { QL_DPRINT1(ha,"ecore_ptt_acquire failed\n"); return (-1); } ecore_mcp_get_flash_size(p_hwfn, p_ptt, flash_size); ecore_ptt_release(p_hwfn, p_ptt); return (0); } static int qlnx_alloc_mem_arrays(qlnx_host_t *ha) { bzero(&ha->txq_array[0], (sizeof(struct qlnx_tx_queue) * QLNX_MAX_RSS)); bzero(&ha->rxq_array[0], (sizeof(struct qlnx_rx_queue) * QLNX_MAX_RSS)); bzero(&ha->sb_array[0], (sizeof(struct ecore_sb_info) * QLNX_MAX_RSS)); return 0; } static void qlnx_init_fp(qlnx_host_t *ha) { int rss_id, txq_array_index, tc; for (rss_id = 0; rss_id < ha->num_rss; rss_id++) { struct qlnx_fastpath *fp = &ha->fp_array[rss_id]; fp->rss_id = rss_id; fp->edev = ha; fp->sb_info = &ha->sb_array[rss_id]; fp->rxq = &ha->rxq_array[rss_id]; fp->rxq->rxq_id = rss_id; for (tc = 0; tc < ha->num_tc; tc++) { txq_array_index = tc * ha->num_rss + rss_id; fp->txq[tc] = &ha->txq_array[txq_array_index]; fp->txq[tc]->index = txq_array_index; } snprintf(fp->name, sizeof(fp->name), "%s-fp-%d", qlnx_name_str, rss_id); fp->tx_ring_full = 0; /* reset all the statistics counters */ fp->tx_pkts_processed = 0; fp->tx_pkts_freed = 0; fp->tx_pkts_transmitted = 0; fp->tx_pkts_completed = 0; #ifdef QLNX_TRACE_PERF_DATA fp->tx_pkts_trans_ctx = 0; fp->tx_pkts_compl_ctx = 0; fp->tx_pkts_trans_fp = 0; fp->tx_pkts_compl_fp = 0; fp->tx_pkts_compl_intr = 0; #endif fp->tx_lso_wnd_min_len = 0; fp->tx_defrag = 0; fp->tx_nsegs_gt_elem_left = 0; fp->tx_tso_max_nsegs = 0; fp->tx_tso_min_nsegs = 0; fp->err_tx_nsegs_gt_elem_left = 0; fp->err_tx_dmamap_create = 0; fp->err_tx_defrag_dmamap_load = 0; fp->err_tx_non_tso_max_seg = 0; fp->err_tx_dmamap_load = 0; fp->err_tx_defrag = 0; fp->err_tx_free_pkt_null = 0; fp->err_tx_cons_idx_conflict = 0; fp->rx_pkts = 0; fp->err_m_getcl = 0; fp->err_m_getjcl = 0; } return; } void qlnx_free_mem_sb(qlnx_host_t *ha, struct ecore_sb_info *sb_info) { struct ecore_dev *cdev; cdev = &ha->cdev; if (sb_info->sb_virt) { OSAL_DMA_FREE_COHERENT(cdev, ((void *)sb_info->sb_virt), (sb_info->sb_phys), (sizeof(*sb_info->sb_virt))); sb_info->sb_virt = NULL; } } static int qlnx_sb_init(struct ecore_dev *cdev, struct ecore_sb_info *sb_info, void *sb_virt_addr, bus_addr_t sb_phy_addr, u16 sb_id) { struct ecore_hwfn *p_hwfn; int hwfn_index, rc; u16 rel_sb_id; hwfn_index = sb_id % cdev->num_hwfns; p_hwfn = &cdev->hwfns[hwfn_index]; rel_sb_id = sb_id / cdev->num_hwfns; QL_DPRINT2(((qlnx_host_t *)cdev), "hwfn_index = %d p_hwfn = %p sb_id = 0x%x rel_sb_id = 0x%x \ sb_info = %p sb_virt_addr = %p sb_phy_addr = %p\n", hwfn_index, p_hwfn, sb_id, rel_sb_id, sb_info, sb_virt_addr, (void *)sb_phy_addr); rc = ecore_int_sb_init(p_hwfn, p_hwfn->p_main_ptt, sb_info, sb_virt_addr, sb_phy_addr, rel_sb_id); return rc; } /* This function allocates fast-path status block memory */ int qlnx_alloc_mem_sb(qlnx_host_t *ha, struct ecore_sb_info *sb_info, u16 sb_id) { struct status_block_e4 *sb_virt; bus_addr_t sb_phys; int rc; uint32_t size; struct ecore_dev *cdev; cdev = &ha->cdev; size = sizeof(*sb_virt); sb_virt = OSAL_DMA_ALLOC_COHERENT(cdev, (&sb_phys), size); if (!sb_virt) { QL_DPRINT1(ha, "Status block allocation failed\n"); return -ENOMEM; } rc = qlnx_sb_init(cdev, sb_info, sb_virt, sb_phys, sb_id); if (rc) { OSAL_DMA_FREE_COHERENT(cdev, sb_virt, sb_phys, size); } return rc; } static void qlnx_free_rx_buffers(qlnx_host_t *ha, struct qlnx_rx_queue *rxq) { int i; struct sw_rx_data *rx_buf; for (i = 0; i < rxq->num_rx_buffers; i++) { rx_buf = &rxq->sw_rx_ring[i]; if (rx_buf->data != NULL) { if (rx_buf->map != NULL) { bus_dmamap_unload(ha->rx_tag, rx_buf->map); bus_dmamap_destroy(ha->rx_tag, rx_buf->map); rx_buf->map = NULL; } m_freem(rx_buf->data); rx_buf->data = NULL; } } return; } static void qlnx_free_mem_rxq(qlnx_host_t *ha, struct qlnx_rx_queue *rxq) { struct ecore_dev *cdev; int i; cdev = &ha->cdev; qlnx_free_rx_buffers(ha, rxq); for (i = 0; i < ETH_TPA_MAX_AGGS_NUM; i++) { qlnx_free_tpa_mbuf(ha, &rxq->tpa_info[i]); if (rxq->tpa_info[i].mpf != NULL) m_freem(rxq->tpa_info[i].mpf); } bzero((void *)&rxq->sw_rx_ring[0], (sizeof (struct sw_rx_data) * RX_RING_SIZE)); /* Free the real RQ ring used by FW */ if (rxq->rx_bd_ring.p_virt_addr) { ecore_chain_free(cdev, &rxq->rx_bd_ring); rxq->rx_bd_ring.p_virt_addr = NULL; } /* Free the real completion ring used by FW */ if (rxq->rx_comp_ring.p_virt_addr && rxq->rx_comp_ring.pbl_sp.p_virt_table) { ecore_chain_free(cdev, &rxq->rx_comp_ring); rxq->rx_comp_ring.p_virt_addr = NULL; rxq->rx_comp_ring.pbl_sp.p_virt_table = NULL; } #ifdef QLNX_SOFT_LRO { struct lro_ctrl *lro; lro = &rxq->lro; tcp_lro_free(lro); } #endif /* #ifdef QLNX_SOFT_LRO */ return; } static int qlnx_alloc_rx_buffer(qlnx_host_t *ha, struct qlnx_rx_queue *rxq) { register struct mbuf *mp; uint16_t rx_buf_size; struct sw_rx_data *sw_rx_data; struct eth_rx_bd *rx_bd; dma_addr_t dma_addr; bus_dmamap_t map; bus_dma_segment_t segs[1]; int nsegs; int ret; rx_buf_size = rxq->rx_buf_size; mp = m_getjcl(M_NOWAIT, MT_DATA, M_PKTHDR, rx_buf_size); if (mp == NULL) { QL_DPRINT1(ha, "Failed to allocate Rx data\n"); return -ENOMEM; } mp->m_len = mp->m_pkthdr.len = rx_buf_size; map = (bus_dmamap_t)0; ret = bus_dmamap_load_mbuf_sg(ha->rx_tag, map, mp, segs, &nsegs, BUS_DMA_NOWAIT); dma_addr = segs[0].ds_addr; if (ret || !dma_addr || (nsegs != 1)) { m_freem(mp); QL_DPRINT1(ha, "bus_dmamap_load failed[%d, 0x%016llx, %d]\n", ret, (long long unsigned int)dma_addr, nsegs); return -ENOMEM; } sw_rx_data = &rxq->sw_rx_ring[rxq->sw_rx_prod]; sw_rx_data->data = mp; sw_rx_data->dma_addr = dma_addr; sw_rx_data->map = map; /* Advance PROD and get BD pointer */ rx_bd = (struct eth_rx_bd *)ecore_chain_produce(&rxq->rx_bd_ring); rx_bd->addr.hi = htole32(U64_HI(dma_addr)); rx_bd->addr.lo = htole32(U64_LO(dma_addr)); bus_dmamap_sync(ha->rx_tag, map, BUS_DMASYNC_PREREAD); rxq->sw_rx_prod = (rxq->sw_rx_prod + 1) & (RX_RING_SIZE - 1); return 0; } static int qlnx_alloc_tpa_mbuf(qlnx_host_t *ha, uint16_t rx_buf_size, struct qlnx_agg_info *tpa) { struct mbuf *mp; dma_addr_t dma_addr; bus_dmamap_t map; bus_dma_segment_t segs[1]; int nsegs; int ret; struct sw_rx_data *rx_buf; mp = m_getjcl(M_NOWAIT, MT_DATA, M_PKTHDR, rx_buf_size); if (mp == NULL) { QL_DPRINT1(ha, "Failed to allocate Rx data\n"); return -ENOMEM; } mp->m_len = mp->m_pkthdr.len = rx_buf_size; map = (bus_dmamap_t)0; ret = bus_dmamap_load_mbuf_sg(ha->rx_tag, map, mp, segs, &nsegs, BUS_DMA_NOWAIT); dma_addr = segs[0].ds_addr; if (ret || !dma_addr || (nsegs != 1)) { m_freem(mp); QL_DPRINT1(ha, "bus_dmamap_load failed[%d, 0x%016llx, %d]\n", ret, (long long unsigned int)dma_addr, nsegs); return -ENOMEM; } rx_buf = &tpa->rx_buf; memset(rx_buf, 0, sizeof (struct sw_rx_data)); rx_buf->data = mp; rx_buf->dma_addr = dma_addr; rx_buf->map = map; bus_dmamap_sync(ha->rx_tag, map, BUS_DMASYNC_PREREAD); return (0); } static void qlnx_free_tpa_mbuf(qlnx_host_t *ha, struct qlnx_agg_info *tpa) { struct sw_rx_data *rx_buf; rx_buf = &tpa->rx_buf; if (rx_buf->data != NULL) { if (rx_buf->map != NULL) { bus_dmamap_unload(ha->rx_tag, rx_buf->map); bus_dmamap_destroy(ha->rx_tag, rx_buf->map); rx_buf->map = NULL; } m_freem(rx_buf->data); rx_buf->data = NULL; } return; } /* This function allocates all memory needed per Rx queue */ static int qlnx_alloc_mem_rxq(qlnx_host_t *ha, struct qlnx_rx_queue *rxq) { int i, rc, num_allocated; struct ecore_dev *cdev; cdev = &ha->cdev; rxq->num_rx_buffers = RX_RING_SIZE; rxq->rx_buf_size = ha->rx_buf_size; /* Allocate the parallel driver ring for Rx buffers */ bzero((void *)&rxq->sw_rx_ring[0], (sizeof (struct sw_rx_data) * RX_RING_SIZE)); /* Allocate FW Rx ring */ rc = ecore_chain_alloc(cdev, ECORE_CHAIN_USE_TO_CONSUME_PRODUCE, ECORE_CHAIN_MODE_NEXT_PTR, ECORE_CHAIN_CNT_TYPE_U16, RX_RING_SIZE, sizeof(struct eth_rx_bd), &rxq->rx_bd_ring, NULL); if (rc) goto err; /* Allocate FW completion ring */ rc = ecore_chain_alloc(cdev, ECORE_CHAIN_USE_TO_CONSUME, ECORE_CHAIN_MODE_PBL, ECORE_CHAIN_CNT_TYPE_U16, RX_RING_SIZE, sizeof(union eth_rx_cqe), &rxq->rx_comp_ring, NULL); if (rc) goto err; /* Allocate buffers for the Rx ring */ for (i = 0; i < ETH_TPA_MAX_AGGS_NUM; i++) { rc = qlnx_alloc_tpa_mbuf(ha, rxq->rx_buf_size, &rxq->tpa_info[i]); if (rc) break; } for (i = 0; i < rxq->num_rx_buffers; i++) { rc = qlnx_alloc_rx_buffer(ha, rxq); if (rc) break; } num_allocated = i; if (!num_allocated) { QL_DPRINT1(ha, "Rx buffers allocation failed\n"); goto err; } else if (num_allocated < rxq->num_rx_buffers) { QL_DPRINT1(ha, "Allocated less buffers than" " desired (%d allocated)\n", num_allocated); } #ifdef QLNX_SOFT_LRO { struct lro_ctrl *lro; lro = &rxq->lro; if (tcp_lro_init_args(lro, ifp, 0, rxq->num_rx_buffers)) { QL_DPRINT1(ha, "tcp_lro_init[%d] failed\n", rxq->rxq_id); goto err; } lro->ifp = ha->ifp; } #endif /* #ifdef QLNX_SOFT_LRO */ return 0; err: qlnx_free_mem_rxq(ha, rxq); return -ENOMEM; } static void qlnx_free_mem_txq(qlnx_host_t *ha, struct qlnx_fastpath *fp, struct qlnx_tx_queue *txq) { struct ecore_dev *cdev; cdev = &ha->cdev; bzero((void *)&txq->sw_tx_ring[0], (sizeof (struct sw_tx_bd) * TX_RING_SIZE)); /* Free the real RQ ring used by FW */ if (txq->tx_pbl.p_virt_addr) { ecore_chain_free(cdev, &txq->tx_pbl); txq->tx_pbl.p_virt_addr = NULL; } return; } /* This function allocates all memory needed per Tx queue */ static int qlnx_alloc_mem_txq(qlnx_host_t *ha, struct qlnx_fastpath *fp, struct qlnx_tx_queue *txq) { int ret = ECORE_SUCCESS; union eth_tx_bd_types *p_virt; struct ecore_dev *cdev; cdev = &ha->cdev; bzero((void *)&txq->sw_tx_ring[0], (sizeof (struct sw_tx_bd) * TX_RING_SIZE)); /* Allocate the real Tx ring to be used by FW */ ret = ecore_chain_alloc(cdev, ECORE_CHAIN_USE_TO_CONSUME_PRODUCE, ECORE_CHAIN_MODE_PBL, ECORE_CHAIN_CNT_TYPE_U16, TX_RING_SIZE, sizeof(*p_virt), &txq->tx_pbl, NULL); if (ret != ECORE_SUCCESS) { goto err; } txq->num_tx_buffers = TX_RING_SIZE; return 0; err: qlnx_free_mem_txq(ha, fp, txq); return -ENOMEM; } static void qlnx_free_tx_br(qlnx_host_t *ha, struct qlnx_fastpath *fp) { struct mbuf *mp; - struct ifnet *ifp = ha->ifp; + if_t ifp = ha->ifp; if (mtx_initialized(&fp->tx_mtx)) { if (fp->tx_br != NULL) { mtx_lock(&fp->tx_mtx); while ((mp = drbr_dequeue(ifp, fp->tx_br)) != NULL) { fp->tx_pkts_freed++; m_freem(mp); } mtx_unlock(&fp->tx_mtx); buf_ring_free(fp->tx_br, M_DEVBUF); fp->tx_br = NULL; } mtx_destroy(&fp->tx_mtx); } return; } static void qlnx_free_mem_fp(qlnx_host_t *ha, struct qlnx_fastpath *fp) { int tc; qlnx_free_mem_sb(ha, fp->sb_info); qlnx_free_mem_rxq(ha, fp->rxq); for (tc = 0; tc < ha->num_tc; tc++) qlnx_free_mem_txq(ha, fp, fp->txq[tc]); return; } static int qlnx_alloc_tx_br(qlnx_host_t *ha, struct qlnx_fastpath *fp) { snprintf(fp->tx_mtx_name, sizeof(fp->tx_mtx_name), "qlnx%d_fp%d_tx_mq_lock", ha->dev_unit, fp->rss_id); mtx_init(&fp->tx_mtx, fp->tx_mtx_name, NULL, MTX_DEF); fp->tx_br = buf_ring_alloc(TX_RING_SIZE, M_DEVBUF, M_NOWAIT, &fp->tx_mtx); if (fp->tx_br == NULL) { QL_DPRINT1(ha, "buf_ring_alloc failed for fp[%d, %d]\n", ha->dev_unit, fp->rss_id); return -ENOMEM; } return 0; } static int qlnx_alloc_mem_fp(qlnx_host_t *ha, struct qlnx_fastpath *fp) { int rc, tc; rc = qlnx_alloc_mem_sb(ha, fp->sb_info, fp->rss_id); if (rc) goto err; if (ha->rx_jumbo_buf_eq_mtu) { if (ha->max_frame_size <= MCLBYTES) ha->rx_buf_size = MCLBYTES; else if (ha->max_frame_size <= MJUMPAGESIZE) ha->rx_buf_size = MJUMPAGESIZE; else if (ha->max_frame_size <= MJUM9BYTES) ha->rx_buf_size = MJUM9BYTES; else if (ha->max_frame_size <= MJUM16BYTES) ha->rx_buf_size = MJUM16BYTES; } else { if (ha->max_frame_size <= MCLBYTES) ha->rx_buf_size = MCLBYTES; else ha->rx_buf_size = MJUMPAGESIZE; } rc = qlnx_alloc_mem_rxq(ha, fp->rxq); if (rc) goto err; for (tc = 0; tc < ha->num_tc; tc++) { rc = qlnx_alloc_mem_txq(ha, fp, fp->txq[tc]); if (rc) goto err; } return 0; err: qlnx_free_mem_fp(ha, fp); return -ENOMEM; } static void qlnx_free_mem_load(qlnx_host_t *ha) { int i; for (i = 0; i < ha->num_rss; i++) { struct qlnx_fastpath *fp = &ha->fp_array[i]; qlnx_free_mem_fp(ha, fp); } return; } static int qlnx_alloc_mem_load(qlnx_host_t *ha) { int rc = 0, rss_id; for (rss_id = 0; rss_id < ha->num_rss; rss_id++) { struct qlnx_fastpath *fp = &ha->fp_array[rss_id]; rc = qlnx_alloc_mem_fp(ha, fp); if (rc) break; } return (rc); } static int qlnx_start_vport(struct ecore_dev *cdev, u8 vport_id, u16 mtu, u8 drop_ttl0_flg, u8 inner_vlan_removal_en_flg, u8 tx_switching, u8 hw_lro_enable) { int rc, i; struct ecore_sp_vport_start_params vport_start_params = { 0 }; qlnx_host_t *ha __unused; ha = (qlnx_host_t *)cdev; vport_start_params.remove_inner_vlan = inner_vlan_removal_en_flg; vport_start_params.tx_switching = 0; vport_start_params.handle_ptp_pkts = 0; vport_start_params.only_untagged = 0; vport_start_params.drop_ttl0 = drop_ttl0_flg; vport_start_params.tpa_mode = (hw_lro_enable ? ECORE_TPA_MODE_RSC : ECORE_TPA_MODE_NONE); vport_start_params.max_buffers_per_cqe = QLNX_TPA_MAX_AGG_BUFFERS; vport_start_params.vport_id = vport_id; vport_start_params.mtu = mtu; QL_DPRINT2(ha, "Setting mtu to %d and VPORT ID = %d\n", mtu, vport_id); for_each_hwfn(cdev, i) { struct ecore_hwfn *p_hwfn = &cdev->hwfns[i]; vport_start_params.concrete_fid = p_hwfn->hw_info.concrete_fid; vport_start_params.opaque_fid = p_hwfn->hw_info.opaque_fid; rc = ecore_sp_vport_start(p_hwfn, &vport_start_params); if (rc) { QL_DPRINT1(ha, "Failed to start VPORT V-PORT %d" " with MTU %d\n" , vport_id, mtu); return -ENOMEM; } ecore_hw_start_fastpath(p_hwfn); QL_DPRINT2(ha, "Started V-PORT %d with MTU %d\n", vport_id, mtu); } return 0; } static int qlnx_update_vport(struct ecore_dev *cdev, struct qlnx_update_vport_params *params) { struct ecore_sp_vport_update_params sp_params; int rc, i, j, fp_index; struct ecore_hwfn *p_hwfn; struct ecore_rss_params *rss; qlnx_host_t *ha = (qlnx_host_t *)cdev; struct qlnx_fastpath *fp; memset(&sp_params, 0, sizeof(sp_params)); /* Translate protocol params into sp params */ sp_params.vport_id = params->vport_id; sp_params.update_vport_active_rx_flg = params->update_vport_active_rx_flg; sp_params.vport_active_rx_flg = params->vport_active_rx_flg; sp_params.update_vport_active_tx_flg = params->update_vport_active_tx_flg; sp_params.vport_active_tx_flg = params->vport_active_tx_flg; sp_params.update_inner_vlan_removal_flg = params->update_inner_vlan_removal_flg; sp_params.inner_vlan_removal_flg = params->inner_vlan_removal_flg; sp_params.sge_tpa_params = params->sge_tpa_params; /* RSS - is a bit tricky, since upper-layer isn't familiar with hwfns. * We need to re-fix the rss values per engine for CMT. */ if (params->rss_params->update_rss_config) sp_params.rss_params = params->rss_params; else sp_params.rss_params = NULL; for_each_hwfn(cdev, i) { p_hwfn = &cdev->hwfns[i]; if ((cdev->num_hwfns > 1) && params->rss_params->update_rss_config && params->rss_params->rss_enable) { rss = params->rss_params; for (j = 0; j < ECORE_RSS_IND_TABLE_SIZE; j++) { fp_index = ((cdev->num_hwfns * j) + i) % ha->num_rss; fp = &ha->fp_array[fp_index]; rss->rss_ind_table[j] = fp->rxq->handle; } for (j = 0; j < ECORE_RSS_IND_TABLE_SIZE;) { QL_DPRINT3(ha, "%p %p %p %p %p %p %p %p \n", rss->rss_ind_table[j], rss->rss_ind_table[j+1], rss->rss_ind_table[j+2], rss->rss_ind_table[j+3], rss->rss_ind_table[j+4], rss->rss_ind_table[j+5], rss->rss_ind_table[j+6], rss->rss_ind_table[j+7]); j += 8; } } sp_params.opaque_fid = p_hwfn->hw_info.opaque_fid; QL_DPRINT1(ha, "Update sp vport ID=%d\n", params->vport_id); rc = ecore_sp_vport_update(p_hwfn, &sp_params, ECORE_SPQ_MODE_EBLOCK, NULL); if (rc) { QL_DPRINT1(ha, "Failed to update VPORT\n"); return rc; } QL_DPRINT2(ha, "Updated V-PORT %d: tx_active_flag %d, \ rx_active_flag %d [tx_update %d], [rx_update %d]\n", params->vport_id, params->vport_active_tx_flg, params->vport_active_rx_flg, params->update_vport_active_tx_flg, params->update_vport_active_rx_flg); } return 0; } static void qlnx_reuse_rx_data(struct qlnx_rx_queue *rxq) { struct eth_rx_bd *rx_bd_cons = ecore_chain_consume(&rxq->rx_bd_ring); struct eth_rx_bd *rx_bd_prod = ecore_chain_produce(&rxq->rx_bd_ring); struct sw_rx_data *sw_rx_data_cons = &rxq->sw_rx_ring[rxq->sw_rx_cons]; struct sw_rx_data *sw_rx_data_prod = &rxq->sw_rx_ring[rxq->sw_rx_prod]; sw_rx_data_prod->data = sw_rx_data_cons->data; memcpy(rx_bd_prod, rx_bd_cons, sizeof(struct eth_rx_bd)); rxq->sw_rx_cons = (rxq->sw_rx_cons + 1) & (RX_RING_SIZE - 1); rxq->sw_rx_prod = (rxq->sw_rx_prod + 1) & (RX_RING_SIZE - 1); return; } static void qlnx_update_rx_prod(struct ecore_hwfn *p_hwfn, struct qlnx_rx_queue *rxq) { uint16_t bd_prod; uint16_t cqe_prod; union { struct eth_rx_prod_data rx_prod_data; uint32_t data32; } rx_prods; bd_prod = ecore_chain_get_prod_idx(&rxq->rx_bd_ring); cqe_prod = ecore_chain_get_prod_idx(&rxq->rx_comp_ring); /* Update producers */ rx_prods.rx_prod_data.bd_prod = htole16(bd_prod); rx_prods.rx_prod_data.cqe_prod = htole16(cqe_prod); /* Make sure that the BD and SGE data is updated before updating the * producers since FW might read the BD/SGE right after the producer * is updated. */ wmb(); internal_ram_wr(p_hwfn, rxq->hw_rxq_prod_addr, sizeof(rx_prods), &rx_prods.data32); /* mmiowb is needed to synchronize doorbell writes from more than one * processor. It guarantees that the write arrives to the device before * the napi lock is released and another qlnx_poll is called (possibly * on another CPU). Without this barrier, the next doorbell can bypass * this doorbell. This is applicable to IA64/Altix systems. */ wmb(); return; } static uint32_t qlnx_hash_key[] = { ((0x6d << 24)|(0x5a << 16)|(0x56 << 8)|0xda), ((0x25 << 24)|(0x5b << 16)|(0x0e << 8)|0xc2), ((0x41 << 24)|(0x67 << 16)|(0x25 << 8)|0x3d), ((0x43 << 24)|(0xa3 << 16)|(0x8f << 8)|0xb0), ((0xd0 << 24)|(0xca << 16)|(0x2b << 8)|0xcb), ((0xae << 24)|(0x7b << 16)|(0x30 << 8)|0xb4), ((0x77 << 24)|(0xcb << 16)|(0x2d << 8)|0xa3), ((0x80 << 24)|(0x30 << 16)|(0xf2 << 8)|0x0c), ((0x6a << 24)|(0x42 << 16)|(0xb7 << 8)|0x3b), ((0xbe << 24)|(0xac << 16)|(0x01 << 8)|0xfa)}; static int qlnx_start_queues(qlnx_host_t *ha) { int rc, tc, i, vport_id = 0, drop_ttl0_flg = 1, vlan_removal_en = 1, tx_switching = 0, hw_lro_enable = 0; struct ecore_dev *cdev = &ha->cdev; struct ecore_rss_params *rss_params = &ha->rss_params; struct qlnx_update_vport_params vport_update_params; - struct ifnet *ifp; + if_t ifp; struct ecore_hwfn *p_hwfn; struct ecore_sge_tpa_params tpa_params; struct ecore_queue_start_common_params qparams; struct qlnx_fastpath *fp; ifp = ha->ifp; QL_DPRINT1(ha, "Num RSS = %d\n", ha->num_rss); if (!ha->num_rss) { QL_DPRINT1(ha, "Cannot update V-VPORT as active as there" " are no Rx queues\n"); return -EINVAL; } #ifndef QLNX_SOFT_LRO - hw_lro_enable = ifp->if_capenable & IFCAP_LRO; + hw_lro_enable = if_getcapenable(ifp) & IFCAP_LRO; #endif /* #ifndef QLNX_SOFT_LRO */ - rc = qlnx_start_vport(cdev, vport_id, ifp->if_mtu, drop_ttl0_flg, + rc = qlnx_start_vport(cdev, vport_id, if_getmtu(ifp), drop_ttl0_flg, vlan_removal_en, tx_switching, hw_lro_enable); if (rc) { QL_DPRINT1(ha, "Start V-PORT failed %d\n", rc); return rc; } QL_DPRINT2(ha, "Start vport ramrod passed, " "vport_id = %d, MTU = %d, vlan_removal_en = %d\n", - vport_id, (int)(ifp->if_mtu + 0xe), vlan_removal_en); + vport_id, (int)(if_getmtu(ifp) + 0xe), vlan_removal_en); for_each_rss(i) { struct ecore_rxq_start_ret_params rx_ret_params; struct ecore_txq_start_ret_params tx_ret_params; fp = &ha->fp_array[i]; p_hwfn = &cdev->hwfns[(fp->rss_id % cdev->num_hwfns)]; bzero(&qparams, sizeof(struct ecore_queue_start_common_params)); bzero(&rx_ret_params, sizeof (struct ecore_rxq_start_ret_params)); qparams.queue_id = i ; qparams.vport_id = vport_id; qparams.stats_id = vport_id; qparams.p_sb = fp->sb_info; qparams.sb_idx = RX_PI; rc = ecore_eth_rx_queue_start(p_hwfn, p_hwfn->hw_info.opaque_fid, &qparams, fp->rxq->rx_buf_size, /* bd_max_bytes */ /* bd_chain_phys_addr */ fp->rxq->rx_bd_ring.p_phys_addr, /* cqe_pbl_addr */ ecore_chain_get_pbl_phys(&fp->rxq->rx_comp_ring), /* cqe_pbl_size */ ecore_chain_get_page_cnt(&fp->rxq->rx_comp_ring), &rx_ret_params); if (rc) { QL_DPRINT1(ha, "Start RXQ #%d failed %d\n", i, rc); return rc; } fp->rxq->hw_rxq_prod_addr = rx_ret_params.p_prod; fp->rxq->handle = rx_ret_params.p_handle; fp->rxq->hw_cons_ptr = &fp->sb_info->sb_virt->pi_array[RX_PI]; qlnx_update_rx_prod(p_hwfn, fp->rxq); for (tc = 0; tc < ha->num_tc; tc++) { struct qlnx_tx_queue *txq = fp->txq[tc]; bzero(&qparams, sizeof(struct ecore_queue_start_common_params)); bzero(&tx_ret_params, sizeof (struct ecore_txq_start_ret_params)); qparams.queue_id = txq->index / cdev->num_hwfns ; qparams.vport_id = vport_id; qparams.stats_id = vport_id; qparams.p_sb = fp->sb_info; qparams.sb_idx = TX_PI(tc); rc = ecore_eth_tx_queue_start(p_hwfn, p_hwfn->hw_info.opaque_fid, &qparams, tc, /* bd_chain_phys_addr */ ecore_chain_get_pbl_phys(&txq->tx_pbl), ecore_chain_get_page_cnt(&txq->tx_pbl), &tx_ret_params); if (rc) { QL_DPRINT1(ha, "Start TXQ #%d failed %d\n", txq->index, rc); return rc; } txq->doorbell_addr = tx_ret_params.p_doorbell; txq->handle = tx_ret_params.p_handle; txq->hw_cons_ptr = &fp->sb_info->sb_virt->pi_array[TX_PI(tc)]; SET_FIELD(txq->tx_db.data.params, ETH_DB_DATA_DEST, DB_DEST_XCM); SET_FIELD(txq->tx_db.data.params, ETH_DB_DATA_AGG_CMD, DB_AGG_CMD_SET); SET_FIELD(txq->tx_db.data.params, ETH_DB_DATA_AGG_VAL_SEL, DQ_XCM_ETH_TX_BD_PROD_CMD); txq->tx_db.data.agg_flags = DQ_XCM_ETH_DQ_CF_CMD; } } /* Fill struct with RSS params */ if (ha->num_rss > 1) { rss_params->update_rss_config = 1; rss_params->rss_enable = 1; rss_params->update_rss_capabilities = 1; rss_params->update_rss_ind_table = 1; rss_params->update_rss_key = 1; rss_params->rss_caps = ECORE_RSS_IPV4 | ECORE_RSS_IPV6 | ECORE_RSS_IPV4_TCP | ECORE_RSS_IPV6_TCP; rss_params->rss_table_size_log = 7; /* 2^7 = 128 */ for (i = 0; i < ECORE_RSS_IND_TABLE_SIZE; i++) { fp = &ha->fp_array[(i % ha->num_rss)]; rss_params->rss_ind_table[i] = fp->rxq->handle; } for (i = 0; i < ECORE_RSS_KEY_SIZE; i++) rss_params->rss_key[i] = (__le32)qlnx_hash_key[i]; } else { memset(rss_params, 0, sizeof(*rss_params)); } /* Prepare and send the vport enable */ memset(&vport_update_params, 0, sizeof(vport_update_params)); vport_update_params.vport_id = vport_id; vport_update_params.update_vport_active_tx_flg = 1; vport_update_params.vport_active_tx_flg = 1; vport_update_params.update_vport_active_rx_flg = 1; vport_update_params.vport_active_rx_flg = 1; vport_update_params.rss_params = rss_params; vport_update_params.update_inner_vlan_removal_flg = 1; vport_update_params.inner_vlan_removal_flg = 1; if (hw_lro_enable) { memset(&tpa_params, 0, sizeof (struct ecore_sge_tpa_params)); tpa_params.max_buffers_per_cqe = QLNX_TPA_MAX_AGG_BUFFERS; tpa_params.update_tpa_en_flg = 1; tpa_params.tpa_ipv4_en_flg = 1; tpa_params.tpa_ipv6_en_flg = 1; tpa_params.update_tpa_param_flg = 1; tpa_params.tpa_pkt_split_flg = 0; tpa_params.tpa_hdr_data_split_flg = 0; tpa_params.tpa_gro_consistent_flg = 0; tpa_params.tpa_max_aggs_num = ETH_TPA_MAX_AGGS_NUM; tpa_params.tpa_max_size = (uint16_t)(-1); - tpa_params.tpa_min_size_to_start = ifp->if_mtu/2; - tpa_params.tpa_min_size_to_cont = ifp->if_mtu/2; + tpa_params.tpa_min_size_to_start = if_getmtu(ifp) / 2; + tpa_params.tpa_min_size_to_cont = if_getmtu(ifp) / 2; vport_update_params.sge_tpa_params = &tpa_params; } rc = qlnx_update_vport(cdev, &vport_update_params); if (rc) { QL_DPRINT1(ha, "Update V-PORT failed %d\n", rc); return rc; } return 0; } static int qlnx_drain_txq(qlnx_host_t *ha, struct qlnx_fastpath *fp, struct qlnx_tx_queue *txq) { uint16_t hw_bd_cons; uint16_t ecore_cons_idx; QL_DPRINT2(ha, "enter\n"); hw_bd_cons = le16toh(*txq->hw_cons_ptr); while (hw_bd_cons != (ecore_cons_idx = ecore_chain_get_cons_idx(&txq->tx_pbl))) { mtx_lock(&fp->tx_mtx); (void)qlnx_tx_int(ha, fp, txq); mtx_unlock(&fp->tx_mtx); qlnx_mdelay(__func__, 2); hw_bd_cons = le16toh(*txq->hw_cons_ptr); } QL_DPRINT2(ha, "[%d, %d]: done\n", fp->rss_id, txq->index); return 0; } static int qlnx_stop_queues(qlnx_host_t *ha) { struct qlnx_update_vport_params vport_update_params; struct ecore_dev *cdev; struct qlnx_fastpath *fp; int rc, tc, i; cdev = &ha->cdev; /* Disable the vport */ memset(&vport_update_params, 0, sizeof(vport_update_params)); vport_update_params.vport_id = 0; vport_update_params.update_vport_active_tx_flg = 1; vport_update_params.vport_active_tx_flg = 0; vport_update_params.update_vport_active_rx_flg = 1; vport_update_params.vport_active_rx_flg = 0; vport_update_params.rss_params = &ha->rss_params; vport_update_params.rss_params->update_rss_config = 0; vport_update_params.rss_params->rss_enable = 0; vport_update_params.update_inner_vlan_removal_flg = 0; vport_update_params.inner_vlan_removal_flg = 0; QL_DPRINT1(ha, "Update vport ID= %d\n", vport_update_params.vport_id); rc = qlnx_update_vport(cdev, &vport_update_params); if (rc) { QL_DPRINT1(ha, "Failed to update vport\n"); return rc; } /* Flush Tx queues. If needed, request drain from MCP */ for_each_rss(i) { fp = &ha->fp_array[i]; for (tc = 0; tc < ha->num_tc; tc++) { struct qlnx_tx_queue *txq = fp->txq[tc]; rc = qlnx_drain_txq(ha, fp, txq); if (rc) return rc; } } /* Stop all Queues in reverse order*/ for (i = ha->num_rss - 1; i >= 0; i--) { struct ecore_hwfn *p_hwfn = &cdev->hwfns[(i % cdev->num_hwfns)]; fp = &ha->fp_array[i]; /* Stop the Tx Queue(s)*/ for (tc = 0; tc < ha->num_tc; tc++) { int tx_queue_id __unused; tx_queue_id = tc * ha->num_rss + i; rc = ecore_eth_tx_queue_stop(p_hwfn, fp->txq[tc]->handle); if (rc) { QL_DPRINT1(ha, "Failed to stop TXQ #%d\n", tx_queue_id); return rc; } } /* Stop the Rx Queue*/ rc = ecore_eth_rx_queue_stop(p_hwfn, fp->rxq->handle, false, false); if (rc) { QL_DPRINT1(ha, "Failed to stop RXQ #%d\n", i); return rc; } } /* Stop the vport */ for_each_hwfn(cdev, i) { struct ecore_hwfn *p_hwfn = &cdev->hwfns[i]; rc = ecore_sp_vport_stop(p_hwfn, p_hwfn->hw_info.opaque_fid, 0); if (rc) { QL_DPRINT1(ha, "Failed to stop VPORT\n"); return rc; } } return rc; } static int qlnx_set_ucast_rx_mac(qlnx_host_t *ha, enum ecore_filter_opcode opcode, unsigned char mac[ETH_ALEN]) { struct ecore_filter_ucast ucast; struct ecore_dev *cdev; int rc; cdev = &ha->cdev; bzero(&ucast, sizeof(struct ecore_filter_ucast)); ucast.opcode = opcode; ucast.type = ECORE_FILTER_MAC; ucast.is_rx_filter = 1; ucast.vport_to_add_to = 0; memcpy(&ucast.mac[0], mac, ETH_ALEN); rc = ecore_filter_ucast_cmd(cdev, &ucast, ECORE_SPQ_MODE_CB, NULL); return (rc); } static int qlnx_remove_all_ucast_mac(qlnx_host_t *ha) { struct ecore_filter_ucast ucast; struct ecore_dev *cdev; int rc; bzero(&ucast, sizeof(struct ecore_filter_ucast)); ucast.opcode = ECORE_FILTER_REPLACE; ucast.type = ECORE_FILTER_MAC; ucast.is_rx_filter = 1; cdev = &ha->cdev; rc = ecore_filter_ucast_cmd(cdev, &ucast, ECORE_SPQ_MODE_CB, NULL); return (rc); } static int qlnx_remove_all_mcast_mac(qlnx_host_t *ha) { struct ecore_filter_mcast *mcast; struct ecore_dev *cdev; int rc, i; cdev = &ha->cdev; mcast = &ha->ecore_mcast; bzero(mcast, sizeof(struct ecore_filter_mcast)); mcast->opcode = ECORE_FILTER_REMOVE; for (i = 0; i < QLNX_MAX_NUM_MULTICAST_ADDRS; i++) { if (ha->mcast[i].addr[0] || ha->mcast[i].addr[1] || ha->mcast[i].addr[2] || ha->mcast[i].addr[3] || ha->mcast[i].addr[4] || ha->mcast[i].addr[5]) { memcpy(&mcast->mac[i][0], &ha->mcast[i].addr[0], ETH_ALEN); mcast->num_mc_addrs++; } } mcast = &ha->ecore_mcast; rc = ecore_filter_mcast_cmd(cdev, mcast, ECORE_SPQ_MODE_CB, NULL); bzero(ha->mcast, (sizeof(qlnx_mcast_t) * QLNX_MAX_NUM_MULTICAST_ADDRS)); ha->nmcast = 0; return (rc); } static int qlnx_clean_filters(qlnx_host_t *ha) { int rc = 0; /* Remove all unicast macs */ rc = qlnx_remove_all_ucast_mac(ha); if (rc) return rc; /* Remove all multicast macs */ rc = qlnx_remove_all_mcast_mac(ha); if (rc) return rc; rc = qlnx_set_ucast_rx_mac(ha, ECORE_FILTER_FLUSH, ha->primary_mac); return (rc); } static int qlnx_set_rx_accept_filter(qlnx_host_t *ha, uint8_t filter) { struct ecore_filter_accept_flags accept; int rc = 0; struct ecore_dev *cdev; cdev = &ha->cdev; bzero(&accept, sizeof(struct ecore_filter_accept_flags)); accept.update_rx_mode_config = 1; accept.rx_accept_filter = filter; accept.update_tx_mode_config = 1; accept.tx_accept_filter = ECORE_ACCEPT_UCAST_MATCHED | ECORE_ACCEPT_MCAST_MATCHED | ECORE_ACCEPT_BCAST; rc = ecore_filter_accept_cmd(cdev, 0, accept, false, false, ECORE_SPQ_MODE_CB, NULL); return (rc); } static int qlnx_set_rx_mode(qlnx_host_t *ha) { int rc = 0; uint8_t filter; rc = qlnx_set_ucast_rx_mac(ha, ECORE_FILTER_REPLACE, ha->primary_mac); if (rc) return rc; rc = qlnx_remove_all_mcast_mac(ha); if (rc) return rc; filter = ECORE_ACCEPT_UCAST_MATCHED | ECORE_ACCEPT_MCAST_MATCHED | ECORE_ACCEPT_BCAST; if (qlnx_vf_device(ha) == 0) { filter |= ECORE_ACCEPT_UCAST_UNMATCHED; filter |= ECORE_ACCEPT_MCAST_UNMATCHED; } ha->filter = filter; rc = qlnx_set_rx_accept_filter(ha, filter); return (rc); } static int qlnx_set_link(qlnx_host_t *ha, bool link_up) { int i, rc = 0; struct ecore_dev *cdev; struct ecore_hwfn *hwfn; struct ecore_ptt *ptt; if (qlnx_vf_device(ha) == 0) return (0); cdev = &ha->cdev; for_each_hwfn(cdev, i) { hwfn = &cdev->hwfns[i]; ptt = ecore_ptt_acquire(hwfn); if (!ptt) return -EBUSY; rc = ecore_mcp_set_link(hwfn, ptt, link_up); ecore_ptt_release(hwfn, ptt); if (rc) return rc; } return (rc); } static uint64_t qlnx_get_counter(if_t ifp, ift_counter cnt) { qlnx_host_t *ha; uint64_t count; ha = (qlnx_host_t *)if_getsoftc(ifp); switch (cnt) { case IFCOUNTER_IPACKETS: count = ha->hw_stats.common.rx_ucast_pkts + ha->hw_stats.common.rx_mcast_pkts + ha->hw_stats.common.rx_bcast_pkts; break; case IFCOUNTER_IERRORS: count = ha->hw_stats.common.rx_crc_errors + ha->hw_stats.common.rx_align_errors + ha->hw_stats.common.rx_oversize_packets + ha->hw_stats.common.rx_undersize_packets; break; case IFCOUNTER_OPACKETS: count = ha->hw_stats.common.tx_ucast_pkts + ha->hw_stats.common.tx_mcast_pkts + ha->hw_stats.common.tx_bcast_pkts; break; case IFCOUNTER_OERRORS: count = ha->hw_stats.common.tx_err_drop_pkts; break; case IFCOUNTER_COLLISIONS: return (0); case IFCOUNTER_IBYTES: count = ha->hw_stats.common.rx_ucast_bytes + ha->hw_stats.common.rx_mcast_bytes + ha->hw_stats.common.rx_bcast_bytes; break; case IFCOUNTER_OBYTES: count = ha->hw_stats.common.tx_ucast_bytes + ha->hw_stats.common.tx_mcast_bytes + ha->hw_stats.common.tx_bcast_bytes; break; case IFCOUNTER_IMCASTS: count = ha->hw_stats.common.rx_mcast_bytes; break; case IFCOUNTER_OMCASTS: count = ha->hw_stats.common.tx_mcast_bytes; break; case IFCOUNTER_IQDROPS: case IFCOUNTER_OQDROPS: case IFCOUNTER_NOPROTO: default: return (if_get_counter_default(ifp, cnt)); } return (count); } static void qlnx_timer(void *arg) { qlnx_host_t *ha; ha = (qlnx_host_t *)arg; if (ha->error_recovery) { ha->error_recovery = 0; taskqueue_enqueue(ha->err_taskqueue, &ha->err_task); return; } ecore_get_vport_stats(&ha->cdev, &ha->hw_stats); if (ha->storm_stats_gather) qlnx_sample_storm_stats(ha); callout_reset(&ha->qlnx_callout, hz, qlnx_timer, ha); return; } static int qlnx_load(qlnx_host_t *ha) { int i; int rc = 0; device_t dev; dev = ha->pci_dev; QL_DPRINT2(ha, "enter\n"); rc = qlnx_alloc_mem_arrays(ha); if (rc) goto qlnx_load_exit0; qlnx_init_fp(ha); rc = qlnx_alloc_mem_load(ha); if (rc) goto qlnx_load_exit1; QL_DPRINT2(ha, "Allocated %d RSS queues on %d TC/s\n", ha->num_rss, ha->num_tc); for (i = 0; i < ha->num_rss; i++) { if ((rc = bus_setup_intr(dev, ha->irq_vec[i].irq, (INTR_TYPE_NET | INTR_MPSAFE), NULL, qlnx_fp_isr, &ha->irq_vec[i], &ha->irq_vec[i].handle))) { QL_DPRINT1(ha, "could not setup interrupt\n"); goto qlnx_load_exit2; } QL_DPRINT2(ha, "rss_id = %d irq_rid %d \ irq %p handle %p\n", i, ha->irq_vec[i].irq_rid, ha->irq_vec[i].irq, ha->irq_vec[i].handle); bus_bind_intr(dev, ha->irq_vec[i].irq, (i % mp_ncpus)); } rc = qlnx_start_queues(ha); if (rc) goto qlnx_load_exit2; QL_DPRINT2(ha, "Start VPORT, RXQ and TXQ succeeded\n"); /* Add primary mac and set Rx filters */ rc = qlnx_set_rx_mode(ha); if (rc) goto qlnx_load_exit2; /* Ask for link-up using current configuration */ qlnx_set_link(ha, true); if (qlnx_vf_device(ha) == 0) qlnx_link_update(&ha->cdev.hwfns[0]); ha->state = QLNX_STATE_OPEN; bzero(&ha->hw_stats, sizeof(struct ecore_eth_stats)); if (ha->flags.callout_init) callout_reset(&ha->qlnx_callout, hz, qlnx_timer, ha); goto qlnx_load_exit0; qlnx_load_exit2: qlnx_free_mem_load(ha); qlnx_load_exit1: ha->num_rss = 0; qlnx_load_exit0: QL_DPRINT2(ha, "exit [%d]\n", rc); return rc; } static void qlnx_drain_soft_lro(qlnx_host_t *ha) { #ifdef QLNX_SOFT_LRO - struct ifnet *ifp; + if_t ifp; int i; ifp = ha->ifp; - if (ifp->if_capenable & IFCAP_LRO) { + if (if_getcapenable(ifp) & IFCAP_LRO) { for (i = 0; i < ha->num_rss; i++) { struct qlnx_fastpath *fp = &ha->fp_array[i]; struct lro_ctrl *lro; lro = &fp->rxq->lro; tcp_lro_flush_all(lro); } } #endif /* #ifdef QLNX_SOFT_LRO */ return; } static void qlnx_unload(qlnx_host_t *ha) { struct ecore_dev *cdev; device_t dev; int i; cdev = &ha->cdev; dev = ha->pci_dev; QL_DPRINT2(ha, "enter\n"); QL_DPRINT1(ha, " QLNX STATE = %d\n",ha->state); if (ha->state == QLNX_STATE_OPEN) { qlnx_set_link(ha, false); qlnx_clean_filters(ha); qlnx_stop_queues(ha); ecore_hw_stop_fastpath(cdev); for (i = 0; i < ha->num_rss; i++) { if (ha->irq_vec[i].handle) { (void)bus_teardown_intr(dev, ha->irq_vec[i].irq, ha->irq_vec[i].handle); ha->irq_vec[i].handle = NULL; } } qlnx_drain_fp_taskqueues(ha); qlnx_drain_soft_lro(ha); qlnx_free_mem_load(ha); } if (ha->flags.callout_init) callout_drain(&ha->qlnx_callout); qlnx_mdelay(__func__, 1000); ha->state = QLNX_STATE_CLOSED; QL_DPRINT2(ha, "exit\n"); return; } static int qlnx_grc_dumpsize(qlnx_host_t *ha, uint32_t *num_dwords, int hwfn_index) { int rval = -1; struct ecore_hwfn *p_hwfn; struct ecore_ptt *p_ptt; ecore_dbg_set_app_ver(ecore_dbg_get_fw_func_ver()); p_hwfn = &ha->cdev.hwfns[hwfn_index]; p_ptt = ecore_ptt_acquire(p_hwfn); if (!p_ptt) { QL_DPRINT1(ha, "ecore_ptt_acquire failed\n"); return (rval); } rval = ecore_dbg_grc_get_dump_buf_size(p_hwfn, p_ptt, num_dwords); if (rval == DBG_STATUS_OK) rval = 0; else { QL_DPRINT1(ha, "ecore_dbg_grc_get_dump_buf_size failed" "[0x%x]\n", rval); } ecore_ptt_release(p_hwfn, p_ptt); return (rval); } static int qlnx_idle_chk_size(qlnx_host_t *ha, uint32_t *num_dwords, int hwfn_index) { int rval = -1; struct ecore_hwfn *p_hwfn; struct ecore_ptt *p_ptt; ecore_dbg_set_app_ver(ecore_dbg_get_fw_func_ver()); p_hwfn = &ha->cdev.hwfns[hwfn_index]; p_ptt = ecore_ptt_acquire(p_hwfn); if (!p_ptt) { QL_DPRINT1(ha, "ecore_ptt_acquire failed\n"); return (rval); } rval = ecore_dbg_idle_chk_get_dump_buf_size(p_hwfn, p_ptt, num_dwords); if (rval == DBG_STATUS_OK) rval = 0; else { QL_DPRINT1(ha, "ecore_dbg_idle_chk_get_dump_buf_size failed" " [0x%x]\n", rval); } ecore_ptt_release(p_hwfn, p_ptt); return (rval); } static void qlnx_sample_storm_stats(qlnx_host_t *ha) { int i, index; struct ecore_dev *cdev; qlnx_storm_stats_t *s_stats; uint32_t reg; struct ecore_ptt *p_ptt; struct ecore_hwfn *hwfn; if (ha->storm_stats_index >= QLNX_STORM_STATS_SAMPLES_PER_HWFN) { ha->storm_stats_gather = 0; return; } cdev = &ha->cdev; for_each_hwfn(cdev, i) { hwfn = &cdev->hwfns[i]; p_ptt = ecore_ptt_acquire(hwfn); if (!p_ptt) return; index = ha->storm_stats_index + (i * QLNX_STORM_STATS_SAMPLES_PER_HWFN); s_stats = &ha->storm_stats[index]; /* XSTORM */ reg = XSEM_REG_FAST_MEMORY + SEM_FAST_REG_STORM_ACTIVE_CYCLES_BB_K2; s_stats->xstorm_active_cycles = ecore_rd(hwfn, p_ptt, reg); reg = XSEM_REG_FAST_MEMORY + SEM_FAST_REG_STORM_STALL_CYCLES_BB_K2; s_stats->xstorm_stall_cycles = ecore_rd(hwfn, p_ptt, reg); reg = XSEM_REG_FAST_MEMORY + SEM_FAST_REG_IDLE_SLEEPING_CYCLES_BB_K2; s_stats->xstorm_sleeping_cycles = ecore_rd(hwfn, p_ptt, reg); reg = XSEM_REG_FAST_MEMORY + SEM_FAST_REG_IDLE_INACTIVE_CYCLES_BB_K2; s_stats->xstorm_inactive_cycles = ecore_rd(hwfn, p_ptt, reg); /* YSTORM */ reg = YSEM_REG_FAST_MEMORY + SEM_FAST_REG_STORM_ACTIVE_CYCLES_BB_K2; s_stats->ystorm_active_cycles = ecore_rd(hwfn, p_ptt, reg); reg = YSEM_REG_FAST_MEMORY + SEM_FAST_REG_STORM_STALL_CYCLES_BB_K2; s_stats->ystorm_stall_cycles = ecore_rd(hwfn, p_ptt, reg); reg = YSEM_REG_FAST_MEMORY + SEM_FAST_REG_IDLE_SLEEPING_CYCLES_BB_K2; s_stats->ystorm_sleeping_cycles = ecore_rd(hwfn, p_ptt, reg); reg = YSEM_REG_FAST_MEMORY + SEM_FAST_REG_IDLE_INACTIVE_CYCLES_BB_K2; s_stats->ystorm_inactive_cycles = ecore_rd(hwfn, p_ptt, reg); /* PSTORM */ reg = PSEM_REG_FAST_MEMORY + SEM_FAST_REG_STORM_ACTIVE_CYCLES_BB_K2; s_stats->pstorm_active_cycles = ecore_rd(hwfn, p_ptt, reg); reg = PSEM_REG_FAST_MEMORY + SEM_FAST_REG_STORM_STALL_CYCLES_BB_K2; s_stats->pstorm_stall_cycles = ecore_rd(hwfn, p_ptt, reg); reg = PSEM_REG_FAST_MEMORY + SEM_FAST_REG_IDLE_SLEEPING_CYCLES_BB_K2; s_stats->pstorm_sleeping_cycles = ecore_rd(hwfn, p_ptt, reg); reg = PSEM_REG_FAST_MEMORY + SEM_FAST_REG_IDLE_INACTIVE_CYCLES_BB_K2; s_stats->pstorm_inactive_cycles = ecore_rd(hwfn, p_ptt, reg); /* TSTORM */ reg = TSEM_REG_FAST_MEMORY + SEM_FAST_REG_STORM_ACTIVE_CYCLES_BB_K2; s_stats->tstorm_active_cycles = ecore_rd(hwfn, p_ptt, reg); reg = TSEM_REG_FAST_MEMORY + SEM_FAST_REG_STORM_STALL_CYCLES_BB_K2; s_stats->tstorm_stall_cycles = ecore_rd(hwfn, p_ptt, reg); reg = TSEM_REG_FAST_MEMORY + SEM_FAST_REG_IDLE_SLEEPING_CYCLES_BB_K2; s_stats->tstorm_sleeping_cycles = ecore_rd(hwfn, p_ptt, reg); reg = TSEM_REG_FAST_MEMORY + SEM_FAST_REG_IDLE_INACTIVE_CYCLES_BB_K2; s_stats->tstorm_inactive_cycles = ecore_rd(hwfn, p_ptt, reg); /* MSTORM */ reg = MSEM_REG_FAST_MEMORY + SEM_FAST_REG_STORM_ACTIVE_CYCLES_BB_K2; s_stats->mstorm_active_cycles = ecore_rd(hwfn, p_ptt, reg); reg = MSEM_REG_FAST_MEMORY + SEM_FAST_REG_STORM_STALL_CYCLES_BB_K2; s_stats->mstorm_stall_cycles = ecore_rd(hwfn, p_ptt, reg); reg = MSEM_REG_FAST_MEMORY + SEM_FAST_REG_IDLE_SLEEPING_CYCLES_BB_K2; s_stats->mstorm_sleeping_cycles = ecore_rd(hwfn, p_ptt, reg); reg = MSEM_REG_FAST_MEMORY + SEM_FAST_REG_IDLE_INACTIVE_CYCLES_BB_K2; s_stats->mstorm_inactive_cycles = ecore_rd(hwfn, p_ptt, reg); /* USTORM */ reg = USEM_REG_FAST_MEMORY + SEM_FAST_REG_STORM_ACTIVE_CYCLES_BB_K2; s_stats->ustorm_active_cycles = ecore_rd(hwfn, p_ptt, reg); reg = USEM_REG_FAST_MEMORY + SEM_FAST_REG_STORM_STALL_CYCLES_BB_K2; s_stats->ustorm_stall_cycles = ecore_rd(hwfn, p_ptt, reg); reg = USEM_REG_FAST_MEMORY + SEM_FAST_REG_IDLE_SLEEPING_CYCLES_BB_K2; s_stats->ustorm_sleeping_cycles = ecore_rd(hwfn, p_ptt, reg); reg = USEM_REG_FAST_MEMORY + SEM_FAST_REG_IDLE_INACTIVE_CYCLES_BB_K2; s_stats->ustorm_inactive_cycles = ecore_rd(hwfn, p_ptt, reg); ecore_ptt_release(hwfn, p_ptt); } ha->storm_stats_index++; return; } /* * Name: qlnx_dump_buf8 * Function: dumps a buffer as bytes */ static void qlnx_dump_buf8(qlnx_host_t *ha, const char *msg, void *dbuf, uint32_t len) { device_t dev; uint32_t i = 0; uint8_t *buf; dev = ha->pci_dev; buf = dbuf; device_printf(dev, "%s: %s 0x%x dump start\n", __func__, msg, len); while (len >= 16) { device_printf(dev,"0x%08x:" " %02x %02x %02x %02x %02x %02x %02x %02x" " %02x %02x %02x %02x %02x %02x %02x %02x\n", i, buf[0], buf[1], buf[2], buf[3], buf[4], buf[5], buf[6], buf[7], buf[8], buf[9], buf[10], buf[11], buf[12], buf[13], buf[14], buf[15]); i += 16; len -= 16; buf += 16; } switch (len) { case 1: device_printf(dev,"0x%08x: %02x\n", i, buf[0]); break; case 2: device_printf(dev,"0x%08x: %02x %02x\n", i, buf[0], buf[1]); break; case 3: device_printf(dev,"0x%08x: %02x %02x %02x\n", i, buf[0], buf[1], buf[2]); break; case 4: device_printf(dev,"0x%08x: %02x %02x %02x %02x\n", i, buf[0], buf[1], buf[2], buf[3]); break; case 5: device_printf(dev,"0x%08x:" " %02x %02x %02x %02x %02x\n", i, buf[0], buf[1], buf[2], buf[3], buf[4]); break; case 6: device_printf(dev,"0x%08x:" " %02x %02x %02x %02x %02x %02x\n", i, buf[0], buf[1], buf[2], buf[3], buf[4], buf[5]); break; case 7: device_printf(dev,"0x%08x:" " %02x %02x %02x %02x %02x %02x %02x\n", i, buf[0], buf[1], buf[2], buf[3], buf[4], buf[5], buf[6]); break; case 8: device_printf(dev,"0x%08x:" " %02x %02x %02x %02x %02x %02x %02x %02x\n", i, buf[0], buf[1], buf[2], buf[3], buf[4], buf[5], buf[6], buf[7]); break; case 9: device_printf(dev,"0x%08x:" " %02x %02x %02x %02x %02x %02x %02x %02x" " %02x\n", i, buf[0], buf[1], buf[2], buf[3], buf[4], buf[5], buf[6], buf[7], buf[8]); break; case 10: device_printf(dev,"0x%08x:" " %02x %02x %02x %02x %02x %02x %02x %02x" " %02x %02x\n", i, buf[0], buf[1], buf[2], buf[3], buf[4], buf[5], buf[6], buf[7], buf[8], buf[9]); break; case 11: device_printf(dev,"0x%08x:" " %02x %02x %02x %02x %02x %02x %02x %02x" " %02x %02x %02x\n", i, buf[0], buf[1], buf[2], buf[3], buf[4], buf[5], buf[6], buf[7], buf[8], buf[9], buf[10]); break; case 12: device_printf(dev,"0x%08x:" " %02x %02x %02x %02x %02x %02x %02x %02x" " %02x %02x %02x %02x\n", i, buf[0], buf[1], buf[2], buf[3], buf[4], buf[5], buf[6], buf[7], buf[8], buf[9], buf[10], buf[11]); break; case 13: device_printf(dev,"0x%08x:" " %02x %02x %02x %02x %02x %02x %02x %02x" " %02x %02x %02x %02x %02x\n", i, buf[0], buf[1], buf[2], buf[3], buf[4], buf[5], buf[6], buf[7], buf[8], buf[9], buf[10], buf[11], buf[12]); break; case 14: device_printf(dev,"0x%08x:" " %02x %02x %02x %02x %02x %02x %02x %02x" " %02x %02x %02x %02x %02x %02x\n", i, buf[0], buf[1], buf[2], buf[3], buf[4], buf[5], buf[6], buf[7], buf[8], buf[9], buf[10], buf[11], buf[12], buf[13]); break; case 15: device_printf(dev,"0x%08x:" " %02x %02x %02x %02x %02x %02x %02x %02x" " %02x %02x %02x %02x %02x %02x %02x\n", i, buf[0], buf[1], buf[2], buf[3], buf[4], buf[5], buf[6], buf[7], buf[8], buf[9], buf[10], buf[11], buf[12], buf[13], buf[14]); break; default: break; } device_printf(dev, "%s: %s dump end\n", __func__, msg); return; } #ifdef CONFIG_ECORE_SRIOV static void __qlnx_osal_iov_vf_cleanup(struct ecore_hwfn *p_hwfn, uint8_t rel_vf_id) { struct ecore_public_vf_info *vf_info; vf_info = ecore_iov_get_public_vf_info(p_hwfn, rel_vf_id, false); if (!vf_info) return; /* Clear the VF mac */ memset(vf_info->forced_mac, 0, ETH_ALEN); vf_info->forced_vlan = 0; return; } void qlnx_osal_iov_vf_cleanup(void *p_hwfn, uint8_t relative_vf_id) { __qlnx_osal_iov_vf_cleanup(p_hwfn, relative_vf_id); return; } static int __qlnx_iov_chk_ucast(struct ecore_hwfn *p_hwfn, int vfid, struct ecore_filter_ucast *params) { struct ecore_public_vf_info *vf; if (!ecore_iov_vf_has_vport_instance(p_hwfn, vfid)) { QL_DPRINT1(((qlnx_host_t *)p_hwfn->p_dev), "VF[%d] vport not initialized\n", vfid); return ECORE_INVAL; } vf = ecore_iov_get_public_vf_info(p_hwfn, vfid, true); if (!vf) return -EINVAL; /* No real decision to make; Store the configured MAC */ if (params->type == ECORE_FILTER_MAC || params->type == ECORE_FILTER_MAC_VLAN) memcpy(params->mac, vf->forced_mac, ETH_ALEN); return 0; } int qlnx_iov_chk_ucast(void *p_hwfn, int vfid, void *params) { return (__qlnx_iov_chk_ucast(p_hwfn, vfid, params)); } static int __qlnx_iov_update_vport(struct ecore_hwfn *hwfn, uint8_t vfid, struct ecore_sp_vport_update_params *params, uint16_t * tlvs) { if (!ecore_iov_vf_has_vport_instance(hwfn, vfid)) { QL_DPRINT1(((qlnx_host_t *)hwfn->p_dev), "VF[%d] vport not initialized\n", vfid); return ECORE_INVAL; } /* Untrusted VFs can't even be trusted to know that fact. * Simply indicate everything is configured fine, and trace * configuration 'behind their back'. */ if (!(*tlvs & BIT(ECORE_IOV_VP_UPDATE_ACCEPT_PARAM))) return 0; return 0; } int qlnx_iov_update_vport(void *hwfn, uint8_t vfid, void *params, uint16_t *tlvs) { return(__qlnx_iov_update_vport(hwfn, vfid, params, tlvs)); } static int qlnx_find_hwfn_index(struct ecore_hwfn *p_hwfn) { int i; struct ecore_dev *cdev; cdev = p_hwfn->p_dev; for (i = 0; i < cdev->num_hwfns; i++) { if (&cdev->hwfns[i] == p_hwfn) break; } if (i >= cdev->num_hwfns) return (-1); return (i); } static int __qlnx_pf_vf_msg(struct ecore_hwfn *p_hwfn, uint16_t rel_vf_id) { qlnx_host_t *ha = (qlnx_host_t *)p_hwfn->p_dev; int i; QL_DPRINT2(ha, "ha = %p cdev = %p p_hwfn = %p rel_vf_id = %d\n", ha, p_hwfn->p_dev, p_hwfn, rel_vf_id); if ((i = qlnx_find_hwfn_index(p_hwfn)) == -1) return (-1); if (ha->sriov_task[i].pf_taskqueue != NULL) { atomic_testandset_32(&ha->sriov_task[i].flags, QLNX_SRIOV_TASK_FLAGS_VF_PF_MSG); taskqueue_enqueue(ha->sriov_task[i].pf_taskqueue, &ha->sriov_task[i].pf_task); } return (ECORE_SUCCESS); } int qlnx_pf_vf_msg(void *p_hwfn, uint16_t relative_vf_id) { return (__qlnx_pf_vf_msg(p_hwfn, relative_vf_id)); } static void __qlnx_vf_flr_update(struct ecore_hwfn *p_hwfn) { qlnx_host_t *ha = (qlnx_host_t *)p_hwfn->p_dev; int i; if (!ha->sriov_initialized) return; QL_DPRINT2(ha, "ha = %p cdev = %p p_hwfn = %p \n", ha, p_hwfn->p_dev, p_hwfn); if ((i = qlnx_find_hwfn_index(p_hwfn)) == -1) return; if (ha->sriov_task[i].pf_taskqueue != NULL) { atomic_testandset_32(&ha->sriov_task[i].flags, QLNX_SRIOV_TASK_FLAGS_VF_FLR_UPDATE); taskqueue_enqueue(ha->sriov_task[i].pf_taskqueue, &ha->sriov_task[i].pf_task); } return; } void qlnx_vf_flr_update(void *p_hwfn) { __qlnx_vf_flr_update(p_hwfn); return; } #ifndef QLNX_VF static void qlnx_vf_bulleting_update(struct ecore_hwfn *p_hwfn) { qlnx_host_t *ha = (qlnx_host_t *)p_hwfn->p_dev; int i; QL_DPRINT2(ha, "ha = %p cdev = %p p_hwfn = %p \n", ha, p_hwfn->p_dev, p_hwfn); if ((i = qlnx_find_hwfn_index(p_hwfn)) == -1) return; QL_DPRINT2(ha, "ha = %p cdev = %p p_hwfn = %p i = %d\n", ha, p_hwfn->p_dev, p_hwfn, i); if (ha->sriov_task[i].pf_taskqueue != NULL) { atomic_testandset_32(&ha->sriov_task[i].flags, QLNX_SRIOV_TASK_FLAGS_BULLETIN_UPDATE); taskqueue_enqueue(ha->sriov_task[i].pf_taskqueue, &ha->sriov_task[i].pf_task); } } static void qlnx_initialize_sriov(qlnx_host_t *ha) { device_t dev; nvlist_t *pf_schema, *vf_schema; int iov_error; dev = ha->pci_dev; pf_schema = pci_iov_schema_alloc_node(); vf_schema = pci_iov_schema_alloc_node(); pci_iov_schema_add_unicast_mac(vf_schema, "mac-addr", 0, NULL); pci_iov_schema_add_bool(vf_schema, "allow-set-mac", IOV_SCHEMA_HASDEFAULT, FALSE); pci_iov_schema_add_bool(vf_schema, "allow-promisc", IOV_SCHEMA_HASDEFAULT, FALSE); pci_iov_schema_add_uint16(vf_schema, "num-queues", IOV_SCHEMA_HASDEFAULT, 1); iov_error = pci_iov_attach(dev, pf_schema, vf_schema); if (iov_error != 0) { ha->sriov_initialized = 0; } else { device_printf(dev, "SRIOV initialized\n"); ha->sriov_initialized = 1; } return; } static void qlnx_sriov_disable(qlnx_host_t *ha) { struct ecore_dev *cdev; int i, j; cdev = &ha->cdev; ecore_iov_set_vfs_to_disable(cdev, true); for_each_hwfn(cdev, i) { struct ecore_hwfn *hwfn = &cdev->hwfns[i]; struct ecore_ptt *ptt = ecore_ptt_acquire(hwfn); if (!ptt) { QL_DPRINT1(ha, "Failed to acquire ptt\n"); return; } /* Clean WFQ db and configure equal weight for all vports */ ecore_clean_wfq_db(hwfn, ptt); ecore_for_each_vf(hwfn, j) { int k = 0; if (!ecore_iov_is_valid_vfid(hwfn, j, true, false)) continue; if (ecore_iov_is_vf_started(hwfn, j)) { /* Wait until VF is disabled before releasing */ for (k = 0; k < 100; k++) { if (!ecore_iov_is_vf_stopped(hwfn, j)) { qlnx_mdelay(__func__, 10); } else break; } } if (k < 100) ecore_iov_release_hw_for_vf(&cdev->hwfns[i], ptt, j); else { QL_DPRINT1(ha, "Timeout waiting for VF's FLR to end\n"); } } ecore_ptt_release(hwfn, ptt); } ecore_iov_set_vfs_to_disable(cdev, false); return; } static void qlnx_sriov_enable_qid_config(struct ecore_hwfn *hwfn, u16 vfid, struct ecore_iov_vf_init_params *params) { u16 base, i; /* Since we have an equal resource distribution per-VF, and we assume * PF has acquired the ECORE_PF_L2_QUE first queues, we start setting * sequentially from there. */ base = FEAT_NUM(hwfn, ECORE_PF_L2_QUE) + vfid * params->num_queues; params->rel_vf_id = vfid; for (i = 0; i < params->num_queues; i++) { params->req_rx_queue[i] = base + i; params->req_tx_queue[i] = base + i; } /* PF uses indices 0 for itself; Set vport/RSS afterwards */ params->vport_id = vfid + 1; params->rss_eng_id = vfid + 1; return; } static int qlnx_iov_init(device_t dev, uint16_t num_vfs, const nvlist_t *nvlist_params) { qlnx_host_t *ha; struct ecore_dev *cdev; struct ecore_iov_vf_init_params params; int ret, j, i; uint32_t max_vfs; if ((ha = device_get_softc(dev)) == NULL) { device_printf(dev, "%s: cannot get softc\n", __func__); return (-1); } if (qlnx_create_pf_taskqueues(ha) != 0) goto qlnx_iov_init_err0; cdev = &ha->cdev; max_vfs = RESC_NUM(&cdev->hwfns[0], ECORE_VPORT); QL_DPRINT2(ha," dev = %p enter num_vfs = %d max_vfs = %d\n", dev, num_vfs, max_vfs); if (num_vfs >= max_vfs) { QL_DPRINT1(ha, "Can start at most %d VFs\n", (RESC_NUM(&cdev->hwfns[0], ECORE_VPORT) - 1)); goto qlnx_iov_init_err0; } ha->vf_attr = malloc(((sizeof (qlnx_vf_attr_t) * num_vfs)), M_QLNXBUF, M_NOWAIT); if (ha->vf_attr == NULL) goto qlnx_iov_init_err0; memset(¶ms, 0, sizeof(params)); /* Initialize HW for VF access */ for_each_hwfn(cdev, j) { struct ecore_hwfn *hwfn = &cdev->hwfns[j]; struct ecore_ptt *ptt = ecore_ptt_acquire(hwfn); /* Make sure not to use more than 16 queues per VF */ params.num_queues = min_t(int, (FEAT_NUM(hwfn, ECORE_VF_L2_QUE) / num_vfs), 16); if (!ptt) { QL_DPRINT1(ha, "Failed to acquire ptt\n"); goto qlnx_iov_init_err1; } for (i = 0; i < num_vfs; i++) { if (!ecore_iov_is_valid_vfid(hwfn, i, false, true)) continue; qlnx_sriov_enable_qid_config(hwfn, i, ¶ms); ret = ecore_iov_init_hw_for_vf(hwfn, ptt, ¶ms); if (ret) { QL_DPRINT1(ha, "Failed to enable VF[%d]\n", i); ecore_ptt_release(hwfn, ptt); goto qlnx_iov_init_err1; } } ecore_ptt_release(hwfn, ptt); } ha->num_vfs = num_vfs; qlnx_inform_vf_link_state(&cdev->hwfns[0], ha); QL_DPRINT2(ha," dev = %p exit num_vfs = %d\n", dev, num_vfs); return (0); qlnx_iov_init_err1: qlnx_sriov_disable(ha); qlnx_iov_init_err0: qlnx_destroy_pf_taskqueues(ha); ha->num_vfs = 0; return (-1); } static void qlnx_iov_uninit(device_t dev) { qlnx_host_t *ha; if ((ha = device_get_softc(dev)) == NULL) { device_printf(dev, "%s: cannot get softc\n", __func__); return; } QL_DPRINT2(ha," dev = %p enter\n", dev); qlnx_sriov_disable(ha); qlnx_destroy_pf_taskqueues(ha); free(ha->vf_attr, M_QLNXBUF); ha->vf_attr = NULL; ha->num_vfs = 0; QL_DPRINT2(ha," dev = %p exit\n", dev); return; } static int qlnx_iov_add_vf(device_t dev, uint16_t vfnum, const nvlist_t *params) { qlnx_host_t *ha; qlnx_vf_attr_t *vf_attr; unsigned const char *mac; size_t size; struct ecore_hwfn *p_hwfn; if ((ha = device_get_softc(dev)) == NULL) { device_printf(dev, "%s: cannot get softc\n", __func__); return (-1); } QL_DPRINT2(ha," dev = %p enter vfnum = %d\n", dev, vfnum); if (vfnum > (ha->num_vfs - 1)) { QL_DPRINT1(ha, " VF[%d] is greater than max allowed [%d]\n", vfnum, (ha->num_vfs - 1)); } vf_attr = &ha->vf_attr[vfnum]; if (nvlist_exists_binary(params, "mac-addr")) { mac = nvlist_get_binary(params, "mac-addr", &size); bcopy(mac, vf_attr->mac_addr, ETHER_ADDR_LEN); device_printf(dev, "%s: mac_addr = %02x:%02x:%02x:%02x:%02x:%02x\n", __func__, vf_attr->mac_addr[0], vf_attr->mac_addr[1], vf_attr->mac_addr[2], vf_attr->mac_addr[3], vf_attr->mac_addr[4], vf_attr->mac_addr[5]); p_hwfn = &ha->cdev.hwfns[0]; ecore_iov_bulletin_set_mac(p_hwfn, vf_attr->mac_addr, vfnum); } QL_DPRINT2(ha," dev = %p exit vfnum = %d\n", dev, vfnum); return (0); } static void qlnx_handle_vf_msg(qlnx_host_t *ha, struct ecore_hwfn *p_hwfn) { uint64_t events[ECORE_VF_ARRAY_LENGTH]; struct ecore_ptt *ptt; int i; ptt = ecore_ptt_acquire(p_hwfn); if (!ptt) { QL_DPRINT1(ha, "Can't acquire PTT; re-scheduling\n"); __qlnx_pf_vf_msg(p_hwfn, 0); return; } ecore_iov_pf_get_pending_events(p_hwfn, events); QL_DPRINT2(ha, "Event mask of VF events:" "0x%" PRIu64 "0x%" PRIu64 " 0x%" PRIu64 "\n", events[0], events[1], events[2]); ecore_for_each_vf(p_hwfn, i) { /* Skip VFs with no pending messages */ if (!(events[i / 64] & (1ULL << (i % 64)))) continue; QL_DPRINT2(ha, "Handling VF message from VF 0x%02x [Abs 0x%02x]\n", i, p_hwfn->p_dev->p_iov_info->first_vf_in_pf + i); /* Copy VF's message to PF's request buffer for that VF */ if (ecore_iov_copy_vf_msg(p_hwfn, ptt, i)) continue; ecore_iov_process_mbx_req(p_hwfn, ptt, i); } ecore_ptt_release(p_hwfn, ptt); return; } static void qlnx_handle_vf_flr_update(qlnx_host_t *ha, struct ecore_hwfn *p_hwfn) { struct ecore_ptt *ptt; int ret; ptt = ecore_ptt_acquire(p_hwfn); if (!ptt) { QL_DPRINT1(ha, "Can't acquire PTT; re-scheduling\n"); __qlnx_vf_flr_update(p_hwfn); return; } ret = ecore_iov_vf_flr_cleanup(p_hwfn, ptt); if (ret) { QL_DPRINT1(ha, "ecore_iov_vf_flr_cleanup failed; re-scheduling\n"); } ecore_ptt_release(p_hwfn, ptt); return; } static void qlnx_handle_bulletin_update(qlnx_host_t *ha, struct ecore_hwfn *p_hwfn) { struct ecore_ptt *ptt; int i; ptt = ecore_ptt_acquire(p_hwfn); if (!ptt) { QL_DPRINT1(ha, "Can't acquire PTT; re-scheduling\n"); qlnx_vf_bulleting_update(p_hwfn); return; } ecore_for_each_vf(p_hwfn, i) { QL_DPRINT1(ha, "ecore_iov_post_vf_bulletin[%p, %d]\n", p_hwfn, i); ecore_iov_post_vf_bulletin(p_hwfn, i, ptt); } ecore_ptt_release(p_hwfn, ptt); return; } static void qlnx_pf_taskqueue(void *context, int pending) { struct ecore_hwfn *p_hwfn; qlnx_host_t *ha; int i; p_hwfn = context; if (p_hwfn == NULL) return; ha = (qlnx_host_t *)(p_hwfn->p_dev); if ((i = qlnx_find_hwfn_index(p_hwfn)) == -1) return; if (atomic_testandclear_32(&ha->sriov_task[i].flags, QLNX_SRIOV_TASK_FLAGS_VF_PF_MSG)) qlnx_handle_vf_msg(ha, p_hwfn); if (atomic_testandclear_32(&ha->sriov_task[i].flags, QLNX_SRIOV_TASK_FLAGS_VF_FLR_UPDATE)) qlnx_handle_vf_flr_update(ha, p_hwfn); if (atomic_testandclear_32(&ha->sriov_task[i].flags, QLNX_SRIOV_TASK_FLAGS_BULLETIN_UPDATE)) qlnx_handle_bulletin_update(ha, p_hwfn); return; } static int qlnx_create_pf_taskqueues(qlnx_host_t *ha) { int i; uint8_t tq_name[32]; for (i = 0; i < ha->cdev.num_hwfns; i++) { struct ecore_hwfn *p_hwfn = &ha->cdev.hwfns[i]; bzero(tq_name, sizeof (tq_name)); snprintf(tq_name, sizeof (tq_name), "ql_pf_tq_%d", i); TASK_INIT(&ha->sriov_task[i].pf_task, 0, qlnx_pf_taskqueue, p_hwfn); ha->sriov_task[i].pf_taskqueue = taskqueue_create(tq_name, M_NOWAIT, taskqueue_thread_enqueue, &ha->sriov_task[i].pf_taskqueue); if (ha->sriov_task[i].pf_taskqueue == NULL) return (-1); taskqueue_start_threads(&ha->sriov_task[i].pf_taskqueue, 1, PI_NET, "%s", tq_name); QL_DPRINT1(ha, "%p\n", ha->sriov_task[i].pf_taskqueue); } return (0); } static void qlnx_destroy_pf_taskqueues(qlnx_host_t *ha) { int i; for (i = 0; i < ha->cdev.num_hwfns; i++) { if (ha->sriov_task[i].pf_taskqueue != NULL) { taskqueue_drain(ha->sriov_task[i].pf_taskqueue, &ha->sriov_task[i].pf_task); taskqueue_free(ha->sriov_task[i].pf_taskqueue); ha->sriov_task[i].pf_taskqueue = NULL; } } return; } static void qlnx_inform_vf_link_state(struct ecore_hwfn *p_hwfn, qlnx_host_t *ha) { struct ecore_mcp_link_capabilities caps; struct ecore_mcp_link_params params; struct ecore_mcp_link_state link; int i; if (!p_hwfn->pf_iov_info) return; memset(¶ms, 0, sizeof(struct ecore_mcp_link_params)); memset(&link, 0, sizeof(struct ecore_mcp_link_state)); memset(&caps, 0, sizeof(struct ecore_mcp_link_capabilities)); memcpy(&caps, ecore_mcp_get_link_capabilities(p_hwfn), sizeof(caps)); memcpy(&link, ecore_mcp_get_link_state(p_hwfn), sizeof(link)); memcpy(¶ms, ecore_mcp_get_link_params(p_hwfn), sizeof(params)); QL_DPRINT2(ha, "called\n"); /* Update bulletin of all future possible VFs with link configuration */ for (i = 0; i < p_hwfn->p_dev->p_iov_info->total_vfs; i++) { /* Modify link according to the VF's configured link state */ link.link_up = false; if (ha->link_up) { link.link_up = true; /* Set speed according to maximum supported by HW. * that is 40G for regular devices and 100G for CMT * mode devices. */ link.speed = (p_hwfn->p_dev->num_hwfns > 1) ? 100000 : link.speed; } QL_DPRINT2(ha, "link [%d] = %d\n", i, link.link_up); ecore_iov_set_link(p_hwfn, i, ¶ms, &link, &caps); } qlnx_vf_bulleting_update(p_hwfn); return; } #endif /* #ifndef QLNX_VF */ #endif /* #ifdef CONFIG_ECORE_SRIOV */ diff --git a/sys/dev/qlnx/qlnxr/qlnxr_cm.c b/sys/dev/qlnx/qlnxr/qlnxr_cm.c index 1f418f83ad37..569dbc6d7472 100644 --- a/sys/dev/qlnx/qlnxr/qlnxr_cm.c +++ b/sys/dev/qlnx/qlnxr/qlnxr_cm.c @@ -1,859 +1,859 @@ /* * Copyright (c) 2018-2019 Cavium, Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include "qlnxr_def.h" #include "rdma_common.h" #include "qlnxr_cm.h" void qlnxr_inc_sw_gsi_cons(struct qlnxr_qp_hwq_info *info) { info->gsi_cons = (info->gsi_cons + 1) % info->max_wr; } void qlnxr_store_gsi_qp_cq(struct qlnxr_dev *dev, struct qlnxr_qp *qp, struct ib_qp_init_attr *attrs) { QL_DPRINT12(dev->ha, "enter\n"); dev->gsi_qp_created = 1; dev->gsi_sqcq = get_qlnxr_cq((attrs->send_cq)); dev->gsi_rqcq = get_qlnxr_cq((attrs->recv_cq)); dev->gsi_qp = qp; QL_DPRINT12(dev->ha, "exit\n"); return; } void qlnxr_ll2_complete_tx_packet(void *cxt, uint8_t connection_handle, void *cookie, dma_addr_t first_frag_addr, bool b_last_fragment, bool b_last_packet) { struct qlnxr_dev *dev = (struct qlnxr_dev *)cxt; struct ecore_roce_ll2_packet *pkt = cookie; struct qlnxr_cq *cq = dev->gsi_sqcq; struct qlnxr_qp *qp = dev->gsi_qp; unsigned long flags; QL_DPRINT12(dev->ha, "enter\n"); qlnx_dma_free_coherent(&dev->ha->cdev, pkt->header.vaddr, pkt->header.baddr, pkt->header.len); kfree(pkt); spin_lock_irqsave(&qp->q_lock, flags); qlnxr_inc_sw_gsi_cons(&qp->sq); spin_unlock_irqrestore(&qp->q_lock, flags); if (cq->ibcq.comp_handler) (*cq->ibcq.comp_handler) (&cq->ibcq, cq->ibcq.cq_context); QL_DPRINT12(dev->ha, "exit\n"); return; } void qlnxr_ll2_complete_rx_packet(void *cxt, struct ecore_ll2_comp_rx_data *data) { struct qlnxr_dev *dev = (struct qlnxr_dev *)cxt; struct qlnxr_cq *cq = dev->gsi_rqcq; // struct qlnxr_qp *qp = dev->gsi_qp; struct qlnxr_qp *qp = NULL; unsigned long flags; // uint32_t delay_count = 0, gsi_cons = 0; //void * dest_va; QL_DPRINT12(dev->ha, "enter\n"); if (data->u.data_length_error) { /* TODO: add statistic */ } if (data->cookie == NULL) { QL_DPRINT12(dev->ha, "cookie is NULL, bad sign\n"); } if (data->qp_id == 1) { qp = dev->gsi_qp; } else { /* TODO: This will be needed for UD QP support */ /* For RoCEv1 this is invalid */ QL_DPRINT12(dev->ha, "invalid QP\n"); return; } /* note: currently only one recv sg is supported */ QL_DPRINT12(dev->ha, "MAD received on QP : %x\n", data->rx_buf_addr); spin_lock_irqsave(&qp->q_lock, flags); qp->rqe_wr_id[qp->rq.gsi_cons].rc = data->u.data_length_error ? -EINVAL : 0; qp->rqe_wr_id[qp->rq.gsi_cons].vlan_id = data->vlan; /* note: length stands for data length i.e. GRH is excluded */ qp->rqe_wr_id[qp->rq.gsi_cons].sg_list[0].length = data->length.data_length; *((u32 *)&qp->rqe_wr_id[qp->rq.gsi_cons].smac[0]) = ntohl(data->opaque_data_0); *((u16 *)&qp->rqe_wr_id[qp->rq.gsi_cons].smac[4]) = ntohs((u16)data->opaque_data_1); qlnxr_inc_sw_gsi_cons(&qp->rq); spin_unlock_irqrestore(&qp->q_lock, flags); if (cq->ibcq.comp_handler) (*cq->ibcq.comp_handler) (&cq->ibcq, cq->ibcq.cq_context); QL_DPRINT12(dev->ha, "exit\n"); return; } void qlnxr_ll2_release_rx_packet(void *cxt, u8 connection_handle, void *cookie, dma_addr_t rx_buf_addr, bool b_last_packet) { /* Do nothing... */ } static void qlnxr_destroy_gsi_cq(struct qlnxr_dev *dev, struct ib_qp_init_attr *attrs) { struct ecore_rdma_destroy_cq_in_params iparams; struct ecore_rdma_destroy_cq_out_params oparams; struct qlnxr_cq *cq; QL_DPRINT12(dev->ha, "enter\n"); cq = get_qlnxr_cq((attrs->send_cq)); iparams.icid = cq->icid; ecore_rdma_destroy_cq(dev->rdma_ctx, &iparams, &oparams); ecore_chain_free(&dev->ha->cdev, &cq->pbl); cq = get_qlnxr_cq((attrs->recv_cq)); /* if a dedicated recv_cq was used, delete it too */ if (iparams.icid != cq->icid) { iparams.icid = cq->icid; ecore_rdma_destroy_cq(dev->rdma_ctx, &iparams, &oparams); ecore_chain_free(&dev->ha->cdev, &cq->pbl); } QL_DPRINT12(dev->ha, "exit\n"); return; } static inline int qlnxr_check_gsi_qp_attrs(struct qlnxr_dev *dev, struct ib_qp_init_attr *attrs) { QL_DPRINT12(dev->ha, "enter\n"); if (attrs->cap.max_recv_sge > QLNXR_GSI_MAX_RECV_SGE) { QL_DPRINT11(dev->ha, "(attrs->cap.max_recv_sge > QLNXR_GSI_MAX_RECV_SGE)\n"); return -EINVAL; } if (attrs->cap.max_recv_wr > QLNXR_GSI_MAX_RECV_WR) { QL_DPRINT11(dev->ha, "(attrs->cap.max_recv_wr > QLNXR_GSI_MAX_RECV_WR)\n"); return -EINVAL; } if (attrs->cap.max_send_wr > QLNXR_GSI_MAX_SEND_WR) { QL_DPRINT11(dev->ha, "(attrs->cap.max_send_wr > QLNXR_GSI_MAX_SEND_WR)\n"); return -EINVAL; } QL_DPRINT12(dev->ha, "exit\n"); return 0; } static int qlnxr_ll2_post_tx(struct qlnxr_dev *dev, struct ecore_roce_ll2_packet *pkt) { enum ecore_ll2_roce_flavor_type roce_flavor; struct ecore_ll2_tx_pkt_info ll2_tx_pkt; int rc; int i; QL_DPRINT12(dev->ha, "enter\n"); memset(&ll2_tx_pkt, 0, sizeof(ll2_tx_pkt)); if (pkt->roce_mode != ROCE_V1) { QL_DPRINT11(dev->ha, "roce_mode != ROCE_V1\n"); return (-1); } roce_flavor = (pkt->roce_mode == ROCE_V1) ? ECORE_LL2_ROCE : ECORE_LL2_RROCE; ll2_tx_pkt.num_of_bds = 1 /* hdr */ + pkt->n_seg; ll2_tx_pkt.vlan = 0; /* ??? */ ll2_tx_pkt.tx_dest = ECORE_LL2_TX_DEST_NW; ll2_tx_pkt.ecore_roce_flavor = roce_flavor; ll2_tx_pkt.first_frag = pkt->header.baddr; ll2_tx_pkt.first_frag_len = pkt->header.len; ll2_tx_pkt.cookie = pkt; ll2_tx_pkt.enable_ip_cksum = 1; // Only for RoCEv2:IPv4 /* tx header */ rc = ecore_ll2_prepare_tx_packet(dev->rdma_ctx, dev->gsi_ll2_handle, &ll2_tx_pkt, 1); if (rc) { QL_DPRINT11(dev->ha, "ecore_ll2_prepare_tx_packet failed\n"); /* TX failed while posting header - release resources*/ qlnx_dma_free_coherent(&dev->ha->cdev, pkt->header.vaddr, pkt->header.baddr, pkt->header.len); kfree(pkt); return rc; } /* tx payload */ for (i = 0; i < pkt->n_seg; i++) { rc = ecore_ll2_set_fragment_of_tx_packet(dev->rdma_ctx, dev->gsi_ll2_handle, pkt->payload[i].baddr, pkt->payload[i].len); if (rc) { /* if failed not much to do here, partial packet has * been posted we can't free memory, will need to wait * for completion */ QL_DPRINT11(dev->ha, "ecore_ll2_set_fragment_of_tx_packet failed\n"); return rc; } } struct ecore_ll2_stats stats = {0}; rc = ecore_ll2_get_stats(dev->rdma_ctx, dev->gsi_ll2_handle, &stats); if (rc) { QL_DPRINT11(dev->ha, "failed to obtain ll2 stats\n"); } QL_DPRINT12(dev->ha, "exit\n"); return 0; } int qlnxr_ll2_stop(struct qlnxr_dev *dev) { int rc; QL_DPRINT12(dev->ha, "enter\n"); if (dev->gsi_ll2_handle == 0xFF) return 0; /* remove LL2 MAC address filter */ rc = qlnx_rdma_ll2_set_mac_filter(dev->rdma_ctx, dev->gsi_ll2_mac_address, NULL); rc = ecore_ll2_terminate_connection(dev->rdma_ctx, dev->gsi_ll2_handle); ecore_ll2_release_connection(dev->rdma_ctx, dev->gsi_ll2_handle); dev->gsi_ll2_handle = 0xFF; QL_DPRINT12(dev->ha, "exit rc = %d\n", rc); return rc; } int qlnxr_ll2_start(struct qlnxr_dev *dev, struct ib_qp_init_attr *attrs, struct qlnxr_qp *qp) { struct ecore_ll2_acquire_data data; struct ecore_ll2_cbs cbs; int rc; QL_DPRINT12(dev->ha, "enter\n"); /* configure and start LL2 */ cbs.rx_comp_cb = qlnxr_ll2_complete_rx_packet; cbs.tx_comp_cb = qlnxr_ll2_complete_tx_packet; cbs.rx_release_cb = qlnxr_ll2_release_rx_packet; cbs.tx_release_cb = qlnxr_ll2_complete_tx_packet; cbs.cookie = dev; dev->gsi_ll2_handle = 0xFF; memset(&data, 0, sizeof(data)); data.input.conn_type = ECORE_LL2_TYPE_ROCE; - data.input.mtu = dev->ha->ifp->if_mtu; + data.input.mtu = if_getmtu(dev->ha->ifp); data.input.rx_num_desc = 8 * 1024; data.input.rx_drop_ttl0_flg = 1; data.input.rx_vlan_removal_en = 0; data.input.tx_num_desc = 8 * 1024; data.input.tx_tc = 0; data.input.tx_dest = ECORE_LL2_TX_DEST_NW; data.input.ai_err_packet_too_big = ECORE_LL2_DROP_PACKET; data.input.ai_err_no_buf = ECORE_LL2_DROP_PACKET; data.input.gsi_enable = 1; data.p_connection_handle = &dev->gsi_ll2_handle; data.cbs = &cbs; rc = ecore_ll2_acquire_connection(dev->rdma_ctx, &data); if (rc) { QL_DPRINT11(dev->ha, "ecore_ll2_acquire_connection failed: %d\n", rc); return rc; } QL_DPRINT11(dev->ha, "ll2 connection acquired successfully\n"); rc = ecore_ll2_establish_connection(dev->rdma_ctx, dev->gsi_ll2_handle); if (rc) { QL_DPRINT11(dev->ha, "ecore_ll2_establish_connection failed\n", rc); goto err1; } QL_DPRINT11(dev->ha, "ll2 connection established successfully\n"); rc = qlnx_rdma_ll2_set_mac_filter(dev->rdma_ctx, NULL, dev->ha->primary_mac); if (rc) { QL_DPRINT11(dev->ha, "qlnx_rdma_ll2_set_mac_filter failed\n", rc); goto err2; } QL_DPRINT12(dev->ha, "exit rc = %d\n", rc); return 0; err2: ecore_ll2_terminate_connection(dev->rdma_ctx, dev->gsi_ll2_handle); err1: ecore_ll2_release_connection(dev->rdma_ctx, dev->gsi_ll2_handle); QL_DPRINT12(dev->ha, "exit rc = %d\n", rc); return rc; } struct ib_qp* qlnxr_create_gsi_qp(struct qlnxr_dev *dev, struct ib_qp_init_attr *attrs, struct qlnxr_qp *qp) { int rc; QL_DPRINT12(dev->ha, "enter\n"); rc = qlnxr_check_gsi_qp_attrs(dev, attrs); if (rc) { QL_DPRINT11(dev->ha, "qlnxr_check_gsi_qp_attrs failed\n"); return ERR_PTR(rc); } rc = qlnxr_ll2_start(dev, attrs, qp); if (rc) { QL_DPRINT11(dev->ha, "qlnxr_ll2_start failed\n"); return ERR_PTR(rc); } /* create QP */ qp->ibqp.qp_num = 1; qp->rq.max_wr = attrs->cap.max_recv_wr; qp->sq.max_wr = attrs->cap.max_send_wr; qp->rqe_wr_id = kzalloc(qp->rq.max_wr * sizeof(*qp->rqe_wr_id), GFP_KERNEL); if (!qp->rqe_wr_id) { QL_DPRINT11(dev->ha, "(!qp->rqe_wr_id)\n"); goto err; } qp->wqe_wr_id = kzalloc(qp->sq.max_wr * sizeof(*qp->wqe_wr_id), GFP_KERNEL); if (!qp->wqe_wr_id) { QL_DPRINT11(dev->ha, "(!qp->wqe_wr_id)\n"); goto err; } qlnxr_store_gsi_qp_cq(dev, qp, attrs); memcpy(dev->gsi_ll2_mac_address, dev->ha->primary_mac, ETH_ALEN); /* the GSI CQ is handled by the driver so remove it from the FW */ qlnxr_destroy_gsi_cq(dev, attrs); dev->gsi_rqcq->cq_type = QLNXR_CQ_TYPE_GSI; dev->gsi_rqcq->cq_type = QLNXR_CQ_TYPE_GSI; QL_DPRINT12(dev->ha, "exit &qp->ibqp = %p\n", &qp->ibqp); return &qp->ibqp; err: kfree(qp->rqe_wr_id); rc = qlnxr_ll2_stop(dev); QL_DPRINT12(dev->ha, "exit with error\n"); return ERR_PTR(-ENOMEM); } int qlnxr_destroy_gsi_qp(struct qlnxr_dev *dev) { int rc = 0; QL_DPRINT12(dev->ha, "enter\n"); rc = qlnxr_ll2_stop(dev); QL_DPRINT12(dev->ha, "exit rc = %d\n", rc); return (rc); } static inline bool qlnxr_get_vlan_id_gsi(struct ib_ah_attr *ah_attr, u16 *vlan_id) { u16 tmp_vlan_id; union ib_gid *dgid = &ah_attr->grh.dgid; tmp_vlan_id = (dgid->raw[11] << 8) | dgid->raw[12]; if (tmp_vlan_id < 0x1000) { *vlan_id = tmp_vlan_id; return true; } else { *vlan_id = 0; return false; } } #define QLNXR_MAX_UD_HEADER_SIZE (100) #define QLNXR_GSI_QPN (1) static inline int qlnxr_gsi_build_header(struct qlnxr_dev *dev, struct qlnxr_qp *qp, const struct ib_send_wr *swr, struct ib_ud_header *udh, int *roce_mode) { bool has_vlan = false, has_grh_ipv6 = true; struct ib_ah_attr *ah_attr = &get_qlnxr_ah((ud_wr(swr)->ah))->attr; struct ib_global_route *grh = &ah_attr->grh; union ib_gid sgid; int send_size = 0; u16 vlan_id = 0; u16 ether_type; int rc = 0; int ip_ver = 0; bool has_udp = false; int i; send_size = 0; for (i = 0; i < swr->num_sge; ++i) send_size += swr->sg_list[i].length; has_vlan = qlnxr_get_vlan_id_gsi(ah_attr, &vlan_id); ether_type = ETH_P_ROCE; *roce_mode = ROCE_V1; if (grh->sgid_index < QLNXR_MAX_SGID) sgid = dev->sgid_tbl[grh->sgid_index]; else sgid = dev->sgid_tbl[0]; rc = ib_ud_header_init(send_size, false /* LRH */, true /* ETH */, has_vlan, has_grh_ipv6, ip_ver, has_udp, 0 /* immediate */, udh); if (rc) { QL_DPRINT11(dev->ha, "gsi post send: failed to init header\n"); return rc; } /* ENET + VLAN headers*/ memcpy(udh->eth.dmac_h, ah_attr->dmac, ETH_ALEN); memcpy(udh->eth.smac_h, dev->ha->primary_mac, ETH_ALEN); if (has_vlan) { udh->eth.type = htons(ETH_P_8021Q); udh->vlan.tag = htons(vlan_id); udh->vlan.type = htons(ether_type); } else { udh->eth.type = htons(ether_type); } for (int j = 0; j < 4; j++) { QL_DPRINT12(dev->ha, "destination mac: %x\n", udh->eth.dmac_h[j]); } for (int j = 0; j < 4; j++) { QL_DPRINT12(dev->ha, "source mac: %x\n", udh->eth.smac_h[j]); } QL_DPRINT12(dev->ha, "QP: %p, opcode: %d, wq: %lx, roce: %x, hops:%d," "imm : %d, vlan :%d, AH: %p\n", qp, swr->opcode, swr->wr_id, *roce_mode, grh->hop_limit, 0, has_vlan, get_qlnxr_ah((ud_wr(swr)->ah))); if (has_grh_ipv6) { /* GRH / IPv6 header */ udh->grh.traffic_class = grh->traffic_class; udh->grh.flow_label = grh->flow_label; udh->grh.hop_limit = grh->hop_limit; udh->grh.destination_gid = grh->dgid; memcpy(&udh->grh.source_gid.raw, &sgid.raw, sizeof(udh->grh.source_gid.raw)); QL_DPRINT12(dev->ha, "header: tc: %x, flow_label : %x, " "hop_limit: %x \n", udh->grh.traffic_class, udh->grh.flow_label, udh->grh.hop_limit); for (i = 0; i < 16; i++) { QL_DPRINT12(dev->ha, "udh dgid = %x\n", udh->grh.destination_gid.raw[i]); } for (i = 0; i < 16; i++) { QL_DPRINT12(dev->ha, "udh sgid = %x\n", udh->grh.source_gid.raw[i]); } udh->grh.next_header = 0x1b; } #ifdef DEFINE_IB_UD_HEADER_INIT_UDP_PRESENT /* This is for RoCEv2 */ else { /* IPv4 header */ u32 ipv4_addr; udh->ip4.protocol = IPPROTO_UDP; udh->ip4.tos = htonl(grh->flow_label); udh->ip4.frag_off = htons(IP_DF); udh->ip4.ttl = grh->hop_limit; ipv4_addr = qedr_get_ipv4_from_gid(sgid.raw); udh->ip4.saddr = ipv4_addr; ipv4_addr = qedr_get_ipv4_from_gid(grh->dgid.raw); udh->ip4.daddr = ipv4_addr; /* note: checksum is calculated by the device */ } #endif /* BTH */ udh->bth.solicited_event = !!(swr->send_flags & IB_SEND_SOLICITED); udh->bth.pkey = QLNXR_ROCE_PKEY_DEFAULT;/* TODO: ib_get_cahced_pkey?! */ //udh->bth.destination_qpn = htonl(ud_wr(swr)->remote_qpn); udh->bth.destination_qpn = OSAL_CPU_TO_BE32(ud_wr(swr)->remote_qpn); //udh->bth.psn = htonl((qp->sq_psn++) & ((1 << 24) - 1)); udh->bth.psn = OSAL_CPU_TO_BE32((qp->sq_psn++) & ((1 << 24) - 1)); udh->bth.opcode = IB_OPCODE_UD_SEND_ONLY; /* DETH */ //udh->deth.qkey = htonl(0x80010000); /* qp->qkey */ /* TODO: what is?! */ //udh->deth.source_qpn = htonl(QLNXR_GSI_QPN); udh->deth.qkey = OSAL_CPU_TO_BE32(0x80010000); /* qp->qkey */ /* TODO: what is?! */ udh->deth.source_qpn = OSAL_CPU_TO_BE32(QLNXR_GSI_QPN); QL_DPRINT12(dev->ha, "exit\n"); return 0; } static inline int qlnxr_gsi_build_packet(struct qlnxr_dev *dev, struct qlnxr_qp *qp, const struct ib_send_wr *swr, struct ecore_roce_ll2_packet **p_packet) { u8 ud_header_buffer[QLNXR_MAX_UD_HEADER_SIZE]; struct ecore_roce_ll2_packet *packet; int roce_mode, header_size; struct ib_ud_header udh; int i, rc; QL_DPRINT12(dev->ha, "enter\n"); *p_packet = NULL; rc = qlnxr_gsi_build_header(dev, qp, swr, &udh, &roce_mode); if (rc) { QL_DPRINT11(dev->ha, "qlnxr_gsi_build_header failed rc = %d\n", rc); return rc; } header_size = ib_ud_header_pack(&udh, &ud_header_buffer); packet = kzalloc(sizeof(*packet), GFP_ATOMIC); if (!packet) { QL_DPRINT11(dev->ha, "packet == NULL\n"); return -ENOMEM; } packet->header.vaddr = qlnx_dma_alloc_coherent(&dev->ha->cdev, &packet->header.baddr, header_size); if (!packet->header.vaddr) { QL_DPRINT11(dev->ha, "packet->header.vaddr == NULL\n"); kfree(packet); return -ENOMEM; } if (memcmp(udh.eth.smac_h, udh.eth.dmac_h, ETH_ALEN)) packet->tx_dest = ECORE_ROCE_LL2_TX_DEST_NW; else packet->tx_dest = ECORE_ROCE_LL2_TX_DEST_LB; packet->roce_mode = roce_mode; memcpy(packet->header.vaddr, ud_header_buffer, header_size); packet->header.len = header_size; packet->n_seg = swr->num_sge; qp->wqe_wr_id[qp->sq.prod].bytes_len = IB_GRH_BYTES; //RDMA_GRH_BYTES for (i = 0; i < packet->n_seg; i++) { packet->payload[i].baddr = swr->sg_list[i].addr; packet->payload[i].len = swr->sg_list[i].length; qp->wqe_wr_id[qp->sq.prod].bytes_len += packet->payload[i].len; QL_DPRINT11(dev->ha, "baddr: %p, len: %d\n", packet->payload[i].baddr, packet->payload[i].len); } *p_packet = packet; QL_DPRINT12(dev->ha, "exit, packet->n_seg: %d\n", packet->n_seg); return 0; } int qlnxr_gsi_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr, const struct ib_send_wr **bad_wr) { struct ecore_roce_ll2_packet *pkt = NULL; struct qlnxr_qp *qp = get_qlnxr_qp(ibqp); struct qlnxr_dev *dev = qp->dev; unsigned long flags; int rc; QL_DPRINT12(dev->ha, "exit\n"); if (qp->state != ECORE_ROCE_QP_STATE_RTS) { QL_DPRINT11(dev->ha, "(qp->state != ECORE_ROCE_QP_STATE_RTS)\n"); *bad_wr = wr; return -EINVAL; } if (wr->num_sge > RDMA_MAX_SGE_PER_SQ_WQE) { QL_DPRINT11(dev->ha, "(wr->num_sge > RDMA_MAX_SGE_PER_SQ_WQE)\n"); rc = -EINVAL; goto err; } if (wr->opcode != IB_WR_SEND) { QL_DPRINT11(dev->ha, "(wr->opcode > IB_WR_SEND)\n"); rc = -EINVAL; goto err; } spin_lock_irqsave(&qp->q_lock, flags); rc = qlnxr_gsi_build_packet(dev, qp, wr, &pkt); if(rc) { spin_unlock_irqrestore(&qp->q_lock, flags); QL_DPRINT11(dev->ha, "qlnxr_gsi_build_packet failed\n"); goto err; } rc = qlnxr_ll2_post_tx(dev, pkt); if (!rc) { qp->wqe_wr_id[qp->sq.prod].wr_id = wr->wr_id; qp->wqe_wr_id[qp->sq.prod].signaled = !!(wr->send_flags & IB_SEND_SIGNALED); qp->wqe_wr_id[qp->sq.prod].opcode = IB_WC_SEND; qlnxr_inc_sw_prod(&qp->sq); QL_DPRINT11(dev->ha, "packet sent over gsi qp\n"); } else { QL_DPRINT11(dev->ha, "qlnxr_ll2_post_tx failed\n"); rc = -EAGAIN; *bad_wr = wr; } spin_unlock_irqrestore(&qp->q_lock, flags); if (wr->next != NULL) { *bad_wr = wr->next; rc=-EINVAL; } QL_DPRINT12(dev->ha, "exit\n"); return rc; err: *bad_wr = wr; QL_DPRINT12(dev->ha, "exit error\n"); return rc; } #define QLNXR_LL2_RX_BUFFER_SIZE (4 * 1024) int qlnxr_gsi_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *wr, const struct ib_recv_wr **bad_wr) { struct qlnxr_dev *dev = get_qlnxr_dev((ibqp->device)); struct qlnxr_qp *qp = get_qlnxr_qp(ibqp); unsigned long flags; int rc = 0; QL_DPRINT12(dev->ha, "enter, wr: %p\n", wr); if ((qp->state != ECORE_ROCE_QP_STATE_RTR) && (qp->state != ECORE_ROCE_QP_STATE_RTS)) { *bad_wr = wr; QL_DPRINT11(dev->ha, "exit 0\n"); return -EINVAL; } spin_lock_irqsave(&qp->q_lock, flags); while (wr) { if (wr->num_sge > QLNXR_GSI_MAX_RECV_SGE) { QL_DPRINT11(dev->ha, "exit 1\n"); goto err; } rc = ecore_ll2_post_rx_buffer(dev->rdma_ctx, dev->gsi_ll2_handle, wr->sg_list[0].addr, wr->sg_list[0].length, 0 /* cookie */, 1 /* notify_fw */); if (rc) { QL_DPRINT11(dev->ha, "exit 2\n"); goto err; } memset(&qp->rqe_wr_id[qp->rq.prod], 0, sizeof(qp->rqe_wr_id[qp->rq.prod])); qp->rqe_wr_id[qp->rq.prod].sg_list[0] = wr->sg_list[0]; qp->rqe_wr_id[qp->rq.prod].wr_id = wr->wr_id; qlnxr_inc_sw_prod(&qp->rq); wr = wr->next; } spin_unlock_irqrestore(&qp->q_lock, flags); QL_DPRINT12(dev->ha, "exit rc = %d\n", rc); return rc; err: spin_unlock_irqrestore(&qp->q_lock, flags); *bad_wr = wr; QL_DPRINT12(dev->ha, "exit with -ENOMEM\n"); return -ENOMEM; } int qlnxr_gsi_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc) { struct qlnxr_dev *dev = get_qlnxr_dev((ibcq->device)); struct qlnxr_cq *cq = get_qlnxr_cq(ibcq); struct qlnxr_qp *qp = dev->gsi_qp; unsigned long flags; int i = 0; QL_DPRINT12(dev->ha, "enter\n"); spin_lock_irqsave(&cq->cq_lock, flags); while (i < num_entries && qp->rq.cons != qp->rq.gsi_cons) { memset(&wc[i], 0, sizeof(*wc)); wc[i].qp = &qp->ibqp; wc[i].wr_id = qp->rqe_wr_id[qp->rq.cons].wr_id; wc[i].opcode = IB_WC_RECV; wc[i].pkey_index = 0; wc[i].status = (qp->rqe_wr_id[qp->rq.cons].rc)? IB_WC_GENERAL_ERR:IB_WC_SUCCESS; /* 0 - currently only one recv sg is supported */ wc[i].byte_len = qp->rqe_wr_id[qp->rq.cons].sg_list[0].length; wc[i].wc_flags |= IB_WC_GRH | IB_WC_IP_CSUM_OK; memcpy(&wc[i].smac, qp->rqe_wr_id[qp->rq.cons].smac, ETH_ALEN); wc[i].wc_flags |= IB_WC_WITH_SMAC; if (qp->rqe_wr_id[qp->rq.cons].vlan_id) { wc[i].wc_flags |= IB_WC_WITH_VLAN; wc[i].vlan_id = qp->rqe_wr_id[qp->rq.cons].vlan_id; } qlnxr_inc_sw_cons(&qp->rq); i++; } while (i < num_entries && qp->sq.cons != qp->sq.gsi_cons) { memset(&wc[i], 0, sizeof(*wc)); wc[i].qp = &qp->ibqp; wc[i].wr_id = qp->wqe_wr_id[qp->sq.cons].wr_id; wc[i].opcode = IB_WC_SEND; wc[i].status = IB_WC_SUCCESS; qlnxr_inc_sw_cons(&qp->sq); i++; } spin_unlock_irqrestore(&cq->cq_lock, flags); QL_DPRINT12(dev->ha, "exit i = %d\n", i); return i; } diff --git a/sys/dev/qlnx/qlnxr/qlnxr_os.c b/sys/dev/qlnx/qlnxr/qlnxr_os.c index aa8386808ebb..0b1a7b67cb1c 100644 --- a/sys/dev/qlnx/qlnxr/qlnxr_os.c +++ b/sys/dev/qlnx/qlnxr/qlnxr_os.c @@ -1,1314 +1,1323 @@ /* * Copyright (c) 2018-2019 Cavium, Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ /* * File: qlnxr_os.c */ #include __FBSDID("$FreeBSD$"); #include "qlnxr_def.h" SYSCTL_NODE(_dev, OID_AUTO, qnxr, CTLFLAG_RW | CTLFLAG_MPSAFE, 0, "Qlogic RDMA module"); uint32_t delayed_ack = 0; SYSCTL_UINT(_dev_qnxr, OID_AUTO, delayed_ack, CTLFLAG_RW, &delayed_ack, 1, "iWARP: Delayed Ack: 0 - Disabled 1 - Enabled. Default: Disabled"); uint32_t timestamp = 1; SYSCTL_UINT(_dev_qnxr, OID_AUTO, timestamp, CTLFLAG_RW, ×tamp, 1, "iWARP: Timestamp: 0 - Disabled 1 - Enabled. Default:Enabled"); uint32_t rcv_wnd_size = 0; SYSCTL_UINT(_dev_qnxr, OID_AUTO, rcv_wnd_size, CTLFLAG_RW, &rcv_wnd_size, 1, "iWARP: Receive Window Size in K. Default 1M"); uint32_t crc_needed = 1; SYSCTL_UINT(_dev_qnxr, OID_AUTO, crc_needed, CTLFLAG_RW, &crc_needed, 1, "iWARP: CRC needed 0 - Disabled 1 - Enabled. Default:Enabled"); uint32_t peer2peer = 1; SYSCTL_UINT(_dev_qnxr, OID_AUTO, peer2peer, CTLFLAG_RW, &peer2peer, 1, "iWARP: Support peer2peer ULPs 0 - Disabled 1 - Enabled. Default:Enabled"); uint32_t mpa_enhanced = 1; SYSCTL_UINT(_dev_qnxr, OID_AUTO, mpa_enhanced, CTLFLAG_RW, &mpa_enhanced, 1, "iWARP: MPA Enhanced mode. Default:1"); uint32_t rtr_type = 7; SYSCTL_UINT(_dev_qnxr, OID_AUTO, rtr_type, CTLFLAG_RW, &rtr_type, 1, "iWARP: RDMAP opcode to use for the RTR message: BITMAP 1: RDMA_SEND 2: RDMA_WRITE 4: RDMA_READ. Default: 7"); #define QNXR_WQ_MULTIPLIER_MIN (1) #define QNXR_WQ_MULTIPLIER_MAX (7) #define QNXR_WQ_MULTIPLIER_DFT (3) uint32_t wq_multiplier= QNXR_WQ_MULTIPLIER_DFT; SYSCTL_UINT(_dev_qnxr, OID_AUTO, wq_multiplier, CTLFLAG_RW, &wq_multiplier, 1, " When creating a WQ the actual number of WQE created will" " be multiplied by this number (default is 3)."); static ssize_t show_rev(struct device *device, struct device_attribute *attr, char *buf) { struct qlnxr_dev *dev = dev_get_drvdata(device); return sprintf(buf, "0x%x\n", dev->cdev->vendor_id); } static ssize_t show_hca_type(struct device *device, struct device_attribute *attr, char *buf) { struct qlnxr_dev *dev = dev_get_drvdata(device); return sprintf(buf, "QLogic0x%x\n", dev->cdev->device_id); } static ssize_t show_fw_ver(struct device *device, struct device_attribute *attr, char *buf) { struct qlnxr_dev *dev = dev_get_drvdata(device); uint32_t fw_ver = (uint32_t) dev->attr.fw_ver; return sprintf(buf, "%d.%d.%d\n", (fw_ver >> 24) & 0xff, (fw_ver >> 16) & 0xff, (fw_ver >> 8) & 0xff); } static ssize_t show_board(struct device *device, struct device_attribute *attr, char *buf) { struct qlnxr_dev *dev = dev_get_drvdata(device); return sprintf(buf, "%x\n", dev->cdev->device_id); } static DEVICE_ATTR(hw_rev, S_IRUGO, show_rev, NULL); static DEVICE_ATTR(hca_type, S_IRUGO, show_hca_type, NULL); static DEVICE_ATTR(fw_ver, S_IRUGO, show_fw_ver, NULL); static DEVICE_ATTR(board_id, S_IRUGO, show_board, NULL); static struct device_attribute *qlnxr_class_attributes[] = { &dev_attr_hw_rev, &dev_attr_hca_type, &dev_attr_fw_ver, &dev_attr_board_id }; static void qlnxr_ib_dispatch_event(qlnxr_dev_t *dev, uint8_t port_num, enum ib_event_type type) { struct ib_event ibev; QL_DPRINT12(dev->ha, "enter\n"); ibev.device = &dev->ibdev; ibev.element.port_num = port_num; ibev.event = type; ib_dispatch_event(&ibev); QL_DPRINT12(dev->ha, "exit\n"); } static int __qlnxr_iw_destroy_listen(struct iw_cm_id *cm_id) { qlnxr_iw_destroy_listen(cm_id); return (0); } static int qlnxr_register_device(qlnxr_dev_t *dev) { struct ib_device *ibdev; struct iw_cm_verbs *iwcm; int ret; QL_DPRINT12(dev->ha, "enter\n"); ibdev = &dev->ibdev; #define qlnxr_ib_ah qlnxr_ah #define qlnxr_ib_cq qlnxr_cq #define qlnxr_ib_pd qlnxr_pd #define qlnxr_ib_qp qlnxr_qp #define qlnxr_ib_srq qlnxr_srq #define qlnxr_ib_ucontext qlnxr_ucontext INIT_IB_DEVICE_OPS(&ibdev->ops, qlnxr, QLNXR); strlcpy(ibdev->name, "qlnxr%d", IB_DEVICE_NAME_MAX); memset(&ibdev->node_guid, 0, sizeof(ibdev->node_guid)); memcpy(&ibdev->node_guid, dev->ha->primary_mac, ETHER_ADDR_LEN); memcpy(ibdev->node_desc, QLNXR_NODE_DESC, sizeof(QLNXR_NODE_DESC)); ibdev->owner = THIS_MODULE; ibdev->uverbs_abi_ver = 7; ibdev->local_dma_lkey = 0; ibdev->uverbs_cmd_mask = (1ull << IB_USER_VERBS_CMD_GET_CONTEXT) | (1ull << IB_USER_VERBS_CMD_QUERY_DEVICE) | (1ull << IB_USER_VERBS_CMD_QUERY_PORT) | (1ull << IB_USER_VERBS_CMD_ALLOC_PD) | (1ull << IB_USER_VERBS_CMD_DEALLOC_PD) | (1ull << IB_USER_VERBS_CMD_REG_MR) | (1ull << IB_USER_VERBS_CMD_DEREG_MR) | (1ull << IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL) | (1ull << IB_USER_VERBS_CMD_CREATE_CQ) | (1ull << IB_USER_VERBS_CMD_DESTROY_CQ) | (1ull << IB_USER_VERBS_CMD_REQ_NOTIFY_CQ) | (1ull << IB_USER_VERBS_CMD_CREATE_QP) | (1ull << IB_USER_VERBS_CMD_MODIFY_QP) | (1ull << IB_USER_VERBS_CMD_QUERY_QP) | (1ull << IB_USER_VERBS_CMD_DESTROY_QP) | (1ull << IB_USER_VERBS_CMD_POLL_CQ) | (1ull << IB_USER_VERBS_CMD_POST_SEND) | (1ull << IB_USER_VERBS_CMD_POST_RECV); if (QLNX_IS_IWARP(dev)) { ibdev->node_type = RDMA_NODE_RNIC; ibdev->query_gid = qlnxr_iw_query_gid; } else { ibdev->node_type = RDMA_NODE_IB_CA; ibdev->query_gid = qlnxr_query_gid; ibdev->uverbs_cmd_mask |= (1ull << IB_USER_VERBS_CMD_CREATE_SRQ) | (1ull << IB_USER_VERBS_CMD_DESTROY_SRQ) | (1ull << IB_USER_VERBS_CMD_QUERY_SRQ) | (1ull << IB_USER_VERBS_CMD_MODIFY_SRQ) | (1ull << IB_USER_VERBS_CMD_POST_SRQ_RECV); ibdev->create_srq = qlnxr_create_srq; ibdev->destroy_srq = qlnxr_destroy_srq; ibdev->modify_srq = qlnxr_modify_srq; ibdev->query_srq = qlnxr_query_srq; ibdev->post_srq_recv = qlnxr_post_srq_recv; } ibdev->phys_port_cnt = 1; ibdev->num_comp_vectors = dev->num_cnq; /* mandatory verbs. */ ibdev->query_device = qlnxr_query_device; ibdev->query_port = qlnxr_query_port; ibdev->modify_port = qlnxr_modify_port; ibdev->alloc_ucontext = qlnxr_alloc_ucontext; ibdev->dealloc_ucontext = qlnxr_dealloc_ucontext; /* mandatory to support user space verbs consumer. */ ibdev->mmap = qlnxr_mmap; ibdev->alloc_pd = qlnxr_alloc_pd; ibdev->dealloc_pd = qlnxr_dealloc_pd; ibdev->create_cq = qlnxr_create_cq; ibdev->destroy_cq = qlnxr_destroy_cq; ibdev->resize_cq = qlnxr_resize_cq; ibdev->req_notify_cq = qlnxr_arm_cq; ibdev->create_qp = qlnxr_create_qp; ibdev->modify_qp = qlnxr_modify_qp; ibdev->query_qp = qlnxr_query_qp; ibdev->destroy_qp = qlnxr_destroy_qp; ibdev->query_pkey = qlnxr_query_pkey; ibdev->create_ah = qlnxr_create_ah; ibdev->destroy_ah = qlnxr_destroy_ah; ibdev->query_ah = qlnxr_query_ah; ibdev->modify_ah = qlnxr_modify_ah; ibdev->get_dma_mr = qlnxr_get_dma_mr; ibdev->dereg_mr = qlnxr_dereg_mr; ibdev->reg_user_mr = qlnxr_reg_user_mr; ibdev->alloc_mr = qlnxr_alloc_mr; ibdev->map_mr_sg = qlnxr_map_mr_sg; ibdev->get_port_immutable = qlnxr_get_port_immutable; ibdev->poll_cq = qlnxr_poll_cq; ibdev->post_send = qlnxr_post_send; ibdev->post_recv = qlnxr_post_recv; ibdev->process_mad = qlnxr_process_mad; ibdev->dma_device = &dev->pdev.dev; ibdev->get_link_layer = qlnxr_link_layer; if (QLNX_IS_IWARP(dev)) { iwcm = kmalloc(sizeof(*iwcm), GFP_KERNEL); device_printf(dev->ha->pci_dev, "device is IWARP\n"); if (iwcm == NULL) return (-ENOMEM); ibdev->iwcm = iwcm; iwcm->connect = qlnxr_iw_connect; iwcm->accept = qlnxr_iw_accept; iwcm->reject = qlnxr_iw_reject; iwcm->create_listen = qlnxr_iw_create_listen; iwcm->destroy_listen = __qlnxr_iw_destroy_listen; iwcm->add_ref = qlnxr_iw_qp_add_ref; iwcm->rem_ref = qlnxr_iw_qp_rem_ref; iwcm->get_qp = qlnxr_iw_get_qp; } ret = ib_register_device(ibdev, NULL); if (ret) { kfree(iwcm); } QL_DPRINT12(dev->ha, "exit\n"); return ret; } #define HILO_U64(hi, lo) ((((u64)(hi)) << 32) + (lo)) static void qlnxr_intr(void *handle) { struct qlnxr_cnq *cnq = handle; struct qlnxr_cq *cq; struct regpair *cq_handle; u16 hw_comp_cons, sw_comp_cons; qlnx_host_t *ha; ha = cnq->dev->ha; QL_DPRINT12(ha, "enter cnq = %p\n", handle); ecore_sb_ack(cnq->sb, IGU_INT_DISABLE, 0 /*do not update*/); ecore_sb_update_sb_idx(cnq->sb); hw_comp_cons = le16_to_cpu(*cnq->hw_cons_ptr); sw_comp_cons = ecore_chain_get_cons_idx(&cnq->pbl); rmb(); QL_DPRINT12(ha, "enter cnq = %p hw_comp_cons = 0x%x sw_comp_cons = 0x%x\n", handle, hw_comp_cons, sw_comp_cons); while (sw_comp_cons != hw_comp_cons) { cq_handle = (struct regpair *)ecore_chain_consume(&cnq->pbl); cq = (struct qlnxr_cq *)(uintptr_t)HILO_U64(cq_handle->hi, cq_handle->lo); if (cq == NULL) { QL_DPRINT11(ha, "cq == NULL\n"); break; } if (cq->sig != QLNXR_CQ_MAGIC_NUMBER) { QL_DPRINT11(ha, "cq->sig = 0x%x QLNXR_CQ_MAGIC_NUMBER = 0x%x\n", cq->sig, QLNXR_CQ_MAGIC_NUMBER); break; } cq->arm_flags = 0; if (!cq->destroyed && cq->ibcq.comp_handler) { QL_DPRINT11(ha, "calling comp_handler = %p " "ibcq = %p cq_context = 0x%x\n", &cq->ibcq, cq->ibcq.cq_context); (*cq->ibcq.comp_handler) (&cq->ibcq, cq->ibcq.cq_context); } cq->cnq_notif++; sw_comp_cons = ecore_chain_get_cons_idx(&cnq->pbl); cnq->n_comp++; } ecore_rdma_cnq_prod_update(cnq->dev->rdma_ctx, cnq->index, sw_comp_cons); ecore_sb_ack(cnq->sb, IGU_INT_ENABLE, 1 /*update*/); QL_DPRINT12(ha, "exit cnq = %p\n", handle); return; } static void qlnxr_release_irqs(struct qlnxr_dev *dev) { int i; qlnx_host_t *ha; ha = dev->ha; QL_DPRINT12(ha, "enter\n"); for (i = 0; i < dev->num_cnq; i++) { if (dev->cnq_array[i].irq_handle) (void)bus_teardown_intr(dev->ha->pci_dev, dev->cnq_array[i].irq, dev->cnq_array[i].irq_handle); if (dev->cnq_array[i].irq) (void) bus_release_resource(dev->ha->pci_dev, SYS_RES_IRQ, dev->cnq_array[i].irq_rid, dev->cnq_array[i].irq); } QL_DPRINT12(ha, "exit\n"); return; } static int qlnxr_setup_irqs(struct qlnxr_dev *dev) { int start_irq_rid; int i; qlnx_host_t *ha; ha = dev->ha; start_irq_rid = dev->sb_start + 2; QL_DPRINT12(ha, "enter start_irq_rid = %d num_rss = %d\n", start_irq_rid, dev->ha->num_rss); for (i = 0; i < dev->num_cnq; i++) { dev->cnq_array[i].irq_rid = start_irq_rid + i; dev->cnq_array[i].irq = bus_alloc_resource_any(dev->ha->pci_dev, SYS_RES_IRQ, &dev->cnq_array[i].irq_rid, (RF_ACTIVE | RF_SHAREABLE)); if (dev->cnq_array[i].irq == NULL) { QL_DPRINT11(ha, "bus_alloc_resource_any failed irq_rid = %d\n", dev->cnq_array[i].irq_rid); goto qlnxr_setup_irqs_err; } if (bus_setup_intr(dev->ha->pci_dev, dev->cnq_array[i].irq, (INTR_TYPE_NET | INTR_MPSAFE), NULL, qlnxr_intr, &dev->cnq_array[i], &dev->cnq_array[i].irq_handle)) { QL_DPRINT11(ha, "bus_setup_intr failed\n"); goto qlnxr_setup_irqs_err; } QL_DPRINT12(ha, "irq_rid = %d irq = %p irq_handle = %p\n", dev->cnq_array[i].irq_rid, dev->cnq_array[i].irq, dev->cnq_array[i].irq_handle); } QL_DPRINT12(ha, "exit\n"); return (0); qlnxr_setup_irqs_err: qlnxr_release_irqs(dev); QL_DPRINT12(ha, "exit -1\n"); return (-1); } static void qlnxr_free_resources(struct qlnxr_dev *dev) { int i; qlnx_host_t *ha; ha = dev->ha; QL_DPRINT12(ha, "enter dev->num_cnq = %d\n", dev->num_cnq); if (QLNX_IS_IWARP(dev)) { if (dev->iwarp_wq != NULL) destroy_workqueue(dev->iwarp_wq); } for (i = 0; i < dev->num_cnq; i++) { qlnx_free_mem_sb(dev->ha, &dev->sb_array[i]); ecore_chain_free(&dev->ha->cdev, &dev->cnq_array[i].pbl); } bzero(dev->cnq_array, (sizeof(struct qlnxr_cnq) * QLNXR_MAX_MSIX)); bzero(dev->sb_array, (sizeof(struct ecore_sb_info) * QLNXR_MAX_MSIX)); bzero(dev->sgid_tbl, (sizeof(union ib_gid) * QLNXR_MAX_SGID)); if (mtx_initialized(&dev->idr_lock)) mtx_destroy(&dev->idr_lock); if (mtx_initialized(&dev->sgid_lock)) mtx_destroy(&dev->sgid_lock); QL_DPRINT12(ha, "exit\n"); return; } static int qlnxr_alloc_resources(struct qlnxr_dev *dev) { uint16_t n_entries; int i, rc; qlnx_host_t *ha; ha = dev->ha; QL_DPRINT12(ha, "enter\n"); bzero(dev->sgid_tbl, (sizeof (union ib_gid) * QLNXR_MAX_SGID)); mtx_init(&dev->idr_lock, "idr_lock", NULL, MTX_DEF); mtx_init(&dev->sgid_lock, "sgid_lock", NULL, MTX_DEF); idr_init(&dev->qpidr); bzero(dev->sb_array, (sizeof (struct ecore_sb_info) * QLNXR_MAX_MSIX)); bzero(dev->cnq_array, (sizeof (struct qlnxr_cnq) * QLNXR_MAX_MSIX)); dev->sb_start = ecore_rdma_get_sb_id(dev->rdma_ctx, 0); QL_DPRINT12(ha, "dev->sb_start = 0x%x\n", dev->sb_start); /* Allocate CNQ PBLs */ n_entries = min_t(u32, ECORE_RDMA_MAX_CNQ_SIZE, QLNXR_ROCE_MAX_CNQ_SIZE); for (i = 0; i < dev->num_cnq; i++) { rc = qlnx_alloc_mem_sb(dev->ha, &dev->sb_array[i], dev->sb_start + i); if (rc) goto qlnxr_alloc_resources_exit; rc = ecore_chain_alloc(&dev->ha->cdev, ECORE_CHAIN_USE_TO_CONSUME_PRODUCE, ECORE_CHAIN_MODE_PBL, ECORE_CHAIN_CNT_TYPE_U16, n_entries, sizeof(struct regpair *), &dev->cnq_array[i].pbl, NULL); /* configure cnq, except name since ibdev.name is still NULL */ dev->cnq_array[i].dev = dev; dev->cnq_array[i].sb = &dev->sb_array[i]; dev->cnq_array[i].hw_cons_ptr = &(dev->sb_array[i].sb_virt->pi_array[ECORE_ROCE_PROTOCOL_INDEX]); dev->cnq_array[i].index = i; sprintf(dev->cnq_array[i].name, "qlnxr%d@pci:%d", i, (dev->ha->pci_func)); } QL_DPRINT12(ha, "exit\n"); return 0; qlnxr_alloc_resources_exit: qlnxr_free_resources(dev); QL_DPRINT12(ha, "exit -ENOMEM\n"); return -ENOMEM; } void qlnxr_affiliated_event(void *context, u8 e_code, void *fw_handle) { #define EVENT_TYPE_NOT_DEFINED 0 #define EVENT_TYPE_CQ 1 #define EVENT_TYPE_QP 2 #define EVENT_TYPE_GENERAL 3 struct qlnxr_dev *dev = (struct qlnxr_dev *)context; struct regpair *async_handle = (struct regpair *)fw_handle; u64 roceHandle64 = ((u64)async_handle->hi << 32) + async_handle->lo; struct qlnxr_cq *cq = (struct qlnxr_cq *)(uintptr_t)roceHandle64; struct qlnxr_qp *qp = (struct qlnxr_qp *)(uintptr_t)roceHandle64; u8 event_type = EVENT_TYPE_NOT_DEFINED; struct ib_event event; qlnx_host_t *ha; ha = dev->ha; QL_DPRINT12(ha, "enter context = %p e_code = 0x%x fw_handle = %p\n", context, e_code, fw_handle); if (QLNX_IS_IWARP(dev)) { switch (e_code) { case ECORE_IWARP_EVENT_CQ_OVERFLOW: event.event = IB_EVENT_CQ_ERR; event_type = EVENT_TYPE_CQ; break; default: QL_DPRINT12(ha, "unsupported event %d on handle=%llx\n", e_code, roceHandle64); break; } } else { switch (e_code) { case ROCE_ASYNC_EVENT_CQ_OVERFLOW_ERR: event.event = IB_EVENT_CQ_ERR; event_type = EVENT_TYPE_CQ; break; case ROCE_ASYNC_EVENT_SQ_DRAINED: event.event = IB_EVENT_SQ_DRAINED; event_type = EVENT_TYPE_QP; break; case ROCE_ASYNC_EVENT_QP_CATASTROPHIC_ERR: event.event = IB_EVENT_QP_FATAL; event_type = EVENT_TYPE_QP; break; case ROCE_ASYNC_EVENT_LOCAL_INVALID_REQUEST_ERR: event.event = IB_EVENT_QP_REQ_ERR; event_type = EVENT_TYPE_QP; break; case ROCE_ASYNC_EVENT_LOCAL_ACCESS_ERR: event.event = IB_EVENT_QP_ACCESS_ERR; event_type = EVENT_TYPE_QP; break; /* NOTE the following are not implemented in FW * ROCE_ASYNC_EVENT_CQ_ERR * ROCE_ASYNC_EVENT_COMM_EST */ /* TODO associate the following events - * ROCE_ASYNC_EVENT_SRQ_LIMIT * ROCE_ASYNC_EVENT_LAST_WQE_REACHED * ROCE_ASYNC_EVENT_LOCAL_CATASTROPHIC_ERR (un-affiliated) */ default: QL_DPRINT12(ha, "unsupported event 0x%x on fw_handle = %p\n", e_code, fw_handle); break; } } switch (event_type) { case EVENT_TYPE_CQ: if (cq && cq->sig == QLNXR_CQ_MAGIC_NUMBER) { struct ib_cq *ibcq = &cq->ibcq; if (ibcq->event_handler) { event.device = ibcq->device; event.element.cq = ibcq; ibcq->event_handler(&event, ibcq->cq_context); } } else { QL_DPRINT11(ha, "CQ event with invalid CQ pointer" " Handle = %llx\n", roceHandle64); } QL_DPRINT12(ha, "CQ event 0x%x on handle = %p\n", e_code, cq); break; case EVENT_TYPE_QP: if (qp && qp->sig == QLNXR_QP_MAGIC_NUMBER) { struct ib_qp *ibqp = &qp->ibqp; if (ibqp->event_handler) { event.device = ibqp->device; event.element.qp = ibqp; ibqp->event_handler(&event, ibqp->qp_context); } } else { QL_DPRINT11(ha, "QP event 0x%x with invalid QP pointer" " qp handle = %p\n", e_code, roceHandle64); } QL_DPRINT12(ha, "QP event 0x%x on qp handle = %p\n", e_code, qp); break; case EVENT_TYPE_GENERAL: break; default: break; } QL_DPRINT12(ha, "exit\n"); return; } void qlnxr_unaffiliated_event(void *context, u8 e_code) { struct qlnxr_dev *dev = (struct qlnxr_dev *)context; qlnx_host_t *ha; ha = dev->ha; QL_DPRINT12(ha, "enter/exit \n"); return; } static int qlnxr_set_device_attr(struct qlnxr_dev *dev) { struct ecore_rdma_device *ecore_attr; struct qlnxr_device_attr *attr; u32 page_size; ecore_attr = ecore_rdma_query_device(dev->rdma_ctx); page_size = ~dev->attr.page_size_caps + 1; if(page_size > PAGE_SIZE) { QL_DPRINT12(dev->ha, "Kernel page size : %ld is smaller than" " minimum page size : %ld required by qlnxr\n", PAGE_SIZE, page_size); return -ENODEV; } attr = &dev->attr; attr->vendor_id = ecore_attr->vendor_id; attr->vendor_part_id = ecore_attr->vendor_part_id; QL_DPRINT12(dev->ha, "in qlnxr_set_device_attr, vendor : %x device : %x\n", attr->vendor_id, attr->vendor_part_id); attr->hw_ver = ecore_attr->hw_ver; attr->fw_ver = ecore_attr->fw_ver; attr->node_guid = ecore_attr->node_guid; attr->sys_image_guid = ecore_attr->sys_image_guid; attr->max_cnq = ecore_attr->max_cnq; attr->max_sge = ecore_attr->max_sge; attr->max_inline = ecore_attr->max_inline; attr->max_sqe = min_t(u32, ecore_attr->max_wqe, QLNXR_MAX_SQE); attr->max_rqe = min_t(u32, ecore_attr->max_wqe, QLNXR_MAX_RQE); attr->max_qp_resp_rd_atomic_resc = ecore_attr->max_qp_resp_rd_atomic_resc; attr->max_qp_req_rd_atomic_resc = ecore_attr->max_qp_req_rd_atomic_resc; attr->max_dev_resp_rd_atomic_resc = ecore_attr->max_dev_resp_rd_atomic_resc; attr->max_cq = ecore_attr->max_cq; attr->max_qp = ecore_attr->max_qp; attr->max_mr = ecore_attr->max_mr; attr->max_mr_size = ecore_attr->max_mr_size; attr->max_cqe = min_t(u64, ecore_attr->max_cqe, QLNXR_MAX_CQES); attr->max_mw = ecore_attr->max_mw; attr->max_fmr = ecore_attr->max_fmr; attr->max_mr_mw_fmr_pbl = ecore_attr->max_mr_mw_fmr_pbl; attr->max_mr_mw_fmr_size = ecore_attr->max_mr_mw_fmr_size; attr->max_pd = ecore_attr->max_pd; attr->max_ah = ecore_attr->max_ah; attr->max_pkey = ecore_attr->max_pkey; attr->max_srq = ecore_attr->max_srq; attr->max_srq_wr = ecore_attr->max_srq_wr; //attr->dev_caps = ecore_attr->dev_caps; attr->page_size_caps = ecore_attr->page_size_caps; attr->dev_ack_delay = ecore_attr->dev_ack_delay; attr->reserved_lkey = ecore_attr->reserved_lkey; attr->bad_pkey_counter = ecore_attr->bad_pkey_counter; attr->max_stats_queues = ecore_attr->max_stats_queues; return 0; } static int qlnxr_init_hw(struct qlnxr_dev *dev) { struct ecore_rdma_events events; struct ecore_rdma_add_user_out_params out_params; struct ecore_rdma_cnq_params *cur_pbl; struct ecore_rdma_start_in_params *in_params; dma_addr_t p_phys_table; u32 page_cnt; int rc = 0; int i; qlnx_host_t *ha; ha = dev->ha; QL_DPRINT12(ha, "enter\n"); in_params = kzalloc(sizeof(*in_params), GFP_KERNEL); if (!in_params) { rc = -ENOMEM; goto out; } bzero(&out_params, sizeof(struct ecore_rdma_add_user_out_params)); bzero(&events, sizeof(struct ecore_rdma_events)); in_params->desired_cnq = dev->num_cnq; for (i = 0; i < dev->num_cnq; i++) { cur_pbl = &in_params->cnq_pbl_list[i]; page_cnt = ecore_chain_get_page_cnt(&dev->cnq_array[i].pbl); cur_pbl->num_pbl_pages = page_cnt; p_phys_table = ecore_chain_get_pbl_phys(&dev->cnq_array[i].pbl); cur_pbl->pbl_ptr = (u64)p_phys_table; } events.affiliated_event = qlnxr_affiliated_event; events.unaffiliated_event = qlnxr_unaffiliated_event; events.context = dev; in_params->events = &events; in_params->roce.cq_mode = ECORE_RDMA_CQ_MODE_32_BITS; in_params->max_mtu = dev->ha->max_frame_size; if (QLNX_IS_IWARP(dev)) { if (delayed_ack) in_params->iwarp.flags |= ECORE_IWARP_DA_EN; if (timestamp) in_params->iwarp.flags |= ECORE_IWARP_TS_EN; in_params->iwarp.rcv_wnd_size = rcv_wnd_size*1024; in_params->iwarp.crc_needed = crc_needed; in_params->iwarp.ooo_num_rx_bufs = (MAX_RXMIT_CONNS * in_params->iwarp.rcv_wnd_size) / in_params->max_mtu; in_params->iwarp.mpa_peer2peer = peer2peer; in_params->iwarp.mpa_rev = mpa_enhanced ? ECORE_MPA_REV2 : ECORE_MPA_REV1; in_params->iwarp.mpa_rtr = rtr_type; } memcpy(&in_params->mac_addr[0], dev->ha->primary_mac, ETH_ALEN); rc = ecore_rdma_start(dev->rdma_ctx, in_params); if (rc) goto out; rc = ecore_rdma_add_user(dev->rdma_ctx, &out_params); if (rc) goto out; dev->db_addr = (void *)(uintptr_t)out_params.dpi_addr; dev->db_phys_addr = out_params.dpi_phys_addr; dev->db_size = out_params.dpi_size; dev->dpi = out_params.dpi; qlnxr_set_device_attr(dev); QL_DPRINT12(ha, "cdev->doorbells = %p, db_phys_addr = %p db_size = 0x%x\n", (void *)ha->cdev.doorbells, (void *)ha->cdev.db_phys_addr, ha->cdev.db_size); QL_DPRINT12(ha, "db_addr = %p db_phys_addr = %p db_size = 0x%x dpi = 0x%x\n", (void *)dev->db_addr, (void *)dev->db_phys_addr, dev->db_size, dev->dpi); out: kfree(in_params); QL_DPRINT12(ha, "exit\n"); return rc; } static void qlnxr_build_sgid_mac(union ib_gid *sgid, unsigned char *mac_addr, bool is_vlan, u16 vlan_id) { sgid->global.subnet_prefix = OSAL_CPU_TO_BE64(0xfe80000000000000LL); sgid->raw[8] = mac_addr[0] ^ 2; sgid->raw[9] = mac_addr[1]; sgid->raw[10] = mac_addr[2]; if (is_vlan) { sgid->raw[11] = vlan_id >> 8; sgid->raw[12] = vlan_id & 0xff; } else { sgid->raw[11] = 0xff; sgid->raw[12] = 0xfe; } sgid->raw[13] = mac_addr[3]; sgid->raw[14] = mac_addr[4]; sgid->raw[15] = mac_addr[5]; } static bool qlnxr_add_sgid(struct qlnxr_dev *dev, union ib_gid *new_sgid); -static void -qlnxr_add_ip_based_gid(struct qlnxr_dev *dev, struct ifnet *ifp) -{ - struct ifaddr *ifa; +struct qlnx_cb_s { + struct qlnxr_dev *dev; union ib_gid gid; +}; - CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { - if (ifa->ifa_addr && ifa->ifa_addr->sa_family == AF_INET) { - QL_DPRINT12(dev->ha, "IP address : %x\n", ((struct sockaddr_in *) ifa->ifa_addr)->sin_addr.s_addr); - ipv6_addr_set_v4mapped( - ((struct sockaddr_in *) ifa->ifa_addr)->sin_addr.s_addr, - (struct in6_addr *)&gid); - QL_DPRINT12(dev->ha, "gid generated : %llx\n", gid); +static u_int +qlnxr_add_ip_based_gid_cb(void *arg, struct ifaddr *ifa, u_int count) +{ + struct qlnx_cb_s *cba = arg; - qlnxr_add_sgid(dev, &gid); - } - } + QL_DPRINT12(cba->dev->ha, "IP address : %x\n", ((struct sockaddr_in *) ifa->ifa_addr)->sin_addr.s_addr); + ipv6_addr_set_v4mapped( + ((struct sockaddr_in *) ifa->ifa_addr)->sin_addr.s_addr, + (struct in6_addr *)&cba->gid); + QL_DPRINT12(cba->dev->ha, "gid generated : %llx\n", cba->gid); + + qlnxr_add_sgid(cba->dev, &cba->gid); + return (1); +} + +static void +qlnxr_add_ip_based_gid(struct qlnxr_dev *dev, if_t ifp) +{ + struct qlnx_cb_s cba; + + if_foreach_addr_type(ifp, AF_INET, qlnxr_add_ip_based_gid_cb, &cba); for (int i = 0; i < 16; i++) { - QL_DPRINT12(dev->ha, "gid generated : %x\n", gid.raw[i]); + QL_DPRINT12(dev->ha, "gid generated : %x\n", cba.gid.raw[i]); } } static bool qlnxr_add_sgid(struct qlnxr_dev *dev, union ib_gid *new_sgid) { union ib_gid zero_sgid = { { 0 } }; int i; //unsigned long flags; mtx_lock(&dev->sgid_lock); for (i = 0; i < QLNXR_MAX_SGID; i++) { if (!memcmp(&dev->sgid_tbl[i], &zero_sgid, sizeof(union ib_gid))) { /* found free entry */ memcpy(&dev->sgid_tbl[i], new_sgid, sizeof(union ib_gid)); QL_DPRINT12(dev->ha, "copying sgid : %llx\n", *new_sgid); mtx_unlock(&dev->sgid_lock); //TODO ib_dispatch event here? return true; } else if (!memcmp(&dev->sgid_tbl[i], new_sgid, sizeof(union ib_gid))) { /* entry already present, no addition required */ mtx_unlock(&dev->sgid_lock); QL_DPRINT12(dev->ha, "sgid present : %llx\n", *new_sgid); return false; } } if (i == QLNXR_MAX_SGID) { QL_DPRINT12(dev->ha, "didn't find an empty entry in sgid_tbl\n"); } mtx_unlock(&dev->sgid_lock); return false; } static bool qlnxr_del_sgid(struct qlnxr_dev *dev, union ib_gid *gid) { int found = false; int i; //unsigned long flags; QL_DPRINT12(dev->ha, "removing gid %llx %llx\n", gid->global.interface_id, gid->global.subnet_prefix); mtx_lock(&dev->sgid_lock); /* first is the default sgid which cannot be deleted */ for (i = 1; i < QLNXR_MAX_SGID; i++) { if (!memcmp(&dev->sgid_tbl[i], gid, sizeof(union ib_gid))) { /* found matching entry */ memset(&dev->sgid_tbl[i], 0, sizeof(union ib_gid)); found = true; break; } } mtx_unlock(&dev->sgid_lock); return found; } static void qlnxr_add_sgids(struct qlnxr_dev *dev) { qlnx_host_t *ha = dev->ha; u16 vlan_id; bool is_vlan; union ib_gid vgid; qlnxr_add_ip_based_gid(dev, ha->ifp); /* MAC/VLAN base GIDs */ is_vlan = is_vlan_dev(ha->ifp); vlan_id = (is_vlan) ? vlan_dev_vlan_id(ha->ifp) : 0; qlnxr_build_sgid_mac(&vgid, ha->primary_mac, is_vlan, vlan_id); qlnxr_add_sgid(dev, &vgid); } static int qlnxr_add_default_sgid(struct qlnxr_dev *dev) { /* GID Index 0 - Invariant manufacturer-assigned EUI-64 */ union ib_gid *sgid = &dev->sgid_tbl[0]; struct ecore_rdma_device *qattr; qlnx_host_t *ha; ha = dev->ha; qattr = ecore_rdma_query_device(dev->rdma_ctx); if(sgid == NULL) QL_DPRINT12(ha, "sgid = NULL?\n"); sgid->global.subnet_prefix = OSAL_CPU_TO_BE64(0xfe80000000000000LL); QL_DPRINT12(ha, "node_guid = %llx", dev->attr.node_guid); memcpy(&sgid->raw[8], &qattr->node_guid, sizeof(qattr->node_guid)); //memcpy(&sgid->raw[8], &dev->attr.node_guid, // sizeof(dev->attr.node_guid)); QL_DPRINT12(ha, "DEFAULT sgid=[%x][%x][%x][%x][%x][%x][%x][%x][%x][%x][%x][%x][%x][%x][%x][%x]\n", sgid->raw[0], sgid->raw[1], sgid->raw[2], sgid->raw[3], sgid->raw[4], sgid->raw[5], sgid->raw[6], sgid->raw[7], sgid->raw[8], sgid->raw[9], sgid->raw[10], sgid->raw[11], sgid->raw[12], sgid->raw[13], sgid->raw[14], sgid->raw[15]); return 0; } static int qlnxr_addr_event (struct qlnxr_dev *dev, unsigned long event, - struct ifnet *ifp, + if_t ifp, union ib_gid *gid) { bool is_vlan = false; union ib_gid vgid; u16 vlan_id = 0xffff; QL_DPRINT12(dev->ha, "Link event occured\n"); is_vlan = is_vlan_dev(dev->ha->ifp); vlan_id = (is_vlan) ? vlan_dev_vlan_id(dev->ha->ifp) : 0; switch (event) { case NETDEV_UP : qlnxr_add_sgid(dev, gid); if (is_vlan) { qlnxr_build_sgid_mac(&vgid, dev->ha->primary_mac, is_vlan, vlan_id); qlnxr_add_sgid(dev, &vgid); } break; case NETDEV_DOWN : qlnxr_del_sgid(dev, gid); if (is_vlan) { qlnxr_build_sgid_mac(&vgid, dev->ha->primary_mac, is_vlan, vlan_id); qlnxr_del_sgid(dev, &vgid); } break; default : break; } return 1; } static int qlnxr_inetaddr_event(struct notifier_block *notifier, unsigned long event, void *ptr) { struct ifaddr *ifa = ptr; union ib_gid gid; struct qlnxr_dev *dev = container_of(notifier, struct qlnxr_dev, nb_inet); qlnx_host_t *ha = dev->ha; ipv6_addr_set_v4mapped( ((struct sockaddr_in *) ifa->ifa_addr)->sin_addr.s_addr, (struct in6_addr *)&gid); return qlnxr_addr_event(dev, event, ha->ifp, &gid); } static int qlnxr_register_inet(struct qlnxr_dev *dev) { int ret; dev->nb_inet.notifier_call = qlnxr_inetaddr_event; ret = register_inetaddr_notifier(&dev->nb_inet); if (ret) { QL_DPRINT12(dev->ha, "Failed to register inetaddr\n"); return ret; } /* TODO : add for CONFIG_IPV6) */ return 0; } static int qlnxr_build_sgid_tbl(struct qlnxr_dev *dev) { qlnxr_add_default_sgid(dev); qlnxr_add_sgids(dev); return 0; } static struct qlnx_rdma_if qlnxr_drv; static void * qlnxr_add(void *eth_dev) { struct qlnxr_dev *dev; int ret; //device_t pci_dev; qlnx_host_t *ha; ha = eth_dev; QL_DPRINT12(ha, "enter [ha = %p]\n", ha); dev = (struct qlnxr_dev *)ib_alloc_device(sizeof(struct qlnxr_dev)); if (dev == NULL) return (NULL); dev->ha = eth_dev; dev->cdev = &ha->cdev; /* Added to extend Application support */ linux_pci_attach_device(dev->ha->pci_dev, NULL, NULL, &dev->pdev); dev->rdma_ctx = &ha->cdev.hwfns[0]; dev->wq_multiplier = wq_multiplier; dev->num_cnq = QLNX_NUM_CNQ; QL_DPRINT12(ha, "ha = %p dev = %p ha->cdev = %p\n", ha, dev, &ha->cdev); QL_DPRINT12(ha, "dev->cdev = %p dev->rdma_ctx = %p\n", dev->cdev, dev->rdma_ctx); ret = qlnxr_alloc_resources(dev); if (ret) goto qlnxr_add_err; ret = qlnxr_setup_irqs(dev); if (ret) { qlnxr_free_resources(dev); goto qlnxr_add_err; } ret = qlnxr_init_hw(dev); if (ret) { qlnxr_release_irqs(dev); qlnxr_free_resources(dev); goto qlnxr_add_err; } qlnxr_register_device(dev); for (int i = 0; i < ARRAY_SIZE(qlnxr_class_attributes); ++i) { if (device_create_file(&dev->ibdev.dev, qlnxr_class_attributes[i])) goto sysfs_err; } qlnxr_build_sgid_tbl(dev); //ret = qlnxr_register_inet(dev); QL_DPRINT12(ha, "exit\n"); if (!test_and_set_bit(QLNXR_ENET_STATE_BIT, &dev->enet_state)) { QL_DPRINT12(ha, "dispatching IB_PORT_ACITVE event\n"); qlnxr_ib_dispatch_event(dev, QLNXR_PORT, IB_EVENT_PORT_ACTIVE); } return (dev); sysfs_err: for (int i = 0; i < ARRAY_SIZE(qlnxr_class_attributes); ++i) { device_remove_file(&dev->ibdev.dev, qlnxr_class_attributes[i]); } ib_unregister_device(&dev->ibdev); qlnxr_add_err: ib_dealloc_device(&dev->ibdev); QL_DPRINT12(ha, "exit failed\n"); return (NULL); } static void qlnxr_remove_sysfiles(struct qlnxr_dev *dev) { int i; for (i = 0; i < ARRAY_SIZE(qlnxr_class_attributes); ++i) device_remove_file(&dev->ibdev.dev, qlnxr_class_attributes[i]); } static int qlnxr_remove(void *eth_dev, void *qlnx_rdma_dev) { struct qlnxr_dev *dev; qlnx_host_t *ha; dev = qlnx_rdma_dev; ha = eth_dev; if ((ha == NULL) || (dev == NULL)) return (0); QL_DPRINT12(ha, "enter ha = %p qlnx_rdma_dev = %p pd_count = %d\n", ha, qlnx_rdma_dev, dev->pd_count); qlnxr_ib_dispatch_event(dev, QLNXR_PORT, IB_EVENT_PORT_ERR); if (QLNX_IS_IWARP(dev)) { if (dev->pd_count) return (EBUSY); } ib_unregister_device(&dev->ibdev); if (QLNX_IS_ROCE(dev)) { if (dev->pd_count) return (EBUSY); } ecore_rdma_remove_user(dev->rdma_ctx, dev->dpi); ecore_rdma_stop(dev->rdma_ctx); qlnxr_release_irqs(dev); qlnxr_free_resources(dev); qlnxr_remove_sysfiles(dev); ib_dealloc_device(&dev->ibdev); linux_pci_detach_device(&dev->pdev); QL_DPRINT12(ha, "exit ha = %p qlnx_rdma_dev = %p\n", ha, qlnx_rdma_dev); return (0); } int qlnx_rdma_ll2_set_mac_filter(void *rdma_ctx, uint8_t *old_mac_address, uint8_t *new_mac_address) { struct ecore_hwfn *p_hwfn = rdma_ctx; struct qlnx_host *ha; int ret = 0; ha = (struct qlnx_host *)(p_hwfn->p_dev); QL_DPRINT2(ha, "enter rdma_ctx (%p)\n", rdma_ctx); if (old_mac_address) ecore_llh_remove_mac_filter(p_hwfn->p_dev, 0, old_mac_address); if (new_mac_address) ret = ecore_llh_add_mac_filter(p_hwfn->p_dev, 0, new_mac_address); QL_DPRINT2(ha, "exit rdma_ctx (%p)\n", rdma_ctx); return (ret); } static void qlnxr_mac_address_change(struct qlnxr_dev *dev) { qlnx_host_t *ha; ha = dev->ha; QL_DPRINT12(ha, "enter/exit\n"); return; } static void qlnxr_notify(void *eth_dev, void *qlnx_rdma_dev, enum qlnx_rdma_event event) { struct qlnxr_dev *dev; qlnx_host_t *ha; dev = qlnx_rdma_dev; if (dev == NULL) return; ha = dev->ha; QL_DPRINT12(ha, "enter (%p, %d)\n", qlnx_rdma_dev, event); switch (event) { case QLNX_ETHDEV_UP: if (!test_and_set_bit(QLNXR_ENET_STATE_BIT, &dev->enet_state)) qlnxr_ib_dispatch_event(dev, QLNXR_PORT, IB_EVENT_PORT_ACTIVE); break; case QLNX_ETHDEV_CHANGE_ADDR: qlnxr_mac_address_change(dev); break; case QLNX_ETHDEV_DOWN: if (test_and_set_bit(QLNXR_ENET_STATE_BIT, &dev->enet_state)) qlnxr_ib_dispatch_event(dev, QLNXR_PORT, IB_EVENT_PORT_ERR); break; } QL_DPRINT12(ha, "exit (%p, %d)\n", qlnx_rdma_dev, event); return; } static int qlnxr_mod_load(void) { int ret; qlnxr_drv.add = qlnxr_add; qlnxr_drv.remove = qlnxr_remove; qlnxr_drv.notify = qlnxr_notify; ret = qlnx_rdma_register_if(&qlnxr_drv); return (ret); } static int qlnxr_mod_unload(void) { int ret; ret = qlnx_rdma_deregister_if(&qlnxr_drv); return (ret); } static int qlnxr_event_handler(module_t mod, int event, void *arg) { int ret = 0; switch (event) { case MOD_LOAD: ret = qlnxr_mod_load(); break; case MOD_UNLOAD: ret = qlnxr_mod_unload(); break; default: break; } return (ret); } static moduledata_t qlnxr_mod_info = { .name = "qlnxr", .evhand = qlnxr_event_handler, }; MODULE_VERSION(qlnxr, 1); MODULE_DEPEND(qlnxr, if_qlnxe, 1, 1, 1); MODULE_DEPEND(qlnxr, ibcore, 1, 1, 1); MODULE_DEPEND(qlnxr, linuxkpi, 1, 1, 1); DECLARE_MODULE(qlnxr, qlnxr_mod_info, SI_SUB_LAST, SI_ORDER_ANY); diff --git a/sys/dev/qlnx/qlnxr/qlnxr_verbs.c b/sys/dev/qlnx/qlnxr/qlnxr_verbs.c index 310ef2dc6e9f..bc2da02eae45 100644 --- a/sys/dev/qlnx/qlnxr/qlnxr_verbs.c +++ b/sys/dev/qlnx/qlnxr/qlnxr_verbs.c @@ -1,6392 +1,6392 @@ /* * Copyright (c) 2018-2019 Cavium, Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ /* * File: qlnxr_verbs.c */ #include __FBSDID("$FreeBSD$"); #include "qlnxr_def.h" #include "rdma_common.h" #include "qlnxr_roce.h" #include "qlnxr_cm.h" #define HILO_U64(hi, lo) ((((u64)(hi)) << 32) + (lo)) #define TYPEPTR_ADDR_SET(type_ptr, field, vaddr) \ do { \ (type_ptr)->field.hi = cpu_to_le32(upper_32_bits(vaddr));\ (type_ptr)->field.lo = cpu_to_le32(lower_32_bits(vaddr));\ } while (0) #define RQ_SGE_SET(sge, vaddr, vlength, vflags) \ do { \ TYPEPTR_ADDR_SET(sge, addr, vaddr); \ (sge)->length = cpu_to_le32(vlength); \ (sge)->flags = cpu_to_le32(vflags); \ } while (0) #define SRQ_HDR_SET(hdr, vwr_id, num_sge) \ do { \ TYPEPTR_ADDR_SET(hdr, wr_id, vwr_id); \ (hdr)->num_sges = num_sge; \ } while (0) #define SRQ_SGE_SET(sge, vaddr, vlength, vlkey) \ do { \ TYPEPTR_ADDR_SET(sge, addr, vaddr); \ (sge)->length = cpu_to_le32(vlength); \ (sge)->l_key = cpu_to_le32(vlkey); \ } while (0) #define NIPQUAD(addr) \ ((unsigned char *)&addr)[0], \ ((unsigned char *)&addr)[1], \ ((unsigned char *)&addr)[2], \ ((unsigned char *)&addr)[3] static int qlnxr_check_srq_params(struct qlnxr_dev *dev, struct ib_srq_init_attr *attrs); static int qlnxr_init_srq_user_params(struct ib_ucontext *ib_ctx, struct qlnxr_srq *srq, struct qlnxr_create_srq_ureq *ureq, int access, int dmasync); static int qlnxr_alloc_srq_kernel_params(struct qlnxr_srq *srq, struct qlnxr_dev *dev, struct ib_srq_init_attr *init_attr); static int qlnxr_copy_srq_uresp(struct qlnxr_dev *dev, struct qlnxr_srq *srq, struct ib_udata *udata); static void qlnxr_free_srq_user_params(struct qlnxr_srq *srq); static void qlnxr_free_srq_kernel_params(struct qlnxr_srq *srq); static u32 qlnxr_srq_elem_left(struct qlnxr_srq_hwq_info *hw_srq); int qlnxr_iw_query_gid(struct ib_device *ibdev, u8 port, int index, union ib_gid *sgid) { struct qlnxr_dev *dev; qlnx_host_t *ha; dev = get_qlnxr_dev(ibdev); ha = dev->ha; QL_DPRINT12(ha, "enter\n"); memset(sgid->raw, 0, sizeof(sgid->raw)); memcpy(sgid->raw, dev->ha->primary_mac, sizeof (dev->ha->primary_mac)); QL_DPRINT12(ha, "exit\n"); return 0; } int qlnxr_query_gid(struct ib_device *ibdev, u8 port, int index, union ib_gid *sgid) { struct qlnxr_dev *dev; qlnx_host_t *ha; dev = get_qlnxr_dev(ibdev); ha = dev->ha; QL_DPRINT12(ha, "enter index: %d\n", index); #if 0 int ret = 0; /* @@@: if DEFINE_ROCE_GID_TABLE to be used here */ //if (!rdma_cap_roce_gid_table(ibdev, port)) { if (!(rdma_protocol_roce(ibdev, port) && ibdev->add_gid && ibdev->del_gid)) { QL_DPRINT11(ha, "acquire gid failed\n"); return -ENODEV; } ret = ib_get_cached_gid(ibdev, port, index, sgid, NULL); if (ret == -EAGAIN) { memcpy(sgid, &zgid, sizeof(*sgid)); return 0; } #endif if ((index >= QLNXR_MAX_SGID) || (index < 0)) { QL_DPRINT12(ha, "invalid gid index %d\n", index); memset(sgid, 0, sizeof(*sgid)); return -EINVAL; } memcpy(sgid, &dev->sgid_tbl[index], sizeof(*sgid)); QL_DPRINT12(ha, "exit : %p\n", sgid); return 0; } int qlnxr_create_srq(struct ib_srq *ibsrq, struct ib_srq_init_attr *init_attr, struct ib_udata *udata) { struct qlnxr_dev *dev; qlnx_host_t *ha; struct ecore_rdma_destroy_srq_in_params destroy_in_params; struct ecore_rdma_create_srq_out_params out_params; struct ecore_rdma_create_srq_in_params in_params; u64 pbl_base_addr, phy_prod_pair_addr; struct qlnxr_srq_hwq_info *hw_srq; struct qlnxr_ucontext *ctx; struct qlnxr_create_srq_ureq ureq; u32 page_cnt, page_size; struct qlnxr_srq *srq = get_qlnxr_srq(ibsrq); int ret = 0; dev = get_qlnxr_dev(ibsrq->device); ha = dev->ha; QL_DPRINT12(ha, "enter\n"); ret = qlnxr_check_srq_params(dev, init_attr); srq->dev = dev; hw_srq = &srq->hw_srq; spin_lock_init(&srq->lock); memset(&in_params, 0, sizeof(in_params)); if (udata) { ctx = rdma_udata_to_drv_context( udata, struct qlnxr_ucontext, ibucontext); memset(&ureq, 0, sizeof(ureq)); if (ib_copy_from_udata(&ureq, udata, min(sizeof(ureq), udata->inlen))) { QL_DPRINT11(ha, "problem" " copying data from user space\n"); goto err0; } ret = qlnxr_init_srq_user_params(&ctx->ibucontext, srq, &ureq, 0, 0); if (ret) goto err0; page_cnt = srq->usrq.pbl_info.num_pbes; pbl_base_addr = srq->usrq.pbl_tbl->pa; phy_prod_pair_addr = hw_srq->phy_prod_pair_addr; // @@@ : if DEFINE_IB_UMEM_PAGE_SHIFT // page_size = BIT(srq->usrq.umem->page_shift); // else page_size = srq->usrq.umem->page_size; } else { struct ecore_chain *pbl; ret = qlnxr_alloc_srq_kernel_params(srq, dev, init_attr); if (ret) goto err0; pbl = &hw_srq->pbl; page_cnt = ecore_chain_get_page_cnt(pbl); pbl_base_addr = ecore_chain_get_pbl_phys(pbl); phy_prod_pair_addr = hw_srq->phy_prod_pair_addr; page_size = pbl->elem_per_page << 4; } in_params.pd_id = get_qlnxr_pd(ibsrq->pd)->pd_id; in_params.pbl_base_addr = pbl_base_addr; in_params.prod_pair_addr = phy_prod_pair_addr; in_params.num_pages = page_cnt; in_params.page_size = page_size; ret = ecore_rdma_create_srq(dev->rdma_ctx, &in_params, &out_params); if (ret) goto err1; srq->srq_id = out_params.srq_id; if (udata) { ret = qlnxr_copy_srq_uresp(dev, srq, udata); if (ret) goto err2; } QL_DPRINT12(ha, "created srq with srq_id = 0x%0x\n", srq->srq_id); return (0); err2: memset(&in_params, 0, sizeof(in_params)); destroy_in_params.srq_id = srq->srq_id; ecore_rdma_destroy_srq(dev->rdma_ctx, &destroy_in_params); err1: if (udata) qlnxr_free_srq_user_params(srq); else qlnxr_free_srq_kernel_params(srq); err0: return (-EFAULT); } void qlnxr_destroy_srq(struct ib_srq *ibsrq, struct ib_udata *udata) { struct qlnxr_dev *dev; struct qlnxr_srq *srq; qlnx_host_t *ha; struct ecore_rdma_destroy_srq_in_params in_params; srq = get_qlnxr_srq(ibsrq); dev = srq->dev; ha = dev->ha; memset(&in_params, 0, sizeof(in_params)); in_params.srq_id = srq->srq_id; ecore_rdma_destroy_srq(dev->rdma_ctx, &in_params); if (ibsrq->pd->uobject && ibsrq->pd->uobject->context) qlnxr_free_srq_user_params(srq); else qlnxr_free_srq_kernel_params(srq); QL_DPRINT12(ha, "destroyed srq_id=0x%0x\n", srq->srq_id); } int qlnxr_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr, enum ib_srq_attr_mask attr_mask, struct ib_udata *udata) { struct qlnxr_dev *dev; struct qlnxr_srq *srq; qlnx_host_t *ha; struct ecore_rdma_modify_srq_in_params in_params; int ret = 0; srq = get_qlnxr_srq(ibsrq); dev = srq->dev; ha = dev->ha; QL_DPRINT12(ha, "enter\n"); if (attr_mask & IB_SRQ_MAX_WR) { QL_DPRINT12(ha, "invalid attribute mask=0x%x" " specified for %p\n", attr_mask, srq); return -EINVAL; } if (attr_mask & IB_SRQ_LIMIT) { if (attr->srq_limit >= srq->hw_srq.max_wr) { QL_DPRINT12(ha, "invalid srq_limit=0x%x" " (max_srq_limit = 0x%x)\n", attr->srq_limit, srq->hw_srq.max_wr); return -EINVAL; } memset(&in_params, 0, sizeof(in_params)); in_params.srq_id = srq->srq_id; in_params.wqe_limit = attr->srq_limit; ret = ecore_rdma_modify_srq(dev->rdma_ctx, &in_params); if (ret) return ret; } QL_DPRINT12(ha, "modified srq with srq_id = 0x%0x\n", srq->srq_id); return 0; } int qlnxr_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *srq_attr) { struct qlnxr_dev *dev; struct qlnxr_srq *srq; qlnx_host_t *ha; struct ecore_rdma_device *qattr; srq = get_qlnxr_srq(ibsrq); dev = srq->dev; ha = dev->ha; //qattr = &dev->attr; qattr = ecore_rdma_query_device(dev->rdma_ctx); QL_DPRINT12(ha, "enter\n"); if (!dev->rdma_ctx) { QL_DPRINT12(ha, "called with invalid params" " rdma_ctx is NULL\n"); return -EINVAL; } srq_attr->srq_limit = qattr->max_srq; srq_attr->max_wr = qattr->max_srq_wr; srq_attr->max_sge = qattr->max_sge; QL_DPRINT12(ha, "exit\n"); return 0; } /* Increment srq wr producer by one */ static void qlnxr_inc_srq_wr_prod (struct qlnxr_srq_hwq_info *info) { info->wr_prod_cnt++; } /* Increment srq wr consumer by one */ static void qlnxr_inc_srq_wr_cons(struct qlnxr_srq_hwq_info *info) { info->wr_cons_cnt++; } /* get_port_immutable verb is not available in FreeBSD */ #if 0 int qlnxr_roce_port_immutable(struct ib_device *ibdev, u8 port_num, struct ib_port_immutable *immutable) { struct qlnxr_dev *dev; qlnx_host_t *ha; dev = get_qlnxr_dev(ibdev); ha = dev->ha; QL_DPRINT12(ha, "entered but not implemented!!!\n"); } #endif int qlnxr_post_srq_recv(struct ib_srq *ibsrq, const struct ib_recv_wr *wr, const struct ib_recv_wr **bad_wr) { struct qlnxr_dev *dev; struct qlnxr_srq *srq; qlnx_host_t *ha; struct qlnxr_srq_hwq_info *hw_srq; struct ecore_chain *pbl; unsigned long flags; int status = 0; u32 num_sge, offset; srq = get_qlnxr_srq(ibsrq); dev = srq->dev; ha = dev->ha; hw_srq = &srq->hw_srq; QL_DPRINT12(ha, "enter\n"); spin_lock_irqsave(&srq->lock, flags); pbl = &srq->hw_srq.pbl; while (wr) { struct rdma_srq_wqe_header *hdr; int i; if (!qlnxr_srq_elem_left(hw_srq) || wr->num_sge > srq->hw_srq.max_sges) { QL_DPRINT11(ha, "WR cannot be posted" " (%d, %d) || (%d > %d)\n", hw_srq->wr_prod_cnt, hw_srq->wr_cons_cnt, wr->num_sge, srq->hw_srq.max_sges); status = -ENOMEM; *bad_wr = wr; break; } hdr = ecore_chain_produce(pbl); num_sge = wr->num_sge; /* Set number of sge and WR id in header */ SRQ_HDR_SET(hdr, wr->wr_id, num_sge); /* PBL is maintained in case of WR granularity. * So increment WR producer in case we post a WR. */ qlnxr_inc_srq_wr_prod(hw_srq); hw_srq->wqe_prod++; hw_srq->sge_prod++; QL_DPRINT12(ha, "SRQ WR : SGEs: %d with wr_id[%d] = %llx\n", wr->num_sge, hw_srq->wqe_prod, wr->wr_id); for (i = 0; i < wr->num_sge; i++) { struct rdma_srq_sge *srq_sge = ecore_chain_produce(pbl); /* Set SGE length, lkey and address */ SRQ_SGE_SET(srq_sge, wr->sg_list[i].addr, wr->sg_list[i].length, wr->sg_list[i].lkey); QL_DPRINT12(ha, "[%d]: len %d, key %x, addr %x:%x\n", i, srq_sge->length, srq_sge->l_key, srq_sge->addr.hi, srq_sge->addr.lo); hw_srq->sge_prod++; } wmb(); /* * SRQ prod is 8 bytes. Need to update SGE prod in index * in first 4 bytes and need to update WQE prod in next * 4 bytes. */ *(srq->hw_srq.virt_prod_pair_addr) = hw_srq->sge_prod; offset = offsetof(struct rdma_srq_producers, wqe_prod); *((u8 *)srq->hw_srq.virt_prod_pair_addr + offset) = hw_srq->wqe_prod; /* Flush prod after updating it */ wmb(); wr = wr->next; } QL_DPRINT12(ha, "Elements in SRQ: %d\n", ecore_chain_get_elem_left(pbl)); spin_unlock_irqrestore(&srq->lock, flags); QL_DPRINT12(ha, "exit\n"); return status; } int qlnxr_query_device(struct ib_device *ibdev, struct ib_device_attr *attr, struct ib_udata *udata) { struct qlnxr_dev *dev; struct ecore_rdma_device *qattr; qlnx_host_t *ha; dev = get_qlnxr_dev(ibdev); ha = dev->ha; QL_DPRINT12(ha, "enter\n"); if (udata->inlen || udata->outlen) return -EINVAL; if (dev->rdma_ctx == NULL) { return -EINVAL; } qattr = ecore_rdma_query_device(dev->rdma_ctx); memset(attr, 0, sizeof *attr); attr->fw_ver = qattr->fw_ver; attr->sys_image_guid = qattr->sys_image_guid; attr->max_mr_size = qattr->max_mr_size; attr->page_size_cap = qattr->page_size_caps; attr->vendor_id = qattr->vendor_id; attr->vendor_part_id = qattr->vendor_part_id; attr->hw_ver = qattr->hw_ver; attr->max_qp = qattr->max_qp; attr->device_cap_flags = IB_DEVICE_CURR_QP_STATE_MOD | IB_DEVICE_RC_RNR_NAK_GEN | IB_DEVICE_LOCAL_DMA_LKEY | IB_DEVICE_MEM_MGT_EXTENSIONS; attr->max_sge = qattr->max_sge; attr->max_sge_rd = qattr->max_sge; attr->max_cq = qattr->max_cq; attr->max_cqe = qattr->max_cqe; attr->max_mr = qattr->max_mr; attr->max_mw = qattr->max_mw; attr->max_pd = qattr->max_pd; attr->atomic_cap = dev->atomic_cap; attr->max_fmr = qattr->max_fmr; attr->max_map_per_fmr = 16; /* TBD: FMR */ /* There is an implicit assumption in some of the ib_xxx apps that the * qp_rd_atom is smaller than the qp_init_rd_atom. Specifically, in * communication the qp_rd_atom is passed to the other side and used as * init_rd_atom without check device capabilities for init_rd_atom. * for this reason, we set the qp_rd_atom to be the minimum between the * two...There is an additional assumption in mlx4 driver that the * values are power of two, fls is performed on the value - 1, which * in fact gives a larger power of two for values which are not a power * of two. This should be fixed in mlx4 driver, but until then -> * we provide a value that is a power of two in our code. */ attr->max_qp_init_rd_atom = 1 << (fls(qattr->max_qp_req_rd_atomic_resc) - 1); attr->max_qp_rd_atom = min(1 << (fls(qattr->max_qp_resp_rd_atomic_resc) - 1), attr->max_qp_init_rd_atom); attr->max_srq = qattr->max_srq; attr->max_srq_sge = qattr->max_srq_sge; attr->max_srq_wr = qattr->max_srq_wr; /* TODO: R&D to more properly configure the following */ attr->local_ca_ack_delay = qattr->dev_ack_delay; attr->max_fast_reg_page_list_len = qattr->max_mr/8; attr->max_pkeys = QLNXR_ROCE_PKEY_MAX; attr->max_ah = qattr->max_ah; QL_DPRINT12(ha, "exit\n"); return 0; } static inline void get_link_speed_and_width(int speed, uint8_t *ib_speed, uint8_t *ib_width) { switch (speed) { case 1000: *ib_speed = IB_SPEED_SDR; *ib_width = IB_WIDTH_1X; break; case 10000: *ib_speed = IB_SPEED_QDR; *ib_width = IB_WIDTH_1X; break; case 20000: *ib_speed = IB_SPEED_DDR; *ib_width = IB_WIDTH_4X; break; case 25000: *ib_speed = IB_SPEED_EDR; *ib_width = IB_WIDTH_1X; break; case 40000: *ib_speed = IB_SPEED_QDR; *ib_width = IB_WIDTH_4X; break; case 50000: *ib_speed = IB_SPEED_QDR; *ib_width = IB_WIDTH_4X; // TODO doesn't add up to 50... break; case 100000: *ib_speed = IB_SPEED_EDR; *ib_width = IB_WIDTH_4X; break; default: /* Unsupported */ *ib_speed = IB_SPEED_SDR; *ib_width = IB_WIDTH_1X; } return; } int qlnxr_query_port(struct ib_device *ibdev, uint8_t port, struct ib_port_attr *attr) { struct qlnxr_dev *dev; struct ecore_rdma_port *rdma_port; qlnx_host_t *ha; dev = get_qlnxr_dev(ibdev); ha = dev->ha; QL_DPRINT12(ha, "enter\n"); if (port > 1) { QL_DPRINT12(ha, "port [%d] > 1 \n", port); return -EINVAL; } if (dev->rdma_ctx == NULL) { QL_DPRINT12(ha, "rdma_ctx == NULL\n"); return -EINVAL; } rdma_port = ecore_rdma_query_port(dev->rdma_ctx); memset(attr, 0, sizeof *attr); if (rdma_port->port_state == ECORE_RDMA_PORT_UP) { attr->state = IB_PORT_ACTIVE; attr->phys_state = 5; } else { attr->state = IB_PORT_DOWN; attr->phys_state = 3; } attr->max_mtu = IB_MTU_4096; - attr->active_mtu = iboe_get_mtu(dev->ha->ifp->if_mtu); + attr->active_mtu = iboe_get_mtu(if_getmtu(dev->ha->ifp)); attr->lid = 0; attr->lmc = 0; attr->sm_lid = 0; attr->sm_sl = 0; attr->port_cap_flags = 0; if (QLNX_IS_IWARP(dev)) { attr->gid_tbl_len = 1; attr->pkey_tbl_len = 1; } else { attr->gid_tbl_len = QLNXR_MAX_SGID; attr->pkey_tbl_len = QLNXR_ROCE_PKEY_TABLE_LEN; } attr->bad_pkey_cntr = rdma_port->pkey_bad_counter; attr->qkey_viol_cntr = 0; get_link_speed_and_width(rdma_port->link_speed, &attr->active_speed, &attr->active_width); attr->max_msg_sz = rdma_port->max_msg_size; attr->max_vl_num = 4; /* TODO -> figure this one out... */ QL_DPRINT12(ha, "state = %d phys_state = %d " " link_speed = %d active_speed = %d active_width = %d" " attr->gid_tbl_len = %d attr->pkey_tbl_len = %d" " max_msg_sz = 0x%x max_vl_num = 0x%x \n", attr->state, attr->phys_state, rdma_port->link_speed, attr->active_speed, attr->active_width, attr->gid_tbl_len, attr->pkey_tbl_len, attr->max_msg_sz, attr->max_vl_num); QL_DPRINT12(ha, "exit\n"); return 0; } int qlnxr_modify_port(struct ib_device *ibdev, uint8_t port, int mask, struct ib_port_modify *props) { struct qlnxr_dev *dev; qlnx_host_t *ha; dev = get_qlnxr_dev(ibdev); ha = dev->ha; QL_DPRINT12(ha, "enter\n"); if (port > 1) { QL_DPRINT12(ha, "port (%d) > 1\n", port); return -EINVAL; } QL_DPRINT12(ha, "exit\n"); return 0; } enum rdma_link_layer qlnxr_link_layer(struct ib_device *ibdev, uint8_t port_num) { struct qlnxr_dev *dev; qlnx_host_t *ha; dev = get_qlnxr_dev(ibdev); ha = dev->ha; QL_DPRINT12(ha, "ibdev = %p port_num = 0x%x\n", ibdev, port_num); return IB_LINK_LAYER_ETHERNET; } int qlnxr_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata) { struct ib_device *ibdev = ibpd->device; struct qlnxr_pd *pd = get_qlnxr_pd(ibpd); u16 pd_id; int rc; struct qlnxr_dev *dev; qlnx_host_t *ha; dev = get_qlnxr_dev(ibdev); ha = dev->ha; QL_DPRINT12(ha, "ibdev = %p udata = %p enter\n", ibdev, udata); if (dev->rdma_ctx == NULL) { QL_DPRINT11(ha, "dev->rdma_ctx = NULL\n"); rc = -1; goto err; } rc = ecore_rdma_alloc_pd(dev->rdma_ctx, &pd_id); if (rc) { QL_DPRINT11(ha, "ecore_rdma_alloc_pd failed\n"); goto err; } pd->pd_id = pd_id; if (udata) { rc = ib_copy_to_udata(udata, &pd->pd_id, sizeof(pd->pd_id)); if (rc) { QL_DPRINT11(ha, "ib_copy_to_udata failed\n"); ecore_rdma_free_pd(dev->rdma_ctx, pd_id); goto err; } pd->uctx = rdma_udata_to_drv_context( udata, struct qlnxr_ucontext, ibucontext); pd->uctx->pd = pd; } atomic_add_rel_32(&dev->pd_count, 1); QL_DPRINT12(ha, "exit [pd, pd_id, pd_count] = [%p, 0x%x, %d]\n", pd, pd_id, dev->pd_count); return (0); err: QL_DPRINT12(ha, "exit -1\n"); return (rc); } void qlnxr_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata) { struct qlnxr_pd *pd; struct qlnxr_dev *dev; qlnx_host_t *ha; pd = get_qlnxr_pd(ibpd); dev = get_qlnxr_dev((ibpd->device)); ha = dev->ha; QL_DPRINT12(ha, "enter\n"); if (pd == NULL) { QL_DPRINT11(ha, "pd = NULL\n"); } else { ecore_rdma_free_pd(dev->rdma_ctx, pd->pd_id); atomic_subtract_rel_32(&dev->pd_count, 1); QL_DPRINT12(ha, "exit [pd, pd_id, pd_count] = [%p, 0x%x, %d]\n", pd, pd->pd_id, dev->pd_count); } QL_DPRINT12(ha, "exit\n"); } #define ROCE_WQE_ELEM_SIZE sizeof(struct rdma_sq_sge) #define RDMA_MAX_SGE_PER_SRQ (4) /* Should be part of HSI */ /* Should be part of HSI */ #define RDMA_MAX_SRQ_WQE_SIZE (RDMA_MAX_SGE_PER_SRQ + 1) /* +1 for header */ #define DB_ADDR_SHIFT(addr) ((addr) << DB_PWM_ADDR_OFFSET_SHIFT) static void qlnxr_cleanup_user(struct qlnxr_dev *, struct qlnxr_qp *); static void qlnxr_cleanup_kernel(struct qlnxr_dev *, struct qlnxr_qp *); int qlnxr_query_pkey(struct ib_device *ibdev, u8 port, u16 index, u16 *pkey) { struct qlnxr_dev *dev; qlnx_host_t *ha; dev = get_qlnxr_dev(ibdev); ha = dev->ha; QL_DPRINT12(ha, "enter index = 0x%x\n", index); if (index > QLNXR_ROCE_PKEY_TABLE_LEN) return -EINVAL; *pkey = QLNXR_ROCE_PKEY_DEFAULT; QL_DPRINT12(ha, "exit\n"); return 0; } static inline bool qlnxr_get_vlan_id_qp(qlnx_host_t *ha, struct ib_qp_attr *attr, int attr_mask, u16 *vlan_id) { bool ret = false; u16 tmp_vlan_id; union ib_gid *dgid; QL_DPRINT12(ha, "enter \n"); *vlan_id = 0; dgid = &attr->ah_attr.grh.dgid; tmp_vlan_id = (dgid->raw[11] << 8) | dgid->raw[12]; if (!(tmp_vlan_id & ~EVL_VLID_MASK)) { *vlan_id = tmp_vlan_id; ret = true; } QL_DPRINT12(ha, "exit vlan_id = 0x%x ret = %d \n", *vlan_id, ret); return (ret); } static inline void get_gid_info(struct ib_qp *ibqp, struct ib_qp_attr *attr, int attr_mask, struct qlnxr_dev *dev, struct qlnxr_qp *qp, struct ecore_rdma_modify_qp_in_params *qp_params) { int i; qlnx_host_t *ha; ha = dev->ha; QL_DPRINT12(ha, "enter\n"); memcpy(&qp_params->sgid.bytes[0], &dev->sgid_tbl[qp->sgid_idx].raw[0], sizeof(qp_params->sgid.bytes)); memcpy(&qp_params->dgid.bytes[0], &attr->ah_attr.grh.dgid.raw[0], sizeof(qp_params->dgid)); qlnxr_get_vlan_id_qp(ha, attr, attr_mask, &qp_params->vlan_id); for (i = 0; i < (sizeof(qp_params->sgid.dwords)/sizeof(uint32_t)); i++) { qp_params->sgid.dwords[i] = ntohl(qp_params->sgid.dwords[i]); qp_params->dgid.dwords[i] = ntohl(qp_params->dgid.dwords[i]); } QL_DPRINT12(ha, "exit\n"); return; } static int qlnxr_add_mmap(struct qlnxr_ucontext *uctx, u64 phy_addr, unsigned long len) { struct qlnxr_mm *mm; qlnx_host_t *ha; ha = uctx->dev->ha; QL_DPRINT12(ha, "enter\n"); mm = kzalloc(sizeof(*mm), GFP_KERNEL); if (mm == NULL) { QL_DPRINT11(ha, "mm = NULL\n"); return -ENOMEM; } mm->key.phy_addr = phy_addr; /* This function might be called with a length which is not a multiple * of PAGE_SIZE, while the mapping is PAGE_SIZE grained and the kernel * forces this granularity by increasing the requested size if needed. * When qedr_mmap is called, it will search the list with the updated * length as a key. To prevent search failures, the length is rounded up * in advance to PAGE_SIZE. */ mm->key.len = roundup(len, PAGE_SIZE); INIT_LIST_HEAD(&mm->entry); mutex_lock(&uctx->mm_list_lock); list_add(&mm->entry, &uctx->mm_head); mutex_unlock(&uctx->mm_list_lock); QL_DPRINT12(ha, "added (addr=0x%llx,len=0x%lx) for ctx=%p\n", (unsigned long long)mm->key.phy_addr, (unsigned long)mm->key.len, uctx); return 0; } static bool qlnxr_search_mmap(struct qlnxr_ucontext *uctx, u64 phy_addr, unsigned long len) { bool found = false; struct qlnxr_mm *mm; qlnx_host_t *ha; ha = uctx->dev->ha; QL_DPRINT12(ha, "enter\n"); mutex_lock(&uctx->mm_list_lock); list_for_each_entry(mm, &uctx->mm_head, entry) { if (len != mm->key.len || phy_addr != mm->key.phy_addr) continue; found = true; break; } mutex_unlock(&uctx->mm_list_lock); QL_DPRINT12(ha, "searched for (addr=0x%llx,len=0x%lx) for ctx=%p, found=%d\n", mm->key.phy_addr, mm->key.len, uctx, found); return found; } int qlnxr_alloc_ucontext(struct ib_ucontext *uctx, struct ib_udata *udata) { int rc; struct qlnxr_ucontext *ctx = get_qlnxr_ucontext(uctx); struct qlnxr_alloc_ucontext_resp uresp; struct qlnxr_dev *dev = get_qlnxr_dev(uctx->device); qlnx_host_t *ha = dev->ha; struct ecore_rdma_add_user_out_params oparams; if (!udata) return -EFAULT; rc = ecore_rdma_add_user(dev->rdma_ctx, &oparams); if (rc) { QL_DPRINT12(ha, "Failed to allocate a DPI for a new RoCE application " ",rc = %d. To overcome this, consider to increase " "the number of DPIs, increase the doorbell BAR size " "or just close unnecessary RoCE applications. In " "order to increase the number of DPIs consult the " "README\n", rc); goto err; } ctx->dpi = oparams.dpi; ctx->dpi_addr = oparams.dpi_addr; ctx->dpi_phys_addr = oparams.dpi_phys_addr; ctx->dpi_size = oparams.dpi_size; INIT_LIST_HEAD(&ctx->mm_head); mutex_init(&ctx->mm_list_lock); memset(&uresp, 0, sizeof(uresp)); uresp.dpm_enabled = offsetof(struct qlnxr_alloc_ucontext_resp, dpm_enabled) < udata->outlen ? dev->user_dpm_enabled : 0; //TODO: figure this out uresp.wids_enabled = offsetof(struct qlnxr_alloc_ucontext_resp, wids_enabled) < udata->outlen ? 1 : 0; //TODO: figure this out uresp.wid_count = offsetof(struct qlnxr_alloc_ucontext_resp, wid_count) < udata->outlen ? oparams.wid_count : 0; //TODO: figure this out uresp.db_pa = ctx->dpi_phys_addr; uresp.db_size = ctx->dpi_size; uresp.max_send_wr = dev->attr.max_sqe; uresp.max_recv_wr = dev->attr.max_rqe; uresp.max_srq_wr = dev->attr.max_srq_wr; uresp.sges_per_send_wr = QLNXR_MAX_SQE_ELEMENTS_PER_SQE; uresp.sges_per_recv_wr = QLNXR_MAX_RQE_ELEMENTS_PER_RQE; uresp.sges_per_srq_wr = dev->attr.max_srq_sge; uresp.max_cqes = QLNXR_MAX_CQES; rc = ib_copy_to_udata(udata, &uresp, sizeof(uresp)); if (rc) goto err; ctx->dev = dev; rc = qlnxr_add_mmap(ctx, ctx->dpi_phys_addr, ctx->dpi_size); if (rc) goto err; QL_DPRINT12(ha, "Allocated user context %p\n", &ctx->ibucontext); return (0); err: return (rc); } void qlnxr_dealloc_ucontext(struct ib_ucontext *ibctx) { struct qlnxr_ucontext *uctx = get_qlnxr_ucontext(ibctx); struct qlnxr_dev *dev = uctx->dev; qlnx_host_t *ha = dev->ha; struct qlnxr_mm *mm, *tmp; QL_DPRINT12(ha, "Deallocating user context %p\n", uctx); if (dev) { ecore_rdma_remove_user(uctx->dev->rdma_ctx, uctx->dpi); } list_for_each_entry_safe(mm, tmp, &uctx->mm_head, entry) { QL_DPRINT12(ha, "deleted addr= 0x%llx, len = 0x%lx for" " ctx=%p\n", mm->key.phy_addr, mm->key.len, uctx); list_del(&mm->entry); kfree(mm); } } int qlnxr_mmap(struct ib_ucontext *context, struct vm_area_struct *vma) { struct qlnxr_ucontext *ucontext = get_qlnxr_ucontext(context); struct qlnxr_dev *dev = get_qlnxr_dev((context->device)); unsigned long vm_page = vma->vm_pgoff << PAGE_SHIFT; u64 unmapped_db; unsigned long len = (vma->vm_end - vma->vm_start); int rc = 0; bool found; qlnx_host_t *ha; ha = dev->ha; unmapped_db = dev->db_phys_addr + (ucontext->dpi * ucontext->dpi_size); QL_DPRINT12(ha, "qedr_mmap enter vm_page=0x%lx" " vm_pgoff=0x%lx unmapped_db=0x%llx db_size=%x, len=%lx\n", vm_page, vma->vm_pgoff, unmapped_db, dev->db_size, len); if ((vma->vm_start & (PAGE_SIZE - 1)) || (len & (PAGE_SIZE - 1))) { QL_DPRINT11(ha, "Vma_start not page aligned " "vm_start = %ld vma_end = %ld\n", vma->vm_start, vma->vm_end); return -EINVAL; } found = qlnxr_search_mmap(ucontext, vm_page, len); if (!found) { QL_DPRINT11(ha, "Vma_pgoff not found in mapped array = %ld\n", vma->vm_pgoff); return -EINVAL; } QL_DPRINT12(ha, "Mapping doorbell bar\n"); if ((vm_page < unmapped_db) || ((vm_page + len) > (unmapped_db + ucontext->dpi_size))) { QL_DPRINT11(ha, "failed pages are outside of dpi;" "page address=0x%lx, unmapped_db=0x%lx, dpi_size=0x%x\n", vm_page, unmapped_db, ucontext->dpi_size); return -EINVAL; } if (vma->vm_flags & VM_READ) { QL_DPRINT11(ha, "failed mmap, cannot map doorbell bar for read\n"); return -EINVAL; } vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot); rc = io_remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff, len, vma->vm_page_prot); QL_DPRINT12(ha, "exit [%d]\n", rc); return rc; } struct ib_mr * qlnxr_get_dma_mr(struct ib_pd *ibpd, int acc) { struct qlnxr_mr *mr; struct qlnxr_dev *dev = get_qlnxr_dev((ibpd->device)); struct qlnxr_pd *pd = get_qlnxr_pd(ibpd); int rc; qlnx_host_t *ha; ha = dev->ha; QL_DPRINT12(ha, "enter\n"); if (acc & IB_ACCESS_MW_BIND) { QL_DPRINT12(ha, "Unsupported access flags received for dma mr\n"); } mr = kzalloc(sizeof(*mr), GFP_KERNEL); if (!mr) { rc = -ENOMEM; QL_DPRINT12(ha, "kzalloc(mr) failed %d\n", rc); goto err0; } mr->type = QLNXR_MR_DMA; rc = ecore_rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid); if (rc) { QL_DPRINT12(ha, "ecore_rdma_alloc_tid failed %d\n", rc); goto err1; } /* index only, 18 bit long, lkey = itid << 8 | key */ mr->hw_mr.tid_type = ECORE_RDMA_TID_REGISTERED_MR; mr->hw_mr.pd = pd->pd_id; mr->hw_mr.local_read = 1; mr->hw_mr.local_write = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0; mr->hw_mr.remote_read = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0; mr->hw_mr.remote_write = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0; mr->hw_mr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0; mr->hw_mr.dma_mr = true; rc = ecore_rdma_register_tid(dev->rdma_ctx, &mr->hw_mr); if (rc) { QL_DPRINT12(ha, "ecore_rdma_register_tid failed %d\n", rc); goto err2; } mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key; if (mr->hw_mr.remote_write || mr->hw_mr.remote_read || mr->hw_mr.remote_atomic) { mr->ibmr.rkey = mr->hw_mr.itid << 8 | mr->hw_mr.key; } QL_DPRINT12(ha, "lkey = %x\n", mr->ibmr.lkey); return &mr->ibmr; err2: ecore_rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid); err1: kfree(mr); err0: QL_DPRINT12(ha, "exit [%d]\n", rc); return ERR_PTR(rc); } static void qlnxr_free_pbl(struct qlnxr_dev *dev, struct qlnxr_pbl_info *pbl_info, struct qlnxr_pbl *pbl) { int i; qlnx_host_t *ha; ha = dev->ha; QL_DPRINT12(ha, "enter\n"); for (i = 0; i < pbl_info->num_pbls; i++) { if (!pbl[i].va) continue; qlnx_dma_free_coherent(&dev->ha->cdev, pbl[i].va, pbl[i].pa, pbl_info->pbl_size); } kfree(pbl); QL_DPRINT12(ha, "exit\n"); return; } #define MIN_FW_PBL_PAGE_SIZE (4*1024) #define MAX_FW_PBL_PAGE_SIZE (64*1024) #define NUM_PBES_ON_PAGE(_page_size) (_page_size / sizeof(u64)) #define MAX_PBES_ON_PAGE NUM_PBES_ON_PAGE(MAX_FW_PBL_PAGE_SIZE) #define MAX_PBES_TWO_LAYER (MAX_PBES_ON_PAGE*MAX_PBES_ON_PAGE) static struct qlnxr_pbl * qlnxr_alloc_pbl_tbl(struct qlnxr_dev *dev, struct qlnxr_pbl_info *pbl_info, gfp_t flags) { void *va; dma_addr_t pa; dma_addr_t *pbl_main_tbl; struct qlnxr_pbl *pbl_table; int i; qlnx_host_t *ha; ha = dev->ha; QL_DPRINT12(ha, "enter\n"); pbl_table = kzalloc(sizeof(*pbl_table) * pbl_info->num_pbls, flags); if (!pbl_table) { QL_DPRINT12(ha, "pbl_table = NULL\n"); return NULL; } for (i = 0; i < pbl_info->num_pbls; i++) { va = qlnx_dma_alloc_coherent(&dev->ha->cdev, &pa, pbl_info->pbl_size); if (!va) { QL_DPRINT11(ha, "Failed to allocate pbl#%d\n", i); goto err; } memset(va, 0, pbl_info->pbl_size); pbl_table[i].va = va; pbl_table[i].pa = pa; } /* Two-Layer PBLs, if we have more than one pbl we need to initialize * the first one with physical pointers to all of the rest */ pbl_main_tbl = (dma_addr_t *)pbl_table[0].va; for (i = 0; i < pbl_info->num_pbls - 1; i++) pbl_main_tbl[i] = pbl_table[i + 1].pa; QL_DPRINT12(ha, "exit\n"); return pbl_table; err: qlnxr_free_pbl(dev, pbl_info, pbl_table); QL_DPRINT12(ha, "exit with error\n"); return NULL; } static int qlnxr_prepare_pbl_tbl(struct qlnxr_dev *dev, struct qlnxr_pbl_info *pbl_info, u32 num_pbes, int two_layer_capable) { u32 pbl_capacity; u32 pbl_size; u32 num_pbls; qlnx_host_t *ha; ha = dev->ha; QL_DPRINT12(ha, "enter\n"); if ((num_pbes > MAX_PBES_ON_PAGE) && two_layer_capable) { if (num_pbes > MAX_PBES_TWO_LAYER) { QL_DPRINT11(ha, "prepare pbl table: too many pages %d\n", num_pbes); return -EINVAL; } /* calculate required pbl page size */ pbl_size = MIN_FW_PBL_PAGE_SIZE; pbl_capacity = NUM_PBES_ON_PAGE(pbl_size) * NUM_PBES_ON_PAGE(pbl_size); while (pbl_capacity < num_pbes) { pbl_size *= 2; pbl_capacity = pbl_size / sizeof(u64); pbl_capacity = pbl_capacity * pbl_capacity; } num_pbls = DIV_ROUND_UP(num_pbes, NUM_PBES_ON_PAGE(pbl_size)); num_pbls++; /* One for the layer0 ( points to the pbls) */ pbl_info->two_layered = true; } else { /* One layered PBL */ num_pbls = 1; pbl_size = max_t(u32, MIN_FW_PBL_PAGE_SIZE, \ roundup_pow_of_two((num_pbes * sizeof(u64)))); pbl_info->two_layered = false; } pbl_info->num_pbls = num_pbls; pbl_info->pbl_size = pbl_size; pbl_info->num_pbes = num_pbes; QL_DPRINT12(ha, "prepare pbl table: num_pbes=%d, num_pbls=%d pbl_size=%d\n", pbl_info->num_pbes, pbl_info->num_pbls, pbl_info->pbl_size); return 0; } static void qlnxr_populate_pbls(struct qlnxr_dev *dev, struct ib_umem *umem, struct qlnxr_pbl *pbl, struct qlnxr_pbl_info *pbl_info) { struct regpair *pbe; struct qlnxr_pbl *pbl_tbl; struct scatterlist *sg; int shift, pg_cnt, pages, pbe_cnt, total_num_pbes = 0; qlnx_host_t *ha; int entry; ha = dev->ha; QL_DPRINT12(ha, "enter\n"); if (!pbl_info) { QL_DPRINT11(ha, "PBL_INFO not initialized\n"); return; } if (!pbl_info->num_pbes) { QL_DPRINT11(ha, "pbl_info->num_pbes == 0\n"); return; } /* If we have a two layered pbl, the first pbl points to the rest * of the pbls and the first entry lays on the second pbl in the table */ if (pbl_info->two_layered) pbl_tbl = &pbl[1]; else pbl_tbl = pbl; pbe = (struct regpair *)pbl_tbl->va; if (!pbe) { QL_DPRINT12(ha, "pbe is NULL\n"); return; } pbe_cnt = 0; shift = ilog2(umem->page_size); for_each_sg(umem->sg_head.sgl, sg, umem->nmap, entry) { pages = sg_dma_len(sg) >> shift; for (pg_cnt = 0; pg_cnt < pages; pg_cnt++) { /* store the page address in pbe */ pbe->lo = cpu_to_le32(sg_dma_address(sg) + (umem->page_size * pg_cnt)); pbe->hi = cpu_to_le32(upper_32_bits ((sg_dma_address(sg) + umem->page_size * pg_cnt))); QL_DPRINT12(ha, "Populate pbl table:" " pbe->addr=0x%x:0x%x " " pbe_cnt = %d total_num_pbes=%d" " pbe=%p\n", pbe->lo, pbe->hi, pbe_cnt, total_num_pbes, pbe); pbe_cnt ++; total_num_pbes ++; pbe++; if (total_num_pbes == pbl_info->num_pbes) return; /* if the given pbl is full storing the pbes, * move to next pbl. */ if (pbe_cnt == (pbl_info->pbl_size / sizeof(u64))) { pbl_tbl++; pbe = (struct regpair *)pbl_tbl->va; pbe_cnt = 0; } } } QL_DPRINT12(ha, "exit\n"); return; } static void free_mr_info(struct qlnxr_dev *dev, struct mr_info *info) { struct qlnxr_pbl *pbl, *tmp; qlnx_host_t *ha; ha = dev->ha; QL_DPRINT12(ha, "enter\n"); if (info->pbl_table) list_add_tail(&info->pbl_table->list_entry, &info->free_pbl_list); if (!list_empty(&info->inuse_pbl_list)) list_splice(&info->inuse_pbl_list, &info->free_pbl_list); list_for_each_entry_safe(pbl, tmp, &info->free_pbl_list, list_entry) { list_del(&pbl->list_entry); qlnxr_free_pbl(dev, &info->pbl_info, pbl); } QL_DPRINT12(ha, "exit\n"); return; } static int qlnxr_init_mr_info(struct qlnxr_dev *dev, struct mr_info *info, size_t page_list_len, bool two_layered) { int rc; struct qlnxr_pbl *tmp; qlnx_host_t *ha; ha = dev->ha; QL_DPRINT12(ha, "enter\n"); INIT_LIST_HEAD(&info->free_pbl_list); INIT_LIST_HEAD(&info->inuse_pbl_list); rc = qlnxr_prepare_pbl_tbl(dev, &info->pbl_info, page_list_len, two_layered); if (rc) { QL_DPRINT11(ha, "qlnxr_prepare_pbl_tbl [%d]\n", rc); goto done; } info->pbl_table = qlnxr_alloc_pbl_tbl(dev, &info->pbl_info, GFP_KERNEL); if (!info->pbl_table) { rc = -ENOMEM; QL_DPRINT11(ha, "qlnxr_alloc_pbl_tbl returned NULL\n"); goto done; } QL_DPRINT12(ha, "pbl_table_pa = %pa\n", &info->pbl_table->pa); /* in usual case we use 2 PBLs, so we add one to free * list and allocating another one */ tmp = qlnxr_alloc_pbl_tbl(dev, &info->pbl_info, GFP_KERNEL); if (!tmp) { QL_DPRINT11(ha, "Extra PBL is not allocated\n"); goto done; /* it's OK if second allocation fails, so rc = 0*/ } list_add_tail(&tmp->list_entry, &info->free_pbl_list); QL_DPRINT12(ha, "extra pbl_table_pa = %pa\n", &tmp->pa); done: if (rc) free_mr_info(dev, info); QL_DPRINT12(ha, "exit [%d]\n", rc); return rc; } struct ib_mr * qlnxr_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len, u64 usr_addr, int acc, struct ib_udata *udata) { int rc = -ENOMEM; struct qlnxr_dev *dev = get_qlnxr_dev((ibpd->device)); struct qlnxr_mr *mr; struct qlnxr_pd *pd; qlnx_host_t *ha; ha = dev->ha; QL_DPRINT12(ha, "enter\n"); pd = get_qlnxr_pd(ibpd); QL_DPRINT12(ha, "qedr_register user mr pd = %d" " start = %lld, len = %lld, usr_addr = %lld, acc = %d\n", pd->pd_id, start, len, usr_addr, acc); if (acc & IB_ACCESS_REMOTE_WRITE && !(acc & IB_ACCESS_LOCAL_WRITE)) { QL_DPRINT11(ha, "(acc & IB_ACCESS_REMOTE_WRITE &&" " !(acc & IB_ACCESS_LOCAL_WRITE))\n"); return ERR_PTR(-EINVAL); } mr = kzalloc(sizeof(*mr), GFP_KERNEL); if (!mr) { QL_DPRINT11(ha, "kzalloc(mr) failed\n"); return ERR_PTR(rc); } mr->type = QLNXR_MR_USER; mr->umem = ib_umem_get(ibpd->uobject->context, start, len, acc, 0); if (IS_ERR(mr->umem)) { rc = -EFAULT; QL_DPRINT11(ha, "ib_umem_get failed [%p]\n", mr->umem); goto err0; } rc = qlnxr_init_mr_info(dev, &mr->info, ib_umem_page_count(mr->umem), 1); if (rc) { QL_DPRINT11(ha, "qlnxr_init_mr_info failed [%d]\n", rc); goto err1; } qlnxr_populate_pbls(dev, mr->umem, mr->info.pbl_table, &mr->info.pbl_info); rc = ecore_rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid); if (rc) { QL_DPRINT11(ha, "roce alloc tid returned an error %d\n", rc); goto err1; } /* index only, 18 bit long, lkey = itid << 8 | key */ mr->hw_mr.tid_type = ECORE_RDMA_TID_REGISTERED_MR; mr->hw_mr.key = 0; mr->hw_mr.pd = pd->pd_id; mr->hw_mr.local_read = 1; mr->hw_mr.local_write = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0; mr->hw_mr.remote_read = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0; mr->hw_mr.remote_write = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0; mr->hw_mr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0; mr->hw_mr.mw_bind = false; /* TBD MW BIND */ mr->hw_mr.pbl_ptr = mr->info.pbl_table[0].pa; mr->hw_mr.pbl_two_level = mr->info.pbl_info.two_layered; mr->hw_mr.pbl_page_size_log = ilog2(mr->info.pbl_info.pbl_size); mr->hw_mr.page_size_log = ilog2(mr->umem->page_size); /* for the MR pages */ mr->hw_mr.fbo = ib_umem_offset(mr->umem); mr->hw_mr.length = len; mr->hw_mr.vaddr = usr_addr; mr->hw_mr.zbva = false; /* TBD figure when this should be true */ mr->hw_mr.phy_mr = false; /* Fast MR - True, Regular Register False */ mr->hw_mr.dma_mr = false; rc = ecore_rdma_register_tid(dev->rdma_ctx, &mr->hw_mr); if (rc) { QL_DPRINT11(ha, "roce register tid returned an error %d\n", rc); goto err2; } mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key; if (mr->hw_mr.remote_write || mr->hw_mr.remote_read || mr->hw_mr.remote_atomic) mr->ibmr.rkey = mr->hw_mr.itid << 8 | mr->hw_mr.key; QL_DPRINT12(ha, "register user mr lkey: %x\n", mr->ibmr.lkey); return (&mr->ibmr); err2: ecore_rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid); err1: qlnxr_free_pbl(dev, &mr->info.pbl_info, mr->info.pbl_table); err0: kfree(mr); QL_DPRINT12(ha, "exit [%d]\n", rc); return (ERR_PTR(rc)); } int qlnxr_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata) { struct qlnxr_mr *mr = get_qlnxr_mr(ib_mr); struct qlnxr_dev *dev = get_qlnxr_dev((ib_mr->device)); int rc = 0; qlnx_host_t *ha; ha = dev->ha; QL_DPRINT12(ha, "enter\n"); if ((mr->type != QLNXR_MR_DMA) && (mr->type != QLNXR_MR_FRMR)) qlnxr_free_pbl(dev, &mr->info.pbl_info, mr->info.pbl_table); /* it could be user registered memory. */ if (mr->umem) ib_umem_release(mr->umem); kfree(mr->pages); kfree(mr); QL_DPRINT12(ha, "exit\n"); return rc; } static int qlnxr_copy_cq_uresp(struct qlnxr_dev *dev, struct qlnxr_cq *cq, struct ib_udata *udata) { struct qlnxr_create_cq_uresp uresp; int rc; qlnx_host_t *ha; ha = dev->ha; QL_DPRINT12(ha, "enter\n"); memset(&uresp, 0, sizeof(uresp)); uresp.db_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_UCM_RDMA_CQ_CONS_32BIT); uresp.icid = cq->icid; rc = ib_copy_to_udata(udata, &uresp, sizeof(uresp)); if (rc) { QL_DPRINT12(ha, "ib_copy_to_udata error cqid=0x%x[%d]\n", cq->icid, rc); } QL_DPRINT12(ha, "exit [%d]\n", rc); return rc; } static void consume_cqe(struct qlnxr_cq *cq) { if (cq->latest_cqe == cq->toggle_cqe) cq->pbl_toggle ^= RDMA_RESIZE_CQ_RAMROD_DATA_TOGGLE_BIT_MASK; cq->latest_cqe = ecore_chain_consume(&cq->pbl); } static inline int qlnxr_align_cq_entries(int entries) { u64 size, aligned_size; /* We allocate an extra entry that we don't report to the FW. * Why? * The CQE size is 32 bytes but the FW writes in chunks of 64 bytes * (for performance purposes). Allocating an extra entry and telling * the FW we have less prevents overwriting the first entry in case of * a wrap i.e. when the FW writes the last entry and the application * hasn't read the first one. */ size = (entries + 1) * QLNXR_CQE_SIZE; /* We align to PAGE_SIZE. * Why? * Since the CQ is going to be mapped and the mapping is anyhow in whole * kernel pages we benefit from the possibly extra CQEs. */ aligned_size = ALIGN(size, PAGE_SIZE); /* note: for CQs created in user space the result of this function * should match the size mapped in user space */ return (aligned_size / QLNXR_CQE_SIZE); } static inline int qlnxr_init_user_queue(struct ib_ucontext *ib_ctx, struct qlnxr_dev *dev, struct qlnxr_userq *q, u64 buf_addr, size_t buf_len, int access, int dmasync, int alloc_and_init) { int page_cnt; int rc; qlnx_host_t *ha; ha = dev->ha; QL_DPRINT12(ha, "enter\n"); q->buf_addr = buf_addr; q->buf_len = buf_len; QL_DPRINT12(ha, "buf_addr : %llx, buf_len : %x, access : %x" " dmasync : %x\n", q->buf_addr, q->buf_len, access, dmasync); q->umem = ib_umem_get(ib_ctx, q->buf_addr, q->buf_len, access, dmasync); if (IS_ERR(q->umem)) { QL_DPRINT11(ha, "ib_umem_get failed [%lx]\n", PTR_ERR(q->umem)); return PTR_ERR(q->umem); } page_cnt = ib_umem_page_count(q->umem); rc = qlnxr_prepare_pbl_tbl(dev, &q->pbl_info, page_cnt, 0 /* SQ and RQ don't support dual layer pbl. * CQ may, but this is yet uncoded. */); if (rc) { QL_DPRINT11(ha, "qlnxr_prepare_pbl_tbl failed [%d]\n", rc); goto err; } if (alloc_and_init) { q->pbl_tbl = qlnxr_alloc_pbl_tbl(dev, &q->pbl_info, GFP_KERNEL); if (!q->pbl_tbl) { QL_DPRINT11(ha, "qlnxr_alloc_pbl_tbl failed\n"); rc = -ENOMEM; goto err; } qlnxr_populate_pbls(dev, q->umem, q->pbl_tbl, &q->pbl_info); } else { q->pbl_tbl = kzalloc(sizeof(*q->pbl_tbl), GFP_KERNEL); if (!q->pbl_tbl) { QL_DPRINT11(ha, "qlnxr_alloc_pbl_tbl failed\n"); rc = -ENOMEM; goto err; } } QL_DPRINT12(ha, "exit\n"); return 0; err: ib_umem_release(q->umem); q->umem = NULL; QL_DPRINT12(ha, "exit [%d]\n", rc); return rc; } int qlnxr_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, struct ib_udata *udata) { struct qlnxr_ucontext *ctx; struct ecore_rdma_destroy_cq_out_params destroy_oparams; struct ecore_rdma_destroy_cq_in_params destroy_iparams; struct qlnxr_dev *dev; struct ecore_rdma_create_cq_in_params params; struct qlnxr_create_cq_ureq ureq; int vector = attr->comp_vector; int entries = attr->cqe; struct qlnxr_cq *cq = get_qlnxr_cq(ibcq); int chain_entries, rc, page_cnt; u64 pbl_ptr; u16 icid; qlnx_host_t *ha; dev = get_qlnxr_dev(ibcq->device); ha = dev->ha; QL_DPRINT12(ha, "called from %s. entries = %d, " "vector = %d\n", (udata ? "User Lib" : "Kernel"), entries, vector); memset(¶ms, 0, sizeof(struct ecore_rdma_create_cq_in_params)); memset(&destroy_iparams, 0, sizeof(struct ecore_rdma_destroy_cq_in_params)); memset(&destroy_oparams, 0, sizeof(struct ecore_rdma_destroy_cq_out_params)); if (entries > QLNXR_MAX_CQES) { QL_DPRINT11(ha, "the number of entries %d is too high. " "Must be equal or below %d.\n", entries, QLNXR_MAX_CQES); return -EINVAL; } chain_entries = qlnxr_align_cq_entries(entries); chain_entries = min_t(int, chain_entries, QLNXR_MAX_CQES); if (udata) { ctx = rdma_udata_to_drv_context( udata, struct qlnxr_ucontext, ibucontext); memset(&ureq, 0, sizeof(ureq)); if (ib_copy_from_udata(&ureq, udata, min(sizeof(ureq), udata->inlen))) { QL_DPRINT11(ha, "ib_copy_from_udata failed\n"); goto err0; } if (!ureq.len) { QL_DPRINT11(ha, "ureq.len == 0\n"); goto err0; } cq->cq_type = QLNXR_CQ_TYPE_USER; qlnxr_init_user_queue(&ctx->ibucontext, dev, &cq->q, ureq.addr, ureq.len, IB_ACCESS_LOCAL_WRITE, 1, 1); pbl_ptr = cq->q.pbl_tbl->pa; page_cnt = cq->q.pbl_info.num_pbes; cq->ibcq.cqe = chain_entries; } else { ctx = NULL; cq->cq_type = QLNXR_CQ_TYPE_KERNEL; rc = ecore_chain_alloc(&dev->ha->cdev, ECORE_CHAIN_USE_TO_CONSUME, ECORE_CHAIN_MODE_PBL, ECORE_CHAIN_CNT_TYPE_U32, chain_entries, sizeof(union roce_cqe), &cq->pbl, NULL); if (rc) goto err1; page_cnt = ecore_chain_get_page_cnt(&cq->pbl); pbl_ptr = ecore_chain_get_pbl_phys(&cq->pbl); cq->ibcq.cqe = cq->pbl.capacity; } params.cq_handle_hi = upper_32_bits((uintptr_t)cq); params.cq_handle_lo = lower_32_bits((uintptr_t)cq); params.cnq_id = vector; params.cq_size = chain_entries - 1; params.pbl_num_pages = page_cnt; params.pbl_ptr = pbl_ptr; params.pbl_two_level = 0; if (udata) { params.dpi = ctx->dpi; } else { params.dpi = dev->dpi; } rc = ecore_rdma_create_cq(dev->rdma_ctx, ¶ms, &icid); if (rc) goto err2; cq->icid = icid; cq->sig = QLNXR_CQ_MAGIC_NUMBER; spin_lock_init(&cq->cq_lock); if (udata) { rc = qlnxr_copy_cq_uresp(dev, cq, udata); if (rc) goto err3; } else { /* Generate doorbell address. * Configure bits 3-9 with DQ_PWM_OFFSET_UCM_RDMA_CQ_CONS_32BIT. * TODO: consider moving to device scope as it is a function of * the device. * TODO: add ifdef if plan to support 16 bit. */ cq->db_addr = dev->db_addr + DB_ADDR_SHIFT(DQ_PWM_OFFSET_UCM_RDMA_CQ_CONS_32BIT); cq->db.data.icid = cq->icid; cq->db.data.params = DB_AGG_CMD_SET << RDMA_PWM_VAL32_DATA_AGG_CMD_SHIFT; /* point to the very last element, passing it we will toggle */ cq->toggle_cqe = ecore_chain_get_last_elem(&cq->pbl); cq->pbl_toggle = RDMA_RESIZE_CQ_RAMROD_DATA_TOGGLE_BIT_MASK; /* must be different from pbl_toggle */ cq->latest_cqe = NULL; consume_cqe(cq); cq->cq_cons = ecore_chain_get_cons_idx_u32(&cq->pbl); } QL_DPRINT12(ha, "exit icid = 0x%0x, addr = %p," " number of entries = 0x%x\n", cq->icid, cq, params.cq_size); QL_DPRINT12(ha,"cq_addr = %p\n", cq); return (0); err3: destroy_iparams.icid = cq->icid; ecore_rdma_destroy_cq(dev->rdma_ctx, &destroy_iparams, &destroy_oparams); err2: if (udata) qlnxr_free_pbl(dev, &cq->q.pbl_info, cq->q.pbl_tbl); else ecore_chain_free(&dev->ha->cdev, &cq->pbl); err1: if (udata) ib_umem_release(cq->q.umem); err0: QL_DPRINT12(ha, "exit error\n"); return (-EINVAL); } int qlnxr_resize_cq(struct ib_cq *ibcq, int new_cnt, struct ib_udata *udata) { int status = 0; struct qlnxr_dev *dev = get_qlnxr_dev((ibcq->device)); qlnx_host_t *ha; ha = dev->ha; QL_DPRINT12(ha, "enter/exit\n"); return status; } void qlnxr_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata) { struct qlnxr_dev *dev = get_qlnxr_dev((ibcq->device)); struct ecore_rdma_destroy_cq_out_params oparams; struct ecore_rdma_destroy_cq_in_params iparams; struct qlnxr_cq *cq = get_qlnxr_cq(ibcq); int rc = 0; qlnx_host_t *ha; ha = dev->ha; QL_DPRINT12(ha, "enter cq_id = %d\n", cq->icid); cq->destroyed = 1; /* TODO: Syncronize irq of the CNQ the CQ belongs to for validation * that all completions with notification are dealt with. The rest * of the completions are not interesting */ /* GSIs CQs are handled by driver, so they don't exist in the FW */ if (cq->cq_type != QLNXR_CQ_TYPE_GSI) { iparams.icid = cq->icid; rc = ecore_rdma_destroy_cq(dev->rdma_ctx, &iparams, &oparams); if (rc) { QL_DPRINT12(ha, "ecore_rdma_destroy_cq failed cq_id = %d\n", cq->icid); return; } QL_DPRINT12(ha, "free cq->pbl cq_id = %d\n", cq->icid); ecore_chain_free(&dev->ha->cdev, &cq->pbl); } if (udata) { qlnxr_free_pbl(dev, &cq->q.pbl_info, cq->q.pbl_tbl); ib_umem_release(cq->q.umem); } cq->sig = ~cq->sig; QL_DPRINT12(ha, "exit cq_id = %d\n", cq->icid); } static int qlnxr_check_qp_attrs(struct ib_pd *ibpd, struct qlnxr_dev *dev, struct ib_qp_init_attr *attrs, struct ib_udata *udata) { struct ecore_rdma_device *qattr; qlnx_host_t *ha; qattr = ecore_rdma_query_device(dev->rdma_ctx); ha = dev->ha; QL_DPRINT12(ha, "enter\n"); QL_DPRINT12(ha, "attrs->sq_sig_type = %d\n", attrs->sq_sig_type); QL_DPRINT12(ha, "attrs->qp_type = %d\n", attrs->qp_type); QL_DPRINT12(ha, "attrs->create_flags = %d\n", attrs->create_flags); QL_DPRINT12(ha, "attrs->port_num = %d\n", attrs->port_num); QL_DPRINT12(ha, "attrs->cap.max_send_wr = 0x%x\n", attrs->cap.max_send_wr); QL_DPRINT12(ha, "attrs->cap.max_recv_wr = 0x%x\n", attrs->cap.max_recv_wr); QL_DPRINT12(ha, "attrs->cap.max_send_sge = 0x%x\n", attrs->cap.max_send_sge); QL_DPRINT12(ha, "attrs->cap.max_recv_sge = 0x%x\n", attrs->cap.max_recv_sge); QL_DPRINT12(ha, "attrs->cap.max_inline_data = 0x%x\n", attrs->cap.max_inline_data); QL_DPRINT12(ha, "\n\nqattr->vendor_id = 0x%x\n", qattr->vendor_id); QL_DPRINT12(ha, "qattr->vendor_part_id = 0x%x\n", qattr->vendor_part_id); QL_DPRINT12(ha, "qattr->hw_ver = 0x%x\n", qattr->hw_ver); QL_DPRINT12(ha, "qattr->fw_ver = %p\n", (void *)qattr->fw_ver); QL_DPRINT12(ha, "qattr->node_guid = %p\n", (void *)qattr->node_guid); QL_DPRINT12(ha, "qattr->sys_image_guid = %p\n", (void *)qattr->sys_image_guid); QL_DPRINT12(ha, "qattr->max_cnq = 0x%x\n", qattr->max_cnq); QL_DPRINT12(ha, "qattr->max_sge = 0x%x\n", qattr->max_sge); QL_DPRINT12(ha, "qattr->max_srq_sge = 0x%x\n", qattr->max_srq_sge); QL_DPRINT12(ha, "qattr->max_inline = 0x%x\n", qattr->max_inline); QL_DPRINT12(ha, "qattr->max_wqe = 0x%x\n", qattr->max_wqe); QL_DPRINT12(ha, "qattr->max_srq_wqe = 0x%x\n", qattr->max_srq_wqe); QL_DPRINT12(ha, "qattr->max_qp_resp_rd_atomic_resc = 0x%x\n", qattr->max_qp_resp_rd_atomic_resc); QL_DPRINT12(ha, "qattr->max_qp_req_rd_atomic_resc = 0x%x\n", qattr->max_qp_req_rd_atomic_resc); QL_DPRINT12(ha, "qattr->max_dev_resp_rd_atomic_resc = 0x%x\n", qattr->max_dev_resp_rd_atomic_resc); QL_DPRINT12(ha, "qattr->max_cq = 0x%x\n", qattr->max_cq); QL_DPRINT12(ha, "qattr->max_qp = 0x%x\n", qattr->max_qp); QL_DPRINT12(ha, "qattr->max_srq = 0x%x\n", qattr->max_srq); QL_DPRINT12(ha, "qattr->max_mr = 0x%x\n", qattr->max_mr); QL_DPRINT12(ha, "qattr->max_mr_size = %p\n", (void *)qattr->max_mr_size); QL_DPRINT12(ha, "qattr->max_cqe = 0x%x\n", qattr->max_cqe); QL_DPRINT12(ha, "qattr->max_mw = 0x%x\n", qattr->max_mw); QL_DPRINT12(ha, "qattr->max_fmr = 0x%x\n", qattr->max_fmr); QL_DPRINT12(ha, "qattr->max_mr_mw_fmr_pbl = 0x%x\n", qattr->max_mr_mw_fmr_pbl); QL_DPRINT12(ha, "qattr->max_mr_mw_fmr_size = %p\n", (void *)qattr->max_mr_mw_fmr_size); QL_DPRINT12(ha, "qattr->max_pd = 0x%x\n", qattr->max_pd); QL_DPRINT12(ha, "qattr->max_ah = 0x%x\n", qattr->max_ah); QL_DPRINT12(ha, "qattr->max_pkey = 0x%x\n", qattr->max_pkey); QL_DPRINT12(ha, "qattr->max_srq_wr = 0x%x\n", qattr->max_srq_wr); QL_DPRINT12(ha, "qattr->max_stats_queues = 0x%x\n", qattr->max_stats_queues); //QL_DPRINT12(ha, "qattr->dev_caps = 0x%x\n", qattr->dev_caps); QL_DPRINT12(ha, "qattr->page_size_caps = %p\n", (void *)qattr->page_size_caps); QL_DPRINT12(ha, "qattr->dev_ack_delay = 0x%x\n", qattr->dev_ack_delay); QL_DPRINT12(ha, "qattr->reserved_lkey = 0x%x\n", qattr->reserved_lkey); QL_DPRINT12(ha, "qattr->bad_pkey_counter = 0x%x\n", qattr->bad_pkey_counter); if ((attrs->qp_type == IB_QPT_GSI) && udata) { QL_DPRINT12(ha, "unexpected udata when creating GSI QP\n"); return -EINVAL; } if (udata && !(ibpd->uobject && ibpd->uobject->context)) { QL_DPRINT12(ha, "called from user without context\n"); return -EINVAL; } /* QP0... attrs->qp_type == IB_QPT_GSI */ if (attrs->qp_type != IB_QPT_RC && attrs->qp_type != IB_QPT_GSI) { QL_DPRINT12(ha, "unsupported qp type=0x%x requested\n", attrs->qp_type); return -EINVAL; } if (attrs->qp_type == IB_QPT_GSI && attrs->srq) { QL_DPRINT12(ha, "cannot create GSI qp with SRQ\n"); return -EINVAL; } /* Skip the check for QP1 to support CM size of 128 */ if (attrs->cap.max_send_wr > qattr->max_wqe) { QL_DPRINT12(ha, "cannot create a SQ with %d elements " " (max_send_wr=0x%x)\n", attrs->cap.max_send_wr, qattr->max_wqe); return -EINVAL; } if (!attrs->srq && (attrs->cap.max_recv_wr > qattr->max_wqe)) { QL_DPRINT12(ha, "cannot create a RQ with %d elements" " (max_recv_wr=0x%x)\n", attrs->cap.max_recv_wr, qattr->max_wqe); return -EINVAL; } if (attrs->cap.max_inline_data > qattr->max_inline) { QL_DPRINT12(ha, "unsupported inline data size=0x%x " "requested (max_inline=0x%x)\n", attrs->cap.max_inline_data, qattr->max_inline); return -EINVAL; } if (attrs->cap.max_send_sge > qattr->max_sge) { QL_DPRINT12(ha, "unsupported send_sge=0x%x " "requested (max_send_sge=0x%x)\n", attrs->cap.max_send_sge, qattr->max_sge); return -EINVAL; } if (attrs->cap.max_recv_sge > qattr->max_sge) { QL_DPRINT12(ha, "unsupported recv_sge=0x%x requested " " (max_recv_sge=0x%x)\n", attrs->cap.max_recv_sge, qattr->max_sge); return -EINVAL; } /* unprivileged user space cannot create special QP */ if (ibpd->uobject && attrs->qp_type == IB_QPT_GSI) { QL_DPRINT12(ha, "userspace can't create special QPs of type=0x%x\n", attrs->qp_type); return -EINVAL; } /* allow creating only one GSI type of QP */ if (attrs->qp_type == IB_QPT_GSI && dev->gsi_qp_created) { QL_DPRINT12(ha, "create qp: GSI special QPs already created.\n"); return -EINVAL; } /* verify consumer QPs are not trying to use GSI QP's CQ */ if ((attrs->qp_type != IB_QPT_GSI) && (dev->gsi_qp_created)) { struct qlnxr_cq *send_cq = get_qlnxr_cq(attrs->send_cq); struct qlnxr_cq *recv_cq = get_qlnxr_cq(attrs->recv_cq); if ((send_cq->cq_type == QLNXR_CQ_TYPE_GSI) || (recv_cq->cq_type == QLNXR_CQ_TYPE_GSI)) { QL_DPRINT11(ha, "consumer QP cannot use GSI CQs.\n"); return -EINVAL; } } QL_DPRINT12(ha, "exit\n"); return 0; } static int qlnxr_copy_srq_uresp(struct qlnxr_dev *dev, struct qlnxr_srq *srq, struct ib_udata *udata) { struct qlnxr_create_srq_uresp uresp; qlnx_host_t *ha; int rc; ha = dev->ha; QL_DPRINT12(ha, "enter\n"); memset(&uresp, 0, sizeof(uresp)); uresp.srq_id = srq->srq_id; rc = ib_copy_to_udata(udata, &uresp, sizeof(uresp)); QL_DPRINT12(ha, "exit [%d]\n", rc); return rc; } static void qlnxr_copy_rq_uresp(struct qlnxr_dev *dev, struct qlnxr_create_qp_uresp *uresp, struct qlnxr_qp *qp) { qlnx_host_t *ha; ha = dev->ha; /* Return if QP is associated with SRQ instead of RQ */ QL_DPRINT12(ha, "enter qp->srq = %p\n", qp->srq); if (qp->srq) return; /* iWARP requires two doorbells per RQ. */ if (QLNX_IS_IWARP(dev)) { uresp->rq_db_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_IWARP_RQ_PROD); uresp->rq_db2_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_FLAGS); QL_DPRINT12(ha, "uresp->rq_db_offset = 0x%x " "uresp->rq_db2_offset = 0x%x\n", uresp->rq_db_offset, uresp->rq_db2_offset); } else { uresp->rq_db_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_ROCE_RQ_PROD); } uresp->rq_icid = qp->icid; QL_DPRINT12(ha, "exit\n"); return; } static void qlnxr_copy_sq_uresp(struct qlnxr_dev *dev, struct qlnxr_create_qp_uresp *uresp, struct qlnxr_qp *qp) { qlnx_host_t *ha; ha = dev->ha; QL_DPRINT12(ha, "enter\n"); uresp->sq_db_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD); /* iWARP uses the same cid for rq and sq*/ if (QLNX_IS_IWARP(dev)) { uresp->sq_icid = qp->icid; QL_DPRINT12(ha, "uresp->sq_icid = 0x%x\n", uresp->sq_icid); } else uresp->sq_icid = qp->icid + 1; QL_DPRINT12(ha, "exit\n"); return; } static int qlnxr_copy_qp_uresp(struct qlnxr_dev *dev, struct qlnxr_qp *qp, struct ib_udata *udata) { int rc; struct qlnxr_create_qp_uresp uresp; qlnx_host_t *ha; ha = dev->ha; QL_DPRINT12(ha, "enter qp->icid =0x%x\n", qp->icid); memset(&uresp, 0, sizeof(uresp)); qlnxr_copy_sq_uresp(dev, &uresp, qp); qlnxr_copy_rq_uresp(dev, &uresp, qp); uresp.atomic_supported = dev->atomic_cap != IB_ATOMIC_NONE; uresp.qp_id = qp->qp_id; rc = ib_copy_to_udata(udata, &uresp, sizeof(uresp)); QL_DPRINT12(ha, "exit [%d]\n", rc); return rc; } static void qlnxr_set_common_qp_params(struct qlnxr_dev *dev, struct qlnxr_qp *qp, struct qlnxr_pd *pd, struct ib_qp_init_attr *attrs) { qlnx_host_t *ha; ha = dev->ha; QL_DPRINT12(ha, "enter\n"); spin_lock_init(&qp->q_lock); atomic_set(&qp->refcnt, 1); qp->pd = pd; qp->sig = QLNXR_QP_MAGIC_NUMBER; qp->qp_type = attrs->qp_type; qp->max_inline_data = ROCE_REQ_MAX_INLINE_DATA_SIZE; qp->sq.max_sges = attrs->cap.max_send_sge; qp->state = ECORE_ROCE_QP_STATE_RESET; qp->signaled = (attrs->sq_sig_type == IB_SIGNAL_ALL_WR) ? true : false; qp->sq_cq = get_qlnxr_cq(attrs->send_cq); qp->rq_cq = get_qlnxr_cq(attrs->recv_cq); qp->dev = dev; if (!attrs->srq) { /* QP is associated with RQ instead of SRQ */ qp->rq.max_sges = attrs->cap.max_recv_sge; QL_DPRINT12(ha, "RQ params:\trq_max_sges = %d, rq_cq_id = %d\n", qp->rq.max_sges, qp->rq_cq->icid); } else { qp->srq = get_qlnxr_srq(attrs->srq); } QL_DPRINT12(ha, "QP params:\tpd = %d, qp_type = %d, max_inline_data = %d," " state = %d, signaled = %d, use_srq=%d\n", pd->pd_id, qp->qp_type, qp->max_inline_data, qp->state, qp->signaled, ((attrs->srq) ? 1 : 0)); QL_DPRINT12(ha, "SQ params:\tsq_max_sges = %d, sq_cq_id = %d\n", qp->sq.max_sges, qp->sq_cq->icid); return; } static int qlnxr_check_srq_params(struct qlnxr_dev *dev, struct ib_srq_init_attr *attrs) { struct ecore_rdma_device *qattr; qlnx_host_t *ha; ha = dev->ha; qattr = ecore_rdma_query_device(dev->rdma_ctx); QL_DPRINT12(ha, "enter\n"); if (attrs->attr.max_wr > qattr->max_srq_wqe) { QL_DPRINT12(ha, "unsupported srq_wr=0x%x" " requested (max_srq_wr=0x%x)\n", attrs->attr.max_wr, qattr->max_srq_wr); return -EINVAL; } if (attrs->attr.max_sge > qattr->max_sge) { QL_DPRINT12(ha, "unsupported sge=0x%x requested (max_srq_sge=0x%x)\n", attrs->attr.max_sge, qattr->max_sge); return -EINVAL; } if (attrs->attr.srq_limit > attrs->attr.max_wr) { QL_DPRINT12(ha, "unsupported srq_limit=0x%x requested" " (max_srq_limit=0x%x)\n", attrs->attr.srq_limit, attrs->attr.srq_limit); return -EINVAL; } QL_DPRINT12(ha, "exit\n"); return 0; } static void qlnxr_free_srq_user_params(struct qlnxr_srq *srq) { struct qlnxr_dev *dev = srq->dev; qlnx_host_t *ha; ha = dev->ha; QL_DPRINT12(ha, "enter\n"); qlnxr_free_pbl(srq->dev, &srq->usrq.pbl_info, srq->usrq.pbl_tbl); ib_umem_release(srq->usrq.umem); ib_umem_release(srq->prod_umem); QL_DPRINT12(ha, "exit\n"); return; } static void qlnxr_free_srq_kernel_params(struct qlnxr_srq *srq) { struct qlnxr_srq_hwq_info *hw_srq = &srq->hw_srq; struct qlnxr_dev *dev = srq->dev; qlnx_host_t *ha; ha = dev->ha; QL_DPRINT12(ha, "enter\n"); ecore_chain_free(dev->cdev, &hw_srq->pbl); qlnx_dma_free_coherent(&dev->cdev, hw_srq->virt_prod_pair_addr, hw_srq->phy_prod_pair_addr, sizeof(struct rdma_srq_producers)); QL_DPRINT12(ha, "exit\n"); return; } static int qlnxr_init_srq_user_params(struct ib_ucontext *ib_ctx, struct qlnxr_srq *srq, struct qlnxr_create_srq_ureq *ureq, int access, int dmasync) { struct scatterlist *sg; int rc; struct qlnxr_dev *dev = srq->dev; qlnx_host_t *ha; ha = dev->ha; QL_DPRINT12(ha, "enter\n"); rc = qlnxr_init_user_queue(ib_ctx, srq->dev, &srq->usrq, ureq->srq_addr, ureq->srq_len, access, dmasync, 1); if (rc) return rc; srq->prod_umem = ib_umem_get(ib_ctx, ureq->prod_pair_addr, sizeof(struct rdma_srq_producers), access, dmasync); if (IS_ERR(srq->prod_umem)) { qlnxr_free_pbl(srq->dev, &srq->usrq.pbl_info, srq->usrq.pbl_tbl); ib_umem_release(srq->usrq.umem); QL_DPRINT12(ha, "ib_umem_get failed for producer [%p]\n", PTR_ERR(srq->prod_umem)); return PTR_ERR(srq->prod_umem); } sg = srq->prod_umem->sg_head.sgl; srq->hw_srq.phy_prod_pair_addr = sg_dma_address(sg); QL_DPRINT12(ha, "exit\n"); return 0; } static int qlnxr_alloc_srq_kernel_params(struct qlnxr_srq *srq, struct qlnxr_dev *dev, struct ib_srq_init_attr *init_attr) { struct qlnxr_srq_hwq_info *hw_srq = &srq->hw_srq; dma_addr_t phy_prod_pair_addr; u32 num_elems, max_wr; void *va; int rc; qlnx_host_t *ha; ha = dev->ha; QL_DPRINT12(ha, "enter\n"); va = qlnx_dma_alloc_coherent(&dev->cdev, &phy_prod_pair_addr, sizeof(struct rdma_srq_producers)); if (!va) { QL_DPRINT11(ha, "qlnx_dma_alloc_coherent failed for produceer\n"); return -ENOMEM; } hw_srq->phy_prod_pair_addr = phy_prod_pair_addr; hw_srq->virt_prod_pair_addr = va; max_wr = init_attr->attr.max_wr; num_elems = max_wr * RDMA_MAX_SRQ_WQE_SIZE; rc = ecore_chain_alloc(dev->cdev, ECORE_CHAIN_USE_TO_CONSUME_PRODUCE, ECORE_CHAIN_MODE_PBL, ECORE_CHAIN_CNT_TYPE_U32, num_elems, ECORE_RDMA_SRQ_WQE_ELEM_SIZE, &hw_srq->pbl, NULL); if (rc) { QL_DPRINT11(ha, "ecore_chain_alloc failed [%d]\n", rc); goto err0; } hw_srq->max_wr = max_wr; hw_srq->num_elems = num_elems; hw_srq->max_sges = RDMA_MAX_SGE_PER_SRQ; QL_DPRINT12(ha, "exit\n"); return 0; err0: qlnx_dma_free_coherent(&dev->cdev, va, phy_prod_pair_addr, sizeof(struct rdma_srq_producers)); QL_DPRINT12(ha, "exit [%d]\n", rc); return rc; } static inline void qlnxr_init_common_qp_in_params(struct qlnxr_dev *dev, struct qlnxr_pd *pd, struct qlnxr_qp *qp, struct ib_qp_init_attr *attrs, bool fmr_and_reserved_lkey, struct ecore_rdma_create_qp_in_params *params) { qlnx_host_t *ha; ha = dev->ha; QL_DPRINT12(ha, "enter\n"); /* QP handle to be written in an async event */ params->qp_handle_async_lo = lower_32_bits((uintptr_t)qp); params->qp_handle_async_hi = upper_32_bits((uintptr_t)qp); params->signal_all = (attrs->sq_sig_type == IB_SIGNAL_ALL_WR); params->fmr_and_reserved_lkey = fmr_and_reserved_lkey; params->pd = pd->pd_id; params->dpi = pd->uctx ? pd->uctx->dpi : dev->dpi; params->sq_cq_id = get_qlnxr_cq(attrs->send_cq)->icid; params->stats_queue = 0; params->rq_cq_id = get_qlnxr_cq(attrs->recv_cq)->icid; if (qp->srq) { /* QP is associated with SRQ instead of RQ */ params->srq_id = qp->srq->srq_id; params->use_srq = true; QL_DPRINT11(ha, "exit srq_id = 0x%x use_srq = 0x%x\n", params->srq_id, params->use_srq); return; } params->srq_id = 0; params->use_srq = false; QL_DPRINT12(ha, "exit\n"); return; } static inline void qlnxr_qp_user_print( struct qlnxr_dev *dev, struct qlnxr_qp *qp) { QL_DPRINT12((dev->ha), "qp=%p. sq_addr=0x%llx, sq_len=%zd, " "rq_addr=0x%llx, rq_len=%zd\n", qp, qp->usq.buf_addr, qp->usq.buf_len, qp->urq.buf_addr, qp->urq.buf_len); return; } static int qlnxr_idr_add(struct qlnxr_dev *dev, void *ptr, u32 id) { u32 newid; int rc; qlnx_host_t *ha; ha = dev->ha; QL_DPRINT12(ha, "enter\n"); if (!QLNX_IS_IWARP(dev)) return 0; do { if (!idr_pre_get(&dev->qpidr, GFP_KERNEL)) { QL_DPRINT11(ha, "idr_pre_get failed\n"); return -ENOMEM; } mtx_lock(&dev->idr_lock); rc = idr_get_new_above(&dev->qpidr, ptr, id, &newid); mtx_unlock(&dev->idr_lock); } while (rc == -EAGAIN); QL_DPRINT12(ha, "exit [%d]\n", rc); return rc; } static void qlnxr_idr_remove(struct qlnxr_dev *dev, u32 id) { qlnx_host_t *ha; ha = dev->ha; QL_DPRINT12(ha, "enter\n"); if (!QLNX_IS_IWARP(dev)) return; mtx_lock(&dev->idr_lock); idr_remove(&dev->qpidr, id); mtx_unlock(&dev->idr_lock); QL_DPRINT12(ha, "exit \n"); return; } static inline void qlnxr_iwarp_populate_user_qp(struct qlnxr_dev *dev, struct qlnxr_qp *qp, struct ecore_rdma_create_qp_out_params *out_params) { qlnx_host_t *ha; ha = dev->ha; QL_DPRINT12(ha, "enter\n"); qp->usq.pbl_tbl->va = out_params->sq_pbl_virt; qp->usq.pbl_tbl->pa = out_params->sq_pbl_phys; qlnxr_populate_pbls(dev, qp->usq.umem, qp->usq.pbl_tbl, &qp->usq.pbl_info); if (qp->srq) { QL_DPRINT11(ha, "qp->srq = %p\n", qp->srq); return; } qp->urq.pbl_tbl->va = out_params->rq_pbl_virt; qp->urq.pbl_tbl->pa = out_params->rq_pbl_phys; qlnxr_populate_pbls(dev, qp->urq.umem, qp->urq.pbl_tbl, &qp->urq.pbl_info); QL_DPRINT12(ha, "exit\n"); return; } static int qlnxr_create_user_qp(struct qlnxr_dev *dev, struct qlnxr_qp *qp, struct ib_pd *ibpd, struct ib_udata *udata, struct ib_qp_init_attr *attrs) { struct ecore_rdma_destroy_qp_out_params d_out_params; struct ecore_rdma_create_qp_in_params in_params; struct ecore_rdma_create_qp_out_params out_params; struct qlnxr_pd *pd = get_qlnxr_pd(ibpd); struct ib_ucontext *ib_ctx = NULL; struct qlnxr_create_qp_ureq ureq; int alloc_and_init = QLNX_IS_ROCE(dev); int rc = -EINVAL; qlnx_host_t *ha; ha = dev->ha; QL_DPRINT12(ha, "enter\n"); ib_ctx = ibpd->uobject->context; memset(&ureq, 0, sizeof(ureq)); rc = ib_copy_from_udata(&ureq, udata, sizeof(ureq)); if (rc) { QL_DPRINT11(ha, "ib_copy_from_udata failed [%d]\n", rc); return rc; } /* SQ - read access only (0), dma sync not required (0) */ rc = qlnxr_init_user_queue(ib_ctx, dev, &qp->usq, ureq.sq_addr, ureq.sq_len, 0, 0, alloc_and_init); if (rc) { QL_DPRINT11(ha, "qlnxr_init_user_queue failed [%d]\n", rc); return rc; } if (!qp->srq) { /* RQ - read access only (0), dma sync not required (0) */ rc = qlnxr_init_user_queue(ib_ctx, dev, &qp->urq, ureq.rq_addr, ureq.rq_len, 0, 0, alloc_and_init); if (rc) { QL_DPRINT11(ha, "qlnxr_init_user_queue failed [%d]\n", rc); return rc; } } memset(&in_params, 0, sizeof(in_params)); qlnxr_init_common_qp_in_params(dev, pd, qp, attrs, false, &in_params); in_params.qp_handle_lo = ureq.qp_handle_lo; in_params.qp_handle_hi = ureq.qp_handle_hi; in_params.sq_num_pages = qp->usq.pbl_info.num_pbes; in_params.sq_pbl_ptr = qp->usq.pbl_tbl->pa; if (!qp->srq) { in_params.rq_num_pages = qp->urq.pbl_info.num_pbes; in_params.rq_pbl_ptr = qp->urq.pbl_tbl->pa; } qp->ecore_qp = ecore_rdma_create_qp(dev->rdma_ctx, &in_params, &out_params); if (!qp->ecore_qp) { rc = -ENOMEM; QL_DPRINT11(ha, "ecore_rdma_create_qp failed\n"); goto err1; } if (QLNX_IS_IWARP(dev)) qlnxr_iwarp_populate_user_qp(dev, qp, &out_params); qp->qp_id = out_params.qp_id; qp->icid = out_params.icid; rc = qlnxr_copy_qp_uresp(dev, qp, udata); if (rc) { QL_DPRINT11(ha, "qlnxr_copy_qp_uresp failed\n"); goto err; } qlnxr_qp_user_print(dev, qp); QL_DPRINT12(ha, "exit\n"); return 0; err: rc = ecore_rdma_destroy_qp(dev->rdma_ctx, qp->ecore_qp, &d_out_params); if (rc) QL_DPRINT12(ha, "fatal fault\n"); err1: qlnxr_cleanup_user(dev, qp); QL_DPRINT12(ha, "exit[%d]\n", rc); return rc; } static void qlnxr_set_roce_db_info(struct qlnxr_dev *dev, struct qlnxr_qp *qp) { qlnx_host_t *ha; ha = dev->ha; QL_DPRINT12(ha, "enter qp = %p qp->srq %p\n", qp, qp->srq); qp->sq.db = dev->db_addr + DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD); qp->sq.db_data.data.icid = qp->icid + 1; if (!qp->srq) { qp->rq.db = dev->db_addr + DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_ROCE_RQ_PROD); qp->rq.db_data.data.icid = qp->icid; } QL_DPRINT12(ha, "exit\n"); return; } static void qlnxr_set_iwarp_db_info(struct qlnxr_dev *dev, struct qlnxr_qp *qp) { qlnx_host_t *ha; ha = dev->ha; QL_DPRINT12(ha, "enter qp = %p qp->srq %p\n", qp, qp->srq); qp->sq.db = dev->db_addr + DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD); qp->sq.db_data.data.icid = qp->icid; if (!qp->srq) { qp->rq.db = dev->db_addr + DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_IWARP_RQ_PROD); qp->rq.db_data.data.icid = qp->icid; qp->rq.iwarp_db2 = dev->db_addr + DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_FLAGS); qp->rq.iwarp_db2_data.data.icid = qp->icid; qp->rq.iwarp_db2_data.data.value = DQ_TCM_IWARP_POST_RQ_CF_CMD; } QL_DPRINT12(ha, "qp->sq.db = %p qp->sq.db_data.data.icid =0x%x\n" "\t\t\tqp->rq.db = %p qp->rq.db_data.data.icid =0x%x\n" "\t\t\tqp->rq.iwarp_db2 = %p qp->rq.iwarp_db2.data.icid =0x%x" " qp->rq.iwarp_db2.data.prod_val =0x%x\n", qp->sq.db, qp->sq.db_data.data.icid, qp->rq.db, qp->rq.db_data.data.icid, qp->rq.iwarp_db2, qp->rq.iwarp_db2_data.data.icid, qp->rq.iwarp_db2_data.data.value); QL_DPRINT12(ha, "exit\n"); return; } static int qlnxr_roce_create_kernel_qp(struct qlnxr_dev *dev, struct qlnxr_qp *qp, struct ecore_rdma_create_qp_in_params *in_params, u32 n_sq_elems, u32 n_rq_elems) { struct ecore_rdma_create_qp_out_params out_params; int rc; qlnx_host_t *ha; ha = dev->ha; QL_DPRINT12(ha, "enter\n"); rc = ecore_chain_alloc( dev->cdev, ECORE_CHAIN_USE_TO_PRODUCE, ECORE_CHAIN_MODE_PBL, ECORE_CHAIN_CNT_TYPE_U32, n_sq_elems, QLNXR_SQE_ELEMENT_SIZE, &qp->sq.pbl, NULL); if (rc) { QL_DPRINT11(ha, "ecore_chain_alloc qp->sq.pbl failed[%d]\n", rc); return rc; } in_params->sq_num_pages = ecore_chain_get_page_cnt(&qp->sq.pbl); in_params->sq_pbl_ptr = ecore_chain_get_pbl_phys(&qp->sq.pbl); if (!qp->srq) { rc = ecore_chain_alloc( dev->cdev, ECORE_CHAIN_USE_TO_CONSUME_PRODUCE, ECORE_CHAIN_MODE_PBL, ECORE_CHAIN_CNT_TYPE_U32, n_rq_elems, QLNXR_RQE_ELEMENT_SIZE, &qp->rq.pbl, NULL); if (rc) { QL_DPRINT11(ha, "ecore_chain_alloc qp->rq.pbl failed[%d]\n", rc); return rc; } in_params->rq_num_pages = ecore_chain_get_page_cnt(&qp->rq.pbl); in_params->rq_pbl_ptr = ecore_chain_get_pbl_phys(&qp->rq.pbl); } qp->ecore_qp = ecore_rdma_create_qp(dev->rdma_ctx, in_params, &out_params); if (!qp->ecore_qp) { QL_DPRINT11(ha, "qp->ecore_qp == NULL\n"); return -EINVAL; } qp->qp_id = out_params.qp_id; qp->icid = out_params.icid; qlnxr_set_roce_db_info(dev, qp); QL_DPRINT12(ha, "exit\n"); return 0; } static int qlnxr_iwarp_create_kernel_qp(struct qlnxr_dev *dev, struct qlnxr_qp *qp, struct ecore_rdma_create_qp_in_params *in_params, u32 n_sq_elems, u32 n_rq_elems) { struct ecore_rdma_destroy_qp_out_params d_out_params; struct ecore_rdma_create_qp_out_params out_params; struct ecore_chain_ext_pbl ext_pbl; int rc; qlnx_host_t *ha; ha = dev->ha; QL_DPRINT12(ha, "enter\n"); in_params->sq_num_pages = ECORE_CHAIN_PAGE_CNT(n_sq_elems, QLNXR_SQE_ELEMENT_SIZE, ECORE_CHAIN_MODE_PBL); in_params->rq_num_pages = ECORE_CHAIN_PAGE_CNT(n_rq_elems, QLNXR_RQE_ELEMENT_SIZE, ECORE_CHAIN_MODE_PBL); QL_DPRINT12(ha, "n_sq_elems = 0x%x" " n_rq_elems = 0x%x in_params\n" "\t\t\tqp_handle_lo\t\t= 0x%08x\n" "\t\t\tqp_handle_hi\t\t= 0x%08x\n" "\t\t\tqp_handle_async_lo\t\t= 0x%08x\n" "\t\t\tqp_handle_async_hi\t\t= 0x%08x\n" "\t\t\tuse_srq\t\t\t= 0x%x\n" "\t\t\tsignal_all\t\t= 0x%x\n" "\t\t\tfmr_and_reserved_lkey\t= 0x%x\n" "\t\t\tpd\t\t\t= 0x%x\n" "\t\t\tdpi\t\t\t= 0x%x\n" "\t\t\tsq_cq_id\t\t\t= 0x%x\n" "\t\t\tsq_num_pages\t\t= 0x%x\n" "\t\t\tsq_pbl_ptr\t\t= %p\n" "\t\t\tmax_sq_sges\t\t= 0x%x\n" "\t\t\trq_cq_id\t\t\t= 0x%x\n" "\t\t\trq_num_pages\t\t= 0x%x\n" "\t\t\trq_pbl_ptr\t\t= %p\n" "\t\t\tsrq_id\t\t\t= 0x%x\n" "\t\t\tstats_queue\t\t= 0x%x\n", n_sq_elems, n_rq_elems, in_params->qp_handle_lo, in_params->qp_handle_hi, in_params->qp_handle_async_lo, in_params->qp_handle_async_hi, in_params->use_srq, in_params->signal_all, in_params->fmr_and_reserved_lkey, in_params->pd, in_params->dpi, in_params->sq_cq_id, in_params->sq_num_pages, (void *)in_params->sq_pbl_ptr, in_params->max_sq_sges, in_params->rq_cq_id, in_params->rq_num_pages, (void *)in_params->rq_pbl_ptr, in_params->srq_id, in_params->stats_queue ); memset(&out_params, 0, sizeof (struct ecore_rdma_create_qp_out_params)); memset(&ext_pbl, 0, sizeof (struct ecore_chain_ext_pbl)); qp->ecore_qp = ecore_rdma_create_qp(dev->rdma_ctx, in_params, &out_params); if (!qp->ecore_qp) { QL_DPRINT11(ha, "ecore_rdma_create_qp failed\n"); return -EINVAL; } /* Now we allocate the chain */ ext_pbl.p_pbl_virt = out_params.sq_pbl_virt; ext_pbl.p_pbl_phys = out_params.sq_pbl_phys; QL_DPRINT12(ha, "ext_pbl.p_pbl_virt = %p " "ext_pbl.p_pbl_phys = %p\n", ext_pbl.p_pbl_virt, ext_pbl.p_pbl_phys); rc = ecore_chain_alloc( dev->cdev, ECORE_CHAIN_USE_TO_PRODUCE, ECORE_CHAIN_MODE_PBL, ECORE_CHAIN_CNT_TYPE_U32, n_sq_elems, QLNXR_SQE_ELEMENT_SIZE, &qp->sq.pbl, &ext_pbl); if (rc) { QL_DPRINT11(ha, "ecore_chain_alloc qp->sq.pbl failed rc = %d\n", rc); goto err; } ext_pbl.p_pbl_virt = out_params.rq_pbl_virt; ext_pbl.p_pbl_phys = out_params.rq_pbl_phys; QL_DPRINT12(ha, "ext_pbl.p_pbl_virt = %p " "ext_pbl.p_pbl_phys = %p\n", ext_pbl.p_pbl_virt, ext_pbl.p_pbl_phys); if (!qp->srq) { rc = ecore_chain_alloc( dev->cdev, ECORE_CHAIN_USE_TO_CONSUME_PRODUCE, ECORE_CHAIN_MODE_PBL, ECORE_CHAIN_CNT_TYPE_U32, n_rq_elems, QLNXR_RQE_ELEMENT_SIZE, &qp->rq.pbl, &ext_pbl); if (rc) { QL_DPRINT11(ha,, "ecore_chain_alloc qp->rq.pbl" " failed rc = %d\n", rc); goto err; } } QL_DPRINT12(ha, "qp_id = 0x%x icid =0x%x\n", out_params.qp_id, out_params.icid); qp->qp_id = out_params.qp_id; qp->icid = out_params.icid; qlnxr_set_iwarp_db_info(dev, qp); QL_DPRINT12(ha, "exit\n"); return 0; err: ecore_rdma_destroy_qp(dev->rdma_ctx, qp->ecore_qp, &d_out_params); QL_DPRINT12(ha, "exit rc = %d\n", rc); return rc; } static int qlnxr_create_kernel_qp(struct qlnxr_dev *dev, struct qlnxr_qp *qp, struct ib_pd *ibpd, struct ib_qp_init_attr *attrs) { struct ecore_rdma_create_qp_in_params in_params; struct qlnxr_pd *pd = get_qlnxr_pd(ibpd); int rc = -EINVAL; u32 n_rq_elems; u32 n_sq_elems; u32 n_sq_entries; struct ecore_rdma_device *qattr = ecore_rdma_query_device(dev->rdma_ctx); qlnx_host_t *ha; ha = dev->ha; QL_DPRINT12(ha, "enter\n"); memset(&in_params, 0, sizeof(in_params)); /* A single work request may take up to MAX_SQ_WQE_SIZE elements in * the ring. The ring should allow at least a single WR, even if the * user requested none, due to allocation issues. * We should add an extra WR since the prod and cons indices of * wqe_wr_id are managed in such a way that the WQ is considered full * when (prod+1)%max_wr==cons. We currently don't do that because we * double the number of entries due an iSER issue that pushes far more * WRs than indicated. If we decline its ib_post_send() then we get * error prints in the dmesg we'd like to avoid. */ qp->sq.max_wr = min_t(u32, attrs->cap.max_send_wr * dev->wq_multiplier, qattr->max_wqe); qp->wqe_wr_id = kzalloc(qp->sq.max_wr * sizeof(*qp->wqe_wr_id), GFP_KERNEL); if (!qp->wqe_wr_id) { QL_DPRINT11(ha, "failed SQ shadow memory allocation\n"); return -ENOMEM; } /* QP handle to be written in CQE */ in_params.qp_handle_lo = lower_32_bits((uintptr_t)qp); in_params.qp_handle_hi = upper_32_bits((uintptr_t)qp); /* A single work request may take up to MAX_RQ_WQE_SIZE elements in * the ring. There ring should allow at least a single WR, even if the * user requested none, due to allocation issues. */ qp->rq.max_wr = (u16)max_t(u32, attrs->cap.max_recv_wr, 1); /* Allocate driver internal RQ array */ if (!qp->srq) { qp->rqe_wr_id = kzalloc(qp->rq.max_wr * sizeof(*qp->rqe_wr_id), GFP_KERNEL); if (!qp->rqe_wr_id) { QL_DPRINT11(ha, "failed RQ shadow memory allocation\n"); kfree(qp->wqe_wr_id); return -ENOMEM; } } //qlnxr_init_common_qp_in_params(dev, pd, qp, attrs, true, &in_params); in_params.qp_handle_async_lo = lower_32_bits((uintptr_t)qp); in_params.qp_handle_async_hi = upper_32_bits((uintptr_t)qp); in_params.signal_all = (attrs->sq_sig_type == IB_SIGNAL_ALL_WR); in_params.fmr_and_reserved_lkey = true; in_params.pd = pd->pd_id; in_params.dpi = pd->uctx ? pd->uctx->dpi : dev->dpi; in_params.sq_cq_id = get_qlnxr_cq(attrs->send_cq)->icid; in_params.stats_queue = 0; in_params.rq_cq_id = get_qlnxr_cq(attrs->recv_cq)->icid; if (qp->srq) { /* QP is associated with SRQ instead of RQ */ in_params.srq_id = qp->srq->srq_id; in_params.use_srq = true; QL_DPRINT11(ha, "exit srq_id = 0x%x use_srq = 0x%x\n", in_params.srq_id, in_params.use_srq); } else { in_params.srq_id = 0; in_params.use_srq = false; } n_sq_entries = attrs->cap.max_send_wr; n_sq_entries = min_t(u32, n_sq_entries, qattr->max_wqe); n_sq_entries = max_t(u32, n_sq_entries, 1); n_sq_elems = n_sq_entries * QLNXR_MAX_SQE_ELEMENTS_PER_SQE; n_rq_elems = qp->rq.max_wr * QLNXR_MAX_RQE_ELEMENTS_PER_RQE; if (QLNX_IS_ROCE(dev)) { rc = qlnxr_roce_create_kernel_qp(dev, qp, &in_params, n_sq_elems, n_rq_elems); } else { rc = qlnxr_iwarp_create_kernel_qp(dev, qp, &in_params, n_sq_elems, n_rq_elems); } if (rc) qlnxr_cleanup_kernel(dev, qp); QL_DPRINT12(ha, "exit [%d]\n", rc); return rc; } struct ib_qp * qlnxr_create_qp(struct ib_pd *ibpd, struct ib_qp_init_attr *attrs, struct ib_udata *udata) { struct qlnxr_dev *dev = get_qlnxr_dev(ibpd->device); struct qlnxr_pd *pd = get_qlnxr_pd(ibpd); struct qlnxr_qp *qp; int rc = 0; qlnx_host_t *ha; ha = dev->ha; QL_DPRINT12(ha, "enter\n"); rc = qlnxr_check_qp_attrs(ibpd, dev, attrs, udata); if (rc) { QL_DPRINT11(ha, "qlnxr_check_qp_attrs failed [%d]\n", rc); return ERR_PTR(rc); } QL_DPRINT12(ha, "called from %s, event_handle=%p," " eepd=%p sq_cq=%p, sq_icid=%d, rq_cq=%p, rq_icid=%d\n", (udata ? "user library" : "kernel"), attrs->event_handler, pd, get_qlnxr_cq(attrs->send_cq), get_qlnxr_cq(attrs->send_cq)->icid, get_qlnxr_cq(attrs->recv_cq), get_qlnxr_cq(attrs->recv_cq)->icid); qp = qlnx_zalloc(sizeof(struct qlnxr_qp)); if (!qp) { QL_DPRINT11(ha, "kzalloc(qp) failed\n"); return ERR_PTR(-ENOMEM); } qlnxr_set_common_qp_params(dev, qp, pd, attrs); if (attrs->qp_type == IB_QPT_GSI) { QL_DPRINT11(ha, "calling qlnxr_create_gsi_qp\n"); return qlnxr_create_gsi_qp(dev, attrs, qp); } if (udata) { rc = qlnxr_create_user_qp(dev, qp, ibpd, udata, attrs); if (rc) { QL_DPRINT11(ha, "qlnxr_create_user_qp failed\n"); goto err; } } else { rc = qlnxr_create_kernel_qp(dev, qp, ibpd, attrs); if (rc) { QL_DPRINT11(ha, "qlnxr_create_kernel_qp failed\n"); goto err; } } qp->ibqp.qp_num = qp->qp_id; rc = qlnxr_idr_add(dev, qp, qp->qp_id); if (rc) { QL_DPRINT11(ha, "qlnxr_idr_add failed\n"); goto err; } QL_DPRINT12(ha, "exit [%p]\n", &qp->ibqp); return &qp->ibqp; err: QL_DPRINT12(ha, "failed exit\n"); return ERR_PTR(-EFAULT); } static enum ib_qp_state qlnxr_get_ibqp_state(enum ecore_roce_qp_state qp_state) { enum ib_qp_state state = IB_QPS_ERR; switch (qp_state) { case ECORE_ROCE_QP_STATE_RESET: state = IB_QPS_RESET; break; case ECORE_ROCE_QP_STATE_INIT: state = IB_QPS_INIT; break; case ECORE_ROCE_QP_STATE_RTR: state = IB_QPS_RTR; break; case ECORE_ROCE_QP_STATE_RTS: state = IB_QPS_RTS; break; case ECORE_ROCE_QP_STATE_SQD: state = IB_QPS_SQD; break; case ECORE_ROCE_QP_STATE_ERR: state = IB_QPS_ERR; break; case ECORE_ROCE_QP_STATE_SQE: state = IB_QPS_SQE; break; } return state; } static enum ecore_roce_qp_state qlnxr_get_state_from_ibqp( enum ib_qp_state qp_state) { enum ecore_roce_qp_state ecore_qp_state; ecore_qp_state = ECORE_ROCE_QP_STATE_ERR; switch (qp_state) { case IB_QPS_RESET: ecore_qp_state = ECORE_ROCE_QP_STATE_RESET; break; case IB_QPS_INIT: ecore_qp_state = ECORE_ROCE_QP_STATE_INIT; break; case IB_QPS_RTR: ecore_qp_state = ECORE_ROCE_QP_STATE_RTR; break; case IB_QPS_RTS: ecore_qp_state = ECORE_ROCE_QP_STATE_RTS; break; case IB_QPS_SQD: ecore_qp_state = ECORE_ROCE_QP_STATE_SQD; break; case IB_QPS_ERR: ecore_qp_state = ECORE_ROCE_QP_STATE_ERR; break; default: ecore_qp_state = ECORE_ROCE_QP_STATE_ERR; break; } return (ecore_qp_state); } static void qlnxr_reset_qp_hwq_info(struct qlnxr_qp_hwq_info *qph) { ecore_chain_reset(&qph->pbl); qph->prod = qph->cons = 0; qph->wqe_cons = 0; qph->db_data.data.value = cpu_to_le16(0); return; } static int qlnxr_update_qp_state(struct qlnxr_dev *dev, struct qlnxr_qp *qp, enum ecore_roce_qp_state new_state) { int status = 0; uint32_t reg_addr; struct ecore_dev *cdev; qlnx_host_t *ha; ha = dev->ha; cdev = &ha->cdev; QL_DPRINT12(ha, "enter qp = %p new_state = 0x%x qp->state = 0x%x\n", qp, new_state, qp->state); if (new_state == qp->state) { return 0; } switch (qp->state) { case ECORE_ROCE_QP_STATE_RESET: switch (new_state) { case ECORE_ROCE_QP_STATE_INIT: qp->prev_wqe_size = 0; qlnxr_reset_qp_hwq_info(&qp->sq); if (!(qp->srq)) qlnxr_reset_qp_hwq_info(&qp->rq); break; default: status = -EINVAL; break; }; break; case ECORE_ROCE_QP_STATE_INIT: /* INIT->XXX */ switch (new_state) { case ECORE_ROCE_QP_STATE_RTR: /* Update doorbell (in case post_recv was done before move to RTR) */ if (qp->srq) break; wmb(); //writel(qp->rq.db_data.raw, qp->rq.db); //if (QLNX_IS_IWARP(dev)) // writel(qp->rq.iwarp_db2_data.raw, // qp->rq.iwarp_db2); reg_addr = (uint32_t)((uint8_t *)qp->rq.db - (uint8_t *)cdev->doorbells); bus_write_4(ha->pci_dbells, reg_addr, qp->rq.db_data.raw); bus_barrier(ha->pci_dbells, 0, 0, BUS_SPACE_BARRIER_READ); if (QLNX_IS_IWARP(dev)) { reg_addr = (uint32_t)((uint8_t *)qp->rq.iwarp_db2 - (uint8_t *)cdev->doorbells); bus_write_4(ha->pci_dbells, reg_addr,\ qp->rq.iwarp_db2_data.raw); bus_barrier(ha->pci_dbells, 0, 0,\ BUS_SPACE_BARRIER_READ); } mmiowb(); break; case ECORE_ROCE_QP_STATE_ERR: /* TBD:flush qps... */ break; default: /* invalid state change. */ status = -EINVAL; break; }; break; case ECORE_ROCE_QP_STATE_RTR: /* RTR->XXX */ switch (new_state) { case ECORE_ROCE_QP_STATE_RTS: break; case ECORE_ROCE_QP_STATE_ERR: break; default: /* invalid state change. */ status = -EINVAL; break; }; break; case ECORE_ROCE_QP_STATE_RTS: /* RTS->XXX */ switch (new_state) { case ECORE_ROCE_QP_STATE_SQD: break; case ECORE_ROCE_QP_STATE_ERR: break; default: /* invalid state change. */ status = -EINVAL; break; }; break; case ECORE_ROCE_QP_STATE_SQD: /* SQD->XXX */ switch (new_state) { case ECORE_ROCE_QP_STATE_RTS: case ECORE_ROCE_QP_STATE_ERR: break; default: /* invalid state change. */ status = -EINVAL; break; }; break; case ECORE_ROCE_QP_STATE_ERR: /* ERR->XXX */ switch (new_state) { case ECORE_ROCE_QP_STATE_RESET: if ((qp->rq.prod != qp->rq.cons) || (qp->sq.prod != qp->sq.cons)) { QL_DPRINT11(ha, "Error->Reset with rq/sq " "not empty rq.prod=0x%x rq.cons=0x%x" " sq.prod=0x%x sq.cons=0x%x\n", qp->rq.prod, qp->rq.cons, qp->sq.prod, qp->sq.cons); status = -EINVAL; } break; default: status = -EINVAL; break; }; break; default: status = -EINVAL; break; }; QL_DPRINT12(ha, "exit\n"); return status; } int qlnxr_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int attr_mask, struct ib_udata *udata) { int rc = 0; struct qlnxr_qp *qp = get_qlnxr_qp(ibqp); struct qlnxr_dev *dev = get_qlnxr_dev(&qp->dev->ibdev); struct ecore_rdma_modify_qp_in_params qp_params = { 0 }; enum ib_qp_state old_qp_state, new_qp_state; struct ecore_rdma_device *qattr = ecore_rdma_query_device(dev->rdma_ctx); qlnx_host_t *ha; ha = dev->ha; QL_DPRINT12(ha, "enter qp = %p attr_mask = 0x%x, state = %d udata = %p\n", qp, attr_mask, attr->qp_state, udata); old_qp_state = qlnxr_get_ibqp_state(qp->state); if (attr_mask & IB_QP_STATE) new_qp_state = attr->qp_state; else new_qp_state = old_qp_state; if (QLNX_IS_ROCE(dev)) { if (!ib_modify_qp_is_ok(old_qp_state, new_qp_state, ibqp->qp_type, attr_mask )) { QL_DPRINT12(ha, "invalid attribute mask=0x%x" " specified for qpn=0x%x of type=0x%x \n" " old_qp_state=0x%x, new_qp_state=0x%x\n", attr_mask, qp->qp_id, ibqp->qp_type, old_qp_state, new_qp_state); rc = -EINVAL; goto err; } } /* translate the masks... */ if (attr_mask & IB_QP_STATE) { SET_FIELD(qp_params.modify_flags, ECORE_RDMA_MODIFY_QP_VALID_NEW_STATE, 1); qp_params.new_state = qlnxr_get_state_from_ibqp(attr->qp_state); } // TBD consider changing ecore to be a flag as well... if (attr_mask & IB_QP_EN_SQD_ASYNC_NOTIFY) qp_params.sqd_async = true; if (attr_mask & IB_QP_PKEY_INDEX) { SET_FIELD(qp_params.modify_flags, ECORE_ROCE_MODIFY_QP_VALID_PKEY, 1); if (attr->pkey_index >= QLNXR_ROCE_PKEY_TABLE_LEN) { rc = -EINVAL; goto err; } qp_params.pkey = QLNXR_ROCE_PKEY_DEFAULT; } if (attr_mask & IB_QP_QKEY) { qp->qkey = attr->qkey; } /* tbd consider splitting in ecore.. */ if (attr_mask & IB_QP_ACCESS_FLAGS) { SET_FIELD(qp_params.modify_flags, ECORE_RDMA_MODIFY_QP_VALID_RDMA_OPS_EN, 1); qp_params.incoming_rdma_read_en = attr->qp_access_flags & IB_ACCESS_REMOTE_READ; qp_params.incoming_rdma_write_en = attr->qp_access_flags & IB_ACCESS_REMOTE_WRITE; qp_params.incoming_atomic_en = attr->qp_access_flags & IB_ACCESS_REMOTE_ATOMIC; } if (attr_mask & (IB_QP_AV | IB_QP_PATH_MTU)) { if (attr_mask & IB_QP_PATH_MTU) { if (attr->path_mtu < IB_MTU_256 || attr->path_mtu > IB_MTU_4096) { QL_DPRINT12(ha, "Only MTU sizes of 256, 512, 1024," " 2048 and 4096 are supported " " attr->path_mtu = [%d]\n", attr->path_mtu); rc = -EINVAL; goto err; } qp->mtu = min(ib_mtu_enum_to_int(attr->path_mtu), ib_mtu_enum_to_int( - iboe_get_mtu(dev->ha->ifp->if_mtu))); + iboe_get_mtu(if_getmtu(dev->ha->ifp)))); } if (qp->mtu == 0) { qp->mtu = ib_mtu_enum_to_int( - iboe_get_mtu(dev->ha->ifp->if_mtu)); + iboe_get_mtu(if_getmtu(dev->ha->ifp))); QL_DPRINT12(ha, "fixing zetoed MTU to qp->mtu = %d\n", qp->mtu); } SET_FIELD(qp_params.modify_flags, ECORE_ROCE_MODIFY_QP_VALID_ADDRESS_VECTOR, 1); qp_params.traffic_class_tos = attr->ah_attr.grh.traffic_class; qp_params.flow_label = attr->ah_attr.grh.flow_label; qp_params.hop_limit_ttl = attr->ah_attr.grh.hop_limit; qp->sgid_idx = attr->ah_attr.grh.sgid_index; get_gid_info(ibqp, attr, attr_mask, dev, qp, &qp_params); rc = qlnxr_get_dmac(dev, &attr->ah_attr, qp_params.remote_mac_addr); if (rc) return rc; qp_params.use_local_mac = true; memcpy(qp_params.local_mac_addr, dev->ha->primary_mac, ETH_ALEN); QL_DPRINT12(ha, "dgid=0x%x:0x%x:0x%x:0x%x\n", qp_params.dgid.dwords[0], qp_params.dgid.dwords[1], qp_params.dgid.dwords[2], qp_params.dgid.dwords[3]); QL_DPRINT12(ha, "sgid=0x%x:0x%x:0x%x:0x%x\n", qp_params.sgid.dwords[0], qp_params.sgid.dwords[1], qp_params.sgid.dwords[2], qp_params.sgid.dwords[3]); QL_DPRINT12(ha, "remote_mac=[0x%x:0x%x:0x%x:0x%x:0x%x:0x%x]\n", qp_params.remote_mac_addr[0], qp_params.remote_mac_addr[1], qp_params.remote_mac_addr[2], qp_params.remote_mac_addr[3], qp_params.remote_mac_addr[4], qp_params.remote_mac_addr[5]); qp_params.mtu = qp->mtu; } if (qp_params.mtu == 0) { /* stay with current MTU */ if (qp->mtu) { qp_params.mtu = qp->mtu; } else { qp_params.mtu = ib_mtu_enum_to_int( - iboe_get_mtu(dev->ha->ifp->if_mtu)); + iboe_get_mtu(if_getmtu(dev->ha->ifp))); } } if (attr_mask & IB_QP_TIMEOUT) { SET_FIELD(qp_params.modify_flags, \ ECORE_ROCE_MODIFY_QP_VALID_ACK_TIMEOUT, 1); qp_params.ack_timeout = attr->timeout; if (attr->timeout) { u32 temp; /* 12.7.34 LOCAL ACK TIMEOUT * Value representing the transport (ACK) timeout for * use by the remote, expressed as (4.096 μS*2Local ACK * Timeout) */ /* We use 1UL since the temporal value may be overflow * 32 bits */ temp = 4096 * (1UL << attr->timeout) / 1000 / 1000; qp_params.ack_timeout = temp; /* FW requires [msec] */ } else qp_params.ack_timeout = 0; /* infinite */ } if (attr_mask & IB_QP_RETRY_CNT) { SET_FIELD(qp_params.modify_flags,\ ECORE_ROCE_MODIFY_QP_VALID_RETRY_CNT, 1); qp_params.retry_cnt = attr->retry_cnt; } if (attr_mask & IB_QP_RNR_RETRY) { SET_FIELD(qp_params.modify_flags, ECORE_ROCE_MODIFY_QP_VALID_RNR_RETRY_CNT, 1); qp_params.rnr_retry_cnt = attr->rnr_retry; } if (attr_mask & IB_QP_RQ_PSN) { SET_FIELD(qp_params.modify_flags, ECORE_ROCE_MODIFY_QP_VALID_RQ_PSN, 1); qp_params.rq_psn = attr->rq_psn; qp->rq_psn = attr->rq_psn; } if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC) { if (attr->max_rd_atomic > qattr->max_qp_req_rd_atomic_resc) { rc = -EINVAL; QL_DPRINT12(ha, "unsupported max_rd_atomic=%d, supported=%d\n", attr->max_rd_atomic, qattr->max_qp_req_rd_atomic_resc); goto err; } SET_FIELD(qp_params.modify_flags, ECORE_RDMA_MODIFY_QP_VALID_MAX_RD_ATOMIC_REQ, 1); qp_params.max_rd_atomic_req = attr->max_rd_atomic; } if (attr_mask & IB_QP_MIN_RNR_TIMER) { SET_FIELD(qp_params.modify_flags, ECORE_ROCE_MODIFY_QP_VALID_MIN_RNR_NAK_TIMER, 1); qp_params.min_rnr_nak_timer = attr->min_rnr_timer; } if (attr_mask & IB_QP_SQ_PSN) { SET_FIELD(qp_params.modify_flags, ECORE_ROCE_MODIFY_QP_VALID_SQ_PSN, 1); qp_params.sq_psn = attr->sq_psn; qp->sq_psn = attr->sq_psn; } if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) { if (attr->max_dest_rd_atomic > qattr->max_qp_resp_rd_atomic_resc) { QL_DPRINT12(ha, "unsupported max_dest_rd_atomic=%d, " "supported=%d\n", attr->max_dest_rd_atomic, qattr->max_qp_resp_rd_atomic_resc); rc = -EINVAL; goto err; } SET_FIELD(qp_params.modify_flags, ECORE_RDMA_MODIFY_QP_VALID_MAX_RD_ATOMIC_RESP, 1); qp_params.max_rd_atomic_resp = attr->max_dest_rd_atomic; } if (attr_mask & IB_QP_DEST_QPN) { SET_FIELD(qp_params.modify_flags, ECORE_ROCE_MODIFY_QP_VALID_DEST_QP, 1); qp_params.dest_qp = attr->dest_qp_num; qp->dest_qp_num = attr->dest_qp_num; } /* * Update the QP state before the actual ramrod to prevent a race with * fast path. Modifying the QP state to error will cause the device to * flush the CQEs and while polling the flushed CQEs will considered as * a potential issue if the QP isn't in error state. */ if ((attr_mask & IB_QP_STATE) && (qp->qp_type != IB_QPT_GSI) && (!udata) && (qp_params.new_state == ECORE_ROCE_QP_STATE_ERR)) qp->state = ECORE_ROCE_QP_STATE_ERR; if (qp->qp_type != IB_QPT_GSI) rc = ecore_rdma_modify_qp(dev->rdma_ctx, qp->ecore_qp, &qp_params); if (attr_mask & IB_QP_STATE) { if ((qp->qp_type != IB_QPT_GSI) && (!udata)) rc = qlnxr_update_qp_state(dev, qp, qp_params.new_state); qp->state = qp_params.new_state; } err: QL_DPRINT12(ha, "exit\n"); return rc; } static int qlnxr_to_ib_qp_acc_flags(struct ecore_rdma_query_qp_out_params *params) { int ib_qp_acc_flags = 0; if (params->incoming_rdma_write_en) ib_qp_acc_flags |= IB_ACCESS_REMOTE_WRITE; if (params->incoming_rdma_read_en) ib_qp_acc_flags |= IB_ACCESS_REMOTE_READ; if (params->incoming_atomic_en) ib_qp_acc_flags |= IB_ACCESS_REMOTE_ATOMIC; if (true) /* FIXME -> local write ?? */ ib_qp_acc_flags |= IB_ACCESS_LOCAL_WRITE; return ib_qp_acc_flags; } static enum ib_mtu qlnxr_mtu_int_to_enum(u16 mtu) { enum ib_mtu ib_mtu_size; switch (mtu) { case 256: ib_mtu_size = IB_MTU_256; break; case 512: ib_mtu_size = IB_MTU_512; break; case 1024: ib_mtu_size = IB_MTU_1024; break; case 2048: ib_mtu_size = IB_MTU_2048; break; case 4096: ib_mtu_size = IB_MTU_4096; break; default: ib_mtu_size = IB_MTU_1024; break; } return (ib_mtu_size); } int qlnxr_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr, int attr_mask, struct ib_qp_init_attr *qp_init_attr) { int rc = 0; struct ecore_rdma_query_qp_out_params params; struct qlnxr_qp *qp = get_qlnxr_qp(ibqp); struct qlnxr_dev *dev = qp->dev; qlnx_host_t *ha; ha = dev->ha; QL_DPRINT12(ha, "enter\n"); memset(¶ms, 0, sizeof(params)); rc = ecore_rdma_query_qp(dev->rdma_ctx, qp->ecore_qp, ¶ms); if (rc) goto err; memset(qp_attr, 0, sizeof(*qp_attr)); memset(qp_init_attr, 0, sizeof(*qp_init_attr)); qp_attr->qp_state = qlnxr_get_ibqp_state(params.state); qp_attr->cur_qp_state = qlnxr_get_ibqp_state(params.state); /* In some cases in iWARP qelr will ask for the state only */ if (QLNX_IS_IWARP(dev) && (attr_mask == IB_QP_STATE)) { QL_DPRINT11(ha, "only state requested\n"); return 0; } qp_attr->path_mtu = qlnxr_mtu_int_to_enum(params.mtu); qp_attr->path_mig_state = IB_MIG_MIGRATED; qp_attr->rq_psn = params.rq_psn; qp_attr->sq_psn = params.sq_psn; qp_attr->dest_qp_num = params.dest_qp; qp_attr->qp_access_flags = qlnxr_to_ib_qp_acc_flags(¶ms); QL_DPRINT12(ha, "qp_state = 0x%x cur_qp_state = 0x%x " "path_mtu = %d qp_access_flags = 0x%x\n", qp_attr->qp_state, qp_attr->cur_qp_state, qp_attr->path_mtu, qp_attr->qp_access_flags); qp_attr->cap.max_send_wr = qp->sq.max_wr; qp_attr->cap.max_recv_wr = qp->rq.max_wr; qp_attr->cap.max_send_sge = qp->sq.max_sges; qp_attr->cap.max_recv_sge = qp->rq.max_sges; qp_attr->cap.max_inline_data = qp->max_inline_data; qp_init_attr->cap = qp_attr->cap; memcpy(&qp_attr->ah_attr.grh.dgid.raw[0], ¶ms.dgid.bytes[0], sizeof(qp_attr->ah_attr.grh.dgid.raw)); qp_attr->ah_attr.grh.flow_label = params.flow_label; qp_attr->ah_attr.grh.sgid_index = qp->sgid_idx; qp_attr->ah_attr.grh.hop_limit = params.hop_limit_ttl; qp_attr->ah_attr.grh.traffic_class = params.traffic_class_tos; qp_attr->ah_attr.ah_flags = IB_AH_GRH; qp_attr->ah_attr.port_num = 1; /* FIXME -> check this */ qp_attr->ah_attr.sl = 0;/* FIXME -> check this */ qp_attr->timeout = params.timeout; qp_attr->rnr_retry = params.rnr_retry; qp_attr->retry_cnt = params.retry_cnt; qp_attr->min_rnr_timer = params.min_rnr_nak_timer; qp_attr->pkey_index = params.pkey_index; qp_attr->port_num = 1; /* FIXME -> check this */ qp_attr->ah_attr.src_path_bits = 0; qp_attr->ah_attr.static_rate = 0; qp_attr->alt_pkey_index = 0; qp_attr->alt_port_num = 0; qp_attr->alt_timeout = 0; memset(&qp_attr->alt_ah_attr, 0, sizeof(qp_attr->alt_ah_attr)); qp_attr->sq_draining = (params.state == ECORE_ROCE_QP_STATE_SQD) ? 1 : 0; qp_attr->max_dest_rd_atomic = params.max_dest_rd_atomic; qp_attr->max_rd_atomic = params.max_rd_atomic; qp_attr->en_sqd_async_notify = (params.sqd_async)? 1 : 0; QL_DPRINT12(ha, "max_inline_data=%d\n", qp_attr->cap.max_inline_data); err: QL_DPRINT12(ha, "exit\n"); return rc; } static void qlnxr_cleanup_user(struct qlnxr_dev *dev, struct qlnxr_qp *qp) { qlnx_host_t *ha; ha = dev->ha; QL_DPRINT12(ha, "enter\n"); if (qp->usq.umem) ib_umem_release(qp->usq.umem); qp->usq.umem = NULL; if (qp->urq.umem) ib_umem_release(qp->urq.umem); qp->urq.umem = NULL; QL_DPRINT12(ha, "exit\n"); return; } static void qlnxr_cleanup_kernel(struct qlnxr_dev *dev, struct qlnxr_qp *qp) { qlnx_host_t *ha; ha = dev->ha; QL_DPRINT12(ha, "enter\n"); if (qlnxr_qp_has_sq(qp)) { QL_DPRINT12(ha, "freeing SQ\n"); ha->qlnxr_debug = 1; // ecore_chain_free(dev->cdev, &qp->sq.pbl); ha->qlnxr_debug = 0; kfree(qp->wqe_wr_id); } if (qlnxr_qp_has_rq(qp)) { QL_DPRINT12(ha, "freeing RQ\n"); ha->qlnxr_debug = 1; // ecore_chain_free(dev->cdev, &qp->rq.pbl); ha->qlnxr_debug = 0; kfree(qp->rqe_wr_id); } QL_DPRINT12(ha, "exit\n"); return; } static int qlnxr_free_qp_resources(struct qlnxr_dev *dev, struct qlnxr_qp *qp, struct ib_udata *udata) { int rc = 0; qlnx_host_t *ha; struct ecore_rdma_destroy_qp_out_params d_out_params; ha = dev->ha; QL_DPRINT12(ha, "enter\n"); #if 0 if (qp->qp_type != IB_QPT_GSI) { rc = ecore_rdma_destroy_qp(dev->rdma_ctx, qp->ecore_qp, &d_out_params); if (rc) return rc; } if (udata) qlnxr_cleanup_user(dev, qp); else qlnxr_cleanup_kernel(dev, qp); #endif if (udata) qlnxr_cleanup_user(dev, qp); else qlnxr_cleanup_kernel(dev, qp); if (qp->qp_type != IB_QPT_GSI) { rc = ecore_rdma_destroy_qp(dev->rdma_ctx, qp->ecore_qp, &d_out_params); if (rc) return rc; } QL_DPRINT12(ha, "exit\n"); return 0; } int qlnxr_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata) { struct qlnxr_qp *qp = get_qlnxr_qp(ibqp); struct qlnxr_dev *dev = qp->dev; int rc = 0; struct ib_qp_attr attr; int attr_mask = 0; qlnx_host_t *ha; ha = dev->ha; QL_DPRINT12(ha, "enter qp = %p, qp_type=%d\n", qp, qp->qp_type); qp->destroyed = 1; if (QLNX_IS_ROCE(dev) && (qp->state != (ECORE_ROCE_QP_STATE_RESET | ECORE_ROCE_QP_STATE_ERR | ECORE_ROCE_QP_STATE_INIT))) { attr.qp_state = IB_QPS_ERR; attr_mask |= IB_QP_STATE; /* change the QP state to ERROR */ qlnxr_modify_qp(ibqp, &attr, attr_mask, NULL); } if (qp->qp_type == IB_QPT_GSI) qlnxr_destroy_gsi_qp(dev); qp->sig = ~qp->sig; qlnxr_free_qp_resources(dev, qp, udata); if (atomic_dec_and_test(&qp->refcnt)) { /* TODO: only for iWARP? */ qlnxr_idr_remove(dev, qp->qp_id); } QL_DPRINT12(ha, "exit\n"); return rc; } static inline int qlnxr_wq_is_full(struct qlnxr_qp_hwq_info *wq) { return (((wq->prod + 1) % wq->max_wr) == wq->cons); } static int sge_data_len(struct ib_sge *sg_list, int num_sge) { int i, len = 0; for (i = 0; i < num_sge; i++) len += sg_list[i].length; return len; } static void swap_wqe_data64(u64 *p) { int i; for (i = 0; i < QLNXR_SQE_ELEMENT_SIZE / sizeof(u64); i++, p++) *p = cpu_to_be64(cpu_to_le64(*p)); } static u32 qlnxr_prepare_sq_inline_data(struct qlnxr_dev *dev, struct qlnxr_qp *qp, u8 *wqe_size, const struct ib_send_wr *wr, const struct ib_send_wr **bad_wr, u8 *bits, u8 bit) { int i, seg_siz; char *seg_prt, *wqe; u32 data_size = sge_data_len(wr->sg_list, wr->num_sge); qlnx_host_t *ha; ha = dev->ha; QL_DPRINT12(ha, "enter[%d]\n", data_size); if (data_size > ROCE_REQ_MAX_INLINE_DATA_SIZE) { QL_DPRINT12(ha, "Too much inline data in WR:[%d, %d]\n", data_size, ROCE_REQ_MAX_INLINE_DATA_SIZE); *bad_wr = wr; return 0; } if (!data_size) return data_size; /* set the bit */ *bits |= bit; seg_prt = wqe = NULL; seg_siz = 0; /* copy data inline */ for (i = 0; i < wr->num_sge; i++) { u32 len = wr->sg_list[i].length; void *src = (void *)(uintptr_t)wr->sg_list[i].addr; while (len > 0) { u32 cur; /* new segment required */ if (!seg_siz) { wqe = (char *)ecore_chain_produce(&qp->sq.pbl); seg_prt = wqe; seg_siz = sizeof(struct rdma_sq_common_wqe); (*wqe_size)++; } /* calculate currently allowed length */ cur = MIN(len, seg_siz); memcpy(seg_prt, src, cur); /* update segment variables */ seg_prt += cur; seg_siz -= cur; /* update sge variables */ src += cur; len -= cur; /* swap fully-completed segments */ if (!seg_siz) swap_wqe_data64((u64 *)wqe); } } /* swap last not completed segment */ if (seg_siz) swap_wqe_data64((u64 *)wqe); QL_DPRINT12(ha, "exit\n"); return data_size; } static u32 qlnxr_prepare_sq_sges(struct qlnxr_dev *dev, struct qlnxr_qp *qp, u8 *wqe_size, const struct ib_send_wr *wr) { int i; u32 data_size = 0; qlnx_host_t *ha; ha = dev->ha; QL_DPRINT12(ha, "enter wr->num_sge = %d \n", wr->num_sge); for (i = 0; i < wr->num_sge; i++) { struct rdma_sq_sge *sge = ecore_chain_produce(&qp->sq.pbl); TYPEPTR_ADDR_SET(sge, addr, wr->sg_list[i].addr); sge->l_key = cpu_to_le32(wr->sg_list[i].lkey); sge->length = cpu_to_le32(wr->sg_list[i].length); data_size += wr->sg_list[i].length; } if (wqe_size) *wqe_size += wr->num_sge; QL_DPRINT12(ha, "exit data_size = %d\n", data_size); return data_size; } static u32 qlnxr_prepare_sq_rdma_data(struct qlnxr_dev *dev, struct qlnxr_qp *qp, struct rdma_sq_rdma_wqe_1st *rwqe, struct rdma_sq_rdma_wqe_2nd *rwqe2, const struct ib_send_wr *wr, const struct ib_send_wr **bad_wr) { qlnx_host_t *ha; u32 ret = 0; ha = dev->ha; QL_DPRINT12(ha, "enter\n"); rwqe2->r_key = cpu_to_le32(rdma_wr(wr)->rkey); TYPEPTR_ADDR_SET(rwqe2, remote_va, rdma_wr(wr)->remote_addr); if (wr->send_flags & IB_SEND_INLINE) { u8 flags = 0; SET_FIELD2(flags, RDMA_SQ_RDMA_WQE_1ST_INLINE_FLG, 1); return qlnxr_prepare_sq_inline_data(dev, qp, &rwqe->wqe_size, wr, bad_wr, &rwqe->flags, flags); } ret = qlnxr_prepare_sq_sges(dev, qp, &rwqe->wqe_size, wr); QL_DPRINT12(ha, "exit ret = 0x%x\n", ret); return (ret); } static u32 qlnxr_prepare_sq_send_data(struct qlnxr_dev *dev, struct qlnxr_qp *qp, struct rdma_sq_send_wqe *swqe, struct rdma_sq_send_wqe *swqe2, const struct ib_send_wr *wr, const struct ib_send_wr **bad_wr) { qlnx_host_t *ha; u32 ret = 0; ha = dev->ha; QL_DPRINT12(ha, "enter\n"); memset(swqe2, 0, sizeof(*swqe2)); if (wr->send_flags & IB_SEND_INLINE) { u8 flags = 0; SET_FIELD2(flags, RDMA_SQ_SEND_WQE_INLINE_FLG, 1); return qlnxr_prepare_sq_inline_data(dev, qp, &swqe->wqe_size, wr, bad_wr, &swqe->flags, flags); } ret = qlnxr_prepare_sq_sges(dev, qp, &swqe->wqe_size, wr); QL_DPRINT12(ha, "exit ret = 0x%x\n", ret); return (ret); } static void qlnx_handle_completed_mrs(struct qlnxr_dev *dev, struct mr_info *info) { qlnx_host_t *ha; ha = dev->ha; int work = info->completed - info->completed_handled - 1; QL_DPRINT12(ha, "enter [%d]\n", work); while (work-- > 0 && !list_empty(&info->inuse_pbl_list)) { struct qlnxr_pbl *pbl; /* Free all the page list that are possible to be freed * (all the ones that were invalidated), under the assumption * that if an FMR was completed successfully that means that * if there was an invalidate operation before it also ended */ pbl = list_first_entry(&info->inuse_pbl_list, struct qlnxr_pbl, list_entry); list_del(&pbl->list_entry); list_add_tail(&pbl->list_entry, &info->free_pbl_list); info->completed_handled++; } QL_DPRINT12(ha, "exit\n"); return; } static int qlnxr_prepare_reg(struct qlnxr_qp *qp, struct rdma_sq_fmr_wqe_1st *fwqe1, const struct ib_reg_wr *wr) { struct qlnxr_mr *mr = get_qlnxr_mr(wr->mr); struct rdma_sq_fmr_wqe_2nd *fwqe2; fwqe2 = (struct rdma_sq_fmr_wqe_2nd *)ecore_chain_produce(&qp->sq.pbl); fwqe1->addr.hi = upper_32_bits(mr->ibmr.iova); fwqe1->addr.lo = lower_32_bits(mr->ibmr.iova); fwqe1->l_key = wr->key; fwqe2->access_ctrl = 0; SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_REMOTE_READ, !!(wr->access & IB_ACCESS_REMOTE_READ)); SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_REMOTE_WRITE, !!(wr->access & IB_ACCESS_REMOTE_WRITE)); SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_ENABLE_ATOMIC, !!(wr->access & IB_ACCESS_REMOTE_ATOMIC)); SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_LOCAL_READ, 1); SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_LOCAL_WRITE, !!(wr->access & IB_ACCESS_LOCAL_WRITE)); fwqe2->fmr_ctrl = 0; SET_FIELD2(fwqe2->fmr_ctrl, RDMA_SQ_FMR_WQE_2ND_PAGE_SIZE_LOG, ilog2(mr->ibmr.page_size) - 12); fwqe2->length_hi = 0; /* TODO - figure out why length is only 32bit.. */ fwqe2->length_lo = mr->ibmr.length; fwqe2->pbl_addr.hi = upper_32_bits(mr->info.pbl_table->pa); fwqe2->pbl_addr.lo = lower_32_bits(mr->info.pbl_table->pa); qp->wqe_wr_id[qp->sq.prod].mr = mr; return 0; } static enum ib_wc_opcode qlnxr_ib_to_wc_opcode(enum ib_wr_opcode opcode) { switch (opcode) { case IB_WR_RDMA_WRITE: case IB_WR_RDMA_WRITE_WITH_IMM: return IB_WC_RDMA_WRITE; case IB_WR_SEND_WITH_IMM: case IB_WR_SEND: case IB_WR_SEND_WITH_INV: return IB_WC_SEND; case IB_WR_RDMA_READ: return IB_WC_RDMA_READ; case IB_WR_ATOMIC_CMP_AND_SWP: return IB_WC_COMP_SWAP; case IB_WR_ATOMIC_FETCH_AND_ADD: return IB_WC_FETCH_ADD; case IB_WR_REG_MR: return IB_WC_REG_MR; case IB_WR_LOCAL_INV: return IB_WC_LOCAL_INV; default: return IB_WC_SEND; } } static inline bool qlnxr_can_post_send(struct qlnxr_qp *qp, const struct ib_send_wr *wr) { int wq_is_full, err_wr, pbl_is_full; struct qlnxr_dev *dev = qp->dev; qlnx_host_t *ha; ha = dev->ha; QL_DPRINT12(ha, "enter[qp, wr] = [%p,%p]\n", qp, wr); /* prevent SQ overflow and/or processing of a bad WR */ err_wr = wr->num_sge > qp->sq.max_sges; wq_is_full = qlnxr_wq_is_full(&qp->sq); pbl_is_full = ecore_chain_get_elem_left_u32(&qp->sq.pbl) < QLNXR_MAX_SQE_ELEMENTS_PER_SQE; if (wq_is_full || err_wr || pbl_is_full) { if (wq_is_full && !(qp->err_bitmap & QLNXR_QP_ERR_SQ_FULL)) { qp->err_bitmap |= QLNXR_QP_ERR_SQ_FULL; QL_DPRINT12(ha, "error: WQ is full. Post send on QP failed" " (this error appears only once) " "[qp, wr, qp->err_bitmap]=[%p, %p, 0x%x]\n", qp, wr, qp->err_bitmap); } if (err_wr && !(qp->err_bitmap & QLNXR_QP_ERR_BAD_SR)) { qp->err_bitmap |= QLNXR_QP_ERR_BAD_SR; QL_DPRINT12(ha, "error: WQ is bad. Post send on QP failed" " (this error appears only once) " "[qp, wr, qp->err_bitmap]=[%p, %p, 0x%x]\n", qp, wr, qp->err_bitmap); } if (pbl_is_full && !(qp->err_bitmap & QLNXR_QP_ERR_SQ_PBL_FULL)) { qp->err_bitmap |= QLNXR_QP_ERR_SQ_PBL_FULL; QL_DPRINT12(ha, "error: WQ PBL is full. Post send on QP failed" " (this error appears only once) " "[qp, wr, qp->err_bitmap]=[%p, %p, 0x%x]\n", qp, wr, qp->err_bitmap); } return false; } QL_DPRINT12(ha, "exit[qp, wr] = [%p,%p]\n", qp, wr); return true; } int qlnxr_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr, const struct ib_send_wr **bad_wr) { struct qlnxr_dev *dev = get_qlnxr_dev(ibqp->device); struct qlnxr_qp *qp = get_qlnxr_qp(ibqp); unsigned long flags; int status = 0, rc = 0; bool comp; qlnx_host_t *ha; uint32_t reg_addr; *bad_wr = NULL; ha = dev->ha; QL_DPRINT12(ha, "exit[ibqp, wr, bad_wr] = [%p, %p, %p]\n", ibqp, wr, bad_wr); - if (!(ha->ifp->if_drv_flags & IFF_DRV_RUNNING)) + if (!(if_getdrvflags(ha->ifp) & IFF_DRV_RUNNING)) return -EINVAL; if (qp->qp_type == IB_QPT_GSI) return qlnxr_gsi_post_send(ibqp, wr, bad_wr); spin_lock_irqsave(&qp->q_lock, flags); if (QLNX_IS_ROCE(dev) && (qp->state != ECORE_ROCE_QP_STATE_RTS) && (qp->state != ECORE_ROCE_QP_STATE_ERR) && (qp->state != ECORE_ROCE_QP_STATE_SQD)) { spin_unlock_irqrestore(&qp->q_lock, flags); *bad_wr = wr; QL_DPRINT11(ha, "QP in wrong state! QP icid=0x%x state %d\n", qp->icid, qp->state); return -EINVAL; } if (!wr) { QL_DPRINT11(ha, "Got an empty post send???\n"); } while (wr) { struct rdma_sq_common_wqe *wqe; struct rdma_sq_send_wqe *swqe; struct rdma_sq_send_wqe *swqe2; struct rdma_sq_rdma_wqe_1st *rwqe; struct rdma_sq_rdma_wqe_2nd *rwqe2; struct rdma_sq_local_inv_wqe *iwqe; struct rdma_sq_atomic_wqe *awqe1; struct rdma_sq_atomic_wqe *awqe2; struct rdma_sq_atomic_wqe *awqe3; struct rdma_sq_fmr_wqe_1st *fwqe1; if (!qlnxr_can_post_send(qp, wr)) { status = -ENOMEM; *bad_wr = wr; break; } wqe = ecore_chain_produce(&qp->sq.pbl); qp->wqe_wr_id[qp->sq.prod].signaled = !!(wr->send_flags & IB_SEND_SIGNALED) || qp->signaled; /* common fields */ wqe->flags = 0; wqe->flags |= (RDMA_SQ_SEND_WQE_COMP_FLG_MASK << RDMA_SQ_SEND_WQE_COMP_FLG_SHIFT); SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_SE_FLG, \ !!(wr->send_flags & IB_SEND_SOLICITED)); comp = (!!(wr->send_flags & IB_SEND_SIGNALED)) || (qp->signaled); SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_COMP_FLG, comp); SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_RD_FENCE_FLG, \ !!(wr->send_flags & IB_SEND_FENCE)); wqe->prev_wqe_size = qp->prev_wqe_size; qp->wqe_wr_id[qp->sq.prod].opcode = qlnxr_ib_to_wc_opcode(wr->opcode); switch (wr->opcode) { case IB_WR_SEND_WITH_IMM: wqe->req_type = RDMA_SQ_REQ_TYPE_SEND_WITH_IMM; swqe = (struct rdma_sq_send_wqe *)wqe; swqe->wqe_size = 2; swqe2 = (struct rdma_sq_send_wqe *) ecore_chain_produce(&qp->sq.pbl); swqe->inv_key_or_imm_data = cpu_to_le32(wr->ex.imm_data); swqe->length = cpu_to_le32( qlnxr_prepare_sq_send_data(dev, qp, swqe, swqe2, wr, bad_wr)); qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size; qp->prev_wqe_size = swqe->wqe_size; qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length; QL_DPRINT12(ha, "SEND w/ IMM length = %d imm data=%x\n", swqe->length, wr->ex.imm_data); break; case IB_WR_SEND: wqe->req_type = RDMA_SQ_REQ_TYPE_SEND; swqe = (struct rdma_sq_send_wqe *)wqe; swqe->wqe_size = 2; swqe2 = (struct rdma_sq_send_wqe *) ecore_chain_produce(&qp->sq.pbl); swqe->length = cpu_to_le32( qlnxr_prepare_sq_send_data(dev, qp, swqe, swqe2, wr, bad_wr)); qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size; qp->prev_wqe_size = swqe->wqe_size; qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length; QL_DPRINT12(ha, "SEND w/o IMM length = %d\n", swqe->length); break; case IB_WR_SEND_WITH_INV: wqe->req_type = RDMA_SQ_REQ_TYPE_SEND_WITH_INVALIDATE; swqe = (struct rdma_sq_send_wqe *)wqe; swqe2 = (struct rdma_sq_send_wqe *) ecore_chain_produce(&qp->sq.pbl); swqe->wqe_size = 2; swqe->inv_key_or_imm_data = cpu_to_le32(wr->ex.invalidate_rkey); swqe->length = cpu_to_le32(qlnxr_prepare_sq_send_data(dev, qp, swqe, swqe2, wr, bad_wr)); qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size; qp->prev_wqe_size = swqe->wqe_size; qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length; QL_DPRINT12(ha, "SEND w INVALIDATE length = %d\n", swqe->length); break; case IB_WR_RDMA_WRITE_WITH_IMM: wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_WR_WITH_IMM; rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe; rwqe->wqe_size = 2; rwqe->imm_data = htonl(cpu_to_le32(wr->ex.imm_data)); rwqe2 = (struct rdma_sq_rdma_wqe_2nd *) ecore_chain_produce(&qp->sq.pbl); rwqe->length = cpu_to_le32(qlnxr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2, wr, bad_wr)); qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size; qp->prev_wqe_size = rwqe->wqe_size; qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length; QL_DPRINT12(ha, "RDMA WRITE w/ IMM length = %d imm data=%x\n", rwqe->length, rwqe->imm_data); break; case IB_WR_RDMA_WRITE: wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_WR; rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe; rwqe->wqe_size = 2; rwqe2 = (struct rdma_sq_rdma_wqe_2nd *) ecore_chain_produce(&qp->sq.pbl); rwqe->length = cpu_to_le32(qlnxr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2, wr, bad_wr)); qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size; qp->prev_wqe_size = rwqe->wqe_size; qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length; QL_DPRINT12(ha, "RDMA WRITE w/o IMM length = %d\n", rwqe->length); break; case IB_WR_RDMA_READ_WITH_INV: QL_DPRINT12(ha, "RDMA READ WITH INVALIDATE not supported\n"); *bad_wr = wr; rc = -EINVAL; break; case IB_WR_RDMA_READ: wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_RD; rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe; rwqe->wqe_size = 2; rwqe2 = (struct rdma_sq_rdma_wqe_2nd *) ecore_chain_produce(&qp->sq.pbl); rwqe->length = cpu_to_le32(qlnxr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2, wr, bad_wr)); qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size; qp->prev_wqe_size = rwqe->wqe_size; qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length; QL_DPRINT12(ha, "RDMA READ length = %d\n", rwqe->length); break; case IB_WR_ATOMIC_CMP_AND_SWP: case IB_WR_ATOMIC_FETCH_AND_ADD: QL_DPRINT12(ha, "ATOMIC operation = %s\n", ((wr->opcode == IB_WR_ATOMIC_CMP_AND_SWP) ? "IB_WR_ATOMIC_CMP_AND_SWP" : "IB_WR_ATOMIC_FETCH_AND_ADD")); awqe1 = (struct rdma_sq_atomic_wqe *)wqe; awqe1->prev_wqe_size = 4; awqe2 = (struct rdma_sq_atomic_wqe *) ecore_chain_produce(&qp->sq.pbl); TYPEPTR_ADDR_SET(awqe2, remote_va, \ atomic_wr(wr)->remote_addr); awqe2->r_key = cpu_to_le32(atomic_wr(wr)->rkey); awqe3 = (struct rdma_sq_atomic_wqe *) ecore_chain_produce(&qp->sq.pbl); if (wr->opcode == IB_WR_ATOMIC_FETCH_AND_ADD) { wqe->req_type = RDMA_SQ_REQ_TYPE_ATOMIC_ADD; TYPEPTR_ADDR_SET(awqe3, swap_data, atomic_wr(wr)->compare_add); } else { wqe->req_type = RDMA_SQ_REQ_TYPE_ATOMIC_CMP_AND_SWAP; TYPEPTR_ADDR_SET(awqe3, swap_data, atomic_wr(wr)->swap); TYPEPTR_ADDR_SET(awqe3, cmp_data, atomic_wr(wr)->compare_add); } qlnxr_prepare_sq_sges(dev, qp, NULL, wr); qp->wqe_wr_id[qp->sq.prod].wqe_size = awqe1->prev_wqe_size; qp->prev_wqe_size = awqe1->prev_wqe_size; break; case IB_WR_LOCAL_INV: QL_DPRINT12(ha, "INVALIDATE length (IB_WR_LOCAL_INV)\n"); iwqe = (struct rdma_sq_local_inv_wqe *)wqe; iwqe->prev_wqe_size = 1; iwqe->req_type = RDMA_SQ_REQ_TYPE_LOCAL_INVALIDATE; iwqe->inv_l_key = wr->ex.invalidate_rkey; qp->wqe_wr_id[qp->sq.prod].wqe_size = iwqe->prev_wqe_size; qp->prev_wqe_size = iwqe->prev_wqe_size; break; case IB_WR_REG_MR: QL_DPRINT12(ha, "IB_WR_REG_MR\n"); wqe->req_type = RDMA_SQ_REQ_TYPE_FAST_MR; fwqe1 = (struct rdma_sq_fmr_wqe_1st *)wqe; fwqe1->wqe_size = 2; rc = qlnxr_prepare_reg(qp, fwqe1, reg_wr(wr)); if (rc) { QL_DPRINT11(ha, "IB_WR_REG_MR failed rc=%d\n", rc); *bad_wr = wr; break; } qp->wqe_wr_id[qp->sq.prod].wqe_size = fwqe1->wqe_size; qp->prev_wqe_size = fwqe1->wqe_size; break; default: QL_DPRINT12(ha, "Invalid Opcode 0x%x!\n", wr->opcode); rc = -EINVAL; *bad_wr = wr; break; } if (*bad_wr) { /* * restore prod to its position before this WR was processed */ ecore_chain_set_prod(&qp->sq.pbl, le16_to_cpu(qp->sq.db_data.data.value), wqe); /* restore prev_wqe_size */ qp->prev_wqe_size = wqe->prev_wqe_size; status = rc; QL_DPRINT12(ha, "failed *bad_wr = %p\n", *bad_wr); break; /* out of the loop */ } qp->wqe_wr_id[qp->sq.prod].wr_id = wr->wr_id; qlnxr_inc_sw_prod(&qp->sq); qp->sq.db_data.data.value++; wr = wr->next; } /* Trigger doorbell * If there was a failure in the first WR then it will be triggered in * vane. However this is not harmful (as long as the producer value is * unchanged). For performance reasons we avoid checking for this * redundant doorbell. */ wmb(); //writel(qp->sq.db_data.raw, qp->sq.db); reg_addr = (uint32_t)((uint8_t *)qp->sq.db - (uint8_t *)ha->cdev.doorbells); bus_write_4(ha->pci_dbells, reg_addr, qp->sq.db_data.raw); bus_barrier(ha->pci_dbells, 0, 0, BUS_SPACE_BARRIER_READ); mmiowb(); spin_unlock_irqrestore(&qp->q_lock, flags); QL_DPRINT12(ha, "exit[ibqp, wr, bad_wr] = [%p, %p, %p]\n", ibqp, wr, bad_wr); return status; } static u32 qlnxr_srq_elem_left(struct qlnxr_srq_hwq_info *hw_srq) { u32 used; /* Calculate number of elements used based on producer * count and consumer count and subtract it from max * work request supported so that we get elements left. */ used = hw_srq->wr_prod_cnt - hw_srq->wr_cons_cnt; return hw_srq->max_wr - used; } int qlnxr_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *wr, const struct ib_recv_wr **bad_wr) { struct qlnxr_qp *qp = get_qlnxr_qp(ibqp); struct qlnxr_dev *dev = qp->dev; unsigned long flags; int status = 0; qlnx_host_t *ha; uint32_t reg_addr; ha = dev->ha; - if (!(ha->ifp->if_drv_flags & IFF_DRV_RUNNING)) + if (!(if_getdrvflags(ha->ifp) & IFF_DRV_RUNNING)) return -EINVAL; QL_DPRINT12(ha, "enter\n"); if (qp->qp_type == IB_QPT_GSI) { QL_DPRINT12(ha, "(qp->qp_type = IB_QPT_GSI)\n"); return qlnxr_gsi_post_recv(ibqp, wr, bad_wr); } if (qp->srq) { QL_DPRINT11(ha, "qp->srq [%p]" " QP is associated with SRQ, cannot post RQ buffers\n", qp->srq); return -EINVAL; } spin_lock_irqsave(&qp->q_lock, flags); if (qp->state == ECORE_ROCE_QP_STATE_RESET) { spin_unlock_irqrestore(&qp->q_lock, flags); *bad_wr = wr; QL_DPRINT11(ha, "qp->qp_type = ECORE_ROCE_QP_STATE_RESET\n"); return -EINVAL; } while (wr) { int i; if ((ecore_chain_get_elem_left_u32(&qp->rq.pbl) < QLNXR_MAX_RQE_ELEMENTS_PER_RQE) || (wr->num_sge > qp->rq.max_sges)) { status = -ENOMEM; *bad_wr = wr; break; } for (i = 0; i < wr->num_sge; i++) { u32 flags = 0; struct rdma_rq_sge *rqe = ecore_chain_produce(&qp->rq.pbl); /* first one must include the number of SGE in the list */ if (!i) SET_FIELD(flags, RDMA_RQ_SGE_NUM_SGES, wr->num_sge); SET_FIELD(flags, RDMA_RQ_SGE_L_KEY, wr->sg_list[i].lkey); RQ_SGE_SET(rqe, wr->sg_list[i].addr, \ wr->sg_list[i].length, flags); } /* Special case of no sges. FW requires between 1-4 sges... * in this case we need to post 1 sge with length zero. this is * because rdma write with immediate consumes an RQ. */ if (!wr->num_sge) { u32 flags = 0; struct rdma_rq_sge *rqe = ecore_chain_produce(&qp->rq.pbl); /* first one must include the number of SGE in the list */ SET_FIELD(flags, RDMA_RQ_SGE_L_KEY, 0); SET_FIELD(flags, RDMA_RQ_SGE_NUM_SGES, 1); //RQ_SGE_SET(rqe, 0, 0, flags); rqe->addr.hi = 0; rqe->addr.lo = 0; rqe->length = 0; rqe->flags = cpu_to_le32(flags); i = 1; } qp->rqe_wr_id[qp->rq.prod].wr_id = wr->wr_id; qp->rqe_wr_id[qp->rq.prod].wqe_size = i; qlnxr_inc_sw_prod(&qp->rq); wmb(); qp->rq.db_data.data.value++; // writel(qp->rq.db_data.raw, qp->rq.db); mmiowb(); // if (QLNX_IS_IWARP(dev)) { // writel(qp->rq.iwarp_db2_data.raw, qp->rq.iwarp_db2); // mmiowb(); /* for second doorbell */ // } reg_addr = (uint32_t)((uint8_t *)qp->rq.db - (uint8_t *)ha->cdev.doorbells); bus_write_4(ha->pci_dbells, reg_addr, qp->rq.db_data.raw); bus_barrier(ha->pci_dbells, 0, 0, BUS_SPACE_BARRIER_READ); if (QLNX_IS_IWARP(dev)) { reg_addr = (uint32_t)((uint8_t *)qp->rq.iwarp_db2 - (uint8_t *)ha->cdev.doorbells); bus_write_4(ha->pci_dbells, reg_addr, \ qp->rq.iwarp_db2_data.raw); bus_barrier(ha->pci_dbells, 0, 0, \ BUS_SPACE_BARRIER_READ); } wr = wr->next; } spin_unlock_irqrestore(&qp->q_lock, flags); QL_DPRINT12(ha, "exit status = 0x%x\n", status); return status; } /* In fmr we need to increase the number of fmr completed counter for the fmr * algorithm determining whether we can free a pbl or not. * we need to perform this whether the work request was signaled or not. for * this purpose we call this function from the condition that checks if a wr * should be skipped, to make sure we don't miss it ( possibly this fmr * operation was not signalted) */ static inline void qlnxr_chk_if_fmr(struct qlnxr_qp *qp) { if (qp->wqe_wr_id[qp->sq.cons].opcode == IB_WC_REG_MR) qp->wqe_wr_id[qp->sq.cons].mr->info.completed++; } static int process_req(struct qlnxr_dev *dev, struct qlnxr_qp *qp, struct qlnxr_cq *cq, int num_entries, struct ib_wc *wc, u16 hw_cons, enum ib_wc_status status, int force) { u16 cnt = 0; qlnx_host_t *ha = dev->ha; QL_DPRINT12(ha, "enter\n"); while (num_entries && qp->sq.wqe_cons != hw_cons) { if (!qp->wqe_wr_id[qp->sq.cons].signaled && !force) { qlnxr_chk_if_fmr(qp); /* skip WC */ goto next_cqe; } /* fill WC */ wc->status = status; wc->vendor_err = 0; wc->wc_flags = 0; wc->src_qp = qp->id; wc->qp = &qp->ibqp; // common section wc->wr_id = qp->wqe_wr_id[qp->sq.cons].wr_id; wc->opcode = qp->wqe_wr_id[qp->sq.cons].opcode; switch (wc->opcode) { case IB_WC_RDMA_WRITE: wc->byte_len = qp->wqe_wr_id[qp->sq.cons].bytes_len; QL_DPRINT12(ha, "opcode = IB_WC_RDMA_WRITE bytes = %d\n", qp->wqe_wr_id[qp->sq.cons].bytes_len); break; case IB_WC_COMP_SWAP: case IB_WC_FETCH_ADD: wc->byte_len = 8; break; case IB_WC_REG_MR: qp->wqe_wr_id[qp->sq.cons].mr->info.completed++; break; case IB_WC_RDMA_READ: case IB_WC_SEND: QL_DPRINT12(ha, "opcode = 0x%x \n", wc->opcode); break; default: ;//DP_ERR("TBD ERROR"); } num_entries--; wc++; cnt++; next_cqe: while (qp->wqe_wr_id[qp->sq.cons].wqe_size--) ecore_chain_consume(&qp->sq.pbl); qlnxr_inc_sw_cons(&qp->sq); } QL_DPRINT12(ha, "exit cnt = 0x%x\n", cnt); return cnt; } static int qlnxr_poll_cq_req(struct qlnxr_dev *dev, struct qlnxr_qp *qp, struct qlnxr_cq *cq, int num_entries, struct ib_wc *wc, struct rdma_cqe_requester *req) { int cnt = 0; qlnx_host_t *ha = dev->ha; QL_DPRINT12(ha, "enter req->status = 0x%x\n", req->status); switch (req->status) { case RDMA_CQE_REQ_STS_OK: cnt = process_req(dev, qp, cq, num_entries, wc, req->sq_cons, IB_WC_SUCCESS, 0); break; case RDMA_CQE_REQ_STS_WORK_REQUEST_FLUSHED_ERR: if (qp->state != ECORE_ROCE_QP_STATE_ERR) cnt = process_req(dev, qp, cq, num_entries, wc, req->sq_cons, IB_WC_WR_FLUSH_ERR, 1); break; default: /* other errors case */ /* process all WQE before the cosumer */ qp->state = ECORE_ROCE_QP_STATE_ERR; cnt = process_req(dev, qp, cq, num_entries, wc, req->sq_cons - 1, IB_WC_SUCCESS, 0); wc += cnt; /* if we have extra WC fill it with actual error info */ if (cnt < num_entries) { enum ib_wc_status wc_status; switch (req->status) { case RDMA_CQE_REQ_STS_BAD_RESPONSE_ERR: wc_status = IB_WC_BAD_RESP_ERR; break; case RDMA_CQE_REQ_STS_LOCAL_LENGTH_ERR: wc_status = IB_WC_LOC_LEN_ERR; break; case RDMA_CQE_REQ_STS_LOCAL_QP_OPERATION_ERR: wc_status = IB_WC_LOC_QP_OP_ERR; break; case RDMA_CQE_REQ_STS_LOCAL_PROTECTION_ERR: wc_status = IB_WC_LOC_PROT_ERR; break; case RDMA_CQE_REQ_STS_MEMORY_MGT_OPERATION_ERR: wc_status = IB_WC_MW_BIND_ERR; break; case RDMA_CQE_REQ_STS_REMOTE_INVALID_REQUEST_ERR: wc_status = IB_WC_REM_INV_REQ_ERR; break; case RDMA_CQE_REQ_STS_REMOTE_ACCESS_ERR: wc_status = IB_WC_REM_ACCESS_ERR; break; case RDMA_CQE_REQ_STS_REMOTE_OPERATION_ERR: wc_status = IB_WC_REM_OP_ERR; break; case RDMA_CQE_REQ_STS_RNR_NAK_RETRY_CNT_ERR: wc_status = IB_WC_RNR_RETRY_EXC_ERR; break; case RDMA_CQE_REQ_STS_TRANSPORT_RETRY_CNT_ERR: wc_status = IB_WC_RETRY_EXC_ERR; break; default: wc_status = IB_WC_GENERAL_ERR; } cnt += process_req(dev, qp, cq, 1, wc, req->sq_cons, wc_status, 1 /* force use of WC */); } } QL_DPRINT12(ha, "exit cnt = %d\n", cnt); return cnt; } static void __process_resp_one(struct qlnxr_dev *dev, struct qlnxr_qp *qp, struct qlnxr_cq *cq, struct ib_wc *wc, struct rdma_cqe_responder *resp, u64 wr_id) { enum ib_wc_status wc_status = IB_WC_SUCCESS; qlnx_host_t *ha = dev->ha; QL_DPRINT12(ha, "enter qp = %p resp->status = 0x%x\n", qp, resp->status); wc->opcode = IB_WC_RECV; wc->wc_flags = 0; switch (resp->status) { case RDMA_CQE_RESP_STS_LOCAL_ACCESS_ERR: wc_status = IB_WC_LOC_ACCESS_ERR; break; case RDMA_CQE_RESP_STS_LOCAL_LENGTH_ERR: wc_status = IB_WC_LOC_LEN_ERR; break; case RDMA_CQE_RESP_STS_LOCAL_QP_OPERATION_ERR: wc_status = IB_WC_LOC_QP_OP_ERR; break; case RDMA_CQE_RESP_STS_LOCAL_PROTECTION_ERR: wc_status = IB_WC_LOC_PROT_ERR; break; case RDMA_CQE_RESP_STS_MEMORY_MGT_OPERATION_ERR: wc_status = IB_WC_MW_BIND_ERR; break; case RDMA_CQE_RESP_STS_REMOTE_INVALID_REQUEST_ERR: wc_status = IB_WC_REM_INV_RD_REQ_ERR; break; case RDMA_CQE_RESP_STS_OK: if (resp->flags & QLNXR_RESP_IMM) { wc->ex.imm_data = le32_to_cpu(resp->imm_data_or_inv_r_Key); wc->wc_flags |= IB_WC_WITH_IMM; if (resp->flags & QLNXR_RESP_RDMA) wc->opcode = IB_WC_RECV_RDMA_WITH_IMM; if (resp->flags & QLNXR_RESP_INV) { QL_DPRINT11(ha, "Invalid flags QLNXR_RESP_INV [0x%x]" "qp = %p qp->id = 0x%x cq = %p" " cq->icid = 0x%x\n", resp->flags, qp, qp->id, cq, cq->icid ); } } else if (resp->flags & QLNXR_RESP_INV) { wc->ex.imm_data = le32_to_cpu(resp->imm_data_or_inv_r_Key); wc->wc_flags |= IB_WC_WITH_INVALIDATE; if (resp->flags & QLNXR_RESP_RDMA) { QL_DPRINT11(ha, "Invalid flags QLNXR_RESP_RDMA [0x%x]" "qp = %p qp->id = 0x%x cq = %p" " cq->icid = 0x%x\n", resp->flags, qp, qp->id, cq, cq->icid ); } } else if (resp->flags & QLNXR_RESP_RDMA) { QL_DPRINT11(ha, "Invalid flags QLNXR_RESP_RDMA [0x%x]" "qp = %p qp->id = 0x%x cq = %p cq->icid = 0x%x\n", resp->flags, qp, qp->id, cq, cq->icid ); } break; default: wc_status = IB_WC_GENERAL_ERR; } /* fill WC */ wc->status = wc_status; wc->vendor_err = 0; wc->src_qp = qp->id; wc->qp = &qp->ibqp; wc->wr_id = wr_id; QL_DPRINT12(ha, "exit status = 0x%x\n", wc_status); return; } static int process_resp_one_srq(struct qlnxr_dev *dev, struct qlnxr_qp *qp, struct qlnxr_cq *cq, struct ib_wc *wc, struct rdma_cqe_responder *resp) { struct qlnxr_srq *srq = qp->srq; u64 wr_id; qlnx_host_t *ha = dev->ha; QL_DPRINT12(ha, "enter\n"); wr_id = HILO_U64(resp->srq_wr_id.hi, resp->srq_wr_id.lo); if (resp->status == RDMA_CQE_RESP_STS_WORK_REQUEST_FLUSHED_ERR) { wc->status = IB_WC_WR_FLUSH_ERR; wc->vendor_err = 0; wc->wr_id = wr_id; wc->byte_len = 0; wc->src_qp = qp->id; wc->qp = &qp->ibqp; wc->wr_id = wr_id; } else { __process_resp_one(dev, qp, cq, wc, resp, wr_id); } /* PBL is maintained in case of WR granularity. * So increment WR consumer after consuming WR */ srq->hw_srq.wr_cons_cnt++; QL_DPRINT12(ha, "exit\n"); return 1; } static int process_resp_one(struct qlnxr_dev *dev, struct qlnxr_qp *qp, struct qlnxr_cq *cq, struct ib_wc *wc, struct rdma_cqe_responder *resp) { qlnx_host_t *ha = dev->ha; u64 wr_id = qp->rqe_wr_id[qp->rq.cons].wr_id; QL_DPRINT12(ha, "enter\n"); __process_resp_one(dev, qp, cq, wc, resp, wr_id); while (qp->rqe_wr_id[qp->rq.cons].wqe_size--) ecore_chain_consume(&qp->rq.pbl); qlnxr_inc_sw_cons(&qp->rq); QL_DPRINT12(ha, "exit\n"); return 1; } static int process_resp_flush(struct qlnxr_qp *qp, int num_entries, struct ib_wc *wc, u16 hw_cons) { u16 cnt = 0; qlnx_host_t *ha = qp->dev->ha; QL_DPRINT12(ha, "enter\n"); while (num_entries && qp->rq.wqe_cons != hw_cons) { /* fill WC */ wc->status = IB_WC_WR_FLUSH_ERR; wc->vendor_err = 0; wc->wc_flags = 0; wc->src_qp = qp->id; wc->byte_len = 0; wc->wr_id = qp->rqe_wr_id[qp->rq.cons].wr_id; wc->qp = &qp->ibqp; num_entries--; wc++; cnt++; while (qp->rqe_wr_id[qp->rq.cons].wqe_size--) ecore_chain_consume(&qp->rq.pbl); qlnxr_inc_sw_cons(&qp->rq); } QL_DPRINT12(ha, "exit cnt = 0x%x\n", cnt); return cnt; } static void try_consume_resp_cqe(struct qlnxr_cq *cq, struct qlnxr_qp *qp, struct rdma_cqe_responder *resp, int *update) { if (le16_to_cpu(resp->rq_cons) == qp->rq.wqe_cons) { consume_cqe(cq); *update |= 1; } } static int qlnxr_poll_cq_resp_srq(struct qlnxr_dev *dev, struct qlnxr_qp *qp, struct qlnxr_cq *cq, int num_entries, struct ib_wc *wc, struct rdma_cqe_responder *resp, int *update) { int cnt; qlnx_host_t *ha = dev->ha; QL_DPRINT12(ha, "enter\n"); cnt = process_resp_one_srq(dev, qp, cq, wc, resp); consume_cqe(cq); *update |= 1; QL_DPRINT12(ha, "exit cnt = 0x%x\n", cnt); return cnt; } static int qlnxr_poll_cq_resp(struct qlnxr_dev *dev, struct qlnxr_qp *qp, struct qlnxr_cq *cq, int num_entries, struct ib_wc *wc, struct rdma_cqe_responder *resp, int *update) { int cnt; qlnx_host_t *ha = dev->ha; QL_DPRINT12(ha, "enter\n"); if (resp->status == RDMA_CQE_RESP_STS_WORK_REQUEST_FLUSHED_ERR) { cnt = process_resp_flush(qp, num_entries, wc, resp->rq_cons); try_consume_resp_cqe(cq, qp, resp, update); } else { cnt = process_resp_one(dev, qp, cq, wc, resp); consume_cqe(cq); *update |= 1; } QL_DPRINT12(ha, "exit cnt = 0x%x\n", cnt); return cnt; } static void try_consume_req_cqe(struct qlnxr_cq *cq, struct qlnxr_qp *qp, struct rdma_cqe_requester *req, int *update) { if (le16_to_cpu(req->sq_cons) == qp->sq.wqe_cons) { consume_cqe(cq); *update |= 1; } } static void doorbell_cq(struct qlnxr_dev *dev, struct qlnxr_cq *cq, u32 cons, u8 flags) { uint64_t reg_addr; qlnx_host_t *ha = dev->ha; QL_DPRINT12(ha, "enter\n"); wmb(); cq->db.data.agg_flags = flags; cq->db.data.value = cpu_to_le32(cons); reg_addr = (uint64_t)((uint8_t *)cq->db_addr - (uint8_t *)(ha->cdev.doorbells)); bus_write_8(ha->pci_dbells, reg_addr, cq->db.raw); bus_barrier(ha->pci_dbells, 0, 0, BUS_SPACE_BARRIER_READ); QL_DPRINT12(ha, "exit\n"); return; //#ifdef __LP64__ // writeq(cq->db.raw, cq->db_addr); //#else /* Note that since the FW allows 64 bit write only, in 32bit systems * the value of db_addr must be low enough. This is currently not * enforced. */ // writel(cq->db.raw & 0xffffffff, cq->db_addr); // mmiowb(); //#endif } static int is_valid_cqe(struct qlnxr_cq *cq, union rdma_cqe *cqe) { struct rdma_cqe_requester *resp_cqe = &cqe->req; return (resp_cqe->flags & RDMA_RESIZE_CQ_RAMROD_DATA_TOGGLE_BIT_MASK) == cq->pbl_toggle; } int qlnxr_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc) { struct qlnxr_cq *cq = get_qlnxr_cq(ibcq); struct qlnxr_dev *dev = get_qlnxr_dev((ibcq->device)); int done = 0; union rdma_cqe *cqe = cq->latest_cqe; int update = 0; u32 old_cons, new_cons; unsigned long flags; qlnx_host_t *ha = dev->ha; QL_DPRINT12(ha, "enter\n"); - if (!(ha->ifp->if_drv_flags & IFF_DRV_RUNNING)) + if (!(if_getdrvflags(ha->ifp) & IFF_DRV_RUNNING)) return -EINVAL; if (cq->destroyed) { QL_DPRINT11(ha, "called after destroy for cq %p (icid=%d)\n", cq, cq->icid); return 0; } if (cq->cq_type == QLNXR_CQ_TYPE_GSI) return qlnxr_gsi_poll_cq(ibcq, num_entries, wc); spin_lock_irqsave(&cq->cq_lock, flags); old_cons = ecore_chain_get_cons_idx_u32(&cq->pbl); while (num_entries && is_valid_cqe(cq, cqe)) { int cnt = 0; struct qlnxr_qp *qp; struct rdma_cqe_requester *resp_cqe; enum rdma_cqe_type cqe_type; /* prevent speculative reads of any field of CQE */ rmb(); resp_cqe = &cqe->req; qp = (struct qlnxr_qp *)(uintptr_t)HILO_U64(resp_cqe->qp_handle.hi, resp_cqe->qp_handle.lo); if (!qp) { QL_DPRINT11(ha, "qp = NULL\n"); break; } wc->qp = &qp->ibqp; cqe_type = GET_FIELD(resp_cqe->flags, RDMA_CQE_REQUESTER_TYPE); switch (cqe_type) { case RDMA_CQE_TYPE_REQUESTER: cnt = qlnxr_poll_cq_req(dev, qp, cq, num_entries, wc, &cqe->req); try_consume_req_cqe(cq, qp, &cqe->req, &update); break; case RDMA_CQE_TYPE_RESPONDER_RQ: cnt = qlnxr_poll_cq_resp(dev, qp, cq, num_entries, wc, &cqe->resp, &update); break; case RDMA_CQE_TYPE_RESPONDER_SRQ: cnt = qlnxr_poll_cq_resp_srq(dev, qp, cq, num_entries, wc, &cqe->resp, &update); break; case RDMA_CQE_TYPE_INVALID: default: QL_DPRINT11(ha, "cqe type [0x%x] invalid\n", cqe_type); break; } num_entries -= cnt; wc += cnt; done += cnt; cqe = cq->latest_cqe; } new_cons = ecore_chain_get_cons_idx_u32(&cq->pbl); cq->cq_cons += new_cons - old_cons; if (update) { /* doorbell notifies abount latest VALID entry, * but chain already point to the next INVALID one */ doorbell_cq(dev, cq, cq->cq_cons - 1, cq->arm_flags); QL_DPRINT12(ha, "cq = %p cons = 0x%x " "arm_flags = 0x%x db.icid = 0x%x\n", cq, (cq->cq_cons - 1), cq->arm_flags, cq->db.data.icid); } spin_unlock_irqrestore(&cq->cq_lock, flags); QL_DPRINT12(ha, "exit\n"); return done; } int qlnxr_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags) { struct qlnxr_cq *cq = get_qlnxr_cq(ibcq); unsigned long sflags; struct qlnxr_dev *dev; qlnx_host_t *ha; dev = get_qlnxr_dev((ibcq->device)); ha = dev->ha; QL_DPRINT12(ha, "enter ibcq = %p flags = 0x%x " "cp = %p cons = 0x%x cq_type = 0x%x\n", ibcq, flags, cq, cq->cq_cons, cq->cq_type); - if (!(ha->ifp->if_drv_flags & IFF_DRV_RUNNING)) + if (!(if_getdrvflags(ha->ifp) & IFF_DRV_RUNNING)) return -EINVAL; if (cq->destroyed) { QL_DPRINT11(ha, "cq was already destroyed cq = %p icid=%d\n", cq, cq->icid); return -EINVAL; } if (cq->cq_type == QLNXR_CQ_TYPE_GSI) { return 0; } spin_lock_irqsave(&cq->cq_lock, sflags); cq->arm_flags = 0; if (flags & IB_CQ_SOLICITED) { cq->arm_flags |= DQ_UCM_ROCE_CQ_ARM_SE_CF_CMD; } if (flags & IB_CQ_NEXT_COMP) { cq->arm_flags |= DQ_UCM_ROCE_CQ_ARM_CF_CMD; } doorbell_cq(dev, cq, (cq->cq_cons - 1), cq->arm_flags); spin_unlock_irqrestore(&cq->cq_lock, sflags); QL_DPRINT12(ha, "exit ibcq = %p flags = 0x%x\n", ibcq, flags); return 0; } static struct qlnxr_mr * __qlnxr_alloc_mr(struct ib_pd *ibpd, int max_page_list_len) { struct qlnxr_pd *pd = get_qlnxr_pd(ibpd); struct qlnxr_dev *dev = get_qlnxr_dev((ibpd->device)); struct qlnxr_mr *mr; int rc = -ENOMEM; qlnx_host_t *ha; ha = dev->ha; QL_DPRINT12(ha, "enter ibpd = %p pd = %p " " pd_id = %d max_page_list_len = %d\n", ibpd, pd, pd->pd_id, max_page_list_len); mr = kzalloc(sizeof(*mr), GFP_KERNEL); if (!mr) { QL_DPRINT11(ha, "kzalloc(mr) failed\n"); return ERR_PTR(rc); } mr->dev = dev; mr->type = QLNXR_MR_FRMR; rc = qlnxr_init_mr_info(dev, &mr->info, max_page_list_len, 1 /* allow dual layer pbl */); if (rc) { QL_DPRINT11(ha, "qlnxr_init_mr_info failed\n"); goto err0; } rc = ecore_rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid); if (rc) { QL_DPRINT11(ha, "ecore_rdma_alloc_tid failed\n"); goto err0; } /* index only, 18 bit long, lkey = itid << 8 | key */ mr->hw_mr.tid_type = ECORE_RDMA_TID_FMR; mr->hw_mr.key = 0; mr->hw_mr.pd = pd->pd_id; mr->hw_mr.local_read = 1; mr->hw_mr.local_write = 0; mr->hw_mr.remote_read = 0; mr->hw_mr.remote_write = 0; mr->hw_mr.remote_atomic = 0; mr->hw_mr.mw_bind = false; /* TBD MW BIND */ mr->hw_mr.pbl_ptr = 0; /* Will be supplied during post */ mr->hw_mr.pbl_two_level = mr->info.pbl_info.two_layered; mr->hw_mr.pbl_page_size_log = ilog2(mr->info.pbl_info.pbl_size); mr->hw_mr.fbo = 0; mr->hw_mr.length = 0; mr->hw_mr.vaddr = 0; mr->hw_mr.zbva = false; /* TBD figure when this should be true */ mr->hw_mr.phy_mr = true; /* Fast MR - True, Regular Register False */ mr->hw_mr.dma_mr = false; rc = ecore_rdma_register_tid(dev->rdma_ctx, &mr->hw_mr); if (rc) { QL_DPRINT11(ha, "ecore_rdma_register_tid failed\n"); goto err1; } mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key; mr->ibmr.rkey = mr->ibmr.lkey; QL_DPRINT12(ha, "exit mr = %p mr->ibmr.lkey = 0x%x\n", mr, mr->ibmr.lkey); return mr; err1: ecore_rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid); err0: kfree(mr); QL_DPRINT12(ha, "exit\n"); return ERR_PTR(rc); } struct ib_mr * qlnxr_alloc_mr(struct ib_pd *ibpd, enum ib_mr_type mr_type, u32 max_num_sg, struct ib_udata *udata) { struct qlnxr_dev *dev; struct qlnxr_mr *mr; qlnx_host_t *ha; dev = get_qlnxr_dev(ibpd->device); ha = dev->ha; QL_DPRINT12(ha, "enter\n"); if (mr_type != IB_MR_TYPE_MEM_REG) return ERR_PTR(-EINVAL); mr = __qlnxr_alloc_mr(ibpd, max_num_sg); if (IS_ERR(mr)) return ERR_PTR(-EINVAL); QL_DPRINT12(ha, "exit mr = %p &mr->ibmr = %p\n", mr, &mr->ibmr); return &mr->ibmr; } static int qlnxr_set_page(struct ib_mr *ibmr, u64 addr) { struct qlnxr_mr *mr = get_qlnxr_mr(ibmr); struct qlnxr_pbl *pbl_table; struct regpair *pbe; struct qlnxr_dev *dev; qlnx_host_t *ha; u32 pbes_in_page; dev = mr->dev; ha = dev->ha; if (unlikely(mr->npages == mr->info.pbl_info.num_pbes)) { QL_DPRINT12(ha, "fails mr->npages %d\n", mr->npages); return -ENOMEM; } QL_DPRINT12(ha, "mr->npages %d addr = %p enter\n", mr->npages, ((void *)addr)); pbes_in_page = mr->info.pbl_info.pbl_size / sizeof(u64); pbl_table = mr->info.pbl_table + (mr->npages / pbes_in_page); pbe = (struct regpair *)pbl_table->va; pbe += mr->npages % pbes_in_page; pbe->lo = cpu_to_le32((u32)addr); pbe->hi = cpu_to_le32((u32)upper_32_bits(addr)); mr->npages++; QL_DPRINT12(ha, "mr->npages %d addr = %p exit \n", mr->npages, ((void *)addr)); return 0; } int qlnxr_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents, unsigned int *sg_offset) { int ret; struct qlnxr_mr *mr = get_qlnxr_mr(ibmr); qlnx_host_t *ha; if (mr == NULL) return (-1); if (mr->dev == NULL) return (-1); ha = mr->dev->ha; QL_DPRINT12(ha, "enter\n"); mr->npages = 0; qlnx_handle_completed_mrs(mr->dev, &mr->info); ret = ib_sg_to_pages(ibmr, sg, sg_nents, NULL, qlnxr_set_page); QL_DPRINT12(ha, "exit ret = %d\n", ret); return (ret); } int qlnxr_create_ah(struct ib_ah *ibah, struct ib_ah_attr *attr, u32 flags, struct ib_udata *udata) { struct qlnxr_dev *dev; qlnx_host_t *ha; struct qlnxr_ah *ah = get_qlnxr_ah(ibah); dev = get_qlnxr_dev(ibah->device); ha = dev->ha; QL_DPRINT12(ha, "in create_ah\n"); ah->attr = *attr; return (0); } void qlnxr_destroy_ah(struct ib_ah *ibah, u32 flags) { } int qlnxr_query_ah(struct ib_ah *ibah, struct ib_ah_attr *attr) { struct qlnxr_dev *dev; qlnx_host_t *ha; dev = get_qlnxr_dev((ibah->device)); ha = dev->ha; QL_DPRINT12(ha, "Query AH not supported\n"); return -EINVAL; } int qlnxr_modify_ah(struct ib_ah *ibah, struct ib_ah_attr *attr) { struct qlnxr_dev *dev; qlnx_host_t *ha; dev = get_qlnxr_dev((ibah->device)); ha = dev->ha; QL_DPRINT12(ha, "Modify AH not supported\n"); return -ENOSYS; } int qlnxr_process_mad(struct ib_device *ibdev, int process_mad_flags, u8 port_num, const struct ib_wc *in_wc, const struct ib_grh *in_grh, const struct ib_mad_hdr *mad_hdr, size_t in_mad_size, struct ib_mad_hdr *out_mad, size_t *out_mad_size, u16 *out_mad_pkey_index) { struct qlnxr_dev *dev; qlnx_host_t *ha; dev = get_qlnxr_dev(ibdev); ha = dev->ha; QL_DPRINT12(ha, "process mad not supported\n"); return -ENOSYS; // QL_DPRINT12(ha, "qlnxr_process_mad in_mad %x %x %x %x %x %x %x %x\n", // in_mad->mad_hdr.attr_id, in_mad->mad_hdr.base_version, // in_mad->mad_hdr.attr_mod, in_mad->mad_hdr.class_specific, // in_mad->mad_hdr.class_version, in_mad->mad_hdr.method, // in_mad->mad_hdr.mgmt_class, in_mad->mad_hdr.status); // return IB_MAD_RESULT_SUCCESS; } int qlnxr_get_port_immutable(struct ib_device *ibdev, u8 port_num, struct ib_port_immutable *immutable) { struct qlnxr_dev *dev; qlnx_host_t *ha; struct ib_port_attr attr; int err; dev = get_qlnxr_dev(ibdev); ha = dev->ha; QL_DPRINT12(ha, "enter\n"); err = qlnxr_query_port(ibdev, port_num, &attr); if (err) return err; if (QLNX_IS_IWARP(dev)) { immutable->pkey_tbl_len = 1; immutable->gid_tbl_len = 1; immutable->core_cap_flags = RDMA_CORE_PORT_IWARP; immutable->max_mad_size = 0; } else { immutable->pkey_tbl_len = attr.pkey_tbl_len; immutable->gid_tbl_len = attr.gid_tbl_len; immutable->core_cap_flags = RDMA_CORE_PORT_IBA_ROCE; immutable->max_mad_size = IB_MGMT_MAD_SIZE; } QL_DPRINT12(ha, "exit\n"); return 0; } /***** iWARP related functions *************/ static void qlnxr_iw_mpa_request(void *context, struct ecore_iwarp_cm_event_params *params) { struct qlnxr_iw_listener *listener = (struct qlnxr_iw_listener *)context; struct qlnxr_dev *dev = listener->dev; struct qlnxr_iw_ep *ep; struct iw_cm_event event; struct sockaddr_in *laddr; struct sockaddr_in *raddr; qlnx_host_t *ha; ha = dev->ha; QL_DPRINT12(ha, "enter\n"); if (params->cm_info->ip_version != ECORE_TCP_IPV4) { QL_DPRINT11(ha, "only IPv4 supported [0x%x]\n", params->cm_info->ip_version); return; } ep = kzalloc(sizeof(*ep), GFP_ATOMIC); if (!ep) { QL_DPRINT11(ha, "kzalloc{ep) failed\n"); return; } ep->dev = dev; ep->ecore_context = params->ep_context; memset(&event, 0, sizeof(event)); event.event = IW_CM_EVENT_CONNECT_REQUEST; event.status = params->status; laddr = (struct sockaddr_in *)&event.local_addr; raddr = (struct sockaddr_in *)&event.remote_addr; laddr->sin_family = AF_INET; raddr->sin_family = AF_INET; laddr->sin_port = htons(params->cm_info->local_port); raddr->sin_port = htons(params->cm_info->remote_port); laddr->sin_addr.s_addr = htonl(params->cm_info->local_ip[0]); raddr->sin_addr.s_addr = htonl(params->cm_info->remote_ip[0]); event.provider_data = (void *)ep; event.private_data = (void *)params->cm_info->private_data; event.private_data_len = (u8)params->cm_info->private_data_len; event.ord = params->cm_info->ord; event.ird = params->cm_info->ird; listener->cm_id->event_handler(listener->cm_id, &event); QL_DPRINT12(ha, "exit\n"); return; } static void qlnxr_iw_issue_event(void *context, struct ecore_iwarp_cm_event_params *params, enum iw_cm_event_type event_type, char *str) { struct qlnxr_iw_ep *ep = (struct qlnxr_iw_ep *)context; struct qlnxr_dev *dev = ep->dev; struct iw_cm_event event; qlnx_host_t *ha; ha = dev->ha; QL_DPRINT12(ha, "enter\n"); memset(&event, 0, sizeof(event)); event.status = params->status; event.event = event_type; if (params->cm_info != NULL) { event.ird = params->cm_info->ird; event.ord = params->cm_info->ord; QL_DPRINT12(ha, "ord=[%d] \n", event.ord); QL_DPRINT12(ha, "ird=[%d] \n", event.ird); event.private_data_len = params->cm_info->private_data_len; event.private_data = (void *)params->cm_info->private_data; QL_DPRINT12(ha, "private_data_len=[%d] \n", event.private_data_len); } QL_DPRINT12(ha, "event=[%d] %s\n", event.event, str); QL_DPRINT12(ha, "status=[%d] \n", event.status); if (ep) { if (ep->cm_id) ep->cm_id->event_handler(ep->cm_id, &event); else QL_DPRINT11(ha, "ep->cm_id == NULL \n"); } else { QL_DPRINT11(ha, "ep == NULL \n"); } QL_DPRINT12(ha, "exit\n"); return; } static void qlnxr_iw_close_event(void *context, struct ecore_iwarp_cm_event_params *params) { struct qlnxr_iw_ep *ep = (struct qlnxr_iw_ep *)context; struct qlnxr_dev *dev = ep->dev; qlnx_host_t *ha; ha = dev->ha; QL_DPRINT12(ha, "enter\n"); if (ep->cm_id) { qlnxr_iw_issue_event(context, params, IW_CM_EVENT_CLOSE, "IW_CM_EVENT_EVENT_CLOSE"); ep->cm_id->rem_ref(ep->cm_id); ep->cm_id = NULL; } QL_DPRINT12(ha, "exit\n"); return; } static void qlnxr_iw_passive_complete(void *context, struct ecore_iwarp_cm_event_params *params) { struct qlnxr_iw_ep *ep = (struct qlnxr_iw_ep *)context; struct qlnxr_dev *dev = ep->dev; qlnx_host_t *ha; ha = dev->ha; /* We will only reach the following state if MPA_REJECT was called on * passive. In this case there will be no associated QP. */ if ((params->status == -ECONNREFUSED) && (ep->qp == NULL)) { QL_DPRINT11(ha, "PASSIVE connection refused releasing ep...\n"); kfree(ep); return; } /* We always issue an established event, however, ofed does not look * at event code for established. So if there was a failure, we follow * with close... */ qlnxr_iw_issue_event(context, params, IW_CM_EVENT_ESTABLISHED, "IW_CM_EVENT_ESTABLISHED"); if (params->status < 0) { qlnxr_iw_close_event(context, params); } return; } struct qlnxr_discon_work { struct work_struct work; struct qlnxr_iw_ep *ep; enum ecore_iwarp_event_type event; int status; }; static void qlnxr_iw_disconnect_worker(struct work_struct *work) { struct qlnxr_discon_work *dwork = container_of(work, struct qlnxr_discon_work, work); struct ecore_rdma_modify_qp_in_params qp_params = { 0 }; struct qlnxr_iw_ep *ep = dwork->ep; struct qlnxr_dev *dev = ep->dev; struct qlnxr_qp *qp = ep->qp; struct iw_cm_event event; if (qp->destroyed) { kfree(dwork); qlnxr_iw_qp_rem_ref(&qp->ibqp); return; } memset(&event, 0, sizeof(event)); event.status = dwork->status; event.event = IW_CM_EVENT_DISCONNECT; /* Success means graceful disconnect was requested. modifying * to SQD is translated to graceful disconnect. O/w reset is sent */ if (dwork->status) qp_params.new_state = ECORE_ROCE_QP_STATE_ERR; else qp_params.new_state = ECORE_ROCE_QP_STATE_SQD; kfree(dwork); if (ep->cm_id) ep->cm_id->event_handler(ep->cm_id, &event); SET_FIELD(qp_params.modify_flags, ECORE_RDMA_MODIFY_QP_VALID_NEW_STATE, 1); ecore_rdma_modify_qp(dev->rdma_ctx, qp->ecore_qp, &qp_params); qlnxr_iw_qp_rem_ref(&qp->ibqp); return; } void qlnxr_iw_disconnect_event(void *context, struct ecore_iwarp_cm_event_params *params) { struct qlnxr_discon_work *work; struct qlnxr_iw_ep *ep = (struct qlnxr_iw_ep *)context; struct qlnxr_dev *dev = ep->dev; struct qlnxr_qp *qp = ep->qp; work = kzalloc(sizeof(*work), GFP_ATOMIC); if (!work) return; qlnxr_iw_qp_add_ref(&qp->ibqp); work->ep = ep; work->event = params->event; work->status = params->status; INIT_WORK(&work->work, qlnxr_iw_disconnect_worker); queue_work(dev->iwarp_wq, &work->work); return; } static int qlnxr_iw_mpa_reply(void *context, struct ecore_iwarp_cm_event_params *params) { struct qlnxr_iw_ep *ep = (struct qlnxr_iw_ep *)context; struct qlnxr_dev *dev = ep->dev; struct ecore_iwarp_send_rtr_in rtr_in; int rc; qlnx_host_t *ha; ha = dev->ha; QL_DPRINT12(ha, "enter\n"); - if (!(ha->ifp->if_drv_flags & IFF_DRV_RUNNING)) + if (!(if_getdrvflags(ha->ifp) & IFF_DRV_RUNNING)) return -EINVAL; bzero(&rtr_in, sizeof(struct ecore_iwarp_send_rtr_in)); rtr_in.ep_context = params->ep_context; rc = ecore_iwarp_send_rtr(dev->rdma_ctx, &rtr_in); QL_DPRINT12(ha, "exit rc = %d\n", rc); return rc; } void qlnxr_iw_qp_event(void *context, struct ecore_iwarp_cm_event_params *params, enum ib_event_type ib_event, char *str) { struct qlnxr_iw_ep *ep = (struct qlnxr_iw_ep *)context; struct qlnxr_dev *dev = ep->dev; struct ib_qp *ibqp = &(ep->qp->ibqp); struct ib_event event; qlnx_host_t *ha; ha = dev->ha; QL_DPRINT12(ha, "[context, event, event_handler] = [%p, 0x%x, %s, %p] enter\n", context, params->event, str, ibqp->event_handler); if (ibqp->event_handler) { event.event = ib_event; event.device = ibqp->device; event.element.qp = ibqp; ibqp->event_handler(&event, ibqp->qp_context); } return; } int qlnxr_iw_event_handler(void *context, struct ecore_iwarp_cm_event_params *params) { struct qlnxr_iw_ep *ep = (struct qlnxr_iw_ep *)context; struct qlnxr_dev *dev = ep->dev; qlnx_host_t *ha; ha = dev->ha; QL_DPRINT12(ha, "[context, event] = [%p, 0x%x] " "enter\n", context, params->event); switch (params->event) { /* Passive side request received */ case ECORE_IWARP_EVENT_MPA_REQUEST: qlnxr_iw_mpa_request(context, params); break; case ECORE_IWARP_EVENT_ACTIVE_MPA_REPLY: qlnxr_iw_mpa_reply(context, params); break; /* Passive side established ( ack on mpa response ) */ case ECORE_IWARP_EVENT_PASSIVE_COMPLETE: ep->during_connect = 0; qlnxr_iw_passive_complete(context, params); break; /* Active side reply received */ case ECORE_IWARP_EVENT_ACTIVE_COMPLETE: ep->during_connect = 0; qlnxr_iw_issue_event(context, params, IW_CM_EVENT_CONNECT_REPLY, "IW_CM_EVENT_CONNECT_REPLY"); if (params->status < 0) { struct qlnxr_iw_ep *ep = (struct qlnxr_iw_ep *)context; ep->cm_id->rem_ref(ep->cm_id); ep->cm_id = NULL; } break; case ECORE_IWARP_EVENT_DISCONNECT: qlnxr_iw_disconnect_event(context, params); break; case ECORE_IWARP_EVENT_CLOSE: ep->during_connect = 0; qlnxr_iw_close_event(context, params); break; case ECORE_IWARP_EVENT_RQ_EMPTY: qlnxr_iw_qp_event(context, params, IB_EVENT_QP_FATAL, "IWARP_EVENT_RQ_EMPTY"); break; case ECORE_IWARP_EVENT_IRQ_FULL: qlnxr_iw_qp_event(context, params, IB_EVENT_QP_FATAL, "IWARP_EVENT_IRQ_FULL"); break; case ECORE_IWARP_EVENT_LLP_TIMEOUT: qlnxr_iw_qp_event(context, params, IB_EVENT_QP_FATAL, "IWARP_EVENT_LLP_TIMEOUT"); break; case ECORE_IWARP_EVENT_REMOTE_PROTECTION_ERROR: qlnxr_iw_qp_event(context, params, IB_EVENT_QP_ACCESS_ERR, "IWARP_EVENT_REMOTE_PROTECTION_ERROR"); break; case ECORE_IWARP_EVENT_CQ_OVERFLOW: qlnxr_iw_qp_event(context, params, IB_EVENT_QP_FATAL, "QED_IWARP_EVENT_CQ_OVERFLOW"); break; case ECORE_IWARP_EVENT_QP_CATASTROPHIC: qlnxr_iw_qp_event(context, params, IB_EVENT_QP_FATAL, "QED_IWARP_EVENT_QP_CATASTROPHIC"); break; case ECORE_IWARP_EVENT_LOCAL_ACCESS_ERROR: qlnxr_iw_qp_event(context, params, IB_EVENT_QP_ACCESS_ERR, "IWARP_EVENT_LOCAL_ACCESS_ERROR"); break; case ECORE_IWARP_EVENT_REMOTE_OPERATION_ERROR: qlnxr_iw_qp_event(context, params, IB_EVENT_QP_FATAL, "IWARP_EVENT_REMOTE_OPERATION_ERROR"); break; case ECORE_IWARP_EVENT_TERMINATE_RECEIVED: QL_DPRINT12(ha, "Got terminate message" " ECORE_IWARP_EVENT_TERMINATE_RECEIVED\n"); break; default: QL_DPRINT12(ha, "Unknown event [0x%x] received \n", params->event); break; }; QL_DPRINT12(ha, "[context, event] = [%p, 0x%x] " "exit\n", context, params->event); return 0; } static int qlnxr_addr4_resolve(struct qlnxr_dev *dev, struct sockaddr_in *src_in, struct sockaddr_in *dst_in, u8 *dst_mac) { int rc; rc = arpresolve(dev->ha->ifp, 0, NULL, (struct sockaddr *)dst_in, dst_mac, NULL, NULL); QL_DPRINT12(dev->ha, "rc = %d " "sa_len = 0x%x sa_family = 0x%x IP Address = %d.%d.%d.%d " "Dest MAC %02x:%02x:%02x:%02x:%02x:%02x\n", rc, dst_in->sin_len, dst_in->sin_family, NIPQUAD((dst_in->sin_addr.s_addr)), dst_mac[0], dst_mac[1], dst_mac[2], dst_mac[3], dst_mac[4], dst_mac[5]); return rc; } int qlnxr_iw_connect(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param) { struct qlnxr_dev *dev; struct ecore_iwarp_connect_out out_params; struct ecore_iwarp_connect_in in_params; struct qlnxr_iw_ep *ep; struct qlnxr_qp *qp; struct sockaddr_in *laddr; struct sockaddr_in *raddr; int rc = 0; qlnx_host_t *ha; dev = get_qlnxr_dev((cm_id->device)); ha = dev->ha; QL_DPRINT12(ha, "[cm_id, conn_param] = [%p, %p] " "enter \n", cm_id, conn_param); - if (!(ha->ifp->if_drv_flags & IFF_DRV_RUNNING)) + if (!(if_getdrvflags(ha->ifp) & IFF_DRV_RUNNING)) return -EINVAL; qp = idr_find(&dev->qpidr, conn_param->qpn); laddr = (struct sockaddr_in *)&cm_id->local_addr; raddr = (struct sockaddr_in *)&cm_id->remote_addr; QL_DPRINT12(ha, "local = [%d.%d.%d.%d, %d] remote = [%d.%d.%d.%d, %d]\n", NIPQUAD((laddr->sin_addr.s_addr)), laddr->sin_port, NIPQUAD((raddr->sin_addr.s_addr)), raddr->sin_port); ep = kzalloc(sizeof(*ep), GFP_KERNEL); if (!ep) { QL_DPRINT11(ha, "struct qlnxr_iw_ep " "alloc memory failed\n"); return -ENOMEM; } ep->dev = dev; ep->qp = qp; cm_id->add_ref(cm_id); ep->cm_id = cm_id; memset(&in_params, 0, sizeof (struct ecore_iwarp_connect_in)); memset(&out_params, 0, sizeof (struct ecore_iwarp_connect_out)); in_params.event_cb = qlnxr_iw_event_handler; in_params.cb_context = ep; in_params.cm_info.ip_version = ECORE_TCP_IPV4; in_params.cm_info.remote_ip[0] = ntohl(raddr->sin_addr.s_addr); in_params.cm_info.local_ip[0] = ntohl(laddr->sin_addr.s_addr); in_params.cm_info.remote_port = ntohs(raddr->sin_port); in_params.cm_info.local_port = ntohs(laddr->sin_port); in_params.cm_info.vlan = 0; - in_params.mss = dev->ha->ifp->if_mtu - 40; + in_params.mss = if_getmtu(dev->ha->ifp) - 40; QL_DPRINT12(ha, "remote_ip = [%d.%d.%d.%d] " "local_ip = [%d.%d.%d.%d] remote_port = %d local_port = %d " "vlan = %d\n", NIPQUAD((in_params.cm_info.remote_ip[0])), NIPQUAD((in_params.cm_info.local_ip[0])), in_params.cm_info.remote_port, in_params.cm_info.local_port, in_params.cm_info.vlan); rc = qlnxr_addr4_resolve(dev, laddr, raddr, (u8 *)in_params.remote_mac_addr); if (rc) { QL_DPRINT11(ha, "qlnxr_addr4_resolve failed\n"); goto err; } QL_DPRINT12(ha, "ord = %d ird=%d private_data=%p" " private_data_len=%d rq_psn=%d\n", conn_param->ord, conn_param->ird, conn_param->private_data, conn_param->private_data_len, qp->rq_psn); in_params.cm_info.ord = conn_param->ord; in_params.cm_info.ird = conn_param->ird; in_params.cm_info.private_data = conn_param->private_data; in_params.cm_info.private_data_len = conn_param->private_data_len; in_params.qp = qp->ecore_qp; memcpy(in_params.local_mac_addr, dev->ha->primary_mac, ETH_ALEN); rc = ecore_iwarp_connect(dev->rdma_ctx, &in_params, &out_params); if (rc) { QL_DPRINT12(ha, "ecore_iwarp_connect failed\n"); goto err; } QL_DPRINT12(ha, "exit\n"); return rc; err: cm_id->rem_ref(cm_id); kfree(ep); QL_DPRINT12(ha, "exit [%d]\n", rc); return rc; } int qlnxr_iw_create_listen(struct iw_cm_id *cm_id, int backlog) { struct qlnxr_dev *dev; struct qlnxr_iw_listener *listener; struct ecore_iwarp_listen_in iparams; struct ecore_iwarp_listen_out oparams; struct sockaddr_in *laddr; qlnx_host_t *ha; int rc; dev = get_qlnxr_dev((cm_id->device)); ha = dev->ha; QL_DPRINT12(ha, "enter\n"); - if (!(ha->ifp->if_drv_flags & IFF_DRV_RUNNING)) + if (!(if_getdrvflags(ha->ifp) & IFF_DRV_RUNNING)) return -EINVAL; laddr = (struct sockaddr_in *)&cm_id->local_addr; listener = kzalloc(sizeof(*listener), GFP_KERNEL); if (listener == NULL) { QL_DPRINT11(ha, "listener memory alloc failed\n"); return -ENOMEM; } listener->dev = dev; cm_id->add_ref(cm_id); listener->cm_id = cm_id; listener->backlog = backlog; memset(&iparams, 0, sizeof (struct ecore_iwarp_listen_in)); memset(&oparams, 0, sizeof (struct ecore_iwarp_listen_out)); iparams.cb_context = listener; iparams.event_cb = qlnxr_iw_event_handler; iparams.max_backlog = backlog; iparams.ip_version = ECORE_TCP_IPV4; iparams.ip_addr[0] = ntohl(laddr->sin_addr.s_addr); iparams.port = ntohs(laddr->sin_port); iparams.vlan = 0; QL_DPRINT12(ha, "[%d.%d.%d.%d, %d] iparamsport=%d\n", NIPQUAD((laddr->sin_addr.s_addr)), laddr->sin_port, iparams.port); rc = ecore_iwarp_create_listen(dev->rdma_ctx, &iparams, &oparams); if (rc) { QL_DPRINT11(ha, "ecore_iwarp_create_listen failed rc = %d\n", rc); goto err; } listener->ecore_handle = oparams.handle; cm_id->provider_data = listener; QL_DPRINT12(ha, "exit\n"); return rc; err: cm_id->rem_ref(cm_id); kfree(listener); QL_DPRINT12(ha, "exit [%d]\n", rc); return rc; } void qlnxr_iw_destroy_listen(struct iw_cm_id *cm_id) { struct qlnxr_iw_listener *listener = cm_id->provider_data; struct qlnxr_dev *dev = get_qlnxr_dev((cm_id->device)); int rc = 0; qlnx_host_t *ha; ha = dev->ha; QL_DPRINT12(ha, "enter\n"); if (listener->ecore_handle) rc = ecore_iwarp_destroy_listen(dev->rdma_ctx, listener->ecore_handle); cm_id->rem_ref(cm_id); QL_DPRINT12(ha, "exit [%d]\n", rc); return; } int qlnxr_iw_accept(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param) { struct qlnxr_iw_ep *ep = (struct qlnxr_iw_ep *)cm_id->provider_data; struct qlnxr_dev *dev = ep->dev; struct qlnxr_qp *qp; struct ecore_iwarp_accept_in params; int rc; qlnx_host_t *ha; ha = dev->ha; QL_DPRINT12(ha, "enter qpid=%d\n", conn_param->qpn); - if (!(ha->ifp->if_drv_flags & IFF_DRV_RUNNING)) + if (!(if_getdrvflags(ha->ifp) & IFF_DRV_RUNNING)) return -EINVAL; qp = idr_find(&dev->qpidr, conn_param->qpn); if (!qp) { QL_DPRINT11(ha, "idr_find failed invalid qpn = %d\n", conn_param->qpn); return -EINVAL; } ep->qp = qp; qp->ep = ep; cm_id->add_ref(cm_id); ep->cm_id = cm_id; params.ep_context = ep->ecore_context; params.cb_context = ep; params.qp = ep->qp->ecore_qp; params.private_data = conn_param->private_data; params.private_data_len = conn_param->private_data_len; params.ird = conn_param->ird; params.ord = conn_param->ord; rc = ecore_iwarp_accept(dev->rdma_ctx, ¶ms); if (rc) { QL_DPRINT11(ha, "ecore_iwarp_accept failed %d\n", rc); goto err; } QL_DPRINT12(ha, "exit\n"); return 0; err: cm_id->rem_ref(cm_id); QL_DPRINT12(ha, "exit rc = %d\n", rc); return rc; } int qlnxr_iw_reject(struct iw_cm_id *cm_id, const void *pdata, u8 pdata_len) { struct qlnxr_iw_ep *ep = (struct qlnxr_iw_ep *)cm_id->provider_data; struct qlnxr_dev *dev = ep->dev; struct ecore_iwarp_reject_in params; int rc; params.ep_context = ep->ecore_context; params.cb_context = ep; params.private_data = pdata; params.private_data_len = pdata_len; ep->qp = NULL; rc = ecore_iwarp_reject(dev->rdma_ctx, ¶ms); return rc; } void qlnxr_iw_qp_add_ref(struct ib_qp *ibqp) { struct qlnxr_qp *qp = get_qlnxr_qp(ibqp); qlnx_host_t *ha; ha = qp->dev->ha; QL_DPRINT12(ha, "enter ibqp = %p\n", ibqp); atomic_inc(&qp->refcnt); QL_DPRINT12(ha, "exit \n"); return; } void qlnxr_iw_qp_rem_ref(struct ib_qp *ibqp) { struct qlnxr_qp *qp = get_qlnxr_qp(ibqp); qlnx_host_t *ha; ha = qp->dev->ha; QL_DPRINT12(ha, "enter ibqp = %p qp = %p\n", ibqp, qp); if (atomic_dec_and_test(&qp->refcnt)) { qlnxr_idr_remove(qp->dev, qp->qp_id); } QL_DPRINT12(ha, "exit \n"); return; } struct ib_qp * qlnxr_iw_get_qp(struct ib_device *ibdev, int qpn) { struct qlnxr_dev *dev = get_qlnxr_dev(ibdev); struct ib_qp *qp; qlnx_host_t *ha; ha = dev->ha; QL_DPRINT12(ha, "enter dev = %p ibdev = %p qpn = %d\n", dev, ibdev, qpn); qp = idr_find(&dev->qpidr, qpn); QL_DPRINT12(ha, "exit qp = %p\n", qp); return (qp); }