diff --git a/sys/dev/bnxt/bnxt_en/bnxt.h b/sys/dev/bnxt/bnxt_en/bnxt.h index 5556914fb61e..64482a656e9d 100644 --- a/sys/dev/bnxt/bnxt_en/bnxt.h +++ b/sys/dev/bnxt/bnxt_en/bnxt.h @@ -1,1397 +1,1398 @@ /*- * Broadcom NetXtreme-C/E network driver. * * Copyright (c) 2016 Broadcom, All Rights Reserved. * The term Broadcom refers to Broadcom Limited and/or its subsidiaries * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS' * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF * THE POSSIBILITY OF SUCH DAMAGE. */ #include #ifndef _BNXT_H #define _BNXT_H #include #include #include #include #include #include #include #include #include #include #include #include "hsi_struct_def.h" #include "bnxt_dcb.h" #include "bnxt_auxbus_compat.h" #define DFLT_HWRM_CMD_TIMEOUT 500 /* PCI IDs */ #define BROADCOM_VENDOR_ID 0x14E4 #define BCM57301 0x16c8 #define BCM57302 0x16c9 #define BCM57304 0x16ca #define BCM57311 0x16ce #define BCM57312 0x16cf #define BCM57314 0x16df #define BCM57402 0x16d0 #define BCM57402_NPAR 0x16d4 #define BCM57404 0x16d1 #define BCM57404_NPAR 0x16e7 #define BCM57406 0x16d2 #define BCM57406_NPAR 0x16e8 #define BCM57407 0x16d5 #define BCM57407_NPAR 0x16ea #define BCM57407_SFP 0x16e9 #define BCM57412 0x16d6 #define BCM57412_NPAR1 0x16de #define BCM57412_NPAR2 0x16eb #define BCM57414 0x16d7 #define BCM57414_NPAR1 0x16ec #define BCM57414_NPAR2 0x16ed #define BCM57416 0x16d8 #define BCM57416_NPAR1 0x16ee #define BCM57416_NPAR2 0x16ef #define BCM57416_SFP 0x16e3 #define BCM57417 0x16d9 #define BCM57417_NPAR1 0x16c0 #define BCM57417_NPAR2 0x16cc #define BCM57417_SFP 0x16e2 #define BCM57454 0x1614 #define BCM58700 0x16cd #define BCM57508 0x1750 #define BCM57504 0x1751 #define BCM57504_NPAR 0x1801 #define BCM57502 0x1752 #define BCM57608 0x1760 #define BCM57604 0x1761 #define BCM57602 0x1762 #define BCM57601 0x1763 #define NETXTREME_C_VF1 0x16cb #define NETXTREME_C_VF2 0x16e1 #define NETXTREME_C_VF3 0x16e5 #define NETXTREME_E_VF1 0x16c1 #define NETXTREME_E_VF2 0x16d3 #define NETXTREME_E_VF3 0x16dc #define EVENT_DATA1_RESET_NOTIFY_FATAL(data1) \ (((data1) & \ HWRM_ASYNC_EVENT_CMPL_RESET_NOTIFY_EVENT_DATA1_REASON_CODE_MASK) ==\ HWRM_ASYNC_EVENT_CMPL_RESET_NOTIFY_EVENT_DATA1_REASON_CODE_FW_EXCEPTION_FATAL) #define BNXT_EVENT_ERROR_REPORT_TYPE(data1) \ (((data1) & \ HWRM_ASYNC_EVENT_CMPL_ERROR_REPORT_BASE_EVENT_DATA1_ERROR_TYPE_MASK) >> \ HWRM_ASYNC_EVENT_CMPL_ERROR_REPORT_BASE_EVENT_DATA1_ERROR_TYPE_SFT) #define BNXT_EVENT_INVALID_SIGNAL_DATA(data2) \ (((data2) & \ HWRM_ASYNC_EVENT_CMPL_ERROR_REPORT_INVALID_SIGNAL_EVENT_DATA2_PIN_ID_MASK) >> \ HWRM_ASYNC_EVENT_CMPL_ERROR_REPORT_INVALID_SIGNAL_EVENT_DATA2_PIN_ID_SFT) #define BNXT_EVENT_DBR_EPOCH(data) \ (((data) & HWRM_ASYNC_EVENT_CMPL_ERROR_REPORT_DOORBELL_DROP_THRESHOLD_EVENT_DATA1_EPOCH_MASK) >> \ HWRM_ASYNC_EVENT_CMPL_ERROR_REPORT_DOORBELL_DROP_THRESHOLD_EVENT_DATA1_EPOCH_SFT) #define BNXT_EVENT_THERMAL_THRESHOLD_TEMP(data2) \ (((data2) & \ HWRM_ASYNC_EVENT_CMPL_ERROR_REPORT_THERMAL_EVENT_DATA2_THRESHOLD_TEMP_MASK) >> \ HWRM_ASYNC_EVENT_CMPL_ERROR_REPORT_THERMAL_EVENT_DATA2_THRESHOLD_TEMP_SFT) #define EVENT_DATA2_NVM_ERR_ADDR(data2) \ (((data2) & \ HWRM_ASYNC_EVENT_CMPL_ERROR_REPORT_NVM_EVENT_DATA2_ERR_ADDR_MASK) >> \ HWRM_ASYNC_EVENT_CMPL_ERROR_REPORT_NVM_EVENT_DATA2_ERR_ADDR_SFT) #define EVENT_DATA1_THERMAL_THRESHOLD_DIR_INCREASING(data1) \ (((data1) & \ HWRM_ASYNC_EVENT_CMPL_ERROR_REPORT_THERMAL_EVENT_DATA1_TRANSITION_DIR) == \ HWRM_ASYNC_EVENT_CMPL_ERROR_REPORT_THERMAL_EVENT_DATA1_TRANSITION_DIR_INCREASING) #define EVENT_DATA1_NVM_ERR_TYPE_WRITE(data1) \ (((data1) & \ HWRM_ASYNC_EVENT_CMPL_ERROR_REPORT_NVM_EVENT_DATA1_NVM_ERR_TYPE_MASK) == \ HWRM_ASYNC_EVENT_CMPL_ERROR_REPORT_NVM_EVENT_DATA1_NVM_ERR_TYPE_WRITE) #define EVENT_DATA1_NVM_ERR_TYPE_ERASE(data1) \ (((data1) & \ HWRM_ASYNC_EVENT_CMPL_ERROR_REPORT_NVM_EVENT_DATA1_NVM_ERR_TYPE_MASK) == \ HWRM_ASYNC_EVENT_CMPL_ERROR_REPORT_NVM_EVENT_DATA1_NVM_ERR_TYPE_ERASE) #define EVENT_DATA1_THERMAL_THRESHOLD_TYPE(data1) \ ((data1) & HWRM_ASYNC_EVENT_CMPL_ERROR_REPORT_THERMAL_EVENT_DATA1_THRESHOLD_TYPE_MASK) #define BNXT_EVENT_THERMAL_CURRENT_TEMP(data2) \ ((data2) & HWRM_ASYNC_EVENT_CMPL_ERROR_REPORT_THERMAL_EVENT_DATA2_CURRENT_TEMP_MASK) #define EVENT_DATA1_RESET_NOTIFY_FW_ACTIVATION(data1) \ (((data1) & \ HWRM_ASYNC_EVENT_CMPL_RESET_NOTIFY_EVENT_DATA1_REASON_CODE_MASK) ==\ HWRM_ASYNC_EVENT_CMPL_RESET_NOTIFY_EVENT_DATA1_REASON_CODE_FW_ACTIVATION) #define EVENT_DATA2_RESET_NOTIFY_FW_STATUS_CODE(data2) \ ((data2) & \ HWRM_ASYNC_EVENT_CMPL_RESET_NOTIFY_EVENT_DATA2_FW_STATUS_CODE_MASK) #define EVENT_DATA1_RECOVERY_ENABLED(data1) \ !!((data1) & \ HWRM_ASYNC_EVENT_CMPL_ERROR_RECOVERY_EVENT_DATA1_FLAGS_RECOVERY_ENABLED) #define EVENT_DATA1_RECOVERY_MASTER_FUNC(data1) \ !!((data1) & \ HWRM_ASYNC_EVENT_CMPL_ERROR_RECOVERY_EVENT_DATA1_FLAGS_MASTER_FUNC) #define INVALID_STATS_CTX_ID -1 /* Maximum numbers of RX and TX descriptors. iflib requires this to be a power * of two. The hardware has no particular limitation. */ #define BNXT_MAX_RXD ((INT32_MAX >> 1) + 1) #define BNXT_MAX_TXD ((INT32_MAX >> 1) + 1) #define CSUM_OFFLOAD (CSUM_IP_TSO|CSUM_IP6_TSO|CSUM_IP| \ CSUM_IP_UDP|CSUM_IP_TCP|CSUM_IP_SCTP| \ CSUM_IP6_UDP|CSUM_IP6_TCP|CSUM_IP6_SCTP) #define BNXT_MAX_MTU 9600 #define BNXT_RSS_HASH_TYPE_TCPV4 0 #define BNXT_RSS_HASH_TYPE_UDPV4 1 #define BNXT_RSS_HASH_TYPE_IPV4 2 #define BNXT_RSS_HASH_TYPE_TCPV6 3 #define BNXT_RSS_HASH_TYPE_UDPV6 4 #define BNXT_RSS_HASH_TYPE_IPV6 5 #define BNXT_GET_RSS_PROFILE_ID(rss_hash_type) ((rss_hash_type >> 1) & 0x1F) #define BNXT_NO_MORE_WOL_FILTERS 0xFFFF #define bnxt_wol_supported(softc) (!((softc)->flags & BNXT_FLAG_VF) && \ ((softc)->flags & BNXT_FLAG_WOL_CAP )) /* 64-bit doorbell */ #define DBR_INDEX_MASK 0x0000000000ffffffULL #define DBR_PI_LO_MASK 0xff000000UL #define DBR_PI_LO_SFT 24 #define DBR_EPOCH_MASK 0x01000000UL #define DBR_EPOCH_SFT 24 #define DBR_TOGGLE_MASK 0x06000000UL #define DBR_TOGGLE_SFT 25 #define DBR_XID_MASK 0x000fffff00000000ULL #define DBR_XID_SFT 32 #define DBR_PI_HI_MASK 0xf0000000000000ULL #define DBR_PI_HI_SFT 52 #define DBR_PATH_L2 (0x1ULL << 56) #define DBR_VALID (0x1ULL << 58) #define DBR_TYPE_SQ (0x0ULL << 60) #define DBR_TYPE_RQ (0x1ULL << 60) #define DBR_TYPE_SRQ (0x2ULL << 60) #define DBR_TYPE_SRQ_ARM (0x3ULL << 60) #define DBR_TYPE_CQ (0x4ULL << 60) #define DBR_TYPE_CQ_ARMSE (0x5ULL << 60) #define DBR_TYPE_CQ_ARMALL (0x6ULL << 60) #define DBR_TYPE_CQ_ARMENA (0x7ULL << 60) #define DBR_TYPE_SRQ_ARMENA (0x8ULL << 60) #define DBR_TYPE_CQ_CUTOFF_ACK (0x9ULL << 60) #define DBR_TYPE_NQ (0xaULL << 60) #define DBR_TYPE_NQ_ARM (0xbULL << 60) #define DBR_TYPE_PUSH_START (0xcULL << 60) #define DBR_TYPE_PUSH_END (0xdULL << 60) #define DBR_TYPE_NQ_MASK (0xeULL << 60) #define DBR_TYPE_NULL (0xfULL << 60) #define BNXT_MAX_L2_QUEUES 128 #define BNXT_ROCE_IRQ_COUNT 9 #define BNXT_MAX_NUM_QUEUES (BNXT_MAX_L2_QUEUES + BNXT_ROCE_IRQ_COUNT) /* Completion related defines */ #define CMP_VALID(cmp, v_bit) \ ((!!(((struct cmpl_base *)(cmp))->info3_v & htole32(CMPL_BASE_V))) == !!(v_bit) ) /* Chip class phase 5 */ #define BNXT_CHIP_P5(sc) ((sc->flags & BNXT_FLAG_CHIP_P5)) /* Chip class phase 7 */ #define BNXT_CHIP_P7(sc) ((sc->flags & BNXT_FLAG_CHIP_P7)) /* Chip class phase 5 plus */ #define BNXT_CHIP_P5_PLUS(sc) \ (BNXT_CHIP_P5(sc) || BNXT_CHIP_P7(sc)) #define DB_PF_OFFSET_P5 0x10000 #define DB_VF_OFFSET_P5 0x4000 #define NQ_VALID(cmp, v_bit) \ ((!!(((nq_cn_t *)(cmp))->v & htole32(NQ_CN_V))) == !!(v_bit) ) #ifndef DIV_ROUND_UP #define DIV_ROUND_UP(n, d) (((n) + (d) - 1) / (d)) #endif #ifndef roundup #define roundup(x, y) ((((x) + ((y) - 1)) / (y)) * (y)) #endif #define NEXT_CP_CONS_V(ring, cons, v_bit) do { \ if (__predict_false(++(cons) == (ring)->ring_size)) \ ((cons) = 0, (v_bit) = !v_bit); \ } while (0) #define RING_NEXT(ring, idx) (__predict_false(idx + 1 == (ring)->ring_size) ? \ 0 : idx + 1) #define CMPL_PREFETCH_NEXT(cpr, idx) \ __builtin_prefetch(&((struct cmpl_base *)(cpr)->ring.vaddr)[((idx) +\ (CACHE_LINE_SIZE / sizeof(struct cmpl_base))) & \ ((cpr)->ring.ring_size - 1)]) /* Lock macros */ #define BNXT_HWRM_LOCK_INIT(_softc, _name) \ mtx_init(&(_softc)->hwrm_lock, _name, "BNXT HWRM Lock", MTX_DEF) #define BNXT_HWRM_LOCK(_softc) mtx_lock(&(_softc)->hwrm_lock) #define BNXT_HWRM_UNLOCK(_softc) mtx_unlock(&(_softc)->hwrm_lock) #define BNXT_HWRM_LOCK_DESTROY(_softc) mtx_destroy(&(_softc)->hwrm_lock) #define BNXT_HWRM_LOCK_ASSERT(_softc) mtx_assert(&(_softc)->hwrm_lock, \ MA_OWNED) #define BNXT_IS_FLOW_CTRL_CHANGED(link_info) \ ((link_info->last_flow_ctrl.tx != link_info->flow_ctrl.tx) || \ (link_info->last_flow_ctrl.rx != link_info->flow_ctrl.rx) || \ (link_info->last_flow_ctrl.autoneg != link_info->flow_ctrl.autoneg)) /* Chip info */ #define BNXT_TSO_SIZE UINT16_MAX #define min_t(type, x, y) ({ \ type __min1 = (x); \ type __min2 = (y); \ __min1 < __min2 ? __min1 : __min2; }) #define max_t(type, x, y) ({ \ type __max1 = (x); \ type __max2 = (y); \ __max1 > __max2 ? __max1 : __max2; }) #define clamp_t(type, _x, min, max) min_t(type, max_t(type, _x, min), max) #define BNXT_IFMEDIA_ADD(supported, fw_speed, ifm_speed) do { \ if ((supported) & HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_ ## fw_speed) \ ifmedia_add(softc->media, IFM_ETHER | (ifm_speed), 0, NULL); \ } while(0) #define BNXT_MIN_FRAME_SIZE 52 /* Frames must be padded to this size for some A0 chips */ #define BNXT_RX_STATS_EXT_OFFSET(counter) \ (offsetof(struct rx_port_stats_ext, counter) / 8) #define BNXT_RX_STATS_EXT_NUM_LEGACY \ BNXT_RX_STATS_EXT_OFFSET(rx_fec_corrected_blocks) #define BNXT_TX_STATS_EXT_OFFSET(counter) \ (offsetof(struct tx_port_stats_ext, counter) / 8) extern const char bnxt_driver_version[]; typedef void (*bnxt_doorbell_tx)(void *, uint16_t idx); typedef void (*bnxt_doorbell_rx)(void *, uint16_t idx); typedef void (*bnxt_doorbell_rx_cq)(void *, bool); typedef void (*bnxt_doorbell_tx_cq)(void *, bool); typedef void (*bnxt_doorbell_nq)(void *, bool); typedef struct bnxt_doorbell_ops { bnxt_doorbell_tx bnxt_db_tx; bnxt_doorbell_rx bnxt_db_rx; bnxt_doorbell_rx_cq bnxt_db_rx_cq; bnxt_doorbell_tx_cq bnxt_db_tx_cq; bnxt_doorbell_nq bnxt_db_nq; } bnxt_dooorbell_ops_t; /* NVRAM access */ enum bnxt_nvm_directory_type { BNX_DIR_TYPE_UNUSED = 0, BNX_DIR_TYPE_PKG_LOG = 1, BNX_DIR_TYPE_UPDATE = 2, BNX_DIR_TYPE_CHIMP_PATCH = 3, BNX_DIR_TYPE_BOOTCODE = 4, BNX_DIR_TYPE_VPD = 5, BNX_DIR_TYPE_EXP_ROM_MBA = 6, BNX_DIR_TYPE_AVS = 7, BNX_DIR_TYPE_PCIE = 8, BNX_DIR_TYPE_PORT_MACRO = 9, BNX_DIR_TYPE_APE_FW = 10, BNX_DIR_TYPE_APE_PATCH = 11, BNX_DIR_TYPE_KONG_FW = 12, BNX_DIR_TYPE_KONG_PATCH = 13, BNX_DIR_TYPE_BONO_FW = 14, BNX_DIR_TYPE_BONO_PATCH = 15, BNX_DIR_TYPE_TANG_FW = 16, BNX_DIR_TYPE_TANG_PATCH = 17, BNX_DIR_TYPE_BOOTCODE_2 = 18, BNX_DIR_TYPE_CCM = 19, BNX_DIR_TYPE_PCI_CFG = 20, BNX_DIR_TYPE_TSCF_UCODE = 21, BNX_DIR_TYPE_ISCSI_BOOT = 22, BNX_DIR_TYPE_ISCSI_BOOT_IPV6 = 24, BNX_DIR_TYPE_ISCSI_BOOT_IPV4N6 = 25, BNX_DIR_TYPE_ISCSI_BOOT_CFG6 = 26, BNX_DIR_TYPE_EXT_PHY = 27, BNX_DIR_TYPE_SHARED_CFG = 40, BNX_DIR_TYPE_PORT_CFG = 41, BNX_DIR_TYPE_FUNC_CFG = 42, BNX_DIR_TYPE_MGMT_CFG = 48, BNX_DIR_TYPE_MGMT_DATA = 49, BNX_DIR_TYPE_MGMT_WEB_DATA = 50, BNX_DIR_TYPE_MGMT_WEB_META = 51, BNX_DIR_TYPE_MGMT_EVENT_LOG = 52, BNX_DIR_TYPE_MGMT_AUDIT_LOG = 53 }; enum bnxnvm_pkglog_field_index { BNX_PKG_LOG_FIELD_IDX_INSTALLED_TIMESTAMP = 0, BNX_PKG_LOG_FIELD_IDX_PKG_DESCRIPTION = 1, BNX_PKG_LOG_FIELD_IDX_PKG_VERSION = 2, BNX_PKG_LOG_FIELD_IDX_PKG_TIMESTAMP = 3, BNX_PKG_LOG_FIELD_IDX_PKG_CHECKSUM = 4, BNX_PKG_LOG_FIELD_IDX_INSTALLED_ITEMS = 5, BNX_PKG_LOG_FIELD_IDX_INSTALLED_MASK = 6 }; #define BNX_DIR_ORDINAL_FIRST 0 #define BNX_DIR_EXT_NONE 0 struct bnxt_bar_info { struct resource *res; bus_space_tag_t tag; bus_space_handle_t handle; bus_size_t size; int rid; }; struct bnxt_flow_ctrl { bool rx; bool tx; bool autoneg; }; struct bnxt_link_info { uint8_t media_type; uint8_t transceiver; uint8_t phy_addr; uint8_t phy_link_status; uint8_t wire_speed; uint8_t loop_back; uint8_t link_up; uint8_t last_link_up; uint8_t duplex; uint8_t last_duplex; uint8_t last_phy_type; struct bnxt_flow_ctrl flow_ctrl; struct bnxt_flow_ctrl last_flow_ctrl; uint8_t duplex_setting; uint8_t auto_mode; #define PHY_VER_LEN 3 uint8_t phy_ver[PHY_VER_LEN]; uint8_t phy_type; #define BNXT_PHY_STATE_ENABLED 0 #define BNXT_PHY_STATE_DISABLED 1 uint8_t phy_state; uint16_t link_speed; uint16_t support_speeds; uint16_t support_speeds2; uint16_t support_pam4_speeds; uint16_t auto_link_speeds; uint16_t auto_link_speeds2; uint16_t auto_pam4_link_speeds; uint16_t force_link_speed; uint16_t force_link_speeds2; uint16_t force_pam4_link_speed; bool force_pam4_speed; bool force_speed2_nrz; bool force_pam4_56_speed2; bool force_pam4_112_speed2; uint16_t advertising; uint16_t advertising_pam4; uint32_t preemphasis; uint16_t support_auto_speeds; uint16_t support_force_speeds; uint16_t support_pam4_auto_speeds; uint16_t support_pam4_force_speeds; uint16_t support_auto_speeds2; uint16_t support_force_speeds2; #define BNXT_SIG_MODE_NRZ HWRM_PORT_PHY_QCFG_OUTPUT_SIGNAL_MODE_NRZ #define BNXT_SIG_MODE_PAM4 HWRM_PORT_PHY_QCFG_OUTPUT_SIGNAL_MODE_PAM4 #define BNXT_SIG_MODE_PAM4_112 HWRM_PORT_PHY_QCFG_OUTPUT_SIGNAL_MODE_PAM4_112 uint8_t req_signal_mode; uint8_t active_fec_sig_mode; uint8_t sig_mode; /* copy of requested setting */ uint8_t autoneg; #define BNXT_AUTONEG_SPEED 1 #define BNXT_AUTONEG_FLOW_CTRL 2 uint8_t req_duplex; uint16_t req_link_speed; uint8_t module_status; struct hwrm_port_phy_qcfg_output phy_qcfg_resp; uint8_t active_lanes; }; enum bnxt_phy_type { BNXT_MEDIA_CR = 0, BNXT_MEDIA_LR, BNXT_MEDIA_SR, BNXT_MEDIA_ER, BNXT_MEDIA_KR, BNXT_MEDIA_AC, BNXT_MEDIA_BASECX, BNXT_MEDIA_BASET, BNXT_MEDIA_BASEKX, BNXT_MEDIA_BASESGMII, BNXT_MEDIA_END }; enum bnxt_cp_type { BNXT_DEFAULT, BNXT_TX, BNXT_RX, BNXT_SHARED }; struct bnxt_queue_info { uint8_t queue_id; uint8_t queue_profile; }; struct bnxt_func_info { uint32_t fw_fid; uint8_t mac_addr[ETHER_ADDR_LEN]; uint16_t max_rsscos_ctxs; uint16_t max_cp_rings; uint16_t max_tx_rings; uint16_t max_rx_rings; uint16_t max_hw_ring_grps; uint16_t max_irqs; uint16_t max_l2_ctxs; uint16_t max_vnics; uint16_t max_stat_ctxs; }; struct bnxt_pf_info { #define BNXT_FIRST_PF_FID 1 #define BNXT_FIRST_VF_FID 128 uint8_t port_id; uint32_t first_vf_id; uint16_t active_vfs; uint16_t max_vfs; uint32_t max_encap_records; uint32_t max_decap_records; uint32_t max_tx_em_flows; uint32_t max_tx_wm_flows; uint32_t max_rx_em_flows; uint32_t max_rx_wm_flows; unsigned long *vf_event_bmap; uint16_t hwrm_cmd_req_pages; void *hwrm_cmd_req_addr[4]; bus_addr_t hwrm_cmd_req_dma_addr[4]; }; struct bnxt_vf_info { uint16_t fw_fid; uint8_t mac_addr[ETHER_ADDR_LEN]; uint16_t max_rsscos_ctxs; uint16_t max_cp_rings; uint16_t max_tx_rings; uint16_t max_rx_rings; uint16_t max_hw_ring_grps; uint16_t max_l2_ctxs; uint16_t max_irqs; uint16_t max_vnics; uint16_t max_stat_ctxs; uint32_t vlan; #define BNXT_VF_QOS 0x1 #define BNXT_VF_SPOOFCHK 0x2 #define BNXT_VF_LINK_FORCED 0x4 #define BNXT_VF_LINK_UP 0x8 uint32_t flags; uint32_t func_flags; /* func cfg flags */ uint32_t min_tx_rate; uint32_t max_tx_rate; void *hwrm_cmd_req_addr; bus_addr_t hwrm_cmd_req_dma_addr; }; #define BNXT_PF(softc) (!((softc)->flags & BNXT_FLAG_VF)) #define BNXT_VF(softc) ((softc)->flags & BNXT_FLAG_VF) struct bnxt_vlan_tag { SLIST_ENTRY(bnxt_vlan_tag) next; uint64_t filter_id; uint16_t tag; }; struct bnxt_vnic_info { uint16_t id; uint16_t def_ring_grp; uint16_t cos_rule; uint16_t lb_rule; uint16_t mru; uint32_t rx_mask; struct iflib_dma_info mc_list; int mc_list_count; #define BNXT_MAX_MC_ADDRS 16 uint32_t flags; #define BNXT_VNIC_FLAG_DEFAULT 0x01 #define BNXT_VNIC_FLAG_BD_STALL 0x02 #define BNXT_VNIC_FLAG_VLAN_STRIP 0x04 uint64_t filter_id; uint16_t rss_id; uint32_t rss_hash_type; uint8_t rss_hash_key[HW_HASH_KEY_SIZE]; struct iflib_dma_info rss_hash_key_tbl; struct iflib_dma_info rss_grp_tbl; SLIST_HEAD(vlan_head, bnxt_vlan_tag) vlan_tags; struct iflib_dma_info vlan_tag_list; }; struct bnxt_grp_info { uint16_t stats_ctx; uint16_t grp_id; uint16_t rx_ring_id; uint16_t cp_ring_id; uint16_t ag_ring_id; }; #define EPOCH_ARR_SZ 4096 struct bnxt_ring { uint64_t paddr; vm_offset_t doorbell; caddr_t vaddr; struct bnxt_softc *softc; uint32_t ring_size; /* Must be a power of two */ uint16_t id; /* Logical ID */ uint16_t phys_id; uint16_t idx; struct bnxt_full_tpa_start *tpa_start; union { u64 db_key64; u32 db_key32; }; uint32_t db_ring_mask; uint32_t db_epoch_mask; uint8_t db_epoch_shift; uint64_t epoch_arr[EPOCH_ARR_SZ]; bool epoch_bit; }; struct bnxt_cp_ring { struct bnxt_ring ring; struct if_irq irq; uint32_t cons; uint32_t raw_cons; bool v_bit; /* Value of valid bit */ struct ctx_hw_stats *stats; uint32_t stats_ctx_id; uint32_t last_idx; /* Used by RX rings only * set to the last read pidx */ uint64_t int_count; uint8_t toggle; uint8_t type; #define Q_TYPE_TX 1 #define Q_TYPE_RX 2 }; struct bnxt_full_tpa_start { struct rx_tpa_start_cmpl low; struct rx_tpa_start_cmpl_hi high; }; /* All the version information for the part */ #define BNXT_VERSTR_SIZE (3*3+2+1) /* ie: "255.255.255\0" */ #define BNXT_NAME_SIZE 17 #define FW_VER_STR_LEN 32 #define BC_HWRM_STR_LEN 21 struct bnxt_ver_info { uint8_t hwrm_if_major; uint8_t hwrm_if_minor; uint8_t hwrm_if_update; char hwrm_if_ver[BNXT_VERSTR_SIZE]; char driver_hwrm_if_ver[BNXT_VERSTR_SIZE]; char mgmt_fw_ver[FW_VER_STR_LEN]; char netctrl_fw_ver[FW_VER_STR_LEN]; char roce_fw_ver[FW_VER_STR_LEN]; char fw_ver_str[FW_VER_STR_LEN]; char phy_ver[BNXT_VERSTR_SIZE]; char pkg_ver[64]; char hwrm_fw_name[BNXT_NAME_SIZE]; char mgmt_fw_name[BNXT_NAME_SIZE]; char netctrl_fw_name[BNXT_NAME_SIZE]; char roce_fw_name[BNXT_NAME_SIZE]; char phy_vendor[BNXT_NAME_SIZE]; char phy_partnumber[BNXT_NAME_SIZE]; uint16_t chip_num; uint8_t chip_rev; uint8_t chip_metal; uint8_t chip_bond_id; uint8_t chip_type; uint8_t hwrm_min_major; uint8_t hwrm_min_minor; uint8_t hwrm_min_update; uint64_t fw_ver_code; #define BNXT_FW_VER_CODE(maj, min, bld, rsv) \ ((uint64_t)(maj) << 48 | (uint64_t)(min) << 32 | (uint64_t)(bld) << 16 | (rsv)) #define BNXT_FW_MAJ(softc) ((softc)->ver_info->fw_ver_code >> 48) #define BNXT_FW_MIN(softc) (((softc)->ver_info->fw_ver_code >> 32) & 0xffff) #define BNXT_FW_BLD(softc) (((softc)->ver_info->fw_ver_code >> 16) & 0xffff) #define BNXT_FW_RSV(softc) (((softc)->ver_info->fw_ver_code) & 0xffff) struct sysctl_ctx_list ver_ctx; struct sysctl_oid *ver_oid; }; struct bnxt_nvram_info { uint16_t mfg_id; uint16_t device_id; uint32_t sector_size; uint32_t size; uint32_t reserved_size; uint32_t available_size; struct sysctl_ctx_list nvm_ctx; struct sysctl_oid *nvm_oid; }; struct bnxt_func_qcfg { uint16_t alloc_completion_rings; uint16_t alloc_tx_rings; uint16_t alloc_rx_rings; uint16_t alloc_vnics; }; struct bnxt_hw_lro { uint16_t enable; uint16_t is_mode_gro; uint16_t max_agg_segs; uint16_t max_aggs; uint32_t min_agg_len; }; /* The hardware supports certain page sizes. Use the supported page sizes * to allocate the rings. */ #if (PAGE_SHIFT < 12) #define BNXT_PAGE_SHIFT 12 #elif (PAGE_SHIFT <= 13) #define BNXT_PAGE_SHIFT PAGE_SHIFT #elif (PAGE_SHIFT < 16) #define BNXT_PAGE_SHIFT 13 #else #define BNXT_PAGE_SHIFT 16 #endif #define BNXT_PAGE_SIZE (1 << BNXT_PAGE_SHIFT) #define MAX_CTX_PAGES (BNXT_PAGE_SIZE / 8) #define MAX_CTX_TOTAL_PAGES (MAX_CTX_PAGES * MAX_CTX_PAGES) struct bnxt_ring_mem_info { int nr_pages; int page_size; uint16_t flags; #define BNXT_RMEM_VALID_PTE_FLAG 1 #define BNXT_RMEM_RING_PTE_FLAG 2 #define BNXT_RMEM_USE_FULL_PAGE_FLAG 4 uint16_t depth; struct bnxt_ctx_mem_type *ctx_mem; struct iflib_dma_info *pg_arr; struct iflib_dma_info pg_tbl; int vmem_size; void **vmem; }; struct bnxt_ctx_pg_info { uint32_t entries; uint32_t nr_pages; struct iflib_dma_info ctx_arr[MAX_CTX_PAGES]; struct bnxt_ring_mem_info ring_mem; struct bnxt_ctx_pg_info **ctx_pg_tbl; }; #define BNXT_MAX_TQM_SP_RINGS 1 #define BNXT_MAX_TQM_FP_LEGACY_RINGS 8 #define BNXT_MAX_TQM_FP_RINGS 9 #define BNXT_MAX_TQM_LEGACY_RINGS \ (BNXT_MAX_TQM_SP_RINGS + BNXT_MAX_TQM_FP_LEGACY_RINGS) #define BNXT_MAX_TQM_RINGS \ (BNXT_MAX_TQM_SP_RINGS + BNXT_MAX_TQM_FP_RINGS) #define BNXT_BACKING_STORE_CFG_LEGACY_LEN 256 #define BNXT_BACKING_STORE_CFG_LEN \ sizeof(struct hwrm_func_backing_store_cfg_input) #define BNXT_SET_CTX_PAGE_ATTR(attr) \ do { \ if (BNXT_PAGE_SIZE == 0x2000) \ attr = HWRM_FUNC_BACKING_STORE_CFG_INPUT_SRQ_PG_SIZE_PG_8K; \ else if (BNXT_PAGE_SIZE == 0x10000) \ attr = HWRM_FUNC_BACKING_STORE_CFG_INPUT_SRQ_PG_SIZE_PG_64K; \ else \ attr = HWRM_FUNC_BACKING_STORE_CFG_INPUT_SRQ_PG_SIZE_PG_4K; \ } while (0) struct bnxt_ctx_mem_type { u16 type; u16 entry_size; u32 flags; #define BNXT_CTX_MEM_TYPE_VALID HWRM_FUNC_BACKING_STORE_QCAPS_V2_OUTPUT_FLAGS_TYPE_VALID u32 instance_bmap; u8 init_value; u8 entry_multiple; u16 init_offset; #define BNXT_CTX_INIT_INVALID_OFFSET 0xffff u32 max_entries; u32 min_entries; u8 last:1; u8 mem_valid:1; u8 split_entry_cnt; #define BNXT_MAX_SPLIT_ENTRY 4 union { struct { u32 qp_l2_entries; u32 qp_qp1_entries; }; u32 srq_l2_entries; u32 cq_l2_entries; u32 vnic_entries; struct { u32 mrav_av_entries; u32 mrav_num_entries_units; }; u32 split[BNXT_MAX_SPLIT_ENTRY]; }; struct bnxt_ctx_pg_info *pg_info; }; #define BNXT_CTX_QP HWRM_FUNC_BACKING_STORE_CFG_V2_INPUT_TYPE_QP #define BNXT_CTX_SRQ HWRM_FUNC_BACKING_STORE_CFG_V2_INPUT_TYPE_SRQ #define BNXT_CTX_CQ HWRM_FUNC_BACKING_STORE_CFG_V2_INPUT_TYPE_CQ #define BNXT_CTX_VNIC HWRM_FUNC_BACKING_STORE_CFG_V2_INPUT_TYPE_VNIC #define BNXT_CTX_STAT HWRM_FUNC_BACKING_STORE_CFG_V2_INPUT_TYPE_STAT #define BNXT_CTX_STQM HWRM_FUNC_BACKING_STORE_CFG_V2_INPUT_TYPE_SP_TQM_RING #define BNXT_CTX_FTQM HWRM_FUNC_BACKING_STORE_CFG_V2_INPUT_TYPE_FP_TQM_RING #define BNXT_CTX_MRAV HWRM_FUNC_BACKING_STORE_CFG_V2_INPUT_TYPE_MRAV #define BNXT_CTX_TIM HWRM_FUNC_BACKING_STORE_CFG_V2_INPUT_TYPE_TIM #define BNXT_CTX_TKC HWRM_FUNC_BACKING_STORE_CFG_V2_INPUT_TYPE_TKC #define BNXT_CTX_RKC HWRM_FUNC_BACKING_STORE_CFG_V2_INPUT_TYPE_RKC #define BNXT_CTX_MTQM HWRM_FUNC_BACKING_STORE_CFG_V2_INPUT_TYPE_MP_TQM_RING #define BNXT_CTX_SQDBS HWRM_FUNC_BACKING_STORE_CFG_V2_INPUT_TYPE_SQ_DB_SHADOW #define BNXT_CTX_RQDBS HWRM_FUNC_BACKING_STORE_CFG_V2_INPUT_TYPE_RQ_DB_SHADOW #define BNXT_CTX_SRQDBS HWRM_FUNC_BACKING_STORE_CFG_V2_INPUT_TYPE_SRQ_DB_SHADOW #define BNXT_CTX_CQDBS HWRM_FUNC_BACKING_STORE_CFG_V2_INPUT_TYPE_CQ_DB_SHADOW #define BNXT_CTX_QTKC HWRM_FUNC_BACKING_STORE_CFG_V2_INPUT_TYPE_QUIC_TKC #define BNXT_CTX_QRKC HWRM_FUNC_BACKING_STORE_CFG_V2_INPUT_TYPE_QUIC_RKC #define BNXT_CTX_MAX (BNXT_CTX_TIM + 1) #define BNXT_CTX_L2_MAX (BNXT_CTX_FTQM + 1) #define BNXT_CTX_V2_MAX (HWRM_FUNC_BACKING_STORE_CFG_V2_INPUT_TYPE_ROCE_HWRM_TRACE + 1) #define BNXT_CTX_SRT_TRACE HWRM_FUNC_BACKING_STORE_QCFG_V2_OUTPUT_TYPE_SRT_TRACE #define BNXT_CTX_ROCE_HWRM_TRACE HWRM_FUNC_BACKING_STORE_CFG_V2_INPUT_TYPE_ROCE_HWRM_TRACE #define BNXT_CTX_INV ((u16)-1) struct bnxt_ctx_mem_info { u8 tqm_fp_rings_count; u32 flags; #define BNXT_CTX_FLAG_INITED 0x01 struct bnxt_ctx_mem_type ctx_arr[BNXT_CTX_V2_MAX]; }; struct bnxt_hw_resc { uint16_t min_rsscos_ctxs; uint16_t max_rsscos_ctxs; uint16_t min_cp_rings; uint16_t max_cp_rings; uint16_t resv_cp_rings; uint16_t min_tx_rings; uint16_t max_tx_rings; uint16_t resv_tx_rings; uint16_t max_tx_sch_inputs; uint16_t min_rx_rings; uint16_t max_rx_rings; uint16_t resv_rx_rings; uint16_t min_hw_ring_grps; uint16_t max_hw_ring_grps; uint16_t resv_hw_ring_grps; uint16_t min_l2_ctxs; uint16_t max_l2_ctxs; uint16_t min_vnics; uint16_t max_vnics; uint16_t resv_vnics; uint16_t min_stat_ctxs; uint16_t max_stat_ctxs; uint16_t resv_stat_ctxs; uint16_t max_nqs; uint16_t max_irqs; uint16_t resv_irqs; }; enum bnxt_type_ets { BNXT_TYPE_ETS_TSA = 0, BNXT_TYPE_ETS_PRI2TC, BNXT_TYPE_ETS_TCBW, BNXT_TYPE_ETS_MAX }; static const char *const BNXT_ETS_TYPE_STR[] = { "tsa", "pri2tc", "tcbw", }; static const char *const BNXT_ETS_HELP_STR[] = { "X is 1 (strict), 0 (ets)", "TC values for pri 0 to 7", "TC BW values for pri 0 to 7, Sum should be 100", }; #define BNXT_HWRM_MAX_REQ_LEN (softc->hwrm_max_req_len) struct bnxt_softc_list { SLIST_ENTRY(bnxt_softc_list) next; struct bnxt_softc *softc; }; #ifndef BIT_ULL #define BIT_ULL(nr) (1ULL << (nr)) #endif struct bnxt_aux_dev { struct auxiliary_device aux_dev; struct bnxt_en_dev *edev; int id; }; struct bnxt_msix_tbl { uint32_t entry; uint32_t vector; }; enum bnxt_health_severity { SEVERITY_NORMAL = 0, SEVERITY_WARNING, SEVERITY_RECOVERABLE, SEVERITY_FATAL, }; enum bnxt_health_remedy { REMEDY_DEVLINK_RECOVER, REMEDY_POWER_CYCLE_DEVICE, REMEDY_POWER_CYCLE_HOST, REMEDY_FW_UPDATE, REMEDY_HW_REPLACE, }; struct bnxt_fw_health { u32 flags; u32 polling_dsecs; u32 master_func_wait_dsecs; u32 normal_func_wait_dsecs; u32 post_reset_wait_dsecs; u32 post_reset_max_wait_dsecs; u32 regs[4]; u32 mapped_regs[4]; #define BNXT_FW_HEALTH_REG 0 #define BNXT_FW_HEARTBEAT_REG 1 #define BNXT_FW_RESET_CNT_REG 2 #define BNXT_FW_RESET_INPROG_REG 3 u32 fw_reset_inprog_reg_mask; u32 last_fw_heartbeat; u32 last_fw_reset_cnt; u8 enabled:1; u8 primary:1; u8 status_reliable:1; u8 resets_reliable:1; u8 tmr_multiplier; u8 tmr_counter; u8 fw_reset_seq_cnt; u32 fw_reset_seq_regs[16]; u32 fw_reset_seq_vals[16]; u32 fw_reset_seq_delay_msec[16]; u32 echo_req_data1; u32 echo_req_data2; struct devlink_health_reporter *fw_reporter; struct mutex lock; enum bnxt_health_severity severity; enum bnxt_health_remedy remedy; u32 arrests; u32 discoveries; u32 survivals; u32 fatalities; u32 diagnoses; }; #define BNXT_FW_HEALTH_REG_TYPE_MASK 3 #define BNXT_FW_HEALTH_REG_TYPE_CFG 0 #define BNXT_FW_HEALTH_REG_TYPE_GRC 1 #define BNXT_FW_HEALTH_REG_TYPE_BAR0 2 #define BNXT_FW_HEALTH_REG_TYPE_BAR1 3 #define BNXT_FW_HEALTH_REG_TYPE(reg) ((reg) & BNXT_FW_HEALTH_REG_TYPE_MASK) #define BNXT_FW_HEALTH_REG_OFF(reg) ((reg) & ~BNXT_FW_HEALTH_REG_TYPE_MASK) #define BNXT_FW_HEALTH_WIN_BASE 0x3000 #define BNXT_FW_HEALTH_WIN_MAP_OFF 8 #define BNXT_FW_HEALTH_WIN_OFF(reg) (BNXT_FW_HEALTH_WIN_BASE + \ ((reg) & BNXT_GRC_OFFSET_MASK)) #define BNXT_FW_STATUS_HEALTH_MSK 0xffff #define BNXT_FW_STATUS_HEALTHY 0x8000 #define BNXT_FW_STATUS_SHUTDOWN 0x100000 #define BNXT_FW_STATUS_RECOVERING 0x400000 #define BNXT_FW_IS_HEALTHY(sts) (((sts) & BNXT_FW_STATUS_HEALTH_MSK) ==\ BNXT_FW_STATUS_HEALTHY) #define BNXT_FW_IS_BOOTING(sts) (((sts) & BNXT_FW_STATUS_HEALTH_MSK) < \ BNXT_FW_STATUS_HEALTHY) #define BNXT_FW_IS_ERR(sts) (((sts) & BNXT_FW_STATUS_HEALTH_MSK) > \ BNXT_FW_STATUS_HEALTHY) #define BNXT_FW_IS_RECOVERING(sts) (BNXT_FW_IS_ERR(sts) && \ ((sts) & BNXT_FW_STATUS_RECOVERING)) #define BNXT_FW_RETRY 5 #define BNXT_FW_IF_RETRY 10 #define BNXT_FW_SLOT_RESET_RETRY 4 #define BNXT_GRCPF_REG_CHIMP_COMM 0x0 #define BNXT_GRCPF_REG_CHIMP_COMM_TRIGGER 0x100 #define BNXT_GRCPF_REG_WINDOW_BASE_OUT 0x400 #define BNXT_GRCPF_REG_SYNC_TIME 0x480 #define BNXT_GRCPF_REG_SYNC_TIME_ADJ 0x488 #define BNXT_GRCPF_REG_SYNC_TIME_ADJ_PER_MSK 0xffffffUL #define BNXT_GRCPF_REG_SYNC_TIME_ADJ_PER_SFT 0 #define BNXT_GRCPF_REG_SYNC_TIME_ADJ_VAL_MSK 0x1f000000UL #define BNXT_GRCPF_REG_SYNC_TIME_ADJ_VAL_SFT 24 #define BNXT_GRCPF_REG_SYNC_TIME_ADJ_SIGN_MSK 0x20000000UL #define BNXT_GRCPF_REG_SYNC_TIME_ADJ_SIGN_SFT 29 #define BNXT_GRC_REG_STATUS_P5 0x520 #define BNXT_GRCPF_REG_KONG_COMM 0xA00 #define BNXT_GRCPF_REG_KONG_COMM_TRIGGER 0xB00 #define BNXT_CAG_REG_LEGACY_INT_STATUS 0x4014 #define BNXT_CAG_REG_BASE 0x300000 #define BNXT_GRC_REG_CHIP_NUM 0x48 #define BNXT_GRC_REG_BASE 0x260000 #define BNXT_TS_REG_TIMESYNC_TS0_LOWER 0x640180c #define BNXT_TS_REG_TIMESYNC_TS0_UPPER 0x6401810 #define BNXT_GRC_BASE_MASK 0xfffff000 #define BNXT_GRC_OFFSET_MASK 0x00000ffc #define NQE_CN_TYPE(type) ((type) & NQ_CN_TYPE_MASK) #define NQE_CN_TOGGLE(type) (((type) & NQ_CN_TOGGLE_MASK) >> \ NQ_CN_TOGGLE_SFT) #define DB_EPOCH(ring, idx) (((idx) & (ring)->db_epoch_mask) << \ ((ring)->db_epoch_shift)) #define DB_TOGGLE(tgl) ((tgl) << DBR_TOGGLE_SFT) #define DB_RING_IDX_CMP(ring, idx) (((idx) & (ring)->db_ring_mask) | \ DB_EPOCH(ring, idx)) #define DB_RING_IDX(ring, idx, bit) (((idx) & (ring)->db_ring_mask) | \ ((bit) << (24))) struct bnxt_softc { device_t dev; if_ctx_t ctx; if_softc_ctx_t scctx; if_shared_ctx_t sctx; if_t ifp; uint32_t domain; uint32_t bus; uint32_t slot; uint32_t function; uint32_t dev_fn; struct ifmedia *media; struct bnxt_ctx_mem_info *ctx_mem; struct bnxt_hw_resc hw_resc; struct bnxt_softc_list list; struct bnxt_bar_info hwrm_bar; struct bnxt_bar_info doorbell_bar; struct bnxt_link_info link_info; #define BNXT_FLAG_VF 0x0001 #define BNXT_FLAG_NPAR 0x0002 #define BNXT_FLAG_WOL_CAP 0x0004 #define BNXT_FLAG_SHORT_CMD 0x0008 #define BNXT_FLAG_FW_CAP_NEW_RM 0x0010 #define BNXT_FLAG_CHIP_P5 0x0020 #define BNXT_FLAG_TPA 0x0040 #define BNXT_FLAG_FW_CAP_EXT_STATS 0x0080 #define BNXT_FLAG_MULTI_HOST 0x0100 #define BNXT_FLAG_MULTI_ROOT 0x0200 #define BNXT_FLAG_ROCEV1_CAP 0x0400 #define BNXT_FLAG_ROCEV2_CAP 0x0800 #define BNXT_FLAG_ROCE_CAP (BNXT_FLAG_ROCEV1_CAP | BNXT_FLAG_ROCEV2_CAP) #define BNXT_FLAG_CHIP_P7 0x1000 uint32_t flags; #define BNXT_STATE_LINK_CHANGE (0) #define BNXT_STATE_MAX (BNXT_STATE_LINK_CHANGE + 1) bitstr_t *state_bv; uint32_t total_irqs; struct bnxt_msix_tbl *irq_tbl; struct bnxt_func_info func; struct bnxt_func_qcfg fn_qcfg; struct bnxt_pf_info pf; struct bnxt_vf_info vf; uint16_t hwrm_cmd_seq; uint32_t hwrm_cmd_timeo; /* milliseconds */ struct iflib_dma_info hwrm_cmd_resp; struct iflib_dma_info hwrm_short_cmd_req_addr; /* Interrupt info for HWRM */ struct if_irq irq; struct mtx hwrm_lock; uint16_t hwrm_max_req_len; uint16_t hwrm_max_ext_req_len; uint32_t hwrm_spec_code; #define BNXT_MAX_QUEUE 8 uint8_t max_tc; uint8_t max_lltc; struct bnxt_queue_info tx_q_info[BNXT_MAX_QUEUE]; struct bnxt_queue_info rx_q_info[BNXT_MAX_QUEUE]; uint8_t tc_to_qidx[BNXT_MAX_QUEUE]; uint8_t tx_q_ids[BNXT_MAX_QUEUE]; uint8_t rx_q_ids[BNXT_MAX_QUEUE]; uint8_t tx_max_q; uint8_t rx_max_q; uint8_t is_asym_q; struct bnxt_ieee_ets *ieee_ets; struct bnxt_ieee_pfc *ieee_pfc; uint8_t dcbx_cap; uint8_t default_pri; uint8_t max_dscp_value; uint64_t admin_ticks; struct iflib_dma_info hw_rx_port_stats; struct iflib_dma_info hw_tx_port_stats; struct rx_port_stats *rx_port_stats; struct tx_port_stats *tx_port_stats; struct iflib_dma_info hw_tx_port_stats_ext; struct iflib_dma_info hw_rx_port_stats_ext; struct tx_port_stats_ext *tx_port_stats_ext; struct rx_port_stats_ext *rx_port_stats_ext; uint16_t fw_rx_stats_ext_size; uint16_t fw_tx_stats_ext_size; uint16_t hw_ring_stats_size; uint8_t tx_pri2cos_idx[8]; uint8_t rx_pri2cos_idx[8]; bool pri2cos_valid; uint64_t tx_bytes_pri[8]; uint64_t tx_packets_pri[8]; uint64_t rx_bytes_pri[8]; uint64_t rx_packets_pri[8]; uint8_t port_count; int num_cp_rings; struct bnxt_cp_ring *nq_rings; struct bnxt_ring *tx_rings; struct bnxt_cp_ring *tx_cp_rings; struct iflib_dma_info tx_stats[BNXT_MAX_NUM_QUEUES]; int ntxqsets; struct bnxt_vnic_info vnic_info; struct bnxt_ring *ag_rings; struct bnxt_ring *rx_rings; struct bnxt_cp_ring *rx_cp_rings; struct bnxt_grp_info *grp_info; struct iflib_dma_info rx_stats[BNXT_MAX_NUM_QUEUES]; int nrxqsets; uint16_t rx_buf_size; struct bnxt_cp_ring def_cp_ring; struct bnxt_cp_ring def_nq_ring; struct iflib_dma_info def_cp_ring_mem; struct iflib_dma_info def_nq_ring_mem; struct task def_cp_task; int db_size; int db_offset; int legacy_db_size; struct bnxt_doorbell_ops db_ops; struct sysctl_ctx_list hw_stats; struct sysctl_oid *hw_stats_oid; struct sysctl_ctx_list hw_lro_ctx; struct sysctl_oid *hw_lro_oid; struct sysctl_ctx_list flow_ctrl_ctx; struct sysctl_oid *flow_ctrl_oid; struct sysctl_ctx_list dcb_ctx; struct sysctl_oid *dcb_oid; struct bnxt_ver_info *ver_info; struct bnxt_nvram_info *nvm_info; bool wol; bool is_dev_init; struct bnxt_hw_lro hw_lro; uint8_t wol_filter_id; uint16_t rx_coal_usecs; uint16_t rx_coal_usecs_irq; uint16_t rx_coal_frames; uint16_t rx_coal_frames_irq; uint16_t tx_coal_usecs; uint16_t tx_coal_usecs_irq; uint16_t tx_coal_frames; uint16_t tx_coal_frames_irq; #define BNXT_USEC_TO_COAL_TIMER(x) ((x) * 25 / 2) #define BNXT_DEF_STATS_COAL_TICKS 1000000 #define BNXT_MIN_STATS_COAL_TICKS 250000 #define BNXT_MAX_STATS_COAL_TICKS 1000000 uint64_t fw_cap; #define BNXT_FW_CAP_SHORT_CMD BIT_ULL(0) #define BNXT_FW_CAP_LLDP_AGENT BIT_ULL(1) #define BNXT_FW_CAP_DCBX_AGENT BIT_ULL(2) #define BNXT_FW_CAP_NEW_RM BIT_ULL(3) #define BNXT_FW_CAP_IF_CHANGE BIT_ULL(4) #define BNXT_FW_CAP_LINK_ADMIN BIT_ULL(5) #define BNXT_FW_CAP_VF_RES_MIN_GUARANTEED BIT_ULL(6) #define BNXT_FW_CAP_KONG_MB_CHNL BIT_ULL(7) #define BNXT_FW_CAP_ADMIN_MTU BIT_ULL(8) #define BNXT_FW_CAP_ADMIN_PF BIT_ULL(9) #define BNXT_FW_CAP_OVS_64BIT_HANDLE BIT_ULL(10) #define BNXT_FW_CAP_TRUSTED_VF BIT_ULL(11) #define BNXT_FW_CAP_VF_VNIC_NOTIFY BIT_ULL(12) #define BNXT_FW_CAP_ERROR_RECOVERY BIT_ULL(13) #define BNXT_FW_CAP_PKG_VER BIT_ULL(14) #define BNXT_FW_CAP_CFA_ADV_FLOW BIT_ULL(15) #define BNXT_FW_CAP_CFA_RFS_RING_TBL_IDX_V2 BIT_ULL(16) #define BNXT_FW_CAP_PCIE_STATS_SUPPORTED BIT_ULL(17) #define BNXT_FW_CAP_EXT_STATS_SUPPORTED BIT_ULL(18) #define BNXT_FW_CAP_SECURE_MODE BIT_ULL(19) #define BNXT_FW_CAP_ERR_RECOVER_RELOAD BIT_ULL(20) #define BNXT_FW_CAP_HOT_RESET BIT_ULL(21) #define BNXT_FW_CAP_CRASHDUMP BIT_ULL(23) #define BNXT_FW_CAP_VLAN_RX_STRIP BIT_ULL(24) #define BNXT_FW_CAP_VLAN_TX_INSERT BIT_ULL(25) #define BNXT_FW_CAP_EXT_HW_STATS_SUPPORTED BIT_ULL(26) #define BNXT_FW_CAP_CFA_EEM BIT_ULL(27) #define BNXT_FW_CAP_DBG_QCAPS BIT_ULL(29) #define BNXT_FW_CAP_RING_MONITOR BIT_ULL(30) #define BNXT_FW_CAP_ECN_STATS BIT_ULL(31) #define BNXT_FW_CAP_TRUFLOW BIT_ULL(32) #define BNXT_FW_CAP_VF_CFG_FOR_PF BIT_ULL(33) #define BNXT_FW_CAP_PTP_PPS BIT_ULL(34) #define BNXT_FW_CAP_HOT_RESET_IF BIT_ULL(35) #define BNXT_FW_CAP_LIVEPATCH BIT_ULL(36) #define BNXT_FW_CAP_NPAR_1_2 BIT_ULL(37) #define BNXT_FW_CAP_RSS_HASH_TYPE_DELTA BIT_ULL(38) #define BNXT_FW_CAP_PTP_RTC BIT_ULL(39) #define BNXT_FW_CAP_TRUFLOW_EN BIT_ULL(40) #define BNXT_TRUFLOW_EN(bp) ((bp)->fw_cap & BNXT_FW_CAP_TRUFLOW_EN) #define BNXT_FW_CAP_RX_ALL_PKT_TS BIT_ULL(41) #define BNXT_FW_CAP_BACKING_STORE_V2 BIT_ULL(42) #define BNXT_FW_CAP_DBR_SUPPORTED BIT_ULL(43) #define BNXT_FW_CAP_GENERIC_STATS BIT_ULL(44) #define BNXT_FW_CAP_DBR_PACING_SUPPORTED BIT_ULL(45) #define BNXT_FW_CAP_PTP_PTM BIT_ULL(46) #define BNXT_FW_CAP_CFA_NTUPLE_RX_EXT_IP_PROTO BIT_ULL(47) #define BNXT_FW_CAP_ENABLE_RDMA_SRIOV BIT_ULL(48) #define BNXT_FW_CAP_RSS_TCAM BIT_ULL(49) #define BNXT_FW_CAP_SW_MAX_RESOURCE_LIMITS BIT_ULL(61) #define BNXT_SW_RES_LMT(bp) ((bp)->fw_cap & BNXT_FW_CAP_SW_MAX_RESOURCE_LIMITS) uint32_t lpi_tmr_lo; uint32_t lpi_tmr_hi; /* copied from flags and flags2 in hwrm_port_phy_qcaps_output */ uint16_t phy_flags; #define BNXT_PHY_FL_EEE_CAP HWRM_PORT_PHY_QCAPS_OUTPUT_FLAGS_EEE_SUPPORTED #define BNXT_PHY_FL_EXT_LPBK HWRM_PORT_PHY_QCAPS_OUTPUT_FLAGS_EXTERNAL_LPBK_SUPPORTED #define BNXT_PHY_FL_AN_PHY_LPBK HWRM_PORT_PHY_QCAPS_OUTPUT_FLAGS_AUTONEG_LPBK_SUPPORTED #define BNXT_PHY_FL_SHARED_PORT_CFG HWRM_PORT_PHY_QCAPS_OUTPUT_FLAGS_SHARED_PHY_CFG_SUPPORTED #define BNXT_PHY_FL_PORT_STATS_NO_RESET HWRM_PORT_PHY_QCAPS_OUTPUT_FLAGS_CUMULATIVE_COUNTERS_ON_RESET #define BNXT_PHY_FL_NO_PHY_LPBK HWRM_PORT_PHY_QCAPS_OUTPUT_FLAGS_LOCAL_LPBK_NOT_SUPPORTED #define BNXT_PHY_FL_FW_MANAGED_LKDN HWRM_PORT_PHY_QCAPS_OUTPUT_FLAGS_FW_MANAGED_LINK_DOWN #define BNXT_PHY_FL_NO_FCS HWRM_PORT_PHY_QCAPS_OUTPUT_FLAGS_NO_FCS #define BNXT_PHY_FL_NO_PAUSE (HWRM_PORT_PHY_QCAPS_OUTPUT_FLAGS2_PAUSE_UNSUPPORTED << 8) #define BNXT_PHY_FL_NO_PFC (HWRM_PORT_PHY_QCAPS_OUTPUT_FLAGS2_PFC_UNSUPPORTED << 8) #define BNXT_PHY_FL_BANK_SEL (HWRM_PORT_PHY_QCAPS_OUTPUT_FLAGS2_BANK_ADDR_SUPPORTED << 8) #define BNXT_PHY_FL_SPEEDS2 (HWRM_PORT_PHY_QCAPS_OUTPUT_FLAGS2_SPEEDS2_SUPPORTED << 8) struct bnxt_aux_dev *aux_dev; struct net_device *net_dev; struct mtx en_ops_lock; uint8_t port_partition_type; struct bnxt_en_dev *edev; unsigned long state; #define BNXT_STATE_OPEN 0 #define BNXT_STATE_IN_SP_TASK 1 #define BNXT_STATE_READ_STATS 2 #define BNXT_STATE_FW_RESET_DET 3 #define BNXT_STATE_IN_FW_RESET 4 #define BNXT_STATE_ABORT_ERR 5 #define BNXT_STATE_FW_FATAL_COND 6 #define BNXT_STATE_DRV_REGISTERED 7 #define BNXT_STATE_PCI_CHANNEL_IO_FROZEN 8 #define BNXT_STATE_NAPI_DISABLED 9 #define BNXT_STATE_L2_FILTER_RETRY 10 #define BNXT_STATE_FW_ACTIVATE 11 #define BNXT_STATE_RECOVER 12 #define BNXT_STATE_FW_NON_FATAL_COND 13 #define BNXT_STATE_FW_ACTIVATE_RESET 14 #define BNXT_STATE_HALF_OPEN 15 #define BNXT_NO_FW_ACCESS(bp) \ test_bit(BNXT_STATE_FW_FATAL_COND, &(bp)->state) struct pci_dev *pdev; struct work_struct sp_task; unsigned long sp_event; #define BNXT_RX_MASK_SP_EVENT 0 #define BNXT_RX_NTP_FLTR_SP_EVENT 1 #define BNXT_LINK_CHNG_SP_EVENT 2 #define BNXT_HWRM_EXEC_FWD_REQ_SP_EVENT 3 #define BNXT_VXLAN_ADD_PORT_SP_EVENT 4 #define BNXT_VXLAN_DEL_PORT_SP_EVENT 5 #define BNXT_RESET_TASK_SP_EVENT 6 #define BNXT_RST_RING_SP_EVENT 7 #define BNXT_HWRM_PF_UNLOAD_SP_EVENT 8 #define BNXT_PERIODIC_STATS_SP_EVENT 9 #define BNXT_HWRM_PORT_MODULE_SP_EVENT 10 #define BNXT_RESET_TASK_SILENT_SP_EVENT 11 #define BNXT_GENEVE_ADD_PORT_SP_EVENT 12 #define BNXT_GENEVE_DEL_PORT_SP_EVENT 13 #define BNXT_LINK_SPEED_CHNG_SP_EVENT 14 #define BNXT_FLOW_STATS_SP_EVENT 15 #define BNXT_UPDATE_PHY_SP_EVENT 16 #define BNXT_RING_COAL_NOW_SP_EVENT 17 #define BNXT_FW_RESET_NOTIFY_SP_EVENT 18 #define BNXT_FW_EXCEPTION_SP_EVENT 19 #define BNXT_VF_VNIC_CHANGE_SP_EVENT 20 #define BNXT_LINK_CFG_CHANGE_SP_EVENT 21 #define BNXT_PTP_CURRENT_TIME_EVENT 22 #define BNXT_FW_ECHO_REQUEST_SP_EVENT 23 #define BNXT_VF_CFG_CHNG_SP_EVENT 24 struct delayed_work fw_reset_task; int fw_reset_state; #define BNXT_FW_RESET_STATE_POLL_VF 1 #define BNXT_FW_RESET_STATE_RESET_FW 2 #define BNXT_FW_RESET_STATE_ENABLE_DEV 3 #define BNXT_FW_RESET_STATE_POLL_FW 4 #define BNXT_FW_RESET_STATE_OPENING 5 #define BNXT_FW_RESET_STATE_POLL_FW_DOWN 6 u16 fw_reset_min_dsecs; #define BNXT_DFLT_FW_RST_MIN_DSECS 20 u16 fw_reset_max_dsecs; #define BNXT_DFLT_FW_RST_MAX_DSECS 60 unsigned long fw_reset_timestamp; struct bnxt_fw_health *fw_health; + char board_partno[64]; }; struct bnxt_filter_info { STAILQ_ENTRY(bnxt_filter_info) next; uint64_t fw_l2_filter_id; #define INVALID_MAC_INDEX ((uint16_t)-1) uint16_t mac_index; /* Filter Characteristics */ uint32_t flags; uint32_t enables; uint8_t l2_addr[ETHER_ADDR_LEN]; uint8_t l2_addr_mask[ETHER_ADDR_LEN]; uint16_t l2_ovlan; uint16_t l2_ovlan_mask; uint16_t l2_ivlan; uint16_t l2_ivlan_mask; uint8_t t_l2_addr[ETHER_ADDR_LEN]; uint8_t t_l2_addr_mask[ETHER_ADDR_LEN]; uint16_t t_l2_ovlan; uint16_t t_l2_ovlan_mask; uint16_t t_l2_ivlan; uint16_t t_l2_ivlan_mask; uint8_t tunnel_type; uint16_t mirror_vnic_id; uint32_t vni; uint8_t pri_hint; uint64_t l2_filter_id_hint; }; #define I2C_DEV_ADDR_A0 0xa0 #define BNXT_MAX_PHY_I2C_RESP_SIZE 64 /* Function declarations */ void bnxt_report_link(struct bnxt_softc *softc); bool bnxt_check_hwrm_version(struct bnxt_softc *softc); struct bnxt_softc *bnxt_find_dev(uint32_t domain, uint32_t bus, uint32_t dev_fn, char *name); int bnxt_read_sfp_module_eeprom_info(struct bnxt_softc *bp, uint16_t i2c_addr, uint16_t page_number, uint8_t bank, bool bank_sel_en, uint16_t start_addr, uint16_t data_length, uint8_t *buf); void bnxt_dcb_init(struct bnxt_softc *softc); void bnxt_dcb_free(struct bnxt_softc *softc); uint8_t bnxt_dcb_setdcbx(struct bnxt_softc *softc, uint8_t mode); uint8_t bnxt_dcb_getdcbx(struct bnxt_softc *softc); int bnxt_dcb_ieee_getets(struct bnxt_softc *softc, struct bnxt_ieee_ets *ets); int bnxt_dcb_ieee_setets(struct bnxt_softc *softc, struct bnxt_ieee_ets *ets); uint8_t get_phy_type(struct bnxt_softc *softc); int bnxt_dcb_ieee_getpfc(struct bnxt_softc *softc, struct bnxt_ieee_pfc *pfc); int bnxt_dcb_ieee_setpfc(struct bnxt_softc *softc, struct bnxt_ieee_pfc *pfc); int bnxt_dcb_ieee_setapp(struct bnxt_softc *softc, struct bnxt_dcb_app *app); int bnxt_dcb_ieee_delapp(struct bnxt_softc *softc, struct bnxt_dcb_app *app); int bnxt_dcb_ieee_listapp(struct bnxt_softc *softc, struct bnxt_dcb_app *app, size_t nitems, int *num_inputs); #endif /* _BNXT_H */ diff --git a/sys/dev/bnxt/bnxt_en/bnxt_ulp.c b/sys/dev/bnxt/bnxt_en/bnxt_ulp.c index 677c9c99b74e..c6d862a36a9a 100644 --- a/sys/dev/bnxt/bnxt_en/bnxt_ulp.c +++ b/sys/dev/bnxt/bnxt_en/bnxt_ulp.c @@ -1,529 +1,530 @@ /*- * Broadcom NetXtreme-C/E network driver. * * Copyright (c) 2024 Broadcom, All Rights Reserved. * The term Broadcom refers to Broadcom Limited and/or its subsidiaries * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS' * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF * THE POSSIBILITY OF SUCH DAMAGE. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "hsi_struct_def.h" #include "bnxt.h" #include "bnxt_hwrm.h" #include "bnxt_ulp.h" void bnxt_destroy_irq(struct bnxt_softc *softc); static int bnxt_register_dev(struct bnxt_en_dev *edev, int ulp_id, struct bnxt_ulp_ops *ulp_ops, void *handle) { struct bnxt_softc *bp = edev->softc; struct bnxt_ulp *ulp; int rc = 0; if (ulp_id >= BNXT_MAX_ULP) return -EINVAL; mtx_lock(&bp->en_ops_lock); ulp = &edev->ulp_tbl[ulp_id]; if (rcu_access_pointer(ulp->ulp_ops)) { device_printf(bp->dev, "ulp id %d already registered\n", ulp_id); rc = -EBUSY; goto exit; } edev->flags &= ~BNXT_EN_FLAG_ULP_STOPPED; atomic_set(&ulp->ref_count, 0); ulp->handle = handle; rcu_assign_pointer(ulp->ulp_ops, ulp_ops); if (ulp_id == BNXT_ROCE_ULP) { if (test_bit(BNXT_STATE_OPEN, &bp->state) && bp->is_dev_init) bnxt_hwrm_vnic_cfg(bp, &bp->vnic_info); } exit: mtx_unlock(&bp->en_ops_lock); return rc; } static int bnxt_unregister_dev(struct bnxt_en_dev *edev, int ulp_id) { struct bnxt_softc *bp = edev->softc; struct bnxt_ulp *ulp; int i = 0; if (ulp_id >= BNXT_MAX_ULP) return -EINVAL; ulp = &edev->ulp_tbl[ulp_id]; if (!rcu_access_pointer(ulp->ulp_ops)) { device_printf(bp->dev, "ulp id %d not registered\n", ulp_id); return -EINVAL; } if (ulp_id == BNXT_ROCE_ULP && ulp->msix_requested) edev->en_ops->bnxt_free_msix(edev, ulp_id); mtx_lock(&bp->en_ops_lock); RCU_INIT_POINTER(ulp->ulp_ops, NULL); synchronize_rcu(); ulp->max_async_event_id = 0; ulp->async_events_bmap = NULL; while (atomic_read(&ulp->ref_count) != 0 && i < 10) { msleep(100); i++; } mtx_unlock(&bp->en_ops_lock); return 0; } static void bnxt_fill_msix_vecs(struct bnxt_softc *bp, struct bnxt_msix_entry *ent) { struct bnxt_en_dev *edev = bp->edev; int num_msix, idx, i; num_msix = edev->ulp_tbl[BNXT_ROCE_ULP].msix_requested; idx = edev->ulp_tbl[BNXT_ROCE_ULP].msix_base; for (i = 0; i < num_msix; i++) { ent[i].vector = bp->irq_tbl[idx + i].vector; ent[i].ring_idx = idx + i; if (BNXT_CHIP_P5_PLUS(bp)) ent[i].db_offset = bp->db_offset; else ent[i].db_offset = (idx + i) * 0x80; } } static int bnxt_req_msix_vecs(struct bnxt_en_dev *edev, int ulp_id, struct bnxt_msix_entry *ent, int num_msix) { struct bnxt_softc *bp = edev->softc; int avail_msix, idx; if (ulp_id != BNXT_ROCE_ULP) return -EINVAL; if (edev->ulp_tbl[ulp_id].msix_requested) return -EAGAIN; idx = bp->total_irqs - BNXT_ROCE_IRQ_COUNT; avail_msix = BNXT_ROCE_IRQ_COUNT; mtx_lock(&bp->en_ops_lock); edev->ulp_tbl[ulp_id].msix_base = idx; edev->ulp_tbl[ulp_id].msix_requested = avail_msix; bnxt_fill_msix_vecs(bp, ent); edev->flags |= BNXT_EN_FLAG_MSIX_REQUESTED; mtx_unlock(&bp->en_ops_lock); return avail_msix; } static int bnxt_free_msix_vecs(struct bnxt_en_dev *edev, int ulp_id) { struct bnxt_softc *bp = edev->softc; if (ulp_id != BNXT_ROCE_ULP) return -EINVAL; if (!(edev->flags & BNXT_EN_FLAG_MSIX_REQUESTED)) return 0; mtx_lock(&bp->en_ops_lock); edev->ulp_tbl[ulp_id].msix_requested = 0; edev->flags &= ~BNXT_EN_FLAG_MSIX_REQUESTED; if (edev->flags & BNXT_EN_FLAG_ULP_STOPPED) goto stopped; stopped: mtx_unlock(&bp->en_ops_lock); return 0; } int bnxt_get_ulp_msix_num(struct bnxt_softc *bp) { if (bnxt_ulp_registered(bp->edev, BNXT_ROCE_ULP)) { struct bnxt_en_dev *edev = bp->edev; return edev->ulp_tbl[BNXT_ROCE_ULP].msix_requested; } return 0; } int bnxt_get_ulp_msix_base(struct bnxt_softc *bp) { if (bnxt_ulp_registered(bp->edev, BNXT_ROCE_ULP)) { struct bnxt_en_dev *edev = bp->edev; if (edev->ulp_tbl[BNXT_ROCE_ULP].msix_requested) return edev->ulp_tbl[BNXT_ROCE_ULP].msix_base; } return 0; } static int bnxt_send_msg(struct bnxt_en_dev *edev, int ulp_id, struct bnxt_fw_msg *fw_msg) { struct bnxt_softc *softc = edev->softc; int rc; if ((ulp_id != BNXT_ROCE_ULP) && softc->fw_reset_state) return -EBUSY; rc = bnxt_hwrm_passthrough(softc, fw_msg->msg, fw_msg->msg_len, fw_msg->resp, fw_msg->resp_max_len, fw_msg->timeout); return rc; } static void bnxt_ulp_get(struct bnxt_ulp *ulp) { atomic_inc(&ulp->ref_count); } static void bnxt_ulp_put(struct bnxt_ulp *ulp) { atomic_dec(&ulp->ref_count); } void bnxt_ulp_stop(struct bnxt_softc *bp) { struct bnxt_en_dev *edev = bp->edev; struct bnxt_ulp_ops *ops; int i; if (!edev) return; edev->flags |= BNXT_EN_FLAG_ULP_STOPPED; edev->en_state = bp->state; for (i = 0; i < BNXT_MAX_ULP; i++) { struct bnxt_ulp *ulp = &edev->ulp_tbl[i]; ops = ulp->ulp_ops; if (!ops || !ops->ulp_stop) continue; ops->ulp_stop(ulp->handle); } } void bnxt_ulp_start(struct bnxt_softc *bp, int err) { struct bnxt_en_dev *edev = bp->edev; struct bnxt_ulp_ops *ops; int i; if (!edev) return; edev->flags &= ~BNXT_EN_FLAG_ULP_STOPPED; edev->en_state = bp->state; if (err) return; for (i = 0; i < BNXT_MAX_ULP; i++) { struct bnxt_ulp *ulp = &edev->ulp_tbl[i]; ops = ulp->ulp_ops; if (!ops || !ops->ulp_start) continue; ops->ulp_start(ulp->handle); } } void bnxt_ulp_sriov_cfg(struct bnxt_softc *bp, int num_vfs) { struct bnxt_en_dev *edev = bp->edev; struct bnxt_ulp_ops *ops; int i; if (!edev) return; for (i = 0; i < BNXT_MAX_ULP; i++) { struct bnxt_ulp *ulp = &edev->ulp_tbl[i]; rcu_read_lock(); ops = rcu_dereference(ulp->ulp_ops); if (!ops || !ops->ulp_sriov_config) { rcu_read_unlock(); continue; } bnxt_ulp_get(ulp); rcu_read_unlock(); ops->ulp_sriov_config(ulp->handle, num_vfs); bnxt_ulp_put(ulp); } } void bnxt_ulp_shutdown(struct bnxt_softc *bp) { struct bnxt_en_dev *edev = bp->edev; struct bnxt_ulp_ops *ops; int i; if (!edev) return; for (i = 0; i < BNXT_MAX_ULP; i++) { struct bnxt_ulp *ulp = &edev->ulp_tbl[i]; ops = ulp->ulp_ops; if (!ops || !ops->ulp_shutdown) continue; ops->ulp_shutdown(ulp->handle); } } void bnxt_ulp_irq_stop(struct bnxt_softc *bp) { struct bnxt_en_dev *edev = bp->edev; struct bnxt_ulp_ops *ops; if (!edev || !(edev->flags & BNXT_EN_FLAG_MSIX_REQUESTED)) return; if (bnxt_ulp_registered(bp->edev, BNXT_ROCE_ULP)) { struct bnxt_ulp *ulp = &edev->ulp_tbl[BNXT_ROCE_ULP]; if (!ulp->msix_requested) return; ops = ulp->ulp_ops; if (!ops || !ops->ulp_irq_stop) return; ops->ulp_irq_stop(ulp->handle); } } void bnxt_ulp_async_events(struct bnxt_softc *bp, struct hwrm_async_event_cmpl *cmpl) { u16 event_id = le16_to_cpu(cmpl->event_id); struct bnxt_en_dev *edev = bp->edev; struct bnxt_ulp_ops *ops; int i; if (!edev) return; rcu_read_lock(); for (i = 0; i < BNXT_MAX_ULP; i++) { struct bnxt_ulp *ulp = &edev->ulp_tbl[i]; ops = rcu_dereference(ulp->ulp_ops); if (!ops || !ops->ulp_async_notifier) continue; if (!ulp->async_events_bmap || event_id > ulp->max_async_event_id) continue; /* Read max_async_event_id first before testing the bitmap. */ rmb(); if (edev->flags & BNXT_EN_FLAG_ULP_STOPPED) continue; if (test_bit(event_id, ulp->async_events_bmap)) ops->ulp_async_notifier(ulp->handle, cmpl); } rcu_read_unlock(); } static int bnxt_register_async_events(struct bnxt_en_dev *edev, int ulp_id, unsigned long *events_bmap, u16 max_id) { struct bnxt_softc *bp = edev->softc; struct bnxt_ulp *ulp; if (ulp_id >= BNXT_MAX_ULP) return -EINVAL; mtx_lock(&bp->en_ops_lock); ulp = &edev->ulp_tbl[ulp_id]; ulp->async_events_bmap = events_bmap; wmb(); ulp->max_async_event_id = max_id; bnxt_hwrm_func_drv_rgtr(bp, events_bmap, max_id + 1, true); mtx_unlock(&bp->en_ops_lock); return 0; } void bnxt_destroy_irq(struct bnxt_softc *softc) { kfree(softc->irq_tbl); } static int bnxt_populate_irq(struct bnxt_softc *softc) { struct resource_list *rl = NULL; struct resource_list_entry *rle = NULL; struct bnxt_msix_tbl *irq_tbl = NULL; struct pci_devinfo *dinfo = NULL; int i; softc->total_irqs = softc->scctx->isc_nrxqsets + BNXT_ROCE_IRQ_COUNT; irq_tbl = kzalloc(softc->total_irqs * sizeof(*softc->irq_tbl), GFP_KERNEL); if (!irq_tbl) { device_printf(softc->dev, "Failed to allocate IRQ table\n"); return -1; } dinfo = device_get_ivars(softc->pdev->dev.bsddev); rl = &dinfo->resources; rle = resource_list_find(rl, SYS_RES_IRQ, 1); softc->pdev->dev.irq_start = rle->start; softc->pdev->dev.irq_end = rle->start + softc->total_irqs; for (i = 0; i < softc->total_irqs; i++) { irq_tbl[i].entry = i; irq_tbl[i].vector = softc->pdev->dev.irq_start + i; } softc->irq_tbl = irq_tbl; return 0; } static const struct bnxt_en_ops bnxt_en_ops_tbl = { .bnxt_register_device = bnxt_register_dev, .bnxt_unregister_device = bnxt_unregister_dev, .bnxt_request_msix = bnxt_req_msix_vecs, .bnxt_free_msix = bnxt_free_msix_vecs, .bnxt_send_fw_msg = bnxt_send_msg, .bnxt_register_fw_async_events = bnxt_register_async_events, }; void bnxt_aux_dev_release(struct device *dev) { struct bnxt_aux_dev *bnxt_adev = container_of(dev, struct bnxt_aux_dev, aux_dev.dev); struct bnxt_softc *bp = bnxt_adev->edev->softc; kfree(bnxt_adev->edev); bnxt_adev->edev = NULL; bp->edev = NULL; } static inline void bnxt_set_edev_info(struct bnxt_en_dev *edev, struct bnxt_softc *bp) { edev->en_ops = &bnxt_en_ops_tbl; edev->net = bp->ifp; edev->pdev = bp->pdev; edev->softc = bp; edev->l2_db_size = bp->db_size; edev->l2_db_offset = bp->db_offset; mtx_init(&bp->en_ops_lock, "Ethernet ops lock", NULL, MTX_DEF); if (bp->flags & BNXT_FLAG_ROCEV1_CAP) edev->flags |= BNXT_EN_FLAG_ROCEV1_CAP; if (bp->flags & BNXT_FLAG_ROCEV2_CAP) edev->flags |= BNXT_EN_FLAG_ROCEV2_CAP; if (bp->is_asym_q) edev->flags |= BNXT_EN_FLAG_ASYM_Q; if (BNXT_SW_RES_LMT(bp)) edev->flags |= BNXT_EN_FLAG_SW_RES_LMT; edev->hwrm_bar = bp->hwrm_bar; edev->port_partition_type = bp->port_partition_type; edev->ulp_version = BNXT_ULP_VERSION; + memcpy(edev->board_part_number, bp->board_partno, BNXT_VPD_PN_FLD_LEN - 1); } int bnxt_rdma_aux_device_del(struct bnxt_softc *softc) { struct bnxt_aux_dev *bnxt_adev = softc->aux_dev; struct auxiliary_device *adev; adev = &bnxt_adev->aux_dev; auxiliary_device_delete(adev); auxiliary_device_uninit(adev); bnxt_destroy_irq(softc); return 0; } int bnxt_rdma_aux_device_add(struct bnxt_softc *bp) { struct bnxt_aux_dev *bnxt_adev = bp->aux_dev; struct bnxt_en_dev *edev = bnxt_adev->edev; struct auxiliary_device *aux_dev; int ret = -1; if (bnxt_populate_irq(bp)) return ret; device_printf(bp->dev, "V:D:SV:SD %x:%x:%x:%x, irq 0x%x, " "devfn 0x%x, cla 0x%x, rev 0x%x, msi_en 0x%x\n", bp->pdev->vendor, bp->pdev->device, bp->pdev->subsystem_vendor, bp->pdev->subsystem_device, bp->pdev->irq, bp->pdev->devfn, bp->pdev->class, bp->pdev->revision, bp->pdev->msi_enabled); aux_dev = &bnxt_adev->aux_dev; aux_dev->id = bnxt_adev->id; aux_dev->name = "rdma"; aux_dev->dev.parent = &bp->pdev->dev; aux_dev->dev.release = bnxt_aux_dev_release; if (!edev) { edev = kzalloc(sizeof(*edev), GFP_KERNEL); if (!edev) return -ENOMEM; } bnxt_set_edev_info(edev, bp); bnxt_adev->edev = edev; bp->edev = edev; ret = auxiliary_device_init(aux_dev); if (ret) goto err_free_edev; ret = auxiliary_device_add(aux_dev); if (ret) goto err_dev_uninit; return 0; err_dev_uninit: auxiliary_device_uninit(aux_dev); err_free_edev: kfree(edev); bnxt_adev->edev = NULL; bp->edev = NULL; return ret; } diff --git a/sys/dev/bnxt/bnxt_en/bnxt_ulp.h b/sys/dev/bnxt/bnxt_en/bnxt_ulp.h index 7d7ecbd2f536..53bb51b07135 100644 --- a/sys/dev/bnxt/bnxt_en/bnxt_ulp.h +++ b/sys/dev/bnxt/bnxt_en/bnxt_ulp.h @@ -1,167 +1,169 @@ /*- * Broadcom NetXtreme-C/E network driver. * * Copyright (c) 2024 Broadcom, All Rights Reserved. * The term Broadcom refers to Broadcom Limited and/or its subsidiaries * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS' * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF * THE POSSIBILITY OF SUCH DAMAGE. */ #ifndef BNXT_ULP_H #define BNXT_ULP_H #include #include "bnxt.h" #define BNXT_ROCE_ULP 0 #define BNXT_OTHER_ULP 1 #define BNXT_MAX_ULP 2 #define BNXT_MIN_ROCE_CP_RINGS 2 #define BNXT_MIN_ROCE_STAT_CTXS 1 struct hwrm_async_event_cmpl; struct bnxt_softc; struct bnxt_bar_info; struct bnxt_msix_entry { uint32_t vector; uint32_t ring_idx; uint32_t db_offset; }; struct bnxt_ulp_ops { void (*ulp_async_notifier)(void *, struct hwrm_async_event_cmpl *); void (*ulp_stop)(void *); void (*ulp_start)(void *); void (*ulp_sriov_config)(void *, int); void (*ulp_shutdown)(void *); void (*ulp_irq_stop)(void *); void (*ulp_irq_restart)(void *, struct bnxt_msix_entry *); }; struct bnxt_fw_msg { void *msg; int msg_len; void *resp; int resp_max_len; int timeout; }; struct bnxt_ulp { void *handle; struct bnxt_ulp_ops __rcu *ulp_ops; unsigned long *async_events_bmap; u16 max_async_event_id; u16 msix_requested; u16 msix_base; atomic_t ref_count; }; struct bnxt_en_dev { struct ifnet *net; struct pci_dev *pdev; struct bnxt_softc *softc; u32 flags; #define BNXT_EN_FLAG_ROCEV1_CAP 0x1 #define BNXT_EN_FLAG_ROCEV2_CAP 0x2 #define BNXT_EN_FLAG_ROCE_CAP (BNXT_EN_FLAG_ROCEV1_CAP | \ BNXT_EN_FLAG_ROCEV2_CAP) #define BNXT_EN_FLAG_MSIX_REQUESTED 0x4 #define BNXT_EN_FLAG_ULP_STOPPED 0x8 #define BNXT_EN_FLAG_ASYM_Q 0x10 #define BNXT_EN_FLAG_MULTI_HOST 0x20 #define BNXT_EN_FLAG_SW_RES_LMT 0x400 #define BNXT_EN_ASYM_Q(edev) ((edev)->flags & BNXT_EN_FLAG_ASYM_Q) #define BNXT_EN_MH(edev) ((edev)->flags & BNXT_EN_FLAG_MULTI_HOST) #define BNXT_EN_SW_RES_LMT(edev) ((edev)->flags & BNXT_EN_FLAG_SW_RES_LMT) const struct bnxt_en_ops *en_ops; struct bnxt_ulp ulp_tbl[BNXT_MAX_ULP]; int l2_db_offset; /* Doorbell BAR offset * of non-cacheable. */ int l2_db_size; /* Doorbell BAR size in * bytes mapped by L2 * driver. */ int l2_db_size_nc; /* Doorbell BAR size in * bytes mapped as non- * cacheable. */ u32 ulp_version; /* bnxt_re checks the * ulp_version is correct * to ensure compatibility * with bnxt_en. */ #define BNXT_ULP_VERSION 0x695a0008 /* Change this when any interface * structure or API changes * between bnxt_en and bnxt_re. */ unsigned long en_state; void __iomem *bar0; u16 hw_ring_stats_size; u16 pf_port_id; u8 port_partition_type; #define BNXT_EN_NPAR(edev) ((edev)->port_partition_type) u8 port_count; struct bnxt_dbr *en_dbr; struct bnxt_bar_info hwrm_bar; u32 espeed; uint8_t lanes; + #define BNXT_VPD_PN_FLD_LEN 32 + char board_part_number[BNXT_VPD_PN_FLD_LEN]; }; struct bnxt_en_ops { int (*bnxt_register_device)(struct bnxt_en_dev *, int, struct bnxt_ulp_ops *, void *); int (*bnxt_unregister_device)(struct bnxt_en_dev *, int); int (*bnxt_request_msix)(struct bnxt_en_dev *, int, struct bnxt_msix_entry *, int); int (*bnxt_free_msix)(struct bnxt_en_dev *, int); int (*bnxt_send_fw_msg)(struct bnxt_en_dev *, int, struct bnxt_fw_msg *); int (*bnxt_register_fw_async_events)(struct bnxt_en_dev *, int, unsigned long *, u16); int (*bnxt_dbr_complete)(struct bnxt_en_dev *, int, u32); }; static inline bool bnxt_ulp_registered(struct bnxt_en_dev *edev, int ulp_id) { if (edev && rcu_access_pointer(edev->ulp_tbl[ulp_id].ulp_ops)) return true; return false; } int bnxt_get_ulp_msix_num(struct bnxt_softc *bp); int bnxt_get_ulp_msix_base(struct bnxt_softc *bp); int bnxt_get_ulp_stat_ctxs(struct bnxt_softc *bp); void bnxt_ulp_stop(struct bnxt_softc *bp); void bnxt_ulp_start(struct bnxt_softc *bp, int err); void bnxt_ulp_sriov_cfg(struct bnxt_softc *bp, int num_vfs); void bnxt_ulp_shutdown(struct bnxt_softc *bp); void bnxt_ulp_irq_stop(struct bnxt_softc *bp); void bnxt_ulp_irq_restart(struct bnxt_softc *bp, int err); void bnxt_ulp_async_events(struct bnxt_softc *bp, struct hwrm_async_event_cmpl *cmpl); struct bnxt_en_dev *bnxt_ulp_probe(struct net_device *dev); void bnxt_aux_dev_release(struct device *dev); int bnxt_rdma_aux_device_add(struct bnxt_softc *bp); int bnxt_rdma_aux_device_del(struct bnxt_softc *bp); #endif diff --git a/sys/dev/bnxt/bnxt_en/if_bnxt.c b/sys/dev/bnxt/bnxt_en/if_bnxt.c index 94775457b7e3..dea6fd68181e 100644 --- a/sys/dev/bnxt/bnxt_en/if_bnxt.c +++ b/sys/dev/bnxt/bnxt_en/if_bnxt.c @@ -1,5320 +1,5327 @@ /*- * Broadcom NetXtreme-C/E network driver. * * Copyright (c) 2016 Broadcom, All Rights Reserved. * The term Broadcom refers to Broadcom Limited and/or its subsidiaries * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS' * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF * THE POSSIBILITY OF SUCH DAMAGE. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #define WANT_NATIVE_PCI_GET_SLOT #include #include #include #include #include #include #include #include #include "opt_inet.h" #include "opt_inet6.h" #include "opt_rss.h" #include "ifdi_if.h" #include "bnxt.h" #include "bnxt_hwrm.h" #include "bnxt_ioctl.h" #include "bnxt_sysctl.h" #include "hsi_struct_def.h" #include "bnxt_mgmt.h" #include "bnxt_ulp.h" #include "bnxt_auxbus_compat.h" /* * PCI Device ID Table */ static const pci_vendor_info_t bnxt_vendor_info_array[] = { PVID(BROADCOM_VENDOR_ID, BCM57301, "Broadcom BCM57301 NetXtreme-C 10Gb Ethernet Controller"), PVID(BROADCOM_VENDOR_ID, BCM57302, "Broadcom BCM57302 NetXtreme-C 10Gb/25Gb Ethernet Controller"), PVID(BROADCOM_VENDOR_ID, BCM57304, "Broadcom BCM57304 NetXtreme-C 10Gb/25Gb/40Gb/50Gb Ethernet Controller"), PVID(BROADCOM_VENDOR_ID, BCM57311, "Broadcom BCM57311 NetXtreme-C 10Gb Ethernet"), PVID(BROADCOM_VENDOR_ID, BCM57312, "Broadcom BCM57312 NetXtreme-C 10Gb/25Gb Ethernet"), PVID(BROADCOM_VENDOR_ID, BCM57314, "Broadcom BCM57314 NetXtreme-C 10Gb/25Gb/40Gb/50Gb Ethernet"), PVID(BROADCOM_VENDOR_ID, BCM57402, "Broadcom BCM57402 NetXtreme-E 10Gb Ethernet Controller"), PVID(BROADCOM_VENDOR_ID, BCM57402_NPAR, "Broadcom BCM57402 NetXtreme-E Partition"), PVID(BROADCOM_VENDOR_ID, BCM57404, "Broadcom BCM57404 NetXtreme-E 10Gb/25Gb Ethernet Controller"), PVID(BROADCOM_VENDOR_ID, BCM57404_NPAR, "Broadcom BCM57404 NetXtreme-E Partition"), PVID(BROADCOM_VENDOR_ID, BCM57406, "Broadcom BCM57406 NetXtreme-E 10GBase-T Ethernet Controller"), PVID(BROADCOM_VENDOR_ID, BCM57406_NPAR, "Broadcom BCM57406 NetXtreme-E Partition"), PVID(BROADCOM_VENDOR_ID, BCM57407, "Broadcom BCM57407 NetXtreme-E 10GBase-T Ethernet Controller"), PVID(BROADCOM_VENDOR_ID, BCM57407_NPAR, "Broadcom BCM57407 NetXtreme-E Ethernet Partition"), PVID(BROADCOM_VENDOR_ID, BCM57407_SFP, "Broadcom BCM57407 NetXtreme-E 25Gb Ethernet Controller"), PVID(BROADCOM_VENDOR_ID, BCM57412, "Broadcom BCM57412 NetXtreme-E 10Gb Ethernet"), PVID(BROADCOM_VENDOR_ID, BCM57412_NPAR1, "Broadcom BCM57412 NetXtreme-E Ethernet Partition"), PVID(BROADCOM_VENDOR_ID, BCM57412_NPAR2, "Broadcom BCM57412 NetXtreme-E Ethernet Partition"), PVID(BROADCOM_VENDOR_ID, BCM57414, "Broadcom BCM57414 NetXtreme-E 10Gb/25Gb Ethernet"), PVID(BROADCOM_VENDOR_ID, BCM57414_NPAR1, "Broadcom BCM57414 NetXtreme-E Ethernet Partition"), PVID(BROADCOM_VENDOR_ID, BCM57414_NPAR2, "Broadcom BCM57414 NetXtreme-E Ethernet Partition"), PVID(BROADCOM_VENDOR_ID, BCM57416, "Broadcom BCM57416 NetXtreme-E 10GBase-T Ethernet"), PVID(BROADCOM_VENDOR_ID, BCM57416_NPAR1, "Broadcom BCM57416 NetXtreme-E Ethernet Partition"), PVID(BROADCOM_VENDOR_ID, BCM57416_NPAR2, "Broadcom BCM57416 NetXtreme-E Ethernet Partition"), PVID(BROADCOM_VENDOR_ID, BCM57416_SFP, "Broadcom BCM57416 NetXtreme-E 10Gb Ethernet"), PVID(BROADCOM_VENDOR_ID, BCM57417, "Broadcom BCM57417 NetXtreme-E 10GBase-T Ethernet"), PVID(BROADCOM_VENDOR_ID, BCM57417_NPAR1, "Broadcom BCM57417 NetXtreme-E Ethernet Partition"), PVID(BROADCOM_VENDOR_ID, BCM57417_NPAR2, "Broadcom BCM57417 NetXtreme-E Ethernet Partition"), PVID(BROADCOM_VENDOR_ID, BCM57417_SFP, "Broadcom BCM57417 NetXtreme-E 10Gb/25Gb Ethernet"), PVID(BROADCOM_VENDOR_ID, BCM57454, "Broadcom BCM57454 NetXtreme-E 10Gb/25Gb/40Gb/50Gb/100Gb Ethernet"), PVID(BROADCOM_VENDOR_ID, BCM58700, "Broadcom BCM58700 Nitro 1Gb/2.5Gb/10Gb Ethernet"), PVID(BROADCOM_VENDOR_ID, BCM57508, "Broadcom BCM57508 NetXtreme-E 10Gb/25Gb/50Gb/100Gb/200Gb Ethernet"), PVID(BROADCOM_VENDOR_ID, BCM57504, "Broadcom BCM57504 NetXtreme-E 10Gb/25Gb/50Gb/100Gb/200Gb Ethernet"), PVID(BROADCOM_VENDOR_ID, BCM57504_NPAR, "Broadcom BCM57504 NetXtreme-E Ethernet Partition"), PVID(BROADCOM_VENDOR_ID, BCM57502, "Broadcom BCM57502 NetXtreme-E 10Gb/25Gb/50Gb/100Gb/200Gb Ethernet"), PVID(BROADCOM_VENDOR_ID, BCM57608, "Broadcom BCM57608 NetXtreme-E 25Gb/50Gb/100Gb/200Gb/400Gb Ethernet"), PVID(BROADCOM_VENDOR_ID, BCM57604, "Broadcom BCM57604 NetXtreme-E 25Gb/50Gb/100Gb/200Gb Ethernet"), PVID(BROADCOM_VENDOR_ID, BCM57602, "Broadcom BCM57602 NetXtreme-E 25Gb/50Gb Ethernet"), PVID(BROADCOM_VENDOR_ID, BCM57601, "Broadcom BCM57601 NetXtreme-E 25Gb/50Gb Ethernet"), PVID(BROADCOM_VENDOR_ID, NETXTREME_C_VF1, "Broadcom NetXtreme-C Ethernet Virtual Function"), PVID(BROADCOM_VENDOR_ID, NETXTREME_C_VF2, "Broadcom NetXtreme-C Ethernet Virtual Function"), PVID(BROADCOM_VENDOR_ID, NETXTREME_C_VF3, "Broadcom NetXtreme-C Ethernet Virtual Function"), PVID(BROADCOM_VENDOR_ID, NETXTREME_E_VF1, "Broadcom NetXtreme-E Ethernet Virtual Function"), PVID(BROADCOM_VENDOR_ID, NETXTREME_E_VF2, "Broadcom NetXtreme-E Ethernet Virtual Function"), PVID(BROADCOM_VENDOR_ID, NETXTREME_E_VF3, "Broadcom NetXtreme-E Ethernet Virtual Function"), /* required last entry */ PVID_END }; /* * Function prototypes */ SLIST_HEAD(softc_list, bnxt_softc_list) pf_list; int bnxt_num_pfs = 0; void process_nq(struct bnxt_softc *softc, uint16_t nqid); static void *bnxt_register(device_t dev); /* Soft queue setup and teardown */ static int bnxt_tx_queues_alloc(if_ctx_t ctx, caddr_t *vaddrs, uint64_t *paddrs, int ntxqs, int ntxqsets); static int bnxt_rx_queues_alloc(if_ctx_t ctx, caddr_t *vaddrs, uint64_t *paddrs, int nrxqs, int nrxqsets); static void bnxt_queues_free(if_ctx_t ctx); /* Device setup and teardown */ static int bnxt_attach_pre(if_ctx_t ctx); static int bnxt_attach_post(if_ctx_t ctx); static int bnxt_detach(if_ctx_t ctx); /* Device configuration */ static void bnxt_init(if_ctx_t ctx); static void bnxt_stop(if_ctx_t ctx); static void bnxt_multi_set(if_ctx_t ctx); static int bnxt_mtu_set(if_ctx_t ctx, uint32_t mtu); static void bnxt_media_status(if_ctx_t ctx, struct ifmediareq * ifmr); static int bnxt_media_change(if_ctx_t ctx); static int bnxt_promisc_set(if_ctx_t ctx, int flags); static uint64_t bnxt_get_counter(if_ctx_t, ift_counter); static void bnxt_update_admin_status(if_ctx_t ctx); static void bnxt_if_timer(if_ctx_t ctx, uint16_t qid); /* Interrupt enable / disable */ static void bnxt_intr_enable(if_ctx_t ctx); static int bnxt_rx_queue_intr_enable(if_ctx_t ctx, uint16_t qid); static int bnxt_tx_queue_intr_enable(if_ctx_t ctx, uint16_t qid); static void bnxt_disable_intr(if_ctx_t ctx); static int bnxt_msix_intr_assign(if_ctx_t ctx, int msix); /* vlan support */ static void bnxt_vlan_register(if_ctx_t ctx, uint16_t vtag); static void bnxt_vlan_unregister(if_ctx_t ctx, uint16_t vtag); /* ioctl */ static int bnxt_priv_ioctl(if_ctx_t ctx, u_long command, caddr_t data); static int bnxt_shutdown(if_ctx_t ctx); static int bnxt_suspend(if_ctx_t ctx); static int bnxt_resume(if_ctx_t ctx); /* Internal support functions */ static int bnxt_probe_phy(struct bnxt_softc *softc); static void bnxt_add_media_types(struct bnxt_softc *softc); static int bnxt_pci_mapping(struct bnxt_softc *softc); static void bnxt_pci_mapping_free(struct bnxt_softc *softc); static int bnxt_update_link(struct bnxt_softc *softc, bool chng_link_state); static int bnxt_handle_def_cp(void *arg); static int bnxt_handle_isr(void *arg); static void bnxt_clear_ids(struct bnxt_softc *softc); static void inline bnxt_do_enable_intr(struct bnxt_cp_ring *cpr); static void inline bnxt_do_disable_intr(struct bnxt_cp_ring *cpr); static void bnxt_mark_cpr_invalid(struct bnxt_cp_ring *cpr); static void bnxt_def_cp_task(void *context, int pending); static void bnxt_handle_async_event(struct bnxt_softc *softc, struct cmpl_base *cmpl); static uint64_t bnxt_get_baudrate(struct bnxt_link_info *link); static void bnxt_get_wol_settings(struct bnxt_softc *softc); static int bnxt_wol_config(if_ctx_t ctx); static bool bnxt_if_needs_restart(if_ctx_t, enum iflib_restart_event); static int bnxt_i2c_req(if_ctx_t ctx, struct ifi2creq *i2c); static void bnxt_get_port_module_status(struct bnxt_softc *softc); static void bnxt_rdma_aux_device_init(struct bnxt_softc *softc); static void bnxt_rdma_aux_device_uninit(struct bnxt_softc *softc); static void bnxt_queue_fw_reset_work(struct bnxt_softc *bp, unsigned long delay); void bnxt_queue_sp_work(struct bnxt_softc *bp); void bnxt_fw_reset(struct bnxt_softc *bp); /* * Device Interface Declaration */ static device_method_t bnxt_methods[] = { /* Device interface */ DEVMETHOD(device_register, bnxt_register), DEVMETHOD(device_probe, iflib_device_probe), DEVMETHOD(device_attach, iflib_device_attach), DEVMETHOD(device_detach, iflib_device_detach), DEVMETHOD(device_shutdown, iflib_device_shutdown), DEVMETHOD(device_suspend, iflib_device_suspend), DEVMETHOD(device_resume, iflib_device_resume), DEVMETHOD_END }; static driver_t bnxt_driver = { "bnxt", bnxt_methods, sizeof(struct bnxt_softc), }; DRIVER_MODULE(bnxt, pci, bnxt_driver, 0, 0); MODULE_LICENSE("Dual BSD/GPL"); MODULE_DEPEND(if_bnxt, pci, 1, 1, 1); MODULE_DEPEND(if_bnxt, ether, 1, 1, 1); MODULE_DEPEND(if_bnxt, iflib, 1, 1, 1); MODULE_DEPEND(if_bnxt, linuxkpi, 1, 1, 1); MODULE_VERSION(if_bnxt, 1); IFLIB_PNP_INFO(pci, bnxt, bnxt_vendor_info_array); void writel_fbsd(struct bnxt_softc *bp, u32, u8, u32); u32 readl_fbsd(struct bnxt_softc *bp, u32, u8); u32 readl_fbsd(struct bnxt_softc *bp, u32 reg_off, u8 bar_idx) { if (!bar_idx) return bus_space_read_4(bp->doorbell_bar.tag, bp->doorbell_bar.handle, reg_off); else return bus_space_read_4(bp->hwrm_bar.tag, bp->hwrm_bar.handle, reg_off); } void writel_fbsd(struct bnxt_softc *bp, u32 reg_off, u8 bar_idx, u32 val) { if (!bar_idx) bus_space_write_4(bp->doorbell_bar.tag, bp->doorbell_bar.handle, reg_off, htole32(val)); else bus_space_write_4(bp->hwrm_bar.tag, bp->hwrm_bar.handle, reg_off, htole32(val)); } static DEFINE_IDA(bnxt_aux_dev_ids); static device_method_t bnxt_iflib_methods[] = { DEVMETHOD(ifdi_tx_queues_alloc, bnxt_tx_queues_alloc), DEVMETHOD(ifdi_rx_queues_alloc, bnxt_rx_queues_alloc), DEVMETHOD(ifdi_queues_free, bnxt_queues_free), DEVMETHOD(ifdi_attach_pre, bnxt_attach_pre), DEVMETHOD(ifdi_attach_post, bnxt_attach_post), DEVMETHOD(ifdi_detach, bnxt_detach), DEVMETHOD(ifdi_init, bnxt_init), DEVMETHOD(ifdi_stop, bnxt_stop), DEVMETHOD(ifdi_multi_set, bnxt_multi_set), DEVMETHOD(ifdi_mtu_set, bnxt_mtu_set), DEVMETHOD(ifdi_media_status, bnxt_media_status), DEVMETHOD(ifdi_media_change, bnxt_media_change), DEVMETHOD(ifdi_promisc_set, bnxt_promisc_set), DEVMETHOD(ifdi_get_counter, bnxt_get_counter), DEVMETHOD(ifdi_update_admin_status, bnxt_update_admin_status), DEVMETHOD(ifdi_timer, bnxt_if_timer), DEVMETHOD(ifdi_intr_enable, bnxt_intr_enable), DEVMETHOD(ifdi_tx_queue_intr_enable, bnxt_tx_queue_intr_enable), DEVMETHOD(ifdi_rx_queue_intr_enable, bnxt_rx_queue_intr_enable), DEVMETHOD(ifdi_intr_disable, bnxt_disable_intr), DEVMETHOD(ifdi_msix_intr_assign, bnxt_msix_intr_assign), DEVMETHOD(ifdi_vlan_register, bnxt_vlan_register), DEVMETHOD(ifdi_vlan_unregister, bnxt_vlan_unregister), DEVMETHOD(ifdi_priv_ioctl, bnxt_priv_ioctl), DEVMETHOD(ifdi_suspend, bnxt_suspend), DEVMETHOD(ifdi_shutdown, bnxt_shutdown), DEVMETHOD(ifdi_resume, bnxt_resume), DEVMETHOD(ifdi_i2c_req, bnxt_i2c_req), DEVMETHOD(ifdi_needs_restart, bnxt_if_needs_restart), DEVMETHOD_END }; static driver_t bnxt_iflib_driver = { "bnxt", bnxt_iflib_methods, sizeof(struct bnxt_softc) }; /* * iflib shared context */ #define BNXT_DRIVER_VERSION "230.0.133.0" const char bnxt_driver_version[] = BNXT_DRIVER_VERSION; extern struct if_txrx bnxt_txrx; static struct if_shared_ctx bnxt_sctx_init = { .isc_magic = IFLIB_MAGIC, .isc_driver = &bnxt_iflib_driver, .isc_nfl = 2, // Number of Free Lists .isc_flags = IFLIB_HAS_RXCQ | IFLIB_HAS_TXCQ | IFLIB_NEED_ETHER_PAD, .isc_q_align = PAGE_SIZE, .isc_tx_maxsize = BNXT_TSO_SIZE + sizeof(struct ether_vlan_header), .isc_tx_maxsegsize = BNXT_TSO_SIZE + sizeof(struct ether_vlan_header), .isc_tso_maxsize = BNXT_TSO_SIZE + sizeof(struct ether_vlan_header), .isc_tso_maxsegsize = BNXT_TSO_SIZE + sizeof(struct ether_vlan_header), .isc_rx_maxsize = BNXT_TSO_SIZE + sizeof(struct ether_vlan_header), .isc_rx_maxsegsize = BNXT_TSO_SIZE + sizeof(struct ether_vlan_header), // Only use a single segment to avoid page size constraints .isc_rx_nsegments = 1, .isc_ntxqs = 3, .isc_nrxqs = 3, .isc_nrxd_min = {16, 16, 16}, .isc_nrxd_default = {PAGE_SIZE / sizeof(struct cmpl_base) * 8, PAGE_SIZE / sizeof(struct rx_prod_pkt_bd), PAGE_SIZE / sizeof(struct rx_prod_pkt_bd)}, .isc_nrxd_max = {BNXT_MAX_RXD, BNXT_MAX_RXD, BNXT_MAX_RXD}, .isc_ntxd_min = {16, 16, 16}, .isc_ntxd_default = {PAGE_SIZE / sizeof(struct cmpl_base) * 2, PAGE_SIZE / sizeof(struct tx_bd_short), /* NQ depth 4096 */ PAGE_SIZE / sizeof(struct cmpl_base) * 16}, .isc_ntxd_max = {BNXT_MAX_TXD, BNXT_MAX_TXD, BNXT_MAX_TXD}, .isc_admin_intrcnt = BNXT_ROCE_IRQ_COUNT, .isc_vendor_info = bnxt_vendor_info_array, .isc_driver_version = bnxt_driver_version, }; #define PCI_SUBSYSTEM_ID 0x2e static struct workqueue_struct *bnxt_pf_wq; extern void bnxt_destroy_irq(struct bnxt_softc *softc); /* * Device Methods */ static void * bnxt_register(device_t dev) { return (&bnxt_sctx_init); } static void bnxt_nq_alloc(struct bnxt_softc *softc, int nqsets) { if (softc->nq_rings) return; softc->nq_rings = malloc(sizeof(struct bnxt_cp_ring) * nqsets, M_DEVBUF, M_NOWAIT | M_ZERO); } static void bnxt_nq_free(struct bnxt_softc *softc) { if (softc->nq_rings) free(softc->nq_rings, M_DEVBUF); softc->nq_rings = NULL; } static void bnxt_set_db_mask(struct bnxt_softc *bp, struct bnxt_ring *db, u32 ring_type) { if (BNXT_CHIP_P7(bp)) { db->db_epoch_mask = db->db_ring_mask + 1; db->db_epoch_shift = DBR_EPOCH_SFT - ilog2(db->db_epoch_mask); } } /* * Device Dependent Configuration Functions */ /* Soft queue setup and teardown */ static int bnxt_tx_queues_alloc(if_ctx_t ctx, caddr_t *vaddrs, uint64_t *paddrs, int ntxqs, int ntxqsets) { struct bnxt_softc *softc; int i; int rc; softc = iflib_get_softc(ctx); if (BNXT_CHIP_P5_PLUS(softc)) { bnxt_nq_alloc(softc, ntxqsets); if (!softc->nq_rings) { device_printf(iflib_get_dev(ctx), "unable to allocate NQ rings\n"); rc = ENOMEM; goto nq_alloc_fail; } } softc->tx_cp_rings = malloc(sizeof(struct bnxt_cp_ring) * ntxqsets, M_DEVBUF, M_NOWAIT | M_ZERO); if (!softc->tx_cp_rings) { device_printf(iflib_get_dev(ctx), "unable to allocate TX completion rings\n"); rc = ENOMEM; goto cp_alloc_fail; } softc->tx_rings = malloc(sizeof(struct bnxt_ring) * ntxqsets, M_DEVBUF, M_NOWAIT | M_ZERO); if (!softc->tx_rings) { device_printf(iflib_get_dev(ctx), "unable to allocate TX rings\n"); rc = ENOMEM; goto ring_alloc_fail; } for (i=0; i < ntxqsets; i++) { rc = iflib_dma_alloc(ctx, sizeof(struct ctx_hw_stats), &softc->tx_stats[i], 0); if (rc) goto dma_alloc_fail; bus_dmamap_sync(softc->tx_stats[i].idi_tag, softc->tx_stats[i].idi_map, BUS_DMASYNC_PREREAD); } for (i = 0; i < ntxqsets; i++) { /* Set up the completion ring */ softc->tx_cp_rings[i].stats_ctx_id = HWRM_NA_SIGNATURE; softc->tx_cp_rings[i].ring.phys_id = (uint16_t)HWRM_NA_SIGNATURE; softc->tx_cp_rings[i].ring.softc = softc; softc->tx_cp_rings[i].ring.idx = i; softc->tx_cp_rings[i].ring.id = (softc->scctx->isc_nrxqsets * 2) + 1 + i; softc->tx_cp_rings[i].ring.doorbell = (BNXT_CHIP_P5_PLUS(softc)) ? softc->legacy_db_size: softc->tx_cp_rings[i].ring.id * 0x80; softc->tx_cp_rings[i].ring.ring_size = softc->scctx->isc_ntxd[0]; softc->tx_cp_rings[i].ring.db_ring_mask = softc->tx_cp_rings[i].ring.ring_size - 1; softc->tx_cp_rings[i].ring.vaddr = vaddrs[i * ntxqs]; softc->tx_cp_rings[i].ring.paddr = paddrs[i * ntxqs]; /* Set up the TX ring */ softc->tx_rings[i].phys_id = (uint16_t)HWRM_NA_SIGNATURE; softc->tx_rings[i].softc = softc; softc->tx_rings[i].idx = i; softc->tx_rings[i].id = (softc->scctx->isc_nrxqsets * 2) + 1 + i; softc->tx_rings[i].doorbell = (BNXT_CHIP_P5_PLUS(softc)) ? softc->legacy_db_size : softc->tx_rings[i].id * 0x80; softc->tx_rings[i].ring_size = softc->scctx->isc_ntxd[1]; softc->tx_rings[i].db_ring_mask = softc->tx_rings[i].ring_size - 1; softc->tx_rings[i].vaddr = vaddrs[i * ntxqs + 1]; softc->tx_rings[i].paddr = paddrs[i * ntxqs + 1]; bnxt_create_tx_sysctls(softc, i); if (BNXT_CHIP_P5_PLUS(softc)) { /* Set up the Notification ring (NQ) */ softc->nq_rings[i].stats_ctx_id = HWRM_NA_SIGNATURE; softc->nq_rings[i].ring.phys_id = (uint16_t)HWRM_NA_SIGNATURE; softc->nq_rings[i].ring.softc = softc; softc->nq_rings[i].ring.idx = i; softc->nq_rings[i].ring.id = i; softc->nq_rings[i].ring.doorbell = (BNXT_CHIP_P5_PLUS(softc)) ? softc->legacy_db_size : softc->nq_rings[i].ring.id * 0x80; softc->nq_rings[i].ring.ring_size = softc->scctx->isc_ntxd[2]; softc->nq_rings[i].ring.db_ring_mask = softc->nq_rings[i].ring.ring_size - 1; softc->nq_rings[i].ring.vaddr = vaddrs[i * ntxqs + 2]; softc->nq_rings[i].ring.paddr = paddrs[i * ntxqs + 2]; softc->nq_rings[i].type = Q_TYPE_TX; } } softc->ntxqsets = ntxqsets; return rc; dma_alloc_fail: for (i = i - 1; i >= 0; i--) iflib_dma_free(&softc->tx_stats[i]); free(softc->tx_rings, M_DEVBUF); ring_alloc_fail: free(softc->tx_cp_rings, M_DEVBUF); cp_alloc_fail: bnxt_nq_free(softc); nq_alloc_fail: return rc; } static void bnxt_queues_free(if_ctx_t ctx) { struct bnxt_softc *softc = iflib_get_softc(ctx); int i; // Free TX queues for (i=0; intxqsets; i++) iflib_dma_free(&softc->tx_stats[i]); free(softc->tx_rings, M_DEVBUF); softc->tx_rings = NULL; free(softc->tx_cp_rings, M_DEVBUF); softc->tx_cp_rings = NULL; softc->ntxqsets = 0; // Free RX queues for (i=0; inrxqsets; i++) iflib_dma_free(&softc->rx_stats[i]); iflib_dma_free(&softc->hw_tx_port_stats); iflib_dma_free(&softc->hw_rx_port_stats); iflib_dma_free(&softc->hw_tx_port_stats_ext); iflib_dma_free(&softc->hw_rx_port_stats_ext); free(softc->grp_info, M_DEVBUF); free(softc->ag_rings, M_DEVBUF); free(softc->rx_rings, M_DEVBUF); free(softc->rx_cp_rings, M_DEVBUF); bnxt_nq_free(softc); } static int bnxt_rx_queues_alloc(if_ctx_t ctx, caddr_t *vaddrs, uint64_t *paddrs, int nrxqs, int nrxqsets) { struct bnxt_softc *softc; int i; int rc; softc = iflib_get_softc(ctx); softc->rx_cp_rings = malloc(sizeof(struct bnxt_cp_ring) * nrxqsets, M_DEVBUF, M_NOWAIT | M_ZERO); if (!softc->rx_cp_rings) { device_printf(iflib_get_dev(ctx), "unable to allocate RX completion rings\n"); rc = ENOMEM; goto cp_alloc_fail; } softc->rx_rings = malloc(sizeof(struct bnxt_ring) * nrxqsets, M_DEVBUF, M_NOWAIT | M_ZERO); if (!softc->rx_rings) { device_printf(iflib_get_dev(ctx), "unable to allocate RX rings\n"); rc = ENOMEM; goto ring_alloc_fail; } softc->ag_rings = malloc(sizeof(struct bnxt_ring) * nrxqsets, M_DEVBUF, M_NOWAIT | M_ZERO); if (!softc->ag_rings) { device_printf(iflib_get_dev(ctx), "unable to allocate aggregation rings\n"); rc = ENOMEM; goto ag_alloc_fail; } softc->grp_info = malloc(sizeof(struct bnxt_grp_info) * nrxqsets, M_DEVBUF, M_NOWAIT | M_ZERO); if (!softc->grp_info) { device_printf(iflib_get_dev(ctx), "unable to allocate ring groups\n"); rc = ENOMEM; goto grp_alloc_fail; } for (i=0; i < nrxqsets; i++) { rc = iflib_dma_alloc(ctx, sizeof(struct ctx_hw_stats), &softc->rx_stats[i], 0); if (rc) goto hw_stats_alloc_fail; bus_dmamap_sync(softc->rx_stats[i].idi_tag, softc->rx_stats[i].idi_map, BUS_DMASYNC_PREREAD); } /* * Additional 512 bytes for future expansion. * To prevent corruption when loaded with newer firmwares with added counters. * This can be deleted when there will be no further additions of counters. */ #define BNXT_PORT_STAT_PADDING 512 rc = iflib_dma_alloc(ctx, sizeof(struct rx_port_stats) + BNXT_PORT_STAT_PADDING, &softc->hw_rx_port_stats, 0); if (rc) goto hw_port_rx_stats_alloc_fail; bus_dmamap_sync(softc->hw_rx_port_stats.idi_tag, softc->hw_rx_port_stats.idi_map, BUS_DMASYNC_PREREAD); rc = iflib_dma_alloc(ctx, sizeof(struct tx_port_stats) + BNXT_PORT_STAT_PADDING, &softc->hw_tx_port_stats, 0); if (rc) goto hw_port_tx_stats_alloc_fail; bus_dmamap_sync(softc->hw_tx_port_stats.idi_tag, softc->hw_tx_port_stats.idi_map, BUS_DMASYNC_PREREAD); softc->rx_port_stats = (void *) softc->hw_rx_port_stats.idi_vaddr; softc->tx_port_stats = (void *) softc->hw_tx_port_stats.idi_vaddr; rc = iflib_dma_alloc(ctx, sizeof(struct rx_port_stats_ext), &softc->hw_rx_port_stats_ext, 0); if (rc) goto hw_port_rx_stats_ext_alloc_fail; bus_dmamap_sync(softc->hw_rx_port_stats_ext.idi_tag, softc->hw_rx_port_stats_ext.idi_map, BUS_DMASYNC_PREREAD); rc = iflib_dma_alloc(ctx, sizeof(struct tx_port_stats_ext), &softc->hw_tx_port_stats_ext, 0); if (rc) goto hw_port_tx_stats_ext_alloc_fail; bus_dmamap_sync(softc->hw_tx_port_stats_ext.idi_tag, softc->hw_tx_port_stats_ext.idi_map, BUS_DMASYNC_PREREAD); softc->rx_port_stats_ext = (void *) softc->hw_rx_port_stats_ext.idi_vaddr; softc->tx_port_stats_ext = (void *) softc->hw_tx_port_stats_ext.idi_vaddr; for (i = 0; i < nrxqsets; i++) { /* Allocation the completion ring */ softc->rx_cp_rings[i].stats_ctx_id = HWRM_NA_SIGNATURE; softc->rx_cp_rings[i].ring.phys_id = (uint16_t)HWRM_NA_SIGNATURE; softc->rx_cp_rings[i].ring.softc = softc; softc->rx_cp_rings[i].ring.idx = i; softc->rx_cp_rings[i].ring.id = i + 1; softc->rx_cp_rings[i].ring.doorbell = (BNXT_CHIP_P5_PLUS(softc)) ? softc->legacy_db_size : softc->rx_cp_rings[i].ring.id * 0x80; /* * If this ring overflows, RX stops working. */ softc->rx_cp_rings[i].ring.ring_size = softc->scctx->isc_nrxd[0]; softc->rx_cp_rings[i].ring.db_ring_mask = softc->rx_cp_rings[i].ring.ring_size - 1; softc->rx_cp_rings[i].ring.vaddr = vaddrs[i * nrxqs]; softc->rx_cp_rings[i].ring.paddr = paddrs[i * nrxqs]; /* Allocate the RX ring */ softc->rx_rings[i].phys_id = (uint16_t)HWRM_NA_SIGNATURE; softc->rx_rings[i].softc = softc; softc->rx_rings[i].idx = i; softc->rx_rings[i].id = i + 1; softc->rx_rings[i].doorbell = (BNXT_CHIP_P5_PLUS(softc)) ? softc->legacy_db_size : softc->rx_rings[i].id * 0x80; softc->rx_rings[i].ring_size = softc->scctx->isc_nrxd[1]; softc->rx_rings[i].db_ring_mask = softc->rx_rings[i].ring_size -1; softc->rx_rings[i].vaddr = vaddrs[i * nrxqs + 1]; softc->rx_rings[i].paddr = paddrs[i * nrxqs + 1]; /* Allocate the TPA start buffer */ softc->rx_rings[i].tpa_start = malloc(sizeof(struct bnxt_full_tpa_start) * (RX_TPA_START_CMPL_AGG_ID_MASK >> RX_TPA_START_CMPL_AGG_ID_SFT), M_DEVBUF, M_NOWAIT | M_ZERO); if (softc->rx_rings[i].tpa_start == NULL) { rc = -ENOMEM; device_printf(softc->dev, "Unable to allocate space for TPA\n"); goto tpa_alloc_fail; } /* Allocate the AG ring */ softc->ag_rings[i].phys_id = (uint16_t)HWRM_NA_SIGNATURE; softc->ag_rings[i].softc = softc; softc->ag_rings[i].idx = i; softc->ag_rings[i].id = nrxqsets + i + 1; softc->ag_rings[i].doorbell = (BNXT_CHIP_P5_PLUS(softc)) ? softc->legacy_db_size : softc->ag_rings[i].id * 0x80; softc->ag_rings[i].ring_size = softc->scctx->isc_nrxd[2]; softc->ag_rings[i].db_ring_mask = softc->ag_rings[i].ring_size - 1; softc->ag_rings[i].vaddr = vaddrs[i * nrxqs + 2]; softc->ag_rings[i].paddr = paddrs[i * nrxqs + 2]; /* Allocate the ring group */ softc->grp_info[i].grp_id = (uint16_t)HWRM_NA_SIGNATURE; softc->grp_info[i].stats_ctx = softc->rx_cp_rings[i].stats_ctx_id; softc->grp_info[i].rx_ring_id = softc->rx_rings[i].phys_id; softc->grp_info[i].ag_ring_id = softc->ag_rings[i].phys_id; softc->grp_info[i].cp_ring_id = softc->rx_cp_rings[i].ring.phys_id; bnxt_create_rx_sysctls(softc, i); } /* * When SR-IOV is enabled, avoid each VF sending PORT_QSTATS * HWRM every sec with which firmware timeouts can happen */ if (BNXT_PF(softc)) bnxt_create_port_stats_sysctls(softc); /* And finally, the VNIC */ softc->vnic_info.id = (uint16_t)HWRM_NA_SIGNATURE; softc->vnic_info.filter_id = -1; softc->vnic_info.def_ring_grp = (uint16_t)HWRM_NA_SIGNATURE; softc->vnic_info.cos_rule = (uint16_t)HWRM_NA_SIGNATURE; softc->vnic_info.lb_rule = (uint16_t)HWRM_NA_SIGNATURE; softc->vnic_info.rx_mask = HWRM_CFA_L2_SET_RX_MASK_INPUT_MASK_BCAST | HWRM_CFA_L2_SET_RX_MASK_INPUT_MASK_ANYVLAN_NONVLAN; softc->vnic_info.mc_list_count = 0; softc->vnic_info.flags = BNXT_VNIC_FLAG_DEFAULT; rc = iflib_dma_alloc(ctx, BNXT_MAX_MC_ADDRS * ETHER_ADDR_LEN, &softc->vnic_info.mc_list, 0); if (rc) goto mc_list_alloc_fail; /* The VNIC RSS Hash Key */ rc = iflib_dma_alloc(ctx, HW_HASH_KEY_SIZE, &softc->vnic_info.rss_hash_key_tbl, 0); if (rc) goto rss_hash_alloc_fail; bus_dmamap_sync(softc->vnic_info.rss_hash_key_tbl.idi_tag, softc->vnic_info.rss_hash_key_tbl.idi_map, BUS_DMASYNC_PREWRITE); memcpy(softc->vnic_info.rss_hash_key_tbl.idi_vaddr, softc->vnic_info.rss_hash_key, HW_HASH_KEY_SIZE); /* Allocate the RSS tables */ rc = iflib_dma_alloc(ctx, HW_HASH_INDEX_SIZE * sizeof(uint16_t), &softc->vnic_info.rss_grp_tbl, 0); if (rc) goto rss_grp_alloc_fail; bus_dmamap_sync(softc->vnic_info.rss_grp_tbl.idi_tag, softc->vnic_info.rss_grp_tbl.idi_map, BUS_DMASYNC_PREWRITE); memset(softc->vnic_info.rss_grp_tbl.idi_vaddr, 0xff, softc->vnic_info.rss_grp_tbl.idi_size); softc->nrxqsets = nrxqsets; return rc; rss_grp_alloc_fail: iflib_dma_free(&softc->vnic_info.rss_hash_key_tbl); rss_hash_alloc_fail: iflib_dma_free(&softc->vnic_info.mc_list); mc_list_alloc_fail: for (i = i - 1; i >= 0; i--) { if (softc->rx_rings[i].tpa_start) free(softc->rx_rings[i].tpa_start, M_DEVBUF); } tpa_alloc_fail: iflib_dma_free(&softc->hw_tx_port_stats_ext); hw_port_tx_stats_ext_alloc_fail: iflib_dma_free(&softc->hw_rx_port_stats_ext); hw_port_rx_stats_ext_alloc_fail: iflib_dma_free(&softc->hw_tx_port_stats); hw_port_tx_stats_alloc_fail: iflib_dma_free(&softc->hw_rx_port_stats); hw_port_rx_stats_alloc_fail: for (i=0; i < nrxqsets; i++) { if (softc->rx_stats[i].idi_vaddr) iflib_dma_free(&softc->rx_stats[i]); } hw_stats_alloc_fail: free(softc->grp_info, M_DEVBUF); grp_alloc_fail: free(softc->ag_rings, M_DEVBUF); ag_alloc_fail: free(softc->rx_rings, M_DEVBUF); ring_alloc_fail: free(softc->rx_cp_rings, M_DEVBUF); cp_alloc_fail: return rc; } static void bnxt_free_hwrm_short_cmd_req(struct bnxt_softc *softc) { if (softc->hwrm_short_cmd_req_addr.idi_vaddr) iflib_dma_free(&softc->hwrm_short_cmd_req_addr); softc->hwrm_short_cmd_req_addr.idi_vaddr = NULL; } static int bnxt_alloc_hwrm_short_cmd_req(struct bnxt_softc *softc) { int rc; rc = iflib_dma_alloc(softc->ctx, softc->hwrm_max_req_len, &softc->hwrm_short_cmd_req_addr, BUS_DMA_NOWAIT); return rc; } static void bnxt_free_ring(struct bnxt_softc *softc, struct bnxt_ring_mem_info *rmem) { int i; for (i = 0; i < rmem->nr_pages; i++) { if (!rmem->pg_arr[i].idi_vaddr) continue; iflib_dma_free(&rmem->pg_arr[i]); rmem->pg_arr[i].idi_vaddr = NULL; } if (rmem->pg_tbl.idi_vaddr) { iflib_dma_free(&rmem->pg_tbl); rmem->pg_tbl.idi_vaddr = NULL; } if (rmem->vmem_size && *rmem->vmem) { free(*rmem->vmem, M_DEVBUF); *rmem->vmem = NULL; } } static void bnxt_init_ctx_mem(struct bnxt_ctx_mem_type *ctxm, void *p, int len) { u8 init_val = ctxm->init_value; u16 offset = ctxm->init_offset; u8 *p2 = p; int i; if (!init_val) return; if (offset == BNXT_CTX_INIT_INVALID_OFFSET) { memset(p, init_val, len); return; } for (i = 0; i < len; i += ctxm->entry_size) *(p2 + i + offset) = init_val; } static int bnxt_alloc_ring(struct bnxt_softc *softc, struct bnxt_ring_mem_info *rmem) { uint64_t valid_bit = 0; int i; int rc; if (rmem->flags & (BNXT_RMEM_VALID_PTE_FLAG | BNXT_RMEM_RING_PTE_FLAG)) valid_bit = PTU_PTE_VALID; if ((rmem->nr_pages > 1 || rmem->depth > 0) && !rmem->pg_tbl.idi_vaddr) { size_t pg_tbl_size = rmem->nr_pages * 8; if (rmem->flags & BNXT_RMEM_USE_FULL_PAGE_FLAG) pg_tbl_size = rmem->page_size; rc = iflib_dma_alloc(softc->ctx, pg_tbl_size, &rmem->pg_tbl, 0); if (rc) return -ENOMEM; } for (i = 0; i < rmem->nr_pages; i++) { uint64_t extra_bits = valid_bit; uint64_t *ptr; rc = iflib_dma_alloc(softc->ctx, rmem->page_size, &rmem->pg_arr[i], 0); if (rc) return -ENOMEM; if (rmem->ctx_mem) bnxt_init_ctx_mem(rmem->ctx_mem, rmem->pg_arr[i].idi_vaddr, rmem->page_size); if (rmem->nr_pages > 1 || rmem->depth > 0) { if (i == rmem->nr_pages - 2 && (rmem->flags & BNXT_RMEM_RING_PTE_FLAG)) extra_bits |= PTU_PTE_NEXT_TO_LAST; else if (i == rmem->nr_pages - 1 && (rmem->flags & BNXT_RMEM_RING_PTE_FLAG)) extra_bits |= PTU_PTE_LAST; ptr = (void *) rmem->pg_tbl.idi_vaddr; ptr[i] = htole64(rmem->pg_arr[i].idi_paddr | extra_bits); } } if (rmem->vmem_size) { *rmem->vmem = malloc(rmem->vmem_size, M_DEVBUF, M_NOWAIT | M_ZERO); if (!(*rmem->vmem)) return -ENOMEM; } return 0; } #define HWRM_FUNC_BACKING_STORE_CFG_INPUT_DFLT_ENABLES \ (HWRM_FUNC_BACKING_STORE_CFG_INPUT_ENABLES_QP | \ HWRM_FUNC_BACKING_STORE_CFG_INPUT_ENABLES_SRQ | \ HWRM_FUNC_BACKING_STORE_CFG_INPUT_ENABLES_CQ | \ HWRM_FUNC_BACKING_STORE_CFG_INPUT_ENABLES_VNIC | \ HWRM_FUNC_BACKING_STORE_CFG_INPUT_ENABLES_STAT) static int bnxt_alloc_ctx_mem_blk(struct bnxt_softc *softc, struct bnxt_ctx_pg_info *ctx_pg) { struct bnxt_ring_mem_info *rmem = &ctx_pg->ring_mem; rmem->page_size = BNXT_PAGE_SIZE; rmem->pg_arr = ctx_pg->ctx_arr; rmem->flags = BNXT_RMEM_VALID_PTE_FLAG; if (rmem->depth >= 1) rmem->flags |= BNXT_RMEM_USE_FULL_PAGE_FLAG; return bnxt_alloc_ring(softc, rmem); } static int bnxt_alloc_ctx_pg_tbls(struct bnxt_softc *softc, struct bnxt_ctx_pg_info *ctx_pg, u32 mem_size, u8 depth, struct bnxt_ctx_mem_type *ctxm) { struct bnxt_ring_mem_info *rmem = &ctx_pg->ring_mem; int rc; if (!mem_size) return -EINVAL; ctx_pg->nr_pages = DIV_ROUND_UP(mem_size, BNXT_PAGE_SIZE); if (ctx_pg->nr_pages > MAX_CTX_TOTAL_PAGES) { ctx_pg->nr_pages = 0; return -EINVAL; } if (ctx_pg->nr_pages > MAX_CTX_PAGES || depth > 1) { int nr_tbls, i; rmem->depth = 2; ctx_pg->ctx_pg_tbl = kzalloc(MAX_CTX_PAGES * sizeof(ctx_pg), GFP_KERNEL); if (!ctx_pg->ctx_pg_tbl) return -ENOMEM; nr_tbls = DIV_ROUND_UP(ctx_pg->nr_pages, MAX_CTX_PAGES); rmem->nr_pages = nr_tbls; rc = bnxt_alloc_ctx_mem_blk(softc, ctx_pg); if (rc) return rc; for (i = 0; i < nr_tbls; i++) { struct bnxt_ctx_pg_info *pg_tbl; pg_tbl = kzalloc(sizeof(*pg_tbl), GFP_KERNEL); if (!pg_tbl) return -ENOMEM; ctx_pg->ctx_pg_tbl[i] = pg_tbl; rmem = &pg_tbl->ring_mem; memcpy(&rmem->pg_tbl, &ctx_pg->ctx_arr[i], sizeof(struct iflib_dma_info)); rmem->depth = 1; rmem->nr_pages = MAX_CTX_PAGES; rmem->ctx_mem = ctxm; if (i == (nr_tbls - 1)) { int rem = ctx_pg->nr_pages % MAX_CTX_PAGES; if (rem) rmem->nr_pages = rem; } rc = bnxt_alloc_ctx_mem_blk(softc, pg_tbl); if (rc) break; } } else { rmem->nr_pages = DIV_ROUND_UP(mem_size, BNXT_PAGE_SIZE); if (rmem->nr_pages > 1 || depth) rmem->depth = 1; rmem->ctx_mem = ctxm; rc = bnxt_alloc_ctx_mem_blk(softc, ctx_pg); } return rc; } static void bnxt_free_ctx_pg_tbls(struct bnxt_softc *softc, struct bnxt_ctx_pg_info *ctx_pg) { struct bnxt_ring_mem_info *rmem = &ctx_pg->ring_mem; if (rmem->depth > 1 || ctx_pg->nr_pages > MAX_CTX_PAGES || ctx_pg->ctx_pg_tbl) { int i, nr_tbls = rmem->nr_pages; for (i = 0; i < nr_tbls; i++) { struct bnxt_ctx_pg_info *pg_tbl; struct bnxt_ring_mem_info *rmem2; pg_tbl = ctx_pg->ctx_pg_tbl[i]; if (!pg_tbl) continue; rmem2 = &pg_tbl->ring_mem; bnxt_free_ring(softc, rmem2); ctx_pg->ctx_arr[i].idi_vaddr = NULL; free(pg_tbl , M_DEVBUF); ctx_pg->ctx_pg_tbl[i] = NULL; } kfree(ctx_pg->ctx_pg_tbl); ctx_pg->ctx_pg_tbl = NULL; } bnxt_free_ring(softc, rmem); ctx_pg->nr_pages = 0; } static int bnxt_setup_ctxm_pg_tbls(struct bnxt_softc *softc, struct bnxt_ctx_mem_type *ctxm, u32 entries, u8 pg_lvl) { struct bnxt_ctx_pg_info *ctx_pg = ctxm->pg_info; int i, rc = 0, n = 1; u32 mem_size; if (!ctxm->entry_size || !ctx_pg) return -EINVAL; if (ctxm->instance_bmap) n = hweight32(ctxm->instance_bmap); if (ctxm->entry_multiple) entries = roundup(entries, ctxm->entry_multiple); entries = clamp_t(u32, entries, ctxm->min_entries, ctxm->max_entries); mem_size = entries * ctxm->entry_size; for (i = 0; i < n && !rc; i++) { ctx_pg[i].entries = entries; rc = bnxt_alloc_ctx_pg_tbls(softc, &ctx_pg[i], mem_size, pg_lvl, ctxm->init_value ? ctxm : NULL); } if (!rc) ctxm->mem_valid = 1; return rc; } static void bnxt_free_ctx_mem(struct bnxt_softc *softc) { struct bnxt_ctx_mem_info *ctx = softc->ctx_mem; u16 type; if (!ctx) return; for (type = 0; type < BNXT_CTX_MAX; type++) { struct bnxt_ctx_mem_type *ctxm = &ctx->ctx_arr[type]; struct bnxt_ctx_pg_info *ctx_pg = ctxm->pg_info; int i, n = 1; if (!ctx_pg) continue; if (ctxm->instance_bmap) n = hweight32(ctxm->instance_bmap); for (i = 0; i < n; i++) bnxt_free_ctx_pg_tbls(softc, &ctx_pg[i]); kfree(ctx_pg); ctxm->pg_info = NULL; } ctx->flags &= ~BNXT_CTX_FLAG_INITED; kfree(ctx); softc->ctx_mem = NULL; } static int bnxt_backing_store_cfg_v2(struct bnxt_softc *softc, u32 ena) { struct bnxt_ctx_mem_info *ctx = softc->ctx_mem; struct bnxt_ctx_mem_type *ctxm; u16 last_type = BNXT_CTX_INV; int rc = 0; u16 type; if (BNXT_PF(softc)) { for (type = BNXT_CTX_SRT_TRACE; type <= BNXT_CTX_ROCE_HWRM_TRACE; type++) { ctxm = &ctx->ctx_arr[type]; if (!(ctxm->flags & BNXT_CTX_MEM_TYPE_VALID)) continue; rc = bnxt_setup_ctxm_pg_tbls(softc, ctxm, ctxm->max_entries, 1); if (rc) { device_printf(softc->dev, "Unable to setup ctx page for type:0x%x.\n", type); rc = 0; continue; } /* ckp TODO: this is trace buffer related stuff, so keeping it diabled now. needs revisit */ //bnxt_bs_trace_init(bp, ctxm, type - BNXT_CTX_SRT_TRACE); last_type = type; } } if (last_type == BNXT_CTX_INV) { if (!ena) return 0; else if (ena & HWRM_FUNC_BACKING_STORE_CFG_INPUT_ENABLES_TIM) last_type = BNXT_CTX_MAX - 1; else last_type = BNXT_CTX_L2_MAX - 1; } ctx->ctx_arr[last_type].last = 1; for (type = 0 ; type < BNXT_CTX_V2_MAX; type++) { ctxm = &ctx->ctx_arr[type]; if (!ctxm->mem_valid) continue; rc = bnxt_hwrm_func_backing_store_cfg_v2(softc, ctxm, ctxm->last); if (rc) return rc; } return 0; } static int bnxt_alloc_ctx_mem(struct bnxt_softc *softc) { struct bnxt_ctx_pg_info *ctx_pg; struct bnxt_ctx_mem_type *ctxm; struct bnxt_ctx_mem_info *ctx; u32 l2_qps, qp1_qps, max_qps; u32 ena, entries_sp, entries; u32 srqs, max_srqs, min; u32 num_mr, num_ah; u32 extra_srqs = 0; u32 extra_qps = 0; u8 pg_lvl = 1; int i, rc; if (!BNXT_CHIP_P5_PLUS(softc)) return 0; rc = bnxt_hwrm_func_backing_store_qcaps(softc); if (rc) { device_printf(softc->dev, "Failed querying context mem capability, rc = %d.\n", rc); return rc; } ctx = softc->ctx_mem; if (!ctx || (ctx->flags & BNXT_CTX_FLAG_INITED)) return 0; ena = 0; if (BNXT_VF(softc)) goto skip_legacy; ctxm = &ctx->ctx_arr[BNXT_CTX_QP]; l2_qps = ctxm->qp_l2_entries; qp1_qps = ctxm->qp_qp1_entries; max_qps = ctxm->max_entries; ctxm = &ctx->ctx_arr[BNXT_CTX_SRQ]; srqs = ctxm->srq_l2_entries; max_srqs = ctxm->max_entries; if (softc->flags & BNXT_FLAG_ROCE_CAP) { pg_lvl = 2; if (BNXT_SW_RES_LMT(softc)) { extra_qps = max_qps - l2_qps - qp1_qps; extra_srqs = max_srqs - srqs; } else { extra_qps = min_t(uint32_t, 65536, max_qps - l2_qps - qp1_qps); extra_srqs = min_t(uint32_t, 8192, max_srqs - srqs); } } ctxm = &ctx->ctx_arr[BNXT_CTX_QP]; rc = bnxt_setup_ctxm_pg_tbls(softc, ctxm, l2_qps + qp1_qps + extra_qps, pg_lvl); if (rc) return rc; ctxm = &ctx->ctx_arr[BNXT_CTX_SRQ]; rc = bnxt_setup_ctxm_pg_tbls(softc, ctxm, srqs + extra_srqs, pg_lvl); if (rc) return rc; ctxm = &ctx->ctx_arr[BNXT_CTX_CQ]; rc = bnxt_setup_ctxm_pg_tbls(softc, ctxm, ctxm->cq_l2_entries + extra_qps * 2, pg_lvl); if (rc) return rc; ctxm = &ctx->ctx_arr[BNXT_CTX_VNIC]; rc = bnxt_setup_ctxm_pg_tbls(softc, ctxm, ctxm->max_entries, 1); if (rc) return rc; ctxm = &ctx->ctx_arr[BNXT_CTX_STAT]; rc = bnxt_setup_ctxm_pg_tbls(softc, ctxm, ctxm->max_entries, 1); if (rc) return rc; if (!(softc->flags & BNXT_FLAG_ROCE_CAP)) goto skip_rdma; ctxm = &ctx->ctx_arr[BNXT_CTX_MRAV]; ctx_pg = ctxm->pg_info; /* 128K extra is needed to accomodate static AH context * allocation by f/w. */ num_mr = min_t(u32, ctxm->max_entries / 2, 1024 * 256); num_ah = min_t(u32, num_mr, 1024 * 128); rc = bnxt_setup_ctxm_pg_tbls(softc, ctxm, num_mr + num_ah, 2); if (rc) return rc; ctx_pg->entries = num_mr + num_ah; ena = HWRM_FUNC_BACKING_STORE_CFG_INPUT_ENABLES_MRAV; if (ctxm->mrav_num_entries_units) ctx_pg->entries = ((num_mr / ctxm->mrav_num_entries_units) << 16) | (num_ah / ctxm->mrav_num_entries_units); ctxm = &ctx->ctx_arr[BNXT_CTX_TIM]; rc = bnxt_setup_ctxm_pg_tbls(softc, ctxm, l2_qps + qp1_qps + extra_qps, 1); if (rc) return rc; ena |= HWRM_FUNC_BACKING_STORE_CFG_INPUT_ENABLES_TIM; skip_rdma: ctxm = &ctx->ctx_arr[BNXT_CTX_STQM]; min = ctxm->min_entries; entries_sp = ctx->ctx_arr[BNXT_CTX_VNIC].vnic_entries + l2_qps + 2 * (extra_qps + qp1_qps) + min; rc = bnxt_setup_ctxm_pg_tbls(softc, ctxm, entries_sp, 2); if (rc) return rc; ctxm = &ctx->ctx_arr[BNXT_CTX_FTQM]; entries = l2_qps + 2 * (extra_qps + qp1_qps); rc = bnxt_setup_ctxm_pg_tbls(softc, ctxm, entries, 2); if (rc) return rc; for (i = 0; i < ctx->tqm_fp_rings_count + 1; i++) { if (i < BNXT_MAX_TQM_LEGACY_RINGS) ena |= HWRM_FUNC_BACKING_STORE_CFG_INPUT_ENABLES_TQM_SP << i; else ena |= HWRM_FUNC_BACKING_STORE_CFG_INPUT_ENABLES_TQM_RING8; } ena |= HWRM_FUNC_BACKING_STORE_CFG_INPUT_DFLT_ENABLES; skip_legacy: if (BNXT_CHIP_P7(softc)) { if (softc->fw_cap & BNXT_FW_CAP_BACKING_STORE_V2) rc = bnxt_backing_store_cfg_v2(softc, ena); } else { rc = bnxt_hwrm_func_backing_store_cfg(softc, ena); } if (rc) { device_printf(softc->dev, "Failed configuring context mem, rc = %d.\n", rc); return rc; } ctx->flags |= BNXT_CTX_FLAG_INITED; return 0; } /* * If we update the index, a write barrier is needed after the write to ensure * the completion ring has space before the RX/TX ring does. Since we can't * make the RX and AG doorbells covered by the same barrier without remapping * MSI-X vectors, we create the barrier over the enture doorbell bar. * TODO: Remap the MSI-X vectors to allow a barrier to only cover the doorbells * for a single ring group. * * A barrier of just the size of the write is used to ensure the ordering * remains correct and no writes are lost. */ static void bnxt_cuw_db_rx(void *db_ptr, uint16_t idx) { struct bnxt_ring *ring = (struct bnxt_ring *) db_ptr; struct bnxt_bar_info *db_bar = &ring->softc->doorbell_bar; bus_space_barrier(db_bar->tag, db_bar->handle, ring->doorbell, 4, BUS_SPACE_BARRIER_WRITE); bus_space_write_4(db_bar->tag, db_bar->handle, ring->doorbell, htole32(RX_DOORBELL_KEY_RX | idx)); } static void bnxt_cuw_db_tx(void *db_ptr, uint16_t idx) { struct bnxt_ring *ring = (struct bnxt_ring *) db_ptr; struct bnxt_bar_info *db_bar = &ring->softc->doorbell_bar; bus_space_barrier(db_bar->tag, db_bar->handle, ring->doorbell, 4, BUS_SPACE_BARRIER_WRITE); bus_space_write_4(db_bar->tag, db_bar->handle, ring->doorbell, htole32(TX_DOORBELL_KEY_TX | idx)); } static void bnxt_cuw_db_cq(void *db_ptr, bool enable_irq) { struct bnxt_cp_ring *cpr = (struct bnxt_cp_ring *) db_ptr; struct bnxt_bar_info *db_bar = &cpr->ring.softc->doorbell_bar; bus_space_barrier(db_bar->tag, db_bar->handle, cpr->ring.doorbell, 4, BUS_SPACE_BARRIER_WRITE); bus_space_write_4(db_bar->tag, db_bar->handle, cpr->ring.doorbell, htole32(CMPL_DOORBELL_KEY_CMPL | ((cpr->cons == UINT32_MAX) ? 0 : (cpr->cons | CMPL_DOORBELL_IDX_VALID)) | ((enable_irq) ? 0 : CMPL_DOORBELL_MASK))); bus_space_barrier(db_bar->tag, db_bar->handle, 0, db_bar->size, BUS_SPACE_BARRIER_WRITE); } static void bnxt_thor_db_rx(void *db_ptr, uint16_t idx) { struct bnxt_ring *ring = (struct bnxt_ring *) db_ptr; struct bnxt_bar_info *db_bar = &ring->softc->doorbell_bar; bus_space_barrier(db_bar->tag, db_bar->handle, ring->doorbell, 8, BUS_SPACE_BARRIER_WRITE); bus_space_write_8(db_bar->tag, db_bar->handle, ring->doorbell, htole64((DBR_PATH_L2 | DBR_TYPE_SRQ | idx) | ((uint64_t)ring->phys_id << DBR_XID_SFT))); } static void bnxt_thor_db_tx(void *db_ptr, uint16_t idx) { struct bnxt_ring *ring = (struct bnxt_ring *) db_ptr; struct bnxt_bar_info *db_bar = &ring->softc->doorbell_bar; bus_space_barrier(db_bar->tag, db_bar->handle, ring->doorbell, 8, BUS_SPACE_BARRIER_WRITE); bus_space_write_8(db_bar->tag, db_bar->handle, ring->doorbell, htole64((DBR_PATH_L2 | DBR_TYPE_SQ | idx) | ((uint64_t)ring->phys_id << DBR_XID_SFT))); } static void bnxt_thor_db_rx_cq(void *db_ptr, bool enable_irq) { struct bnxt_cp_ring *cpr = (struct bnxt_cp_ring *) db_ptr; struct bnxt_bar_info *db_bar = &cpr->ring.softc->doorbell_bar; dbc_dbc_t db_msg = { 0 }; uint32_t cons = cpr->cons; if (cons == UINT32_MAX) cons = 0; else cons = RING_NEXT(&cpr->ring, cons); db_msg.index = ((cons << DBC_DBC_INDEX_SFT) & DBC_DBC_INDEX_MASK); db_msg.type_path_xid = ((cpr->ring.phys_id << DBC_DBC_XID_SFT) & DBC_DBC_XID_MASK) | DBC_DBC_PATH_L2 | ((enable_irq) ? DBC_DBC_TYPE_CQ_ARMALL: DBC_DBC_TYPE_CQ); bus_space_barrier(db_bar->tag, db_bar->handle, cpr->ring.doorbell, 8, BUS_SPACE_BARRIER_WRITE); bus_space_write_8(db_bar->tag, db_bar->handle, cpr->ring.doorbell, htole64(*(uint64_t *)&db_msg)); bus_space_barrier(db_bar->tag, db_bar->handle, 0, db_bar->size, BUS_SPACE_BARRIER_WRITE); } static void bnxt_thor_db_tx_cq(void *db_ptr, bool enable_irq) { struct bnxt_cp_ring *cpr = (struct bnxt_cp_ring *) db_ptr; struct bnxt_bar_info *db_bar = &cpr->ring.softc->doorbell_bar; dbc_dbc_t db_msg = { 0 }; uint32_t cons = cpr->cons; db_msg.index = ((cons << DBC_DBC_INDEX_SFT) & DBC_DBC_INDEX_MASK); db_msg.type_path_xid = ((cpr->ring.phys_id << DBC_DBC_XID_SFT) & DBC_DBC_XID_MASK) | DBC_DBC_PATH_L2 | ((enable_irq) ? DBC_DBC_TYPE_CQ_ARMALL: DBC_DBC_TYPE_CQ); bus_space_barrier(db_bar->tag, db_bar->handle, cpr->ring.doorbell, 8, BUS_SPACE_BARRIER_WRITE); bus_space_write_8(db_bar->tag, db_bar->handle, cpr->ring.doorbell, htole64(*(uint64_t *)&db_msg)); bus_space_barrier(db_bar->tag, db_bar->handle, 0, db_bar->size, BUS_SPACE_BARRIER_WRITE); } static void bnxt_thor_db_nq(void *db_ptr, bool enable_irq) { struct bnxt_cp_ring *cpr = (struct bnxt_cp_ring *) db_ptr; struct bnxt_bar_info *db_bar = &cpr->ring.softc->doorbell_bar; dbc_dbc_t db_msg = { 0 }; uint32_t cons = cpr->cons; db_msg.index = ((cons << DBC_DBC_INDEX_SFT) & DBC_DBC_INDEX_MASK); db_msg.type_path_xid = ((cpr->ring.phys_id << DBC_DBC_XID_SFT) & DBC_DBC_XID_MASK) | DBC_DBC_PATH_L2 | ((enable_irq) ? DBC_DBC_TYPE_NQ_ARM: DBC_DBC_TYPE_NQ); bus_space_barrier(db_bar->tag, db_bar->handle, cpr->ring.doorbell, 8, BUS_SPACE_BARRIER_WRITE); bus_space_write_8(db_bar->tag, db_bar->handle, cpr->ring.doorbell, htole64(*(uint64_t *)&db_msg)); bus_space_barrier(db_bar->tag, db_bar->handle, 0, db_bar->size, BUS_SPACE_BARRIER_WRITE); } static void bnxt_thor2_db_rx(void *db_ptr, uint16_t idx) { struct bnxt_ring *ring = (struct bnxt_ring *) db_ptr; struct bnxt_bar_info *db_bar = &ring->softc->doorbell_bar; uint64_t db_val; if (idx >= ring->ring_size) { device_printf(ring->softc->dev, "%s: BRCM DBG: idx: %d crossed boundary\n", __func__, idx); return; } db_val = ((DBR_PATH_L2 | DBR_TYPE_SRQ | DBR_VALID | idx) | ((uint64_t)ring->phys_id << DBR_XID_SFT)); /* Add the PI index */ db_val |= DB_RING_IDX(ring, idx, ring->epoch_arr[idx]); bus_space_barrier(db_bar->tag, db_bar->handle, ring->doorbell, 8, BUS_SPACE_BARRIER_WRITE); bus_space_write_8(db_bar->tag, db_bar->handle, ring->doorbell, htole64(db_val)); } static void bnxt_thor2_db_tx(void *db_ptr, uint16_t idx) { struct bnxt_ring *ring = (struct bnxt_ring *) db_ptr; struct bnxt_bar_info *db_bar = &ring->softc->doorbell_bar; uint64_t db_val; if (idx >= ring->ring_size) { device_printf(ring->softc->dev, "%s: BRCM DBG: idx: %d crossed boundary\n", __func__, idx); return; } db_val = ((DBR_PATH_L2 | DBR_TYPE_SQ | DBR_VALID | idx) | ((uint64_t)ring->phys_id << DBR_XID_SFT)); /* Add the PI index */ db_val |= DB_RING_IDX(ring, idx, ring->epoch_arr[idx]); bus_space_barrier(db_bar->tag, db_bar->handle, ring->doorbell, 8, BUS_SPACE_BARRIER_WRITE); bus_space_write_8(db_bar->tag, db_bar->handle, ring->doorbell, htole64(db_val)); } static void bnxt_thor2_db_rx_cq(void *db_ptr, bool enable_irq) { struct bnxt_cp_ring *cpr = (struct bnxt_cp_ring *) db_ptr; struct bnxt_bar_info *db_bar = &cpr->ring.softc->doorbell_bar; u64 db_msg = { 0 }; uint32_t cons = cpr->raw_cons; uint32_t toggle = 0; if (cons == UINT32_MAX) cons = 0; if (enable_irq == true) toggle = cpr->toggle; db_msg = DBR_PATH_L2 | ((u64)cpr->ring.phys_id << DBR_XID_SFT) | DBR_VALID | DB_RING_IDX_CMP(&cpr->ring, cons) | DB_TOGGLE(toggle); if (enable_irq) db_msg |= DBR_TYPE_CQ_ARMALL; else db_msg |= DBR_TYPE_CQ; bus_space_barrier(db_bar->tag, db_bar->handle, cpr->ring.doorbell, 8, BUS_SPACE_BARRIER_WRITE); bus_space_write_8(db_bar->tag, db_bar->handle, cpr->ring.doorbell, htole64(*(uint64_t *)&db_msg)); bus_space_barrier(db_bar->tag, db_bar->handle, 0, db_bar->size, BUS_SPACE_BARRIER_WRITE); } static void bnxt_thor2_db_tx_cq(void *db_ptr, bool enable_irq) { struct bnxt_cp_ring *cpr = (struct bnxt_cp_ring *) db_ptr; struct bnxt_bar_info *db_bar = &cpr->ring.softc->doorbell_bar; u64 db_msg = { 0 }; uint32_t cons = cpr->raw_cons; uint32_t toggle = 0; if (enable_irq == true) toggle = cpr->toggle; db_msg = DBR_PATH_L2 | ((u64)cpr->ring.phys_id << DBR_XID_SFT) | DBR_VALID | DB_RING_IDX_CMP(&cpr->ring, cons) | DB_TOGGLE(toggle); if (enable_irq) db_msg |= DBR_TYPE_CQ_ARMALL; else db_msg |= DBR_TYPE_CQ; bus_space_barrier(db_bar->tag, db_bar->handle, cpr->ring.doorbell, 8, BUS_SPACE_BARRIER_WRITE); bus_space_write_8(db_bar->tag, db_bar->handle, cpr->ring.doorbell, htole64(*(uint64_t *)&db_msg)); bus_space_barrier(db_bar->tag, db_bar->handle, 0, db_bar->size, BUS_SPACE_BARRIER_WRITE); } static void bnxt_thor2_db_nq(void *db_ptr, bool enable_irq) { struct bnxt_cp_ring *cpr = (struct bnxt_cp_ring *) db_ptr; struct bnxt_bar_info *db_bar = &cpr->ring.softc->doorbell_bar; u64 db_msg = { 0 }; uint32_t cons = cpr->raw_cons; uint32_t toggle = 0; if (enable_irq == true) toggle = cpr->toggle; db_msg = DBR_PATH_L2 | ((u64)cpr->ring.phys_id << DBR_XID_SFT) | DBR_VALID | DB_RING_IDX_CMP(&cpr->ring, cons) | DB_TOGGLE(toggle); if (enable_irq) db_msg |= DBR_TYPE_NQ_ARM; else db_msg |= DBR_TYPE_NQ_MASK; bus_space_barrier(db_bar->tag, db_bar->handle, cpr->ring.doorbell, 8, BUS_SPACE_BARRIER_WRITE); bus_space_write_8(db_bar->tag, db_bar->handle, cpr->ring.doorbell, htole64(*(uint64_t *)&db_msg)); bus_space_barrier(db_bar->tag, db_bar->handle, 0, db_bar->size, BUS_SPACE_BARRIER_WRITE); } struct bnxt_softc *bnxt_find_dev(uint32_t domain, uint32_t bus, uint32_t dev_fn, char *dev_name) { struct bnxt_softc_list *sc = NULL; SLIST_FOREACH(sc, &pf_list, next) { /* get the softc reference based on device name */ if (dev_name && !strncmp(dev_name, if_name(iflib_get_ifp(sc->softc->ctx)), BNXT_MAX_STR)) { return sc->softc; } /* get the softc reference based on domain,bus,device,function */ if (!dev_name && (domain == sc->softc->domain) && (bus == sc->softc->bus) && (dev_fn == sc->softc->dev_fn)) { return sc->softc; } } return NULL; } static void bnxt_verify_asym_queues(struct bnxt_softc *softc) { uint8_t i, lltc = 0; if (!softc->max_lltc) return; /* Verify that lossless TX and RX queues are in the same index */ for (i = 0; i < softc->max_tc; i++) { if (BNXT_LLQ(softc->tx_q_info[i].queue_profile) && BNXT_LLQ(softc->rx_q_info[i].queue_profile)) lltc++; } softc->max_lltc = min(softc->max_lltc, lltc); } static int bnxt_hwrm_poll(struct bnxt_softc *bp) { struct hwrm_ver_get_output *resp = (void *)bp->hwrm_cmd_resp.idi_vaddr; struct hwrm_ver_get_input req = {0}; int rc; bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_VER_GET); req.hwrm_intf_maj = HWRM_VERSION_MAJOR; req.hwrm_intf_min = HWRM_VERSION_MINOR; req.hwrm_intf_upd = HWRM_VERSION_UPDATE; rc = _hwrm_send_message(bp, &req, sizeof(req)); if (rc) return rc; if (resp->flags & HWRM_VER_GET_OUTPUT_FLAGS_DEV_NOT_RDY) rc = -EAGAIN; return rc; } static void bnxt_rtnl_lock_sp(struct bnxt_softc *bp) { /* We are called from bnxt_sp_task which has BNXT_STATE_IN_SP_TASK * set. If the device is being closed, bnxt_close() may be holding * rtnl() and waiting for BNXT_STATE_IN_SP_TASK to clear. So we * must clear BNXT_STATE_IN_SP_TASK before holding rtnl(). */ clear_bit(BNXT_STATE_IN_SP_TASK, &bp->state); rtnl_lock(); } static void bnxt_rtnl_unlock_sp(struct bnxt_softc *bp) { set_bit(BNXT_STATE_IN_SP_TASK, &bp->state); rtnl_unlock(); } static void bnxt_fw_fatal_close(struct bnxt_softc *softc) { bnxt_disable_intr(softc->ctx); if (pci_is_enabled(softc->pdev)) pci_disable_device(softc->pdev); } static u32 bnxt_fw_health_readl(struct bnxt_softc *bp, int reg_idx) { struct bnxt_fw_health *fw_health = bp->fw_health; u32 reg = fw_health->regs[reg_idx]; u32 reg_type, reg_off, val = 0; reg_type = BNXT_FW_HEALTH_REG_TYPE(reg); reg_off = BNXT_FW_HEALTH_REG_OFF(reg); switch (reg_type) { case BNXT_FW_HEALTH_REG_TYPE_CFG: pci_read_config_dword(bp->pdev, reg_off, &val); break; case BNXT_FW_HEALTH_REG_TYPE_GRC: reg_off = fw_health->mapped_regs[reg_idx]; fallthrough; case BNXT_FW_HEALTH_REG_TYPE_BAR0: val = readl_fbsd(bp, reg_off, 0); break; case BNXT_FW_HEALTH_REG_TYPE_BAR1: val = readl_fbsd(bp, reg_off, 2); break; } if (reg_idx == BNXT_FW_RESET_INPROG_REG) val &= fw_health->fw_reset_inprog_reg_mask; return val; } static void bnxt_fw_reset_close(struct bnxt_softc *bp) { int i; bnxt_ulp_stop(bp); /* When firmware is in fatal state, quiesce device and disable * bus master to prevent any potential bad DMAs before freeing * kernel memory. */ if (test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state)) { u16 val = 0; val = pci_read_config(bp->dev, PCI_SUBSYSTEM_ID, 2); if (val == 0xffff) { bp->fw_reset_min_dsecs = 0; } bnxt_fw_fatal_close(bp); } iflib_request_reset(bp->ctx); bnxt_stop(bp->ctx); bnxt_hwrm_func_drv_unrgtr(bp, false); for (i = bp->nrxqsets-1; i>=0; i--) { if (BNXT_CHIP_P5_PLUS(bp)) iflib_irq_free(bp->ctx, &bp->nq_rings[i].irq); else iflib_irq_free(bp->ctx, &bp->rx_cp_rings[i].irq); } if (pci_is_enabled(bp->pdev)) pci_disable_device(bp->pdev); pci_disable_busmaster(bp->dev); bnxt_free_ctx_mem(bp); } static bool is_bnxt_fw_ok(struct bnxt_softc *bp) { struct bnxt_fw_health *fw_health = bp->fw_health; bool no_heartbeat = false, has_reset = false; u32 val; val = bnxt_fw_health_readl(bp, BNXT_FW_HEARTBEAT_REG); if (val == fw_health->last_fw_heartbeat) no_heartbeat = true; val = bnxt_fw_health_readl(bp, BNXT_FW_RESET_CNT_REG); if (val != fw_health->last_fw_reset_cnt) has_reset = true; if (!no_heartbeat && has_reset) return true; return false; } void bnxt_fw_reset(struct bnxt_softc *bp) { bnxt_rtnl_lock_sp(bp); if (test_bit(BNXT_STATE_OPEN, &bp->state) && !test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) { int tmo; set_bit(BNXT_STATE_IN_FW_RESET, &bp->state); bnxt_fw_reset_close(bp); if ((bp->fw_cap & BNXT_FW_CAP_ERR_RECOVER_RELOAD)) { bp->fw_reset_state = BNXT_FW_RESET_STATE_POLL_FW_DOWN; tmo = HZ / 10; } else { bp->fw_reset_state = BNXT_FW_RESET_STATE_ENABLE_DEV; tmo = bp->fw_reset_min_dsecs * HZ /10; } bnxt_queue_fw_reset_work(bp, tmo); } bnxt_rtnl_unlock_sp(bp); } static void bnxt_queue_fw_reset_work(struct bnxt_softc *bp, unsigned long delay) { if (!(test_bit(BNXT_STATE_IN_FW_RESET, &bp->state))) return; if (BNXT_PF(bp)) queue_delayed_work(bnxt_pf_wq, &bp->fw_reset_task, delay); else schedule_delayed_work(&bp->fw_reset_task, delay); } void bnxt_queue_sp_work(struct bnxt_softc *bp) { if (BNXT_PF(bp)) queue_work(bnxt_pf_wq, &bp->sp_task); else schedule_work(&bp->sp_task); } static void bnxt_fw_reset_writel(struct bnxt_softc *bp, int reg_idx) { struct bnxt_fw_health *fw_health = bp->fw_health; u32 reg = fw_health->fw_reset_seq_regs[reg_idx]; u32 val = fw_health->fw_reset_seq_vals[reg_idx]; u32 reg_type, reg_off, delay_msecs; delay_msecs = fw_health->fw_reset_seq_delay_msec[reg_idx]; reg_type = BNXT_FW_HEALTH_REG_TYPE(reg); reg_off = BNXT_FW_HEALTH_REG_OFF(reg); switch (reg_type) { case BNXT_FW_HEALTH_REG_TYPE_CFG: pci_write_config_dword(bp->pdev, reg_off, val); break; case BNXT_FW_HEALTH_REG_TYPE_GRC: writel_fbsd(bp, BNXT_GRCPF_REG_WINDOW_BASE_OUT + 4, 0, reg_off & BNXT_GRC_BASE_MASK); reg_off = (reg_off & BNXT_GRC_OFFSET_MASK) + 0x2000; fallthrough; case BNXT_FW_HEALTH_REG_TYPE_BAR0: writel_fbsd(bp, reg_off, 0, val); break; case BNXT_FW_HEALTH_REG_TYPE_BAR1: writel_fbsd(bp, reg_off, 2, val); break; } if (delay_msecs) { pci_read_config_dword(bp->pdev, 0, &val); msleep(delay_msecs); } } static void bnxt_reset_all(struct bnxt_softc *bp) { struct bnxt_fw_health *fw_health = bp->fw_health; int i, rc; if (bp->fw_cap & BNXT_FW_CAP_ERR_RECOVER_RELOAD) { bp->fw_reset_timestamp = jiffies; return; } if (fw_health->flags & HWRM_ERROR_RECOVERY_QCFG_OUTPUT_FLAGS_HOST) { for (i = 0; i < fw_health->fw_reset_seq_cnt; i++) bnxt_fw_reset_writel(bp, i); } else if (fw_health->flags & HWRM_ERROR_RECOVERY_QCFG_OUTPUT_FLAGS_CO_CPU) { struct hwrm_fw_reset_input req = {0}; bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_FW_RESET); req.target_id = htole16(HWRM_TARGET_ID_KONG); req.embedded_proc_type = HWRM_FW_RESET_INPUT_EMBEDDED_PROC_TYPE_CHIP; req.selfrst_status = HWRM_FW_RESET_INPUT_SELFRST_STATUS_SELFRSTASAP; req.flags = HWRM_FW_RESET_INPUT_FLAGS_RESET_GRACEFUL; rc = hwrm_send_message(bp, &req, sizeof(req)); if (rc != -ENODEV) device_printf(bp->dev, "Unable to reset FW rc=%d\n", rc); } bp->fw_reset_timestamp = jiffies; } static int __bnxt_alloc_fw_health(struct bnxt_softc *bp) { if (bp->fw_health) return 0; bp->fw_health = kzalloc(sizeof(*bp->fw_health), GFP_KERNEL); if (!bp->fw_health) return -ENOMEM; mutex_init(&bp->fw_health->lock); return 0; } static int bnxt_alloc_fw_health(struct bnxt_softc *bp) { int rc; if (!(bp->fw_cap & BNXT_FW_CAP_HOT_RESET) && !(bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY)) return 0; rc = __bnxt_alloc_fw_health(bp); if (rc) { bp->fw_cap &= ~BNXT_FW_CAP_HOT_RESET; bp->fw_cap &= ~BNXT_FW_CAP_ERROR_RECOVERY; return rc; } return 0; } static inline void __bnxt_map_fw_health_reg(struct bnxt_softc *bp, u32 reg) { writel_fbsd(bp, BNXT_GRCPF_REG_WINDOW_BASE_OUT + BNXT_FW_HEALTH_WIN_MAP_OFF, 0, reg & BNXT_GRC_BASE_MASK); } static int bnxt_map_fw_health_regs(struct bnxt_softc *bp) { struct bnxt_fw_health *fw_health = bp->fw_health; u32 reg_base = 0xffffffff; int i; bp->fw_health->status_reliable = false; bp->fw_health->resets_reliable = false; /* Only pre-map the monitoring GRC registers using window 3 */ for (i = 0; i < 4; i++) { u32 reg = fw_health->regs[i]; if (BNXT_FW_HEALTH_REG_TYPE(reg) != BNXT_FW_HEALTH_REG_TYPE_GRC) continue; if (reg_base == 0xffffffff) reg_base = reg & BNXT_GRC_BASE_MASK; if ((reg & BNXT_GRC_BASE_MASK) != reg_base) return -ERANGE; fw_health->mapped_regs[i] = BNXT_FW_HEALTH_WIN_OFF(reg); } bp->fw_health->status_reliable = true; bp->fw_health->resets_reliable = true; if (reg_base == 0xffffffff) return 0; __bnxt_map_fw_health_reg(bp, reg_base); return 0; } static void bnxt_inv_fw_health_reg(struct bnxt_softc *bp) { struct bnxt_fw_health *fw_health = bp->fw_health; u32 reg_type; if (!fw_health) return; reg_type = BNXT_FW_HEALTH_REG_TYPE(fw_health->regs[BNXT_FW_HEALTH_REG]); if (reg_type == BNXT_FW_HEALTH_REG_TYPE_GRC) fw_health->status_reliable = false; reg_type = BNXT_FW_HEALTH_REG_TYPE(fw_health->regs[BNXT_FW_RESET_CNT_REG]); if (reg_type == BNXT_FW_HEALTH_REG_TYPE_GRC) fw_health->resets_reliable = false; } static int bnxt_hwrm_error_recovery_qcfg(struct bnxt_softc *bp) { struct bnxt_fw_health *fw_health = bp->fw_health; struct hwrm_error_recovery_qcfg_output *resp = (void *)bp->hwrm_cmd_resp.idi_vaddr; struct hwrm_error_recovery_qcfg_input req = {0}; int rc, i; if (!(bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY)) return 0; bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_ERROR_RECOVERY_QCFG); rc = _hwrm_send_message(bp, &req, sizeof(req)); if (rc) goto err_recovery_out; fw_health->flags = le32toh(resp->flags); if ((fw_health->flags & HWRM_ERROR_RECOVERY_QCFG_OUTPUT_FLAGS_CO_CPU) && !(bp->fw_cap & BNXT_FW_CAP_KONG_MB_CHNL)) { rc = -EINVAL; goto err_recovery_out; } fw_health->polling_dsecs = le32toh(resp->driver_polling_freq); fw_health->master_func_wait_dsecs = le32toh(resp->master_func_wait_period); fw_health->normal_func_wait_dsecs = le32toh(resp->normal_func_wait_period); fw_health->post_reset_wait_dsecs = le32toh(resp->master_func_wait_period_after_reset); fw_health->post_reset_max_wait_dsecs = le32toh(resp->max_bailout_time_after_reset); fw_health->regs[BNXT_FW_HEALTH_REG] = le32toh(resp->fw_health_status_reg); fw_health->regs[BNXT_FW_HEARTBEAT_REG] = le32toh(resp->fw_heartbeat_reg); fw_health->regs[BNXT_FW_RESET_CNT_REG] = le32toh(resp->fw_reset_cnt_reg); fw_health->regs[BNXT_FW_RESET_INPROG_REG] = le32toh(resp->reset_inprogress_reg); fw_health->fw_reset_inprog_reg_mask = le32toh(resp->reset_inprogress_reg_mask); fw_health->fw_reset_seq_cnt = resp->reg_array_cnt; if (fw_health->fw_reset_seq_cnt >= 16) { rc = -EINVAL; goto err_recovery_out; } for (i = 0; i < fw_health->fw_reset_seq_cnt; i++) { fw_health->fw_reset_seq_regs[i] = le32toh(resp->reset_reg[i]); fw_health->fw_reset_seq_vals[i] = le32toh(resp->reset_reg_val[i]); fw_health->fw_reset_seq_delay_msec[i] = le32toh(resp->delay_after_reset[i]); } err_recovery_out: if (!rc) rc = bnxt_map_fw_health_regs(bp); if (rc) bp->fw_cap &= ~BNXT_FW_CAP_ERROR_RECOVERY; return rc; } static int bnxt_drv_rgtr(struct bnxt_softc *bp) { int rc; /* determine whether we can support error recovery before * registering with FW */ if (bnxt_alloc_fw_health(bp)) { device_printf(bp->dev, "no memory for firmware error recovery\n"); } else { rc = bnxt_hwrm_error_recovery_qcfg(bp); if (rc) device_printf(bp->dev, "hwrm query error recovery failure rc: %d\n", rc); } rc = bnxt_hwrm_func_drv_rgtr(bp, NULL, 0, false); //sumit dbg: revisit the params if (rc) return -ENODEV; return 0; } static bool bnxt_fw_reset_timeout(struct bnxt_softc *bp) { return time_after(jiffies, bp->fw_reset_timestamp + (bp->fw_reset_max_dsecs * HZ / 10)); } static int bnxt_open(struct bnxt_softc *bp) { int rc = 0; if (BNXT_PF(bp)) rc = bnxt_hwrm_nvm_get_dev_info(bp, &bp->nvm_info->mfg_id, &bp->nvm_info->device_id, &bp->nvm_info->sector_size, &bp->nvm_info->size, &bp->nvm_info->reserved_size, &bp->nvm_info->available_size); /* Get the queue config */ rc = bnxt_hwrm_queue_qportcfg(bp, HWRM_QUEUE_QPORTCFG_INPUT_FLAGS_PATH_TX); if (rc) { device_printf(bp->dev, "reinit: hwrm qportcfg (tx) failed\n"); return rc; } if (bp->is_asym_q) { rc = bnxt_hwrm_queue_qportcfg(bp, HWRM_QUEUE_QPORTCFG_INPUT_FLAGS_PATH_RX); if (rc) { device_printf(bp->dev, "re-init: hwrm qportcfg (rx) failed\n"); return rc; } bnxt_verify_asym_queues(bp); } else { bp->rx_max_q = bp->tx_max_q; memcpy(bp->rx_q_info, bp->tx_q_info, sizeof(bp->rx_q_info)); memcpy(bp->rx_q_ids, bp->tx_q_ids, sizeof(bp->rx_q_ids)); } /* Get the HW capabilities */ rc = bnxt_hwrm_func_qcaps(bp); if (rc) return rc; /* Register the driver with the FW */ rc = bnxt_drv_rgtr(bp); if (rc) return rc; if (bp->hwrm_spec_code >= 0x10803) { rc = bnxt_alloc_ctx_mem(bp); if (rc) { device_printf(bp->dev, "attach: alloc_ctx_mem failed\n"); return rc; } rc = bnxt_hwrm_func_resc_qcaps(bp, true); if (!rc) bp->flags |= BNXT_FLAG_FW_CAP_NEW_RM; } if (BNXT_CHIP_P5_PLUS(bp)) bnxt_hwrm_reserve_pf_rings(bp); /* Get the current configuration of this function */ rc = bnxt_hwrm_func_qcfg(bp); if (rc) { device_printf(bp->dev, "re-init: hwrm func qcfg failed\n"); return rc; } bnxt_msix_intr_assign(bp->ctx, 0); bnxt_init(bp->ctx); bnxt_intr_enable(bp->ctx); if (test_and_clear_bit(BNXT_STATE_FW_RESET_DET, &bp->state)) { if (!test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) { bnxt_ulp_start(bp, 0); } } device_printf(bp->dev, "Network interface is UP and operational\n"); return rc; } static void bnxt_fw_reset_abort(struct bnxt_softc *bp, int rc) { clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state); if (bp->fw_reset_state != BNXT_FW_RESET_STATE_POLL_VF) { bnxt_ulp_start(bp, rc); } bp->fw_reset_state = 0; } static void bnxt_fw_reset_task(struct work_struct *work) { struct bnxt_softc *bp = container_of(work, struct bnxt_softc, fw_reset_task.work); int rc = 0; if (!test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) { device_printf(bp->dev, "bnxt_fw_reset_task() called when not in fw reset mode!\n"); return; } switch (bp->fw_reset_state) { case BNXT_FW_RESET_STATE_POLL_FW_DOWN: { u32 val; val = bnxt_fw_health_readl(bp, BNXT_FW_HEALTH_REG); if (!(val & BNXT_FW_STATUS_SHUTDOWN) && !bnxt_fw_reset_timeout(bp)) { bnxt_queue_fw_reset_work(bp, HZ / 5); return; } if (!bp->fw_health->primary) { u32 wait_dsecs = bp->fw_health->normal_func_wait_dsecs; bp->fw_reset_state = BNXT_FW_RESET_STATE_ENABLE_DEV; bnxt_queue_fw_reset_work(bp, wait_dsecs * HZ / 10); return; } bp->fw_reset_state = BNXT_FW_RESET_STATE_RESET_FW; } fallthrough; case BNXT_FW_RESET_STATE_RESET_FW: bnxt_reset_all(bp); bp->fw_reset_state = BNXT_FW_RESET_STATE_ENABLE_DEV; bnxt_queue_fw_reset_work(bp, bp->fw_reset_min_dsecs * HZ / 10); return; case BNXT_FW_RESET_STATE_ENABLE_DEV: bnxt_inv_fw_health_reg(bp); if (test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state) && !bp->fw_reset_min_dsecs) { u16 val; val = pci_read_config(bp->dev, PCI_SUBSYSTEM_ID, 2); if (val == 0xffff) { if (bnxt_fw_reset_timeout(bp)) { device_printf(bp->dev, "Firmware reset aborted, PCI config space invalid\n"); rc = -ETIMEDOUT; goto fw_reset_abort; } bnxt_queue_fw_reset_work(bp, HZ / 1000); return; } } clear_bit(BNXT_STATE_FW_FATAL_COND, &bp->state); clear_bit(BNXT_STATE_FW_NON_FATAL_COND, &bp->state); if (!pci_is_enabled(bp->pdev)) { if (pci_enable_device(bp->pdev)) { device_printf(bp->dev, "Cannot re-enable PCI device\n"); rc = -ENODEV; goto fw_reset_abort; } } pci_set_master(bp->pdev); bp->fw_reset_state = BNXT_FW_RESET_STATE_POLL_FW; fallthrough; case BNXT_FW_RESET_STATE_POLL_FW: bp->hwrm_cmd_timeo = SHORT_HWRM_CMD_TIMEOUT; rc = bnxt_hwrm_poll(bp); if (rc) { if (bnxt_fw_reset_timeout(bp)) { device_printf(bp->dev, "Firmware reset aborted\n"); goto fw_reset_abort_status; } bnxt_queue_fw_reset_work(bp, HZ / 5); return; } bp->hwrm_cmd_timeo = DFLT_HWRM_CMD_TIMEOUT; bp->fw_reset_state = BNXT_FW_RESET_STATE_OPENING; fallthrough; case BNXT_FW_RESET_STATE_OPENING: rc = bnxt_open(bp); if (rc) { device_printf(bp->dev, "bnxt_open() failed during FW reset\n"); bnxt_fw_reset_abort(bp, rc); rtnl_unlock(); return; } if ((bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY) && bp->fw_health->enabled) { bp->fw_health->last_fw_reset_cnt = bnxt_fw_health_readl(bp, BNXT_FW_RESET_CNT_REG); } bp->fw_reset_state = 0; smp_mb__before_atomic(); clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state); bnxt_ulp_start(bp, 0); clear_bit(BNXT_STATE_FW_ACTIVATE, &bp->state); set_bit(BNXT_STATE_OPEN, &bp->state); rtnl_unlock(); } return; fw_reset_abort_status: if (bp->fw_health->status_reliable || (bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY)) { u32 sts = bnxt_fw_health_readl(bp, BNXT_FW_HEALTH_REG); device_printf(bp->dev, "fw_health_status 0x%x\n", sts); } fw_reset_abort: rtnl_lock(); bnxt_fw_reset_abort(bp, rc); rtnl_unlock(); } static void bnxt_force_fw_reset(struct bnxt_softc *bp) { struct bnxt_fw_health *fw_health = bp->fw_health; u32 wait_dsecs; if (!test_bit(BNXT_STATE_OPEN, &bp->state) || test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) return; bnxt_fw_reset_close(bp); wait_dsecs = fw_health->master_func_wait_dsecs; if (fw_health->primary) { if (fw_health->flags & HWRM_ERROR_RECOVERY_QCFG_OUTPUT_FLAGS_CO_CPU) wait_dsecs = 0; bp->fw_reset_state = BNXT_FW_RESET_STATE_RESET_FW; } else { bp->fw_reset_timestamp = jiffies + wait_dsecs * HZ / 10; wait_dsecs = fw_health->normal_func_wait_dsecs; bp->fw_reset_state = BNXT_FW_RESET_STATE_ENABLE_DEV; } bp->fw_reset_min_dsecs = fw_health->post_reset_wait_dsecs; bp->fw_reset_max_dsecs = fw_health->post_reset_max_wait_dsecs; bnxt_queue_fw_reset_work(bp, wait_dsecs * HZ / 10); } static void bnxt_fw_exception(struct bnxt_softc *bp) { device_printf(bp->dev, "Detected firmware fatal condition, initiating reset\n"); set_bit(BNXT_STATE_FW_FATAL_COND, &bp->state); bnxt_rtnl_lock_sp(bp); bnxt_force_fw_reset(bp); bnxt_rtnl_unlock_sp(bp); } static void __bnxt_fw_recover(struct bnxt_softc *bp) { if (test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state) || test_bit(BNXT_STATE_FW_NON_FATAL_COND, &bp->state)) bnxt_fw_reset(bp); else bnxt_fw_exception(bp); } static void bnxt_devlink_health_fw_report(struct bnxt_softc *bp) { struct bnxt_fw_health *fw_health = bp->fw_health; if (!fw_health) return; if (!fw_health->fw_reporter) { __bnxt_fw_recover(bp); return; } } static void bnxt_sp_task(struct work_struct *work) { struct bnxt_softc *bp = container_of(work, struct bnxt_softc, sp_task); set_bit(BNXT_STATE_IN_SP_TASK, &bp->state); smp_mb__after_atomic(); if (!test_bit(BNXT_STATE_OPEN, &bp->state)) { clear_bit(BNXT_STATE_IN_SP_TASK, &bp->state); return; } if (test_and_clear_bit(BNXT_FW_RESET_NOTIFY_SP_EVENT, &bp->sp_event)) { if (test_bit(BNXT_STATE_FW_FATAL_COND, &bp->state) || test_bit(BNXT_STATE_FW_NON_FATAL_COND, &bp->state)) bnxt_devlink_health_fw_report(bp); else bnxt_fw_reset(bp); } if (test_and_clear_bit(BNXT_FW_EXCEPTION_SP_EVENT, &bp->sp_event)) { if (!is_bnxt_fw_ok(bp)) bnxt_devlink_health_fw_report(bp); } smp_mb__before_atomic(); clear_bit(BNXT_STATE_IN_SP_TASK, &bp->state); } /* Device setup and teardown */ static int bnxt_attach_pre(if_ctx_t ctx) { struct bnxt_softc *softc = iflib_get_softc(ctx); if_softc_ctx_t scctx; int rc = 0; softc->ctx = ctx; softc->dev = iflib_get_dev(ctx); softc->media = iflib_get_media(ctx); softc->scctx = iflib_get_softc_ctx(ctx); softc->sctx = iflib_get_sctx(ctx); scctx = softc->scctx; /* TODO: Better way of detecting NPAR/VF is needed */ switch (pci_get_device(softc->dev)) { case BCM57402_NPAR: case BCM57404_NPAR: case BCM57406_NPAR: case BCM57407_NPAR: case BCM57412_NPAR1: case BCM57412_NPAR2: case BCM57414_NPAR1: case BCM57414_NPAR2: case BCM57416_NPAR1: case BCM57416_NPAR2: case BCM57504_NPAR: softc->flags |= BNXT_FLAG_NPAR; break; case NETXTREME_C_VF1: case NETXTREME_C_VF2: case NETXTREME_C_VF3: case NETXTREME_E_VF1: case NETXTREME_E_VF2: case NETXTREME_E_VF3: softc->flags |= BNXT_FLAG_VF; break; } softc->domain = pci_get_domain(softc->dev); softc->bus = pci_get_bus(softc->dev); softc->slot = pci_get_slot(softc->dev); softc->function = pci_get_function(softc->dev); softc->dev_fn = PCI_DEVFN(softc->slot, softc->function); if (bnxt_num_pfs == 0) SLIST_INIT(&pf_list); bnxt_num_pfs++; softc->list.softc = softc; SLIST_INSERT_HEAD(&pf_list, &softc->list, next); pci_enable_busmaster(softc->dev); if (bnxt_pci_mapping(softc)) { device_printf(softc->dev, "PCI mapping failed\n"); rc = ENXIO; goto pci_map_fail; } softc->pdev = kzalloc(sizeof(*softc->pdev), GFP_KERNEL); if (!softc->pdev) { device_printf(softc->dev, "pdev alloc failed\n"); rc = -ENOMEM; goto free_pci_map; } rc = linux_pci_attach_device(softc->dev, NULL, NULL, softc->pdev); if (rc) { device_printf(softc->dev, "Failed to attach Linux PCI device 0x%x\n", rc); goto pci_attach_fail; } /* HWRM setup/init */ BNXT_HWRM_LOCK_INIT(softc, device_get_nameunit(softc->dev)); rc = bnxt_alloc_hwrm_dma_mem(softc); if (rc) goto dma_fail; /* Get firmware version and compare with driver */ softc->ver_info = malloc(sizeof(struct bnxt_ver_info), M_DEVBUF, M_NOWAIT | M_ZERO); if (softc->ver_info == NULL) { rc = ENOMEM; device_printf(softc->dev, "Unable to allocate space for version info\n"); goto ver_alloc_fail; } /* Default minimum required HWRM version */ softc->ver_info->hwrm_min_major = HWRM_VERSION_MAJOR; softc->ver_info->hwrm_min_minor = HWRM_VERSION_MINOR; softc->ver_info->hwrm_min_update = HWRM_VERSION_UPDATE; rc = bnxt_hwrm_ver_get(softc); if (rc) { device_printf(softc->dev, "attach: hwrm ver get failed\n"); goto ver_fail; } /* Now perform a function reset */ rc = bnxt_hwrm_func_reset(softc); if ((softc->flags & BNXT_FLAG_SHORT_CMD) || softc->hwrm_max_ext_req_len > BNXT_HWRM_MAX_REQ_LEN) { rc = bnxt_alloc_hwrm_short_cmd_req(softc); if (rc) goto hwrm_short_cmd_alloc_fail; } if ((softc->ver_info->chip_num == BCM57508) || (softc->ver_info->chip_num == BCM57504) || (softc->ver_info->chip_num == BCM57504_NPAR) || (softc->ver_info->chip_num == BCM57502) || (softc->ver_info->chip_num == BCM57601) || (softc->ver_info->chip_num == BCM57602) || (softc->ver_info->chip_num == BCM57604)) softc->flags |= BNXT_FLAG_CHIP_P5; if (softc->ver_info->chip_num == BCM57608) softc->flags |= BNXT_FLAG_CHIP_P7; softc->flags |= BNXT_FLAG_TPA; if (BNXT_CHIP_P5_PLUS(softc) && (!softc->ver_info->chip_rev) && (!softc->ver_info->chip_metal)) softc->flags &= ~BNXT_FLAG_TPA; if (BNXT_CHIP_P5_PLUS(softc)) softc->flags &= ~BNXT_FLAG_TPA; /* Get NVRAM info */ if (BNXT_PF(softc)) { if (!bnxt_pf_wq) { bnxt_pf_wq = create_singlethread_workqueue("bnxt_pf_wq"); if (!bnxt_pf_wq) { device_printf(softc->dev, "Unable to create workqueue.\n"); rc = -ENOMEM; goto nvm_alloc_fail; } } softc->nvm_info = malloc(sizeof(struct bnxt_nvram_info), M_DEVBUF, M_NOWAIT | M_ZERO); if (softc->nvm_info == NULL) { rc = ENOMEM; device_printf(softc->dev, "Unable to allocate space for NVRAM info\n"); goto nvm_alloc_fail; } rc = bnxt_hwrm_nvm_get_dev_info(softc, &softc->nvm_info->mfg_id, &softc->nvm_info->device_id, &softc->nvm_info->sector_size, &softc->nvm_info->size, &softc->nvm_info->reserved_size, &softc->nvm_info->available_size); } if (BNXT_CHIP_P5(softc)) { softc->db_ops.bnxt_db_tx = bnxt_thor_db_tx; softc->db_ops.bnxt_db_rx = bnxt_thor_db_rx; softc->db_ops.bnxt_db_rx_cq = bnxt_thor_db_rx_cq; softc->db_ops.bnxt_db_tx_cq = bnxt_thor_db_tx_cq; softc->db_ops.bnxt_db_nq = bnxt_thor_db_nq; } else if (BNXT_CHIP_P7(softc)) { softc->db_ops.bnxt_db_tx = bnxt_thor2_db_tx; softc->db_ops.bnxt_db_rx = bnxt_thor2_db_rx; softc->db_ops.bnxt_db_rx_cq = bnxt_thor2_db_rx_cq; softc->db_ops.bnxt_db_tx_cq = bnxt_thor2_db_tx_cq; softc->db_ops.bnxt_db_nq = bnxt_thor2_db_nq; } else { softc->db_ops.bnxt_db_tx = bnxt_cuw_db_tx; softc->db_ops.bnxt_db_rx = bnxt_cuw_db_rx; softc->db_ops.bnxt_db_rx_cq = bnxt_cuw_db_cq; softc->db_ops.bnxt_db_tx_cq = bnxt_cuw_db_cq; } /* Get the queue config */ rc = bnxt_hwrm_queue_qportcfg(softc, HWRM_QUEUE_QPORTCFG_INPUT_FLAGS_PATH_TX); if (rc) { device_printf(softc->dev, "attach: hwrm qportcfg (tx) failed\n"); goto failed; } if (softc->is_asym_q) { rc = bnxt_hwrm_queue_qportcfg(softc, HWRM_QUEUE_QPORTCFG_INPUT_FLAGS_PATH_RX); if (rc) { device_printf(softc->dev, "attach: hwrm qportcfg (rx) failed\n"); return rc; } bnxt_verify_asym_queues(softc); } else { softc->rx_max_q = softc->tx_max_q; memcpy(softc->rx_q_info, softc->tx_q_info, sizeof(softc->rx_q_info)); memcpy(softc->rx_q_ids, softc->tx_q_ids, sizeof(softc->rx_q_ids)); } /* Get the HW capabilities */ rc = bnxt_hwrm_func_qcaps(softc); if (rc) goto failed; /* * Register the driver with the FW * Register the async events with the FW */ rc = bnxt_drv_rgtr(softc); if (rc) goto failed; if (softc->hwrm_spec_code >= 0x10803) { rc = bnxt_alloc_ctx_mem(softc); if (rc) { device_printf(softc->dev, "attach: alloc_ctx_mem failed\n"); return rc; } rc = bnxt_hwrm_func_resc_qcaps(softc, true); if (!rc) softc->flags |= BNXT_FLAG_FW_CAP_NEW_RM; } /* Get the current configuration of this function */ rc = bnxt_hwrm_func_qcfg(softc); if (rc) { device_printf(softc->dev, "attach: hwrm func qcfg failed\n"); goto failed; } iflib_set_mac(ctx, softc->func.mac_addr); scctx->isc_txrx = &bnxt_txrx; scctx->isc_tx_csum_flags = (CSUM_IP | CSUM_TCP | CSUM_UDP | CSUM_TCP_IPV6 | CSUM_UDP_IPV6 | CSUM_TSO); scctx->isc_capabilities = scctx->isc_capenable = /* These are translated to hwassit bits */ IFCAP_TXCSUM | IFCAP_TXCSUM_IPV6 | IFCAP_TSO4 | IFCAP_TSO6 | /* These are checked by iflib */ IFCAP_LRO | IFCAP_VLAN_HWFILTER | /* These are part of the iflib mask */ IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6 | IFCAP_VLAN_MTU | IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_HWTSO | /* These likely get lost... */ IFCAP_VLAN_HWCSUM | IFCAP_JUMBO_MTU; if (bnxt_wol_supported(softc)) scctx->isc_capabilities |= IFCAP_WOL_MAGIC; bnxt_get_wol_settings(softc); if (softc->wol) scctx->isc_capenable |= IFCAP_WOL_MAGIC; /* Get the queue config */ bnxt_get_wol_settings(softc); if (BNXT_CHIP_P5_PLUS(softc)) bnxt_hwrm_reserve_pf_rings(softc); rc = bnxt_hwrm_func_qcfg(softc); if (rc) { device_printf(softc->dev, "attach: hwrm func qcfg failed\n"); goto failed; } bnxt_clear_ids(softc); if (rc) goto failed; /* Now set up iflib sc */ scctx->isc_tx_nsegments = 31, scctx->isc_tx_tso_segments_max = 31; scctx->isc_tx_tso_size_max = BNXT_TSO_SIZE; scctx->isc_tx_tso_segsize_max = BNXT_TSO_SIZE; scctx->isc_vectors = softc->func.max_cp_rings; scctx->isc_min_frame_size = BNXT_MIN_FRAME_SIZE; scctx->isc_txrx = &bnxt_txrx; if (scctx->isc_nrxd[0] < ((scctx->isc_nrxd[1] * 4) + scctx->isc_nrxd[2])) device_printf(softc->dev, "WARNING: nrxd0 (%d) should be at least 4 * nrxd1 (%d) + nrxd2 (%d). Driver may be unstable\n", scctx->isc_nrxd[0], scctx->isc_nrxd[1], scctx->isc_nrxd[2]); if (scctx->isc_ntxd[0] < scctx->isc_ntxd[1] * 2) device_printf(softc->dev, "WARNING: ntxd0 (%d) should be at least 2 * ntxd1 (%d). Driver may be unstable\n", scctx->isc_ntxd[0], scctx->isc_ntxd[1]); scctx->isc_txqsizes[0] = sizeof(struct cmpl_base) * scctx->isc_ntxd[0]; scctx->isc_txqsizes[1] = sizeof(struct tx_bd_short) * scctx->isc_ntxd[1]; scctx->isc_txqsizes[2] = sizeof(struct cmpl_base) * scctx->isc_ntxd[2]; scctx->isc_rxqsizes[0] = sizeof(struct cmpl_base) * scctx->isc_nrxd[0]; scctx->isc_rxqsizes[1] = sizeof(struct rx_prod_pkt_bd) * scctx->isc_nrxd[1]; scctx->isc_rxqsizes[2] = sizeof(struct rx_prod_pkt_bd) * scctx->isc_nrxd[2]; scctx->isc_nrxqsets_max = min(pci_msix_count(softc->dev)-1, softc->fn_qcfg.alloc_completion_rings - 1); scctx->isc_nrxqsets_max = min(scctx->isc_nrxqsets_max, softc->fn_qcfg.alloc_rx_rings); scctx->isc_nrxqsets_max = min(scctx->isc_nrxqsets_max, softc->fn_qcfg.alloc_vnics); scctx->isc_ntxqsets_max = min(softc->fn_qcfg.alloc_tx_rings, softc->fn_qcfg.alloc_completion_rings - scctx->isc_nrxqsets_max - 1); scctx->isc_rss_table_size = HW_HASH_INDEX_SIZE; scctx->isc_rss_table_mask = scctx->isc_rss_table_size - 1; /* iflib will map and release this bar */ scctx->isc_msix_bar = pci_msix_table_bar(softc->dev); /* * Default settings for HW LRO (TPA): * Disable HW LRO by default * Can be enabled after taking care of 'packet forwarding' */ if (softc->flags & BNXT_FLAG_TPA) { softc->hw_lro.enable = 0; softc->hw_lro.is_mode_gro = 0; softc->hw_lro.max_agg_segs = 5; /* 2^5 = 32 segs */ softc->hw_lro.max_aggs = HWRM_VNIC_TPA_CFG_INPUT_MAX_AGGS_MAX; softc->hw_lro.min_agg_len = 512; } /* Allocate the default completion ring */ softc->def_cp_ring.stats_ctx_id = HWRM_NA_SIGNATURE; softc->def_cp_ring.ring.phys_id = (uint16_t)HWRM_NA_SIGNATURE; softc->def_cp_ring.ring.softc = softc; softc->def_cp_ring.ring.id = 0; softc->def_cp_ring.ring.doorbell = (BNXT_CHIP_P5_PLUS(softc)) ? softc->legacy_db_size : softc->def_cp_ring.ring.id * 0x80; softc->def_cp_ring.ring.ring_size = PAGE_SIZE / sizeof(struct cmpl_base); softc->def_cp_ring.ring.db_ring_mask = softc->def_cp_ring.ring.ring_size -1 ; rc = iflib_dma_alloc(ctx, sizeof(struct cmpl_base) * softc->def_cp_ring.ring.ring_size, &softc->def_cp_ring_mem, 0); softc->def_cp_ring.ring.vaddr = softc->def_cp_ring_mem.idi_vaddr; softc->def_cp_ring.ring.paddr = softc->def_cp_ring_mem.idi_paddr; iflib_config_task_init(ctx, &softc->def_cp_task, bnxt_def_cp_task); rc = bnxt_init_sysctl_ctx(softc); if (rc) goto init_sysctl_failed; if (BNXT_PF(softc)) { rc = bnxt_create_nvram_sysctls(softc->nvm_info); if (rc) goto failed; } arc4rand(softc->vnic_info.rss_hash_key, HW_HASH_KEY_SIZE, 0); softc->vnic_info.rss_hash_type = HWRM_VNIC_RSS_CFG_INPUT_HASH_TYPE_IPV4 | HWRM_VNIC_RSS_CFG_INPUT_HASH_TYPE_TCP_IPV4 | HWRM_VNIC_RSS_CFG_INPUT_HASH_TYPE_UDP_IPV4 | HWRM_VNIC_RSS_CFG_INPUT_HASH_TYPE_IPV6 | HWRM_VNIC_RSS_CFG_INPUT_HASH_TYPE_TCP_IPV6 | HWRM_VNIC_RSS_CFG_INPUT_HASH_TYPE_UDP_IPV6; rc = bnxt_create_config_sysctls_pre(softc); if (rc) goto failed; rc = bnxt_create_hw_lro_sysctls(softc); if (rc) goto failed; rc = bnxt_create_pause_fc_sysctls(softc); if (rc) goto failed; rc = bnxt_create_dcb_sysctls(softc); if (rc) goto failed; set_bit(BNXT_STATE_OPEN, &softc->state); INIT_WORK(&softc->sp_task, bnxt_sp_task); INIT_DELAYED_WORK(&softc->fw_reset_task, bnxt_fw_reset_task); /* Initialize the vlan list */ SLIST_INIT(&softc->vnic_info.vlan_tags); softc->vnic_info.vlan_tag_list.idi_vaddr = NULL; softc->state_bv = bit_alloc(BNXT_STATE_MAX, M_DEVBUF, M_WAITOK|M_ZERO); + if (BNXT_PF(softc)) { + const char *part_num; + + if (pci_get_vpd_readonly(softc->dev, "PN", &part_num) == 0) + snprintf(softc->board_partno, sizeof(softc->board_partno), "%s", part_num); + } + return (rc); failed: bnxt_free_sysctl_ctx(softc); init_sysctl_failed: bnxt_hwrm_func_drv_unrgtr(softc, false); if (BNXT_PF(softc)) free(softc->nvm_info, M_DEVBUF); nvm_alloc_fail: bnxt_free_hwrm_short_cmd_req(softc); hwrm_short_cmd_alloc_fail: ver_fail: free(softc->ver_info, M_DEVBUF); ver_alloc_fail: bnxt_free_hwrm_dma_mem(softc); dma_fail: BNXT_HWRM_LOCK_DESTROY(softc); if (softc->pdev) linux_pci_detach_device(softc->pdev); pci_attach_fail: kfree(softc->pdev); softc->pdev = NULL; free_pci_map: bnxt_pci_mapping_free(softc); pci_map_fail: pci_disable_busmaster(softc->dev); return (rc); } static int bnxt_attach_post(if_ctx_t ctx) { struct bnxt_softc *softc = iflib_get_softc(ctx); if_t ifp = iflib_get_ifp(ctx); int rc; softc->ifp = ifp; bnxt_create_config_sysctls_post(softc); /* Update link state etc... */ rc = bnxt_probe_phy(softc); if (rc) goto failed; /* Needs to be done after probing the phy */ bnxt_create_ver_sysctls(softc); ifmedia_removeall(softc->media); bnxt_add_media_types(softc); ifmedia_set(softc->media, IFM_ETHER | IFM_AUTO); softc->scctx->isc_max_frame_size = if_getmtu(ifp) + ETHER_HDR_LEN + ETHER_CRC_LEN; softc->rx_buf_size = min(softc->scctx->isc_max_frame_size, BNXT_PAGE_SIZE); bnxt_dcb_init(softc); bnxt_rdma_aux_device_init(softc); failed: return rc; } static int bnxt_detach(if_ctx_t ctx) { struct bnxt_softc *softc = iflib_get_softc(ctx); struct bnxt_vlan_tag *tag; struct bnxt_vlan_tag *tmp; int i; bnxt_rdma_aux_device_uninit(softc); cancel_delayed_work_sync(&softc->fw_reset_task); cancel_work_sync(&softc->sp_task); bnxt_dcb_free(softc); SLIST_REMOVE(&pf_list, &softc->list, bnxt_softc_list, next); bnxt_num_pfs--; bnxt_wol_config(ctx); bnxt_do_disable_intr(&softc->def_cp_ring); bnxt_free_sysctl_ctx(softc); bnxt_hwrm_func_reset(softc); bnxt_free_ctx_mem(softc); bnxt_clear_ids(softc); iflib_irq_free(ctx, &softc->def_cp_ring.irq); /* We need to free() these here... */ for (i = softc->nrxqsets-1; i>=0; i--) { if (BNXT_CHIP_P5_PLUS(softc)) iflib_irq_free(ctx, &softc->nq_rings[i].irq); else iflib_irq_free(ctx, &softc->rx_cp_rings[i].irq); } iflib_dma_free(&softc->vnic_info.mc_list); iflib_dma_free(&softc->vnic_info.rss_hash_key_tbl); iflib_dma_free(&softc->vnic_info.rss_grp_tbl); if (softc->vnic_info.vlan_tag_list.idi_vaddr) iflib_dma_free(&softc->vnic_info.vlan_tag_list); SLIST_FOREACH_SAFE(tag, &softc->vnic_info.vlan_tags, next, tmp) free(tag, M_DEVBUF); iflib_dma_free(&softc->def_cp_ring_mem); for (i = 0; i < softc->nrxqsets; i++) free(softc->rx_rings[i].tpa_start, M_DEVBUF); free(softc->ver_info, M_DEVBUF); if (BNXT_PF(softc)) free(softc->nvm_info, M_DEVBUF); bnxt_hwrm_func_drv_unrgtr(softc, false); bnxt_free_hwrm_dma_mem(softc); bnxt_free_hwrm_short_cmd_req(softc); BNXT_HWRM_LOCK_DESTROY(softc); if (!bnxt_num_pfs && bnxt_pf_wq) destroy_workqueue(bnxt_pf_wq); if (softc->pdev) linux_pci_detach_device(softc->pdev); free(softc->state_bv, M_DEVBUF); pci_disable_busmaster(softc->dev); bnxt_pci_mapping_free(softc); return 0; } static void bnxt_hwrm_resource_free(struct bnxt_softc *softc) { int i, rc = 0; rc = bnxt_hwrm_ring_free(softc, HWRM_RING_ALLOC_INPUT_RING_TYPE_L2_CMPL, &softc->def_cp_ring.ring, (uint16_t)HWRM_NA_SIGNATURE); if (rc) goto fail; for (i = 0; i < softc->ntxqsets; i++) { rc = bnxt_hwrm_ring_free(softc, HWRM_RING_ALLOC_INPUT_RING_TYPE_TX, &softc->tx_rings[i], softc->tx_cp_rings[i].ring.phys_id); if (rc) goto fail; rc = bnxt_hwrm_ring_free(softc, HWRM_RING_ALLOC_INPUT_RING_TYPE_L2_CMPL, &softc->tx_cp_rings[i].ring, (uint16_t)HWRM_NA_SIGNATURE); if (rc) goto fail; rc = bnxt_hwrm_stat_ctx_free(softc, &softc->tx_cp_rings[i]); if (rc) goto fail; } rc = bnxt_hwrm_free_filter(softc); if (rc) goto fail; rc = bnxt_hwrm_vnic_free(softc, &softc->vnic_info); if (rc) goto fail; rc = bnxt_hwrm_vnic_ctx_free(softc, softc->vnic_info.rss_id); if (rc) goto fail; for (i = 0; i < softc->nrxqsets; i++) { rc = bnxt_hwrm_ring_grp_free(softc, &softc->grp_info[i]); if (rc) goto fail; rc = bnxt_hwrm_ring_free(softc, HWRM_RING_ALLOC_INPUT_RING_TYPE_RX_AGG, &softc->ag_rings[i], (uint16_t)HWRM_NA_SIGNATURE); if (rc) goto fail; rc = bnxt_hwrm_ring_free(softc, HWRM_RING_ALLOC_INPUT_RING_TYPE_RX, &softc->rx_rings[i], softc->rx_cp_rings[i].ring.phys_id); if (rc) goto fail; rc = bnxt_hwrm_ring_free(softc, HWRM_RING_ALLOC_INPUT_RING_TYPE_L2_CMPL, &softc->rx_cp_rings[i].ring, (uint16_t)HWRM_NA_SIGNATURE); if (rc) goto fail; if (BNXT_CHIP_P5_PLUS(softc)) { rc = bnxt_hwrm_ring_free(softc, HWRM_RING_ALLOC_INPUT_RING_TYPE_NQ, &softc->nq_rings[i].ring, (uint16_t)HWRM_NA_SIGNATURE); if (rc) goto fail; } rc = bnxt_hwrm_stat_ctx_free(softc, &softc->rx_cp_rings[i]); if (rc) goto fail; } fail: return; } static void bnxt_func_reset(struct bnxt_softc *softc) { if (!BNXT_CHIP_P5_PLUS(softc)) { bnxt_hwrm_func_reset(softc); return; } bnxt_hwrm_resource_free(softc); return; } static void bnxt_rss_grp_tbl_init(struct bnxt_softc *softc) { uint16_t *rgt = (uint16_t *) softc->vnic_info.rss_grp_tbl.idi_vaddr; int i, j; for (i = 0, j = 0; i < HW_HASH_INDEX_SIZE; i++) { if (BNXT_CHIP_P5_PLUS(softc)) { rgt[i++] = htole16(softc->rx_rings[j].phys_id); rgt[i] = htole16(softc->rx_cp_rings[j].ring.phys_id); } else { rgt[i] = htole16(softc->grp_info[j].grp_id); } if (++j == softc->nrxqsets) j = 0; } } static void bnxt_get_port_module_status(struct bnxt_softc *softc) { struct bnxt_link_info *link_info = &softc->link_info; struct hwrm_port_phy_qcfg_output *resp = &link_info->phy_qcfg_resp; uint8_t module_status; if (bnxt_update_link(softc, false)) return; module_status = link_info->module_status; switch (module_status) { case HWRM_PORT_PHY_QCFG_OUTPUT_MODULE_STATUS_DISABLETX: case HWRM_PORT_PHY_QCFG_OUTPUT_MODULE_STATUS_PWRDOWN: case HWRM_PORT_PHY_QCFG_OUTPUT_MODULE_STATUS_WARNINGMSG: device_printf(softc->dev, "Unqualified SFP+ module detected on port %d\n", softc->pf.port_id); if (softc->hwrm_spec_code >= 0x10201) { device_printf(softc->dev, "Module part number %s\n", resp->phy_vendor_partnumber); } if (module_status == HWRM_PORT_PHY_QCFG_OUTPUT_MODULE_STATUS_DISABLETX) device_printf(softc->dev, "TX is disabled\n"); if (module_status == HWRM_PORT_PHY_QCFG_OUTPUT_MODULE_STATUS_PWRDOWN) device_printf(softc->dev, "SFP+ module is shutdown\n"); } } static void bnxt_aux_dev_free(struct bnxt_softc *softc) { kfree(softc->aux_dev); softc->aux_dev = NULL; } static struct bnxt_aux_dev *bnxt_aux_dev_init(struct bnxt_softc *softc) { struct bnxt_aux_dev *bnxt_adev; msleep(1000 * 2); bnxt_adev = kzalloc(sizeof(*bnxt_adev), GFP_KERNEL); if (!bnxt_adev) return ERR_PTR(-ENOMEM); return bnxt_adev; } static void bnxt_rdma_aux_device_uninit(struct bnxt_softc *softc) { struct bnxt_aux_dev *bnxt_adev = softc->aux_dev; /* Skip if no auxiliary device init was done. */ if (!(softc->flags & BNXT_FLAG_ROCE_CAP)) return; if (IS_ERR_OR_NULL(bnxt_adev)) return; bnxt_rdma_aux_device_del(softc); if (bnxt_adev->id >= 0) ida_free(&bnxt_aux_dev_ids, bnxt_adev->id); bnxt_aux_dev_free(softc); } static void bnxt_rdma_aux_device_init(struct bnxt_softc *softc) { int rc; if (!(softc->flags & BNXT_FLAG_ROCE_CAP)) return; softc->aux_dev = bnxt_aux_dev_init(softc); if (IS_ERR_OR_NULL(softc->aux_dev)) { device_printf(softc->dev, "Failed to init auxiliary device for ROCE\n"); goto skip_aux_init; } softc->aux_dev->id = ida_alloc(&bnxt_aux_dev_ids, GFP_KERNEL); if (softc->aux_dev->id < 0) { device_printf(softc->dev, "ida alloc failed for ROCE auxiliary device\n"); bnxt_aux_dev_free(softc); goto skip_aux_init; } msleep(1000 * 2); /* If aux bus init fails, continue with netdev init. */ rc = bnxt_rdma_aux_device_add(softc); if (rc) { device_printf(softc->dev, "Failed to add auxiliary device for ROCE\n"); msleep(1000 * 2); ida_free(&bnxt_aux_dev_ids, softc->aux_dev->id); } device_printf(softc->dev, "%s:%d Added auxiliary device (id %d) for ROCE \n", __func__, __LINE__, softc->aux_dev->id); skip_aux_init: return; } /* Device configuration */ static void bnxt_init(if_ctx_t ctx) { struct bnxt_softc *softc = iflib_get_softc(ctx); struct ifmediareq ifmr; int i; int rc; if (!BNXT_CHIP_P5_PLUS(softc)) { rc = bnxt_hwrm_func_reset(softc); if (rc) return; } else if (softc->is_dev_init) { bnxt_stop(ctx); } softc->is_dev_init = true; bnxt_clear_ids(softc); if (BNXT_CHIP_P5_PLUS(softc)) goto skip_def_cp_ring; /* Allocate the default completion ring */ softc->def_cp_ring.cons = UINT32_MAX; softc->def_cp_ring.v_bit = 1; bnxt_mark_cpr_invalid(&softc->def_cp_ring); rc = bnxt_hwrm_ring_alloc(softc, HWRM_RING_ALLOC_INPUT_RING_TYPE_L2_CMPL, &softc->def_cp_ring.ring); bnxt_set_db_mask(softc, &softc->def_cp_ring.ring, HWRM_RING_ALLOC_INPUT_RING_TYPE_L2_CMPL); if (rc) goto fail; skip_def_cp_ring: for (i = 0; i < softc->nrxqsets; i++) { /* Allocate the statistics context */ rc = bnxt_hwrm_stat_ctx_alloc(softc, &softc->rx_cp_rings[i], softc->rx_stats[i].idi_paddr); if (rc) goto fail; if (BNXT_CHIP_P5_PLUS(softc)) { /* Allocate the NQ */ softc->nq_rings[i].cons = 0; softc->nq_rings[i].raw_cons = 0; softc->nq_rings[i].v_bit = 1; softc->nq_rings[i].last_idx = UINT32_MAX; bnxt_mark_cpr_invalid(&softc->nq_rings[i]); rc = bnxt_hwrm_ring_alloc(softc, HWRM_RING_ALLOC_INPUT_RING_TYPE_NQ, &softc->nq_rings[i].ring); bnxt_set_db_mask(softc, &softc->nq_rings[i].ring, HWRM_RING_ALLOC_INPUT_RING_TYPE_NQ); if (rc) goto fail; softc->db_ops.bnxt_db_nq(&softc->nq_rings[i], 1); } /* Allocate the completion ring */ softc->rx_cp_rings[i].cons = UINT32_MAX; softc->rx_cp_rings[i].raw_cons = UINT32_MAX; softc->rx_cp_rings[i].v_bit = 1; softc->rx_cp_rings[i].last_idx = UINT32_MAX; softc->rx_cp_rings[i].toggle = 0; bnxt_mark_cpr_invalid(&softc->rx_cp_rings[i]); rc = bnxt_hwrm_ring_alloc(softc, HWRM_RING_ALLOC_INPUT_RING_TYPE_L2_CMPL, &softc->rx_cp_rings[i].ring); bnxt_set_db_mask(softc, &softc->rx_cp_rings[i].ring, HWRM_RING_ALLOC_INPUT_RING_TYPE_L2_CMPL); if (rc) goto fail; if (BNXT_CHIP_P5_PLUS(softc)) softc->db_ops.bnxt_db_rx_cq(&softc->rx_cp_rings[i], 1); /* Allocate the RX ring */ rc = bnxt_hwrm_ring_alloc(softc, HWRM_RING_ALLOC_INPUT_RING_TYPE_RX, &softc->rx_rings[i]); bnxt_set_db_mask(softc, &softc->rx_rings[i], HWRM_RING_ALLOC_INPUT_RING_TYPE_RX); if (rc) goto fail; softc->db_ops.bnxt_db_rx(&softc->rx_rings[i], 0); /* Allocate the AG ring */ rc = bnxt_hwrm_ring_alloc(softc, HWRM_RING_ALLOC_INPUT_RING_TYPE_RX_AGG, &softc->ag_rings[i]); bnxt_set_db_mask(softc, &softc->ag_rings[i], HWRM_RING_ALLOC_INPUT_RING_TYPE_RX_AGG); if (rc) goto fail; softc->db_ops.bnxt_db_rx(&softc->ag_rings[i], 0); /* Allocate the ring group */ softc->grp_info[i].stats_ctx = softc->rx_cp_rings[i].stats_ctx_id; softc->grp_info[i].rx_ring_id = softc->rx_rings[i].phys_id; softc->grp_info[i].ag_ring_id = softc->ag_rings[i].phys_id; softc->grp_info[i].cp_ring_id = softc->rx_cp_rings[i].ring.phys_id; rc = bnxt_hwrm_ring_grp_alloc(softc, &softc->grp_info[i]); if (rc) goto fail; } /* And now set the default CP / NQ ring for the async */ rc = bnxt_cfg_async_cr(softc); if (rc) goto fail; /* Allocate the VNIC RSS context */ rc = bnxt_hwrm_vnic_ctx_alloc(softc, &softc->vnic_info.rss_id); if (rc) goto fail; /* Allocate the vnic */ softc->vnic_info.def_ring_grp = softc->grp_info[0].grp_id; softc->vnic_info.mru = softc->scctx->isc_max_frame_size; rc = bnxt_hwrm_vnic_alloc(softc, &softc->vnic_info); if (rc) goto fail; rc = bnxt_hwrm_vnic_cfg(softc, &softc->vnic_info); if (rc) goto fail; rc = bnxt_hwrm_vnic_set_hds(softc, &softc->vnic_info); if (rc) goto fail; rc = bnxt_hwrm_set_filter(softc); if (rc) goto fail; bnxt_rss_grp_tbl_init(softc); rc = bnxt_hwrm_rss_cfg(softc, &softc->vnic_info, softc->vnic_info.rss_hash_type); if (rc) goto fail; rc = bnxt_hwrm_vnic_tpa_cfg(softc); if (rc) goto fail; for (i = 0; i < softc->ntxqsets; i++) { /* Allocate the statistics context */ rc = bnxt_hwrm_stat_ctx_alloc(softc, &softc->tx_cp_rings[i], softc->tx_stats[i].idi_paddr); if (rc) goto fail; /* Allocate the completion ring */ softc->tx_cp_rings[i].cons = UINT32_MAX; softc->tx_cp_rings[i].raw_cons = UINT32_MAX; softc->tx_cp_rings[i].v_bit = 1; softc->tx_cp_rings[i].toggle = 0; bnxt_mark_cpr_invalid(&softc->tx_cp_rings[i]); rc = bnxt_hwrm_ring_alloc(softc, HWRM_RING_ALLOC_INPUT_RING_TYPE_L2_CMPL, &softc->tx_cp_rings[i].ring); bnxt_set_db_mask(softc, &softc->tx_cp_rings[i].ring, HWRM_RING_ALLOC_INPUT_RING_TYPE_L2_CMPL); if (rc) goto fail; if (BNXT_CHIP_P5_PLUS(softc)) softc->db_ops.bnxt_db_tx_cq(&softc->tx_cp_rings[i], 1); /* Allocate the TX ring */ rc = bnxt_hwrm_ring_alloc(softc, HWRM_RING_ALLOC_INPUT_RING_TYPE_TX, &softc->tx_rings[i]); bnxt_set_db_mask(softc, &softc->tx_rings[i], HWRM_RING_ALLOC_INPUT_RING_TYPE_TX); if (rc) goto fail; softc->db_ops.bnxt_db_tx(&softc->tx_rings[i], 0); } bnxt_do_enable_intr(&softc->def_cp_ring); bnxt_get_port_module_status(softc); bnxt_media_status(softc->ctx, &ifmr); bnxt_hwrm_cfa_l2_set_rx_mask(softc, &softc->vnic_info); return; fail: bnxt_func_reset(softc); bnxt_clear_ids(softc); return; } static void bnxt_stop(if_ctx_t ctx) { struct bnxt_softc *softc = iflib_get_softc(ctx); softc->is_dev_init = false; bnxt_do_disable_intr(&softc->def_cp_ring); bnxt_func_reset(softc); bnxt_clear_ids(softc); return; } static u_int bnxt_copy_maddr(void *arg, struct sockaddr_dl *sdl, u_int cnt) { uint8_t *mta = arg; if (cnt == BNXT_MAX_MC_ADDRS) return (1); bcopy(LLADDR(sdl), &mta[cnt * ETHER_ADDR_LEN], ETHER_ADDR_LEN); return (1); } static void bnxt_multi_set(if_ctx_t ctx) { struct bnxt_softc *softc = iflib_get_softc(ctx); if_t ifp = iflib_get_ifp(ctx); uint8_t *mta; int mcnt; mta = softc->vnic_info.mc_list.idi_vaddr; bzero(mta, softc->vnic_info.mc_list.idi_size); mcnt = if_foreach_llmaddr(ifp, bnxt_copy_maddr, mta); if (mcnt > BNXT_MAX_MC_ADDRS) { softc->vnic_info.rx_mask |= HWRM_CFA_L2_SET_RX_MASK_INPUT_MASK_ALL_MCAST; bnxt_hwrm_cfa_l2_set_rx_mask(softc, &softc->vnic_info); } else { softc->vnic_info.rx_mask &= ~HWRM_CFA_L2_SET_RX_MASK_INPUT_MASK_ALL_MCAST; bus_dmamap_sync(softc->vnic_info.mc_list.idi_tag, softc->vnic_info.mc_list.idi_map, BUS_DMASYNC_PREWRITE); softc->vnic_info.mc_list_count = mcnt; softc->vnic_info.rx_mask |= HWRM_CFA_L2_SET_RX_MASK_INPUT_MASK_MCAST; if (bnxt_hwrm_cfa_l2_set_rx_mask(softc, &softc->vnic_info)) device_printf(softc->dev, "set_multi: rx_mask set failed\n"); } } static int bnxt_mtu_set(if_ctx_t ctx, uint32_t mtu) { struct bnxt_softc *softc = iflib_get_softc(ctx); if (mtu > BNXT_MAX_MTU) return EINVAL; softc->scctx->isc_max_frame_size = mtu + ETHER_HDR_LEN + ETHER_CRC_LEN; softc->rx_buf_size = min(softc->scctx->isc_max_frame_size, BNXT_PAGE_SIZE); return 0; } static void bnxt_media_status(if_ctx_t ctx, struct ifmediareq * ifmr) { struct bnxt_softc *softc = iflib_get_softc(ctx); struct bnxt_link_info *link_info = &softc->link_info; struct ifmedia_entry *next; uint64_t target_baudrate = bnxt_get_baudrate(link_info); int active_media = IFM_UNKNOWN; bnxt_update_link(softc, true); ifmr->ifm_status = IFM_AVALID; ifmr->ifm_active = IFM_ETHER; if (!link_info->link_up) return; ifmr->ifm_status |= IFM_ACTIVE; if (link_info->duplex == HWRM_PORT_PHY_QCFG_OUTPUT_DUPLEX_CFG_FULL) ifmr->ifm_active |= IFM_FDX; else ifmr->ifm_active |= IFM_HDX; /* * Go through the list of supported media which got prepared * as part of bnxt_add_media_types() using api ifmedia_add(). */ LIST_FOREACH(next, &(iflib_get_media(ctx)->ifm_list), ifm_list) { if (ifmedia_baudrate(next->ifm_media) == target_baudrate) { active_media = next->ifm_media; break; } } ifmr->ifm_active |= active_media; if (link_info->flow_ctrl.rx) ifmr->ifm_active |= IFM_ETH_RXPAUSE; if (link_info->flow_ctrl.tx) ifmr->ifm_active |= IFM_ETH_TXPAUSE; bnxt_report_link(softc); return; } static int bnxt_media_change(if_ctx_t ctx) { struct bnxt_softc *softc = iflib_get_softc(ctx); struct ifmedia *ifm = iflib_get_media(ctx); struct ifmediareq ifmr; int rc; struct bnxt_link_info *link_info = &softc->link_info; if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER) return EINVAL; switch (IFM_SUBTYPE(ifm->ifm_media)) { case IFM_100_T: link_info->autoneg &= ~BNXT_AUTONEG_SPEED; link_info->req_link_speed = HWRM_PORT_PHY_CFG_INPUT_FORCE_LINK_SPEED_100MB; break; case IFM_1000_KX: case IFM_1000_SGMII: case IFM_1000_CX: case IFM_1000_SX: case IFM_1000_LX: link_info->autoneg &= ~BNXT_AUTONEG_SPEED; if (link_info->support_speeds & HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_SPEEDS_1GB) { link_info->req_link_speed = HWRM_PORT_PHY_CFG_INPUT_FORCE_LINK_SPEED_1GB; } else if (link_info->support_speeds2 & HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_SPEEDS2_1GB) { link_info->req_link_speed = HWRM_PORT_PHY_CFG_INPUT_FORCE_LINK_SPEEDS2_1GB; link_info->force_speed2_nrz = true; } break; case IFM_2500_KX: case IFM_2500_T: link_info->autoneg &= ~BNXT_AUTONEG_SPEED; link_info->req_link_speed = HWRM_PORT_PHY_CFG_INPUT_FORCE_LINK_SPEED_2_5GB; break; case IFM_10G_CR1: case IFM_10G_KR: case IFM_10G_LR: case IFM_10G_SR: link_info->autoneg &= ~BNXT_AUTONEG_SPEED; if (link_info->support_speeds & HWRM_PORT_PHY_CFG_INPUT_FORCE_LINK_SPEED_10GB) { link_info->req_link_speed = HWRM_PORT_PHY_CFG_INPUT_FORCE_LINK_SPEED_10GB; } else if (link_info->support_speeds2 & HWRM_PORT_PHY_CFG_INPUT_FORCE_LINK_SPEEDS2_10GB) { link_info->req_link_speed = HWRM_PORT_PHY_CFG_INPUT_FORCE_LINK_SPEEDS2_10GB; link_info->force_speed2_nrz = true; } break; case IFM_20G_KR2: link_info->autoneg &= ~BNXT_AUTONEG_SPEED; link_info->req_link_speed = HWRM_PORT_PHY_CFG_INPUT_FORCE_LINK_SPEED_20GB; break; case IFM_25G_CR: case IFM_25G_KR: case IFM_25G_SR: case IFM_25G_LR: link_info->autoneg &= ~BNXT_AUTONEG_SPEED; if (link_info->support_speeds & HWRM_PORT_PHY_CFG_INPUT_FORCE_LINK_SPEED_25GB) { link_info->req_link_speed = HWRM_PORT_PHY_CFG_INPUT_FORCE_LINK_SPEED_25GB; } else if (link_info->support_speeds2 & HWRM_PORT_PHY_CFG_INPUT_FORCE_LINK_SPEEDS2_25GB) { link_info->req_link_speed = HWRM_PORT_PHY_CFG_INPUT_FORCE_LINK_SPEEDS2_25GB; link_info->force_speed2_nrz = true; } break; case IFM_40G_CR4: case IFM_40G_KR4: case IFM_40G_LR4: case IFM_40G_SR4: case IFM_40G_XLAUI: case IFM_40G_XLAUI_AC: link_info->autoneg &= ~BNXT_AUTONEG_SPEED; if (link_info->support_speeds & HWRM_PORT_PHY_CFG_INPUT_FORCE_LINK_SPEED_40GB) { link_info->req_link_speed = HWRM_PORT_PHY_CFG_INPUT_FORCE_LINK_SPEED_40GB; } else if (link_info->support_speeds2 & HWRM_PORT_PHY_CFG_INPUT_FORCE_LINK_SPEEDS2_40GB) { link_info->req_link_speed = HWRM_PORT_PHY_CFG_INPUT_FORCE_LINK_SPEEDS2_40GB; link_info->force_speed2_nrz = true; } break; case IFM_50G_CR2: case IFM_50G_KR2: case IFM_50G_KR4: case IFM_50G_SR2: case IFM_50G_LR2: link_info->autoneg &= ~BNXT_AUTONEG_SPEED; if (link_info->support_speeds & HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_SPEEDS_50GB) { link_info->req_link_speed = HWRM_PORT_PHY_CFG_INPUT_FORCE_LINK_SPEED_50GB; } else if (link_info->support_speeds2 & HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_SPEEDS2_50GB) { link_info->req_link_speed = HWRM_PORT_PHY_CFG_INPUT_FORCE_LINK_SPEEDS2_50GB; link_info->force_speed2_nrz = true; } break; case IFM_50G_CP: case IFM_50G_LR: case IFM_50G_SR: case IFM_50G_KR_PAM4: link_info->autoneg &= ~BNXT_AUTONEG_SPEED; if (link_info->support_pam4_speeds & HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_PAM4_SPEEDS_50G) { link_info->req_link_speed = HWRM_PORT_PHY_CFG_INPUT_FORCE_PAM4_LINK_SPEED_50GB; link_info->force_pam4_speed = true; } else if (link_info->support_speeds2 & HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_SPEEDS2_50GB_PAM4_56) { link_info->req_link_speed = HWRM_PORT_PHY_CFG_INPUT_FORCE_LINK_SPEEDS2_50GB_PAM4_56; link_info->force_pam4_56_speed2 = true; } break; case IFM_100G_CR4: case IFM_100G_KR4: case IFM_100G_LR4: case IFM_100G_SR4: case IFM_100G_AUI4: link_info->autoneg &= ~BNXT_AUTONEG_SPEED; if (link_info->support_speeds & HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_SPEEDS_100GB) { link_info->req_link_speed = HWRM_PORT_PHY_CFG_INPUT_FORCE_LINK_SPEED_100GB; } else if (link_info->support_speeds2 & HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_SPEEDS2_100GB) { link_info->req_link_speed = HWRM_PORT_PHY_CFG_INPUT_FORCE_LINK_SPEEDS2_100GB; link_info->force_speed2_nrz = true; } break; case IFM_100G_CP2: case IFM_100G_SR2: case IFM_100G_KR2_PAM4: case IFM_100G_AUI2: link_info->autoneg &= ~BNXT_AUTONEG_SPEED; if (link_info->support_pam4_speeds & HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_PAM4_SPEEDS_100G) { link_info->req_link_speed = HWRM_PORT_PHY_CFG_INPUT_FORCE_PAM4_LINK_SPEED_100GB; link_info->force_pam4_speed = true; } else if (link_info->support_speeds2 & HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_SPEEDS2_100GB_PAM4_56) { link_info->req_link_speed = HWRM_PORT_PHY_CFG_INPUT_FORCE_LINK_SPEEDS2_100GB_PAM4_56; link_info->force_pam4_56_speed2 = true; } break; case IFM_100G_KR_PAM4: case IFM_100G_CR_PAM4: case IFM_100G_DR: case IFM_100G_AUI2_AC: link_info->autoneg &= ~BNXT_AUTONEG_SPEED; if (link_info->support_speeds2 & HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_SPEEDS2_100GB_PAM4_112) { link_info->req_link_speed = HWRM_PORT_PHY_CFG_INPUT_FORCE_LINK_SPEEDS2_100GB_PAM4_112; link_info->force_pam4_112_speed2 = true; } break; case IFM_200G_SR4: case IFM_200G_FR4: case IFM_200G_LR4: case IFM_200G_DR4: case IFM_200G_CR4_PAM4: case IFM_200G_KR4_PAM4: link_info->autoneg &= ~BNXT_AUTONEG_SPEED; if (link_info->support_pam4_speeds & HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_PAM4_SPEEDS_200G) { link_info->req_link_speed = HWRM_PORT_PHY_CFG_INPUT_FORCE_PAM4_LINK_SPEED_200GB; link_info->force_pam4_speed = true; } else if (link_info->support_speeds2 & HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_SPEEDS2_200GB_PAM4_56) { link_info->req_link_speed = HWRM_PORT_PHY_CFG_INPUT_FORCE_LINK_SPEEDS2_200GB_PAM4_56; link_info->force_pam4_56_speed2 = true; } break; case IFM_200G_AUI4: link_info->autoneg &= ~BNXT_AUTONEG_SPEED; if (link_info->support_speeds2 & HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_SPEEDS2_200GB_PAM4_112) { link_info->req_link_speed = HWRM_PORT_PHY_CFG_INPUT_FORCE_LINK_SPEEDS2_200GB_PAM4_112; link_info->force_pam4_112_speed2 = true; } break; case IFM_400G_FR8: case IFM_400G_LR8: case IFM_400G_AUI8: link_info->autoneg &= ~BNXT_AUTONEG_SPEED; if (link_info->support_speeds2 & HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_SPEEDS2_400GB_PAM4_56) { link_info->req_link_speed = HWRM_PORT_PHY_CFG_INPUT_FORCE_LINK_SPEEDS2_400GB_PAM4_56; link_info->force_pam4_56_speed2 = true; } break; case IFM_400G_AUI8_AC: case IFM_400G_DR4: link_info->autoneg &= ~BNXT_AUTONEG_SPEED; if (link_info->support_speeds2 & HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_SPEEDS2_400GB_PAM4_112) { link_info->req_link_speed = HWRM_PORT_PHY_CFG_INPUT_FORCE_LINK_SPEEDS2_400GB_PAM4_112; link_info->force_pam4_112_speed2 = true; } break; case IFM_1000_T: link_info->advertising = HWRM_PORT_PHY_CFG_INPUT_AUTO_LINK_SPEED_MASK_1GB; link_info->autoneg |= BNXT_AUTONEG_SPEED; break; case IFM_10G_T: link_info->advertising = HWRM_PORT_PHY_CFG_INPUT_AUTO_LINK_SPEED_MASK_10GB; link_info->autoneg |= BNXT_AUTONEG_SPEED; break; default: device_printf(softc->dev, "Unsupported media type! Using auto\n"); /* Fall-through */ case IFM_AUTO: // Auto link_info->autoneg |= BNXT_AUTONEG_SPEED; break; } rc = bnxt_hwrm_set_link_setting(softc, true, true, true); bnxt_media_status(softc->ctx, &ifmr); return rc; } static int bnxt_promisc_set(if_ctx_t ctx, int flags) { struct bnxt_softc *softc = iflib_get_softc(ctx); if_t ifp = iflib_get_ifp(ctx); int rc; if (if_getflags(ifp) & IFF_ALLMULTI || if_llmaddr_count(ifp) > BNXT_MAX_MC_ADDRS) softc->vnic_info.rx_mask |= HWRM_CFA_L2_SET_RX_MASK_INPUT_MASK_ALL_MCAST; else softc->vnic_info.rx_mask &= ~HWRM_CFA_L2_SET_RX_MASK_INPUT_MASK_ALL_MCAST; if (if_getflags(ifp) & IFF_PROMISC) softc->vnic_info.rx_mask |= HWRM_CFA_L2_SET_RX_MASK_INPUT_MASK_PROMISCUOUS | HWRM_CFA_L2_SET_RX_MASK_INPUT_MASK_ANYVLAN_NONVLAN; else softc->vnic_info.rx_mask &= ~(HWRM_CFA_L2_SET_RX_MASK_INPUT_MASK_PROMISCUOUS); rc = bnxt_hwrm_cfa_l2_set_rx_mask(softc, &softc->vnic_info); return rc; } static uint64_t bnxt_get_counter(if_ctx_t ctx, ift_counter cnt) { if_t ifp = iflib_get_ifp(ctx); if (cnt < IFCOUNTERS) return if_get_counter_default(ifp, cnt); return 0; } static void bnxt_update_admin_status(if_ctx_t ctx) { struct bnxt_softc *softc = iflib_get_softc(ctx); /* * When SR-IOV is enabled, avoid each VF sending this HWRM * request every sec with which firmware timeouts can happen */ if (!BNXT_PF(softc)) return; bnxt_hwrm_port_qstats(softc); if (BNXT_CHIP_P5_PLUS(softc) && (softc->flags & BNXT_FLAG_FW_CAP_EXT_STATS)) bnxt_hwrm_port_qstats_ext(softc); if (BNXT_CHIP_P5_PLUS(softc)) { struct ifmediareq ifmr; if (bit_test(softc->state_bv, BNXT_STATE_LINK_CHANGE)) { bit_clear(softc->state_bv, BNXT_STATE_LINK_CHANGE); bnxt_media_status(softc->ctx, &ifmr); } } return; } static void bnxt_if_timer(if_ctx_t ctx, uint16_t qid) { struct bnxt_softc *softc = iflib_get_softc(ctx); uint64_t ticks_now = ticks; /* Schedule bnxt_update_admin_status() once per sec */ if (ticks_now - softc->admin_ticks >= hz) { softc->admin_ticks = ticks_now; iflib_admin_intr_deferred(ctx); } return; } static void inline bnxt_do_enable_intr(struct bnxt_cp_ring *cpr) { struct bnxt_softc *softc = cpr->ring.softc; if (cpr->ring.phys_id == (uint16_t)HWRM_NA_SIGNATURE) return; if (BNXT_CHIP_P5_PLUS(softc)) softc->db_ops.bnxt_db_nq(cpr, 1); else softc->db_ops.bnxt_db_rx_cq(cpr, 1); } static void inline bnxt_do_disable_intr(struct bnxt_cp_ring *cpr) { struct bnxt_softc *softc = cpr->ring.softc; if (cpr->ring.phys_id == (uint16_t)HWRM_NA_SIGNATURE) return; if (BNXT_CHIP_P5_PLUS(softc)) softc->db_ops.bnxt_db_nq(cpr, 0); else softc->db_ops.bnxt_db_rx_cq(cpr, 0); } /* Enable all interrupts */ static void bnxt_intr_enable(if_ctx_t ctx) { struct bnxt_softc *softc = iflib_get_softc(ctx); int i; bnxt_do_enable_intr(&softc->def_cp_ring); for (i = 0; i < softc->nrxqsets; i++) if (BNXT_CHIP_P5_PLUS(softc)) softc->db_ops.bnxt_db_nq(&softc->nq_rings[i], 1); else softc->db_ops.bnxt_db_rx_cq(&softc->rx_cp_rings[i], 1); return; } /* Enable interrupt for a single queue */ static int bnxt_tx_queue_intr_enable(if_ctx_t ctx, uint16_t qid) { struct bnxt_softc *softc = iflib_get_softc(ctx); if (BNXT_CHIP_P5_PLUS(softc)) softc->db_ops.bnxt_db_nq(&softc->nq_rings[qid], 1); else softc->db_ops.bnxt_db_rx_cq(&softc->tx_cp_rings[qid], 1); return 0; } static void bnxt_process_cmd_cmpl(struct bnxt_softc *softc, hwrm_cmpl_t *cmd_cmpl) { device_printf(softc->dev, "cmd sequence number %d\n", cmd_cmpl->sequence_id); return; } static void bnxt_process_async_msg(struct bnxt_cp_ring *cpr, tx_cmpl_t *cmpl) { struct bnxt_softc *softc = cpr->ring.softc; uint16_t type = cmpl->flags_type & TX_CMPL_TYPE_MASK; switch (type) { case HWRM_CMPL_TYPE_HWRM_DONE: bnxt_process_cmd_cmpl(softc, (hwrm_cmpl_t *)cmpl); break; case HWRM_ASYNC_EVENT_CMPL_TYPE_HWRM_ASYNC_EVENT: bnxt_handle_async_event(softc, (cmpl_base_t *) cmpl); break; default: device_printf(softc->dev, "%s:%d Unhandled async message %x\n", __FUNCTION__, __LINE__, type); break; } } void process_nq(struct bnxt_softc *softc, uint16_t nqid) { struct bnxt_cp_ring *cpr = &softc->nq_rings[nqid]; nq_cn_t *cmp = (nq_cn_t *) cpr->ring.vaddr; struct bnxt_cp_ring *tx_cpr = &softc->tx_cp_rings[nqid]; struct bnxt_cp_ring *rx_cpr = &softc->rx_cp_rings[nqid]; bool v_bit = cpr->v_bit; uint32_t cons = cpr->cons; uint32_t raw_cons = cpr->raw_cons; uint16_t nq_type, nqe_cnt = 0; while (1) { if (!NQ_VALID(&cmp[cons], v_bit)) { goto done; } nq_type = NQ_CN_TYPE_MASK & cmp[cons].type; if (NQE_CN_TYPE(nq_type) != NQ_CN_TYPE_CQ_NOTIFICATION) { bnxt_process_async_msg(cpr, (tx_cmpl_t *)&cmp[cons]); } else { tx_cpr->toggle = NQE_CN_TOGGLE(cmp[cons].type); rx_cpr->toggle = NQE_CN_TOGGLE(cmp[cons].type); } NEXT_CP_CONS_V(&cpr->ring, cons, v_bit); raw_cons++; nqe_cnt++; } done: if (nqe_cnt) { cpr->cons = cons; cpr->raw_cons = raw_cons; cpr->v_bit = v_bit; } } static int bnxt_rx_queue_intr_enable(if_ctx_t ctx, uint16_t qid) { struct bnxt_softc *softc = iflib_get_softc(ctx); if (BNXT_CHIP_P5_PLUS(softc)) { process_nq(softc, qid); softc->db_ops.bnxt_db_nq(&softc->nq_rings[qid], 1); } softc->db_ops.bnxt_db_rx_cq(&softc->rx_cp_rings[qid], 1); return 0; } /* Disable all interrupts */ static void bnxt_disable_intr(if_ctx_t ctx) { struct bnxt_softc *softc = iflib_get_softc(ctx); int i; /* * NOTE: These TX interrupts should never get enabled, so don't * update the index */ for (i = 0; i < softc->nrxqsets; i++) if (BNXT_CHIP_P5_PLUS(softc)) softc->db_ops.bnxt_db_nq(&softc->nq_rings[i], 0); else softc->db_ops.bnxt_db_rx_cq(&softc->rx_cp_rings[i], 0); return; } static int bnxt_msix_intr_assign(if_ctx_t ctx, int msix) { struct bnxt_softc *softc = iflib_get_softc(ctx); struct bnxt_cp_ring *ring; struct if_irq *irq; uint16_t id; int rc; int i; char irq_name[16]; if (BNXT_CHIP_P5_PLUS(softc)) goto skip_default_cp; rc = iflib_irq_alloc_generic(ctx, &softc->def_cp_ring.irq, softc->def_cp_ring.ring.id + 1, IFLIB_INTR_ADMIN, bnxt_handle_def_cp, softc, 0, "def_cp"); if (rc) { device_printf(iflib_get_dev(ctx), "Failed to register default completion ring handler\n"); return rc; } skip_default_cp: for (i=0; iscctx->isc_nrxqsets; i++) { if (BNXT_CHIP_P5_PLUS(softc)) { irq = &softc->nq_rings[i].irq; id = softc->nq_rings[i].ring.id; ring = &softc->nq_rings[i]; } else { irq = &softc->rx_cp_rings[i].irq; id = softc->rx_cp_rings[i].ring.id ; ring = &softc->rx_cp_rings[i]; } snprintf(irq_name, sizeof(irq_name), "rxq%d", i); rc = iflib_irq_alloc_generic(ctx, irq, id + 1, IFLIB_INTR_RX, bnxt_handle_isr, ring, i, irq_name); if (rc) { device_printf(iflib_get_dev(ctx), "Failed to register RX completion ring handler\n"); i--; goto fail; } } for (i=0; iscctx->isc_ntxqsets; i++) iflib_softirq_alloc_generic(ctx, NULL, IFLIB_INTR_TX, NULL, i, "tx_cp"); return rc; fail: for (; i>=0; i--) iflib_irq_free(ctx, &softc->rx_cp_rings[i].irq); iflib_irq_free(ctx, &softc->def_cp_ring.irq); return rc; } /* * We're explicitly allowing duplicates here. They will need to be * removed as many times as they are added. */ static void bnxt_vlan_register(if_ctx_t ctx, uint16_t vtag) { struct bnxt_softc *softc = iflib_get_softc(ctx); struct bnxt_vlan_tag *new_tag; new_tag = malloc(sizeof(struct bnxt_vlan_tag), M_DEVBUF, M_NOWAIT); if (new_tag == NULL) return; new_tag->tag = vtag; new_tag->filter_id = -1; SLIST_INSERT_HEAD(&softc->vnic_info.vlan_tags, new_tag, next); }; static void bnxt_vlan_unregister(if_ctx_t ctx, uint16_t vtag) { struct bnxt_softc *softc = iflib_get_softc(ctx); struct bnxt_vlan_tag *vlan_tag; SLIST_FOREACH(vlan_tag, &softc->vnic_info.vlan_tags, next) { if (vlan_tag->tag == vtag) { SLIST_REMOVE(&softc->vnic_info.vlan_tags, vlan_tag, bnxt_vlan_tag, next); free(vlan_tag, M_DEVBUF); break; } } } static int bnxt_wol_config(if_ctx_t ctx) { struct bnxt_softc *softc = iflib_get_softc(ctx); if_t ifp = iflib_get_ifp(ctx); if (!softc) return -EBUSY; if (!bnxt_wol_supported(softc)) return -ENOTSUP; if (if_getcapenable(ifp) & IFCAP_WOL_MAGIC) { if (!softc->wol) { if (bnxt_hwrm_alloc_wol_fltr(softc)) return -EBUSY; softc->wol = 1; } } else { if (softc->wol) { if (bnxt_hwrm_free_wol_fltr(softc)) return -EBUSY; softc->wol = 0; } } return 0; } static bool bnxt_if_needs_restart(if_ctx_t ctx __unused, enum iflib_restart_event event) { switch (event) { case IFLIB_RESTART_VLAN_CONFIG: default: return (false); } } static int bnxt_shutdown(if_ctx_t ctx) { bnxt_wol_config(ctx); return 0; } static int bnxt_suspend(if_ctx_t ctx) { bnxt_wol_config(ctx); return 0; } static int bnxt_resume(if_ctx_t ctx) { struct bnxt_softc *softc = iflib_get_softc(ctx); bnxt_get_wol_settings(softc); return 0; } static int bnxt_priv_ioctl(if_ctx_t ctx, u_long command, caddr_t data) { struct bnxt_softc *softc = iflib_get_softc(ctx); struct ifreq *ifr = (struct ifreq *)data; struct bnxt_ioctl_header *ioh; size_t iol; int rc = ENOTSUP; struct bnxt_ioctl_data iod_storage, *iod = &iod_storage; switch (command) { case SIOCGPRIVATE_0: if ((rc = priv_check(curthread, PRIV_DRIVER)) != 0) goto exit; ioh = ifr_buffer_get_buffer(ifr); iol = ifr_buffer_get_length(ifr); if (iol > sizeof(iod_storage)) return (EINVAL); if ((rc = copyin(ioh, iod, iol)) != 0) goto exit; switch (iod->hdr.type) { case BNXT_HWRM_NVM_FIND_DIR_ENTRY: { struct bnxt_ioctl_hwrm_nvm_find_dir_entry *find = &iod->find; rc = bnxt_hwrm_nvm_find_dir_entry(softc, find->type, &find->ordinal, find->ext, &find->index, find->use_index, find->search_opt, &find->data_length, &find->item_length, &find->fw_ver); if (rc) { iod->hdr.rc = rc; rc = copyout(&iod->hdr.rc, &ioh->rc, sizeof(ioh->rc)); } else { iod->hdr.rc = 0; rc = copyout(iod, ioh, iol); } goto exit; } case BNXT_HWRM_NVM_READ: { struct bnxt_ioctl_hwrm_nvm_read *rd = &iod->read; struct iflib_dma_info dma_data; size_t offset; size_t remain; size_t csize; /* * Some HWRM versions can't read more than 0x8000 bytes */ rc = iflib_dma_alloc(softc->ctx, min(rd->length, 0x8000), &dma_data, BUS_DMA_NOWAIT); if (rc) break; for (remain = rd->length, offset = 0; remain && offset < rd->length; offset += 0x8000) { csize = min(remain, 0x8000); rc = bnxt_hwrm_nvm_read(softc, rd->index, rd->offset + offset, csize, &dma_data); if (rc) { iod->hdr.rc = rc; rc = copyout(&iod->hdr.rc, &ioh->rc, sizeof(ioh->rc)); break; } else { rc = copyout(dma_data.idi_vaddr, rd->data + offset, csize); iod->hdr.rc = rc; } remain -= csize; } if (rc == 0) rc = copyout(iod, ioh, iol); iflib_dma_free(&dma_data); goto exit; } case BNXT_HWRM_FW_RESET: { struct bnxt_ioctl_hwrm_fw_reset *rst = &iod->reset; rc = bnxt_hwrm_fw_reset(softc, rst->processor, &rst->selfreset); if (rc) { iod->hdr.rc = rc; rc = copyout(&iod->hdr.rc, &ioh->rc, sizeof(ioh->rc)); } else { iod->hdr.rc = 0; rc = copyout(iod, ioh, iol); } goto exit; } case BNXT_HWRM_FW_QSTATUS: { struct bnxt_ioctl_hwrm_fw_qstatus *qstat = &iod->status; rc = bnxt_hwrm_fw_qstatus(softc, qstat->processor, &qstat->selfreset); if (rc) { iod->hdr.rc = rc; rc = copyout(&iod->hdr.rc, &ioh->rc, sizeof(ioh->rc)); } else { iod->hdr.rc = 0; rc = copyout(iod, ioh, iol); } goto exit; } case BNXT_HWRM_NVM_WRITE: { struct bnxt_ioctl_hwrm_nvm_write *wr = &iod->write; rc = bnxt_hwrm_nvm_write(softc, wr->data, true, wr->type, wr->ordinal, wr->ext, wr->attr, wr->option, wr->data_length, wr->keep, &wr->item_length, &wr->index); if (rc) { iod->hdr.rc = rc; rc = copyout(&iod->hdr.rc, &ioh->rc, sizeof(ioh->rc)); } else { iod->hdr.rc = 0; rc = copyout(iod, ioh, iol); } goto exit; } case BNXT_HWRM_NVM_ERASE_DIR_ENTRY: { struct bnxt_ioctl_hwrm_nvm_erase_dir_entry *erase = &iod->erase; rc = bnxt_hwrm_nvm_erase_dir_entry(softc, erase->index); if (rc) { iod->hdr.rc = rc; rc = copyout(&iod->hdr.rc, &ioh->rc, sizeof(ioh->rc)); } else { iod->hdr.rc = 0; rc = copyout(iod, ioh, iol); } goto exit; } case BNXT_HWRM_NVM_GET_DIR_INFO: { struct bnxt_ioctl_hwrm_nvm_get_dir_info *info = &iod->dir_info; rc = bnxt_hwrm_nvm_get_dir_info(softc, &info->entries, &info->entry_length); if (rc) { iod->hdr.rc = rc; rc = copyout(&iod->hdr.rc, &ioh->rc, sizeof(ioh->rc)); } else { iod->hdr.rc = 0; rc = copyout(iod, ioh, iol); } goto exit; } case BNXT_HWRM_NVM_GET_DIR_ENTRIES: { struct bnxt_ioctl_hwrm_nvm_get_dir_entries *get = &iod->dir_entries; struct iflib_dma_info dma_data; rc = iflib_dma_alloc(softc->ctx, get->max_size, &dma_data, BUS_DMA_NOWAIT); if (rc) break; rc = bnxt_hwrm_nvm_get_dir_entries(softc, &get->entries, &get->entry_length, &dma_data); if (rc) { iod->hdr.rc = rc; rc = copyout(&iod->hdr.rc, &ioh->rc, sizeof(ioh->rc)); } else { rc = copyout(dma_data.idi_vaddr, get->data, get->entry_length * get->entries); iod->hdr.rc = rc; if (rc == 0) rc = copyout(iod, ioh, iol); } iflib_dma_free(&dma_data); goto exit; } case BNXT_HWRM_NVM_VERIFY_UPDATE: { struct bnxt_ioctl_hwrm_nvm_verify_update *vrfy = &iod->verify; rc = bnxt_hwrm_nvm_verify_update(softc, vrfy->type, vrfy->ordinal, vrfy->ext); if (rc) { iod->hdr.rc = rc; rc = copyout(&iod->hdr.rc, &ioh->rc, sizeof(ioh->rc)); } else { iod->hdr.rc = 0; rc = copyout(iod, ioh, iol); } goto exit; } case BNXT_HWRM_NVM_INSTALL_UPDATE: { struct bnxt_ioctl_hwrm_nvm_install_update *inst = &iod->install; rc = bnxt_hwrm_nvm_install_update(softc, inst->install_type, &inst->installed_items, &inst->result, &inst->problem_item, &inst->reset_required); if (rc) { iod->hdr.rc = rc; rc = copyout(&iod->hdr.rc, &ioh->rc, sizeof(ioh->rc)); } else { iod->hdr.rc = 0; rc = copyout(iod, ioh, iol); } goto exit; } case BNXT_HWRM_NVM_MODIFY: { struct bnxt_ioctl_hwrm_nvm_modify *mod = &iod->modify; rc = bnxt_hwrm_nvm_modify(softc, mod->index, mod->offset, mod->data, true, mod->length); if (rc) { iod->hdr.rc = rc; rc = copyout(&iod->hdr.rc, &ioh->rc, sizeof(ioh->rc)); } else { iod->hdr.rc = 0; rc = copyout(iod, ioh, iol); } goto exit; } case BNXT_HWRM_FW_GET_TIME: { struct bnxt_ioctl_hwrm_fw_get_time *gtm = &iod->get_time; rc = bnxt_hwrm_fw_get_time(softc, >m->year, >m->month, >m->day, >m->hour, >m->minute, >m->second, >m->millisecond, >m->zone); if (rc) { iod->hdr.rc = rc; rc = copyout(&iod->hdr.rc, &ioh->rc, sizeof(ioh->rc)); } else { iod->hdr.rc = 0; rc = copyout(iod, ioh, iol); } goto exit; } case BNXT_HWRM_FW_SET_TIME: { struct bnxt_ioctl_hwrm_fw_set_time *stm = &iod->set_time; rc = bnxt_hwrm_fw_set_time(softc, stm->year, stm->month, stm->day, stm->hour, stm->minute, stm->second, stm->millisecond, stm->zone); if (rc) { iod->hdr.rc = rc; rc = copyout(&iod->hdr.rc, &ioh->rc, sizeof(ioh->rc)); } else { iod->hdr.rc = 0; rc = copyout(iod, ioh, iol); } goto exit; } } break; } exit: return rc; } static int bnxt_i2c_req(if_ctx_t ctx, struct ifi2creq *i2c) { struct bnxt_softc *softc = iflib_get_softc(ctx); uint8_t *data = i2c->data; int rc; /* No point in going further if phy status indicates * module is not inserted or if it is powered down or * if it is of type 10GBase-T */ if (softc->link_info.module_status > HWRM_PORT_PHY_QCFG_OUTPUT_MODULE_STATUS_WARNINGMSG) return -EOPNOTSUPP; /* This feature is not supported in older firmware versions */ if (!BNXT_CHIP_P5_PLUS(softc) || (softc->hwrm_spec_code < 0x10202)) return -EOPNOTSUPP; rc = bnxt_read_sfp_module_eeprom_info(softc, i2c->dev_addr, 0, 0, 0, i2c->offset, i2c->len, data); return rc; } /* * Support functions */ static int bnxt_probe_phy(struct bnxt_softc *softc) { struct bnxt_link_info *link_info = &softc->link_info; int rc = 0; softc->phy_flags = 0; rc = bnxt_hwrm_phy_qcaps(softc); if (rc) { device_printf(softc->dev, "Probe phy can't get phy capabilities (rc: %x)\n", rc); return rc; } rc = bnxt_update_link(softc, false); if (rc) { device_printf(softc->dev, "Probe phy can't update link (rc: %x)\n", rc); return (rc); } bnxt_get_port_module_status(softc); /*initialize the ethool setting copy with NVM settings */ if (link_info->auto_mode != HWRM_PORT_PHY_QCFG_OUTPUT_AUTO_MODE_NONE) link_info->autoneg |= BNXT_AUTONEG_SPEED; link_info->req_duplex = link_info->duplex_setting; /* NRZ link speed */ if (link_info->autoneg & BNXT_AUTONEG_SPEED) link_info->req_link_speed = link_info->auto_link_speeds; else link_info->req_link_speed = link_info->force_link_speed; /* PAM4 link speed */ if (link_info->auto_pam4_link_speeds) link_info->req_link_speed = link_info->auto_pam4_link_speeds; if (link_info->force_pam4_link_speed) link_info->req_link_speed = link_info->force_pam4_link_speed; return (rc); } static void add_media(struct bnxt_softc *softc, u8 media_type, u16 supported_NRZ_speeds, u16 supported_pam4_speeds, u16 supported_speeds2) { switch (media_type) { case BNXT_MEDIA_CR: BNXT_IFMEDIA_ADD(supported_pam4_speeds, PAM4_SPEEDS_50G, IFM_50G_CP); BNXT_IFMEDIA_ADD(supported_pam4_speeds, PAM4_SPEEDS_100G, IFM_100G_CP2); BNXT_IFMEDIA_ADD(supported_pam4_speeds, PAM4_SPEEDS_200G, IFM_200G_CR4_PAM4); BNXT_IFMEDIA_ADD(supported_NRZ_speeds, SPEEDS_100GB, IFM_100G_CR4); BNXT_IFMEDIA_ADD(supported_NRZ_speeds, SPEEDS_50GB, IFM_50G_CR2); BNXT_IFMEDIA_ADD(supported_NRZ_speeds, SPEEDS_40GB, IFM_40G_CR4); BNXT_IFMEDIA_ADD(supported_NRZ_speeds, SPEEDS_25GB, IFM_25G_CR); BNXT_IFMEDIA_ADD(supported_NRZ_speeds, SPEEDS_10GB, IFM_10G_CR1); BNXT_IFMEDIA_ADD(supported_NRZ_speeds, SPEEDS_1GB, IFM_1000_CX); /* thor2 nrz*/ BNXT_IFMEDIA_ADD(supported_speeds2, SPEEDS2_100GB, IFM_100G_CR4); BNXT_IFMEDIA_ADD(supported_speeds2, SPEEDS2_50GB, IFM_50G_CR2); BNXT_IFMEDIA_ADD(supported_speeds2, SPEEDS2_40GB, IFM_40G_CR4); BNXT_IFMEDIA_ADD(supported_speeds2, SPEEDS2_25GB, IFM_25G_CR); BNXT_IFMEDIA_ADD(supported_speeds2, SPEEDS2_10GB, IFM_10G_CR1); BNXT_IFMEDIA_ADD(supported_speeds2, SPEEDS2_1GB, IFM_1000_CX); /* thor2 PAM56 */ BNXT_IFMEDIA_ADD(supported_speeds2, SPEEDS2_50GB_PAM4_56, IFM_50G_CP); BNXT_IFMEDIA_ADD(supported_speeds2, SPEEDS2_100GB_PAM4_56, IFM_100G_CP2); BNXT_IFMEDIA_ADD(supported_speeds2, SPEEDS2_200GB_PAM4_56, IFM_200G_CR4_PAM4); BNXT_IFMEDIA_ADD(supported_speeds2, SPEEDS2_400GB_PAM4_56, IFM_400G_AUI8); /* thor2 PAM112 */ BNXT_IFMEDIA_ADD(supported_speeds2, SPEEDS2_100GB_PAM4_112, IFM_100G_CR_PAM4); BNXT_IFMEDIA_ADD(supported_speeds2, SPEEDS2_200GB_PAM4_112, IFM_200G_AUI4); BNXT_IFMEDIA_ADD(supported_speeds2, SPEEDS2_400GB_PAM4_112, IFM_400G_AUI8_AC); break; case BNXT_MEDIA_LR: BNXT_IFMEDIA_ADD(supported_pam4_speeds, PAM4_SPEEDS_50G, IFM_50G_LR); BNXT_IFMEDIA_ADD(supported_pam4_speeds, PAM4_SPEEDS_200G, IFM_200G_LR4); BNXT_IFMEDIA_ADD(supported_NRZ_speeds, SPEEDS_100GB, IFM_100G_LR4); BNXT_IFMEDIA_ADD(supported_NRZ_speeds, SPEEDS_50GB, IFM_50G_LR2); BNXT_IFMEDIA_ADD(supported_NRZ_speeds, SPEEDS_40GB, IFM_40G_LR4); BNXT_IFMEDIA_ADD(supported_NRZ_speeds, SPEEDS_25GB, IFM_25G_LR); BNXT_IFMEDIA_ADD(supported_NRZ_speeds, SPEEDS_10GB, IFM_10G_LR); /* thor2 nrz*/ BNXT_IFMEDIA_ADD(supported_speeds2, SPEEDS2_100GB, IFM_100G_LR4); BNXT_IFMEDIA_ADD(supported_speeds2, SPEEDS2_50GB, IFM_50G_LR2); BNXT_IFMEDIA_ADD(supported_speeds2, SPEEDS2_40GB, IFM_40G_LR4); BNXT_IFMEDIA_ADD(supported_speeds2, SPEEDS2_25GB, IFM_25G_LR); BNXT_IFMEDIA_ADD(supported_speeds2, SPEEDS2_10GB, IFM_10G_LR); /* thor2 PAM56 */ BNXT_IFMEDIA_ADD(supported_speeds2, SPEEDS2_50GB_PAM4_56, IFM_50G_LR); BNXT_IFMEDIA_ADD(supported_speeds2, SPEEDS2_100GB_PAM4_56, IFM_100G_AUI2); BNXT_IFMEDIA_ADD(supported_speeds2, SPEEDS2_200GB_PAM4_56, IFM_200G_LR4); BNXT_IFMEDIA_ADD(supported_speeds2, SPEEDS2_400GB_PAM4_56, IFM_400G_LR8); /* thor2 PAM112 */ BNXT_IFMEDIA_ADD(supported_speeds2, SPEEDS2_100GB_PAM4_112, IFM_100G_AUI2_AC); BNXT_IFMEDIA_ADD(supported_speeds2, SPEEDS2_200GB_PAM4_112, IFM_200G_AUI4); BNXT_IFMEDIA_ADD(supported_speeds2, SPEEDS2_400GB_PAM4_112, IFM_400G_AUI8_AC); break; case BNXT_MEDIA_SR: BNXT_IFMEDIA_ADD(supported_pam4_speeds, PAM4_SPEEDS_50G, IFM_50G_SR); BNXT_IFMEDIA_ADD(supported_pam4_speeds, PAM4_SPEEDS_100G, IFM_100G_SR2); BNXT_IFMEDIA_ADD(supported_pam4_speeds, PAM4_SPEEDS_200G, IFM_200G_SR4); BNXT_IFMEDIA_ADD(supported_NRZ_speeds, SPEEDS_100GB, IFM_100G_SR4); BNXT_IFMEDIA_ADD(supported_NRZ_speeds, SPEEDS_50GB, IFM_50G_SR2); BNXT_IFMEDIA_ADD(supported_NRZ_speeds, SPEEDS_40GB, IFM_40G_SR4); BNXT_IFMEDIA_ADD(supported_NRZ_speeds, SPEEDS_25GB, IFM_25G_SR); BNXT_IFMEDIA_ADD(supported_NRZ_speeds, SPEEDS_10GB, IFM_10G_SR); BNXT_IFMEDIA_ADD(supported_NRZ_speeds, SPEEDS_1GB, IFM_1000_SX); /* thor2 nrz*/ BNXT_IFMEDIA_ADD(supported_speeds2, SPEEDS2_100GB, IFM_100G_SR4); BNXT_IFMEDIA_ADD(supported_speeds2, SPEEDS2_50GB, IFM_50G_SR2); BNXT_IFMEDIA_ADD(supported_speeds2, SPEEDS2_40GB, IFM_40G_SR4); BNXT_IFMEDIA_ADD(supported_speeds2, SPEEDS2_25GB, IFM_25G_SR); BNXT_IFMEDIA_ADD(supported_speeds2, SPEEDS2_10GB, IFM_10G_SR); BNXT_IFMEDIA_ADD(supported_speeds2, SPEEDS2_1GB, IFM_1000_SX); /* thor2 PAM56 */ BNXT_IFMEDIA_ADD(supported_speeds2, SPEEDS2_50GB_PAM4_56, IFM_50G_SR); BNXT_IFMEDIA_ADD(supported_speeds2, SPEEDS2_100GB_PAM4_56, IFM_100G_SR2); BNXT_IFMEDIA_ADD(supported_speeds2, SPEEDS2_200GB_PAM4_56, IFM_200G_SR4); BNXT_IFMEDIA_ADD(supported_speeds2, SPEEDS2_400GB_PAM4_56, IFM_400G_AUI8); /* thor2 PAM112 */ BNXT_IFMEDIA_ADD(supported_speeds2, SPEEDS2_100GB_PAM4_112, IFM_100G_AUI2_AC); BNXT_IFMEDIA_ADD(supported_speeds2, SPEEDS2_200GB_PAM4_112, IFM_200G_AUI4); BNXT_IFMEDIA_ADD(supported_speeds2, SPEEDS2_400GB_PAM4_112, IFM_400G_DR4); break; case BNXT_MEDIA_ER: BNXT_IFMEDIA_ADD(supported_NRZ_speeds, SPEEDS_40GB, IFM_40G_ER4); BNXT_IFMEDIA_ADD(supported_NRZ_speeds, SPEEDS_100GB, IFM_100G_AUI4); /* thor2 PAM56 */ BNXT_IFMEDIA_ADD(supported_speeds2, SPEEDS2_50GB_PAM4_56, IFM_50G_LR); BNXT_IFMEDIA_ADD(supported_speeds2, SPEEDS2_100GB_PAM4_56, IFM_100G_AUI2); BNXT_IFMEDIA_ADD(supported_speeds2, SPEEDS2_200GB_PAM4_56, IFM_200G_LR4); BNXT_IFMEDIA_ADD(supported_speeds2, SPEEDS2_400GB_PAM4_56, IFM_400G_FR8); /* thor2 PAM112 */ BNXT_IFMEDIA_ADD(supported_speeds2, SPEEDS2_100GB_PAM4_112, IFM_100G_AUI2_AC); BNXT_IFMEDIA_ADD(supported_speeds2, SPEEDS2_200GB_PAM4_112, IFM_200G_AUI4_AC); BNXT_IFMEDIA_ADD(supported_speeds2, SPEEDS2_400GB_PAM4_112, IFM_400G_AUI8_AC); break; case BNXT_MEDIA_KR: BNXT_IFMEDIA_ADD(supported_pam4_speeds, PAM4_SPEEDS_50G, IFM_50G_KR_PAM4); BNXT_IFMEDIA_ADD(supported_pam4_speeds, PAM4_SPEEDS_100G, IFM_100G_KR2_PAM4); BNXT_IFMEDIA_ADD(supported_pam4_speeds, PAM4_SPEEDS_200G, IFM_200G_KR4_PAM4); BNXT_IFMEDIA_ADD(supported_NRZ_speeds, SPEEDS_100GB, IFM_100G_KR4); BNXT_IFMEDIA_ADD(supported_NRZ_speeds, SPEEDS_50GB, IFM_50G_KR2); BNXT_IFMEDIA_ADD(supported_NRZ_speeds, SPEEDS_50GB, IFM_50G_KR4); BNXT_IFMEDIA_ADD(supported_NRZ_speeds, SPEEDS_40GB, IFM_40G_KR4); BNXT_IFMEDIA_ADD(supported_NRZ_speeds, SPEEDS_25GB, IFM_25G_KR); BNXT_IFMEDIA_ADD(supported_NRZ_speeds, SPEEDS_20GB, IFM_20G_KR2); BNXT_IFMEDIA_ADD(supported_NRZ_speeds, SPEEDS_10GB, IFM_10G_KR); BNXT_IFMEDIA_ADD(supported_NRZ_speeds, SPEEDS_1GB, IFM_1000_KX); break; case BNXT_MEDIA_AC: BNXT_IFMEDIA_ADD(supported_NRZ_speeds, SPEEDS_25GB, IFM_25G_ACC); BNXT_IFMEDIA_ADD(supported_NRZ_speeds, SPEEDS_10GB, IFM_10G_AOC); BNXT_IFMEDIA_ADD(supported_NRZ_speeds, SPEEDS_40GB, IFM_40G_XLAUI); BNXT_IFMEDIA_ADD(supported_NRZ_speeds, SPEEDS_40GB, IFM_40G_XLAUI_AC); BNXT_IFMEDIA_ADD(supported_speeds2, SPEEDS2_25GB, IFM_25G_ACC); BNXT_IFMEDIA_ADD(supported_speeds2, SPEEDS2_10GB, IFM_10G_AOC); BNXT_IFMEDIA_ADD(supported_speeds2, SPEEDS2_40GB, IFM_40G_XLAUI); BNXT_IFMEDIA_ADD(supported_speeds2, SPEEDS2_40GB, IFM_40G_XLAUI_AC); break; case BNXT_MEDIA_BASECX: BNXT_IFMEDIA_ADD(supported_NRZ_speeds, SPEEDS_1GB, IFM_1000_CX); BNXT_IFMEDIA_ADD(supported_speeds2, SPEEDS2_1GB, IFM_1000_CX); break; case BNXT_MEDIA_BASET: BNXT_IFMEDIA_ADD(supported_NRZ_speeds, SPEEDS_10GB, IFM_10G_T); BNXT_IFMEDIA_ADD(supported_NRZ_speeds, SPEEDS_2_5GB, IFM_2500_T); BNXT_IFMEDIA_ADD(supported_NRZ_speeds, SPEEDS_1GB, IFM_1000_T); BNXT_IFMEDIA_ADD(supported_NRZ_speeds, SPEEDS_100MB, IFM_100_T); BNXT_IFMEDIA_ADD(supported_NRZ_speeds, SPEEDS_10MB, IFM_10_T); BNXT_IFMEDIA_ADD(supported_speeds2, SPEEDS2_10GB, IFM_10G_T); BNXT_IFMEDIA_ADD(supported_speeds2, SPEEDS2_1GB, IFM_1000_T); break; case BNXT_MEDIA_BASEKX: BNXT_IFMEDIA_ADD(supported_NRZ_speeds, SPEEDS_10GB, IFM_10G_KR); BNXT_IFMEDIA_ADD(supported_NRZ_speeds, SPEEDS_2_5GB, IFM_2500_KX); BNXT_IFMEDIA_ADD(supported_NRZ_speeds, SPEEDS_1GB, IFM_1000_KX); BNXT_IFMEDIA_ADD(supported_speeds2, SPEEDS2_10GB, IFM_10G_KR); BNXT_IFMEDIA_ADD(supported_speeds2, SPEEDS2_1GB, IFM_1000_KX); break; case BNXT_MEDIA_BASESGMII: BNXT_IFMEDIA_ADD(supported_NRZ_speeds, SPEEDS_1GB, IFM_1000_SGMII); BNXT_IFMEDIA_ADD(supported_speeds2, SPEEDS2_1GB, IFM_1000_SGMII); break; default: break; } return; } static void bnxt_add_media_types(struct bnxt_softc *softc) { struct bnxt_link_info *link_info = &softc->link_info; uint16_t supported_NRZ_speeds = 0, supported_pam4_speeds = 0, supported_speeds2 = 0; uint8_t phy_type = get_phy_type(softc), media_type; supported_NRZ_speeds = link_info->support_speeds; supported_speeds2 = link_info->support_speeds2; supported_pam4_speeds = link_info->support_pam4_speeds; /* Auto is always supported */ ifmedia_add(softc->media, IFM_ETHER | IFM_AUTO, 0, NULL); if (softc->flags & BNXT_FLAG_NPAR) return; switch (phy_type) { case HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_100G_BASECR4: case HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_40G_BASECR4: case HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_25G_BASECR_CA_L: case HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_25G_BASECR_CA_S: case HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_25G_BASECR_CA_N: case HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_BASECR: case HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_50G_BASECR: case HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_100G_BASECR2: case HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_200G_BASECR4: case HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_400G_BASECR8: case HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_100G_BASECR: case HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_200G_BASECR2: case HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_400G_BASECR4: media_type = BNXT_MEDIA_CR; break; case HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_100G_BASELR4: case HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_40G_BASELR4: case HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_BASELR: case HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_50G_BASELR: case HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_100G_BASELR2: case HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_200G_BASELR4: case HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_400G_BASELR8: case HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_100G_BASELR: case HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_200G_BASELR2: case HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_400G_BASELR4: media_type = BNXT_MEDIA_LR; break; case HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_100G_BASESR10: case HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_100G_BASESR4: case HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_40G_BASESR4: case HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_BASESR: case HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_25G_BASESR: case HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_1G_BASESX: case HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_50G_BASESR: case HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_100G_BASESR2: case HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_200G_BASESR4: case HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_400G_BASESR8: case HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_100G_BASESR: case HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_200G_BASESR2: case HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_400G_BASESR4: media_type = BNXT_MEDIA_SR; break; case HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_40G_BASEER4: case HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_100G_BASEER4: case HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_50G_BASEER: case HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_100G_BASEER2: case HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_200G_BASEER4: case HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_400G_BASEER8: case HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_100G_BASEER: case HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_200G_BASEER2: case HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_400G_BASEER4: media_type = BNXT_MEDIA_ER; break; case HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_BASEKR4: case HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_BASEKR2: case HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_BASEKR: media_type = BNXT_MEDIA_KR; break; case HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_40G_ACTIVE_CABLE: media_type = BNXT_MEDIA_AC; break; case HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_1G_BASECX: media_type = BNXT_MEDIA_BASECX; break; case HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_1G_BASET: case HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_BASET: case HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_BASETE: media_type = BNXT_MEDIA_BASET; break; case HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_BASEKX: media_type = BNXT_MEDIA_BASEKX; break; case HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_SGMIIEXTPHY: media_type = BNXT_MEDIA_BASESGMII; break; case HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_UNKNOWN: /* Only Autoneg is supported for TYPE_UNKNOWN */ break; default: /* Only Autoneg is supported for new phy type values */ device_printf(softc->dev, "phy type %d not supported by driver\n", phy_type); break; } switch (link_info->sig_mode) { case BNXT_SIG_MODE_NRZ: if (supported_NRZ_speeds != 0) add_media(softc, media_type, supported_NRZ_speeds, 0, 0); else add_media(softc, media_type, 0, 0, supported_speeds2); break; case BNXT_SIG_MODE_PAM4: if (supported_pam4_speeds != 0) add_media(softc, media_type, 0, supported_pam4_speeds, 0); else add_media(softc, media_type, 0, 0, supported_speeds2); break; case BNXT_SIG_MODE_PAM4_112: add_media(softc, media_type, 0, 0, supported_speeds2); break; } return; } static int bnxt_map_bar(struct bnxt_softc *softc, struct bnxt_bar_info *bar, int bar_num, bool shareable) { uint32_t flag; if (bar->res != NULL) { device_printf(softc->dev, "Bar %d already mapped\n", bar_num); return EDOOFUS; } bar->rid = PCIR_BAR(bar_num); flag = RF_ACTIVE; if (shareable) flag |= RF_SHAREABLE; if ((bar->res = bus_alloc_resource_any(softc->dev, SYS_RES_MEMORY, &bar->rid, flag)) == NULL) { device_printf(softc->dev, "PCI BAR%d mapping failure\n", bar_num); return (ENXIO); } bar->tag = rman_get_bustag(bar->res); bar->handle = rman_get_bushandle(bar->res); bar->size = rman_get_size(bar->res); return 0; } static int bnxt_pci_mapping(struct bnxt_softc *softc) { int rc; rc = bnxt_map_bar(softc, &softc->hwrm_bar, 0, true); if (rc) return rc; rc = bnxt_map_bar(softc, &softc->doorbell_bar, 2, false); return rc; } static void bnxt_pci_mapping_free(struct bnxt_softc *softc) { if (softc->hwrm_bar.res != NULL) bus_release_resource(softc->dev, SYS_RES_MEMORY, softc->hwrm_bar.rid, softc->hwrm_bar.res); softc->hwrm_bar.res = NULL; if (softc->doorbell_bar.res != NULL) bus_release_resource(softc->dev, SYS_RES_MEMORY, softc->doorbell_bar.rid, softc->doorbell_bar.res); softc->doorbell_bar.res = NULL; } static int bnxt_update_link(struct bnxt_softc *softc, bool chng_link_state) { struct bnxt_link_info *link_info = &softc->link_info; uint8_t link_up = link_info->link_up; int rc = 0; rc = bnxt_hwrm_port_phy_qcfg(softc); if (rc) goto exit; /* TODO: need to add more logic to report VF link */ if (chng_link_state) { if (link_info->phy_link_status == HWRM_PORT_PHY_QCFG_OUTPUT_LINK_LINK) link_info->link_up = 1; else link_info->link_up = 0; if (link_up != link_info->link_up) bnxt_report_link(softc); } else { /* always link down if not require to update link state */ link_info->link_up = 0; } exit: return rc; } #define ETHTOOL_SPEED_1000 1000 #define ETHTOOL_SPEED_10000 10000 #define ETHTOOL_SPEED_20000 20000 #define ETHTOOL_SPEED_25000 25000 #define ETHTOOL_SPEED_40000 40000 #define ETHTOOL_SPEED_50000 50000 #define ETHTOOL_SPEED_100000 100000 #define ETHTOOL_SPEED_200000 200000 #define ETHTOOL_SPEED_UNKNOWN -1 static u32 bnxt_fw_to_ethtool_speed(u16 fw_link_speed) { switch (fw_link_speed) { case HWRM_PORT_PHY_QCFG_OUTPUT_LINK_SPEED_1GB: return ETHTOOL_SPEED_1000; case HWRM_PORT_PHY_QCFG_OUTPUT_LINK_SPEED_10GB: return ETHTOOL_SPEED_10000; case HWRM_PORT_PHY_QCFG_OUTPUT_LINK_SPEED_20GB: return ETHTOOL_SPEED_20000; case HWRM_PORT_PHY_QCFG_OUTPUT_LINK_SPEED_25GB: return ETHTOOL_SPEED_25000; case HWRM_PORT_PHY_QCFG_OUTPUT_LINK_SPEED_40GB: return ETHTOOL_SPEED_40000; case HWRM_PORT_PHY_QCFG_OUTPUT_LINK_SPEED_50GB: return ETHTOOL_SPEED_50000; case HWRM_PORT_PHY_QCFG_OUTPUT_LINK_SPEED_100GB: return ETHTOOL_SPEED_100000; case HWRM_PORT_PHY_QCFG_OUTPUT_LINK_SPEED_200GB: return ETHTOOL_SPEED_200000; default: return ETHTOOL_SPEED_UNKNOWN; } } void bnxt_report_link(struct bnxt_softc *softc) { struct bnxt_link_info *link_info = &softc->link_info; const char *duplex = NULL, *flow_ctrl = NULL; const char *signal_mode = ""; if(softc->edev) { softc->edev->espeed = bnxt_fw_to_ethtool_speed(link_info->link_speed); softc->edev->lanes = link_info->active_lanes; } if (link_info->link_up == link_info->last_link_up) { if (!link_info->link_up) return; if ((link_info->duplex == link_info->last_duplex) && (link_info->phy_type == link_info->last_phy_type) && (!(BNXT_IS_FLOW_CTRL_CHANGED(link_info)))) return; } if (link_info->link_up) { if (link_info->duplex == HWRM_PORT_PHY_QCFG_OUTPUT_DUPLEX_CFG_FULL) duplex = "full duplex"; else duplex = "half duplex"; if (link_info->flow_ctrl.tx & link_info->flow_ctrl.rx) flow_ctrl = "FC - receive & transmit"; else if (link_info->flow_ctrl.tx) flow_ctrl = "FC - transmit"; else if (link_info->flow_ctrl.rx) flow_ctrl = "FC - receive"; else flow_ctrl = "FC - none"; if (softc->link_info.phy_qcfg_resp.option_flags & HWRM_PORT_PHY_QCFG_OUTPUT_OPTION_FLAGS_SIGNAL_MODE_KNOWN) { uint8_t sig_mode = softc->link_info.active_fec_sig_mode & HWRM_PORT_PHY_QCFG_OUTPUT_SIGNAL_MODE_MASK; switch (sig_mode) { case BNXT_SIG_MODE_NRZ: signal_mode = "(NRZ) "; break; case BNXT_SIG_MODE_PAM4: signal_mode = "(PAM4 56Gbps) "; break; case BNXT_SIG_MODE_PAM4_112: signal_mode = "(PAM4 112Gbps) "; break; default: break; } link_info->sig_mode = sig_mode; } iflib_link_state_change(softc->ctx, LINK_STATE_UP, IF_Gbps(100)); device_printf(softc->dev, "Link is UP %s %s, %s - %d Mbps \n", duplex, signal_mode, flow_ctrl, (link_info->link_speed * 100)); } else { iflib_link_state_change(softc->ctx, LINK_STATE_DOWN, bnxt_get_baudrate(&softc->link_info)); device_printf(softc->dev, "Link is Down\n"); } link_info->last_link_up = link_info->link_up; link_info->last_duplex = link_info->duplex; link_info->last_phy_type = link_info->phy_type; link_info->last_flow_ctrl.tx = link_info->flow_ctrl.tx; link_info->last_flow_ctrl.rx = link_info->flow_ctrl.rx; link_info->last_flow_ctrl.autoneg = link_info->flow_ctrl.autoneg; /* update media types */ ifmedia_removeall(softc->media); bnxt_add_media_types(softc); ifmedia_set(softc->media, IFM_ETHER | IFM_AUTO); } static int bnxt_handle_isr(void *arg) { struct bnxt_cp_ring *cpr = arg; struct bnxt_softc *softc = cpr->ring.softc; cpr->int_count++; /* Disable further interrupts for this queue */ if (!BNXT_CHIP_P5_PLUS(softc)) softc->db_ops.bnxt_db_rx_cq(cpr, 0); return FILTER_SCHEDULE_THREAD; } static int bnxt_handle_def_cp(void *arg) { struct bnxt_softc *softc = arg; softc->db_ops.bnxt_db_rx_cq(&softc->def_cp_ring, 0); iflib_config_task_enqueue(softc->ctx, &softc->def_cp_task); return FILTER_HANDLED; } static void bnxt_clear_ids(struct bnxt_softc *softc) { int i; softc->def_cp_ring.stats_ctx_id = HWRM_NA_SIGNATURE; softc->def_cp_ring.ring.phys_id = (uint16_t)HWRM_NA_SIGNATURE; softc->def_nq_ring.stats_ctx_id = HWRM_NA_SIGNATURE; softc->def_nq_ring.ring.phys_id = (uint16_t)HWRM_NA_SIGNATURE; for (i = 0; i < softc->ntxqsets; i++) { softc->tx_cp_rings[i].stats_ctx_id = HWRM_NA_SIGNATURE; softc->tx_cp_rings[i].ring.phys_id = (uint16_t)HWRM_NA_SIGNATURE; softc->tx_rings[i].phys_id = (uint16_t)HWRM_NA_SIGNATURE; if (!softc->nq_rings) continue; softc->nq_rings[i].stats_ctx_id = HWRM_NA_SIGNATURE; softc->nq_rings[i].ring.phys_id = (uint16_t)HWRM_NA_SIGNATURE; } for (i = 0; i < softc->nrxqsets; i++) { softc->rx_cp_rings[i].stats_ctx_id = HWRM_NA_SIGNATURE; softc->rx_cp_rings[i].ring.phys_id = (uint16_t)HWRM_NA_SIGNATURE; softc->rx_rings[i].phys_id = (uint16_t)HWRM_NA_SIGNATURE; softc->ag_rings[i].phys_id = (uint16_t)HWRM_NA_SIGNATURE; softc->grp_info[i].grp_id = (uint16_t)HWRM_NA_SIGNATURE; } softc->vnic_info.filter_id = -1; softc->vnic_info.id = (uint16_t)HWRM_NA_SIGNATURE; softc->vnic_info.rss_id = (uint16_t)HWRM_NA_SIGNATURE; memset(softc->vnic_info.rss_grp_tbl.idi_vaddr, 0xff, softc->vnic_info.rss_grp_tbl.idi_size); } static void bnxt_mark_cpr_invalid(struct bnxt_cp_ring *cpr) { struct cmpl_base *cmp = (void *)cpr->ring.vaddr; int i; for (i = 0; i < cpr->ring.ring_size; i++) cmp[i].info3_v = !cpr->v_bit; } static void bnxt_event_error_report(struct bnxt_softc *softc, u32 data1, u32 data2) { u32 err_type = BNXT_EVENT_ERROR_REPORT_TYPE(data1); switch (err_type) { case HWRM_ASYNC_EVENT_CMPL_ERROR_REPORT_BASE_EVENT_DATA1_ERROR_TYPE_INVALID_SIGNAL: device_printf(softc->dev, "1PPS: Received invalid signal on pin%u from the external source. Please fix the signal and reconfigure the pin\n", BNXT_EVENT_INVALID_SIGNAL_DATA(data2)); break; case HWRM_ASYNC_EVENT_CMPL_ERROR_REPORT_BASE_EVENT_DATA1_ERROR_TYPE_PAUSE_STORM: device_printf(softc->dev, "Pause Storm detected!\n"); break; case HWRM_ASYNC_EVENT_CMPL_ERROR_REPORT_BASE_EVENT_DATA1_ERROR_TYPE_DOORBELL_DROP_THRESHOLD: device_printf(softc->dev, "One or more MMIO doorbells dropped by the device! epoch: 0x%x\n", BNXT_EVENT_DBR_EPOCH(data1)); break; case HWRM_ASYNC_EVENT_CMPL_ERROR_REPORT_BASE_EVENT_DATA1_ERROR_TYPE_NVM: { const char *nvm_err_str; if (EVENT_DATA1_NVM_ERR_TYPE_WRITE(data1)) nvm_err_str = "nvm write error"; else if (EVENT_DATA1_NVM_ERR_TYPE_ERASE(data1)) nvm_err_str = "nvm erase error"; else nvm_err_str = "unrecognized nvm error"; device_printf(softc->dev, "%s reported at address 0x%x\n", nvm_err_str, (u32)EVENT_DATA2_NVM_ERR_ADDR(data2)); break; } case HWRM_ASYNC_EVENT_CMPL_ERROR_REPORT_BASE_EVENT_DATA1_ERROR_TYPE_THERMAL_THRESHOLD: { char *threshold_type; char *dir_str; switch (EVENT_DATA1_THERMAL_THRESHOLD_TYPE(data1)) { case HWRM_ASYNC_EVENT_CMPL_ERROR_REPORT_THERMAL_EVENT_DATA1_THRESHOLD_TYPE_WARN: threshold_type = "warning"; break; case HWRM_ASYNC_EVENT_CMPL_ERROR_REPORT_THERMAL_EVENT_DATA1_THRESHOLD_TYPE_CRITICAL: threshold_type = "critical"; break; case HWRM_ASYNC_EVENT_CMPL_ERROR_REPORT_THERMAL_EVENT_DATA1_THRESHOLD_TYPE_FATAL: threshold_type = "fatal"; break; case HWRM_ASYNC_EVENT_CMPL_ERROR_REPORT_THERMAL_EVENT_DATA1_THRESHOLD_TYPE_SHUTDOWN: threshold_type = "shutdown"; break; default: device_printf(softc->dev, "Unknown Thermal threshold type event\n"); return; } if (EVENT_DATA1_THERMAL_THRESHOLD_DIR_INCREASING(data1)) dir_str = "above"; else dir_str = "below"; device_printf(softc->dev, "Chip temperature has gone %s the %s thermal threshold!\n", dir_str, threshold_type); device_printf(softc->dev, "Temperature (In Celsius), Current: %u, threshold: %u\n", BNXT_EVENT_THERMAL_CURRENT_TEMP(data2), BNXT_EVENT_THERMAL_THRESHOLD_TEMP(data2)); break; } case HWRM_ASYNC_EVENT_CMPL_ERROR_REPORT_BASE_EVENT_DATA1_ERROR_TYPE_DUAL_DATA_RATE_NOT_SUPPORTED: device_printf(softc->dev, "Speed change is not supported with dual rate transceivers on this board\n"); break; default: device_printf(softc->dev, "FW reported unknown error type: %u, data1: 0x%x data2: 0x%x\n", err_type, data1, data2); break; } } static void bnxt_handle_async_event(struct bnxt_softc *softc, struct cmpl_base *cmpl) { struct hwrm_async_event_cmpl *ae = (void *)cmpl; uint16_t async_id = le16toh(ae->event_id); struct ifmediareq ifmr; char *type_str; char *status_desc; struct bnxt_fw_health *fw_health; u32 data1 = le32toh(ae->event_data1); u32 data2 = le32toh(ae->event_data2); switch (async_id) { case HWRM_ASYNC_EVENT_CMPL_EVENT_ID_LINK_STATUS_CHANGE: case HWRM_ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CHANGE: case HWRM_ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CFG_CHANGE: if (BNXT_CHIP_P5_PLUS(softc)) bit_set(softc->state_bv, BNXT_STATE_LINK_CHANGE); else bnxt_media_status(softc->ctx, &ifmr); break; case HWRM_ASYNC_EVENT_CMPL_EVENT_ID_ERROR_REPORT: { bnxt_event_error_report(softc, data1, data2); goto async_event_process_exit; } case HWRM_ASYNC_EVENT_CMPL_EVENT_ID_DOORBELL_PACING_THRESHOLD: case HWRM_ASYNC_EVENT_CMPL_EVENT_ID_DOORBELL_PACING_NQ_UPDATE: break; case HWRM_ASYNC_EVENT_CMPL_EVENT_ID_RESET_NOTIFY: { type_str = "Solicited"; if (!softc->fw_health) goto async_event_process_exit; softc->fw_reset_timestamp = jiffies; softc->fw_reset_min_dsecs = ae->timestamp_lo; if (!softc->fw_reset_min_dsecs) softc->fw_reset_min_dsecs = BNXT_DFLT_FW_RST_MIN_DSECS; softc->fw_reset_max_dsecs = le16toh(ae->timestamp_hi); if (!softc->fw_reset_max_dsecs) softc->fw_reset_max_dsecs = BNXT_DFLT_FW_RST_MAX_DSECS; if (EVENT_DATA1_RESET_NOTIFY_FW_ACTIVATION(data1)) { set_bit(BNXT_STATE_FW_ACTIVATE_RESET, &softc->state); } else if (EVENT_DATA1_RESET_NOTIFY_FATAL(data1)) { type_str = "Fatal"; softc->fw_health->fatalities++; set_bit(BNXT_STATE_FW_FATAL_COND, &softc->state); } else if (data2 && BNXT_FW_STATUS_HEALTHY != EVENT_DATA2_RESET_NOTIFY_FW_STATUS_CODE(data2)) { type_str = "Non-fatal"; softc->fw_health->survivals++; set_bit(BNXT_STATE_FW_NON_FATAL_COND, &softc->state); } device_printf(softc->dev, "%s firmware reset event, data1: 0x%x, data2: 0x%x, min wait %u ms, max wait %u ms\n", type_str, data1, data2, softc->fw_reset_min_dsecs * 100, softc->fw_reset_max_dsecs * 100); set_bit(BNXT_FW_RESET_NOTIFY_SP_EVENT, &softc->sp_event); break; } case HWRM_ASYNC_EVENT_CMPL_EVENT_ID_ERROR_RECOVERY: { fw_health = softc->fw_health; status_desc = "healthy"; u32 status; if (!fw_health) goto async_event_process_exit; if (!EVENT_DATA1_RECOVERY_ENABLED(data1)) { fw_health->enabled = false; device_printf(softc->dev, "Driver recovery watchdog is disabled\n"); break; } fw_health->primary = EVENT_DATA1_RECOVERY_MASTER_FUNC(data1); fw_health->tmr_multiplier = DIV_ROUND_UP(fw_health->polling_dsecs * HZ, HZ * 10); fw_health->tmr_counter = fw_health->tmr_multiplier; if (!fw_health->enabled) fw_health->last_fw_heartbeat = bnxt_fw_health_readl(softc, BNXT_FW_HEARTBEAT_REG); fw_health->last_fw_reset_cnt = bnxt_fw_health_readl(softc, BNXT_FW_RESET_CNT_REG); status = bnxt_fw_health_readl(softc, BNXT_FW_HEALTH_REG); if (status != BNXT_FW_STATUS_HEALTHY) status_desc = "unhealthy"; device_printf(softc->dev, "Driver recovery watchdog, role: %s, firmware status: 0x%x (%s), resets: %u\n", fw_health->primary ? "primary" : "backup", status, status_desc, fw_health->last_fw_reset_cnt); if (!fw_health->enabled) { /* Make sure tmr_counter is set and seen by * bnxt_health_check() before setting enabled */ smp_mb(); fw_health->enabled = true; } goto async_event_process_exit; } case HWRM_ASYNC_EVENT_CMPL_EVENT_ID_LINK_MTU_CHANGE: case HWRM_ASYNC_EVENT_CMPL_EVENT_ID_DCB_CONFIG_CHANGE: case HWRM_ASYNC_EVENT_CMPL_EVENT_ID_PORT_CONN_NOT_ALLOWED: case HWRM_ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CFG_NOT_ALLOWED: case HWRM_ASYNC_EVENT_CMPL_EVENT_ID_FUNC_DRVR_UNLOAD: case HWRM_ASYNC_EVENT_CMPL_EVENT_ID_FUNC_DRVR_LOAD: case HWRM_ASYNC_EVENT_CMPL_EVENT_ID_PF_DRVR_UNLOAD: case HWRM_ASYNC_EVENT_CMPL_EVENT_ID_PF_DRVR_LOAD: case HWRM_ASYNC_EVENT_CMPL_EVENT_ID_VF_FLR: case HWRM_ASYNC_EVENT_CMPL_EVENT_ID_VF_MAC_ADDR_CHANGE: case HWRM_ASYNC_EVENT_CMPL_EVENT_ID_PF_VF_COMM_STATUS_CHANGE: case HWRM_ASYNC_EVENT_CMPL_EVENT_ID_VF_CFG_CHANGE: case HWRM_ASYNC_EVENT_CMPL_EVENT_ID_HWRM_ERROR: device_printf(softc->dev, "Unhandled async completion type %u\n", async_id); break; default: dev_dbg(softc->dev, "Unknown Async event completion type %u\n", async_id); break; } bnxt_queue_sp_work(softc); async_event_process_exit: bnxt_ulp_async_events(softc, ae); } static void bnxt_def_cp_task(void *context, int pending) { if_ctx_t ctx = context; struct bnxt_softc *softc = iflib_get_softc(ctx); struct bnxt_cp_ring *cpr = &softc->def_cp_ring; /* Handle completions on the default completion ring */ struct cmpl_base *cmpl; uint32_t cons = cpr->cons; bool v_bit = cpr->v_bit; bool last_v_bit; uint32_t last_cons; uint16_t type; for (;;) { last_cons = cons; last_v_bit = v_bit; NEXT_CP_CONS_V(&cpr->ring, cons, v_bit); cmpl = &((struct cmpl_base *)cpr->ring.vaddr)[cons]; if (!CMP_VALID(cmpl, v_bit)) break; type = le16toh(cmpl->type) & CMPL_BASE_TYPE_MASK; switch (type) { case CMPL_BASE_TYPE_HWRM_ASYNC_EVENT: bnxt_handle_async_event(softc, cmpl); break; case CMPL_BASE_TYPE_TX_L2: case CMPL_BASE_TYPE_RX_L2: case CMPL_BASE_TYPE_RX_L2_V3: case CMPL_BASE_TYPE_RX_AGG: case CMPL_BASE_TYPE_RX_TPA_START: case CMPL_BASE_TYPE_RX_TPA_START_V3: case CMPL_BASE_TYPE_RX_TPA_END: case CMPL_BASE_TYPE_STAT_EJECT: case CMPL_BASE_TYPE_HWRM_DONE: case CMPL_BASE_TYPE_HWRM_FWD_REQ: case CMPL_BASE_TYPE_HWRM_FWD_RESP: case CMPL_BASE_TYPE_CQ_NOTIFICATION: case CMPL_BASE_TYPE_SRQ_EVENT: case CMPL_BASE_TYPE_DBQ_EVENT: case CMPL_BASE_TYPE_QP_EVENT: case CMPL_BASE_TYPE_FUNC_EVENT: dev_dbg(softc->dev, "Unhandled Async event completion type %u\n", type); break; default: dev_dbg(softc->dev, "Unknown Async event completion type %u\n", type); break; } } cpr->cons = last_cons; cpr->v_bit = last_v_bit; softc->db_ops.bnxt_db_rx_cq(cpr, 1); } uint8_t get_phy_type(struct bnxt_softc *softc) { struct bnxt_link_info *link_info = &softc->link_info; uint8_t phy_type = link_info->phy_type; uint16_t supported; if (phy_type != HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_UNKNOWN) return phy_type; /* Deduce the phy type from the media type and supported speeds */ supported = link_info->support_speeds; if (link_info->media_type == HWRM_PORT_PHY_QCFG_OUTPUT_MEDIA_TYPE_TP) return HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_BASET; if (link_info->media_type == HWRM_PORT_PHY_QCFG_OUTPUT_MEDIA_TYPE_DAC) { if (supported & HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_SPEEDS_2_5GB) return HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_BASEKX; if (supported & HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_SPEEDS_20GB) return HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_BASEKR; return HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_BASECR; } if (link_info->media_type == HWRM_PORT_PHY_QCFG_OUTPUT_MEDIA_TYPE_FIBRE) return HWRM_PORT_PHY_QCFG_OUTPUT_PHY_TYPE_BASESR; return phy_type; } bool bnxt_check_hwrm_version(struct bnxt_softc *softc) { char buf[16]; sprintf(buf, "%hhu.%hhu.%hhu", softc->ver_info->hwrm_min_major, softc->ver_info->hwrm_min_minor, softc->ver_info->hwrm_min_update); if (softc->ver_info->hwrm_min_major > softc->ver_info->hwrm_if_major) { device_printf(softc->dev, "WARNING: HWRM version %s is too old (older than %s)\n", softc->ver_info->hwrm_if_ver, buf); return false; } else if(softc->ver_info->hwrm_min_major == softc->ver_info->hwrm_if_major) { if (softc->ver_info->hwrm_min_minor > softc->ver_info->hwrm_if_minor) { device_printf(softc->dev, "WARNING: HWRM version %s is too old (older than %s)\n", softc->ver_info->hwrm_if_ver, buf); return false; } else if (softc->ver_info->hwrm_min_minor == softc->ver_info->hwrm_if_minor) { if (softc->ver_info->hwrm_min_update > softc->ver_info->hwrm_if_update) { device_printf(softc->dev, "WARNING: HWRM version %s is too old (older than %s)\n", softc->ver_info->hwrm_if_ver, buf); return false; } } } return true; } static uint64_t bnxt_get_baudrate(struct bnxt_link_info *link) { switch (link->link_speed) { case HWRM_PORT_PHY_QCFG_OUTPUT_LINK_SPEED_100MB: return IF_Mbps(100); case HWRM_PORT_PHY_QCFG_OUTPUT_LINK_SPEED_1GB: return IF_Gbps(1); case HWRM_PORT_PHY_QCFG_OUTPUT_LINK_SPEED_2GB: return IF_Gbps(2); case HWRM_PORT_PHY_QCFG_OUTPUT_LINK_SPEED_2_5GB: return IF_Mbps(2500); case HWRM_PORT_PHY_QCFG_OUTPUT_LINK_SPEED_10GB: return IF_Gbps(10); case HWRM_PORT_PHY_QCFG_OUTPUT_LINK_SPEED_20GB: return IF_Gbps(20); case HWRM_PORT_PHY_QCFG_OUTPUT_LINK_SPEED_25GB: return IF_Gbps(25); case HWRM_PORT_PHY_QCFG_OUTPUT_LINK_SPEED_40GB: return IF_Gbps(40); case HWRM_PORT_PHY_QCFG_OUTPUT_LINK_SPEED_50GB: return IF_Gbps(50); case HWRM_PORT_PHY_QCFG_OUTPUT_LINK_SPEED_100GB: return IF_Gbps(100); case HWRM_PORT_PHY_QCFG_OUTPUT_LINK_SPEED_10MB: return IF_Mbps(10); case HWRM_PORT_PHY_QCFG_OUTPUT_LINK_SPEED_200GB: return IF_Gbps(200); case HWRM_PORT_PHY_QCFG_OUTPUT_LINK_SPEED_400GB: return IF_Gbps(400); } return IF_Gbps(100); } static void bnxt_get_wol_settings(struct bnxt_softc *softc) { uint16_t wol_handle = 0; if (!bnxt_wol_supported(softc)) return; do { wol_handle = bnxt_hwrm_get_wol_fltrs(softc, wol_handle); } while (wol_handle && wol_handle != BNXT_NO_MORE_WOL_FILTERS); } diff --git a/sys/dev/bnxt/bnxt_re/main.c b/sys/dev/bnxt/bnxt_re/main.c index eb21c770ca5f..dc68854157a0 100644 --- a/sys/dev/bnxt/bnxt_re/main.c +++ b/sys/dev/bnxt/bnxt_re/main.c @@ -1,4475 +1,4494 @@ /* * Copyright (c) 2015-2024, Broadcom. All rights reserved. The term * Broadcom refers to Broadcom Limited and/or its subsidiaries. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in * the documentation and/or other materials provided with the * distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * * Description: Main component of the bnxt_re driver */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "bnxt_re.h" #include "ib_verbs.h" #include "bnxt_re-abi.h" #include "bnxt.h" static char drv_version[] = "Broadcom NetXtreme-C/E RoCE Driver " ROCE_DRV_MODULE_NAME \ " v" ROCE_DRV_MODULE_VERSION " (" ROCE_DRV_MODULE_RELDATE ")\n"; #define BNXT_RE_DESC "Broadcom NetXtreme RoCE" #define BNXT_ADEV_NAME "if_bnxt" MODULE_DESCRIPTION("Broadcom NetXtreme-C/E RoCE Driver"); MODULE_LICENSE("Dual BSD/GPL"); MODULE_DEPEND(bnxt_re, linuxkpi, 1, 1, 1); MODULE_DEPEND(bnxt_re, ibcore, 1, 1, 1); MODULE_DEPEND(bnxt_re, if_bnxt, 1, 1, 1); MODULE_VERSION(bnxt_re, 1); DEFINE_MUTEX(bnxt_re_mutex); /* mutex lock for driver */ static unsigned int restrict_mrs = 0; module_param(restrict_mrs, uint, 0); MODULE_PARM_DESC(restrict_mrs, " Restrict the no. of MRs 0 = 256K , 1 = 64K"); unsigned int restrict_stats = 0; module_param(restrict_stats, uint, 0); MODULE_PARM_DESC(restrict_stats, "Restrict stats query frequency to ethtool coalesce value. Disabled by default"); unsigned int enable_fc = 1; module_param(enable_fc, uint, 0); MODULE_PARM_DESC(enable_fc, "Enable default PFC, CC,ETS during driver load. 1 - fc enable, 0 - fc disable - Default is 1"); unsigned int min_tx_depth = 1; module_param(min_tx_depth, uint, 0); MODULE_PARM_DESC(min_tx_depth, "Minimum TX depth - Default is 1"); static uint8_t max_msix_vec[BNXT_RE_MAX_DEVICES] = {0}; static unsigned int max_msix_vec_argc; module_param_array(max_msix_vec, byte, &max_msix_vec_argc, 0444); MODULE_PARM_DESC(max_msix_vec, "Max MSI-x vectors per PF (2 - 64) - Default is 64"); unsigned int cmdq_shadow_qd = RCFW_CMD_NON_BLOCKING_SHADOW_QD; module_param_named(cmdq_shadow_qd, cmdq_shadow_qd, uint, 0644); MODULE_PARM_DESC(cmdq_shadow_qd, "Perf Stat Debug: Shadow QD Range (1-64) - Default is 64"); /* globals */ struct list_head bnxt_re_dev_list = LINUX_LIST_HEAD_INIT(bnxt_re_dev_list); static int bnxt_re_probe_count; DEFINE_MUTEX(bnxt_re_dev_lock); static u32 gmod_exit; static u32 gadd_dev_inprogress; static void bnxt_re_task(struct work_struct *work_task); static struct workqueue_struct *bnxt_re_wq; static int bnxt_re_query_hwrm_intf_version(struct bnxt_re_dev *rdev); static int bnxt_re_hwrm_qcfg(struct bnxt_re_dev *rdev, u32 *db_len, u32 *offset); static int bnxt_re_ib_init(struct bnxt_re_dev *rdev); static void bnxt_re_ib_init_2(struct bnxt_re_dev *rdev); void _bnxt_re_remove(struct auxiliary_device *adev); void writel_fbsd(struct bnxt_softc *bp, u32, u8, u32); u32 readl_fbsd(struct bnxt_softc *bp, u32, u8); static int bnxt_re_hwrm_dbr_pacing_qcfg(struct bnxt_re_dev *rdev); int bnxt_re_register_netdevice_notifier(struct notifier_block *nb) { int rc; rc = register_netdevice_notifier(nb); return rc; } int bnxt_re_unregister_netdevice_notifier(struct notifier_block *nb) { int rc; rc = unregister_netdevice_notifier(nb); return rc; } void bnxt_re_set_dma_device(struct ib_device *ibdev, struct bnxt_re_dev *rdev) { ibdev->dma_device = &rdev->en_dev->pdev->dev; } void bnxt_re_init_resolve_wq(struct bnxt_re_dev *rdev) { rdev->resolve_wq = create_singlethread_workqueue("bnxt_re_resolve_wq"); INIT_LIST_HEAD(&rdev->mac_wq_list); } void bnxt_re_uninit_resolve_wq(struct bnxt_re_dev *rdev) { struct bnxt_re_resolve_dmac_work *tmp_work = NULL, *tmp_st; if (!rdev->resolve_wq) return; flush_workqueue(rdev->resolve_wq); list_for_each_entry_safe(tmp_work, tmp_st, &rdev->mac_wq_list, list) { list_del(&tmp_work->list); kfree(tmp_work); } destroy_workqueue(rdev->resolve_wq); rdev->resolve_wq = NULL; } u32 readl_fbsd(struct bnxt_softc *bp, u32 reg_off, u8 bar_idx) { if (bar_idx) return bus_space_read_8(bp->doorbell_bar.tag, bp->doorbell_bar.handle, reg_off); else return bus_space_read_8(bp->hwrm_bar.tag, bp->hwrm_bar.handle, reg_off); } void writel_fbsd(struct bnxt_softc *bp, u32 reg_off, u8 bar_idx, u32 val) { if (bar_idx) bus_space_write_8(bp->doorbell_bar.tag, bp->doorbell_bar.handle, reg_off, htole32(val)); else bus_space_write_8(bp->hwrm_bar.tag, bp->hwrm_bar.handle, reg_off, htole32(val)); } static void bnxt_re_update_fifo_occup_slabs(struct bnxt_re_dev *rdev, u32 fifo_occup) { if (fifo_occup > rdev->dbg_stats->dbq.fifo_occup_water_mark) rdev->dbg_stats->dbq.fifo_occup_water_mark = fifo_occup; if (fifo_occup > 8 * rdev->pacing_algo_th) rdev->dbg_stats->dbq.fifo_occup_slab_4++; else if (fifo_occup > 4 * rdev->pacing_algo_th) rdev->dbg_stats->dbq.fifo_occup_slab_3++; else if (fifo_occup > 2 * rdev->pacing_algo_th) rdev->dbg_stats->dbq.fifo_occup_slab_2++; else if (fifo_occup > rdev->pacing_algo_th) rdev->dbg_stats->dbq.fifo_occup_slab_1++; } static void bnxt_re_update_do_pacing_slabs(struct bnxt_re_dev *rdev) { struct bnxt_qplib_db_pacing_data *pacing_data = rdev->qplib_res.pacing_data; if (pacing_data->do_pacing > rdev->dbg_stats->dbq.do_pacing_water_mark) rdev->dbg_stats->dbq.do_pacing_water_mark = pacing_data->do_pacing; if (pacing_data->do_pacing > 16 * rdev->dbr_def_do_pacing) rdev->dbg_stats->dbq.do_pacing_slab_5++; else if (pacing_data->do_pacing > 8 * rdev->dbr_def_do_pacing) rdev->dbg_stats->dbq.do_pacing_slab_4++; else if (pacing_data->do_pacing > 4 * rdev->dbr_def_do_pacing) rdev->dbg_stats->dbq.do_pacing_slab_3++; else if (pacing_data->do_pacing > 2 * rdev->dbr_def_do_pacing) rdev->dbg_stats->dbq.do_pacing_slab_2++; else if (pacing_data->do_pacing > rdev->dbr_def_do_pacing) rdev->dbg_stats->dbq.do_pacing_slab_1++; } static bool bnxt_re_is_qp1_qp(struct bnxt_re_qp *qp) { return qp->ib_qp.qp_type == IB_QPT_GSI; } static struct bnxt_re_qp *bnxt_re_get_qp1_qp(struct bnxt_re_dev *rdev) { struct bnxt_re_qp *qp; mutex_lock(&rdev->qp_lock); list_for_each_entry(qp, &rdev->qp_list, list) { if (bnxt_re_is_qp1_qp(qp)) { mutex_unlock(&rdev->qp_lock); return qp; } } mutex_unlock(&rdev->qp_lock); return NULL; } /* Set the maximum number of each resource that the driver actually wants * to allocate. This may be up to the maximum number the firmware has * reserved for the function. The driver may choose to allocate fewer * resources than the firmware maximum. */ static void bnxt_re_limit_pf_res(struct bnxt_re_dev *rdev) { struct bnxt_qplib_max_res dev_res = {}; struct bnxt_qplib_chip_ctx *cctx; struct bnxt_qplib_dev_attr *attr; struct bnxt_qplib_ctx *hctx; int i; attr = rdev->dev_attr; hctx = rdev->qplib_res.hctx; cctx = rdev->chip_ctx; bnxt_qplib_max_res_supported(cctx, &rdev->qplib_res, &dev_res, false); if (!_is_chip_gen_p5_p7(cctx)) { hctx->qp_ctx.max = min_t(u32, dev_res.max_qp, attr->max_qp); hctx->mrw_ctx.max = min_t(u32, dev_res.max_mr, attr->max_mr); /* To accommodate 16k MRs and 16k AHs, * driver has to allocate 32k backing store memory */ hctx->mrw_ctx.max *= 2; hctx->srq_ctx.max = min_t(u32, dev_res.max_srq, attr->max_srq); hctx->cq_ctx.max = min_t(u32, dev_res.max_cq, attr->max_cq); for (i = 0; i < MAX_TQM_ALLOC_REQ; i++) hctx->tqm_ctx.qcount[i] = attr->tqm_alloc_reqs[i]; } else { hctx->qp_ctx.max = attr->max_qp ? attr->max_qp : dev_res.max_qp; hctx->mrw_ctx.max = attr->max_mr ? attr->max_mr : dev_res.max_mr; hctx->srq_ctx.max = attr->max_srq ? attr->max_srq : dev_res.max_srq; hctx->cq_ctx.max = attr->max_cq ? attr->max_cq : dev_res.max_cq; } } static void bnxt_re_limit_vf_res(struct bnxt_re_dev *rdev, struct bnxt_qplib_vf_res *vf_res, u32 num_vf) { struct bnxt_qplib_chip_ctx *cctx = rdev->chip_ctx; struct bnxt_qplib_max_res dev_res = {}; bnxt_qplib_max_res_supported(cctx, &rdev->qplib_res, &dev_res, true); vf_res->max_qp = dev_res.max_qp / num_vf; vf_res->max_srq = dev_res.max_srq / num_vf; vf_res->max_cq = dev_res.max_cq / num_vf; /* * MR and AH shares the same backing store, the value specified * for max_mrw is split into half by the FW for MR and AH */ vf_res->max_mrw = dev_res.max_mr * 2 / num_vf; vf_res->max_gid = BNXT_RE_MAX_GID_PER_VF; } static void bnxt_re_set_resource_limits(struct bnxt_re_dev *rdev) { struct bnxt_qplib_ctx *hctx; hctx = rdev->qplib_res.hctx; memset(&hctx->vf_res, 0, sizeof(struct bnxt_qplib_vf_res)); bnxt_re_limit_pf_res(rdev); if (rdev->num_vfs) bnxt_re_limit_vf_res(rdev, &hctx->vf_res, rdev->num_vfs); } static void bnxt_re_dettach_irq(struct bnxt_re_dev *rdev) { struct bnxt_qplib_rcfw *rcfw = NULL; struct bnxt_qplib_nq *nq; int indx; rcfw = &rdev->rcfw; for (indx = 0; indx < rdev->nqr.max_init; indx++) { nq = &rdev->nqr.nq[indx]; mutex_lock(&nq->lock); bnxt_qplib_nq_stop_irq(nq, false); mutex_unlock(&nq->lock); } bnxt_qplib_rcfw_stop_irq(rcfw, false); } static void bnxt_re_detach_err_device(struct bnxt_re_dev *rdev) { /* Free the MSIx vectors only so that L2 can proceed with MSIx disable */ bnxt_re_dettach_irq(rdev); /* Set the state as detached to prevent sending any more commands */ set_bit(ERR_DEVICE_DETACHED, &rdev->rcfw.cmdq.flags); set_bit(BNXT_RE_FLAG_ERR_DEVICE_DETACHED, &rdev->flags); wake_up_all(&rdev->rcfw.cmdq.waitq); } #define MAX_DSCP_PRI_TUPLE 64 struct bnxt_re_dcb_work { struct work_struct work; struct bnxt_re_dev *rdev; struct hwrm_async_event_cmpl cmpl; }; static void bnxt_re_init_dcb_wq(struct bnxt_re_dev *rdev) { rdev->dcb_wq = create_singlethread_workqueue("bnxt_re_dcb_wq"); } static void bnxt_re_uninit_dcb_wq(struct bnxt_re_dev *rdev) { if (!rdev->dcb_wq) return; flush_workqueue(rdev->dcb_wq); destroy_workqueue(rdev->dcb_wq); rdev->dcb_wq = NULL; } static void bnxt_re_init_aer_wq(struct bnxt_re_dev *rdev) { rdev->aer_wq = create_singlethread_workqueue("bnxt_re_aer_wq"); } static void bnxt_re_uninit_aer_wq(struct bnxt_re_dev *rdev) { if (!rdev->aer_wq) return; flush_workqueue(rdev->aer_wq); destroy_workqueue(rdev->aer_wq); rdev->aer_wq = NULL; } static int bnxt_re_update_qp1_tos_dscp(struct bnxt_re_dev *rdev) { struct bnxt_re_qp *qp; if (!_is_chip_gen_p5_p7(rdev->chip_ctx)) return 0; qp = bnxt_re_get_qp1_qp(rdev); if (!qp) return 0; qp->qplib_qp.modify_flags = CMDQ_MODIFY_QP_MODIFY_MASK_TOS_DSCP; qp->qplib_qp.tos_dscp = rdev->cc_param.qp1_tos_dscp; return bnxt_qplib_modify_qp(&rdev->qplib_res, &qp->qplib_qp); } static void bnxt_re_reconfigure_dscp(struct bnxt_re_dev *rdev) { struct bnxt_qplib_cc_param *cc_param; struct bnxt_re_tc_rec *tc_rec; bool update_cc = false; u8 dscp_user; int rc; cc_param = &rdev->cc_param; tc_rec = &rdev->tc_rec[0]; if (!(cc_param->roce_dscp_user || cc_param->cnp_dscp_user)) return; if (cc_param->cnp_dscp_user) { dscp_user = (cc_param->cnp_dscp_user & 0x3f); if ((tc_rec->cnp_dscp_bv & (1ul << dscp_user)) && (cc_param->alt_tos_dscp != dscp_user)) { cc_param->alt_tos_dscp = dscp_user; cc_param->mask |= CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_ALT_TOS_DSCP; update_cc = true; } } if (cc_param->roce_dscp_user) { dscp_user = (cc_param->roce_dscp_user & 0x3f); if ((tc_rec->roce_dscp_bv & (1ul << dscp_user)) && (cc_param->tos_dscp != dscp_user)) { cc_param->tos_dscp = dscp_user; cc_param->mask |= CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_TOS_DSCP; update_cc = true; } } if (update_cc) { rc = bnxt_qplib_modify_cc(&rdev->qplib_res, cc_param); if (rc) dev_err(rdev_to_dev(rdev), "Failed to apply cc settings\n"); } } static void bnxt_re_dcb_wq_task(struct work_struct *work) { struct bnxt_qplib_cc_param *cc_param; struct bnxt_re_tc_rec *tc_rec; struct bnxt_re_dev *rdev; struct bnxt_re_dcb_work *dcb_work = container_of(work, struct bnxt_re_dcb_work, work); int rc; rdev = dcb_work->rdev; if (!rdev) goto exit; mutex_lock(&rdev->cc_lock); cc_param = &rdev->cc_param; rc = bnxt_qplib_query_cc_param(&rdev->qplib_res, cc_param); if (rc) { dev_err(rdev_to_dev(rdev), "Failed to query ccparam rc:%d", rc); goto fail; } tc_rec = &rdev->tc_rec[0]; /* * Upon the receival of DCB Async event: * If roce_dscp or cnp_dscp or both (which user configured using configfs) * is in the list, re-program the value using modify_roce_cc command */ bnxt_re_reconfigure_dscp(rdev); cc_param->roce_pri = tc_rec->roce_prio; if (cc_param->qp1_tos_dscp != cc_param->tos_dscp) { cc_param->qp1_tos_dscp = cc_param->tos_dscp; rc = bnxt_re_update_qp1_tos_dscp(rdev); if (rc) { dev_err(rdev_to_dev(rdev), "%s:Failed to modify QP1 rc:%d", __func__, rc); goto fail; } } fail: mutex_unlock(&rdev->cc_lock); exit: kfree(dcb_work); } static int bnxt_re_hwrm_dbr_pacing_broadcast_event(struct bnxt_re_dev *rdev) { struct hwrm_func_dbr_pacing_broadcast_event_output resp = {0}; struct hwrm_func_dbr_pacing_broadcast_event_input req = {0}; struct bnxt_en_dev *en_dev = rdev->en_dev; struct bnxt_fw_msg fw_msg; int rc; memset(&fw_msg, 0, sizeof(fw_msg)); bnxt_re_init_hwrm_hdr(rdev, (void *)&req, HWRM_FUNC_DBR_PACING_BROADCAST_EVENT, -1, -1); bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp, sizeof(resp), BNXT_RE_HWRM_CMD_TIMEOUT(rdev)); rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg); if (rc) { dev_dbg(rdev_to_dev(rdev), "Failed to send dbr pacing broadcast event rc:%d", rc); return rc; } return 0; } static int bnxt_re_hwrm_dbr_pacing_nqlist_query(struct bnxt_re_dev *rdev) { struct hwrm_func_dbr_pacing_nqlist_query_output resp = {0}; struct hwrm_func_dbr_pacing_nqlist_query_input req = {0}; struct bnxt_dbq_nq_list *nq_list = &rdev->nq_list; struct bnxt_en_dev *en_dev = rdev->en_dev; bool primary_found = false; struct bnxt_fw_msg fw_msg; struct bnxt_qplib_nq *nq; int rc, i, j = 1; u16 *nql_ptr; nq = &rdev->nqr.nq[0]; memset(&fw_msg, 0, sizeof(fw_msg)); bnxt_re_init_hwrm_hdr(rdev, (void *)&req, HWRM_FUNC_DBR_PACING_NQLIST_QUERY, -1, -1); bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp, sizeof(resp), BNXT_RE_HWRM_CMD_TIMEOUT(rdev)); rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg); if (rc) { dev_err(rdev_to_dev(rdev), "Failed to send dbr pacing nq list query rc:%d", rc); return rc; } nq_list->num_nql_entries = le32_to_cpu(resp.num_nqs); nql_ptr = &resp.nq_ring_id0; /* populate the nq_list of the primary function with list received * from FW. Fill the NQ IDs of secondary functions from index 1 to * num_nql_entries - 1. Fill the nq_list->nq_id[0] with the * nq_id of the primary pf */ for (i = 0; i < nq_list->num_nql_entries; i++) { u16 nq_id = *nql_ptr; dev_dbg(rdev_to_dev(rdev), "nq_list->nq_id[%d] = %d\n", i, nq_id); if (nq_id != nq->ring_id) { nq_list->nq_id[j] = nq_id; j++; } else { primary_found = true; nq_list->nq_id[0] = nq->ring_id; } nql_ptr++; } if (primary_found) bnxt_qplib_dbr_pacing_set_primary_pf(rdev->chip_ctx, 1); return 0; } static void __wait_for_fifo_occupancy_below_th(struct bnxt_re_dev *rdev) { struct bnxt_qplib_db_pacing_data *pacing_data = rdev->qplib_res.pacing_data; u32 read_val, fifo_occup; bool first_read = true; /* loop shouldn't run infintely as the occupancy usually goes * below pacing algo threshold as soon as pacing kicks in. */ while (1) { read_val = readl_fbsd(rdev->en_dev->softc, rdev->dbr_db_fifo_reg_off, 0); fifo_occup = pacing_data->fifo_max_depth - ((read_val & pacing_data->fifo_room_mask) >> pacing_data->fifo_room_shift); /* Fifo occupancy cannot be greater the MAX FIFO depth */ if (fifo_occup > pacing_data->fifo_max_depth) break; if (first_read) { bnxt_re_update_fifo_occup_slabs(rdev, fifo_occup); first_read = false; } if (fifo_occup < pacing_data->pacing_th) break; } } static void bnxt_re_set_default_pacing_data(struct bnxt_re_dev *rdev) { struct bnxt_qplib_db_pacing_data *pacing_data = rdev->qplib_res.pacing_data; pacing_data->do_pacing = rdev->dbr_def_do_pacing; pacing_data->pacing_th = rdev->pacing_algo_th; pacing_data->alarm_th = pacing_data->pacing_th * BNXT_RE_PACING_ALARM_TH_MULTIPLE(rdev->chip_ctx); } #define CAG_RING_MASK 0x7FF #define CAG_RING_SHIFT 17 #define WATERMARK_MASK 0xFFF #define WATERMARK_SHIFT 0 static bool bnxt_re_check_if_dbq_intr_triggered(struct bnxt_re_dev *rdev) { u32 read_val; int j; for (j = 0; j < 10; j++) { read_val = readl_fbsd(rdev->en_dev->softc, rdev->dbr_aeq_arm_reg_off, 0); dev_dbg(rdev_to_dev(rdev), "AEQ ARM status = 0x%x\n", read_val); if (!read_val) return true; } return false; } int bnxt_re_set_dbq_throttling_reg(struct bnxt_re_dev *rdev, u16 nq_id, u32 throttle) { u32 cag_ring_water_mark = 0, read_val; u32 throttle_val; /* Convert throttle percentage to value */ throttle_val = (rdev->qplib_res.pacing_data->fifo_max_depth * throttle) / 100; if (bnxt_qplib_dbr_pacing_ext_en(rdev->chip_ctx)) { cag_ring_water_mark = (nq_id & CAG_RING_MASK) << CAG_RING_SHIFT | (throttle_val & WATERMARK_MASK); writel_fbsd(rdev->en_dev->softc, rdev->dbr_throttling_reg_off, 0, cag_ring_water_mark); read_val = readl_fbsd(rdev->en_dev->softc , rdev->dbr_throttling_reg_off, 0); dev_dbg(rdev_to_dev(rdev), "%s: dbr_throttling_reg_off read_val = 0x%x\n", __func__, read_val); if (read_val != cag_ring_water_mark) { dev_dbg(rdev_to_dev(rdev), "nq_id = %d write_val=0x%x read_val=0x%x\n", nq_id, cag_ring_water_mark, read_val); return 1; } } writel_fbsd(rdev->en_dev->softc, rdev->dbr_aeq_arm_reg_off, 0, 1); return 0; } static void bnxt_re_set_dbq_throttling_for_non_primary(struct bnxt_re_dev *rdev) { struct bnxt_dbq_nq_list *nq_list; struct bnxt_qplib_nq *nq; int i; nq_list = &rdev->nq_list; /* Run a loop for other Active functions if this is primary function */ if (bnxt_qplib_dbr_pacing_is_primary_pf(rdev->chip_ctx)) { dev_dbg(rdev_to_dev(rdev), "%s: nq_list->num_nql_entries= %d\n", __func__, nq_list->num_nql_entries); nq = &rdev->nqr.nq[0]; for (i = nq_list->num_nql_entries - 1; i > 0; i--) { u16 nq_id = nq_list->nq_id[i]; if (nq) dev_dbg(rdev_to_dev(rdev), "%s: nq_id = %d cur_fn_ring_id = %d\n", __func__, nq_id, nq->ring_id); if (bnxt_re_set_dbq_throttling_reg (rdev, nq_id, 0)) break; bnxt_re_check_if_dbq_intr_triggered(rdev); } } } static void bnxt_re_handle_dbr_nq_pacing_notification(struct bnxt_re_dev *rdev) { struct bnxt_qplib_nq *nq; int rc = 0; nq = &rdev->nqr.nq[0]; /* Query the NQ list*/ rc = bnxt_re_hwrm_dbr_pacing_nqlist_query(rdev); if (rc) { dev_err(rdev_to_dev(rdev), "Failed to Query NQ list rc= %d", rc); return; } /*Configure GRC access for Throttling and aeq_arm register */ writel_fbsd(rdev->en_dev->softc, BNXT_GRCPF_REG_WINDOW_BASE_OUT + 28, 0, rdev->chip_ctx->dbr_aeq_arm_reg & BNXT_GRC_BASE_MASK); rdev->dbr_throttling_reg_off = (rdev->chip_ctx->dbr_throttling_reg & BNXT_GRC_OFFSET_MASK) + 0x8000; rdev->dbr_aeq_arm_reg_off = (rdev->chip_ctx->dbr_aeq_arm_reg & BNXT_GRC_OFFSET_MASK) + 0x8000; bnxt_re_set_dbq_throttling_reg(rdev, nq->ring_id, rdev->dbq_watermark); } static void bnxt_re_dbq_wq_task(struct work_struct *work) { struct bnxt_re_dbq_work *dbq_work = container_of(work, struct bnxt_re_dbq_work, work); struct bnxt_re_dev *rdev; rdev = dbq_work->rdev; if (!rdev) goto exit; switch (dbq_work->event) { case BNXT_RE_DBQ_EVENT_SCHED: dev_dbg(rdev_to_dev(rdev), "%s: Handle DBQ Pacing event\n", __func__); if (!bnxt_qplib_dbr_pacing_ext_en(rdev->chip_ctx)) bnxt_re_hwrm_dbr_pacing_broadcast_event(rdev); else bnxt_re_pacing_alert(rdev); break; case BNXT_RE_DBR_PACING_EVENT: dev_dbg(rdev_to_dev(rdev), "%s: Sched interrupt/pacing worker\n", __func__); if (_is_chip_p7(rdev->chip_ctx)) bnxt_re_pacing_alert(rdev); else if (!rdev->chip_ctx->modes.dbr_pacing_v0) bnxt_re_hwrm_dbr_pacing_qcfg(rdev); break; case BNXT_RE_DBR_NQ_PACING_NOTIFICATION: bnxt_re_handle_dbr_nq_pacing_notification(rdev); /* Issue a broadcast event to notify other functions * that primary changed */ bnxt_re_hwrm_dbr_pacing_broadcast_event(rdev); break; } exit: kfree(dbq_work); } static void bnxt_re_async_notifier(void *handle, struct hwrm_async_event_cmpl *cmpl) { struct bnxt_re_en_dev_info *en_info = auxiliary_get_drvdata(handle); struct bnxt_re_dcb_work *dcb_work; struct bnxt_re_dbq_work *dbq_work; struct bnxt_re_dev *rdev; u16 event_id; u32 data1; u32 data2 = 0; if (!cmpl) { pr_err("Async event, bad completion\n"); return; } if (!en_info || !en_info->en_dev) { pr_err("Async event, bad en_info or en_dev\n"); return; } rdev = en_info->rdev; event_id = le16_to_cpu(cmpl->event_id); data1 = le32_to_cpu(cmpl->event_data1); data2 = le32_to_cpu(cmpl->event_data2); if (!rdev || !rdev_to_dev(rdev)) { dev_dbg(NULL, "Async event, bad rdev or netdev\n"); return; } if (test_bit(BNXT_RE_FLAG_ERR_DEVICE_DETACHED, &rdev->flags) || !test_bit(BNXT_RE_FLAG_NETDEV_REGISTERED, &rdev->flags)) { dev_dbg(NULL, "Async event, device already detached\n"); return; } if (data2 >= 0) dev_dbg(rdev_to_dev(rdev), "Async event_id = %d data1 = %d data2 = %d", event_id, data1, data2); switch (event_id) { case HWRM_ASYNC_EVENT_CMPL_EVENT_ID_DCB_CONFIG_CHANGE: /* Not handling the event in older FWs */ if (!is_qport_service_type_supported(rdev)) break; if (!rdev->dcb_wq) break; dcb_work = kzalloc(sizeof(*dcb_work), GFP_ATOMIC); if (!dcb_work) break; dcb_work->rdev = rdev; memcpy(&dcb_work->cmpl, cmpl, sizeof(*cmpl)); INIT_WORK(&dcb_work->work, bnxt_re_dcb_wq_task); queue_work(rdev->dcb_wq, &dcb_work->work); break; case HWRM_ASYNC_EVENT_CMPL_EVENT_ID_RESET_NOTIFY: if (EVENT_DATA1_RESET_NOTIFY_FATAL(data1)) { /* Set rcfw flag to control commands send to Bono */ set_bit(ERR_DEVICE_DETACHED, &rdev->rcfw.cmdq.flags); /* Set bnxt_re flag to control commands send via L2 driver */ set_bit(BNXT_RE_FLAG_ERR_DEVICE_DETACHED, &rdev->flags); wake_up_all(&rdev->rcfw.cmdq.waitq); } break; case HWRM_ASYNC_EVENT_CMPL_EVENT_ID_DOORBELL_PACING_THRESHOLD: if (!rdev->dbr_pacing) break; dbq_work = kzalloc(sizeof(*dbq_work), GFP_ATOMIC); if (!dbq_work) goto unlock; dbq_work->rdev = rdev; dbq_work->event = BNXT_RE_DBR_PACING_EVENT; INIT_WORK(&dbq_work->work, bnxt_re_dbq_wq_task); queue_work(rdev->dbq_wq, &dbq_work->work); rdev->dbr_sw_stats->dbq_int_recv++; break; case HWRM_ASYNC_EVENT_CMPL_EVENT_ID_DOORBELL_PACING_NQ_UPDATE: if (!rdev->dbr_pacing) break; dbq_work = kzalloc(sizeof(*dbq_work), GFP_ATOMIC); if (!dbq_work) goto unlock; dbq_work->rdev = rdev; dbq_work->event = BNXT_RE_DBR_NQ_PACING_NOTIFICATION; INIT_WORK(&dbq_work->work, bnxt_re_dbq_wq_task); queue_work(rdev->dbq_wq, &dbq_work->work); break; default: break; } unlock: return; } static void bnxt_re_db_fifo_check(struct work_struct *work) { struct bnxt_re_dev *rdev = container_of(work, struct bnxt_re_dev, dbq_fifo_check_work); struct bnxt_qplib_db_pacing_data *pacing_data; u32 pacing_save; if (!mutex_trylock(&rdev->dbq_lock)) return; pacing_data = rdev->qplib_res.pacing_data; pacing_save = rdev->do_pacing_save; __wait_for_fifo_occupancy_below_th(rdev); cancel_delayed_work_sync(&rdev->dbq_pacing_work); if (rdev->dbr_recovery_on) goto recovery_on; if (pacing_save > rdev->dbr_def_do_pacing) { /* Double the do_pacing value during the congestion */ pacing_save = pacing_save << 1; } else { /* * when a new congestion is detected increase the do_pacing * by 8 times. And also increase the pacing_th by 4 times. The * reason to increase pacing_th is to give more space for the * queue to oscillate down without getting empty, but also more * room for the queue to increase without causing another alarm. */ pacing_save = pacing_save << 3; pacing_data->pacing_th = rdev->pacing_algo_th * 4; } if (pacing_save > BNXT_RE_MAX_DBR_DO_PACING) pacing_save = BNXT_RE_MAX_DBR_DO_PACING; pacing_data->do_pacing = pacing_save; rdev->do_pacing_save = pacing_data->do_pacing; pacing_data->alarm_th = pacing_data->pacing_th * BNXT_RE_PACING_ALARM_TH_MULTIPLE(rdev->chip_ctx); recovery_on: schedule_delayed_work(&rdev->dbq_pacing_work, msecs_to_jiffies(rdev->dbq_pacing_time)); rdev->dbr_sw_stats->dbq_pacing_alerts++; mutex_unlock(&rdev->dbq_lock); } static void bnxt_re_pacing_timer_exp(struct work_struct *work) { struct bnxt_re_dev *rdev = container_of(work, struct bnxt_re_dev, dbq_pacing_work.work); struct bnxt_qplib_db_pacing_data *pacing_data; u32 read_val, fifo_occup; struct bnxt_qplib_nq *nq; if (!mutex_trylock(&rdev->dbq_lock)) return; pacing_data = rdev->qplib_res.pacing_data; read_val = readl_fbsd(rdev->en_dev->softc , rdev->dbr_db_fifo_reg_off, 0); fifo_occup = pacing_data->fifo_max_depth - ((read_val & pacing_data->fifo_room_mask) >> pacing_data->fifo_room_shift); if (fifo_occup > pacing_data->pacing_th) goto restart_timer; /* * Instead of immediately going back to the default do_pacing * reduce it by 1/8 times and restart the timer. */ pacing_data->do_pacing = pacing_data->do_pacing - (pacing_data->do_pacing >> 3); pacing_data->do_pacing = max_t(u32, rdev->dbr_def_do_pacing, pacing_data->do_pacing); /* * If the fifo_occup is less than the interrupt enable threshold * enable the interrupt on the primary PF. */ if (rdev->dbq_int_disable && fifo_occup < rdev->pacing_en_int_th) { if (bnxt_qplib_dbr_pacing_is_primary_pf(rdev->chip_ctx)) { if (!rdev->chip_ctx->modes.dbr_pacing_v0) { nq = &rdev->nqr.nq[0]; bnxt_re_set_dbq_throttling_reg(rdev, nq->ring_id, rdev->dbq_watermark); rdev->dbr_sw_stats->dbq_int_en++; rdev->dbq_int_disable = false; } } } if (pacing_data->do_pacing <= rdev->dbr_def_do_pacing) { bnxt_re_set_default_pacing_data(rdev); rdev->dbr_sw_stats->dbq_pacing_complete++; goto dbq_unlock; } restart_timer: schedule_delayed_work(&rdev->dbq_pacing_work, msecs_to_jiffies(rdev->dbq_pacing_time)); bnxt_re_update_do_pacing_slabs(rdev); rdev->dbr_sw_stats->dbq_pacing_resched++; dbq_unlock: rdev->do_pacing_save = pacing_data->do_pacing; mutex_unlock(&rdev->dbq_lock); } void bnxt_re_pacing_alert(struct bnxt_re_dev *rdev) { struct bnxt_qplib_db_pacing_data *pacing_data; if (!rdev->dbr_pacing) return; mutex_lock(&rdev->dbq_lock); pacing_data = rdev->qplib_res.pacing_data; /* * Increase the alarm_th to max so that other user lib instances do not * keep alerting the driver. */ pacing_data->alarm_th = pacing_data->fifo_max_depth; pacing_data->do_pacing = BNXT_RE_MAX_DBR_DO_PACING; cancel_work_sync(&rdev->dbq_fifo_check_work); schedule_work(&rdev->dbq_fifo_check_work); mutex_unlock(&rdev->dbq_lock); } void bnxt_re_schedule_dbq_event(struct bnxt_qplib_res *res) { struct bnxt_re_dbq_work *dbq_work; struct bnxt_re_dev *rdev; rdev = container_of(res, struct bnxt_re_dev, qplib_res); atomic_set(&rdev->dbq_intr_running, 1); if (test_bit(BNXT_RE_FLAG_ERR_DEVICE_DETACHED, &rdev->flags)) goto exit; /* Run the loop to send dbq event to other functions * for newer FW */ if (bnxt_qplib_dbr_pacing_ext_en(rdev->chip_ctx) && !rdev->chip_ctx->modes.dbr_pacing_v0) bnxt_re_set_dbq_throttling_for_non_primary(rdev); dbq_work = kzalloc(sizeof(*dbq_work), GFP_ATOMIC); if (!dbq_work) goto exit; dbq_work->rdev = rdev; dbq_work->event = BNXT_RE_DBQ_EVENT_SCHED; INIT_WORK(&dbq_work->work, bnxt_re_dbq_wq_task); queue_work(rdev->dbq_wq, &dbq_work->work); rdev->dbr_sw_stats->dbq_int_recv++; rdev->dbq_int_disable = true; exit: atomic_set(&rdev->dbq_intr_running, 0); } static void bnxt_re_free_msix(struct bnxt_re_dev *rdev) { struct bnxt_en_dev *en_dev = rdev->en_dev; int rc; rc = en_dev->en_ops->bnxt_free_msix(rdev->en_dev, BNXT_ROCE_ULP); if (rc) dev_err(rdev_to_dev(rdev), "netdev %p free_msix failed! rc = 0x%x", rdev->netdev, rc); } static int bnxt_re_request_msix(struct bnxt_re_dev *rdev) { struct bnxt_en_dev *en_dev = rdev->en_dev; int rc = 0, num_msix_want, num_msix_got; struct bnxt_msix_entry *entry; /* * Request MSIx based on the function type. This is * a temporory solution to enable max VFs when NPAR is * enabled. * TODO - change the scheme with an adapter specific check * as the latest adapters can support more NQs. For now * this change satisfy all adapter versions. */ if (rdev->is_virtfn) num_msix_want = BNXT_RE_MAX_MSIX_VF; else if (BNXT_EN_NPAR(en_dev)) num_msix_want = BNXT_RE_MAX_MSIX_NPAR_PF; else if (_is_chip_gen_p5_p7(rdev->chip_ctx)) num_msix_want = rdev->num_msix_requested ?: BNXT_RE_MAX_MSIX_GEN_P5_PF; else num_msix_want = BNXT_RE_MAX_MSIX_PF; /* * Since MSIX vectors are used for both NQs and CREQ, we should try to * allocate num_online_cpus + 1 by taking into account the CREQ. This * leaves the number of MSIX vectors for NQs match the number of CPUs * and allows the system to be fully utilized */ num_msix_want = min_t(u32, num_msix_want, num_online_cpus() + 1); num_msix_want = min_t(u32, num_msix_want, BNXT_RE_MAX_MSIX); num_msix_want = max_t(u32, num_msix_want, BNXT_RE_MIN_MSIX); entry = rdev->nqr.msix_entries; num_msix_got = en_dev->en_ops->bnxt_request_msix(en_dev, BNXT_ROCE_ULP, entry, num_msix_want); if (num_msix_got < BNXT_RE_MIN_MSIX) { rc = -EINVAL; goto done; } if (num_msix_got != num_msix_want) dev_warn(rdev_to_dev(rdev), "bnxt_request_msix: wanted %d vectors, got %d\n", num_msix_want, num_msix_got); rdev->nqr.num_msix = num_msix_got; return 0; done: if (num_msix_got) bnxt_re_free_msix(rdev); return rc; } static int __wait_for_ib_unregister(struct bnxt_re_dev *rdev, struct bnxt_re_en_dev_info *en_info) { u64 timeout = 0; u32 cur_prod = 0, cur_cons = 0; int retry = 0, rc = 0, ret = 0; cur_prod = rdev->rcfw.cmdq.hwq.prod; cur_cons = rdev->rcfw.cmdq.hwq.cons; timeout = msecs_to_jiffies(BNXT_RE_RECOVERY_IB_UNINIT_WAIT_TIME_MS); retry = BNXT_RE_RECOVERY_IB_UNINIT_WAIT_RETRY; /* During module exit, increase timeout ten-fold to 100 mins to wait * as long as possible for ib_unregister() to complete */ if (rdev->mod_exit) retry *= 10; do { /* * Since the caller of this function invokes with bnxt_re_mutex held, * release it to avoid holding a lock while in wait / sleep mode. */ mutex_unlock(&bnxt_re_mutex); rc = wait_event_timeout(en_info->waitq, en_info->ib_uninit_done, timeout); mutex_lock(&bnxt_re_mutex); if (!bnxt_re_is_rdev_valid(rdev)) break; if (rc) break; if (!RCFW_NO_FW_ACCESS(&rdev->rcfw)) { /* No need to check for cmdq stall during module exit, * wait for ib unregister to complete */ if (!rdev->mod_exit) ret = __check_cmdq_stall(&rdev->rcfw, &cur_prod, &cur_cons); if (ret || en_info->ib_uninit_done) break; } } while (retry--); return rc; } static int bnxt_re_handle_start(struct auxiliary_device *adev) { struct bnxt_re_en_dev_info *en_info = auxiliary_get_drvdata(adev); struct bnxt_re_dev *rdev = NULL; struct ifnet *real_dev; struct bnxt_en_dev *en_dev; struct ifnet *netdev; int rc = 0; if (!en_info || !en_info->en_dev) { pr_err("Start, bad en_info or en_dev\n"); return -EINVAL; } netdev = en_info->en_dev->net; if (en_info->rdev) { dev_info(rdev_to_dev(en_info->rdev), "%s: Device is already added adev %p rdev: %p\n", __func__, adev, en_info->rdev); return 0; } en_dev = en_info->en_dev; real_dev = rdma_vlan_dev_real_dev(netdev); if (!real_dev) real_dev = netdev; rc = bnxt_re_add_device(&rdev, real_dev, en_info->gsi_mode, BNXT_RE_POST_RECOVERY_INIT, en_info->wqe_mode, en_info->num_msix_requested, adev); if (rc) { /* Add device failed. Unregister the device. * This has to be done explicitly as * bnxt_re_stop would not have unregistered */ rtnl_lock(); en_dev->en_ops->bnxt_unregister_device(en_dev, BNXT_ROCE_ULP); rtnl_unlock(); mutex_lock(&bnxt_re_dev_lock); gadd_dev_inprogress--; mutex_unlock(&bnxt_re_dev_lock); return rc; } rdev->adev = adev; rtnl_lock(); bnxt_re_get_link_speed(rdev); rtnl_unlock(); rc = bnxt_re_ib_init(rdev); if (rc) { dev_err(rdev_to_dev(rdev), "Failed ib_init\n"); return rc; } bnxt_re_ib_init_2(rdev); return rc; } static void bnxt_re_stop(void *handle) { struct bnxt_re_en_dev_info *en_info = auxiliary_get_drvdata(handle); struct ifnet *netdev; struct bnxt_re_dev *rdev; struct bnxt_en_dev *en_dev; int rc = 0; rtnl_unlock(); mutex_lock(&bnxt_re_mutex); if (!en_info || !en_info->en_dev) { pr_err("Stop, bad en_info or en_dev\n"); goto exit; } netdev = en_info->en_dev->net; rdev = en_info->rdev; if (!rdev) goto exit; if (!bnxt_re_is_rdev_valid(rdev)) goto exit; /* * Check if fw has undergone reset or is in a fatal condition. * If so, set flags so that no further commands are sent down to FW */ en_dev = rdev->en_dev; if (en_dev->en_state & BNXT_STATE_FW_FATAL_COND || en_dev->en_state & BNXT_STATE_FW_RESET_DET) { /* Set rcfw flag to control commands send to Bono */ set_bit(ERR_DEVICE_DETACHED, &rdev->rcfw.cmdq.flags); /* Set bnxt_re flag to control commands send via L2 driver */ set_bit(BNXT_RE_FLAG_ERR_DEVICE_DETACHED, &rdev->flags); wake_up_all(&rdev->rcfw.cmdq.waitq); } if (test_bit(BNXT_RE_FLAG_STOP_IN_PROGRESS, &rdev->flags)) goto exit; set_bit(BNXT_RE_FLAG_STOP_IN_PROGRESS, &rdev->flags); en_info->wqe_mode = rdev->chip_ctx->modes.wqe_mode; en_info->gsi_mode = rdev->gsi_ctx.gsi_qp_mode; en_info->num_msix_requested = rdev->num_msix_requested; en_info->ib_uninit_done = false; if (rdev->dbr_pacing) bnxt_re_set_pacing_dev_state(rdev); dev_info(rdev_to_dev(rdev), "%s: L2 driver notified to stop." "Attempting to stop and Dispatching event " "to inform the stack\n", __func__); init_waitqueue_head(&en_info->waitq); /* Schedule a work item to handle IB UNINIT for recovery */ bnxt_re_schedule_work(rdev, NETDEV_UNREGISTER, NULL, netdev, rdev->adev); rc = __wait_for_ib_unregister(rdev, en_info); if (!bnxt_re_is_rdev_valid(rdev)) goto exit; if (!rc) { dev_info(rdev_to_dev(rdev), "%s: Attempt to stop failed\n", __func__); bnxt_re_detach_err_device(rdev); goto exit; } bnxt_re_remove_device(rdev, BNXT_RE_PRE_RECOVERY_REMOVE, rdev->adev); exit: mutex_unlock(&bnxt_re_mutex); /* Take rtnl_lock before return, bnxt_re_stop is called with rtnl_lock */ rtnl_lock(); return; } static void bnxt_re_start(void *handle) { rtnl_unlock(); mutex_lock(&bnxt_re_mutex); if (bnxt_re_handle_start((struct auxiliary_device *)handle)) pr_err("Failed to start RoCE device"); mutex_unlock(&bnxt_re_mutex); /* Take rtnl_lock before return, bnxt_re_start is called with rtnl_lock */ rtnl_lock(); return; } static void bnxt_re_shutdown(void *p) { struct bnxt_re_en_dev_info *en_info = auxiliary_get_drvdata(p); struct bnxt_re_dev *rdev; if (!en_info) { pr_err("Shutdown, bad en_info\n"); return; } rtnl_unlock(); mutex_lock(&bnxt_re_mutex); rdev = en_info->rdev; if (!rdev || !bnxt_re_is_rdev_valid(rdev)) goto exit; /* rtnl_lock held by L2 before coming here */ bnxt_re_stopqps_and_ib_uninit(rdev); bnxt_re_remove_device(rdev, BNXT_RE_COMPLETE_REMOVE, rdev->adev); exit: mutex_unlock(&bnxt_re_mutex); rtnl_lock(); return; } static void bnxt_re_stop_irq(void *handle) { struct bnxt_re_en_dev_info *en_info = auxiliary_get_drvdata(handle); struct bnxt_qplib_rcfw *rcfw = NULL; struct bnxt_re_dev *rdev; struct bnxt_qplib_nq *nq; int indx; if (!en_info) { pr_err("Stop irq, bad en_info\n"); return; } rdev = en_info->rdev; if (!rdev) return; rcfw = &rdev->rcfw; for (indx = 0; indx < rdev->nqr.max_init; indx++) { nq = &rdev->nqr.nq[indx]; mutex_lock(&nq->lock); bnxt_qplib_nq_stop_irq(nq, false); mutex_unlock(&nq->lock); } if (test_bit(BNXT_RE_FLAG_ALLOC_RCFW, &rdev->flags)) bnxt_qplib_rcfw_stop_irq(rcfw, false); } static void bnxt_re_start_irq(void *handle, struct bnxt_msix_entry *ent) { struct bnxt_re_en_dev_info *en_info = auxiliary_get_drvdata(handle); struct bnxt_msix_entry *msix_ent = NULL; struct bnxt_qplib_rcfw *rcfw = NULL; struct bnxt_re_dev *rdev; struct bnxt_qplib_nq *nq; int indx, rc, vec; if (!en_info) { pr_err("Start irq, bad en_info\n"); return; } rdev = en_info->rdev; if (!rdev) return; if (test_bit(BNXT_RE_FLAG_ERR_DEVICE_DETACHED, &rdev->flags)) return; msix_ent = rdev->nqr.msix_entries; rcfw = &rdev->rcfw; if (!ent) { /* Not setting the f/w timeout bit in rcfw. * During the driver unload the first command * to f/w will timeout and that will set the * timeout bit. */ dev_err(rdev_to_dev(rdev), "Failed to re-start IRQs\n"); return; } /* Vectors may change after restart, so update with new vectors * in device structure. */ for (indx = 0; indx < rdev->nqr.num_msix; indx++) rdev->nqr.msix_entries[indx].vector = ent[indx].vector; if (test_bit(BNXT_RE_FLAG_ALLOC_RCFW, &rdev->flags)) { rc = bnxt_qplib_rcfw_start_irq(rcfw, msix_ent[BNXT_RE_AEQ_IDX].vector, false); if (rc) { dev_warn(rdev_to_dev(rdev), "Failed to reinit CREQ\n"); return; } } for (indx = 0 ; indx < rdev->nqr.max_init; indx++) { nq = &rdev->nqr.nq[indx]; vec = indx + 1; rc = bnxt_qplib_nq_start_irq(nq, indx, msix_ent[vec].vector, false); if (rc) { dev_warn(rdev_to_dev(rdev), "Failed to reinit NQ index %d\n", indx); return; } } } /* * Except for ulp_async_notifier, the remaining ulp_ops * below are called with rtnl_lock held */ static struct bnxt_ulp_ops bnxt_re_ulp_ops = { .ulp_async_notifier = bnxt_re_async_notifier, .ulp_stop = bnxt_re_stop, .ulp_start = bnxt_re_start, .ulp_shutdown = bnxt_re_shutdown, .ulp_irq_stop = bnxt_re_stop_irq, .ulp_irq_restart = bnxt_re_start_irq, }; static inline const char *bnxt_re_netevent(unsigned long event) { BNXT_RE_NETDEV_EVENT(event, NETDEV_UP); BNXT_RE_NETDEV_EVENT(event, NETDEV_DOWN); BNXT_RE_NETDEV_EVENT(event, NETDEV_CHANGE); BNXT_RE_NETDEV_EVENT(event, NETDEV_REGISTER); BNXT_RE_NETDEV_EVENT(event, NETDEV_UNREGISTER); BNXT_RE_NETDEV_EVENT(event, NETDEV_CHANGEADDR); return "Unknown"; } /* RoCE -> Net driver */ /* Driver registration routines used to let the networking driver (bnxt_en) * to know that the RoCE driver is now installed */ static void bnxt_re_unregister_netdev(struct bnxt_re_dev *rdev) { struct bnxt_en_dev *en_dev = rdev->en_dev; int rc; rtnl_lock(); rc = en_dev->en_ops->bnxt_unregister_device(rdev->en_dev, BNXT_ROCE_ULP); rtnl_unlock(); if (rc) dev_err(rdev_to_dev(rdev), "netdev %p unregister failed! rc = 0x%x", rdev->en_dev->net, rc); clear_bit(BNXT_RE_FLAG_NETDEV_REGISTERED, &rdev->flags); } static int bnxt_re_register_netdev(struct bnxt_re_dev *rdev) { struct bnxt_en_dev *en_dev = rdev->en_dev; int rc = 0; rtnl_lock(); rc = en_dev->en_ops->bnxt_register_device(en_dev, BNXT_ROCE_ULP, &bnxt_re_ulp_ops, rdev->adev); rtnl_unlock(); if (rc) { dev_err(rdev_to_dev(rdev), "netdev %p register failed! rc = 0x%x", rdev->netdev, rc); return rc; } return rc; } static void bnxt_re_set_db_offset(struct bnxt_re_dev *rdev) { struct bnxt_qplib_chip_ctx *cctx; struct bnxt_en_dev *en_dev; struct bnxt_qplib_res *res; u32 l2db_len = 0; u32 offset = 0; u32 barlen; int rc; res = &rdev->qplib_res; en_dev = rdev->en_dev; cctx = rdev->chip_ctx; /* Issue qcfg */ rc = bnxt_re_hwrm_qcfg(rdev, &l2db_len, &offset); if (rc) dev_info(rdev_to_dev(rdev), "Couldn't get DB bar size, Low latency framework is disabled\n"); /* set register offsets for both UC and WC */ if (_is_chip_p7(cctx)) { res->dpi_tbl.ucreg.offset = en_dev->l2_db_offset; res->dpi_tbl.wcreg.offset = en_dev->l2_db_size; } else { res->dpi_tbl.ucreg.offset = res->is_vf ? BNXT_QPLIB_DBR_VF_DB_OFFSET : BNXT_QPLIB_DBR_PF_DB_OFFSET; res->dpi_tbl.wcreg.offset = res->dpi_tbl.ucreg.offset; } /* If WC mapping is disabled by L2 driver then en_dev->l2_db_size * is equal to the DB-Bar actual size. This indicates that L2 * is mapping entire bar as UC-. RoCE driver can't enable WC mapping * in such cases and DB-push will be disabled. */ barlen = pci_resource_len(res->pdev, RCFW_DBR_PCI_BAR_REGION); if (cctx->modes.db_push && l2db_len && en_dev->l2_db_size != barlen) { res->dpi_tbl.wcreg.offset = en_dev->l2_db_size; dev_info(rdev_to_dev(rdev), "Low latency framework is enabled\n"); } return; } static void bnxt_re_set_drv_mode(struct bnxt_re_dev *rdev, u8 mode) { struct bnxt_qplib_chip_ctx *cctx; struct bnxt_en_dev *en_dev; en_dev = rdev->en_dev; cctx = rdev->chip_ctx; cctx->modes.wqe_mode = _is_chip_gen_p5_p7(rdev->chip_ctx) ? mode : BNXT_QPLIB_WQE_MODE_STATIC; cctx->modes.te_bypass = false; if (bnxt_re_hwrm_qcaps(rdev)) dev_err(rdev_to_dev(rdev), "Failed to query hwrm qcaps\n"); /* * TODO: Need a better mechanism for spreading of the * 512 extended PPP pages in the presence of VF and * NPAR, until then not enabling push */ if (_is_chip_p7(rdev->chip_ctx) && cctx->modes.db_push) { if (rdev->is_virtfn || BNXT_EN_NPAR(en_dev)) cctx->modes.db_push = false; } rdev->roce_mode = en_dev->flags & BNXT_EN_FLAG_ROCE_CAP; dev_dbg(rdev_to_dev(rdev), "RoCE is supported on the device - caps:0x%x", rdev->roce_mode); if (!_is_chip_gen_p5_p7(rdev->chip_ctx)) rdev->roce_mode = BNXT_RE_FLAG_ROCEV2_CAP; cctx->hw_stats_size = en_dev->hw_ring_stats_size; } static void bnxt_re_destroy_chip_ctx(struct bnxt_re_dev *rdev) { struct bnxt_qplib_chip_ctx *chip_ctx; struct bnxt_qplib_res *res; if (!rdev->chip_ctx) return; res = &rdev->qplib_res; bnxt_qplib_unmap_db_bar(res); kfree(res->hctx); res->rcfw = NULL; kfree(rdev->dev_attr); rdev->dev_attr = NULL; chip_ctx = rdev->chip_ctx; rdev->chip_ctx = NULL; res->cctx = NULL; res->hctx = NULL; res->pdev = NULL; res->netdev = NULL; kfree(chip_ctx); } static int bnxt_re_setup_chip_ctx(struct bnxt_re_dev *rdev, u8 wqe_mode) { struct bnxt_qplib_chip_ctx *chip_ctx; struct bnxt_en_dev *en_dev; int rc; en_dev = rdev->en_dev; /* Supply pci device to qplib */ rdev->qplib_res.pdev = en_dev->pdev; rdev->qplib_res.netdev = rdev->netdev; rdev->qplib_res.en_dev = en_dev; chip_ctx = kzalloc(sizeof(*chip_ctx), GFP_KERNEL); if (!chip_ctx) return -ENOMEM; rdev->chip_ctx = chip_ctx; rdev->qplib_res.cctx = chip_ctx; rc = bnxt_re_query_hwrm_intf_version(rdev); if (rc) goto fail; rdev->dev_attr = kzalloc(sizeof(*rdev->dev_attr), GFP_KERNEL); if (!rdev->dev_attr) { rc = -ENOMEM; goto fail; } rdev->qplib_res.dattr = rdev->dev_attr; rdev->qplib_res.rcfw = &rdev->rcfw; rdev->qplib_res.is_vf = rdev->is_virtfn; rdev->qplib_res.hctx = kzalloc(sizeof(*rdev->qplib_res.hctx), GFP_KERNEL); if (!rdev->qplib_res.hctx) { rc = -ENOMEM; goto fail; } bnxt_re_set_drv_mode(rdev, wqe_mode); bnxt_re_set_db_offset(rdev); rc = bnxt_qplib_map_db_bar(&rdev->qplib_res); if (rc) goto fail; rc = bnxt_qplib_enable_atomic_ops_to_root(en_dev->pdev); if (rc) dev_dbg(rdev_to_dev(rdev), "platform doesn't support global atomics"); return 0; fail: kfree(rdev->chip_ctx); rdev->chip_ctx = NULL; kfree(rdev->dev_attr); rdev->dev_attr = NULL; kfree(rdev->qplib_res.hctx); rdev->qplib_res.hctx = NULL; return rc; } static u16 bnxt_re_get_rtype(struct bnxt_re_dev *rdev) { return _is_chip_gen_p5_p7(rdev->chip_ctx) ? HWRM_RING_ALLOC_INPUT_RING_TYPE_NQ : HWRM_RING_ALLOC_INPUT_RING_TYPE_ROCE_CMPL; } static int bnxt_re_net_ring_free(struct bnxt_re_dev *rdev, u16 fw_ring_id) { int rc = -EINVAL; struct hwrm_ring_free_input req = {0}; struct hwrm_ring_free_output resp; struct bnxt_en_dev *en_dev = rdev->en_dev; struct bnxt_fw_msg fw_msg; if (!en_dev) return rc; /* To avoid unnecessary error messages during recovery. * HW is anyway in error state. So dont send down the command */ if (test_bit(BNXT_RE_FLAG_ERR_DEVICE_DETACHED, &rdev->flags)) return 0; /* allocation had failed, no need to issue hwrm */ if (fw_ring_id == 0xffff) return 0; memset(&fw_msg, 0, sizeof(fw_msg)); bnxt_re_init_hwrm_hdr(rdev, (void *)&req, HWRM_RING_FREE, -1, -1); req.ring_type = bnxt_re_get_rtype(rdev); req.ring_id = cpu_to_le16(fw_ring_id); bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp, sizeof(resp), DFLT_HWRM_CMD_TIMEOUT); rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg); if (rc) { dev_err(rdev_to_dev(rdev), "Failed to free HW ring with rc = 0x%x", rc); return rc; } dev_dbg(rdev_to_dev(rdev), "HW ring freed with id = 0x%x\n", fw_ring_id); return rc; } static int bnxt_re_net_ring_alloc(struct bnxt_re_dev *rdev, struct bnxt_re_ring_attr *ring_attr, u16 *fw_ring_id) { int rc = -EINVAL; struct hwrm_ring_alloc_input req = {0}; struct hwrm_ring_alloc_output resp; struct bnxt_en_dev *en_dev = rdev->en_dev; struct bnxt_fw_msg fw_msg; if (!en_dev) return rc; memset(&fw_msg, 0, sizeof(fw_msg)); bnxt_re_init_hwrm_hdr(rdev, (void *)&req, HWRM_RING_ALLOC, -1, -1); req.flags = cpu_to_le16(ring_attr->flags); req.enables = 0; req.page_tbl_addr = cpu_to_le64(ring_attr->dma_arr[0]); if (ring_attr->pages > 1) { /* Page size is in log2 units */ req.page_size = BNXT_PAGE_SHIFT; req.page_tbl_depth = 1; } else { req.page_size = 4; req.page_tbl_depth = 0; } req.fbo = 0; /* Association of ring index with doorbell index and MSIX number */ req.logical_id = cpu_to_le16(ring_attr->lrid); req.length = cpu_to_le32(ring_attr->depth + 1); req.ring_type = ring_attr->type; req.int_mode = ring_attr->mode; bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp, sizeof(resp), DFLT_HWRM_CMD_TIMEOUT); rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg); if (rc) { dev_err(rdev_to_dev(rdev), "Failed to allocate HW ring with rc = 0x%x", rc); return rc; } *fw_ring_id = le16_to_cpu(resp.ring_id); dev_dbg(rdev_to_dev(rdev), "HW ring allocated with id = 0x%x at slot 0x%x", resp.ring_id, ring_attr->lrid); return rc; } static int bnxt_re_net_stats_ctx_free(struct bnxt_re_dev *rdev, u32 fw_stats_ctx_id, u16 tid) { struct bnxt_en_dev *en_dev = rdev->en_dev; struct hwrm_stat_ctx_free_input req = {0}; struct hwrm_stat_ctx_free_output resp; struct bnxt_fw_msg fw_msg; int rc = -EINVAL; if (!en_dev) return rc; /* To avoid unnecessary error messages during recovery. * HW is anyway in error state. So dont send down the command */ if (test_bit(BNXT_RE_FLAG_ERR_DEVICE_DETACHED, &rdev->flags)) return 0; memset(&fw_msg, 0, sizeof(fw_msg)); bnxt_re_init_hwrm_hdr(rdev, (void *)&req, HWRM_STAT_CTX_FREE, -1, tid); req.stat_ctx_id = cpu_to_le32(fw_stats_ctx_id); bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp, sizeof(resp), DFLT_HWRM_CMD_TIMEOUT); rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg); if (rc) { dev_err(rdev_to_dev(rdev), "Failed to free HW stats ctx with rc = 0x%x", rc); return rc; } dev_dbg(rdev_to_dev(rdev), "HW stats ctx freed with id = 0x%x", fw_stats_ctx_id); return rc; } static int bnxt_re_net_stats_ctx_alloc(struct bnxt_re_dev *rdev, u16 tid) { struct hwrm_stat_ctx_alloc_output resp = {}; struct hwrm_stat_ctx_alloc_input req = {}; struct bnxt_en_dev *en_dev = rdev->en_dev; struct bnxt_qplib_stats *stat; struct bnxt_qplib_ctx *hctx; struct bnxt_fw_msg fw_msg; int rc = 0; hctx = rdev->qplib_res.hctx; stat = (tid == 0xffff) ? &hctx->stats : &hctx->stats2; stat->fw_id = INVALID_STATS_CTX_ID; if (!en_dev) return -EINVAL; memset(&fw_msg, 0, sizeof(fw_msg)); bnxt_re_init_hwrm_hdr(rdev, (void *)&req, HWRM_STAT_CTX_ALLOC, -1, tid); req.update_period_ms = cpu_to_le32(1000); req.stats_dma_length = rdev->chip_ctx->hw_stats_size; req.stats_dma_addr = cpu_to_le64(stat->dma_map); req.stat_ctx_flags = HWRM_STAT_CTX_ALLOC_INPUT_STAT_CTX_FLAGS_ROCE; bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp, sizeof(resp), DFLT_HWRM_CMD_TIMEOUT); rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg); if (rc) { dev_err(rdev_to_dev(rdev), "Failed to allocate HW stats ctx, rc = 0x%x", rc); return rc; } stat->fw_id = le32_to_cpu(resp.stat_ctx_id); dev_dbg(rdev_to_dev(rdev), "HW stats ctx allocated with id = 0x%x", stat->fw_id); return rc; } static void bnxt_re_net_unregister_async_event(struct bnxt_re_dev *rdev) { const struct bnxt_en_ops *en_ops; if (rdev->is_virtfn || test_bit(BNXT_RE_FLAG_ERR_DEVICE_DETACHED, &rdev->flags)) return; memset(rdev->event_bitmap, 0, sizeof(rdev->event_bitmap)); en_ops = rdev->en_dev->en_ops; if (en_ops->bnxt_register_fw_async_events (rdev->en_dev, BNXT_ROCE_ULP, (unsigned long *)rdev->event_bitmap, HWRM_ASYNC_EVENT_CMPL_EVENT_ID_DOORBELL_PACING_NQ_UPDATE)) dev_err(rdev_to_dev(rdev), "Failed to unregister async event"); } static void bnxt_re_net_register_async_event(struct bnxt_re_dev *rdev) { const struct bnxt_en_ops *en_ops; if (rdev->is_virtfn) return; rdev->event_bitmap[0] |= BIT(HWRM_ASYNC_EVENT_CMPL_EVENT_ID_DCB_CONFIG_CHANGE) | BIT(HWRM_ASYNC_EVENT_CMPL_EVENT_ID_RESET_NOTIFY); rdev->event_bitmap[2] |= BIT(HWRM_ASYNC_EVENT_CMPL_EVENT_ID_ERROR_REPORT - 64); rdev->event_bitmap[2] |= BIT(HWRM_ASYNC_EVENT_CMPL_EVENT_ID_DOORBELL_PACING_THRESHOLD - 64) | BIT(HWRM_ASYNC_EVENT_CMPL_EVENT_ID_DOORBELL_PACING_NQ_UPDATE - 64); en_ops = rdev->en_dev->en_ops; if (en_ops->bnxt_register_fw_async_events (rdev->en_dev, BNXT_ROCE_ULP, (unsigned long *)rdev->event_bitmap, HWRM_ASYNC_EVENT_CMPL_EVENT_ID_DOORBELL_PACING_NQ_UPDATE)) dev_err(rdev_to_dev(rdev), "Failed to reg Async event"); } static int bnxt_re_query_hwrm_intf_version(struct bnxt_re_dev *rdev) { struct bnxt_en_dev *en_dev = rdev->en_dev; struct hwrm_ver_get_output resp = {0}; struct hwrm_ver_get_input req = {0}; struct bnxt_qplib_chip_ctx *cctx; struct bnxt_fw_msg fw_msg; int rc = 0; memset(&fw_msg, 0, sizeof(fw_msg)); bnxt_re_init_hwrm_hdr(rdev, (void *)&req, HWRM_VER_GET, -1, -1); req.hwrm_intf_maj = HWRM_VERSION_MAJOR; req.hwrm_intf_min = HWRM_VERSION_MINOR; req.hwrm_intf_upd = HWRM_VERSION_UPDATE; bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp, sizeof(resp), DFLT_HWRM_CMD_TIMEOUT); rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg); if (rc) { dev_err(rdev_to_dev(rdev), "Failed to query HW version, rc = 0x%x", rc); return rc; } cctx = rdev->chip_ctx; cctx->hwrm_intf_ver = (u64) le16_to_cpu(resp.hwrm_intf_major) << 48 | (u64) le16_to_cpu(resp.hwrm_intf_minor) << 32 | (u64) le16_to_cpu(resp.hwrm_intf_build) << 16 | le16_to_cpu(resp.hwrm_intf_patch); cctx->hwrm_cmd_max_timeout = le16_to_cpu(resp.max_req_timeout); if (!cctx->hwrm_cmd_max_timeout) cctx->hwrm_cmd_max_timeout = RCFW_FW_STALL_MAX_TIMEOUT; cctx->chip_num = le16_to_cpu(resp.chip_num); cctx->chip_rev = resp.chip_rev; cctx->chip_metal = resp.chip_metal; return 0; } /* Query device config using common hwrm */ static int bnxt_re_hwrm_qcfg(struct bnxt_re_dev *rdev, u32 *db_len, u32 *offset) { struct bnxt_en_dev *en_dev = rdev->en_dev; struct hwrm_func_qcfg_output resp = {0}; struct hwrm_func_qcfg_input req = {0}; struct bnxt_fw_msg fw_msg; int rc; memset(&fw_msg, 0, sizeof(fw_msg)); bnxt_re_init_hwrm_hdr(rdev, (void *)&req, HWRM_FUNC_QCFG, -1, -1); req.fid = cpu_to_le16(0xffff); bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp, sizeof(resp), DFLT_HWRM_CMD_TIMEOUT); rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg); if (rc) { dev_err(rdev_to_dev(rdev), "Failed to query config, rc = %#x", rc); return rc; } *db_len = PAGE_ALIGN(le16_to_cpu(resp.l2_doorbell_bar_size_kb) * 1024); *offset = PAGE_ALIGN(le16_to_cpu(resp.legacy_l2_db_size_kb) * 1024); return 0; } /* Query function capabilities using common hwrm */ int bnxt_re_hwrm_qcaps(struct bnxt_re_dev *rdev) { struct bnxt_en_dev *en_dev = rdev->en_dev; struct hwrm_func_qcaps_output resp = {0}; struct hwrm_func_qcaps_input req = {0}; struct bnxt_qplib_chip_ctx *cctx; struct bnxt_fw_msg fw_msg; u8 push_enable = false; int rc; cctx = rdev->chip_ctx; memset(&fw_msg, 0, sizeof(fw_msg)); bnxt_re_init_hwrm_hdr(rdev, (void *)&req, HWRM_FUNC_QCAPS, -1, -1); req.fid = cpu_to_le16(0xffff); bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp, sizeof(resp), DFLT_HWRM_CMD_TIMEOUT); rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg); if (rc) { dev_err(rdev_to_dev(rdev), "Failed to query capabilities, rc = %#x", rc); return rc; } if (_is_chip_p7(rdev->chip_ctx)) push_enable = (resp.flags_ext & HWRM_FUNC_QCAPS_OUTPUT_FLAGS_EXT_PPP_PUSH_MODE_SUPPORTED) ? true : false; else push_enable = (resp.flags & HWRM_FUNC_QCAPS_OUTPUT_FLAGS_WCB_PUSH_MODE) ? true : false; cctx->modes.db_push = push_enable; cctx->modes.dbr_pacing = resp.flags_ext & HWRM_FUNC_QCAPS_OUTPUT_FLAGS_EXT_DBR_PACING_SUPPORTED ? true : false; cctx->modes.dbr_pacing_ext = resp.flags_ext2 & HWRM_FUNC_QCAPS_OUTPUT_FLAGS_EXT2_DBR_PACING_EXT_SUPPORTED ? true : false; cctx->modes.dbr_drop_recov = (resp.flags_ext2 & HWRM_FUNC_QCAPS_OUTPUT_FLAGS_EXT2_SW_DBR_DROP_RECOVERY_SUPPORTED) ? true : false; cctx->modes.dbr_pacing_v0 = (resp.flags_ext2 & HWRM_FUNC_QCAPS_OUTPUT_FLAGS_EXT2_DBR_PACING_V0_SUPPORTED) ? true : false; dev_dbg(rdev_to_dev(rdev), "%s: cctx->modes.dbr_pacing = %d cctx->modes.dbr_pacing_ext = %d, dbr_drop_recov %d\n", __func__, cctx->modes.dbr_pacing, cctx->modes.dbr_pacing_ext, cctx->modes.dbr_drop_recov); return 0; } static int bnxt_re_hwrm_dbr_pacing_qcfg(struct bnxt_re_dev *rdev) { struct bnxt_qplib_db_pacing_data *pacing_data = rdev->qplib_res.pacing_data; struct hwrm_func_dbr_pacing_qcfg_output resp = {0}; struct hwrm_func_dbr_pacing_qcfg_input req = {0}; struct bnxt_en_dev *en_dev = rdev->en_dev; struct bnxt_qplib_chip_ctx *cctx; struct bnxt_fw_msg fw_msg; u32 primary_nq_id; int rc; cctx = rdev->chip_ctx; memset(&fw_msg, 0, sizeof(fw_msg)); bnxt_re_init_hwrm_hdr(rdev, (void *)&req, HWRM_FUNC_DBR_PACING_QCFG, -1, -1); bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp, sizeof(resp), BNXT_RE_HWRM_CMD_TIMEOUT(rdev)); rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg); if (rc) { dev_dbg(rdev_to_dev(rdev), "Failed to query dbr pacing config, rc = %#x", rc); return rc; } primary_nq_id = le32_to_cpu(resp.primary_nq_id); if (primary_nq_id == 0xffffffff && !bnxt_qplib_dbr_pacing_ext_en(rdev->chip_ctx)) { dev_err(rdev_to_dev(rdev), "%s:%d Invoke bnxt_qplib_dbr_pacing_set_primary_pf with 1\n", __func__, __LINE__); bnxt_qplib_dbr_pacing_set_primary_pf(rdev->chip_ctx, 1); } if (bnxt_qplib_dbr_pacing_ext_en(rdev->chip_ctx)) { struct bnxt_qplib_nq *nq; nq = &rdev->nqr.nq[0]; /* Reset the primary capability */ if (nq->ring_id != primary_nq_id) bnxt_qplib_dbr_pacing_set_primary_pf(rdev->chip_ctx, 0); } if ((resp.dbr_stat_db_fifo_reg & HWRM_FUNC_DBR_PACING_QCFG_OUTPUT_DBR_STAT_DB_FIFO_REG_ADDR_SPACE_MASK) == HWRM_FUNC_DBR_PACING_QCFG_OUTPUT_DBR_STAT_DB_FIFO_REG_ADDR_SPACE_GRC) cctx->dbr_stat_db_fifo = resp.dbr_stat_db_fifo_reg & ~HWRM_FUNC_DBR_PACING_QCFG_OUTPUT_DBR_STAT_DB_FIFO_REG_ADDR_SPACE_MASK; if ((resp.dbr_throttling_aeq_arm_reg & HWRM_FUNC_DBR_PACING_QCFG_OUTPUT_DBR_THROTTLING_AEQ_ARM_REG_ADDR_SPACE_MASK) == HWRM_FUNC_DBR_PACING_QCFG_OUTPUT_DBR_THROTTLING_AEQ_ARM_REG_ADDR_SPACE_GRC) { cctx->dbr_aeq_arm_reg = resp.dbr_throttling_aeq_arm_reg & ~HWRM_FUNC_DBR_PACING_QCFG_OUTPUT_DBR_STAT_DB_FIFO_REG_ADDR_SPACE_MASK; cctx->dbr_throttling_reg = cctx->dbr_aeq_arm_reg - 4; } pacing_data->fifo_max_depth = le32_to_cpu(resp.dbr_stat_db_max_fifo_depth); if (!pacing_data->fifo_max_depth) pacing_data->fifo_max_depth = BNXT_RE_MAX_FIFO_DEPTH(cctx); pacing_data->fifo_room_mask = le32_to_cpu(resp.dbr_stat_db_fifo_reg_fifo_room_mask); pacing_data->fifo_room_shift = resp.dbr_stat_db_fifo_reg_fifo_room_shift; dev_dbg(rdev_to_dev(rdev), "%s: nq:0x%x primary_pf:%d db_fifo:0x%x aeq_arm:0x%x i" "fifo_max_depth 0x%x , resp.dbr_stat_db_max_fifo_depth 0x%x);\n", __func__, resp.primary_nq_id, cctx->modes.dbr_primary_pf, cctx->dbr_stat_db_fifo, cctx->dbr_aeq_arm_reg, pacing_data->fifo_max_depth, le32_to_cpu(resp.dbr_stat_db_max_fifo_depth)); return 0; } static int bnxt_re_hwrm_dbr_pacing_cfg(struct bnxt_re_dev *rdev, bool enable) { struct hwrm_func_dbr_pacing_cfg_output resp = {0}; struct hwrm_func_dbr_pacing_cfg_input req = {0}; struct bnxt_en_dev *en_dev = rdev->en_dev; struct bnxt_fw_msg fw_msg; int rc; if (test_bit(BNXT_RE_FLAG_ERR_DEVICE_DETACHED, &rdev->flags)) return 0; memset(&fw_msg, 0, sizeof(fw_msg)); bnxt_re_init_hwrm_hdr(rdev, (void *)&req, HWRM_FUNC_DBR_PACING_CFG, -1, -1); if (enable) { req.flags = HWRM_FUNC_DBR_PACING_CFG_INPUT_FLAGS_DBR_NQ_EVENT_ENABLE; req.enables = cpu_to_le32(HWRM_FUNC_DBR_PACING_CFG_INPUT_ENABLES_PRIMARY_NQ_ID_VALID | HWRM_FUNC_DBR_PACING_CFG_INPUT_ENABLES_PACING_THRESHOLD_VALID); } else { req.flags = HWRM_FUNC_DBR_PACING_CFG_INPUT_FLAGS_DBR_NQ_EVENT_DISABLE; } req.primary_nq_id = cpu_to_le32(rdev->dbq_nq_id); req.pacing_threshold = cpu_to_le32(rdev->dbq_watermark); dev_dbg(rdev_to_dev(rdev), "%s: nq_id = 0x%x pacing_threshold = 0x%x", __func__, req.primary_nq_id, req.pacing_threshold); bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp, sizeof(resp), BNXT_RE_HWRM_CMD_TIMEOUT(rdev)); rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg); if (rc) { dev_dbg(rdev_to_dev(rdev), "Failed to set dbr pacing config, rc = %#x", rc); return rc; } return 0; } /* Net -> RoCE driver */ /* Device */ struct bnxt_re_dev *bnxt_re_from_netdev(struct ifnet *netdev) { struct bnxt_re_dev *rdev; rcu_read_lock(); list_for_each_entry_rcu(rdev, &bnxt_re_dev_list, list) { if (rdev->netdev == netdev) { rcu_read_unlock(); dev_dbg(rdev_to_dev(rdev), "netdev (%p) found, ref_count = 0x%x", netdev, atomic_read(&rdev->ref_count)); return rdev; } } rcu_read_unlock(); return NULL; } static ssize_t show_rev(struct device *device, struct device_attribute *attr, char *buf) { struct bnxt_re_dev *rdev = to_bnxt_re_dev(device, ibdev.dev); return scnprintf(buf, PAGE_SIZE, "0x%x\n", rdev->en_dev->pdev->vendor); } static ssize_t show_hca(struct device *device, struct device_attribute *attr, char *buf) { struct bnxt_re_dev *rdev = to_bnxt_re_dev(device, ibdev.dev); return scnprintf(buf, PAGE_SIZE, "%s\n", rdev->ibdev.node_desc); } +static ssize_t show_board_id(struct device *device, struct device_attribute *attr, + char *buf) +{ + struct bnxt_re_dev *rdev = to_bnxt_re_dev(device, ibdev.dev); + char buffer[BNXT_VPD_PN_FLD_LEN] = {}; + + if (!rdev->is_virtfn) + memcpy(buffer, rdev->en_dev->board_part_number, + BNXT_VPD_PN_FLD_LEN - 1); + else + scnprintf(buffer, BNXT_VPD_PN_FLD_LEN, + "0x%x-VF", rdev->en_dev->pdev->device); + + return scnprintf(buf, PAGE_SIZE, "%s\n", buffer); +} + static DEVICE_ATTR(hw_rev, 0444, show_rev, NULL); static DEVICE_ATTR(hca_type, 0444, show_hca, NULL); +static DEVICE_ATTR(board_id, 0444, show_board_id, NULL); + static struct device_attribute *bnxt_re_attributes[] = { &dev_attr_hw_rev, - &dev_attr_hca_type + &dev_attr_hca_type, + &dev_attr_board_id }; int ib_register_device_compat(struct bnxt_re_dev *rdev) { struct ib_device *ibdev = &rdev->ibdev; char name[IB_DEVICE_NAME_MAX]; memset(name, 0, IB_DEVICE_NAME_MAX); strlcpy(name, "bnxt_re%d", IB_DEVICE_NAME_MAX); strlcpy(ibdev->name, name, IB_DEVICE_NAME_MAX); return ib_register_device(ibdev, NULL); } static int bnxt_re_register_ib(struct bnxt_re_dev *rdev) { struct ib_device *ibdev = &rdev->ibdev; int ret = 0; /* ib device init */ ibdev->owner = THIS_MODULE; ibdev->uverbs_abi_ver = BNXT_RE_ABI_VERSION; ibdev->node_type = RDMA_NODE_IB_CA; strlcpy(ibdev->node_desc, BNXT_RE_DESC " HCA", strlen(BNXT_RE_DESC) + 5); ibdev->phys_port_cnt = 1; bnxt_qplib_get_guid(rdev->dev_addr, (u8 *)&ibdev->node_guid); /* Data path irqs is one less than the max msix vectors */ ibdev->num_comp_vectors = rdev->nqr.num_msix - 1; bnxt_re_set_dma_device(ibdev, rdev); ibdev->local_dma_lkey = BNXT_QPLIB_RSVD_LKEY; /* User space */ ibdev->uverbs_cmd_mask = (1ull << IB_USER_VERBS_CMD_GET_CONTEXT) | (1ull << IB_USER_VERBS_CMD_QUERY_DEVICE) | (1ull << IB_USER_VERBS_CMD_QUERY_PORT) | (1ull << IB_USER_VERBS_CMD_ALLOC_PD) | (1ull << IB_USER_VERBS_CMD_DEALLOC_PD) | (1ull << IB_USER_VERBS_CMD_REG_MR) | (1ull << IB_USER_VERBS_CMD_DEREG_MR) | (1ull << IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL) | (1ull << IB_USER_VERBS_CMD_CREATE_CQ) | (1ull << IB_USER_VERBS_CMD_DESTROY_CQ) | (1ull << IB_USER_VERBS_CMD_CREATE_QP) | (1ull << IB_USER_VERBS_CMD_MODIFY_QP) | (1ull << IB_USER_VERBS_CMD_QUERY_QP) | (1ull << IB_USER_VERBS_CMD_DESTROY_QP) | (1ull << IB_USER_VERBS_CMD_REREG_MR) | (1ull << IB_USER_VERBS_CMD_RESIZE_CQ) | (1ull << IB_USER_VERBS_CMD_CREATE_SRQ) | (1ull << IB_USER_VERBS_CMD_MODIFY_SRQ) | (1ull << IB_USER_VERBS_CMD_QUERY_SRQ) | (1ull << IB_USER_VERBS_CMD_DESTROY_SRQ) | (1ull << IB_USER_VERBS_CMD_ALLOC_MW) | (1ull << IB_USER_VERBS_CMD_DEALLOC_MW) | (1ull << IB_USER_VERBS_CMD_CREATE_AH) | (1ull << IB_USER_VERBS_CMD_MODIFY_AH) | (1ull << IB_USER_VERBS_CMD_QUERY_AH) | (1ull << IB_USER_VERBS_CMD_DESTROY_AH); ibdev->uverbs_ex_cmd_mask = (1ull << IB_USER_VERBS_EX_CMD_MODIFY_QP); ibdev->uverbs_cmd_mask |= (1ull << IB_USER_VERBS_CMD_POLL_CQ); #define bnxt_re_ib_ah bnxt_re_ah #define bnxt_re_ib_cq bnxt_re_cq #define bnxt_re_ib_pd bnxt_re_pd #define bnxt_re_ib_srq bnxt_re_srq #define bnxt_re_ib_ucontext bnxt_re_ucontext INIT_IB_DEVICE_OPS(&ibdev->ops, bnxt_re, BNXT_RE); ibdev->query_device = bnxt_re_query_device; ibdev->modify_device = bnxt_re_modify_device; ibdev->query_port = bnxt_re_query_port; ibdev->modify_port = bnxt_re_modify_port; ibdev->get_port_immutable = bnxt_re_get_port_immutable; ibdev->query_pkey = bnxt_re_query_pkey; ibdev->query_gid = bnxt_re_query_gid; ibdev->get_netdev = bnxt_re_get_netdev; ibdev->add_gid = bnxt_re_add_gid; ibdev->del_gid = bnxt_re_del_gid; ibdev->get_link_layer = bnxt_re_get_link_layer; ibdev->alloc_pd = bnxt_re_alloc_pd; ibdev->dealloc_pd = bnxt_re_dealloc_pd; ibdev->create_ah = bnxt_re_create_ah; ibdev->modify_ah = bnxt_re_modify_ah; ibdev->query_ah = bnxt_re_query_ah; ibdev->destroy_ah = bnxt_re_destroy_ah; ibdev->create_srq = bnxt_re_create_srq; ibdev->modify_srq = bnxt_re_modify_srq; ibdev->query_srq = bnxt_re_query_srq; ibdev->destroy_srq = bnxt_re_destroy_srq; ibdev->post_srq_recv = bnxt_re_post_srq_recv; ibdev->create_qp = bnxt_re_create_qp; ibdev->modify_qp = bnxt_re_modify_qp; ibdev->query_qp = bnxt_re_query_qp; ibdev->destroy_qp = bnxt_re_destroy_qp; ibdev->post_send = bnxt_re_post_send; ibdev->post_recv = bnxt_re_post_recv; ibdev->create_cq = bnxt_re_create_cq; ibdev->modify_cq = bnxt_re_modify_cq; ibdev->destroy_cq = bnxt_re_destroy_cq; ibdev->resize_cq = bnxt_re_resize_cq; ibdev->poll_cq = bnxt_re_poll_cq; ibdev->req_notify_cq = bnxt_re_req_notify_cq; ibdev->get_dma_mr = bnxt_re_get_dma_mr; ibdev->get_hw_stats = bnxt_re_get_hw_stats; ibdev->alloc_hw_stats = bnxt_re_alloc_hw_port_stats; ibdev->dereg_mr = bnxt_re_dereg_mr; ibdev->alloc_mr = bnxt_re_alloc_mr; ibdev->map_mr_sg = bnxt_re_map_mr_sg; ibdev->alloc_mw = bnxt_re_alloc_mw; ibdev->dealloc_mw = bnxt_re_dealloc_mw; ibdev->reg_user_mr = bnxt_re_reg_user_mr; ibdev->rereg_user_mr = bnxt_re_rereg_user_mr; ibdev->disassociate_ucontext = bnxt_re_disassociate_ucntx; ibdev->alloc_ucontext = bnxt_re_alloc_ucontext; ibdev->dealloc_ucontext = bnxt_re_dealloc_ucontext; ibdev->mmap = bnxt_re_mmap; ibdev->process_mad = bnxt_re_process_mad; ret = ib_register_device_compat(rdev); return ret; } static void bnxt_re_dev_dealloc(struct bnxt_re_dev *rdev) { int i = BNXT_RE_REF_WAIT_COUNT; dev_dbg(rdev_to_dev(rdev), "%s:Remove the device %p\n", __func__, rdev); /* Wait for rdev refcount to come down */ while ((atomic_read(&rdev->ref_count) > 1) && i--) msleep(100); if (atomic_read(&rdev->ref_count) > 1) dev_err(rdev_to_dev(rdev), "Failed waiting for ref count to deplete %d", atomic_read(&rdev->ref_count)); atomic_set(&rdev->ref_count, 0); if_rele(rdev->netdev); rdev->netdev = NULL; synchronize_rcu(); kfree(rdev->gid_map); kfree(rdev->dbg_stats); ib_dealloc_device(&rdev->ibdev); } static struct bnxt_re_dev *bnxt_re_dev_alloc(struct ifnet *netdev, struct bnxt_en_dev *en_dev) { struct bnxt_re_dev *rdev; u32 count; /* Allocate bnxt_re_dev instance here */ rdev = (struct bnxt_re_dev *)compat_ib_alloc_device(sizeof(*rdev)); if (!rdev) { pr_err("%s: bnxt_re_dev allocation failure!", ROCE_DRV_MODULE_NAME); return NULL; } /* Default values */ atomic_set(&rdev->ref_count, 0); rdev->netdev = netdev; dev_hold(rdev->netdev); rdev->en_dev = en_dev; rdev->id = rdev->en_dev->pdev->devfn; INIT_LIST_HEAD(&rdev->qp_list); mutex_init(&rdev->qp_lock); mutex_init(&rdev->cc_lock); mutex_init(&rdev->dbq_lock); bnxt_re_clear_rsors_stat(&rdev->stats.rsors); rdev->cosq[0] = rdev->cosq[1] = 0xFFFF; rdev->min_tx_depth = 1; rdev->stats.stats_query_sec = 1; /* Disable priority vlan as the default mode is DSCP based PFC */ rdev->cc_param.disable_prio_vlan_tx = 1; /* Initialize worker for DBR Pacing */ INIT_WORK(&rdev->dbq_fifo_check_work, bnxt_re_db_fifo_check); INIT_DELAYED_WORK(&rdev->dbq_pacing_work, bnxt_re_pacing_timer_exp); rdev->gid_map = kzalloc(sizeof(*(rdev->gid_map)) * BNXT_RE_MAX_SGID_ENTRIES, GFP_KERNEL); if (!rdev->gid_map) { ib_dealloc_device(&rdev->ibdev); return NULL; } for(count = 0; count < BNXT_RE_MAX_SGID_ENTRIES; count++) rdev->gid_map[count] = -1; rdev->dbg_stats = kzalloc(sizeof(*rdev->dbg_stats), GFP_KERNEL); if (!rdev->dbg_stats) { ib_dealloc_device(&rdev->ibdev); return NULL; } return rdev; } static int bnxt_re_handle_unaffi_async_event( struct creq_func_event *unaffi_async) { switch (unaffi_async->event) { case CREQ_FUNC_EVENT_EVENT_TX_WQE_ERROR: case CREQ_FUNC_EVENT_EVENT_TX_DATA_ERROR: case CREQ_FUNC_EVENT_EVENT_RX_WQE_ERROR: case CREQ_FUNC_EVENT_EVENT_RX_DATA_ERROR: case CREQ_FUNC_EVENT_EVENT_CQ_ERROR: case CREQ_FUNC_EVENT_EVENT_TQM_ERROR: case CREQ_FUNC_EVENT_EVENT_CFCQ_ERROR: case CREQ_FUNC_EVENT_EVENT_CFCS_ERROR: case CREQ_FUNC_EVENT_EVENT_CFCC_ERROR: case CREQ_FUNC_EVENT_EVENT_CFCM_ERROR: case CREQ_FUNC_EVENT_EVENT_TIM_ERROR: break; default: return -EINVAL; } return 0; } static int bnxt_re_handle_qp_async_event(void *qp_event, struct bnxt_re_qp *qp) { struct creq_qp_error_notification *err_event; struct ib_event event; unsigned int flags; if (qp->qplib_qp.state == CMDQ_MODIFY_QP_NEW_STATE_ERR && !qp->qplib_qp.is_user) { flags = bnxt_re_lock_cqs(qp); bnxt_qplib_add_flush_qp(&qp->qplib_qp); bnxt_re_unlock_cqs(qp, flags); } memset(&event, 0, sizeof(event)); event.device = &qp->rdev->ibdev; event.element.qp = &qp->ib_qp; event.event = IB_EVENT_QP_FATAL; err_event = qp_event; switch(err_event->res_err_state_reason) { case CFCQ_RES_ERR_STATE_REASON_RES_EXCEED_MAX: case CFCQ_RES_ERR_STATE_REASON_RES_PAYLOAD_LENGTH_MISMATCH: case CFCQ_RES_ERR_STATE_REASON_RES_OPCODE_ERROR: case CFCQ_RES_ERR_STATE_REASON_RES_PSN_SEQ_ERROR_RETRY_LIMIT: case CFCQ_RES_ERR_STATE_REASON_RES_RX_INVALID_R_KEY: case CFCQ_RES_ERR_STATE_REASON_RES_RX_DOMAIN_ERROR: case CFCQ_RES_ERR_STATE_REASON_RES_RX_NO_PERMISSION: case CFCQ_RES_ERR_STATE_REASON_RES_RX_RANGE_ERROR: case CFCQ_RES_ERR_STATE_REASON_RES_TX_INVALID_R_KEY: case CFCQ_RES_ERR_STATE_REASON_RES_TX_DOMAIN_ERROR: case CFCQ_RES_ERR_STATE_REASON_RES_TX_NO_PERMISSION: case CFCQ_RES_ERR_STATE_REASON_RES_TX_RANGE_ERROR: case CFCQ_RES_ERR_STATE_REASON_RES_IVALID_DUP_RKEY: case CFCQ_RES_ERR_STATE_REASON_RES_UNALIGN_ATOMIC: event.event = IB_EVENT_QP_ACCESS_ERR; break; case CFCQ_RES_ERR_STATE_REASON_RES_EXCEEDS_WQE: case CFCQ_RES_ERR_STATE_REASON_RES_WQE_FORMAT_ERROR: case CFCQ_RES_ERR_STATE_REASON_RES_SRQ_LOAD_ERROR: case CFCQ_RES_ERR_STATE_REASON_RES_UNSUPPORTED_OPCODE: case CFCQ_RES_ERR_STATE_REASON_RES_REM_INVALIDATE: event.event = IB_EVENT_QP_REQ_ERR; break; case CFCQ_RES_ERR_STATE_REASON_RES_IRRQ_OFLOW: case CFCQ_RES_ERR_STATE_REASON_RES_CMP_ERROR: case CFCQ_RES_ERR_STATE_REASON_RES_CQ_LOAD_ERROR: case CFCQ_RES_ERR_STATE_REASON_RES_TX_PCI_ERROR: case CFCQ_RES_ERR_STATE_REASON_RES_RX_PCI_ERROR: case CFCQ_RES_ERR_STATE_REASON_RES_MEMORY_ERROR: case CFCQ_RES_ERR_STATE_REASON_RES_SRQ_ERROR: event.event = IB_EVENT_QP_FATAL; break; default: if (qp->qplib_qp.srq) event.event = IB_EVENT_QP_LAST_WQE_REACHED; break; } if (err_event->res_err_state_reason) dev_err(rdev_to_dev(qp->rdev), "%s %s qp_id: %d cons (%d %d) req (%d %d) res (%d %d)\n", __func__, qp->qplib_qp.is_user ? "user" : "kernel", qp->qplib_qp.id, err_event->sq_cons_idx, err_event->rq_cons_idx, err_event->req_slow_path_state, err_event->req_err_state_reason, err_event->res_slow_path_state, err_event->res_err_state_reason); if (event.device && qp->ib_qp.event_handler) qp->ib_qp.event_handler(&event, qp->ib_qp.qp_context); return 0; } static int bnxt_re_handle_cq_async_error(void *event, struct bnxt_re_cq *cq) { struct creq_cq_error_notification *cqerr; bool send = false; cqerr = event; switch (cqerr->cq_err_reason) { case CREQ_CQ_ERROR_NOTIFICATION_CQ_ERR_REASON_REQ_CQ_INVALID_ERROR: case CREQ_CQ_ERROR_NOTIFICATION_CQ_ERR_REASON_REQ_CQ_OVERFLOW_ERROR: case CREQ_CQ_ERROR_NOTIFICATION_CQ_ERR_REASON_REQ_CQ_LOAD_ERROR: case CREQ_CQ_ERROR_NOTIFICATION_CQ_ERR_REASON_RES_CQ_INVALID_ERROR: case CREQ_CQ_ERROR_NOTIFICATION_CQ_ERR_REASON_RES_CQ_OVERFLOW_ERROR: case CREQ_CQ_ERROR_NOTIFICATION_CQ_ERR_REASON_RES_CQ_LOAD_ERROR: send = true; default: break; } if (send && cq->ibcq.event_handler) { struct ib_event ibevent = {}; ibevent.event = IB_EVENT_CQ_ERR; ibevent.element.cq = &cq->ibcq; ibevent.device = &cq->rdev->ibdev; dev_err(rdev_to_dev(cq->rdev), "%s err reason %d\n", __func__, cqerr->cq_err_reason); cq->ibcq.event_handler(&ibevent, cq->ibcq.cq_context); } cq->qplib_cq.is_cq_err_event = true; return 0; } static int bnxt_re_handle_affi_async_event(struct creq_qp_event *affi_async, void *obj) { struct bnxt_qplib_qp *qplqp; struct bnxt_qplib_cq *qplcq; struct bnxt_re_qp *qp; struct bnxt_re_cq *cq; int rc = 0; u8 event; if (!obj) return rc; /* QP was already dead, still return success */ event = affi_async->event; switch (event) { case CREQ_QP_EVENT_EVENT_QP_ERROR_NOTIFICATION: qplqp = obj; qp = container_of(qplqp, struct bnxt_re_qp, qplib_qp); rc = bnxt_re_handle_qp_async_event(affi_async, qp); break; case CREQ_QP_EVENT_EVENT_CQ_ERROR_NOTIFICATION: qplcq = obj; cq = container_of(qplcq, struct bnxt_re_cq, qplib_cq); rc = bnxt_re_handle_cq_async_error(affi_async, cq); break; default: rc = -EINVAL; } return rc; } static int bnxt_re_aeq_handler(struct bnxt_qplib_rcfw *rcfw, void *aeqe, void *obj) { struct creq_func_event *unaffi_async; struct creq_qp_event *affi_async; u8 type; int rc; type = ((struct creq_base *)aeqe)->type; if (type == CREQ_BASE_TYPE_FUNC_EVENT) { unaffi_async = aeqe; rc = bnxt_re_handle_unaffi_async_event(unaffi_async); } else { affi_async = aeqe; rc = bnxt_re_handle_affi_async_event(affi_async, obj); } return rc; } static int bnxt_re_srqn_handler(struct bnxt_qplib_nq *nq, struct bnxt_qplib_srq *handle, u8 event) { struct bnxt_re_srq *srq = to_bnxt_re(handle, struct bnxt_re_srq, qplib_srq); struct ib_event ib_event; if (srq == NULL) { pr_err("%s: SRQ is NULL, SRQN not handled", ROCE_DRV_MODULE_NAME); return -EINVAL; } ib_event.device = &srq->rdev->ibdev; ib_event.element.srq = &srq->ibsrq; if (event == NQ_SRQ_EVENT_EVENT_SRQ_THRESHOLD_EVENT) ib_event.event = IB_EVENT_SRQ_LIMIT_REACHED; else ib_event.event = IB_EVENT_SRQ_ERR; if (srq->ibsrq.event_handler) { /* Lock event_handler? */ (*srq->ibsrq.event_handler)(&ib_event, srq->ibsrq.srq_context); } return 0; } static int bnxt_re_cqn_handler(struct bnxt_qplib_nq *nq, struct bnxt_qplib_cq *handle) { struct bnxt_re_cq *cq = to_bnxt_re(handle, struct bnxt_re_cq, qplib_cq); u32 *cq_ptr; if (cq == NULL) { pr_err("%s: CQ is NULL, CQN not handled", ROCE_DRV_MODULE_NAME); return -EINVAL; } /* CQ already in destroy path. Do not handle any more events */ if (handle->destroyed || !atomic_read(&cq->ibcq.usecnt)) { if (!handle->destroyed) dev_dbg(NULL, "%s: CQ being destroyed, CQN not handled", ROCE_DRV_MODULE_NAME); return 0; } if (cq->ibcq.comp_handler) { if (cq->uctx_cq_page) { cq_ptr = (u32 *)cq->uctx_cq_page; *cq_ptr = cq->qplib_cq.toggle; } /* Lock comp_handler? */ (*cq->ibcq.comp_handler)(&cq->ibcq, cq->ibcq.cq_context); } return 0; } struct bnxt_qplib_nq *bnxt_re_get_nq(struct bnxt_re_dev *rdev) { int min, indx; mutex_lock(&rdev->nqr.load_lock); for (indx = 0, min = 0; indx < (rdev->nqr.num_msix - 1); indx++) { if (rdev->nqr.nq[min].load > rdev->nqr.nq[indx].load) min = indx; } rdev->nqr.nq[min].load++; mutex_unlock(&rdev->nqr.load_lock); return &rdev->nqr.nq[min]; } void bnxt_re_put_nq(struct bnxt_re_dev *rdev, struct bnxt_qplib_nq *nq) { mutex_lock(&rdev->nqr.load_lock); nq->load--; mutex_unlock(&rdev->nqr.load_lock); } static bool bnxt_re_check_min_attr(struct bnxt_re_dev *rdev) { struct bnxt_qplib_dev_attr *attr; bool rc = true; attr = rdev->dev_attr; if (!attr->max_cq || !attr->max_qp || !attr->max_sgid || !attr->max_mr) { dev_err(rdev_to_dev(rdev),"Insufficient RoCE resources"); dev_dbg(rdev_to_dev(rdev), "max_cq = %d, max_qp = %d, max_dpi = %d, max_sgid = %d, max_mr = %d", attr->max_cq, attr->max_qp, attr->max_dpi, attr->max_sgid, attr->max_mr); rc = false; } return rc; } static void bnxt_re_dispatch_event(struct ib_device *ibdev, struct ib_qp *qp, u8 port_num, enum ib_event_type event) { struct ib_event ib_event; ib_event.device = ibdev; if (qp) { ib_event.element.qp = qp; ib_event.event = event; if (qp->event_handler) qp->event_handler(&ib_event, qp->qp_context); } else { ib_event.element.port_num = port_num; ib_event.event = event; ib_dispatch_event(&ib_event); } dev_dbg(rdev_to_dev(to_bnxt_re_dev(ibdev, ibdev)), "ibdev %p Event 0x%x port_num 0x%x", ibdev, event, port_num); } static bool bnxt_re_is_qp1_or_shadow_qp(struct bnxt_re_dev *rdev, struct bnxt_re_qp *qp) { if (rdev->gsi_ctx.gsi_qp_mode == BNXT_RE_GSI_MODE_ALL) return (qp->ib_qp.qp_type == IB_QPT_GSI) || (qp == rdev->gsi_ctx.gsi_sqp); else return (qp->ib_qp.qp_type == IB_QPT_GSI); } static void bnxt_re_stop_all_nonqp1_nonshadow_qps(struct bnxt_re_dev *rdev) { struct bnxt_qplib_qp *qpl_qp; bool dev_detached = false; struct ib_qp_attr qp_attr; int num_qps_stopped = 0; int mask = IB_QP_STATE; struct bnxt_re_qp *qp; unsigned long flags; if (!rdev) return; restart: if (test_bit(BNXT_RE_FLAG_ERR_DEVICE_DETACHED, &rdev->flags)) dev_detached = true; qp_attr.qp_state = IB_QPS_ERR; mutex_lock(&rdev->qp_lock); list_for_each_entry(qp, &rdev->qp_list, list) { qpl_qp = &qp->qplib_qp; if (dev_detached || !bnxt_re_is_qp1_or_shadow_qp(rdev, qp)) { if (qpl_qp->state != CMDQ_MODIFY_QP_NEW_STATE_RESET && qpl_qp->state != CMDQ_MODIFY_QP_NEW_STATE_ERR) { if (dev_detached) { /* * Cant actually send the command down, * marking the state for bookkeeping */ qpl_qp->state = CMDQ_MODIFY_QP_NEW_STATE_ERR; qpl_qp->cur_qp_state = qpl_qp->state; if (!qpl_qp->is_user) { /* Add to flush list */ flags = bnxt_re_lock_cqs(qp); bnxt_qplib_add_flush_qp(qpl_qp); bnxt_re_unlock_cqs(qp, flags); } } else { num_qps_stopped++; bnxt_re_modify_qp(&qp->ib_qp, &qp_attr, mask, NULL); } bnxt_re_dispatch_event(&rdev->ibdev, &qp->ib_qp, 1, IB_EVENT_QP_FATAL); /* * 1. Release qp_lock after a budget to unblock other verb * requests (like qp_destroy) from stack. * 2. Traverse through the qp_list freshly as addition / deletion * might have happened since qp_lock is getting released here. */ if (num_qps_stopped % BNXT_RE_STOP_QPS_BUDGET == 0) { mutex_unlock(&rdev->qp_lock); goto restart; } } } } mutex_unlock(&rdev->qp_lock); } static int bnxt_re_update_gid(struct bnxt_re_dev *rdev) { struct bnxt_qplib_sgid_tbl *sgid_tbl = &rdev->qplib_res.sgid_tbl; struct bnxt_qplib_gid gid; u16 gid_idx, index; int rc = 0; if (!test_bit(BNXT_RE_FLAG_IBDEV_REGISTERED, &rdev->flags)) return 0; if (sgid_tbl == NULL) { dev_err(rdev_to_dev(rdev), "QPLIB: SGID table not allocated"); return -EINVAL; } for (index = 0; index < sgid_tbl->active; index++) { gid_idx = sgid_tbl->hw_id[index]; if (!memcmp(&sgid_tbl->tbl[index], &bnxt_qplib_gid_zero, sizeof(bnxt_qplib_gid_zero))) continue; /* Need to modify the VLAN enable setting of non VLAN GID only * as setting is done for VLAN GID while adding GID * * If disable_prio_vlan_tx is enable, then we'll need to remove the * vlan entry from the sgid_tbl. */ if (sgid_tbl->vlan[index] == true) continue; memcpy(&gid, &sgid_tbl->tbl[index], sizeof(gid)); rc = bnxt_qplib_update_sgid(sgid_tbl, &gid, gid_idx, rdev->dev_addr); } return rc; } static void bnxt_re_clear_cc(struct bnxt_re_dev *rdev) { struct bnxt_qplib_cc_param *cc_param = &rdev->cc_param; if (_is_chip_p7(rdev->chip_ctx)) { cc_param->mask = CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_TOS_DSCP; } else { cc_param->mask = (CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_CC_MODE | CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_ENABLE_CC | CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_TOS_ECN); if (!is_qport_service_type_supported(rdev)) cc_param->mask |= (CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_ALT_VLAN_PCP | CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_ALT_TOS_DSCP | CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_TOS_DSCP); } cc_param->cur_mask = cc_param->mask; if (bnxt_qplib_modify_cc(&rdev->qplib_res, cc_param)) dev_err(rdev_to_dev(rdev), "Failed to modify cc\n"); } static int bnxt_re_setup_cc(struct bnxt_re_dev *rdev) { struct bnxt_qplib_cc_param *cc_param = &rdev->cc_param; int rc; if (_is_chip_p7(rdev->chip_ctx)) { cc_param->enable = 0x0; cc_param->mask = CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_TOS_DSCP; } else { cc_param->enable = 0x1; cc_param->mask = (CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_CC_MODE | CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_ENABLE_CC | CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_TOS_ECN); if (!is_qport_service_type_supported(rdev)) cc_param->mask |= (CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_ALT_VLAN_PCP | CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_ALT_TOS_DSCP | CMDQ_MODIFY_ROCE_CC_MODIFY_MASK_TOS_DSCP); } cc_param->cur_mask = cc_param->mask; rc = bnxt_qplib_modify_cc(&rdev->qplib_res, cc_param); if (rc) { dev_err(rdev_to_dev(rdev), "Failed to modify cc\n"); return rc; } /* Reset the programming mask */ cc_param->mask = 0; if (cc_param->qp1_tos_dscp != cc_param->tos_dscp) { cc_param->qp1_tos_dscp = cc_param->tos_dscp; rc = bnxt_re_update_qp1_tos_dscp(rdev); if (rc) { dev_err(rdev_to_dev(rdev), "%s:Failed to modify QP1:%d", __func__, rc); goto clear; } } return 0; clear: bnxt_re_clear_cc(rdev); return rc; } int bnxt_re_query_hwrm_dscp2pri(struct bnxt_re_dev *rdev, struct bnxt_re_dscp2pri *d2p, u16 *count, u16 target_id) { struct bnxt_en_dev *en_dev = rdev->en_dev; struct hwrm_queue_dscp2pri_qcfg_input req; struct hwrm_queue_dscp2pri_qcfg_output resp; struct bnxt_re_dscp2pri *dscp2pri; struct bnxt_fw_msg fw_msg; u16 in_count = *count; dma_addr_t dma_handle; int rc = 0, i; u16 data_len; u8 *kmem; data_len = *count * sizeof(*dscp2pri); memset(&fw_msg, 0, sizeof(fw_msg)); memset(&req, 0, sizeof(req)); bnxt_re_init_hwrm_hdr(rdev, (void *)&req, HWRM_QUEUE_DSCP2PRI_QCFG, -1, target_id); req.port_id = (target_id == 0xFFFF) ? en_dev->pf_port_id : 1; kmem = dma_zalloc_coherent(&en_dev->pdev->dev, data_len, &dma_handle, GFP_KERNEL); if (!kmem) { dev_err(rdev_to_dev(rdev), "dma_zalloc_coherent failure, length = %u\n", (unsigned)data_len); return -ENOMEM; } req.dest_data_addr = cpu_to_le64(dma_handle); req.dest_data_buffer_size = cpu_to_le16(data_len); bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp, sizeof(resp), DFLT_HWRM_CMD_TIMEOUT); rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg); if (rc) goto out; /* Upload the DSCP-MASK-PRI tuple(s) */ dscp2pri = (struct bnxt_re_dscp2pri *)kmem; for (i = 0; i < le16_to_cpu(resp.entry_cnt) && i < in_count; i++) { d2p[i].dscp = dscp2pri->dscp; d2p[i].mask = dscp2pri->mask; d2p[i].pri = dscp2pri->pri; dscp2pri++; } *count = le16_to_cpu(resp.entry_cnt); out: dma_free_coherent(&en_dev->pdev->dev, data_len, kmem, dma_handle); return rc; } int bnxt_re_prio_vlan_tx_update(struct bnxt_re_dev *rdev) { /* Remove the VLAN from the GID entry */ if (rdev->cc_param.disable_prio_vlan_tx) rdev->qplib_res.prio = false; else rdev->qplib_res.prio = true; return bnxt_re_update_gid(rdev); } int bnxt_re_set_hwrm_dscp2pri(struct bnxt_re_dev *rdev, struct bnxt_re_dscp2pri *d2p, u16 count, u16 target_id) { struct bnxt_en_dev *en_dev = rdev->en_dev; struct hwrm_queue_dscp2pri_cfg_input req; struct hwrm_queue_dscp2pri_cfg_output resp; struct bnxt_fw_msg fw_msg; struct bnxt_re_dscp2pri *dscp2pri; int i, rc, data_len = 3 * 256; dma_addr_t dma_handle; u8 *kmem; memset(&req, 0, sizeof(req)); memset(&fw_msg, 0, sizeof(fw_msg)); bnxt_re_init_hwrm_hdr(rdev, (void *)&req, HWRM_QUEUE_DSCP2PRI_CFG, -1, target_id); req.port_id = (target_id == 0xFFFF) ? en_dev->pf_port_id : 1; kmem = dma_alloc_coherent(&en_dev->pdev->dev, data_len, &dma_handle, GFP_KERNEL); if (!kmem) { dev_err(rdev_to_dev(rdev), "dma_alloc_coherent failure, length = %u\n", (unsigned)data_len); return -ENOMEM; } req.src_data_addr = cpu_to_le64(dma_handle); /* Download the DSCP-MASK-PRI tuple(s) */ dscp2pri = (struct bnxt_re_dscp2pri *)kmem; for (i = 0; i < count; i++) { dscp2pri->dscp = d2p[i].dscp; dscp2pri->mask = d2p[i].mask; dscp2pri->pri = d2p[i].pri; dscp2pri++; } req.entry_cnt = cpu_to_le16(count); bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp, sizeof(resp), DFLT_HWRM_CMD_TIMEOUT); rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg); dma_free_coherent(&en_dev->pdev->dev, data_len, kmem, dma_handle); return rc; } int bnxt_re_query_hwrm_qportcfg(struct bnxt_re_dev *rdev, struct bnxt_re_tc_rec *tc_rec, u16 tid) { u8 max_tc, tc, *qptr, *type_ptr0, *type_ptr1; struct hwrm_queue_qportcfg_output resp = {0}; struct hwrm_queue_qportcfg_input req = {0}; struct bnxt_en_dev *en_dev = rdev->en_dev; struct bnxt_fw_msg fw_msg; bool def_init = false; u8 *tmp_type; u8 cos_id; int rc; memset(&fw_msg, 0, sizeof(fw_msg)); bnxt_re_init_hwrm_hdr(rdev, (void *)&req, HWRM_QUEUE_QPORTCFG, -1, tid); req.port_id = (tid == 0xFFFF) ? en_dev->pf_port_id : 1; if (BNXT_EN_ASYM_Q(en_dev)) req.flags = htole32(HWRM_QUEUE_QPORTCFG_INPUT_FLAGS_PATH_RX); bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp, sizeof(resp), DFLT_HWRM_CMD_TIMEOUT); rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg); if (rc) return rc; if (!resp.max_configurable_queues) return -EINVAL; max_tc = resp.max_configurable_queues; tc_rec->max_tc = max_tc; if (resp.queue_cfg_info & HWRM_QUEUE_QPORTCFG_OUTPUT_QUEUE_CFG_INFO_USE_PROFILE_TYPE) tc_rec->serv_type_enabled = true; qptr = &resp.queue_id0; type_ptr0 = &resp.queue_id0_service_profile_type; type_ptr1 = &resp.queue_id1_service_profile_type; for (tc = 0; tc < max_tc; tc++) { tmp_type = tc ? type_ptr1 + (tc - 1) : type_ptr0; cos_id = *qptr++; /* RoCE CoS queue is the first cos queue. * For MP12 and MP17 order is 405 and 141015. */ if (is_bnxt_roce_queue(rdev, *qptr, *tmp_type)) { tc_rec->cos_id_roce = cos_id; tc_rec->tc_roce = tc; } else if (is_bnxt_cnp_queue(rdev, *qptr, *tmp_type)) { tc_rec->cos_id_cnp = cos_id; tc_rec->tc_cnp = tc; } else if (!def_init) { def_init = true; tc_rec->tc_def = tc; tc_rec->cos_id_def = cos_id; } qptr++; } return rc; } int bnxt_re_hwrm_cos2bw_qcfg(struct bnxt_re_dev *rdev, u16 target_id, struct bnxt_re_cos2bw_cfg *cfg) { struct bnxt_en_dev *en_dev = rdev->en_dev; struct hwrm_queue_cos2bw_qcfg_output resp; struct hwrm_queue_cos2bw_qcfg_input req = {0}; struct bnxt_fw_msg fw_msg; int rc, indx; void *data; memset(&fw_msg, 0, sizeof(fw_msg)); bnxt_re_init_hwrm_hdr(rdev, (void *)&req, HWRM_QUEUE_COS2BW_QCFG, -1, target_id); req.port_id = (target_id == 0xFFFF) ? en_dev->pf_port_id : 1; bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp, sizeof(resp), DFLT_HWRM_CMD_TIMEOUT); rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg); if (rc) return rc; data = &resp.queue_id0 + offsetof(struct bnxt_re_cos2bw_cfg, queue_id); for (indx = 0; indx < 8; indx++, data += (sizeof(cfg->cfg))) { memcpy(&cfg->cfg, data, sizeof(cfg->cfg)); if (indx == 0) cfg->queue_id = resp.queue_id0; cfg++; } return rc; } int bnxt_re_hwrm_cos2bw_cfg(struct bnxt_re_dev *rdev, u16 target_id, struct bnxt_re_cos2bw_cfg *cfg) { struct bnxt_en_dev *en_dev = rdev->en_dev; struct hwrm_queue_cos2bw_cfg_input req = {0}; struct hwrm_queue_cos2bw_cfg_output resp = {0}; struct bnxt_fw_msg fw_msg; void *data; int indx; int rc; memset(&fw_msg, 0, sizeof(fw_msg)); bnxt_re_init_hwrm_hdr(rdev, (void *)&req, HWRM_QUEUE_COS2BW_CFG, -1, target_id); req.port_id = (target_id == 0xFFFF) ? en_dev->pf_port_id : 1; /* Chimp wants enable bit to retain previous * config done by L2 driver */ for (indx = 0; indx < 8; indx++) { if (cfg[indx].queue_id < 40) { req.enables |= cpu_to_le32( HWRM_QUEUE_COS2BW_CFG_INPUT_ENABLES_COS_QUEUE_ID0_VALID << indx); } data = (char *)&req.unused_0 + indx * (sizeof(*cfg) - 4); memcpy(data, &cfg[indx].queue_id, sizeof(*cfg) - 4); if (indx == 0) { req.queue_id0 = cfg[0].queue_id; req.unused_0 = 0; } } memset(&resp, 0, sizeof(resp)); bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp, sizeof(resp), DFLT_HWRM_CMD_TIMEOUT); rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg); return rc; } int bnxt_re_host_pf_id_query(struct bnxt_re_dev *rdev, struct bnxt_qplib_query_fn_info *fn_info, u32 *pf_mask, u32 *first_pf) { struct hwrm_func_host_pf_ids_query_output resp = {0}; struct hwrm_func_host_pf_ids_query_input req; struct bnxt_en_dev *en_dev = rdev->en_dev; struct bnxt_fw_msg fw_msg; int rc; memset(&fw_msg, 0, sizeof(fw_msg)); memset(&req, 0, sizeof(req)); bnxt_re_init_hwrm_hdr(rdev, (void *)&req, HWRM_FUNC_HOST_PF_IDS_QUERY, -1, -1); /* To query the info from the host EPs */ switch (fn_info->host) { case HWRM_FUNC_HOST_PF_IDS_QUERY_INPUT_HOST_SOC: case HWRM_FUNC_HOST_PF_IDS_QUERY_INPUT_HOST_EP_0: case HWRM_FUNC_HOST_PF_IDS_QUERY_INPUT_HOST_EP_1: case HWRM_FUNC_HOST_PF_IDS_QUERY_INPUT_HOST_EP_2: case HWRM_FUNC_HOST_PF_IDS_QUERY_INPUT_HOST_EP_3: req.host = fn_info->host; break; default: req.host = HWRM_FUNC_HOST_PF_IDS_QUERY_INPUT_HOST_EP_0; break; } req.filter = fn_info->filter; if (req.filter > HWRM_FUNC_HOST_PF_IDS_QUERY_INPUT_FILTER_ROCE) req.filter = HWRM_FUNC_HOST_PF_IDS_QUERY_INPUT_FILTER_ALL; bnxt_re_fill_fw_msg(&fw_msg, (void *)&req, sizeof(req), (void *)&resp, sizeof(resp), DFLT_HWRM_CMD_TIMEOUT); rc = en_dev->en_ops->bnxt_send_fw_msg(en_dev, BNXT_ROCE_ULP, &fw_msg); *first_pf = le16_to_cpu(resp.first_pf_id); *pf_mask = le16_to_cpu(resp.pf_ordinal_mask); return rc; } static void bnxt_re_put_stats_ctx(struct bnxt_re_dev *rdev) { struct bnxt_qplib_ctx *hctx; struct bnxt_qplib_res *res; u16 tid = 0xffff; res = &rdev->qplib_res; hctx = res->hctx; if (test_and_clear_bit(BNXT_RE_FLAG_STATS_CTX_ALLOC, &rdev->flags)) { bnxt_re_net_stats_ctx_free(rdev, hctx->stats.fw_id, tid); bnxt_qplib_free_stat_mem(res, &hctx->stats); } } static void bnxt_re_put_stats2_ctx(struct bnxt_re_dev *rdev) { test_and_clear_bit(BNXT_RE_FLAG_STATS_CTX2_ALLOC, &rdev->flags); } static int bnxt_re_get_stats_ctx(struct bnxt_re_dev *rdev) { struct bnxt_qplib_ctx *hctx; struct bnxt_qplib_res *res; u16 tid = 0xffff; int rc; res = &rdev->qplib_res; hctx = res->hctx; rc = bnxt_qplib_alloc_stat_mem(res->pdev, rdev->chip_ctx, &hctx->stats); if (rc) return -ENOMEM; rc = bnxt_re_net_stats_ctx_alloc(rdev, tid); if (rc) goto free_stat_mem; set_bit(BNXT_RE_FLAG_STATS_CTX_ALLOC, &rdev->flags); return 0; free_stat_mem: bnxt_qplib_free_stat_mem(res, &hctx->stats); return rc; } static int bnxt_re_update_dev_attr(struct bnxt_re_dev *rdev) { int rc; rc = bnxt_qplib_get_dev_attr(&rdev->rcfw); if (rc) return rc; if (!bnxt_re_check_min_attr(rdev)) return -EINVAL; return 0; } static void bnxt_re_free_tbls(struct bnxt_re_dev *rdev) { bnxt_qplib_clear_tbls(&rdev->qplib_res); bnxt_qplib_free_tbls(&rdev->qplib_res); } static int bnxt_re_alloc_init_tbls(struct bnxt_re_dev *rdev) { struct bnxt_qplib_chip_ctx *chip_ctx = rdev->chip_ctx; u8 pppp_factor = 0; int rc; /* * TODO: Need a better mechanism for spreading of the * 512 extended PPP pages. For now, spreading it * based on port_count */ if (_is_chip_p7(chip_ctx) && chip_ctx->modes.db_push) pppp_factor = rdev->en_dev->port_count; rc = bnxt_qplib_alloc_tbls(&rdev->qplib_res, pppp_factor); if (rc) return rc; bnxt_qplib_init_tbls(&rdev->qplib_res); set_bit(BNXT_RE_FLAG_TBLS_ALLOCINIT, &rdev->flags); return 0; } static void bnxt_re_clean_nqs(struct bnxt_re_dev *rdev) { struct bnxt_qplib_nq *nq; int i; if (!rdev->nqr.max_init) return; for (i = (rdev->nqr.max_init - 1) ; i >= 0; i--) { nq = &rdev->nqr.nq[i]; bnxt_qplib_disable_nq(nq); bnxt_re_net_ring_free(rdev, nq->ring_id); bnxt_qplib_free_nq_mem(nq); } rdev->nqr.max_init = 0; } static int bnxt_re_setup_nqs(struct bnxt_re_dev *rdev) { struct bnxt_re_ring_attr rattr = {}; struct bnxt_qplib_nq *nq; int rc, i; int depth; u32 offt; u16 vec; mutex_init(&rdev->nqr.load_lock); /* * TODO: Optimize the depth based on the * number of NQs. */ depth = BNXT_QPLIB_NQE_MAX_CNT; for (i = 0; i < rdev->nqr.num_msix - 1; i++) { nq = &rdev->nqr.nq[i]; vec = rdev->nqr.msix_entries[i + 1].vector; offt = rdev->nqr.msix_entries[i + 1].db_offset; nq->hwq.max_elements = depth; rc = bnxt_qplib_alloc_nq_mem(&rdev->qplib_res, nq); if (rc) { dev_err(rdev_to_dev(rdev), "Failed to get mem for NQ %d, rc = 0x%x", i, rc); goto fail_mem; } rattr.dma_arr = nq->hwq.pbl[PBL_LVL_0].pg_map_arr; rattr.pages = nq->hwq.pbl[rdev->nqr.nq[i].hwq.level].pg_count; rattr.type = bnxt_re_get_rtype(rdev); rattr.mode = HWRM_RING_ALLOC_INPUT_INT_MODE_MSIX; rattr.depth = nq->hwq.max_elements - 1; rattr.lrid = rdev->nqr.msix_entries[i + 1].ring_idx; /* Set DBR pacing capability on the first NQ ring only */ if (!i && bnxt_qplib_dbr_pacing_ext_en(rdev->chip_ctx)) rattr.flags = HWRM_RING_ALLOC_INPUT_FLAGS_NQ_DBR_PACING; else rattr.flags = 0; rc = bnxt_re_net_ring_alloc(rdev, &rattr, &nq->ring_id); if (rc) { nq->ring_id = 0xffff; /* Invalid ring-id */ dev_err(rdev_to_dev(rdev), "Failed to get fw id for NQ %d, rc = 0x%x", i, rc); goto fail_ring; } rc = bnxt_qplib_enable_nq(nq, i, vec, offt, &bnxt_re_cqn_handler, &bnxt_re_srqn_handler); if (rc) { dev_err(rdev_to_dev(rdev), "Failed to enable NQ %d, rc = 0x%x", i, rc); goto fail_en; } } rdev->nqr.max_init = i; return 0; fail_en: /* *nq was i'th nq */ bnxt_re_net_ring_free(rdev, nq->ring_id); fail_ring: bnxt_qplib_free_nq_mem(nq); fail_mem: rdev->nqr.max_init = i; return rc; } static void bnxt_re_sysfs_destroy_file(struct bnxt_re_dev *rdev) { int i; for (i = 0; i < ARRAY_SIZE(bnxt_re_attributes); i++) device_remove_file(&rdev->ibdev.dev, bnxt_re_attributes[i]); } static int bnxt_re_sysfs_create_file(struct bnxt_re_dev *rdev) { int i, j, rc = 0; for (i = 0; i < ARRAY_SIZE(bnxt_re_attributes); i++) { rc = device_create_file(&rdev->ibdev.dev, bnxt_re_attributes[i]); if (rc) { dev_err(rdev_to_dev(rdev), "Failed to create IB sysfs with rc = 0x%x", rc); /* Must clean up all created device files */ for (j = 0; j < i; j++) device_remove_file(&rdev->ibdev.dev, bnxt_re_attributes[j]); clear_bit(BNXT_RE_FLAG_IBDEV_REGISTERED, &rdev->flags); ib_unregister_device(&rdev->ibdev); return 1; } } return 0; } /* worker thread for polling periodic events. Now used for QoS programming*/ static void bnxt_re_worker(struct work_struct *work) { struct bnxt_re_dev *rdev = container_of(work, struct bnxt_re_dev, worker.work); int rc; /* QoS is in 30s cadence for PFs*/ if (!rdev->is_virtfn && !rdev->worker_30s--) rdev->worker_30s = 30; /* Use trylock for bnxt_re_dev_lock as this can be * held for long time by debugfs show path while issuing * HWRMS. If the debugfs name update is not done in this * iteration, the driver will check for the same in the * next schedule of the worker i.e after 1 sec. */ if (mutex_trylock(&bnxt_re_dev_lock)) mutex_unlock(&bnxt_re_dev_lock); if (!rdev->stats.stats_query_sec) goto resched; if (test_bit(BNXT_RE_FLAG_ISSUE_CFA_FLOW_STATS, &rdev->flags) && (rdev->is_virtfn || !_is_ext_stats_supported(rdev->dev_attr->dev_cap_flags))) { if (!(rdev->stats.stats_query_counter++ % rdev->stats.stats_query_sec)) { rc = bnxt_re_get_qos_stats(rdev); if (rc && rc != -ENOMEM) clear_bit(BNXT_RE_FLAG_ISSUE_CFA_FLOW_STATS, &rdev->flags); } } resched: schedule_delayed_work(&rdev->worker, msecs_to_jiffies(1000)); } static int bnxt_re_alloc_dbr_sw_stats_mem(struct bnxt_re_dev *rdev) { if (!(rdev->dbr_drop_recov || rdev->dbr_pacing)) return 0; rdev->dbr_sw_stats = kzalloc(sizeof(*rdev->dbr_sw_stats), GFP_KERNEL); if (!rdev->dbr_sw_stats) return -ENOMEM; return 0; } static void bnxt_re_free_dbr_sw_stats_mem(struct bnxt_re_dev *rdev) { kfree(rdev->dbr_sw_stats); rdev->dbr_sw_stats = NULL; } static int bnxt_re_initialize_dbr_drop_recov(struct bnxt_re_dev *rdev) { rdev->dbr_drop_recov_wq = create_singlethread_workqueue("bnxt_re_dbr_drop_recov"); if (!rdev->dbr_drop_recov_wq) { dev_err(rdev_to_dev(rdev), "DBR Drop Revov wq alloc failed!"); return -EINVAL; } rdev->dbr_drop_recov = true; /* Enable configfs setting dbr_drop_recov by default*/ rdev->user_dbr_drop_recov = true; rdev->user_dbr_drop_recov_timeout = BNXT_RE_DBR_RECOV_USERLAND_TIMEOUT; return 0; } static void bnxt_re_deinitialize_dbr_drop_recov(struct bnxt_re_dev *rdev) { if (rdev->dbr_drop_recov_wq) { flush_workqueue(rdev->dbr_drop_recov_wq); destroy_workqueue(rdev->dbr_drop_recov_wq); rdev->dbr_drop_recov_wq = NULL; } rdev->dbr_drop_recov = false; } static int bnxt_re_initialize_dbr_pacing(struct bnxt_re_dev *rdev) { int rc; /* Allocate a page for app use */ rdev->dbr_page = (void *)__get_free_page(GFP_KERNEL); if (!rdev->dbr_page) { dev_err(rdev_to_dev(rdev), "DBR page allocation failed!"); return -ENOMEM; } memset((u8 *)rdev->dbr_page, 0, PAGE_SIZE); rdev->qplib_res.pacing_data = (struct bnxt_qplib_db_pacing_data *)rdev->dbr_page; rc = bnxt_re_hwrm_dbr_pacing_qcfg(rdev); if (rc) { dev_err(rdev_to_dev(rdev), "Failed to query dbr pacing config %d\n", rc); goto fail; } /* Create a work queue for scheduling dbq event */ rdev->dbq_wq = create_singlethread_workqueue("bnxt_re_dbq"); if (!rdev->dbq_wq) { dev_err(rdev_to_dev(rdev), "DBQ wq alloc failed!"); rc = -ENOMEM; goto fail; } /* MAP grc window 2 for reading db fifo depth */ writel_fbsd(rdev->en_dev->softc, BNXT_GRCPF_REG_WINDOW_BASE_OUT + 4, 0, rdev->chip_ctx->dbr_stat_db_fifo & BNXT_GRC_BASE_MASK); rdev->dbr_db_fifo_reg_off = (rdev->chip_ctx->dbr_stat_db_fifo & BNXT_GRC_OFFSET_MASK) + 0x2000; rdev->qplib_res.pacing_data->grc_reg_offset = rdev->dbr_db_fifo_reg_off; rdev->dbr_bar_addr = pci_resource_start(rdev->qplib_res.pdev, 0) + rdev->dbr_db_fifo_reg_off; /* Percentage of DB FIFO */ rdev->dbq_watermark = BNXT_RE_PACING_DBQ_THRESHOLD; rdev->pacing_en_int_th = BNXT_RE_PACING_EN_INT_THRESHOLD; rdev->pacing_algo_th = BNXT_RE_PACING_ALGO_THRESHOLD; rdev->dbq_pacing_time = BNXT_RE_DBR_INT_TIME; rdev->dbr_def_do_pacing = BNXT_RE_DBR_DO_PACING_NO_CONGESTION; rdev->do_pacing_save = rdev->dbr_def_do_pacing; bnxt_re_set_default_pacing_data(rdev); dev_dbg(rdev_to_dev(rdev), "Initialized db pacing\n"); return 0; fail: free_page((u64)rdev->dbr_page); rdev->dbr_page = NULL; return rc; } static void bnxt_re_deinitialize_dbr_pacing(struct bnxt_re_dev *rdev) { if (rdev->dbq_wq) flush_workqueue(rdev->dbq_wq); cancel_work_sync(&rdev->dbq_fifo_check_work); cancel_delayed_work_sync(&rdev->dbq_pacing_work); if (rdev->dbq_wq) { destroy_workqueue(rdev->dbq_wq); rdev->dbq_wq = NULL; } if (rdev->dbr_page) free_page((u64)rdev->dbr_page); rdev->dbr_page = NULL; rdev->dbr_pacing = false; } /* enable_dbr_pacing needs to be done only for older FWs * where host selects primary function. ie. pacing_ext * flags is not set. */ int bnxt_re_enable_dbr_pacing(struct bnxt_re_dev *rdev) { struct bnxt_qplib_nq *nq; nq = &rdev->nqr.nq[0]; rdev->dbq_nq_id = nq->ring_id; if (!bnxt_qplib_dbr_pacing_ext_en(rdev->chip_ctx) && bnxt_qplib_dbr_pacing_is_primary_pf(rdev->chip_ctx)) { if (bnxt_re_hwrm_dbr_pacing_cfg(rdev, true)) { dev_err(rdev_to_dev(rdev), "Failed to set dbr pacing config\n"); return -EIO; } /* MAP grc window 8 for ARMing the NQ DBQ */ writel_fbsd(rdev->en_dev->softc, BNXT_GRCPF_REG_WINDOW_BASE_OUT + 28 , 0, rdev->chip_ctx->dbr_aeq_arm_reg & BNXT_GRC_BASE_MASK); rdev->dbr_aeq_arm_reg_off = (rdev->chip_ctx->dbr_aeq_arm_reg & BNXT_GRC_OFFSET_MASK) + 0x8000; writel_fbsd(rdev->en_dev->softc, rdev->dbr_aeq_arm_reg_off , 0, 1); } return 0; } /* disable_dbr_pacing needs to be done only for older FWs * where host selects primary function. ie. pacing_ext * flags is not set. */ int bnxt_re_disable_dbr_pacing(struct bnxt_re_dev *rdev) { int rc = 0; if (!bnxt_qplib_dbr_pacing_ext_en(rdev->chip_ctx) && bnxt_qplib_dbr_pacing_is_primary_pf(rdev->chip_ctx)) rc = bnxt_re_hwrm_dbr_pacing_cfg(rdev, false); return rc; } static void bnxt_re_ib_uninit(struct bnxt_re_dev *rdev) { if (test_bit(BNXT_RE_FLAG_IBDEV_REGISTERED, &rdev->flags)) { bnxt_re_sysfs_destroy_file(rdev); /* Cleanup ib dev */ ib_unregister_device(&rdev->ibdev); clear_bit(BNXT_RE_FLAG_IBDEV_REGISTERED, &rdev->flags); return; } } static void bnxt_re_dev_uninit(struct bnxt_re_dev *rdev, u8 op_type) { struct bnxt_qplib_dpi *kdpi; int rc, wait_count = BNXT_RE_RES_FREE_WAIT_COUNT; bnxt_re_net_unregister_async_event(rdev); bnxt_re_put_stats2_ctx(rdev); if (test_and_clear_bit(BNXT_RE_FLAG_DEV_LIST_INITIALIZED, &rdev->flags)) { /* did the caller hold the lock? */ mutex_lock(&bnxt_re_dev_lock); list_del_rcu(&rdev->list); mutex_unlock(&bnxt_re_dev_lock); } bnxt_re_uninit_resolve_wq(rdev); bnxt_re_uninit_dcb_wq(rdev); bnxt_re_uninit_aer_wq(rdev); bnxt_re_deinitialize_dbr_drop_recov(rdev); if (bnxt_qplib_dbr_pacing_en(rdev->chip_ctx)) (void)bnxt_re_disable_dbr_pacing(rdev); if (test_and_clear_bit(BNXT_RE_FLAG_WORKER_REG, &rdev->flags)) { cancel_delayed_work_sync(&rdev->worker); } /* Wait for ULPs to release references */ while (atomic_read(&rdev->stats.rsors.cq_count) && --wait_count) usleep_range(500, 1000); if (!wait_count) dev_err(rdev_to_dev(rdev), "CQ resources not freed by stack, count = 0x%x", atomic_read(&rdev->stats.rsors.cq_count)); kdpi = &rdev->dpi_privileged; if (kdpi->umdbr) { /* kernel DPI was allocated with success */ (void)bnxt_qplib_dealloc_dpi(&rdev->qplib_res, kdpi); /* * Driver just need to know no command had failed * during driver load sequence and below command is * required indeed. Piggybacking dpi allocation status. */ } /* Protect the device uninitialization and start_irq/stop_irq L2 * callbacks with rtnl lock to avoid race condition between these calls */ rtnl_lock(); if (test_and_clear_bit(BNXT_RE_FLAG_SETUP_NQ, &rdev->flags)) bnxt_re_clean_nqs(rdev); rtnl_unlock(); if (test_and_clear_bit(BNXT_RE_FLAG_TBLS_ALLOCINIT, &rdev->flags)) bnxt_re_free_tbls(rdev); if (test_and_clear_bit(BNXT_RE_FLAG_RCFW_CHANNEL_INIT, &rdev->flags)) { rc = bnxt_qplib_deinit_rcfw(&rdev->rcfw); if (rc) dev_warn(rdev_to_dev(rdev), "Failed to deinitialize fw, rc = 0x%x", rc); } bnxt_re_put_stats_ctx(rdev); if (test_and_clear_bit(BNXT_RE_FLAG_ALLOC_CTX, &rdev->flags)) bnxt_qplib_free_hwctx(&rdev->qplib_res); rtnl_lock(); if (test_and_clear_bit(BNXT_RE_FLAG_RCFW_CHANNEL_EN, &rdev->flags)) bnxt_qplib_disable_rcfw_channel(&rdev->rcfw); if (rdev->dbr_pacing) bnxt_re_deinitialize_dbr_pacing(rdev); bnxt_re_free_dbr_sw_stats_mem(rdev); if (test_and_clear_bit(BNXT_RE_FLAG_NET_RING_ALLOC, &rdev->flags)) bnxt_re_net_ring_free(rdev, rdev->rcfw.creq.ring_id); if (test_and_clear_bit(BNXT_RE_FLAG_ALLOC_RCFW, &rdev->flags)) bnxt_qplib_free_rcfw_channel(&rdev->qplib_res); if (test_and_clear_bit(BNXT_RE_FLAG_GOT_MSIX, &rdev->flags)) bnxt_re_free_msix(rdev); rtnl_unlock(); bnxt_re_destroy_chip_ctx(rdev); if (op_type != BNXT_RE_PRE_RECOVERY_REMOVE) { if (test_and_clear_bit(BNXT_RE_FLAG_NETDEV_REGISTERED, &rdev->flags)) bnxt_re_unregister_netdev(rdev); } } static int bnxt_re_dev_init(struct bnxt_re_dev *rdev, u8 op_type, u8 wqe_mode) { struct bnxt_re_ring_attr rattr = {}; struct bnxt_qplib_creq_ctx *creq; int vec, offset; int rc = 0; if (op_type != BNXT_RE_POST_RECOVERY_INIT) { /* Registered a new RoCE device instance to netdev */ rc = bnxt_re_register_netdev(rdev); if (rc) return -EINVAL; } set_bit(BNXT_RE_FLAG_NETDEV_REGISTERED, &rdev->flags); rc = bnxt_re_setup_chip_ctx(rdev, wqe_mode); if (rc) { dev_err(rdev_to_dev(rdev), "Failed to get chip context rc 0x%x", rc); bnxt_re_unregister_netdev(rdev); clear_bit(BNXT_RE_FLAG_NETDEV_REGISTERED, &rdev->flags); rc = -EINVAL; return rc; } /* Protect the device initialization and start_irq/stop_irq L2 callbacks * with rtnl lock to avoid race condition between these calls */ rtnl_lock(); rc = bnxt_re_request_msix(rdev); if (rc) { dev_err(rdev_to_dev(rdev), "Requesting MSI-X vectors failed with rc = 0x%x", rc); rc = -EINVAL; goto release_rtnl; } set_bit(BNXT_RE_FLAG_GOT_MSIX, &rdev->flags); /* Establish RCFW Communication Channel to initialize the context memory for the function and all child VFs */ rc = bnxt_qplib_alloc_rcfw_channel(&rdev->qplib_res); if (rc) { dev_err(rdev_to_dev(rdev), "Failed to alloc mem for rcfw, rc = %#x\n", rc); goto release_rtnl; } set_bit(BNXT_RE_FLAG_ALLOC_RCFW, &rdev->flags); creq = &rdev->rcfw.creq; rattr.dma_arr = creq->hwq.pbl[PBL_LVL_0].pg_map_arr; rattr.pages = creq->hwq.pbl[creq->hwq.level].pg_count; rattr.type = bnxt_re_get_rtype(rdev); rattr.mode = HWRM_RING_ALLOC_INPUT_INT_MODE_MSIX; rattr.depth = BNXT_QPLIB_CREQE_MAX_CNT - 1; rattr.lrid = rdev->nqr.msix_entries[BNXT_RE_AEQ_IDX].ring_idx; rc = bnxt_re_net_ring_alloc(rdev, &rattr, &creq->ring_id); if (rc) { creq->ring_id = 0xffff; dev_err(rdev_to_dev(rdev), "Failed to allocate CREQ fw id with rc = 0x%x", rc); goto release_rtnl; } set_bit(BNXT_RE_FLAG_NET_RING_ALLOC, &rdev->flags); if (!rdev->chip_ctx) goto release_rtnl; if (!(_is_chip_p7(rdev->chip_ctx))) { /* Program the NQ ID for DBQ notification */ if (rdev->chip_ctx->modes.dbr_pacing_v0 || bnxt_qplib_dbr_pacing_en(rdev->chip_ctx) || bnxt_qplib_dbr_pacing_ext_en(rdev->chip_ctx)) { rc = bnxt_re_initialize_dbr_pacing(rdev); if (!rc) rdev->dbr_pacing = true; else rdev->dbr_pacing = false; dev_dbg(rdev_to_dev(rdev), "%s: initialize db pacing ret %d\n", __func__, rc); } } vec = rdev->nqr.msix_entries[BNXT_RE_AEQ_IDX].vector; offset = rdev->nqr.msix_entries[BNXT_RE_AEQ_IDX].db_offset; rc = bnxt_qplib_enable_rcfw_channel(&rdev->rcfw, vec, offset, &bnxt_re_aeq_handler); if (rc) { dev_err(rdev_to_dev(rdev), "Failed to enable RCFW channel with rc = 0x%x", rc); goto release_rtnl; } set_bit(BNXT_RE_FLAG_RCFW_CHANNEL_EN, &rdev->flags); rc = bnxt_re_update_dev_attr(rdev); if (rc) goto release_rtnl; bnxt_re_set_resource_limits(rdev); if (!rdev->is_virtfn && !_is_chip_gen_p5_p7(rdev->chip_ctx)) { rc = bnxt_qplib_alloc_hwctx(&rdev->qplib_res); if (rc) { dev_err(rdev_to_dev(rdev), "Failed to alloc hw contexts, rc = 0x%x", rc); goto release_rtnl; } set_bit(BNXT_RE_FLAG_ALLOC_CTX, &rdev->flags); } rc = bnxt_re_get_stats_ctx(rdev); if (rc) goto release_rtnl; rc = bnxt_qplib_init_rcfw(&rdev->rcfw, rdev->is_virtfn); if (rc) { dev_err(rdev_to_dev(rdev), "Failed to initialize fw with rc = 0x%x", rc); goto release_rtnl; } set_bit(BNXT_RE_FLAG_RCFW_CHANNEL_INIT, &rdev->flags); /* Based resource count on the 'new' device caps */ rc = bnxt_re_update_dev_attr(rdev); if (rc) goto release_rtnl; rc = bnxt_re_alloc_init_tbls(rdev); if (rc) { dev_err(rdev_to_dev(rdev), "tbls alloc-init failed rc = %#x", rc); goto release_rtnl; } rc = bnxt_re_setup_nqs(rdev); if (rc) { dev_err(rdev_to_dev(rdev), "NQs alloc-init failed rc = %#x\n", rc); if (rdev->nqr.max_init == 0) goto release_rtnl; dev_warn(rdev_to_dev(rdev), "expected nqs %d available nqs %d\n", rdev->nqr.num_msix, rdev->nqr.max_init); } set_bit(BNXT_RE_FLAG_SETUP_NQ, &rdev->flags); rtnl_unlock(); rc = bnxt_qplib_alloc_dpi(&rdev->qplib_res, &rdev->dpi_privileged, rdev, BNXT_QPLIB_DPI_TYPE_KERNEL); if (rc) goto fail; if (rdev->dbr_pacing) bnxt_re_enable_dbr_pacing(rdev); if (rdev->chip_ctx->modes.dbr_drop_recov) bnxt_re_initialize_dbr_drop_recov(rdev); rc = bnxt_re_alloc_dbr_sw_stats_mem(rdev); if (rc) goto fail; /* This block of code is needed for error recovery support */ if (!rdev->is_virtfn) { struct bnxt_re_tc_rec *tc_rec; tc_rec = &rdev->tc_rec[0]; rc = bnxt_re_query_hwrm_qportcfg(rdev, tc_rec, 0xFFFF); if (rc) { dev_err(rdev_to_dev(rdev), "Failed to query port config rc:%d", rc); return rc; } /* Query f/w defaults of CC params */ rc = bnxt_qplib_query_cc_param(&rdev->qplib_res, &rdev->cc_param); if (rc) dev_warn(rdev_to_dev(rdev), "Failed to query CC defaults\n"); if (1) { rdev->num_vfs = pci_num_vf(rdev->en_dev->pdev); if (rdev->num_vfs) { bnxt_re_set_resource_limits(rdev); bnxt_qplib_set_func_resources(&rdev->qplib_res); } } } INIT_DELAYED_WORK(&rdev->worker, bnxt_re_worker); set_bit(BNXT_RE_FLAG_WORKER_REG, &rdev->flags); schedule_delayed_work(&rdev->worker, msecs_to_jiffies(1000)); bnxt_re_init_dcb_wq(rdev); bnxt_re_init_aer_wq(rdev); bnxt_re_init_resolve_wq(rdev); mutex_lock(&bnxt_re_dev_lock); list_add_tail_rcu(&rdev->list, &bnxt_re_dev_list); /* Added to the list, not in progress anymore */ gadd_dev_inprogress--; set_bit(BNXT_RE_FLAG_DEV_LIST_INITIALIZED, &rdev->flags); mutex_unlock(&bnxt_re_dev_lock); return rc; release_rtnl: rtnl_unlock(); fail: bnxt_re_dev_uninit(rdev, BNXT_RE_COMPLETE_REMOVE); return rc; } static int bnxt_re_ib_init(struct bnxt_re_dev *rdev) { int rc = 0; rc = bnxt_re_register_ib(rdev); if (rc) { dev_err(rdev_to_dev(rdev), "Register IB failed with rc = 0x%x", rc); goto fail; } if (bnxt_re_sysfs_create_file(rdev)) { bnxt_re_stopqps_and_ib_uninit(rdev); goto fail; } set_bit(BNXT_RE_FLAG_IBDEV_REGISTERED, &rdev->flags); set_bit(BNXT_RE_FLAG_ISSUE_ROCE_STATS, &rdev->flags); set_bit(BNXT_RE_FLAG_ISSUE_CFA_FLOW_STATS, &rdev->flags); bnxt_re_dispatch_event(&rdev->ibdev, NULL, 1, IB_EVENT_PORT_ACTIVE); bnxt_re_dispatch_event(&rdev->ibdev, NULL, 1, IB_EVENT_GID_CHANGE); return rc; fail: bnxt_re_dev_uninit(rdev, BNXT_RE_COMPLETE_REMOVE); return rc; } /* wrapper for ib_init funcs */ int _bnxt_re_ib_init(struct bnxt_re_dev *rdev) { return bnxt_re_ib_init(rdev); } /* wrapper for aux init funcs */ int _bnxt_re_ib_init2(struct bnxt_re_dev *rdev) { bnxt_re_ib_init_2(rdev); return 0; /* add return for future proof */ } static void bnxt_re_dev_unreg(struct bnxt_re_dev *rdev) { bnxt_re_dev_dealloc(rdev); } static int bnxt_re_dev_reg(struct bnxt_re_dev **rdev, struct ifnet *netdev, struct bnxt_en_dev *en_dev) { struct ifnet *realdev = NULL; realdev = netdev; if (realdev) dev_dbg(NULL, "%s: realdev = %p netdev = %p\n", __func__, realdev, netdev); /* * Note: * The first argument to bnxt_re_dev_alloc() is 'netdev' and * not 'realdev', since in the case of bonding we want to * register the bonded virtual netdev (master) to the ib stack. * And 'en_dev' (for L2/PCI communication) is the first slave * device (PF0 on the card). * In the case of a regular netdev, both netdev and the en_dev * correspond to the same device. */ *rdev = bnxt_re_dev_alloc(netdev, en_dev); if (!*rdev) { pr_err("%s: netdev %p not handled", ROCE_DRV_MODULE_NAME, netdev); return -ENOMEM; } bnxt_re_hold(*rdev); return 0; } void bnxt_re_get_link_speed(struct bnxt_re_dev *rdev) { rdev->espeed = rdev->en_dev->espeed; rdev->lanes = rdev->en_dev->lanes; return; } void bnxt_re_stopqps_and_ib_uninit(struct bnxt_re_dev *rdev) { dev_dbg(rdev_to_dev(rdev), "%s: Stopping QPs, IB uninit on rdev: %p\n", __func__, rdev); bnxt_re_stop_all_nonqp1_nonshadow_qps(rdev); bnxt_re_ib_uninit(rdev); } void bnxt_re_remove_device(struct bnxt_re_dev *rdev, u8 op_type, struct auxiliary_device *aux_dev) { struct bnxt_re_en_dev_info *en_info; struct bnxt_qplib_cmdq_ctx *cmdq; struct bnxt_qplib_rcfw *rcfw; rcfw = &rdev->rcfw; cmdq = &rcfw->cmdq; if (test_bit(FIRMWARE_STALL_DETECTED, &cmdq->flags)) set_bit(BNXT_RE_FLAG_ERR_DEVICE_DETACHED, &rdev->flags); dev_dbg(rdev_to_dev(rdev), "%s: Removing rdev: %p\n", __func__, rdev); bnxt_re_dev_uninit(rdev, op_type); en_info = auxiliary_get_drvdata(aux_dev); if (en_info) { rtnl_lock(); en_info->rdev = NULL; rtnl_unlock(); if (op_type != BNXT_RE_PRE_RECOVERY_REMOVE) { clear_bit(BNXT_RE_FLAG_EN_DEV_PRIMARY_DEV, &en_info->flags); clear_bit(BNXT_RE_FLAG_EN_DEV_SECONDARY_DEV, &en_info->flags); clear_bit(BNXT_RE_FLAG_EN_DEV_NETDEV_REG, &en_info->flags); } } bnxt_re_dev_unreg(rdev); } int bnxt_re_add_device(struct bnxt_re_dev **rdev, struct ifnet *netdev, u8 qp_mode, u8 op_type, u8 wqe_mode, u32 num_msix_requested, struct auxiliary_device *aux_dev) { struct bnxt_re_en_dev_info *en_info; struct bnxt_en_dev *en_dev; int rc = 0; en_info = auxiliary_get_drvdata(aux_dev); en_dev = en_info->en_dev; mutex_lock(&bnxt_re_dev_lock); /* Check if driver already in mod exit and aux_dev is valid */ if (gmod_exit || !aux_dev) { mutex_unlock(&bnxt_re_dev_lock); return -ENODEV; } /* Add device in progress */ gadd_dev_inprogress++; mutex_unlock(&bnxt_re_dev_lock); rc = bnxt_re_dev_reg(rdev, netdev, en_dev); if (rc) { dev_dbg(NULL, "Failed to create add device for netdev %p\n", netdev); /* * For BNXT_RE_POST_RECOVERY_INIT special case * called from bnxt_re_start, the work is * complete only after, bnxt_re_start completes * bnxt_unregister_device in case of failure. * So bnxt_re_start will decrement gadd_dev_inprogress * in case of failure. */ if (op_type != BNXT_RE_POST_RECOVERY_INIT) { mutex_lock(&bnxt_re_dev_lock); gadd_dev_inprogress--; mutex_unlock(&bnxt_re_dev_lock); } return rc; } if (rc != 0) goto ref_error; /* * num_msix_requested = BNXT_RE_MSIX_FROM_MOD_PARAM indicates fresh driver load. * Otherwaise, this invocation can be the result of lag create / destroy, * err revovery, hot fw upgrade, etc.. */ if (num_msix_requested == BNXT_RE_MSIX_FROM_MOD_PARAM) { if (bnxt_re_probe_count < BNXT_RE_MAX_DEVICES) num_msix_requested = max_msix_vec[bnxt_re_probe_count++]; else /* Consider as default when probe_count exceeds its limit */ num_msix_requested = 0; /* if user specifies only one value, use the same for all PFs */ if (max_msix_vec_argc == 1) num_msix_requested = max_msix_vec[0]; } (*rdev)->num_msix_requested = num_msix_requested; (*rdev)->gsi_ctx.gsi_qp_mode = qp_mode; (*rdev)->adev = aux_dev; (*rdev)->dev_addr = en_dev->softc->func.mac_addr; /* Before updating the rdev pointer in bnxt_re_en_dev_info structure, * take the rtnl lock to avoid accessing invalid rdev pointer from * L2 ULP callbacks. This is applicable in all the places where rdev * pointer is updated in bnxt_re_en_dev_info. */ rtnl_lock(); en_info->rdev = *rdev; rtnl_unlock(); rc = bnxt_re_dev_init(*rdev, op_type, wqe_mode); if (rc) { ref_error: bnxt_re_dev_unreg(*rdev); *rdev = NULL; /* * For BNXT_RE_POST_RECOVERY_INIT special case * called from bnxt_re_start, the work is * complete only after, bnxt_re_start completes * bnxt_unregister_device in case of failure. * So bnxt_re_start will decrement gadd_dev_inprogress * in case of failure. */ if (op_type != BNXT_RE_POST_RECOVERY_INIT) { mutex_lock(&bnxt_re_dev_lock); gadd_dev_inprogress--; mutex_unlock(&bnxt_re_dev_lock); } } dev_dbg(rdev_to_dev(*rdev), "%s: Adding rdev: %p\n", __func__, *rdev); if (!rc) { set_bit(BNXT_RE_FLAG_EN_DEV_NETDEV_REG, &en_info->flags); } return rc; } struct bnxt_re_dev *bnxt_re_get_peer_pf(struct bnxt_re_dev *rdev) { struct pci_dev *pdev_in = rdev->en_dev->pdev; int tmp_bus_num, bus_num = pdev_in->bus->number; int tmp_dev_num, dev_num = PCI_SLOT(pdev_in->devfn); int tmp_func_num, func_num = PCI_FUNC(pdev_in->devfn); struct bnxt_re_dev *tmp_rdev; rcu_read_lock(); list_for_each_entry_rcu(tmp_rdev, &bnxt_re_dev_list, list) { tmp_bus_num = tmp_rdev->en_dev->pdev->bus->number; tmp_dev_num = PCI_SLOT(tmp_rdev->en_dev->pdev->devfn); tmp_func_num = PCI_FUNC(tmp_rdev->en_dev->pdev->devfn); if (bus_num == tmp_bus_num && dev_num == tmp_dev_num && func_num != tmp_func_num) { rcu_read_unlock(); return tmp_rdev; } } rcu_read_unlock(); return NULL; } int bnxt_re_schedule_work(struct bnxt_re_dev *rdev, unsigned long event, struct ifnet *vlan_dev, struct ifnet *netdev, struct auxiliary_device *adev) { struct bnxt_re_work *re_work; /* Allocate for the deferred task */ re_work = kzalloc(sizeof(*re_work), GFP_KERNEL); if (!re_work) return -ENOMEM; re_work->rdev = rdev; re_work->event = event; re_work->vlan_dev = vlan_dev; re_work->adev = adev; INIT_WORK(&re_work->work, bnxt_re_task); if (rdev) atomic_inc(&rdev->sched_count); re_work->netdev = netdev; queue_work(bnxt_re_wq, &re_work->work); return 0; } int bnxt_re_get_slot_pf_count(struct bnxt_re_dev *rdev) { struct pci_dev *pdev_in = rdev->en_dev->pdev; int tmp_bus_num, bus_num = pdev_in->bus->number; int tmp_dev_num, dev_num = PCI_SLOT(pdev_in->devfn); struct bnxt_re_dev *tmp_rdev; int pf_cnt = 0; rcu_read_lock(); list_for_each_entry_rcu(tmp_rdev, &bnxt_re_dev_list, list) { tmp_bus_num = tmp_rdev->en_dev->pdev->bus->number; tmp_dev_num = PCI_SLOT(tmp_rdev->en_dev->pdev->devfn); if (bus_num == tmp_bus_num && dev_num == tmp_dev_num) pf_cnt++; } rcu_read_unlock(); return pf_cnt; } /* Handle all deferred netevents tasks */ static void bnxt_re_task(struct work_struct *work) { struct bnxt_re_en_dev_info *en_info; struct auxiliary_device *aux_dev; struct bnxt_re_work *re_work; struct bnxt_re_dev *rdev; re_work = container_of(work, struct bnxt_re_work, work); mutex_lock(&bnxt_re_mutex); rdev = re_work->rdev; /* * If the previous rdev is deleted due to bond creation * do not handle the event */ if (!bnxt_re_is_rdev_valid(rdev)) goto exit; /* Ignore the event, if the device is not registred with IB stack. This * is to avoid handling any event while the device is added/removed. */ if (rdev && !test_bit(BNXT_RE_FLAG_IBDEV_REGISTERED, &rdev->flags)) { dev_dbg(rdev_to_dev(rdev), "%s: Ignoring netdev event 0x%lx", __func__, re_work->event); goto done; } /* Extra check to silence coverity. We shouldn't handle any event * when rdev is NULL. */ if (!rdev) goto exit; dev_dbg(rdev_to_dev(rdev), "Scheduled work for event 0x%lx", re_work->event); switch (re_work->event) { case NETDEV_UP: bnxt_re_dispatch_event(&rdev->ibdev, NULL, 1, IB_EVENT_PORT_ACTIVE); bnxt_re_net_register_async_event(rdev); break; case NETDEV_DOWN: bnxt_qplib_dbr_pacing_set_primary_pf(rdev->chip_ctx, 0); bnxt_re_stop_all_nonqp1_nonshadow_qps(rdev); bnxt_re_dispatch_event(&rdev->ibdev, NULL, 1, IB_EVENT_PORT_ERR); break; case NETDEV_CHANGE: if (bnxt_re_get_link_state(rdev) == IB_PORT_DOWN) { bnxt_re_stop_all_nonqp1_nonshadow_qps(rdev); bnxt_re_dispatch_event(&rdev->ibdev, NULL, 1, IB_EVENT_PORT_ERR); break; } else if (bnxt_re_get_link_state(rdev) == IB_PORT_ACTIVE) { bnxt_re_dispatch_event(&rdev->ibdev, NULL, 1, IB_EVENT_PORT_ACTIVE); } /* temporarily disable the check for SR2 */ if (!bnxt_qplib_query_cc_param(&rdev->qplib_res, &rdev->cc_param) && !_is_chip_p7(rdev->chip_ctx)) { /* * Disable CC for 10G speed * for non p5 devices */ if (rdev->sl_espeed == SPEED_10000 && !_is_chip_gen_p5_p7(rdev->chip_ctx)) { if (rdev->cc_param.enable) bnxt_re_clear_cc(rdev); } else { if (!rdev->cc_param.enable && rdev->cc_param.admin_enable) bnxt_re_setup_cc(rdev); } } break; case NETDEV_UNREGISTER: bnxt_re_stopqps_and_ib_uninit(rdev); aux_dev = rdev->adev; if (re_work->adev) goto done; bnxt_re_remove_device(rdev, BNXT_RE_COMPLETE_REMOVE, aux_dev); break; default: break; } done: if (rdev) { /* memory barrier to guarantee task completion * before decrementing sched count */ mmiowb(); atomic_dec(&rdev->sched_count); } exit: if (re_work->adev && re_work->event == NETDEV_UNREGISTER) { en_info = auxiliary_get_drvdata(re_work->adev); en_info->ib_uninit_done = true; wake_up(&en_info->waitq); } kfree(re_work); mutex_unlock(&bnxt_re_mutex); } /* "Notifier chain callback can be invoked for the same chain from different CPUs at the same time". For cases when the netdev is already present, our call to the register_netdevice_notifier() will actually get the rtnl_lock() before sending NETDEV_REGISTER and (if up) NETDEV_UP events. But for cases when the netdev is not already present, the notifier chain is subjected to be invoked from different CPUs simultaneously. This is protected by the netdev_mutex. */ static int bnxt_re_netdev_event(struct notifier_block *notifier, unsigned long event, void *ptr) { struct ifnet *real_dev, *netdev; struct bnxt_re_dev *rdev = NULL; netdev = netdev_notifier_info_to_ifp(ptr); real_dev = rdma_vlan_dev_real_dev(netdev); if (!real_dev) real_dev = netdev; /* In case of bonding,this will be bond's rdev */ rdev = bnxt_re_from_netdev(real_dev); if (!rdev) goto exit; dev_info(rdev_to_dev(rdev), "%s: Event = %s (0x%lx), rdev %s (real_dev %s)\n", __func__, bnxt_re_netevent(event), event, rdev ? rdev->netdev ? if_getdname(rdev->netdev) : "->netdev = NULL" : "= NULL", (real_dev == netdev) ? "= netdev" : if_getdname(real_dev)); if (!test_bit(BNXT_RE_FLAG_IBDEV_REGISTERED, &rdev->flags)) goto exit; bnxt_re_hold(rdev); if (real_dev != netdev) { switch (event) { case NETDEV_UP: bnxt_re_schedule_work(rdev, event, netdev, NULL, NULL); break; case NETDEV_DOWN: break; default: break; } goto done; } switch (event) { case NETDEV_CHANGEADDR: if (!_is_chip_gen_p5_p7(rdev->chip_ctx)) bnxt_re_update_shadow_ah(rdev); bnxt_qplib_get_guid(rdev->dev_addr, (u8 *)&rdev->ibdev.node_guid); break; case NETDEV_CHANGE: bnxt_re_get_link_speed(rdev); bnxt_re_schedule_work(rdev, event, NULL, NULL, NULL); break; case NETDEV_UNREGISTER: /* netdev notifier will call NETDEV_UNREGISTER again later since * we are still holding the reference to the netdev */ /* * Workaround to avoid ib_unregister hang. Check for module * reference and dont free up the device if the reference * is non zero. Checking only for PF functions. */ if (rdev) { dev_info(rdev_to_dev(rdev), "bnxt_re:Unreg recvd when module refcnt > 0"); dev_info(rdev_to_dev(rdev), "bnxt_re:Close all apps using bnxt_re devs"); dev_info(rdev_to_dev(rdev), "bnxt_re:Remove the configfs entry created for the device"); dev_info(rdev_to_dev(rdev), "bnxt_re:Refer documentation for details"); goto done; } if (atomic_read(&rdev->sched_count) > 0) goto done; if (!rdev->unreg_sched) { bnxt_re_schedule_work(rdev, NETDEV_UNREGISTER, NULL, NULL, NULL); rdev->unreg_sched = true; goto done; } break; default: break; } done: if (rdev) bnxt_re_put(rdev); exit: return NOTIFY_DONE; } static struct notifier_block bnxt_re_netdev_notifier = { .notifier_call = bnxt_re_netdev_event }; static void bnxt_re_remove_base_interface(struct bnxt_re_dev *rdev, struct auxiliary_device *adev) { bnxt_re_stopqps_and_ib_uninit(rdev); bnxt_re_remove_device(rdev, BNXT_RE_COMPLETE_REMOVE, adev); auxiliary_set_drvdata(adev, NULL); } /* * bnxt_re_remove - Removes the roce aux device * @adev - aux device pointer * * This function removes the roce device. This gets * called in the mod exit path and pci unbind path. * If the rdev is bond interace, destroys the lag * in module exit path, and in pci unbind case * destroys the lag and recreates other base interface. * If the device is already removed in error recovery * path, it just unregister with the L2. */ static void bnxt_re_remove(struct auxiliary_device *adev) { struct bnxt_re_en_dev_info *en_info = auxiliary_get_drvdata(adev); struct bnxt_en_dev *en_dev; struct bnxt_re_dev *rdev; bool primary_dev = false; bool secondary_dev = false; if (!en_info) return; mutex_lock(&bnxt_re_mutex); en_dev = en_info->en_dev; rdev = en_info->rdev; if (rdev && bnxt_re_is_rdev_valid(rdev)) { if (pci_channel_offline(rdev->rcfw.pdev)) set_bit(ERR_DEVICE_DETACHED, &rdev->rcfw.cmdq.flags); if (test_bit(BNXT_RE_FLAG_EN_DEV_PRIMARY_DEV, &en_info->flags)) primary_dev = true; if (test_bit(BNXT_RE_FLAG_EN_DEV_SECONDARY_DEV, &en_info->flags)) secondary_dev = true; /* * en_dev_info of primary device and secondary device have the * same rdev pointer when LAG is configured. This rdev pointer * is rdev of bond interface. */ if (!primary_dev && !secondary_dev) { /* removal of non bond interface */ bnxt_re_remove_base_interface(rdev, adev); } else { /* * removal of bond primary/secondary interface. In this * case bond device is already removed, so rdev->binfo * is NULL. */ auxiliary_set_drvdata(adev, NULL); } } else { /* device is removed from ulp stop, unregister the net dev */ if (test_bit(BNXT_RE_FLAG_EN_DEV_NETDEV_REG, &en_info->flags)) { rtnl_lock(); en_dev->en_ops->bnxt_unregister_device(en_dev, BNXT_ROCE_ULP); rtnl_unlock(); } } mutex_unlock(&bnxt_re_mutex); return; } /* wrapper for all external user context callers */ void _bnxt_re_remove(struct auxiliary_device *adev) { bnxt_re_remove(adev); } static void bnxt_re_ib_init_2(struct bnxt_re_dev *rdev) { int rc; rc = bnxt_re_get_device_stats(rdev); if (rc) dev_err(rdev_to_dev(rdev), "Failed initial device stat query"); bnxt_re_net_register_async_event(rdev); } static int bnxt_re_probe(struct auxiliary_device *adev, const struct auxiliary_device_id *id) { struct bnxt_aux_dev *aux_dev = container_of(adev, struct bnxt_aux_dev, aux_dev); struct bnxt_re_en_dev_info *en_info; struct bnxt_en_dev *en_dev = NULL; struct bnxt_re_dev *rdev; int rc = -ENODEV; if (aux_dev) en_dev = aux_dev->edev; if (!en_dev) return rc; if (en_dev->ulp_version != BNXT_ULP_VERSION) { pr_err("%s: probe error: bnxt_en ulp version magic %x is not compatible!\n", ROCE_DRV_MODULE_NAME, en_dev->ulp_version); return -EINVAL; } en_info = kzalloc(sizeof(*en_info), GFP_KERNEL); if (!en_info) return -ENOMEM; memset(en_info, 0, sizeof(struct bnxt_re_en_dev_info)); en_info->en_dev = en_dev; auxiliary_set_drvdata(adev, en_info); mutex_lock(&bnxt_re_mutex); rc = bnxt_re_add_device(&rdev, en_dev->net, BNXT_RE_GSI_MODE_ALL, BNXT_RE_COMPLETE_INIT, BNXT_QPLIB_WQE_MODE_STATIC, BNXT_RE_MSIX_FROM_MOD_PARAM, adev); if (rc) { mutex_unlock(&bnxt_re_mutex); return rc; } rc = bnxt_re_ib_init(rdev); if (rc) goto err; bnxt_re_ib_init_2(rdev); dev_dbg(rdev_to_dev(rdev), "%s: adev: %p\n", __func__, adev); rdev->adev = adev; mutex_unlock(&bnxt_re_mutex); return 0; err: mutex_unlock(&bnxt_re_mutex); bnxt_re_remove(adev); return rc; } static const struct auxiliary_device_id bnxt_re_id_table[] = { { .name = BNXT_ADEV_NAME ".rdma", }, {}, }; MODULE_DEVICE_TABLE(auxiliary, bnxt_re_id_table); static struct auxiliary_driver bnxt_re_driver = { .name = "rdma", .probe = bnxt_re_probe, .remove = bnxt_re_remove, .id_table = bnxt_re_id_table, }; static int __init bnxt_re_mod_init(void) { int rc = 0; pr_info("%s: %s", ROCE_DRV_MODULE_NAME, drv_version); bnxt_re_wq = create_singlethread_workqueue("bnxt_re"); if (!bnxt_re_wq) return -ENOMEM; rc = bnxt_re_register_netdevice_notifier(&bnxt_re_netdev_notifier); if (rc) { pr_err("%s: Cannot register to netdevice_notifier", ROCE_DRV_MODULE_NAME); goto err_netdev; } INIT_LIST_HEAD(&bnxt_re_dev_list); rc = auxiliary_driver_register(&bnxt_re_driver); if (rc) { pr_err("%s: Failed to register auxiliary driver\n", ROCE_DRV_MODULE_NAME); goto err_auxdrv; } return 0; err_auxdrv: bnxt_re_unregister_netdevice_notifier(&bnxt_re_netdev_notifier); err_netdev: destroy_workqueue(bnxt_re_wq); return rc; } static void __exit bnxt_re_mod_exit(void) { gmod_exit = 1; auxiliary_driver_unregister(&bnxt_re_driver); bnxt_re_unregister_netdevice_notifier(&bnxt_re_netdev_notifier); if (bnxt_re_wq) destroy_workqueue(bnxt_re_wq); } module_init(bnxt_re_mod_init); module_exit(bnxt_re_mod_exit);