diff --git a/sys/dev/iavf/iavf_lib.c b/sys/dev/iavf/iavf_lib.c index 3116ce0501c2..433d31904ea4 100644 --- a/sys/dev/iavf/iavf_lib.c +++ b/sys/dev/iavf/iavf_lib.c @@ -1,1527 +1,1502 @@ /* SPDX-License-Identifier: BSD-3-Clause */ /* Copyright (c) 2024, Intel Corporation * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * 3. Neither the name of the Intel Corporation nor the names of its * contributors may be used to endorse or promote products derived from * this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ /** * @file iavf_lib.c * @brief library code common to both legacy and iflib * * Contains functions common to the iflib and legacy drivers. Includes * hardware initialization and control functions, as well as sysctl handlers * for the sysctls which are shared between the legacy and iflib drivers. */ #include "iavf_iflib.h" #include "iavf_vc_common.h" static void iavf_init_hw(struct iavf_hw *hw, device_t dev); static u_int iavf_mc_filter_apply(void *arg, struct sockaddr_dl *sdl, u_int cnt); /** * iavf_msec_pause - Pause for at least the specified number of milliseconds * @msecs: number of milliseconds to pause for * * Pause execution of the current thread for a specified number of * milliseconds. Used to enforce minimum delay times when waiting for various * hardware events. */ void iavf_msec_pause(int msecs) { pause("iavf_msec_pause", MSEC_2_TICKS(msecs)); } /** * iavf_get_default_rss_key - Get the default RSS key for this driver * @key: output parameter to store the key in * * Copies the driver's default RSS key into the provided key variable. * * @pre assumes that key is not NULL and has at least IAVF_RSS_KEY_SIZE * storage space. */ void iavf_get_default_rss_key(u32 *key) { MPASS(key != NULL); u32 rss_seed[IAVF_RSS_KEY_SIZE_REG] = {0x41b01687, 0x183cfd8c, 0xce880440, 0x580cbc3c, 0x35897377, 0x328b25e1, 0x4fa98922, 0xb7d90c14, 0xd5bad70d, 0xcd15a2c1, 0x0, 0x0, 0x0}; bcopy(rss_seed, key, IAVF_RSS_KEY_SIZE); } /** * iavf_allocate_pci_resources_common - Allocate PCI resources * @sc: the private device softc pointer * * @pre sc->dev is set * * Allocates the common PCI resources used by the driver. * * @returns zero on success, or an error code on failure. */ int iavf_allocate_pci_resources_common(struct iavf_sc *sc) { struct iavf_hw *hw = &sc->hw; device_t dev = sc->dev; int rid; /* Map PCI BAR0 */ rid = PCIR_BAR(0); sc->pci_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, RF_ACTIVE); if (!(sc->pci_mem)) { device_printf(dev, "Unable to allocate bus resource: PCI memory\n"); return (ENXIO); } iavf_init_hw(hw, dev); /* Save off register access information */ sc->osdep.mem_bus_space_tag = rman_get_bustag(sc->pci_mem); sc->osdep.mem_bus_space_handle = rman_get_bushandle(sc->pci_mem); sc->osdep.mem_bus_space_size = rman_get_size(sc->pci_mem); sc->osdep.flush_reg = IAVF_VFGEN_RSTAT; sc->osdep.dev = dev; sc->hw.hw_addr = (u8 *)&sc->osdep.mem_bus_space_handle; sc->hw.back = &sc->osdep; return (0); } /** * iavf_init_hw - Initialize the device HW * @hw: device hardware structure * @dev: the stack device_t pointer * * Attach helper function. Gathers information about the (virtual) hardware * for use elsewhere in the driver. */ static void iavf_init_hw(struct iavf_hw *hw, device_t dev) { /* Save off the information about this board */ hw->vendor_id = pci_get_vendor(dev); hw->device_id = pci_get_device(dev); hw->revision_id = pci_read_config(dev, PCIR_REVID, 1); hw->subsystem_vendor_id = pci_read_config(dev, PCIR_SUBVEND_0, 2); hw->subsystem_device_id = pci_read_config(dev, PCIR_SUBDEV_0, 2); hw->bus.device = pci_get_slot(dev); hw->bus.func = pci_get_function(dev); } /** * iavf_sysctl_current_speed - Sysctl to display the current device speed * @oidp: syctl oid pointer * @arg1: pointer to the device softc typecasted to void * * @arg2: unused sysctl argument * @req: sysctl request structure * * Reads the current speed reported from the physical device into a string for * display by the current_speed sysctl. * * @returns zero or an error code on failure. */ int iavf_sysctl_current_speed(SYSCTL_HANDLER_ARGS) { struct iavf_sc *sc = (struct iavf_sc *)arg1; int error = 0; UNREFERENCED_PARAMETER(arg2); if (iavf_driver_is_detaching(sc)) return (ESHUTDOWN); if (IAVF_CAP_ADV_LINK_SPEED(sc)) error = sysctl_handle_string(oidp, __DECONST(char *, iavf_ext_speed_to_str(iavf_adv_speed_to_ext_speed(sc->link_speed_adv))), 8, req); else error = sysctl_handle_string(oidp, __DECONST(char *, iavf_vc_speed_to_string(sc->link_speed)), 8, req); return (error); } /** * iavf_reset_complete - Wait for a device reset to complete * @hw: pointer to the hardware structure * * Reads the reset registers and waits until they indicate that a device reset * is complete. * * @pre this function may call pause() and must not be called from a context * that cannot sleep. * * @returns zero on success, or EBUSY if it times out waiting for reset. */ int iavf_reset_complete(struct iavf_hw *hw) { u32 reg; /* Wait up to ~10 seconds */ for (int i = 0; i < 100; i++) { reg = rd32(hw, IAVF_VFGEN_RSTAT) & IAVF_VFGEN_RSTAT_VFR_STATE_MASK; if ((reg == VIRTCHNL_VFR_VFACTIVE) || (reg == VIRTCHNL_VFR_COMPLETED)) return (0); iavf_msec_pause(100); } return (EBUSY); } /** * iavf_setup_vc - Setup virtchnl communication * @sc: device private softc * * iavf_attach() helper function. Initializes the admin queue and attempts to * establish contact with the PF by retrying the initial "API version" message * several times or until the PF responds. * * @returns zero on success, or an error code on failure. */ int iavf_setup_vc(struct iavf_sc *sc) { struct iavf_hw *hw = &sc->hw; device_t dev = sc->dev; int error = 0, ret_error = 0, asq_retries = 0; bool send_api_ver_retried = 0; /* Need to set these AQ parameters before initializing AQ */ hw->aq.num_arq_entries = IAVF_AQ_LEN; hw->aq.num_asq_entries = IAVF_AQ_LEN; hw->aq.arq_buf_size = IAVF_AQ_BUF_SZ; hw->aq.asq_buf_size = IAVF_AQ_BUF_SZ; for (int i = 0; i < IAVF_AQ_MAX_ERR; i++) { /* Initialize admin queue */ error = iavf_init_adminq(hw); if (error) { device_printf(dev, "%s: init_adminq failed: %d\n", __func__, error); ret_error = 1; continue; } iavf_dbg_init(sc, "Initialized Admin Queue; starting" " send_api_ver attempt %d", i+1); retry_send: /* Send VF's API version */ error = iavf_send_api_ver(sc); if (error) { iavf_shutdown_adminq(hw); ret_error = 2; device_printf(dev, "%s: unable to send api" " version to PF on attempt %d, error %d\n", __func__, i+1, error); } asq_retries = 0; while (!iavf_asq_done(hw)) { if (++asq_retries > IAVF_AQ_MAX_ERR) { iavf_shutdown_adminq(hw); device_printf(dev, "Admin Queue timeout " "(waiting for send_api_ver), %d more tries...\n", IAVF_AQ_MAX_ERR - (i + 1)); ret_error = 3; break; } iavf_msec_pause(10); } if (asq_retries > IAVF_AQ_MAX_ERR) continue; iavf_dbg_init(sc, "Sent API version message to PF"); /* Verify that the VF accepts the PF's API version */ error = iavf_verify_api_ver(sc); if (error == ETIMEDOUT) { if (!send_api_ver_retried) { /* Resend message, one more time */ send_api_ver_retried = true; device_printf(dev, "%s: Timeout while verifying API version on first" " try!\n", __func__); goto retry_send; } else { device_printf(dev, "%s: Timeout while verifying API version on second" " try!\n", __func__); ret_error = 4; break; } } if (error) { device_printf(dev, "%s: Unable to verify API version," " error %d\n", __func__, error); ret_error = 5; } break; } if (ret_error >= 4) iavf_shutdown_adminq(hw); return (ret_error); } /** * iavf_reset - Requests a VF reset from the PF. * @sc: device private softc * * @pre Requires the VF's Admin Queue to be initialized. * @returns zero on success, or an error code on failure. */ int iavf_reset(struct iavf_sc *sc) { struct iavf_hw *hw = &sc->hw; device_t dev = sc->dev; int error = 0; /* Ask the PF to reset us if we are initiating */ if (!iavf_test_state(&sc->state, IAVF_STATE_RESET_PENDING)) iavf_request_reset(sc); iavf_msec_pause(100); error = iavf_reset_complete(hw); if (error) { device_printf(dev, "%s: VF reset failed\n", __func__); return (error); } pci_enable_busmaster(dev); error = iavf_shutdown_adminq(hw); if (error) { device_printf(dev, "%s: shutdown_adminq failed: %d\n", __func__, error); return (error); } error = iavf_init_adminq(hw); if (error) { device_printf(dev, "%s: init_adminq failed: %d\n", __func__, error); return (error); } /* IFLIB: This is called only in the iflib driver */ iavf_enable_adminq_irq(hw); return (0); } /** * iavf_enable_admin_irq - Enable the administrative interrupt * @hw: pointer to the hardware structure * * Writes to registers to enable the administrative interrupt cause, in order * to handle non-queue related interrupt events. */ void iavf_enable_adminq_irq(struct iavf_hw *hw) { wr32(hw, IAVF_VFINT_DYN_CTL01, IAVF_VFINT_DYN_CTL01_INTENA_MASK | IAVF_VFINT_DYN_CTL01_CLEARPBA_MASK | IAVF_VFINT_DYN_CTL01_ITR_INDX_MASK); wr32(hw, IAVF_VFINT_ICR0_ENA1, IAVF_VFINT_ICR0_ENA1_ADMINQ_MASK); /* flush */ rd32(hw, IAVF_VFGEN_RSTAT); } /** * iavf_disable_admin_irq - Disable the administrative interrupt cause * @hw: pointer to the hardware structure * * Writes to registers to disable the administrative interrupt cause. */ void iavf_disable_adminq_irq(struct iavf_hw *hw) { wr32(hw, IAVF_VFINT_DYN_CTL01, 0); wr32(hw, IAVF_VFINT_ICR0_ENA1, 0); iavf_flush(hw); } /** * iavf_vf_config - Configure this VF over the virtchnl * @sc: device private softc * * iavf_attach() helper function. Asks the PF for this VF's configuration, and * saves the information if it receives it. * * @returns zero on success, or an error code on failure. */ int iavf_vf_config(struct iavf_sc *sc) { struct iavf_hw *hw = &sc->hw; device_t dev = sc->dev; int bufsz, error = 0, ret_error = 0; int asq_retries, retried = 0; retry_config: error = iavf_send_vf_config_msg(sc); if (error) { device_printf(dev, "%s: Unable to send VF config request, attempt %d," " error %d\n", __func__, retried + 1, error); ret_error = 2; } asq_retries = 0; while (!iavf_asq_done(hw)) { if (++asq_retries > IAVF_AQ_MAX_ERR) { device_printf(dev, "%s: Admin Queue timeout " "(waiting for send_vf_config_msg), attempt %d\n", __func__, retried + 1); ret_error = 3; goto fail; } iavf_msec_pause(10); } iavf_dbg_init(sc, "Sent VF config message to PF, attempt %d\n", retried + 1); if (!sc->vf_res) { bufsz = sizeof(struct virtchnl_vf_resource) + (IAVF_MAX_VF_VSI * sizeof(struct virtchnl_vsi_resource)); sc->vf_res = (struct virtchnl_vf_resource *)malloc(bufsz, M_IAVF, M_NOWAIT); if (!sc->vf_res) { device_printf(dev, "%s: Unable to allocate memory for VF configuration" " message from PF on attempt %d\n", __func__, retried + 1); ret_error = 1; goto fail; } } /* Check for VF config response */ error = iavf_get_vf_config(sc); if (error == ETIMEDOUT) { /* The 1st time we timeout, send the configuration message again */ if (!retried) { retried++; goto retry_config; } device_printf(dev, "%s: iavf_get_vf_config() timed out waiting for a response\n", __func__); } if (error) { device_printf(dev, "%s: Unable to get VF configuration from PF after %d tries!\n", __func__, retried + 1); ret_error = 4; } goto done; fail: free(sc->vf_res, M_IAVF); done: return (ret_error); } /** * iavf_print_device_info - Print some device parameters at attach * @sc: device private softc * * Log a message about this virtual device's capabilities at attach time. */ void iavf_print_device_info(struct iavf_sc *sc) { device_t dev = sc->dev; device_printf(dev, "VSIs %d, QPs %d, MSI-X %d, RSS sizes: key %d lut %d\n", sc->vf_res->num_vsis, sc->vf_res->num_queue_pairs, sc->vf_res->max_vectors, sc->vf_res->rss_key_size, sc->vf_res->rss_lut_size); iavf_dbg_info(sc, "Capabilities=%b\n", sc->vf_res->vf_cap_flags, IAVF_PRINTF_VF_OFFLOAD_FLAGS); } /** * iavf_get_vsi_res_from_vf_res - Get VSI parameters and info for this VF * @sc: device private softc * * Get the VSI parameters and information from the general VF resource info * received by the physical device. * * @returns zero on success, or an error code on failure. */ int iavf_get_vsi_res_from_vf_res(struct iavf_sc *sc) { struct iavf_vsi *vsi = &sc->vsi; device_t dev = sc->dev; sc->vsi_res = NULL; for (int i = 0; i < sc->vf_res->num_vsis; i++) { /* XXX: We only use the first VSI we find */ if (sc->vf_res->vsi_res[i].vsi_type == VIRTCHNL_VSI_SRIOV) sc->vsi_res = &sc->vf_res->vsi_res[i]; } if (!sc->vsi_res) { device_printf(dev, "%s: no LAN VSI found\n", __func__); return (EIO); } vsi->id = sc->vsi_res->vsi_id; return (0); } /** * iavf_set_mac_addresses - Set the MAC address for this interface * @sc: device private softc * * Set the permanent MAC address field in the HW structure. If a MAC address * has not yet been set for this device by the physical function, generate one * randomly. */ void iavf_set_mac_addresses(struct iavf_sc *sc) { struct iavf_hw *hw = &sc->hw; device_t dev = sc->dev; u8 addr[ETHER_ADDR_LEN]; /* If no mac address was assigned just make a random one */ if (ETHER_IS_ZERO(hw->mac.addr)) { arc4rand(&addr, sizeof(addr), 0); addr[0] &= 0xFE; addr[0] |= 0x02; memcpy(hw->mac.addr, addr, sizeof(addr)); device_printf(dev, "Generated random MAC address\n"); } memcpy(hw->mac.perm_addr, hw->mac.addr, ETHER_ADDR_LEN); } /** * iavf_init_filters - Initialize filter structures * @sc: device private softc * * Initialize the MAC and VLAN filter list heads. * * @remark this is intended to be called only once during the device attach * process. * * @pre Because it uses M_WAITOK, this function should only be called in * a context that is safe to sleep. */ void iavf_init_filters(struct iavf_sc *sc) { sc->mac_filters = (struct mac_list *)malloc(sizeof(struct iavf_mac_filter), M_IAVF, M_WAITOK | M_ZERO); SLIST_INIT(sc->mac_filters); sc->vlan_filters = (struct vlan_list *)malloc(sizeof(struct iavf_vlan_filter), M_IAVF, M_WAITOK | M_ZERO); SLIST_INIT(sc->vlan_filters); } /** * iavf_free_filters - Release filter lists * @sc: device private softc * * Free the MAC and VLAN filter lists. * * @remark this is intended to be called only once during the device detach * process. */ void iavf_free_filters(struct iavf_sc *sc) { struct iavf_mac_filter *f; struct iavf_vlan_filter *v; while (!SLIST_EMPTY(sc->mac_filters)) { f = SLIST_FIRST(sc->mac_filters); SLIST_REMOVE_HEAD(sc->mac_filters, next); free(f, M_IAVF); } free(sc->mac_filters, M_IAVF); while (!SLIST_EMPTY(sc->vlan_filters)) { v = SLIST_FIRST(sc->vlan_filters); SLIST_REMOVE_HEAD(sc->vlan_filters, next); free(v, M_IAVF); } free(sc->vlan_filters, M_IAVF); } /** * iavf_add_device_sysctls_common - Initialize common device sysctls * @sc: device private softc * * Setup sysctls common to both the iflib and legacy drivers. */ void iavf_add_device_sysctls_common(struct iavf_sc *sc) { device_t dev = sc->dev; struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(dev); struct sysctl_oid_list *ctx_list = SYSCTL_CHILDREN(device_get_sysctl_tree(dev)); SYSCTL_ADD_PROC(ctx, ctx_list, OID_AUTO, "current_speed", CTLTYPE_STRING | CTLFLAG_RD, sc, 0, iavf_sysctl_current_speed, "A", "Current Port Speed"); SYSCTL_ADD_PROC(ctx, ctx_list, OID_AUTO, "tx_itr", CTLTYPE_INT | CTLFLAG_RW, sc, 0, iavf_sysctl_tx_itr, "I", "Immediately set TX ITR value for all queues"); SYSCTL_ADD_PROC(ctx, ctx_list, OID_AUTO, "rx_itr", CTLTYPE_INT | CTLFLAG_RW, sc, 0, iavf_sysctl_rx_itr, "I", "Immediately set RX ITR value for all queues"); SYSCTL_ADD_UQUAD(ctx, ctx_list, OID_AUTO, "admin_irq", CTLFLAG_RD, &sc->admin_irq, "Admin Queue IRQ Handled"); } /** * iavf_add_debug_sysctls_common - Initialize common debug sysctls * @sc: device private softc * @debug_list: pionter to debug sysctl node * * Setup sysctls used for debugging the device driver into the debug sysctl * node. */ void iavf_add_debug_sysctls_common(struct iavf_sc *sc, struct sysctl_oid_list *debug_list) { device_t dev = sc->dev; struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(dev); SYSCTL_ADD_UINT(ctx, debug_list, OID_AUTO, "shared_debug_mask", CTLFLAG_RW, &sc->hw.debug_mask, 0, "Shared code debug message level"); SYSCTL_ADD_UINT(ctx, debug_list, OID_AUTO, "core_debug_mask", CTLFLAG_RW, (unsigned int *)&sc->dbg_mask, 0, "Non-shared code debug message level"); SYSCTL_ADD_PROC(ctx, debug_list, OID_AUTO, "filter_list", CTLTYPE_STRING | CTLFLAG_RD, sc, 0, iavf_sysctl_sw_filter_list, "A", "SW Filter List"); } /** * iavf_sysctl_tx_itr - Sysctl to set the Tx ITR value * @oidp: sysctl oid pointer * @arg1: pointer to the device softc * @arg2: unused sysctl argument * @req: sysctl req pointer * * On read, returns the Tx ITR value for all of the VF queues. On write, * update the Tx ITR registers with the new Tx ITR value. * * @returns zero on success, or an error code on failure. */ int iavf_sysctl_tx_itr(SYSCTL_HANDLER_ARGS) { struct iavf_sc *sc = (struct iavf_sc *)arg1; device_t dev = sc->dev; int requested_tx_itr; int error = 0; UNREFERENCED_PARAMETER(arg2); if (iavf_driver_is_detaching(sc)) return (ESHUTDOWN); requested_tx_itr = sc->tx_itr; error = sysctl_handle_int(oidp, &requested_tx_itr, 0, req); if ((error) || (req->newptr == NULL)) return (error); if (requested_tx_itr < 0 || requested_tx_itr > IAVF_MAX_ITR) { device_printf(dev, "Invalid TX itr value; value must be between 0 and %d\n", IAVF_MAX_ITR); return (EINVAL); } sc->tx_itr = requested_tx_itr; iavf_configure_tx_itr(sc); return (error); } /** * iavf_sysctl_rx_itr - Sysctl to set the Rx ITR value * @oidp: sysctl oid pointer * @arg1: pointer to the device softc * @arg2: unused sysctl argument * @req: sysctl req pointer * * On read, returns the Rx ITR value for all of the VF queues. On write, * update the ITR registers with the new Rx ITR value. * * @returns zero on success, or an error code on failure. */ int iavf_sysctl_rx_itr(SYSCTL_HANDLER_ARGS) { struct iavf_sc *sc = (struct iavf_sc *)arg1; device_t dev = sc->dev; int requested_rx_itr; int error = 0; UNREFERENCED_PARAMETER(arg2); if (iavf_driver_is_detaching(sc)) return (ESHUTDOWN); requested_rx_itr = sc->rx_itr; error = sysctl_handle_int(oidp, &requested_rx_itr, 0, req); if ((error) || (req->newptr == NULL)) return (error); if (requested_rx_itr < 0 || requested_rx_itr > IAVF_MAX_ITR) { device_printf(dev, "Invalid RX itr value; value must be between 0 and %d\n", IAVF_MAX_ITR); return (EINVAL); } sc->rx_itr = requested_rx_itr; iavf_configure_rx_itr(sc); return (error); } /** * iavf_configure_tx_itr - Configure the Tx ITR * @sc: device private softc * * Updates the ITR registers with a new Tx ITR setting. */ void iavf_configure_tx_itr(struct iavf_sc *sc) { struct iavf_hw *hw = &sc->hw; struct iavf_vsi *vsi = &sc->vsi; struct iavf_tx_queue *que = vsi->tx_queues; vsi->tx_itr_setting = sc->tx_itr; for (int i = 0; i < IAVF_NTXQS(vsi); i++, que++) { struct tx_ring *txr = &que->txr; wr32(hw, IAVF_VFINT_ITRN1(IAVF_TX_ITR, i), vsi->tx_itr_setting); txr->itr = vsi->tx_itr_setting; txr->latency = IAVF_AVE_LATENCY; } } /** * iavf_configure_rx_itr - Configure the Rx ITR * @sc: device private softc * * Updates the ITR registers with a new Rx ITR setting. */ void iavf_configure_rx_itr(struct iavf_sc *sc) { struct iavf_hw *hw = &sc->hw; struct iavf_vsi *vsi = &sc->vsi; struct iavf_rx_queue *que = vsi->rx_queues; vsi->rx_itr_setting = sc->rx_itr; for (int i = 0; i < IAVF_NRXQS(vsi); i++, que++) { struct rx_ring *rxr = &que->rxr; wr32(hw, IAVF_VFINT_ITRN1(IAVF_RX_ITR, i), vsi->rx_itr_setting); rxr->itr = vsi->rx_itr_setting; rxr->latency = IAVF_AVE_LATENCY; } } /** * iavf_create_debug_sysctl_tree - Create a debug sysctl node * @sc: device private softc * * Create a sysctl node meant to hold sysctls used to print debug information. * Mark it as CTLFLAG_SKIP so that these sysctls do not show up in the * "sysctl -a" output. * * @returns a pointer to the created sysctl node. */ struct sysctl_oid_list * iavf_create_debug_sysctl_tree(struct iavf_sc *sc) { device_t dev = sc->dev; struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(dev); struct sysctl_oid_list *ctx_list = SYSCTL_CHILDREN(device_get_sysctl_tree(dev)); struct sysctl_oid *debug_node; debug_node = SYSCTL_ADD_NODE(ctx, ctx_list, OID_AUTO, "debug", CTLFLAG_RD | CTLFLAG_SKIP, NULL, "Debug Sysctls"); return (SYSCTL_CHILDREN(debug_node)); } /** * iavf_add_vsi_sysctls - Add sysctls for a given VSI * @dev: device pointer * @vsi: pointer to the VSI * @ctx: sysctl context to add to * @sysctl_name: name of the sysctl node (containing the VSI number) * * Adds a new sysctl node for holding specific sysctls for the given VSI. */ void iavf_add_vsi_sysctls(device_t dev, struct iavf_vsi *vsi, struct sysctl_ctx_list *ctx, const char *sysctl_name) { struct sysctl_oid *tree; struct sysctl_oid_list *child; struct sysctl_oid_list *vsi_list; tree = device_get_sysctl_tree(dev); child = SYSCTL_CHILDREN(tree); vsi->vsi_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, sysctl_name, CTLFLAG_RD, NULL, "VSI Number"); vsi_list = SYSCTL_CHILDREN(vsi->vsi_node); iavf_add_sysctls_eth_stats(ctx, vsi_list, &vsi->eth_stats); } /** * iavf_sysctl_sw_filter_list - Dump software filters * @oidp: sysctl oid pointer * @arg1: pointer to the device softc * @arg2: unused sysctl argument * @req: sysctl req pointer * * On read, generates a string which lists the MAC and VLAN filters added to * this virtual device. Useful for debugging to see whether or not the * expected filters have been configured by software. * * @returns zero on success, or an error code on failure. */ int iavf_sysctl_sw_filter_list(SYSCTL_HANDLER_ARGS) { struct iavf_sc *sc = (struct iavf_sc *)arg1; struct iavf_mac_filter *f; struct iavf_vlan_filter *v; device_t dev = sc->dev; int ftl_len, ftl_counter = 0, error = 0; struct sbuf *buf; UNREFERENCED_2PARAMETER(arg2, oidp); if (iavf_driver_is_detaching(sc)) return (ESHUTDOWN); buf = sbuf_new_for_sysctl(NULL, NULL, 128, req); if (!buf) { device_printf(dev, "Could not allocate sbuf for output.\n"); return (ENOMEM); } sbuf_printf(buf, "\n"); /* Print MAC filters */ sbuf_printf(buf, "MAC Filters:\n"); ftl_len = 0; SLIST_FOREACH(f, sc->mac_filters, next) ftl_len++; if (ftl_len < 1) sbuf_printf(buf, "(none)\n"); else { SLIST_FOREACH(f, sc->mac_filters, next) { sbuf_printf(buf, MAC_FORMAT ", flags %#06x\n", MAC_FORMAT_ARGS(f->macaddr), f->flags); } } /* Print VLAN filters */ sbuf_printf(buf, "VLAN Filters:\n"); ftl_len = 0; SLIST_FOREACH(v, sc->vlan_filters, next) ftl_len++; if (ftl_len < 1) sbuf_printf(buf, "(none)"); else { SLIST_FOREACH(v, sc->vlan_filters, next) { sbuf_printf(buf, "%d, flags %#06x", v->vlan, v->flags); /* don't print '\n' for last entry */ if (++ftl_counter != ftl_len) sbuf_printf(buf, "\n"); } } error = sbuf_finish(buf); if (error) device_printf(dev, "Error finishing sbuf: %d\n", error); sbuf_delete(buf); return (error); } /** * iavf_media_status_common - Get media status for this device * @sc: device softc pointer * @ifmr: ifmedia request structure * * Report the media status for this device into the given ifmr structure. */ void iavf_media_status_common(struct iavf_sc *sc, struct ifmediareq *ifmr) { enum iavf_ext_link_speed ext_speed; iavf_update_link_status(sc); ifmr->ifm_status = IFM_AVALID; ifmr->ifm_active = IFM_ETHER; if (!sc->link_up) return; ifmr->ifm_status |= IFM_ACTIVE; /* Hardware is always full-duplex */ ifmr->ifm_active |= IFM_FDX; /* Based on the link speed reported by the PF over the AdminQ, choose a * PHY type to report. This isn't 100% correct since we don't really * know the underlying PHY type of the PF, but at least we can report * a valid link speed... */ if (IAVF_CAP_ADV_LINK_SPEED(sc)) ext_speed = iavf_adv_speed_to_ext_speed(sc->link_speed_adv); else ext_speed = iavf_vc_speed_to_ext_speed(sc->link_speed); ifmr->ifm_active |= iavf_ext_speed_to_ifmedia(ext_speed); } /** * iavf_media_change_common - Change the media type for this device * @ifp: ifnet structure * * @returns ENODEV because changing the media and speed is not supported. */ int iavf_media_change_common(if_t ifp) { if_printf(ifp, "Changing speed is not supported\n"); return (ENODEV); } /** * iavf_set_initial_baudrate - Set the initial device baudrate * @ifp: ifnet structure * * Set the baudrate for this ifnet structure to the expected initial value of * 40Gbps. This maybe updated to a lower baudrate after the physical function * reports speed to us over the virtchnl interface. */ void iavf_set_initial_baudrate(if_t ifp) { if_setbaudrate(ifp, IF_Gbps(40)); } /** * iavf_add_sysctls_eth_stats - Add ethernet statistics sysctls * @ctx: the sysctl ctx to add to * @child: the node to add the sysctls to * @eth_stats: ethernet stats structure * * Creates sysctls that report the values of the provided ethernet stats * structure. */ void iavf_add_sysctls_eth_stats(struct sysctl_ctx_list *ctx, struct sysctl_oid_list *child, struct iavf_eth_stats *eth_stats) { struct iavf_sysctl_info ctls[] = { {ð_stats->rx_bytes, "good_octets_rcvd", "Good Octets Received"}, {ð_stats->rx_unicast, "ucast_pkts_rcvd", "Unicast Packets Received"}, {ð_stats->rx_multicast, "mcast_pkts_rcvd", "Multicast Packets Received"}, {ð_stats->rx_broadcast, "bcast_pkts_rcvd", "Broadcast Packets Received"}, {ð_stats->rx_discards, "rx_discards", "Discarded RX packets"}, {ð_stats->rx_unknown_protocol, "rx_unknown_proto", "RX unknown protocol packets"}, {ð_stats->tx_bytes, "good_octets_txd", "Good Octets Transmitted"}, {ð_stats->tx_unicast, "ucast_pkts_txd", "Unicast Packets Transmitted"}, {ð_stats->tx_multicast, "mcast_pkts_txd", "Multicast Packets Transmitted"}, {ð_stats->tx_broadcast, "bcast_pkts_txd", "Broadcast Packets Transmitted"}, {ð_stats->tx_errors, "tx_errors", "TX packet errors"}, // end {0,0,0} }; struct iavf_sysctl_info *entry = ctls; while (entry->stat != 0) { SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, entry->name, CTLFLAG_RD, entry->stat, entry->description); entry++; } } /** * iavf_max_vc_speed_to_value - Convert link speed to IF speed value * @link_speeds: bitmap of supported link speeds * * @returns the link speed value for the highest speed reported in the * link_speeds bitmap. */ u64 iavf_max_vc_speed_to_value(u8 link_speeds) { if (link_speeds & VIRTCHNL_LINK_SPEED_40GB) return IF_Gbps(40); if (link_speeds & VIRTCHNL_LINK_SPEED_25GB) return IF_Gbps(25); if (link_speeds & VIRTCHNL_LINK_SPEED_20GB) return IF_Gbps(20); if (link_speeds & VIRTCHNL_LINK_SPEED_10GB) return IF_Gbps(10); if (link_speeds & VIRTCHNL_LINK_SPEED_1GB) return IF_Gbps(1); if (link_speeds & VIRTCHNL_LINK_SPEED_100MB) return IF_Mbps(100); else /* Minimum supported link speed */ return IF_Mbps(100); } /** * iavf_config_rss_reg - Configure RSS using registers * @sc: device private softc * * Configures RSS for this function using the device registers. Called if the * PF does not support configuring RSS over the virtchnl interface. */ void iavf_config_rss_reg(struct iavf_sc *sc) { struct iavf_hw *hw = &sc->hw; struct iavf_vsi *vsi = &sc->vsi; u32 lut = 0; u64 set_hena = 0, hena; int i, j, que_id; u32 rss_seed[IAVF_RSS_KEY_SIZE_REG]; #ifdef RSS u32 rss_hash_config; #endif /* Don't set up RSS if using a single queue */ if (IAVF_NRXQS(vsi) == 1) { wr32(hw, IAVF_VFQF_HENA(0), 0); wr32(hw, IAVF_VFQF_HENA(1), 0); iavf_flush(hw); return; } #ifdef RSS /* Fetch the configured RSS key */ rss_getkey((uint8_t *) &rss_seed); #else iavf_get_default_rss_key(rss_seed); #endif /* Fill out hash function seed */ for (i = 0; i < IAVF_RSS_KEY_SIZE_REG; i++) wr32(hw, IAVF_VFQF_HKEY(i), rss_seed[i]); /* Enable PCTYPES for RSS: */ #ifdef RSS rss_hash_config = rss_gethashconfig(); if (rss_hash_config & RSS_HASHTYPE_RSS_IPV4) set_hena |= ((u64)1 << IAVF_FILTER_PCTYPE_NONF_IPV4_OTHER); if (rss_hash_config & RSS_HASHTYPE_RSS_TCP_IPV4) set_hena |= ((u64)1 << IAVF_FILTER_PCTYPE_NONF_IPV4_TCP); if (rss_hash_config & RSS_HASHTYPE_RSS_UDP_IPV4) set_hena |= ((u64)1 << IAVF_FILTER_PCTYPE_NONF_IPV4_UDP); if (rss_hash_config & RSS_HASHTYPE_RSS_IPV6) set_hena |= ((u64)1 << IAVF_FILTER_PCTYPE_NONF_IPV6_OTHER); if (rss_hash_config & RSS_HASHTYPE_RSS_IPV6_EX) set_hena |= ((u64)1 << IAVF_FILTER_PCTYPE_FRAG_IPV6); if (rss_hash_config & RSS_HASHTYPE_RSS_TCP_IPV6) set_hena |= ((u64)1 << IAVF_FILTER_PCTYPE_NONF_IPV6_TCP); if (rss_hash_config & RSS_HASHTYPE_RSS_UDP_IPV6) set_hena |= ((u64)1 << IAVF_FILTER_PCTYPE_NONF_IPV6_UDP); #else set_hena = IAVF_DEFAULT_RSS_HENA_XL710; #endif hena = (u64)rd32(hw, IAVF_VFQF_HENA(0)) | ((u64)rd32(hw, IAVF_VFQF_HENA(1)) << 32); hena |= set_hena; wr32(hw, IAVF_VFQF_HENA(0), (u32)hena); wr32(hw, IAVF_VFQF_HENA(1), (u32)(hena >> 32)); /* Populate the LUT with max no. of queues in round robin fashion */ for (i = 0, j = 0; i < IAVF_RSS_VSI_LUT_SIZE; i++, j++) { if (j == IAVF_NRXQS(vsi)) j = 0; #ifdef RSS /* * Fetch the RSS bucket id for the given indirection entry. * Cap it at the number of configured buckets (which is * num_rx_queues.) */ que_id = rss_get_indirection_to_bucket(i); que_id = que_id % IAVF_NRXQS(vsi); #else que_id = j; #endif /* lut = 4-byte sliding window of 4 lut entries */ lut = (lut << 8) | (que_id & IAVF_RSS_VF_LUT_ENTRY_MASK); /* On i = 3, we have 4 entries in lut; write to the register */ if ((i & 3) == 3) { wr32(hw, IAVF_VFQF_HLUT(i >> 2), lut); iavf_dbg_rss(sc, "%s: HLUT(%2d): %#010x", __func__, i, lut); } } iavf_flush(hw); } /** * iavf_config_rss_pf - Configure RSS using PF virtchnl messages * @sc: device private softc * * Configure RSS by sending virtchnl messages to the PF. */ void iavf_config_rss_pf(struct iavf_sc *sc) { iavf_send_vc_msg(sc, IAVF_FLAG_AQ_CONFIG_RSS_KEY); iavf_send_vc_msg(sc, IAVF_FLAG_AQ_SET_RSS_HENA); iavf_send_vc_msg(sc, IAVF_FLAG_AQ_CONFIG_RSS_LUT); } /** * iavf_config_rss - setup RSS * @sc: device private softc * * Configures RSS using the method determined by capability flags in the VF * resources structure sent from the PF over the virtchnl interface. * * @remark RSS keys and table are cleared on VF reset. */ void iavf_config_rss(struct iavf_sc *sc) { if (sc->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_RSS_REG) { iavf_dbg_info(sc, "Setting up RSS using VF registers...\n"); iavf_config_rss_reg(sc); } else if (sc->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_RSS_PF) { iavf_dbg_info(sc, "Setting up RSS using messages to PF...\n"); iavf_config_rss_pf(sc); } else device_printf(sc->dev, "VF does not support RSS capability sent by PF.\n"); } /** * iavf_config_promisc - setup promiscuous mode * @sc: device private softc * @flags: promiscuous flags to configure * * Request that promiscuous modes be enabled from the PF * * @returns zero on success, or an error code on failure. */ int iavf_config_promisc(struct iavf_sc *sc, int flags) { if_t ifp = sc->vsi.ifp; sc->promisc_flags = 0; if (flags & IFF_ALLMULTI || if_llmaddr_count(ifp) == MAX_MULTICAST_ADDR) sc->promisc_flags |= FLAG_VF_MULTICAST_PROMISC; if (flags & IFF_PROMISC) sc->promisc_flags |= FLAG_VF_UNICAST_PROMISC; iavf_send_vc_msg(sc, IAVF_FLAG_AQ_CONFIGURE_PROMISC); return (0); } /** * iavf_mc_filter_apply - Program a MAC filter for this VF * @arg: pointer to the device softc * @sdl: MAC multicast address * @cnt: unused parameter * * Program a MAC address multicast filter for this device. Intended * to be used with the map-like function if_foreach_llmaddr(). * * @returns 1 on success, or 0 on failure */ static u_int iavf_mc_filter_apply(void *arg, struct sockaddr_dl *sdl, u_int cnt __unused) { struct iavf_sc *sc = (struct iavf_sc *)arg; int error; error = iavf_add_mac_filter(sc, (u8*)LLADDR(sdl), IAVF_FILTER_MC); return (!error); } /** * iavf_init_multi - Initialize multicast address filters * @sc: device private softc * * Called during initialization to reset multicast address filters to a known * fresh state by deleting all currently active filters. */ void iavf_init_multi(struct iavf_sc *sc) { struct iavf_mac_filter *f; int mcnt = 0; /* First clear any multicast filters */ SLIST_FOREACH(f, sc->mac_filters, next) { if ((f->flags & IAVF_FILTER_USED) && (f->flags & IAVF_FILTER_MC)) { f->flags |= IAVF_FILTER_DEL; mcnt++; } } if (mcnt > 0) iavf_send_vc_msg(sc, IAVF_FLAG_AQ_DEL_MAC_FILTER); } /** * iavf_multi_set - Set multicast filters * @sc: device private softc * * Set multicast MAC filters for this device. If there are too many filters, * this will request the device to go into multicast promiscuous mode instead. */ void iavf_multi_set(struct iavf_sc *sc) { if_t ifp = sc->vsi.ifp; int mcnt = 0; IOCTL_DEBUGOUT("iavf_multi_set: begin"); mcnt = if_llmaddr_count(ifp); if (__predict_false(mcnt == MAX_MULTICAST_ADDR)) { /* Delete MC filters and enable mulitcast promisc instead */ iavf_init_multi(sc); sc->promisc_flags |= FLAG_VF_MULTICAST_PROMISC; iavf_send_vc_msg(sc, IAVF_FLAG_AQ_CONFIGURE_PROMISC); return; } /* If there aren't too many filters, delete existing MC filters */ iavf_init_multi(sc); /* And (re-)install filters for all mcast addresses */ mcnt = if_foreach_llmaddr(ifp, iavf_mc_filter_apply, sc); if (mcnt > 0) iavf_send_vc_msg(sc, IAVF_FLAG_AQ_ADD_MAC_FILTER); } /** * iavf_add_mac_filter - Add a MAC filter to the sc MAC list * @sc: device private softc * @macaddr: MAC address to add * @flags: filter flags * * Add a new MAC filter to the softc MAC filter list. These will later be sent * to the physical function (and ultimately hardware) via the virtchnl * interface. * * @returns zero on success, EEXIST if the filter already exists, and ENOMEM * if we ran out of memory allocating the filter structure. */ int iavf_add_mac_filter(struct iavf_sc *sc, u8 *macaddr, u16 flags) { struct iavf_mac_filter *f; /* Does one already exist? */ f = iavf_find_mac_filter(sc, macaddr); if (f != NULL) { iavf_dbg_filter(sc, "exists: " MAC_FORMAT "\n", MAC_FORMAT_ARGS(macaddr)); return (EEXIST); } /* If not, get a new empty filter */ f = iavf_get_mac_filter(sc); if (f == NULL) { device_printf(sc->dev, "%s: no filters available!!\n", __func__); return (ENOMEM); } iavf_dbg_filter(sc, "marked: " MAC_FORMAT "\n", MAC_FORMAT_ARGS(macaddr)); bcopy(macaddr, f->macaddr, ETHER_ADDR_LEN); f->flags |= (IAVF_FILTER_ADD | IAVF_FILTER_USED); f->flags |= flags; return (0); } /** * iavf_find_mac_filter - Find a MAC filter with the given address * @sc: device private softc * @macaddr: the MAC address to find * * Finds the filter structure in the MAC filter list with the corresponding * MAC address. * * @returns a pointer to the filter structure, or NULL if no such filter * exists in the list yet. */ struct iavf_mac_filter * iavf_find_mac_filter(struct iavf_sc *sc, u8 *macaddr) { struct iavf_mac_filter *f; bool match = FALSE; SLIST_FOREACH(f, sc->mac_filters, next) { if (cmp_etheraddr(f->macaddr, macaddr)) { match = TRUE; break; } } if (!match) f = NULL; return (f); } /** * iavf_get_mac_filter - Get a new MAC address filter * @sc: device private softc * * Allocates a new filter structure and inserts it into the MAC filter list. * * @post the caller must fill in the structure details after calling this * function, but does not need to insert it into the linked list. * * @returns a pointer to the new filter structure, or NULL of we failed to * allocate it. */ struct iavf_mac_filter * iavf_get_mac_filter(struct iavf_sc *sc) { struct iavf_mac_filter *f; f = (struct iavf_mac_filter *)malloc(sizeof(struct iavf_mac_filter), M_IAVF, M_NOWAIT | M_ZERO); if (f) SLIST_INSERT_HEAD(sc->mac_filters, f, next); return (f); } /** * iavf_baudrate_from_link_speed - Convert link speed to baudrate * @sc: device private softc * * @post The link_speed_adv field is in Mbps, so it is multipled by * 1,000,000 before it's returned. * * @returns the adapter link speed in bits/sec */ u64 iavf_baudrate_from_link_speed(struct iavf_sc *sc) { if (sc->vf_res->vf_cap_flags & VIRTCHNL_VF_CAP_ADV_LINK_SPEED) return (sc->link_speed_adv * IAVF_ADV_LINK_SPEED_SCALE); else return iavf_max_vc_speed_to_value(sc->link_speed); } /** * iavf_add_vlan_filter - Add a VLAN filter to the softc VLAN list * @sc: device private softc * @vtag: the VLAN id to filter * * Allocate a new VLAN filter structure and insert it into the VLAN list. */ void iavf_add_vlan_filter(struct iavf_sc *sc, u16 vtag) { struct iavf_vlan_filter *v; v = (struct iavf_vlan_filter *)malloc(sizeof(struct iavf_vlan_filter), M_IAVF, M_WAITOK | M_ZERO); SLIST_INSERT_HEAD(sc->vlan_filters, v, next); v->vlan = vtag; v->flags = IAVF_FILTER_ADD; } /** * iavf_mark_del_vlan_filter - Mark a given VLAN id for deletion * @sc: device private softc * @vtag: the VLAN id to delete * * Marks all VLAN filters matching the given vtag for deletion. * * @returns the number of filters marked for deletion. * * @remark the filters are not removed immediately, but will be removed from * the list by another function that synchronizes over the virtchnl interface. */ int iavf_mark_del_vlan_filter(struct iavf_sc *sc, u16 vtag) { struct iavf_vlan_filter *v; int i = 0; SLIST_FOREACH(v, sc->vlan_filters, next) { if (v->vlan == vtag) { v->flags = IAVF_FILTER_DEL; ++i; } } return (i); } -/** - * iavf_update_msix_devinfo - Fix MSIX values for pci_msix_count() - * @dev: pointer to kernel device - * - * Fix cached MSI-X control register information. This is a workaround - * for an issue where VFs spawned in non-passthrough mode on FreeBSD - * will have their PCI information cached before the PF driver - * finishes updating their PCI information. - * - * @pre Must be called before pci_msix_count() - */ -void -iavf_update_msix_devinfo(device_t dev) -{ - struct pci_devinfo *dinfo; - u32 msix_ctrl; - u8 msix_location; - - dinfo = (struct pci_devinfo *)device_get_ivars(dev); - msix_location = dinfo->cfg.msix.msix_location; - msix_ctrl = pci_read_config(dev, msix_location + PCIR_MSIX_CTRL, 2); - dinfo->cfg.msix.msix_ctrl = msix_ctrl; - dinfo->cfg.msix.msix_msgnum = (msix_ctrl & PCIM_MSIXCTRL_TABLE_SIZE) + 1; -} - /** * iavf_disable_queues_with_retries - Send PF multiple DISABLE_QUEUES messages * @sc: device softc * * Send a virtual channel message to the PF to DISABLE_QUEUES, but resend it up * to IAVF_MAX_DIS_Q_RETRY times if the response says that it wasn't * successful. This is intended to workaround a bug that can appear on the PF. */ void iavf_disable_queues_with_retries(struct iavf_sc *sc) { bool in_detach = iavf_driver_is_detaching(sc); int max_attempts = IAVF_MAX_DIS_Q_RETRY; int msg_count = 0; /* While the driver is detaching, it doesn't care if the queue * disable finishes successfully or not. Just send one message * to just notify the PF driver. */ if (in_detach) max_attempts = 1; while ((msg_count < max_attempts) && atomic_load_acq_32(&sc->queues_enabled)) { msg_count++; iavf_send_vc_msg_sleep(sc, IAVF_FLAG_AQ_DISABLE_QUEUES); } /* Possibly print messages about retry attempts and issues */ if (msg_count > 1) iavf_dbg_vc(sc, "DISABLE_QUEUES messages sent: %d\n", msg_count); if (!in_detach && msg_count >= max_attempts) device_printf(sc->dev, "%s: DISABLE_QUEUES may have failed\n", __func__); } diff --git a/sys/dev/iavf/iavf_lib.h b/sys/dev/iavf/iavf_lib.h index f3ccd9f0c52f..2f874b2e4410 100644 --- a/sys/dev/iavf/iavf_lib.h +++ b/sys/dev/iavf/iavf_lib.h @@ -1,485 +1,484 @@ /* SPDX-License-Identifier: BSD-3-Clause */ /* Copyright (c) 2021, Intel Corporation * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * 3. Neither the name of the Intel Corporation nor the names of its * contributors may be used to endorse or promote products derived from * this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ /** * @file iavf_lib.h * @brief header for structures and functions common to legacy and iflib * * Contains definitions and function declarations which are shared between the * legacy and iflib driver implementation. */ #ifndef _IAVF_LIB_H_ #define _IAVF_LIB_H_ #include #include #include #ifdef RSS #include #endif #include "iavf_debug.h" #include "iavf_osdep.h" #include "iavf_type.h" #include "iavf_prototype.h" MALLOC_DECLARE(M_IAVF); /* * Ring Descriptors Valid Range: 32-4096 Default Value: 1024 This value is the * number of tx/rx descriptors allocated by the driver. Increasing this * value allows the driver to queue more operations. * * Tx descriptors are always 16 bytes, but Rx descriptors can be 32 bytes. * The driver currently always uses 32 byte Rx descriptors. */ #define IAVF_DEFAULT_RING 1024 #define IAVF_MAX_RING 4096 #define IAVF_MIN_RING 64 #define IAVF_RING_INCREMENT 32 #define IAVF_AQ_LEN 256 #define IAVF_AQ_LEN_MAX 1024 /* ** Default number of entries in Tx queue buf_ring. */ #define DEFAULT_TXBRSZ 4096 /* Alignment for rings */ #define DBA_ALIGN 128 /* * Max number of multicast MAC addrs added to the driver's * internal lists before converting to promiscuous mode */ #define MAX_MULTICAST_ADDR 128 /* Byte alignment for Tx/Rx descriptor rings */ #define DBA_ALIGN 128 #define IAVF_MSIX_BAR 3 #define IAVF_ADM_LIMIT 2 #define IAVF_TSO_SIZE ((255*1024)-1) #define IAVF_AQ_BUF_SZ ((u32) 4096) #define IAVF_RX_HDR 128 #define IAVF_RX_LIMIT 512 #define IAVF_RX_ITR 0 #define IAVF_TX_ITR 1 /** * The maximum packet length allowed to be sent or received by the adapter. */ #define IAVF_MAX_FRAME 9728 /** * The minimum packet length allowed to be sent by the adapter. */ #define IAVF_MIN_FRAME 17 #define IAVF_MAX_TX_SEGS 8 #define IAVF_MAX_RX_SEGS 5 #define IAVF_MAX_TSO_SEGS 128 #define IAVF_SPARSE_CHAIN 7 #define IAVF_MIN_TSO_MSS 64 #define IAVF_MAX_TSO_MSS 9668 #define IAVF_MAX_DMA_SEG_SIZE ((16 * 1024) - 1) #define IAVF_AQ_MAX_ERR 30 #define IAVF_MAX_INIT_WAIT 120 #define IAVF_AQ_TIMEOUT (1 * hz) #define IAVF_ADV_LINK_SPEED_SCALE ((u64)1000000) #define IAVF_MAX_DIS_Q_RETRY 10 #define IAVF_RSS_KEY_SIZE_REG 13 #define IAVF_RSS_KEY_SIZE (IAVF_RSS_KEY_SIZE_REG * 4) #define IAVF_RSS_VSI_LUT_SIZE 64 /* X722 -> VSI, X710 -> VF */ #define IAVF_RSS_VSI_LUT_ENTRY_MASK 0x3F #define IAVF_RSS_VF_LUT_ENTRY_MASK 0xF /* Maximum MTU size */ #define IAVF_MAX_MTU (IAVF_MAX_FRAME - \ ETHER_HDR_LEN - ETHER_CRC_LEN - ETHER_VLAN_ENCAP_LEN) /* * Hardware requires that TSO packets have an segment size of at least 64 * bytes. To avoid sending bad frames to the hardware, the driver forces the * MSS for all TSO packets to have a segment size of at least 64 bytes. * * However, if the MTU is reduced below a certain size, then the resulting * larger MSS can result in transmitting segmented frames with a packet size * larger than the MTU. * * Avoid this by preventing the MTU from being lowered below this limit. * Alternative solutions require changing the TCP stack to disable offloading * the segmentation when the requested segment size goes below 64 bytes. */ #define IAVF_MIN_MTU 112 /* * Interrupt Moderation parameters * Multiply ITR values by 2 for real ITR value */ #define IAVF_MAX_ITR 0x0FF0 #define IAVF_ITR_100K 0x0005 #define IAVF_ITR_20K 0x0019 #define IAVF_ITR_8K 0x003E #define IAVF_ITR_4K 0x007A #define IAVF_ITR_1K 0x01F4 #define IAVF_ITR_DYNAMIC 0x8000 #define IAVF_LOW_LATENCY 0 #define IAVF_AVE_LATENCY 1 #define IAVF_BULK_LATENCY 2 /* MacVlan Flags */ #define IAVF_FILTER_USED (u16)(1 << 0) #define IAVF_FILTER_VLAN (u16)(1 << 1) #define IAVF_FILTER_ADD (u16)(1 << 2) #define IAVF_FILTER_DEL (u16)(1 << 3) #define IAVF_FILTER_MC (u16)(1 << 4) /* used in the vlan field of the filter when not a vlan */ #define IAVF_VLAN_ANY -1 #define CSUM_OFFLOAD_IPV4 (CSUM_IP|CSUM_TCP|CSUM_UDP|CSUM_SCTP) #define CSUM_OFFLOAD_IPV6 (CSUM_TCP_IPV6|CSUM_UDP_IPV6|CSUM_SCTP_IPV6) #define CSUM_OFFLOAD (CSUM_OFFLOAD_IPV4|CSUM_OFFLOAD_IPV6|CSUM_TSO) /* Misc flags for iavf_vsi.flags */ #define IAVF_FLAGS_KEEP_TSO4 (1 << 0) #define IAVF_FLAGS_KEEP_TSO6 (1 << 1) #define IAVF_DEFAULT_RSS_HENA_BASE (\ BIT_ULL(IAVF_FILTER_PCTYPE_NONF_IPV4_UDP) | \ BIT_ULL(IAVF_FILTER_PCTYPE_NONF_IPV4_TCP) | \ BIT_ULL(IAVF_FILTER_PCTYPE_NONF_IPV4_SCTP) | \ BIT_ULL(IAVF_FILTER_PCTYPE_NONF_IPV4_OTHER) | \ BIT_ULL(IAVF_FILTER_PCTYPE_FRAG_IPV4) | \ BIT_ULL(IAVF_FILTER_PCTYPE_NONF_IPV6_UDP) | \ BIT_ULL(IAVF_FILTER_PCTYPE_NONF_IPV6_TCP) | \ BIT_ULL(IAVF_FILTER_PCTYPE_NONF_IPV6_SCTP) | \ BIT_ULL(IAVF_FILTER_PCTYPE_NONF_IPV6_OTHER) | \ BIT_ULL(IAVF_FILTER_PCTYPE_FRAG_IPV6)) #define IAVF_DEFAULT_ADV_RSS_HENA (\ BIT_ULL(IAVF_FILTER_PCTYPE_NONF_UNICAST_IPV4_UDP) | \ BIT_ULL(IAVF_FILTER_PCTYPE_NONF_MULTICAST_IPV4_UDP) | \ BIT_ULL(IAVF_FILTER_PCTYPE_NONF_UNICAST_IPV6_UDP) | \ BIT_ULL(IAVF_FILTER_PCTYPE_NONF_MULTICAST_IPV6_UDP) | \ BIT_ULL(IAVF_FILTER_PCTYPE_NONF_IPV4_TCP_SYN_NO_ACK) | \ BIT_ULL(IAVF_FILTER_PCTYPE_NONF_IPV6_TCP_SYN_NO_ACK)) #define IAVF_DEFAULT_RSS_HENA_XL710 (\ IAVF_DEFAULT_RSS_HENA_BASE | \ BIT_ULL(IAVF_FILTER_PCTYPE_L2_PAYLOAD)) #define IAVF_DEFAULT_RSS_HENA_X722 (\ IAVF_DEFAULT_RSS_HENA_XL710 | \ IAVF_DEFAULT_ADV_RSS_HENA) #define IAVF_DEFAULT_RSS_HENA_AVF (\ IAVF_DEFAULT_RSS_HENA_BASE | \ IAVF_DEFAULT_ADV_RSS_HENA) /* For stats sysctl naming */ #define IAVF_QUEUE_NAME_LEN 32 #define IAVF_FLAG_AQ_ENABLE_QUEUES (u32)(1 << 0) #define IAVF_FLAG_AQ_DISABLE_QUEUES (u32)(1 << 1) #define IAVF_FLAG_AQ_ADD_MAC_FILTER (u32)(1 << 2) #define IAVF_FLAG_AQ_ADD_VLAN_FILTER (u32)(1 << 3) #define IAVF_FLAG_AQ_DEL_MAC_FILTER (u32)(1 << 4) #define IAVF_FLAG_AQ_DEL_VLAN_FILTER (u32)(1 << 5) #define IAVF_FLAG_AQ_CONFIGURE_QUEUES (u32)(1 << 6) #define IAVF_FLAG_AQ_MAP_VECTORS (u32)(1 << 7) #define IAVF_FLAG_AQ_HANDLE_RESET (u32)(1 << 8) #define IAVF_FLAG_AQ_CONFIGURE_PROMISC (u32)(1 << 9) #define IAVF_FLAG_AQ_GET_STATS (u32)(1 << 10) #define IAVF_FLAG_AQ_CONFIG_RSS_KEY (u32)(1 << 11) #define IAVF_FLAG_AQ_SET_RSS_HENA (u32)(1 << 12) #define IAVF_FLAG_AQ_GET_RSS_HENA_CAPS (u32)(1 << 13) #define IAVF_FLAG_AQ_CONFIG_RSS_LUT (u32)(1 << 14) #define IAVF_CAP_ADV_LINK_SPEED(_sc) \ ((_sc)->vf_res->vf_cap_flags & VIRTCHNL_VF_CAP_ADV_LINK_SPEED) #define IAVF_NRXQS(_vsi) ((_vsi)->num_rx_queues) #define IAVF_NTXQS(_vsi) ((_vsi)->num_tx_queues) /** * printf %b flag args */ #define IAVF_FLAGS \ "\20\1ENABLE_QUEUES\2DISABLE_QUEUES\3ADD_MAC_FILTER" \ "\4ADD_VLAN_FILTER\5DEL_MAC_FILTER\6DEL_VLAN_FILTER" \ "\7CONFIGURE_QUEUES\10MAP_VECTORS\11HANDLE_RESET" \ "\12CONFIGURE_PROMISC\13GET_STATS\14CONFIG_RSS_KEY" \ "\15SET_RSS_HENA\16GET_RSS_HENA_CAPS\17CONFIG_RSS_LUT" /** * printf %b flag args for offloads from virtchnl.h */ #define IAVF_PRINTF_VF_OFFLOAD_FLAGS \ "\20\1L2" \ "\2IWARP" \ "\3FCOE" \ "\4RSS_AQ" \ "\5RSS_REG" \ "\6WB_ON_ITR" \ "\7REQ_QUEUES" \ "\10ADV_LINK_SPEED" \ "\21VLAN" \ "\22RX_POLLING" \ "\23RSS_PCTYPE_V2" \ "\24RSS_PF" \ "\25ENCAP" \ "\26ENCAP_CSUM" \ "\27RX_ENCAP_CSUM" \ "\30ADQ" /** * @enum iavf_ext_link_speed * @brief Extended link speed enumeration * * Enumeration of possible link speeds that the device could be operating in. * Contains an extended list compared to the virtchnl_link_speed, including * additional higher speeds such as 50GB, and 100GB. * * The enumeration is used to convert between the old virtchnl_link_speed, the * newer advanced speed reporting value specified in Mb/s, and the ifmedia * link speeds reported to the operating system. */ enum iavf_ext_link_speed { IAVF_EXT_LINK_SPEED_UNKNOWN, IAVF_EXT_LINK_SPEED_10MB, IAVF_EXT_LINK_SPEED_100MB, IAVF_EXT_LINK_SPEED_1000MB, IAVF_EXT_LINK_SPEED_2500MB, IAVF_EXT_LINK_SPEED_5GB, IAVF_EXT_LINK_SPEED_10GB, IAVF_EXT_LINK_SPEED_20GB, IAVF_EXT_LINK_SPEED_25GB, IAVF_EXT_LINK_SPEED_40GB, IAVF_EXT_LINK_SPEED_50GB, IAVF_EXT_LINK_SPEED_100GB, }; /** * @struct iavf_sysctl_info * @brief sysctl statistic info * * Structure describing a single statistics sysctl, used for reporting * specific hardware and software statistics via the sysctl interface. */ struct iavf_sysctl_info { u64 *stat; char *name; char *description; }; /* Forward struct declarations */ struct iavf_sc; struct iavf_vsi; /** * @enum iavf_state * @brief Driver state flags * * Used to indicate the status of various driver events. Intended to be * modified only using atomic operations, so that we can use it even in places * which aren't locked. */ enum iavf_state { IAVF_STATE_INITIALIZED, IAVF_STATE_RESET_REQUIRED, IAVF_STATE_RESET_PENDING, IAVF_STATE_RUNNING, /* This entry must be last */ IAVF_STATE_LAST, }; /* Functions for setting and checking driver state. Note the functions take * bit positions, not bitmasks. The atomic_testandset_32 and * atomic_testandclear_32 operations require bit positions, while the * atomic_set_32 and atomic_clear_32 require bitmasks. This can easily lead to * programming error, so we provide wrapper functions to avoid this. */ /** * iavf_set_state - Set the specified state * @s: the state bitmap * @bit: the state to set * * Atomically update the state bitmap with the specified bit set. */ static inline void iavf_set_state(volatile u32 *s, enum iavf_state bit) { /* atomic_set_32 expects a bitmask */ atomic_set_32(s, BIT(bit)); } /** * iavf_clear_state - Clear the specified state * @s: the state bitmap * @bit: the state to clear * * Atomically update the state bitmap with the specified bit cleared. */ static inline void iavf_clear_state(volatile u32 *s, enum iavf_state bit) { /* atomic_clear_32 expects a bitmask */ atomic_clear_32(s, BIT(bit)); } /** * iavf_testandset_state - Test and set the specified state * @s: the state bitmap * @bit: the bit to test * * Atomically update the state bitmap, setting the specified bit. * * @returns the previous value of the bit. */ static inline u32 iavf_testandset_state(volatile u32 *s, enum iavf_state bit) { /* atomic_testandset_32 expects a bit position */ return atomic_testandset_32(s, bit); } /** * iavf_testandclear_state - Test and clear the specified state * @s: the state bitmap * @bit: the bit to test * * Atomically update the state bitmap, clearing the specified bit. * * @returns the previous value of the bit. */ static inline u32 iavf_testandclear_state(volatile u32 *s, enum iavf_state bit) { /* atomic_testandclear_32 expects a bit position */ return atomic_testandclear_32(s, bit); } /** * iavf_test_state - Test the specified state * @s: the state bitmap * @bit: the bit to test * * @returns true if the state is set, false otherwise. * * @remark Use this only if the flow does not need to update the state. If you * must update the state as well, prefer iavf_testandset_state or * iavf_testandclear_state. */ static inline u32 iavf_test_state(volatile u32 *s, enum iavf_state bit) { return (*s & BIT(bit)) ? true : false; } /** * cmp_etheraddr - Compare two ethernet addresses * @ea1: first ethernet address * @ea2: second ethernet address * * Compares two ethernet addresses. * * @returns true if the addresses are equal, false otherwise. */ static inline bool cmp_etheraddr(const u8 *ea1, const u8 *ea2) { bool cmp = FALSE; if ((ea1[0] == ea2[0]) && (ea1[1] == ea2[1]) && (ea1[2] == ea2[2]) && (ea1[3] == ea2[3]) && (ea1[4] == ea2[4]) && (ea1[5] == ea2[5])) cmp = TRUE; return (cmp); } int iavf_send_vc_msg(struct iavf_sc *sc, u32 op); int iavf_send_vc_msg_sleep(struct iavf_sc *sc, u32 op); void iavf_update_link_status(struct iavf_sc *); bool iavf_driver_is_detaching(struct iavf_sc *sc); void iavf_msec_pause(int msecs); void iavf_get_default_rss_key(u32 *key); int iavf_allocate_pci_resources_common(struct iavf_sc *sc); int iavf_reset_complete(struct iavf_hw *hw); int iavf_setup_vc(struct iavf_sc *sc); int iavf_reset(struct iavf_sc *sc); void iavf_enable_adminq_irq(struct iavf_hw *hw); void iavf_disable_adminq_irq(struct iavf_hw *hw); int iavf_vf_config(struct iavf_sc *sc); void iavf_print_device_info(struct iavf_sc *sc); int iavf_get_vsi_res_from_vf_res(struct iavf_sc *sc); void iavf_set_mac_addresses(struct iavf_sc *sc); void iavf_init_filters(struct iavf_sc *sc); void iavf_free_filters(struct iavf_sc *sc); void iavf_add_device_sysctls_common(struct iavf_sc *sc); void iavf_configure_tx_itr(struct iavf_sc *sc); void iavf_configure_rx_itr(struct iavf_sc *sc); struct sysctl_oid_list * iavf_create_debug_sysctl_tree(struct iavf_sc *sc); void iavf_add_debug_sysctls_common(struct iavf_sc *sc, struct sysctl_oid_list *debug_list); void iavf_add_vsi_sysctls(device_t dev, struct iavf_vsi *vsi, struct sysctl_ctx_list *ctx, const char *sysctl_name); void iavf_add_sysctls_eth_stats(struct sysctl_ctx_list *ctx, struct sysctl_oid_list *child, struct iavf_eth_stats *eth_stats); void iavf_media_status_common(struct iavf_sc *sc, struct ifmediareq *ifmr); int iavf_media_change_common(if_t ifp); void iavf_set_initial_baudrate(if_t ifp); u64 iavf_max_vc_speed_to_value(u8 link_speeds); void iavf_config_rss_reg(struct iavf_sc *sc); void iavf_config_rss_pf(struct iavf_sc *sc); void iavf_config_rss(struct iavf_sc *sc); int iavf_config_promisc(struct iavf_sc *sc, int flags); void iavf_init_multi(struct iavf_sc *sc); void iavf_multi_set(struct iavf_sc *sc); int iavf_add_mac_filter(struct iavf_sc *sc, u8 *macaddr, u16 flags); struct iavf_mac_filter * iavf_find_mac_filter(struct iavf_sc *sc, u8 *macaddr); struct iavf_mac_filter * iavf_get_mac_filter(struct iavf_sc *sc); u64 iavf_baudrate_from_link_speed(struct iavf_sc *sc); void iavf_add_vlan_filter(struct iavf_sc *sc, u16 vtag); int iavf_mark_del_vlan_filter(struct iavf_sc *sc, u16 vtag); -void iavf_update_msix_devinfo(device_t dev); void iavf_disable_queues_with_retries(struct iavf_sc *); int iavf_sysctl_current_speed(SYSCTL_HANDLER_ARGS); int iavf_sysctl_tx_itr(SYSCTL_HANDLER_ARGS); int iavf_sysctl_rx_itr(SYSCTL_HANDLER_ARGS); int iavf_sysctl_sw_filter_list(SYSCTL_HANDLER_ARGS); #endif /* _IAVF_LIB_H_ */ diff --git a/sys/dev/iavf/if_iavf_iflib.c b/sys/dev/iavf/if_iavf_iflib.c index d460df6e0d25..e4dd3b1e59a4 100644 --- a/sys/dev/iavf/if_iavf_iflib.c +++ b/sys/dev/iavf/if_iavf_iflib.c @@ -1,2154 +1,2151 @@ /* SPDX-License-Identifier: BSD-3-Clause */ /* Copyright (c) 2024, Intel Corporation * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * 3. Neither the name of the Intel Corporation nor the names of its * contributors may be used to endorse or promote products derived from * this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ /** * @file if_iavf_iflib.c * @brief iflib driver implementation * * Contains the main entry point for the iflib driver implementation. It * implements the various ifdi driver methods, and sets up the module and * driver values to load an iflib driver. */ #include "iavf_iflib.h" #include "iavf_vc_common.h" #include "iavf_drv_info.h" #include "iavf_sysctls_iflib.h" /********************************************************************* * Function prototypes *********************************************************************/ static void *iavf_register(device_t dev); static int iavf_if_attach_pre(if_ctx_t ctx); static int iavf_if_attach_post(if_ctx_t ctx); static int iavf_if_detach(if_ctx_t ctx); static int iavf_if_shutdown(if_ctx_t ctx); static int iavf_if_suspend(if_ctx_t ctx); static int iavf_if_resume(if_ctx_t ctx); static int iavf_if_msix_intr_assign(if_ctx_t ctx, int msix); static void iavf_if_enable_intr(if_ctx_t ctx); static void iavf_if_disable_intr(if_ctx_t ctx); static int iavf_if_rx_queue_intr_enable(if_ctx_t ctx, uint16_t rxqid); static int iavf_if_tx_queue_intr_enable(if_ctx_t ctx, uint16_t txqid); static int iavf_if_tx_queues_alloc(if_ctx_t ctx, caddr_t *vaddrs, uint64_t *paddrs, int ntxqs, int ntxqsets); static int iavf_if_rx_queues_alloc(if_ctx_t ctx, caddr_t *vaddrs, uint64_t *paddrs, int nqs, int nqsets); static void iavf_if_queues_free(if_ctx_t ctx); static void iavf_if_update_admin_status(if_ctx_t ctx); static void iavf_if_multi_set(if_ctx_t ctx); static int iavf_if_mtu_set(if_ctx_t ctx, uint32_t mtu); static void iavf_if_media_status(if_ctx_t ctx, struct ifmediareq *ifmr); static int iavf_if_media_change(if_ctx_t ctx); static int iavf_if_promisc_set(if_ctx_t ctx, int flags); static void iavf_if_timer(if_ctx_t ctx, uint16_t qid); static void iavf_if_vlan_register(if_ctx_t ctx, u16 vtag); static void iavf_if_vlan_unregister(if_ctx_t ctx, u16 vtag); static uint64_t iavf_if_get_counter(if_ctx_t ctx, ift_counter cnt); static void iavf_if_init(if_ctx_t ctx); static void iavf_if_stop(if_ctx_t ctx); static bool iavf_if_needs_restart(if_ctx_t, enum iflib_restart_event); static int iavf_allocate_pci_resources(struct iavf_sc *); static void iavf_free_pci_resources(struct iavf_sc *); static void iavf_setup_interface(struct iavf_sc *); static void iavf_add_device_sysctls(struct iavf_sc *); static void iavf_enable_queue_irq(struct iavf_hw *, int); static void iavf_disable_queue_irq(struct iavf_hw *, int); static void iavf_stop(struct iavf_sc *); static int iavf_del_mac_filter(struct iavf_sc *sc, u8 *macaddr); static int iavf_msix_que(void *); static int iavf_msix_adminq(void *); static void iavf_configure_itr(struct iavf_sc *sc); static int iavf_sysctl_queue_interrupt_table(SYSCTL_HANDLER_ARGS); #ifdef IAVF_DEBUG static int iavf_sysctl_vf_reset(SYSCTL_HANDLER_ARGS); static int iavf_sysctl_vflr_reset(SYSCTL_HANDLER_ARGS); #endif static enum iavf_status iavf_process_adminq(struct iavf_sc *, u16 *); static void iavf_vc_task(void *arg, int pending __unused); static int iavf_setup_vc_tq(struct iavf_sc *sc); static int iavf_vc_sleep_wait(struct iavf_sc *sc, u32 op); /********************************************************************* * FreeBSD Device Interface Entry Points *********************************************************************/ /** * @var iavf_methods * @brief device methods for the iavf driver * * Device method callbacks used to interact with the driver. For iflib this * primarily resolves to the default iflib implementations. */ static device_method_t iavf_methods[] = { /* Device interface */ DEVMETHOD(device_register, iavf_register), DEVMETHOD(device_probe, iflib_device_probe), DEVMETHOD(device_attach, iflib_device_attach), DEVMETHOD(device_detach, iflib_device_detach), DEVMETHOD(device_shutdown, iflib_device_shutdown), DEVMETHOD_END }; static driver_t iavf_driver = { "iavf", iavf_methods, sizeof(struct iavf_sc), }; DRIVER_MODULE(iavf, pci, iavf_driver, 0, 0); MODULE_VERSION(iavf, 1); MODULE_DEPEND(iavf, pci, 1, 1, 1); MODULE_DEPEND(iavf, ether, 1, 1, 1); MODULE_DEPEND(iavf, iflib, 1, 1, 1); IFLIB_PNP_INFO(pci, iavf, iavf_vendor_info_array); /** * @var M_IAVF * @brief main iavf driver allocation type * * malloc(9) allocation type used by the majority of memory allocations in the * iavf iflib driver. */ MALLOC_DEFINE(M_IAVF, "iavf", "iavf driver allocations"); static device_method_t iavf_if_methods[] = { DEVMETHOD(ifdi_attach_pre, iavf_if_attach_pre), DEVMETHOD(ifdi_attach_post, iavf_if_attach_post), DEVMETHOD(ifdi_detach, iavf_if_detach), DEVMETHOD(ifdi_shutdown, iavf_if_shutdown), DEVMETHOD(ifdi_suspend, iavf_if_suspend), DEVMETHOD(ifdi_resume, iavf_if_resume), DEVMETHOD(ifdi_init, iavf_if_init), DEVMETHOD(ifdi_stop, iavf_if_stop), DEVMETHOD(ifdi_msix_intr_assign, iavf_if_msix_intr_assign), DEVMETHOD(ifdi_intr_enable, iavf_if_enable_intr), DEVMETHOD(ifdi_intr_disable, iavf_if_disable_intr), DEVMETHOD(ifdi_rx_queue_intr_enable, iavf_if_rx_queue_intr_enable), DEVMETHOD(ifdi_tx_queue_intr_enable, iavf_if_tx_queue_intr_enable), DEVMETHOD(ifdi_tx_queues_alloc, iavf_if_tx_queues_alloc), DEVMETHOD(ifdi_rx_queues_alloc, iavf_if_rx_queues_alloc), DEVMETHOD(ifdi_queues_free, iavf_if_queues_free), DEVMETHOD(ifdi_update_admin_status, iavf_if_update_admin_status), DEVMETHOD(ifdi_multi_set, iavf_if_multi_set), DEVMETHOD(ifdi_mtu_set, iavf_if_mtu_set), DEVMETHOD(ifdi_media_status, iavf_if_media_status), DEVMETHOD(ifdi_media_change, iavf_if_media_change), DEVMETHOD(ifdi_promisc_set, iavf_if_promisc_set), DEVMETHOD(ifdi_timer, iavf_if_timer), DEVMETHOD(ifdi_vlan_register, iavf_if_vlan_register), DEVMETHOD(ifdi_vlan_unregister, iavf_if_vlan_unregister), DEVMETHOD(ifdi_get_counter, iavf_if_get_counter), DEVMETHOD(ifdi_needs_restart, iavf_if_needs_restart), DEVMETHOD_END }; static driver_t iavf_if_driver = { "iavf_if", iavf_if_methods, sizeof(struct iavf_sc) }; extern struct if_txrx iavf_txrx_hwb; extern struct if_txrx iavf_txrx_dwb; static struct if_shared_ctx iavf_sctx = { .isc_magic = IFLIB_MAGIC, .isc_q_align = PAGE_SIZE, .isc_tx_maxsize = IAVF_MAX_FRAME, .isc_tx_maxsegsize = IAVF_MAX_FRAME, .isc_tso_maxsize = IAVF_TSO_SIZE + sizeof(struct ether_vlan_header), .isc_tso_maxsegsize = IAVF_MAX_DMA_SEG_SIZE, .isc_rx_maxsize = IAVF_MAX_FRAME, .isc_rx_nsegments = IAVF_MAX_RX_SEGS, .isc_rx_maxsegsize = IAVF_MAX_FRAME, .isc_nfl = 1, .isc_ntxqs = 1, .isc_nrxqs = 1, .isc_admin_intrcnt = 1, .isc_vendor_info = iavf_vendor_info_array, .isc_driver_version = __DECONST(char *, iavf_driver_version), .isc_driver = &iavf_if_driver, .isc_flags = IFLIB_NEED_SCRATCH | IFLIB_NEED_ZERO_CSUM | IFLIB_TSO_INIT_IP | IFLIB_IS_VF, .isc_nrxd_min = {IAVF_MIN_RING}, .isc_ntxd_min = {IAVF_MIN_RING}, .isc_nrxd_max = {IAVF_MAX_RING}, .isc_ntxd_max = {IAVF_MAX_RING}, .isc_nrxd_default = {IAVF_DEFAULT_RING}, .isc_ntxd_default = {IAVF_DEFAULT_RING}, }; /*** Functions ***/ /** * iavf_register - iflib callback to obtain the shared context pointer * @dev: the device being registered * * Called when the driver is first being attached to the driver. This function * is used by iflib to obtain a pointer to the shared context structure which * describes the device features. * * @returns a pointer to the iavf shared context structure. */ static void * iavf_register(device_t dev __unused) { return (&iavf_sctx); } /** * iavf_allocate_pci_resources - Allocate PCI resources * @sc: the device private softc * * Allocate PCI resources used by the iflib driver. * * @returns zero or a non-zero error code on failure */ static int iavf_allocate_pci_resources(struct iavf_sc *sc) { return iavf_allocate_pci_resources_common(sc); } /** * iavf_if_attach_pre - Begin attaching the device to the driver * @ctx: the iflib context pointer * * Called by iflib to begin the attach process. Allocates resources and * initializes the hardware for operation. * * @returns zero or a non-zero error code on failure. */ static int iavf_if_attach_pre(if_ctx_t ctx) { device_t dev; struct iavf_sc *sc; struct iavf_hw *hw; struct iavf_vsi *vsi; if_softc_ctx_t scctx; int error = 0; /* Setup pointers */ dev = iflib_get_dev(ctx); sc = iavf_sc_from_ctx(ctx); vsi = &sc->vsi; vsi->back = sc; sc->dev = sc->osdep.dev = dev; hw = &sc->hw; vsi->dev = dev; vsi->hw = &sc->hw; vsi->num_vlans = 0; vsi->ctx = ctx; sc->media = iflib_get_media(ctx); vsi->ifp = iflib_get_ifp(ctx); vsi->shared = scctx = iflib_get_softc_ctx(ctx); iavf_save_tunables(sc); /* Setup VC mutex */ snprintf(sc->vc_mtx_name, sizeof(sc->vc_mtx_name), "%s:vc", device_get_nameunit(dev)); mtx_init(&sc->vc_mtx, sc->vc_mtx_name, NULL, MTX_DEF); /* Do PCI setup - map BAR0, etc */ error = iavf_allocate_pci_resources(sc); if (error) { device_printf(dev, "%s: Allocation of PCI resources failed\n", __func__); goto err_early; } iavf_dbg_init(sc, "Allocated PCI resources and MSI-X vectors\n"); error = iavf_set_mac_type(hw); if (error) { device_printf(dev, "%s: set_mac_type failed: %d\n", __func__, error); goto err_pci_res; } error = iavf_reset_complete(hw); if (error) { device_printf(dev, "%s: Device is still being reset\n", __func__); goto err_pci_res; } iavf_dbg_init(sc, "VF Device is ready for configuration\n"); /* Sets up Admin Queue */ error = iavf_setup_vc(sc); if (error) { device_printf(dev, "%s: Error setting up PF comms, %d\n", __func__, error); goto err_pci_res; } iavf_dbg_init(sc, "PF API version verified\n"); /* Need API version before sending reset message */ error = iavf_reset(sc); if (error) { device_printf(dev, "VF reset failed; reload the driver\n"); goto err_aq; } iavf_dbg_init(sc, "VF reset complete\n"); /* Ask for VF config from PF */ error = iavf_vf_config(sc); if (error) { device_printf(dev, "Error getting configuration from PF: %d\n", error); goto err_aq; } iavf_print_device_info(sc); error = iavf_get_vsi_res_from_vf_res(sc); if (error) goto err_res_buf; iavf_dbg_init(sc, "Resource Acquisition complete\n"); /* Setup taskqueue to service VC messages */ error = iavf_setup_vc_tq(sc); if (error) goto err_vc_tq; iavf_set_mac_addresses(sc); iflib_set_mac(ctx, hw->mac.addr); /* Allocate filter lists */ iavf_init_filters(sc); /* Fill out more iflib parameters */ scctx->isc_ntxqsets_max = scctx->isc_nrxqsets_max = sc->vsi_res->num_queue_pairs; if (vsi->enable_head_writeback) { scctx->isc_txqsizes[0] = roundup2(scctx->isc_ntxd[0] * sizeof(struct iavf_tx_desc) + sizeof(u32), DBA_ALIGN); scctx->isc_txrx = &iavf_txrx_hwb; } else { scctx->isc_txqsizes[0] = roundup2(scctx->isc_ntxd[0] * sizeof(struct iavf_tx_desc), DBA_ALIGN); scctx->isc_txrx = &iavf_txrx_dwb; } scctx->isc_rxqsizes[0] = roundup2(scctx->isc_nrxd[0] * sizeof(union iavf_32byte_rx_desc), DBA_ALIGN); scctx->isc_msix_bar = pci_msix_table_bar(dev); scctx->isc_tx_nsegments = IAVF_MAX_TX_SEGS; scctx->isc_tx_tso_segments_max = IAVF_MAX_TSO_SEGS; scctx->isc_tx_tso_size_max = IAVF_TSO_SIZE; scctx->isc_tx_tso_segsize_max = IAVF_MAX_DMA_SEG_SIZE; scctx->isc_rss_table_size = IAVF_RSS_VSI_LUT_SIZE; scctx->isc_capabilities = scctx->isc_capenable = IAVF_CAPS; scctx->isc_tx_csum_flags = CSUM_OFFLOAD; - /* Update OS cache of MSIX control register values */ - iavf_update_msix_devinfo(dev); - return (0); err_vc_tq: taskqueue_free(sc->vc_tq); err_res_buf: free(sc->vf_res, M_IAVF); err_aq: iavf_shutdown_adminq(hw); err_pci_res: iavf_free_pci_resources(sc); err_early: IAVF_VC_LOCK_DESTROY(sc); return (error); } /** * iavf_vc_task - task used to process VC messages * @arg: device softc * @pending: unused * * Processes the admin queue, in order to process the virtual * channel messages received from the PF. */ static void iavf_vc_task(void *arg, int pending __unused) { struct iavf_sc *sc = (struct iavf_sc *)arg; u16 var; iavf_process_adminq(sc, &var); } /** * iavf_setup_vc_tq - Setup task queues * @sc: device softc * * Create taskqueue and tasklet for processing virtual channel messages. This * is done in a separate non-iflib taskqueue so that the iflib context lock * does not need to be held for VC messages to be processed. * * @returns zero on success, or an error code on failure. */ static int iavf_setup_vc_tq(struct iavf_sc *sc) { device_t dev = sc->dev; int error = 0; TASK_INIT(&sc->vc_task, 0, iavf_vc_task, sc); sc->vc_tq = taskqueue_create_fast("iavf_vc", M_NOWAIT, taskqueue_thread_enqueue, &sc->vc_tq); if (!sc->vc_tq) { device_printf(dev, "taskqueue_create_fast (for VC task) returned NULL!\n"); return (ENOMEM); } error = taskqueue_start_threads(&sc->vc_tq, 1, PI_NET, "%s vc", device_get_nameunit(dev)); if (error) { device_printf(dev, "taskqueue_start_threads (for VC task) error: %d\n", error); taskqueue_free(sc->vc_tq); return (error); } return (error); } /** * iavf_if_attach_post - Finish attaching the device to the driver * @ctx: the iflib context pointer * * Called by iflib after it has setup queues and interrupts. Used to finish up * the attach process for a device. Attach logic which must occur after Tx and * Rx queues are setup belongs here. * * @returns zero or a non-zero error code on failure */ static int iavf_if_attach_post(if_ctx_t ctx) { #ifdef IXL_DEBUG device_t dev = iflib_get_dev(ctx); #endif struct iavf_sc *sc; struct iavf_hw *hw; struct iavf_vsi *vsi; int error = 0; INIT_DBG_DEV(dev, "begin"); sc = iavf_sc_from_ctx(ctx); vsi = &sc->vsi; hw = &sc->hw; /* Save off determined number of queues for interface */ vsi->num_rx_queues = vsi->shared->isc_nrxqsets; vsi->num_tx_queues = vsi->shared->isc_ntxqsets; /* Setup the stack interface */ iavf_setup_interface(sc); iavf_dbg_init(sc, "Interface setup complete\n"); /* Initialize statistics & add sysctls */ bzero(&sc->vsi.eth_stats, sizeof(struct iavf_eth_stats)); iavf_add_device_sysctls(sc); atomic_store_rel_32(&sc->queues_enabled, 0); iavf_set_state(&sc->state, IAVF_STATE_INITIALIZED); /* We want AQ enabled early for init */ iavf_enable_adminq_irq(hw); INIT_DBG_DEV(dev, "end"); return (error); } /** * iavf_if_detach - Detach a device from the driver * @ctx: the iflib context of the device to detach * * Called by iflib to detach a given device from the driver. Clean up any * resources associated with the driver and shut the device down. * * @remark iflib always ignores the return value of IFDI_DETACH, so this * function is effectively not allowed to fail. Instead, it should clean up * and release as much as possible even if something goes wrong. * * @returns zero */ static int iavf_if_detach(if_ctx_t ctx) { struct iavf_sc *sc = iavf_sc_from_ctx(ctx); struct iavf_hw *hw = &sc->hw; device_t dev = sc->dev; enum iavf_status status; INIT_DBG_DEV(dev, "begin"); iavf_clear_state(&sc->state, IAVF_STATE_INITIALIZED); /* Drain admin queue taskqueue */ taskqueue_free(sc->vc_tq); IAVF_VC_LOCK_DESTROY(sc); /* Remove all the media and link information */ ifmedia_removeall(sc->media); iavf_disable_adminq_irq(hw); status = iavf_shutdown_adminq(&sc->hw); if (status != IAVF_SUCCESS) { device_printf(dev, "iavf_shutdown_adminq() failed with status %s\n", iavf_stat_str(hw, status)); } free(sc->vf_res, M_IAVF); sc->vf_res = NULL; iavf_free_pci_resources(sc); iavf_free_filters(sc); INIT_DBG_DEV(dev, "end"); return (0); } /** * iavf_if_shutdown - called by iflib to handle shutdown * @ctx: the iflib context pointer * * Callback for the IFDI_SHUTDOWN iflib function. * * @returns zero or an error code on failure */ static int iavf_if_shutdown(if_ctx_t ctx __unused) { return (0); } /** * iavf_if_suspend - called by iflib to handle suspend * @ctx: the iflib context pointer * * Callback for the IFDI_SUSPEND iflib function. * * @returns zero or an error code on failure */ static int iavf_if_suspend(if_ctx_t ctx __unused) { return (0); } /** * iavf_if_resume - called by iflib to handle resume * @ctx: the iflib context pointer * * Callback for the IFDI_RESUME iflib function. * * @returns zero or an error code on failure */ static int iavf_if_resume(if_ctx_t ctx __unused) { return (0); } /** * iavf_vc_sleep_wait - Sleep for a response from a VC message * @sc: device softc * @op: the op code to sleep on * * Sleep until a response from the PF for the VC message sent by the * given op. * * @returns zero on success, or EWOULDBLOCK if the sleep times out. */ static int iavf_vc_sleep_wait(struct iavf_sc *sc, u32 op) { int error = 0; IAVF_VC_LOCK_ASSERT(sc); iavf_dbg_vc(sc, "Sleeping for op %b\n", op, IAVF_FLAGS); error = mtx_sleep(iavf_vc_get_op_chan(sc, op), &sc->vc_mtx, PRI_MAX, "iavf_vc", IAVF_AQ_TIMEOUT); return (error); } /** * iavf_send_vc_msg_sleep - Send a virtchnl message and wait for a response * @sc: device softc * @op: the op code to send * * Send a virtchnl message to the PF, and sleep or busy wait for a response * from the PF, depending on iflib context lock type. * * @remark this function does not wait if the device is detaching, on kernels * that support indicating to the driver that the device is detaching * * @returns zero or an error code on failure. */ int iavf_send_vc_msg_sleep(struct iavf_sc *sc, u32 op) { if_ctx_t ctx = sc->vsi.ctx; int error = 0; IAVF_VC_LOCK(sc); error = iavf_vc_send_cmd(sc, op); if (error != 0) { iavf_dbg_vc(sc, "Error sending %b: %d\n", op, IAVF_FLAGS, error); goto release_lock; } /* Don't wait for a response if the device is being detached. */ if (!iflib_in_detach(ctx)) { error = iavf_vc_sleep_wait(sc, op); IAVF_VC_LOCK_ASSERT(sc); if (error == EWOULDBLOCK) device_printf(sc->dev, "%b timed out\n", op, IAVF_FLAGS); } release_lock: IAVF_VC_UNLOCK(sc); return (error); } /** * iavf_send_vc_msg - Send a virtchnl message to the PF * @sc: device softc * @op: the op code to send * * Send a virtchnl message to the PF and do not wait for a response. * * @returns zero on success, or an error code on failure. */ int iavf_send_vc_msg(struct iavf_sc *sc, u32 op) { int error = 0; error = iavf_vc_send_cmd(sc, op); if (error != 0) iavf_dbg_vc(sc, "Error sending %b: %d\n", op, IAVF_FLAGS, error); return (error); } /** * iavf_init_queues - initialize Tx and Rx queues * @vsi: the VSI to initialize * * Refresh the Tx and Rx ring contents and update the tail pointers for each * queue. */ static void iavf_init_queues(struct iavf_vsi *vsi) { struct iavf_tx_queue *tx_que = vsi->tx_queues; struct iavf_rx_queue *rx_que = vsi->rx_queues; struct rx_ring *rxr; uint32_t mbuf_sz; mbuf_sz = iflib_get_rx_mbuf_sz(vsi->ctx); MPASS(mbuf_sz <= UINT16_MAX); for (int i = 0; i < vsi->num_tx_queues; i++, tx_que++) iavf_init_tx_ring(vsi, tx_que); for (int i = 0; i < vsi->num_rx_queues; i++, rx_que++) { rxr = &rx_que->rxr; rxr->mbuf_sz = mbuf_sz; wr32(vsi->hw, rxr->tail, 0); } } /** * iavf_if_init - Initialize device for operation * @ctx: the iflib context pointer * * Initializes a device for operation. Called by iflib in response to an * interface up event from the stack. * * @remark this function does not return a value and thus cannot indicate * failure to initialize. */ static void iavf_if_init(if_ctx_t ctx) { struct iavf_sc *sc = iavf_sc_from_ctx(ctx); struct iavf_vsi *vsi = &sc->vsi; struct iavf_hw *hw = &sc->hw; if_t ifp = iflib_get_ifp(ctx); u8 tmpaddr[ETHER_ADDR_LEN]; enum iavf_status status; device_t dev = sc->dev; int error = 0; INIT_DBG_IF(ifp, "begin"); sx_assert(iflib_ctx_lock_get(ctx), SA_XLOCKED); error = iavf_reset_complete(hw); if (error) { device_printf(sc->dev, "%s: VF reset failed\n", __func__); } if (!iavf_check_asq_alive(hw)) { iavf_dbg_info(sc, "ASQ is not alive, re-initializing AQ\n"); pci_enable_busmaster(dev); status = iavf_shutdown_adminq(hw); if (status != IAVF_SUCCESS) { device_printf(dev, "%s: iavf_shutdown_adminq failed: %s\n", __func__, iavf_stat_str(hw, status)); return; } status = iavf_init_adminq(hw); if (status != IAVF_SUCCESS) { device_printf(dev, "%s: iavf_init_adminq failed: %s\n", __func__, iavf_stat_str(hw, status)); return; } } /* Make sure queues are disabled */ iavf_disable_queues_with_retries(sc); bcopy(if_getlladdr(ifp), tmpaddr, ETHER_ADDR_LEN); if (!cmp_etheraddr(hw->mac.addr, tmpaddr) && (iavf_validate_mac_addr(tmpaddr) == IAVF_SUCCESS)) { error = iavf_del_mac_filter(sc, hw->mac.addr); if (error == 0) iavf_send_vc_msg(sc, IAVF_FLAG_AQ_DEL_MAC_FILTER); bcopy(tmpaddr, hw->mac.addr, ETH_ALEN); } error = iavf_add_mac_filter(sc, hw->mac.addr, 0); if (!error || error == EEXIST) iavf_send_vc_msg(sc, IAVF_FLAG_AQ_ADD_MAC_FILTER); iflib_set_mac(ctx, hw->mac.addr); /* Prepare the queues for operation */ iavf_init_queues(vsi); /* Set initial ITR values */ iavf_configure_itr(sc); iavf_send_vc_msg(sc, IAVF_FLAG_AQ_CONFIGURE_QUEUES); /* Set up RSS */ iavf_config_rss(sc); /* Map vectors */ iavf_send_vc_msg(sc, IAVF_FLAG_AQ_MAP_VECTORS); /* Init SW TX ring indices */ if (vsi->enable_head_writeback) iavf_init_tx_cidx(vsi); else iavf_init_tx_rsqs(vsi); /* Configure promiscuous mode */ iavf_config_promisc(sc, if_getflags(ifp)); /* Enable queues */ iavf_send_vc_msg_sleep(sc, IAVF_FLAG_AQ_ENABLE_QUEUES); iavf_set_state(&sc->state, IAVF_STATE_RUNNING); } /** * iavf_if_msix_intr_assign - Assign MSI-X interrupts * @ctx: the iflib context pointer * @msix: the number of MSI-X vectors available * * Called by iflib to assign MSI-X interrupt vectors to queues. Assigns and * sets up vectors for each Tx and Rx queue, as well as the administrative * control interrupt. * * @returns zero or an error code on failure */ static int iavf_if_msix_intr_assign(if_ctx_t ctx, int msix __unused) { struct iavf_sc *sc = iavf_sc_from_ctx(ctx); struct iavf_vsi *vsi = &sc->vsi; struct iavf_rx_queue *rx_que = vsi->rx_queues; struct iavf_tx_queue *tx_que = vsi->tx_queues; int err, i, rid, vector = 0; char buf[16]; MPASS(vsi->shared->isc_nrxqsets > 0); MPASS(vsi->shared->isc_ntxqsets > 0); /* Admin Que is vector 0*/ rid = vector + 1; err = iflib_irq_alloc_generic(ctx, &vsi->irq, rid, IFLIB_INTR_ADMIN, iavf_msix_adminq, sc, 0, "aq"); if (err) { iflib_irq_free(ctx, &vsi->irq); device_printf(iflib_get_dev(ctx), "Failed to register Admin Que handler"); return (err); } /* Now set up the stations */ for (i = 0, vector = 1; i < vsi->shared->isc_nrxqsets; i++, vector++, rx_que++) { rid = vector + 1; snprintf(buf, sizeof(buf), "rxq%d", i); err = iflib_irq_alloc_generic(ctx, &rx_que->que_irq, rid, IFLIB_INTR_RXTX, iavf_msix_que, rx_que, rx_que->rxr.me, buf); if (err) { device_printf(iflib_get_dev(ctx), "Failed to allocate queue RX int vector %d, err: %d\n", i, err); vsi->num_rx_queues = i + 1; goto fail; } rx_que->msix = vector; } bzero(buf, sizeof(buf)); for (i = 0; i < vsi->shared->isc_ntxqsets; i++, tx_que++) { snprintf(buf, sizeof(buf), "txq%d", i); iflib_softirq_alloc_generic(ctx, &vsi->rx_queues[i % vsi->shared->isc_nrxqsets].que_irq, IFLIB_INTR_TX, tx_que, tx_que->txr.me, buf); tx_que->msix = (i % vsi->shared->isc_nrxqsets) + 1; } return (0); fail: iflib_irq_free(ctx, &vsi->irq); rx_que = vsi->rx_queues; for (i = 0; i < vsi->num_rx_queues; i++, rx_que++) iflib_irq_free(ctx, &rx_que->que_irq); return (err); } /** * iavf_if_enable_intr - Enable all interrupts for a device * @ctx: the iflib context pointer * * Called by iflib to request enabling all interrupts. */ static void iavf_if_enable_intr(if_ctx_t ctx) { struct iavf_sc *sc = iavf_sc_from_ctx(ctx); struct iavf_vsi *vsi = &sc->vsi; iavf_enable_intr(vsi); } /** * iavf_if_disable_intr - Disable all interrupts for a device * @ctx: the iflib context pointer * * Called by iflib to request disabling all interrupts. */ static void iavf_if_disable_intr(if_ctx_t ctx) { struct iavf_sc *sc = iavf_sc_from_ctx(ctx); struct iavf_vsi *vsi = &sc->vsi; iavf_disable_intr(vsi); } /** * iavf_if_rx_queue_intr_enable - Enable one Rx queue interrupt * @ctx: the iflib context pointer * @rxqid: Rx queue index * * Enables the interrupt associated with a specified Rx queue. * * @returns zero */ static int iavf_if_rx_queue_intr_enable(if_ctx_t ctx, uint16_t rxqid) { struct iavf_sc *sc = iavf_sc_from_ctx(ctx); struct iavf_vsi *vsi = &sc->vsi; struct iavf_hw *hw = vsi->hw; struct iavf_rx_queue *rx_que = &vsi->rx_queues[rxqid]; iavf_enable_queue_irq(hw, rx_que->msix - 1); return (0); } /** * iavf_if_tx_queue_intr_enable - Enable one Tx queue interrupt * @ctx: the iflib context pointer * @txqid: Tx queue index * * Enables the interrupt associated with a specified Tx queue. * * @returns zero */ static int iavf_if_tx_queue_intr_enable(if_ctx_t ctx, uint16_t txqid) { struct iavf_sc *sc = iavf_sc_from_ctx(ctx); struct iavf_vsi *vsi = &sc->vsi; struct iavf_hw *hw = vsi->hw; struct iavf_tx_queue *tx_que = &vsi->tx_queues[txqid]; iavf_enable_queue_irq(hw, tx_que->msix - 1); return (0); } /** * iavf_if_tx_queues_alloc - Allocate Tx queue memory * @ctx: the iflib context pointer * @vaddrs: Array of virtual addresses * @paddrs: Array of physical addresses * @ntxqs: the number of Tx queues per group (should always be 1) * @ntxqsets: the number of Tx queues * * Allocates memory for the specified number of Tx queues. This includes * memory for the queue structures and the report status array for the queues. * The virtual and physical addresses are saved for later use during * initialization. * * @returns zero or a non-zero error code on failure */ static int iavf_if_tx_queues_alloc(if_ctx_t ctx, caddr_t *vaddrs, uint64_t *paddrs, int ntxqs, int ntxqsets) { struct iavf_sc *sc = iavf_sc_from_ctx(ctx); struct iavf_vsi *vsi = &sc->vsi; if_softc_ctx_t scctx = vsi->shared; struct iavf_tx_queue *que; int i, j, error = 0; MPASS(scctx->isc_ntxqsets > 0); MPASS(ntxqs == 1); MPASS(scctx->isc_ntxqsets == ntxqsets); /* Allocate queue structure memory */ if (!(vsi->tx_queues = (struct iavf_tx_queue *)malloc(sizeof(struct iavf_tx_queue) *ntxqsets, M_IAVF, M_NOWAIT | M_ZERO))) { device_printf(iflib_get_dev(ctx), "Unable to allocate TX ring memory\n"); return (ENOMEM); } for (i = 0, que = vsi->tx_queues; i < ntxqsets; i++, que++) { struct tx_ring *txr = &que->txr; txr->me = i; que->vsi = vsi; if (!vsi->enable_head_writeback) { /* Allocate report status array */ if (!(txr->tx_rsq = (qidx_t *)malloc(sizeof(qidx_t) * scctx->isc_ntxd[0], M_IAVF, M_NOWAIT))) { device_printf(iflib_get_dev(ctx), "failed to allocate tx_rsq memory\n"); error = ENOMEM; goto fail; } /* Init report status array */ for (j = 0; j < scctx->isc_ntxd[0]; j++) txr->tx_rsq[j] = QIDX_INVALID; } /* get the virtual and physical address of the hardware queues */ txr->tail = IAVF_QTX_TAIL1(txr->me); txr->tx_base = (struct iavf_tx_desc *)vaddrs[i * ntxqs]; txr->tx_paddr = paddrs[i * ntxqs]; txr->que = que; } return (0); fail: iavf_if_queues_free(ctx); return (error); } /** * iavf_if_rx_queues_alloc - Allocate Rx queue memory * @ctx: the iflib context pointer * @vaddrs: Array of virtual addresses * @paddrs: Array of physical addresses * @nrxqs: number of Rx queues per group (should always be 1) * @nrxqsets: the number of Rx queues to allocate * * Called by iflib to allocate driver memory for a number of Rx queues. * Allocates memory for the drivers private Rx queue data structure, and saves * the physical and virtual addresses for later use. * * @returns zero or a non-zero error code on failure */ static int iavf_if_rx_queues_alloc(if_ctx_t ctx, caddr_t *vaddrs, uint64_t *paddrs, int nrxqs, int nrxqsets) { struct iavf_sc *sc = iavf_sc_from_ctx(ctx); struct iavf_vsi *vsi = &sc->vsi; struct iavf_rx_queue *que; int i, error = 0; #ifdef INVARIANTS if_softc_ctx_t scctx = vsi->shared; MPASS(scctx->isc_nrxqsets > 0); MPASS(nrxqs == 1); MPASS(scctx->isc_nrxqsets == nrxqsets); #endif /* Allocate queue structure memory */ if (!(vsi->rx_queues = (struct iavf_rx_queue *) malloc(sizeof(struct iavf_rx_queue) * nrxqsets, M_IAVF, M_NOWAIT | M_ZERO))) { device_printf(iflib_get_dev(ctx), "Unable to allocate RX ring memory\n"); error = ENOMEM; goto fail; } for (i = 0, que = vsi->rx_queues; i < nrxqsets; i++, que++) { struct rx_ring *rxr = &que->rxr; rxr->me = i; que->vsi = vsi; /* get the virtual and physical address of the hardware queues */ rxr->tail = IAVF_QRX_TAIL1(rxr->me); rxr->rx_base = (union iavf_rx_desc *)vaddrs[i * nrxqs]; rxr->rx_paddr = paddrs[i * nrxqs]; rxr->que = que; } return (0); fail: iavf_if_queues_free(ctx); return (error); } /** * iavf_if_queues_free - Free driver queue memory * @ctx: the iflib context pointer * * Called by iflib to release memory allocated by the driver when setting up * Tx and Rx queues. * * @remark The ordering of this function and iavf_if_detach is not guaranteed. * It is possible for this function to be called either before or after the * iavf_if_detach. Thus, care must be taken to ensure that either ordering of * iavf_if_detach and iavf_if_queues_free is safe. */ static void iavf_if_queues_free(if_ctx_t ctx) { struct iavf_sc *sc = iavf_sc_from_ctx(ctx); struct iavf_vsi *vsi = &sc->vsi; if (!vsi->enable_head_writeback) { struct iavf_tx_queue *que; int i = 0; for (i = 0, que = vsi->tx_queues; i < vsi->shared->isc_ntxqsets; i++, que++) { struct tx_ring *txr = &que->txr; if (txr->tx_rsq != NULL) { free(txr->tx_rsq, M_IAVF); txr->tx_rsq = NULL; } } } if (vsi->tx_queues != NULL) { free(vsi->tx_queues, M_IAVF); vsi->tx_queues = NULL; } if (vsi->rx_queues != NULL) { free(vsi->rx_queues, M_IAVF); vsi->rx_queues = NULL; } } /** * iavf_check_aq_errors - Check for AdminQ errors * @sc: device softc * * Check the AdminQ registers for errors, and determine whether or not a reset * may be required to resolve them. * * @post if there are errors, the VF device will be stopped and a reset will * be requested. * * @returns zero if there are no issues, EBUSY if the device is resetting, * or EIO if there are any AQ errors. */ static int iavf_check_aq_errors(struct iavf_sc *sc) { struct iavf_hw *hw = &sc->hw; device_t dev = sc->dev; u32 reg, oldreg; u8 aq_error = false; oldreg = reg = rd32(hw, hw->aq.arq.len); /* Check if device is in reset */ if (reg == 0xdeadbeef || reg == 0xffffffff) { device_printf(dev, "VF in reset\n"); return (EBUSY); } /* Check for Admin queue errors */ if (reg & IAVF_VF_ARQLEN1_ARQVFE_MASK) { device_printf(dev, "ARQ VF Error detected\n"); reg &= ~IAVF_VF_ARQLEN1_ARQVFE_MASK; aq_error = true; } if (reg & IAVF_VF_ARQLEN1_ARQOVFL_MASK) { device_printf(dev, "ARQ Overflow Error detected\n"); reg &= ~IAVF_VF_ARQLEN1_ARQOVFL_MASK; aq_error = true; } if (reg & IAVF_VF_ARQLEN1_ARQCRIT_MASK) { device_printf(dev, "ARQ Critical Error detected\n"); reg &= ~IAVF_VF_ARQLEN1_ARQCRIT_MASK; aq_error = true; } if (oldreg != reg) wr32(hw, hw->aq.arq.len, reg); oldreg = reg = rd32(hw, hw->aq.asq.len); if (reg & IAVF_VF_ATQLEN1_ATQVFE_MASK) { device_printf(dev, "ASQ VF Error detected\n"); reg &= ~IAVF_VF_ATQLEN1_ATQVFE_MASK; aq_error = true; } if (reg & IAVF_VF_ATQLEN1_ATQOVFL_MASK) { device_printf(dev, "ASQ Overflow Error detected\n"); reg &= ~IAVF_VF_ATQLEN1_ATQOVFL_MASK; aq_error = true; } if (reg & IAVF_VF_ATQLEN1_ATQCRIT_MASK) { device_printf(dev, "ASQ Critical Error detected\n"); reg &= ~IAVF_VF_ATQLEN1_ATQCRIT_MASK; aq_error = true; } if (oldreg != reg) wr32(hw, hw->aq.asq.len, reg); return (aq_error ? EIO : 0); } /** * iavf_process_adminq - Process adminq responses from the PF * @sc: device softc * @pending: output parameter indicating how many messages remain * * Process the adminq to handle replies from the PF over the virtchnl * connection. * * @returns zero or an iavf_status code on failure */ static enum iavf_status iavf_process_adminq(struct iavf_sc *sc, u16 *pending) { enum iavf_status status = IAVF_SUCCESS; struct iavf_arq_event_info event; struct iavf_hw *hw = &sc->hw; struct virtchnl_msg *v_msg; int error = 0, loop = 0; u32 reg; if (iavf_test_state(&sc->state, IAVF_STATE_RESET_PENDING)) { status = IAVF_ERR_ADMIN_QUEUE_ERROR; goto reenable_interrupt; } error = iavf_check_aq_errors(sc); if (error) { status = IAVF_ERR_ADMIN_QUEUE_CRITICAL_ERROR; goto reenable_interrupt; } event.buf_len = IAVF_AQ_BUF_SZ; event.msg_buf = sc->aq_buffer; bzero(event.msg_buf, IAVF_AQ_BUF_SZ); v_msg = (struct virtchnl_msg *)&event.desc; IAVF_VC_LOCK(sc); /* clean and process any events */ do { status = iavf_clean_arq_element(hw, &event, pending); /* * Also covers normal case when iavf_clean_arq_element() * returns "IAVF_ERR_ADMIN_QUEUE_NO_WORK" */ if (status) break; iavf_vc_completion(sc, v_msg->v_opcode, v_msg->v_retval, event.msg_buf, event.msg_len); bzero(event.msg_buf, IAVF_AQ_BUF_SZ); } while (*pending && (loop++ < IAVF_ADM_LIMIT)); IAVF_VC_UNLOCK(sc); reenable_interrupt: /* Re-enable admin queue interrupt cause */ reg = rd32(hw, IAVF_VFINT_ICR0_ENA1); reg |= IAVF_VFINT_ICR0_ENA1_ADMINQ_MASK; wr32(hw, IAVF_VFINT_ICR0_ENA1, reg); return (status); } /** * iavf_if_update_admin_status - Administrative status task * @ctx: iflib context * * Called by iflib to handle administrative status events. The iavf driver * uses this to process the adminq virtchnl messages outside of interrupt * context. */ static void iavf_if_update_admin_status(if_ctx_t ctx) { struct iavf_sc *sc = iavf_sc_from_ctx(ctx); struct iavf_hw *hw = &sc->hw; u16 pending = 0; iavf_process_adminq(sc, &pending); iavf_update_link_status(sc); /* * If there are still messages to process, reschedule. * Otherwise, re-enable the Admin Queue interrupt. */ if (pending > 0) iflib_admin_intr_deferred(ctx); else iavf_enable_adminq_irq(hw); } /** * iavf_if_multi_set - Set multicast address filters * @ctx: iflib context * * Called by iflib to update the current list of multicast filters for the * device. */ static void iavf_if_multi_set(if_ctx_t ctx) { struct iavf_sc *sc = iavf_sc_from_ctx(ctx); iavf_multi_set(sc); } /** * iavf_if_mtu_set - Set the device MTU * @ctx: iflib context * @mtu: MTU value to set * * Called by iflib to set the device MTU. * * @returns zero on success, or EINVAL if the MTU is invalid. */ static int iavf_if_mtu_set(if_ctx_t ctx, uint32_t mtu) { struct iavf_sc *sc = iavf_sc_from_ctx(ctx); struct iavf_vsi *vsi = &sc->vsi; IOCTL_DEBUGOUT("ioctl: SiOCSIFMTU (Set Interface MTU)"); if (mtu < IAVF_MIN_MTU || mtu > IAVF_MAX_MTU) { device_printf(sc->dev, "mtu %d is not in valid range [%d-%d]\n", mtu, IAVF_MIN_MTU, IAVF_MAX_MTU); return (EINVAL); } vsi->shared->isc_max_frame_size = mtu + ETHER_HDR_LEN + ETHER_CRC_LEN + ETHER_VLAN_ENCAP_LEN; return (0); } /** * iavf_if_media_status - Report current media status * @ctx: iflib context * @ifmr: ifmedia request structure * * Called by iflib to report the current media status in the ifmr. */ static void iavf_if_media_status(if_ctx_t ctx, struct ifmediareq *ifmr) { struct iavf_sc *sc = iavf_sc_from_ctx(ctx); iavf_media_status_common(sc, ifmr); } /** * iavf_if_media_change - Change the current media settings * @ctx: iflib context * * Called by iflib to change the current media settings. * * @returns zero on success, or an error code on failure. */ static int iavf_if_media_change(if_ctx_t ctx) { return iavf_media_change_common(iflib_get_ifp(ctx)); } /** * iavf_if_promisc_set - Set device promiscuous mode * @ctx: iflib context * @flags: promiscuous configuration * * Called by iflib to request that the device enter promiscuous mode. * * @returns zero on success, or an error code on failure. */ static int iavf_if_promisc_set(if_ctx_t ctx, int flags) { struct iavf_sc *sc = iavf_sc_from_ctx(ctx); return iavf_config_promisc(sc, flags); } /** * iavf_if_timer - Periodic timer called by iflib * @ctx: iflib context * @qid: The queue being triggered * * Called by iflib periodically as a timer task, so that the driver can handle * periodic work. * * @remark this timer is only called while the interface is up, even if * IFLIB_ADMIN_ALWAYS_RUN is set. */ static void iavf_if_timer(if_ctx_t ctx, uint16_t qid) { struct iavf_sc *sc = iavf_sc_from_ctx(ctx); struct iavf_hw *hw = &sc->hw; u32 val; if (qid != 0) return; /* Check for when PF triggers a VF reset */ val = rd32(hw, IAVF_VFGEN_RSTAT) & IAVF_VFGEN_RSTAT_VFR_STATE_MASK; if (val != VIRTCHNL_VFR_VFACTIVE && val != VIRTCHNL_VFR_COMPLETED) { iavf_dbg_info(sc, "reset in progress! (%d)\n", val); return; } /* Fire off the adminq task */ iflib_admin_intr_deferred(ctx); /* Update stats */ iavf_request_stats(sc); } /** * iavf_if_vlan_register - Register a VLAN * @ctx: iflib context * @vtag: the VLAN to register * * Register a VLAN filter for a given vtag. */ static void iavf_if_vlan_register(if_ctx_t ctx, u16 vtag) { struct iavf_sc *sc = iavf_sc_from_ctx(ctx); struct iavf_vsi *vsi = &sc->vsi; if ((vtag == 0) || (vtag > 4095)) /* Invalid */ return; /* Add VLAN 0 to list, for untagged traffic */ if (vsi->num_vlans == 0) iavf_add_vlan_filter(sc, 0); iavf_add_vlan_filter(sc, vtag); ++vsi->num_vlans; iavf_send_vc_msg(sc, IAVF_FLAG_AQ_ADD_VLAN_FILTER); } /** * iavf_if_vlan_unregister - Unregister a VLAN * @ctx: iflib context * @vtag: the VLAN to remove * * Unregister (remove) a VLAN filter for the given vtag. */ static void iavf_if_vlan_unregister(if_ctx_t ctx, u16 vtag) { struct iavf_sc *sc = iavf_sc_from_ctx(ctx); struct iavf_vsi *vsi = &sc->vsi; int i = 0; if ((vtag == 0) || (vtag > 4095) || (vsi->num_vlans == 0)) /* Invalid */ return; i = iavf_mark_del_vlan_filter(sc, vtag); vsi->num_vlans -= i; /* Remove VLAN filter 0 if the last VLAN is being removed */ if (vsi->num_vlans == 0) i += iavf_mark_del_vlan_filter(sc, 0); if (i > 0) iavf_send_vc_msg(sc, IAVF_FLAG_AQ_DEL_VLAN_FILTER); } /** * iavf_if_get_counter - Get network statistic counters * @ctx: iflib context * @cnt: The counter to obtain * * Called by iflib to obtain the value of the specified counter. * * @returns the uint64_t counter value. */ static uint64_t iavf_if_get_counter(if_ctx_t ctx, ift_counter cnt) { struct iavf_sc *sc = iavf_sc_from_ctx(ctx); struct iavf_vsi *vsi = &sc->vsi; if_t ifp = iflib_get_ifp(ctx); switch (cnt) { case IFCOUNTER_IPACKETS: return (vsi->ipackets); case IFCOUNTER_IERRORS: return (vsi->ierrors); case IFCOUNTER_OPACKETS: return (vsi->opackets); case IFCOUNTER_OERRORS: return (vsi->oerrors); case IFCOUNTER_COLLISIONS: /* Collisions are by standard impossible in 40G/10G Ethernet */ return (0); case IFCOUNTER_IBYTES: return (vsi->ibytes); case IFCOUNTER_OBYTES: return (vsi->obytes); case IFCOUNTER_IMCASTS: return (vsi->imcasts); case IFCOUNTER_OMCASTS: return (vsi->omcasts); case IFCOUNTER_IQDROPS: return (vsi->iqdrops); case IFCOUNTER_OQDROPS: return (vsi->oqdrops); case IFCOUNTER_NOPROTO: return (vsi->noproto); default: return (if_get_counter_default(ifp, cnt)); } } /* iavf_if_needs_restart - Tell iflib when the driver needs to be reinitialized * @ctx: iflib context * @event: event code to check * * Defaults to returning false for unknown events. * * @returns true if iflib needs to reinit the interface */ static bool iavf_if_needs_restart(if_ctx_t ctx __unused, enum iflib_restart_event event) { switch (event) { case IFLIB_RESTART_VLAN_CONFIG: return (true); default: return (false); } } /** * iavf_free_pci_resources - Free PCI resources * @sc: device softc * * Called to release the PCI resources allocated during attach. May be called * in the error flow of attach_pre, or during detach as part of cleanup. */ static void iavf_free_pci_resources(struct iavf_sc *sc) { struct iavf_vsi *vsi = &sc->vsi; struct iavf_rx_queue *rx_que = vsi->rx_queues; device_t dev = sc->dev; /* We may get here before stations are set up */ if (rx_que == NULL) goto early; /* Release all interrupts */ iflib_irq_free(vsi->ctx, &vsi->irq); for (int i = 0; i < vsi->num_rx_queues; i++, rx_que++) iflib_irq_free(vsi->ctx, &rx_que->que_irq); early: if (sc->pci_mem != NULL) bus_release_resource(dev, SYS_RES_MEMORY, rman_get_rid(sc->pci_mem), sc->pci_mem); } /** * iavf_setup_interface - Setup the device interface * @sc: device softc * * Called to setup some device interface settings, such as the ifmedia * structure. */ static void iavf_setup_interface(struct iavf_sc *sc) { struct iavf_vsi *vsi = &sc->vsi; if_ctx_t ctx = vsi->ctx; if_t ifp = iflib_get_ifp(ctx); iavf_dbg_init(sc, "begin\n"); vsi->shared->isc_max_frame_size = if_getmtu(ifp) + ETHER_HDR_LEN + ETHER_CRC_LEN + ETHER_VLAN_ENCAP_LEN; iavf_set_initial_baudrate(ifp); ifmedia_add(sc->media, IFM_ETHER | IFM_AUTO, 0, NULL); ifmedia_set(sc->media, IFM_ETHER | IFM_AUTO); } /** * iavf_msix_adminq - Admin Queue interrupt handler * @arg: void pointer to the device softc * * Interrupt handler for the non-queue interrupt causes. Primarily this will * be the adminq interrupt, but also includes other miscellaneous causes. * * @returns FILTER_SCHEDULE_THREAD if the admin task needs to be run, otherwise * returns FITLER_HANDLED. */ static int iavf_msix_adminq(void *arg) { struct iavf_sc *sc = (struct iavf_sc *)arg; struct iavf_hw *hw = &sc->hw; u32 reg, mask; ++sc->admin_irq; if (!iavf_test_state(&sc->state, IAVF_STATE_INITIALIZED)) return (FILTER_HANDLED); reg = rd32(hw, IAVF_VFINT_ICR01); /* * For masking off interrupt causes that need to be handled before * they can be re-enabled */ mask = rd32(hw, IAVF_VFINT_ICR0_ENA1); /* Check on the cause */ if (reg & IAVF_VFINT_ICR01_ADMINQ_MASK) { mask &= ~IAVF_VFINT_ICR0_ENA1_ADMINQ_MASK; /* Process messages outside of the iflib context lock */ taskqueue_enqueue(sc->vc_tq, &sc->vc_task); } wr32(hw, IAVF_VFINT_ICR0_ENA1, mask); iavf_enable_adminq_irq(hw); return (FILTER_HANDLED); } /** * iavf_enable_intr - Enable device interrupts * @vsi: the main VSI * * Called to enable all queue interrupts. */ void iavf_enable_intr(struct iavf_vsi *vsi) { struct iavf_hw *hw = vsi->hw; struct iavf_rx_queue *que = vsi->rx_queues; iavf_enable_adminq_irq(hw); for (int i = 0; i < vsi->num_rx_queues; i++, que++) iavf_enable_queue_irq(hw, que->rxr.me); } /** * iavf_disable_intr - Disable device interrupts * @vsi: the main VSI * * Called to disable all interrupts * * @remark we never disable the admin status interrupt. */ void iavf_disable_intr(struct iavf_vsi *vsi) { struct iavf_hw *hw = vsi->hw; struct iavf_rx_queue *que = vsi->rx_queues; for (int i = 0; i < vsi->num_rx_queues; i++, que++) iavf_disable_queue_irq(hw, que->rxr.me); } /** * iavf_enable_queue_irq - Enable IRQ register for a queue interrupt * @hw: hardware structure * @id: IRQ vector to enable * * Writes the IAVF_VFINT_DYN_CTLN1 register to enable a given IRQ interrupt. */ static void iavf_enable_queue_irq(struct iavf_hw *hw, int id) { u32 reg; reg = IAVF_VFINT_DYN_CTLN1_INTENA_MASK | IAVF_VFINT_DYN_CTLN1_CLEARPBA_MASK | IAVF_VFINT_DYN_CTLN1_ITR_INDX_MASK; wr32(hw, IAVF_VFINT_DYN_CTLN1(id), reg); } /** * iavf_disable_queue_irq - Disable IRQ register for a queue interrupt * @hw: hardware structure * @id: IRQ vector to disable * * Writes the IAVF_VFINT_DYN_CTLN1 register to disable a given IRQ interrupt. */ static void iavf_disable_queue_irq(struct iavf_hw *hw, int id) { wr32(hw, IAVF_VFINT_DYN_CTLN1(id), IAVF_VFINT_DYN_CTLN1_ITR_INDX_MASK); rd32(hw, IAVF_VFGEN_RSTAT); } /** * iavf_configure_itr - Get initial ITR values from tunable values. * @sc: device softc * * Load the initial tunable values for the ITR configuration. */ static void iavf_configure_itr(struct iavf_sc *sc) { iavf_configure_tx_itr(sc); iavf_configure_rx_itr(sc); } /** * iavf_set_queue_rx_itr - Update Rx ITR value * @que: Rx queue to update * * Provide a update to the queue RX interrupt moderation value. */ static void iavf_set_queue_rx_itr(struct iavf_rx_queue *que) { struct iavf_vsi *vsi = que->vsi; struct iavf_hw *hw = vsi->hw; struct rx_ring *rxr = &que->rxr; /* Idle, do nothing */ if (rxr->bytes == 0) return; /* Update the hardware if needed */ if (rxr->itr != vsi->rx_itr_setting) { rxr->itr = vsi->rx_itr_setting; wr32(hw, IAVF_VFINT_ITRN1(IAVF_RX_ITR, que->rxr.me), rxr->itr); } } /** * iavf_msix_que - Main Rx queue interrupt handler * @arg: void pointer to the Rx queue * * Main MSI-X interrupt handler for Rx queue interrupts * * @returns FILTER_SCHEDULE_THREAD if the main thread for Rx needs to run, * otherwise returns FILTER_HANDLED. */ static int iavf_msix_que(void *arg) { struct iavf_rx_queue *rx_que = (struct iavf_rx_queue *)arg; struct iavf_sc *sc = rx_que->vsi->back; ++rx_que->irqs; if (!iavf_test_state(&sc->state, IAVF_STATE_RUNNING)) return (FILTER_HANDLED); iavf_set_queue_rx_itr(rx_que); return (FILTER_SCHEDULE_THREAD); } /** * iavf_update_link_status - Update iflib Link status * @sc: device softc * * Notify the iflib stack of changes in link status. Called after the device * receives a virtchnl message indicating a change in link status. */ void iavf_update_link_status(struct iavf_sc *sc) { struct iavf_vsi *vsi = &sc->vsi; u64 baudrate; if (sc->link_up){ if (vsi->link_active == FALSE) { vsi->link_active = TRUE; baudrate = iavf_baudrate_from_link_speed(sc); iavf_dbg_info(sc, "baudrate: %llu\n", (unsigned long long)baudrate); iflib_link_state_change(vsi->ctx, LINK_STATE_UP, baudrate); } } else { /* Link down */ if (vsi->link_active == TRUE) { vsi->link_active = FALSE; iflib_link_state_change(vsi->ctx, LINK_STATE_DOWN, 0); } } } /** * iavf_stop - Stop the interface * @sc: device softc * * This routine disables all traffic on the adapter by disabling interrupts * and sending a message to the PF to tell it to stop the hardware * Tx/Rx LAN queues. */ static void iavf_stop(struct iavf_sc *sc) { iavf_clear_state(&sc->state, IAVF_STATE_RUNNING); iavf_disable_intr(&sc->vsi); iavf_disable_queues_with_retries(sc); } /** * iavf_if_stop - iflib stop handler * @ctx: iflib context * * Call iavf_stop to stop the interface. */ static void iavf_if_stop(if_ctx_t ctx) { struct iavf_sc *sc = iavf_sc_from_ctx(ctx); iavf_stop(sc); } /** * iavf_del_mac_filter - Delete a MAC filter * @sc: device softc * @macaddr: MAC address to remove * * Marks a MAC filter for deletion. * * @returns zero if the filter existed, or ENOENT if it did not. */ static int iavf_del_mac_filter(struct iavf_sc *sc, u8 *macaddr) { struct iavf_mac_filter *f; f = iavf_find_mac_filter(sc, macaddr); if (f == NULL) return (ENOENT); f->flags |= IAVF_FILTER_DEL; return (0); } /** * iavf_init_tx_rsqs - Initialize Report Status array * @vsi: the main VSI * * Set the Report Status queue fields to zero in order to initialize the * queues for transmit. */ void iavf_init_tx_rsqs(struct iavf_vsi *vsi) { if_softc_ctx_t scctx = vsi->shared; struct iavf_tx_queue *tx_que; int i, j; for (i = 0, tx_que = vsi->tx_queues; i < vsi->num_tx_queues; i++, tx_que++) { struct tx_ring *txr = &tx_que->txr; txr->tx_rs_cidx = txr->tx_rs_pidx; /* Initialize the last processed descriptor to be the end of * the ring, rather than the start, so that we avoid an * off-by-one error when calculating how many descriptors are * done in the credits_update function. */ txr->tx_cidx_processed = scctx->isc_ntxd[0] - 1; for (j = 0; j < scctx->isc_ntxd[0]; j++) txr->tx_rsq[j] = QIDX_INVALID; } } /** * iavf_init_tx_cidx - Initialize Tx cidx values * @vsi: the main VSI * * Initialize the tx_cidx_processed values for Tx queues in order to * initialize the Tx queues for transmit. */ void iavf_init_tx_cidx(struct iavf_vsi *vsi) { if_softc_ctx_t scctx = vsi->shared; struct iavf_tx_queue *tx_que; int i; for (i = 0, tx_que = vsi->tx_queues; i < vsi->num_tx_queues; i++, tx_que++) { struct tx_ring *txr = &tx_que->txr; txr->tx_cidx_processed = scctx->isc_ntxd[0] - 1; } } /** * iavf_add_device_sysctls - Add device sysctls for configuration * @sc: device softc * * Add the main sysctl nodes and sysctls for device configuration. */ static void iavf_add_device_sysctls(struct iavf_sc *sc) { struct iavf_vsi *vsi = &sc->vsi; device_t dev = sc->dev; struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(dev); struct sysctl_oid_list *debug_list; iavf_add_device_sysctls_common(sc); debug_list = iavf_create_debug_sysctl_tree(sc); iavf_add_debug_sysctls_common(sc, debug_list); SYSCTL_ADD_PROC(ctx, debug_list, OID_AUTO, "queue_interrupt_table", CTLTYPE_STRING | CTLFLAG_RD, sc, 0, iavf_sysctl_queue_interrupt_table, "A", "View MSI-X indices for TX/RX queues"); #ifdef IAVF_DEBUG SYSCTL_ADD_PROC(ctx, debug_list, OID_AUTO, "do_vf_reset", CTLTYPE_INT | CTLFLAG_WR, sc, 0, iavf_sysctl_vf_reset, "A", "Request a VF reset from PF"); SYSCTL_ADD_PROC(ctx, debug_list, OID_AUTO, "do_vflr_reset", CTLTYPE_INT | CTLFLAG_WR, sc, 0, iavf_sysctl_vflr_reset, "A", "Request a VFLR reset from HW"); #endif /* Add stats sysctls */ iavf_add_vsi_sysctls(dev, vsi, ctx, "vsi"); iavf_add_queues_sysctls(dev, vsi); } /** * iavf_add_queues_sysctls - Add per-queue sysctls * @dev: device pointer * @vsi: the main VSI * * Add sysctls for each Tx and Rx queue. */ void iavf_add_queues_sysctls(device_t dev, struct iavf_vsi *vsi) { struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(dev); struct sysctl_oid_list *vsi_list, *queue_list; struct sysctl_oid *queue_node; char queue_namebuf[32]; struct iavf_rx_queue *rx_que; struct iavf_tx_queue *tx_que; struct tx_ring *txr; struct rx_ring *rxr; vsi_list = SYSCTL_CHILDREN(vsi->vsi_node); /* Queue statistics */ for (int q = 0; q < vsi->num_rx_queues; q++) { bzero(queue_namebuf, sizeof(queue_namebuf)); snprintf(queue_namebuf, IAVF_QUEUE_NAME_LEN, "rxq%02d", q); queue_node = SYSCTL_ADD_NODE(ctx, vsi_list, OID_AUTO, queue_namebuf, CTLFLAG_RD, NULL, "RX Queue #"); queue_list = SYSCTL_CHILDREN(queue_node); rx_que = &(vsi->rx_queues[q]); rxr = &(rx_que->rxr); SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "irqs", CTLFLAG_RD, &(rx_que->irqs), "irqs on this queue (both Tx and Rx)"); SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "packets", CTLFLAG_RD, &(rxr->rx_packets), "Queue Packets Received"); SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "bytes", CTLFLAG_RD, &(rxr->rx_bytes), "Queue Bytes Received"); SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "desc_err", CTLFLAG_RD, &(rxr->desc_errs), "Queue Rx Descriptor Errors"); SYSCTL_ADD_UINT(ctx, queue_list, OID_AUTO, "itr", CTLFLAG_RD, &(rxr->itr), 0, "Queue Rx ITR Interval"); } for (int q = 0; q < vsi->num_tx_queues; q++) { bzero(queue_namebuf, sizeof(queue_namebuf)); snprintf(queue_namebuf, IAVF_QUEUE_NAME_LEN, "txq%02d", q); queue_node = SYSCTL_ADD_NODE(ctx, vsi_list, OID_AUTO, queue_namebuf, CTLFLAG_RD, NULL, "TX Queue #"); queue_list = SYSCTL_CHILDREN(queue_node); tx_que = &(vsi->tx_queues[q]); txr = &(tx_que->txr); SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "tso", CTLFLAG_RD, &(tx_que->tso), "TSO"); SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "mss_too_small", CTLFLAG_RD, &(txr->mss_too_small), "TSO sends with an MSS less than 64"); SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "packets", CTLFLAG_RD, &(txr->tx_packets), "Queue Packets Transmitted"); SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "bytes", CTLFLAG_RD, &(txr->tx_bytes), "Queue Bytes Transmitted"); SYSCTL_ADD_UINT(ctx, queue_list, OID_AUTO, "itr", CTLFLAG_RD, &(txr->itr), 0, "Queue Tx ITR Interval"); } } /** * iavf_driver_is_detaching - Check if the driver is detaching/unloading * @sc: device private softc * * @returns true if the driver is detaching, false otherwise. * * @remark on newer kernels, take advantage of iflib_in_detach in order to * report detachment correctly as early as possible. * * @remark this function is used by various code paths that want to avoid * running if the driver is about to be removed. This includes sysctls and * other driver access points. Note that it does not fully resolve * detach-based race conditions as it is possible for a thread to race with * iflib_in_detach. */ bool iavf_driver_is_detaching(struct iavf_sc *sc) { return (!iavf_test_state(&sc->state, IAVF_STATE_INITIALIZED) || iflib_in_detach(sc->vsi.ctx)); } /** * iavf_sysctl_queue_interrupt_table - Sysctl for displaying Tx queue mapping * @oidp: sysctl oid structure * @arg1: void pointer to device softc * @arg2: unused * @req: sysctl request pointer * * Print out mapping of TX queue indexes and Rx queue indexes to MSI-X vectors. * * @returns zero on success, or an error code on failure. */ static int iavf_sysctl_queue_interrupt_table(SYSCTL_HANDLER_ARGS) { struct iavf_sc *sc = (struct iavf_sc *)arg1; struct iavf_vsi *vsi = &sc->vsi; device_t dev = sc->dev; struct sbuf *buf; int error = 0; struct iavf_rx_queue *rx_que; struct iavf_tx_queue *tx_que; UNREFERENCED_2PARAMETER(arg2, oidp); if (iavf_driver_is_detaching(sc)) return (ESHUTDOWN); buf = sbuf_new_for_sysctl(NULL, NULL, 128, req); if (!buf) { device_printf(dev, "Could not allocate sbuf for output.\n"); return (ENOMEM); } sbuf_cat(buf, "\n"); for (int i = 0; i < vsi->num_rx_queues; i++) { rx_que = &vsi->rx_queues[i]; sbuf_printf(buf, "(rxq %3d): %d\n", i, rx_que->msix); } for (int i = 0; i < vsi->num_tx_queues; i++) { tx_que = &vsi->tx_queues[i]; sbuf_printf(buf, "(txq %3d): %d\n", i, tx_que->msix); } error = sbuf_finish(buf); if (error) device_printf(dev, "Error finishing sbuf: %d\n", error); sbuf_delete(buf); return (error); } #ifdef IAVF_DEBUG #define CTX_ACTIVE(ctx) ((if_getdrvflags(iflib_get_ifp(ctx)) & IFF_DRV_RUNNING)) /** * iavf_sysctl_vf_reset - Request a VF reset * @oidp: sysctl oid pointer * @arg1: void pointer to device softc * @arg2: unused * @req: sysctl request pointer * * Request a VF reset for the device. * * @returns zero on success, or an error code on failure. */ static int iavf_sysctl_vf_reset(SYSCTL_HANDLER_ARGS) { struct iavf_sc *sc = (struct iavf_sc *)arg1; int do_reset = 0, error = 0; UNREFERENCED_PARAMETER(arg2); if (iavf_driver_is_detaching(sc)) return (ESHUTDOWN); error = sysctl_handle_int(oidp, &do_reset, 0, req); if ((error) || (req->newptr == NULL)) return (error); if (do_reset == 1) { iavf_reset(sc); if (CTX_ACTIVE(sc->vsi.ctx)) iflib_request_reset(sc->vsi.ctx); } return (error); } /** * iavf_sysctl_vflr_reset - Trigger a PCIe FLR for the device * @oidp: sysctl oid pointer * @arg1: void pointer to device softc * @arg2: unused * @req: sysctl request pointer * * Sysctl callback to trigger a PCIe FLR. * * @returns zero on success, or an error code on failure. */ static int iavf_sysctl_vflr_reset(SYSCTL_HANDLER_ARGS) { struct iavf_sc *sc = (struct iavf_sc *)arg1; device_t dev = sc->dev; int do_reset = 0, error = 0; UNREFERENCED_PARAMETER(arg2); if (iavf_driver_is_detaching(sc)) return (ESHUTDOWN); error = sysctl_handle_int(oidp, &do_reset, 0, req); if ((error) || (req->newptr == NULL)) return (error); if (do_reset == 1) { if (!pcie_flr(dev, max(pcie_get_max_completion_timeout(dev) / 1000, 10), true)) { device_printf(dev, "PCIE FLR failed\n"); error = EIO; } else if (CTX_ACTIVE(sc->vsi.ctx)) iflib_request_reset(sc->vsi.ctx); } return (error); } #undef CTX_ACTIVE #endif diff --git a/sys/dev/pci/pci.c b/sys/dev/pci/pci.c index 07a9534f6b91..7107fbe4884b 100644 --- a/sys/dev/pci/pci.c +++ b/sys/dev/pci/pci.c @@ -1,6982 +1,6996 @@ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 1997, Stefan Esser * Copyright (c) 2000, Michael Smith * Copyright (c) 2000, BSDi * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice unmodified, this list of conditions, and the following * disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include #include "opt_acpi.h" #include "opt_iommu.h" #include "opt_bus.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #if defined(__i386__) || defined(__amd64__) || defined(__powerpc__) #include #endif #include #include #include #include #ifdef PCI_IOV #include #include #endif #include #include #include #include #include #include "pcib_if.h" #include "pci_if.h" #define PCIR_IS_BIOS(cfg, reg) \ (((cfg)->hdrtype == PCIM_HDRTYPE_NORMAL && reg == PCIR_BIOS) || \ ((cfg)->hdrtype == PCIM_HDRTYPE_BRIDGE && reg == PCIR_BIOS_1)) static device_probe_t pci_probe; static bus_reset_post_t pci_reset_post; static bus_reset_prepare_t pci_reset_prepare; static bus_reset_child_t pci_reset_child; static bus_hint_device_unit_t pci_hint_device_unit; static bus_remap_intr_t pci_remap_intr_method; static pci_get_id_t pci_get_id_method; static int pci_has_quirk(uint32_t devid, int quirk); static pci_addr_t pci_mapbase(uint64_t mapreg); static const char *pci_maptype(uint64_t mapreg); static int pci_maprange(uint64_t mapreg); static pci_addr_t pci_rombase(uint64_t mapreg); static int pci_romsize(uint64_t testval); static void pci_fixancient(pcicfgregs *cfg); static int pci_printf(pcicfgregs *cfg, const char *fmt, ...); static int pci_porten(device_t dev); static int pci_memen(device_t dev); static void pci_assign_interrupt(device_t bus, device_t dev, int force_route); static int pci_add_map(device_t bus, device_t dev, int reg, struct resource_list *rl, int force, int prefetch); static void pci_load_vendor_data(void); static int pci_describe_parse_line(char **ptr, int *vendor, int *device, char **desc); static char *pci_describe_device(device_t dev); static int pci_modevent(module_t mod, int what, void *arg); static void pci_hdrtypedata(device_t pcib, int b, int s, int f, pcicfgregs *cfg); static void pci_read_cap(device_t pcib, pcicfgregs *cfg); static int pci_read_vpd_reg(device_t pcib, pcicfgregs *cfg, int reg, uint32_t *data); #if 0 static int pci_write_vpd_reg(device_t pcib, pcicfgregs *cfg, int reg, uint32_t data); #endif static void pci_read_vpd(device_t pcib, pcicfgregs *cfg); static void pci_mask_msix(device_t dev, u_int index); static void pci_unmask_msix(device_t dev, u_int index); static int pci_msi_blacklisted(void); static int pci_msix_blacklisted(void); static void pci_resume_msi(device_t dev); static void pci_resume_msix(device_t dev); static struct pci_devinfo * pci_fill_devinfo(device_t pcib, device_t bus, int d, int b, int s, int f, uint16_t vid, uint16_t did); static device_method_t pci_methods[] = { /* Device interface */ DEVMETHOD(device_probe, pci_probe), DEVMETHOD(device_attach, pci_attach), DEVMETHOD(device_detach, pci_detach), DEVMETHOD(device_shutdown, bus_generic_shutdown), DEVMETHOD(device_suspend, bus_generic_suspend), DEVMETHOD(device_resume, pci_resume), /* Bus interface */ DEVMETHOD(bus_print_child, pci_print_child), DEVMETHOD(bus_probe_nomatch, pci_probe_nomatch), DEVMETHOD(bus_read_ivar, pci_read_ivar), DEVMETHOD(bus_write_ivar, pci_write_ivar), DEVMETHOD(bus_driver_added, pci_driver_added), DEVMETHOD(bus_setup_intr, pci_setup_intr), DEVMETHOD(bus_teardown_intr, pci_teardown_intr), DEVMETHOD(bus_reset_prepare, pci_reset_prepare), DEVMETHOD(bus_reset_post, pci_reset_post), DEVMETHOD(bus_reset_child, pci_reset_child), DEVMETHOD(bus_get_dma_tag, pci_get_dma_tag), DEVMETHOD(bus_get_resource_list,pci_get_resource_list), DEVMETHOD(bus_set_resource, bus_generic_rl_set_resource), DEVMETHOD(bus_get_resource, bus_generic_rl_get_resource), DEVMETHOD(bus_delete_resource, pci_delete_resource), DEVMETHOD(bus_alloc_resource, pci_alloc_resource), DEVMETHOD(bus_adjust_resource, pci_adjust_resource), DEVMETHOD(bus_release_resource, pci_release_resource), DEVMETHOD(bus_activate_resource, pci_activate_resource), DEVMETHOD(bus_deactivate_resource, pci_deactivate_resource), DEVMETHOD(bus_map_resource, pci_map_resource), DEVMETHOD(bus_unmap_resource, pci_unmap_resource), DEVMETHOD(bus_child_deleted, pci_child_deleted), DEVMETHOD(bus_child_detached, pci_child_detached), DEVMETHOD(bus_child_pnpinfo, pci_child_pnpinfo_method), DEVMETHOD(bus_child_location, pci_child_location_method), DEVMETHOD(bus_get_device_path, pci_get_device_path_method), DEVMETHOD(bus_hint_device_unit, pci_hint_device_unit), DEVMETHOD(bus_remap_intr, pci_remap_intr_method), DEVMETHOD(bus_suspend_child, pci_suspend_child), DEVMETHOD(bus_resume_child, pci_resume_child), DEVMETHOD(bus_rescan, pci_rescan_method), /* PCI interface */ DEVMETHOD(pci_read_config, pci_read_config_method), DEVMETHOD(pci_write_config, pci_write_config_method), DEVMETHOD(pci_enable_busmaster, pci_enable_busmaster_method), DEVMETHOD(pci_disable_busmaster, pci_disable_busmaster_method), DEVMETHOD(pci_enable_io, pci_enable_io_method), DEVMETHOD(pci_disable_io, pci_disable_io_method), DEVMETHOD(pci_get_vpd_ident, pci_get_vpd_ident_method), DEVMETHOD(pci_get_vpd_readonly, pci_get_vpd_readonly_method), DEVMETHOD(pci_get_powerstate, pci_get_powerstate_method), DEVMETHOD(pci_set_powerstate, pci_set_powerstate_method), DEVMETHOD(pci_assign_interrupt, pci_assign_interrupt_method), DEVMETHOD(pci_find_cap, pci_find_cap_method), DEVMETHOD(pci_find_next_cap, pci_find_next_cap_method), DEVMETHOD(pci_find_extcap, pci_find_extcap_method), DEVMETHOD(pci_find_next_extcap, pci_find_next_extcap_method), DEVMETHOD(pci_find_htcap, pci_find_htcap_method), DEVMETHOD(pci_find_next_htcap, pci_find_next_htcap_method), DEVMETHOD(pci_alloc_msi, pci_alloc_msi_method), DEVMETHOD(pci_alloc_msix, pci_alloc_msix_method), DEVMETHOD(pci_enable_msi, pci_enable_msi_method), DEVMETHOD(pci_enable_msix, pci_enable_msix_method), DEVMETHOD(pci_disable_msi, pci_disable_msi_method), DEVMETHOD(pci_remap_msix, pci_remap_msix_method), DEVMETHOD(pci_release_msi, pci_release_msi_method), DEVMETHOD(pci_msi_count, pci_msi_count_method), DEVMETHOD(pci_msix_count, pci_msix_count_method), DEVMETHOD(pci_msix_pba_bar, pci_msix_pba_bar_method), DEVMETHOD(pci_msix_table_bar, pci_msix_table_bar_method), DEVMETHOD(pci_get_id, pci_get_id_method), DEVMETHOD(pci_alloc_devinfo, pci_alloc_devinfo_method), DEVMETHOD(pci_child_added, pci_child_added_method), #ifdef PCI_IOV DEVMETHOD(pci_iov_attach, pci_iov_attach_method), DEVMETHOD(pci_iov_detach, pci_iov_detach_method), DEVMETHOD(pci_create_iov_child, pci_create_iov_child_method), #endif DEVMETHOD_END }; DEFINE_CLASS_0(pci, pci_driver, pci_methods, sizeof(struct pci_softc)); EARLY_DRIVER_MODULE(pci, pcib, pci_driver, pci_modevent, NULL, BUS_PASS_BUS); MODULE_VERSION(pci, 1); static char *pci_vendordata; static size_t pci_vendordata_size; struct pci_quirk { uint32_t devid; /* Vendor/device of the card */ int type; #define PCI_QUIRK_MAP_REG 1 /* PCI map register in weird place */ #define PCI_QUIRK_DISABLE_MSI 2 /* Neither MSI nor MSI-X work */ #define PCI_QUIRK_ENABLE_MSI_VM 3 /* Older chipset in VM where MSI works */ #define PCI_QUIRK_UNMAP_REG 4 /* Ignore PCI map register */ #define PCI_QUIRK_DISABLE_MSIX 5 /* MSI-X doesn't work */ #define PCI_QUIRK_MSI_INTX_BUG 6 /* PCIM_CMD_INTxDIS disables MSI */ #define PCI_QUIRK_REALLOC_BAR 7 /* Can't allocate memory at the default address */ int arg1; int arg2; }; static const struct pci_quirk pci_quirks[] = { /* The Intel 82371AB and 82443MX have a map register at offset 0x90. */ { 0x71138086, PCI_QUIRK_MAP_REG, 0x90, 0 }, { 0x719b8086, PCI_QUIRK_MAP_REG, 0x90, 0 }, /* As does the Serverworks OSB4 (the SMBus mapping register) */ { 0x02001166, PCI_QUIRK_MAP_REG, 0x90, 0 }, /* * MSI doesn't work with the ServerWorks CNB20-HE Host Bridge * or the CMIC-SL (AKA ServerWorks GC_LE). */ { 0x00141166, PCI_QUIRK_DISABLE_MSI, 0, 0 }, { 0x00171166, PCI_QUIRK_DISABLE_MSI, 0, 0 }, /* * MSI doesn't work on earlier Intel chipsets including * E7500, E7501, E7505, 845, 865, 875/E7210, and 855. */ { 0x25408086, PCI_QUIRK_DISABLE_MSI, 0, 0 }, { 0x254c8086, PCI_QUIRK_DISABLE_MSI, 0, 0 }, { 0x25508086, PCI_QUIRK_DISABLE_MSI, 0, 0 }, { 0x25608086, PCI_QUIRK_DISABLE_MSI, 0, 0 }, { 0x25708086, PCI_QUIRK_DISABLE_MSI, 0, 0 }, { 0x25788086, PCI_QUIRK_DISABLE_MSI, 0, 0 }, { 0x35808086, PCI_QUIRK_DISABLE_MSI, 0, 0 }, /* * MSI doesn't work with devices behind the AMD 8131 HT-PCIX * bridge. */ { 0x74501022, PCI_QUIRK_DISABLE_MSI, 0, 0 }, /* * Some virtualization environments emulate an older chipset * but support MSI just fine. QEMU uses the Intel 82440. */ { 0x12378086, PCI_QUIRK_ENABLE_MSI_VM, 0, 0 }, /* * HPET MMIO base address may appear in Bar1 for AMD SB600 SMBus * controller depending on SoftPciRst register (PM_IO 0x55 [7]). * It prevents us from attaching hpet(4) when the bit is unset. * Note this quirk only affects SB600 revision A13 and earlier. * For SB600 A21 and later, firmware must set the bit to hide it. * For SB700 and later, it is unused and hardcoded to zero. */ { 0x43851002, PCI_QUIRK_UNMAP_REG, 0x14, 0 }, /* * Atheros AR8161/AR8162/E2200/E2400/E2500 Ethernet controllers have * a bug that MSI interrupt does not assert if PCIM_CMD_INTxDIS bit * of the command register is set. */ { 0x10911969, PCI_QUIRK_MSI_INTX_BUG, 0, 0 }, { 0xE0911969, PCI_QUIRK_MSI_INTX_BUG, 0, 0 }, { 0xE0A11969, PCI_QUIRK_MSI_INTX_BUG, 0, 0 }, { 0xE0B11969, PCI_QUIRK_MSI_INTX_BUG, 0, 0 }, { 0x10901969, PCI_QUIRK_MSI_INTX_BUG, 0, 0 }, /* * Broadcom BCM5714(S)/BCM5715(S)/BCM5780(S) Ethernet MACs don't * issue MSI interrupts with PCIM_CMD_INTxDIS set either. */ { 0x166814e4, PCI_QUIRK_MSI_INTX_BUG, 0, 0 }, /* BCM5714 */ { 0x166914e4, PCI_QUIRK_MSI_INTX_BUG, 0, 0 }, /* BCM5714S */ { 0x166a14e4, PCI_QUIRK_MSI_INTX_BUG, 0, 0 }, /* BCM5780 */ { 0x166b14e4, PCI_QUIRK_MSI_INTX_BUG, 0, 0 }, /* BCM5780S */ { 0x167814e4, PCI_QUIRK_MSI_INTX_BUG, 0, 0 }, /* BCM5715 */ { 0x167914e4, PCI_QUIRK_MSI_INTX_BUG, 0, 0 }, /* BCM5715S */ /* * HPE Gen 10 VGA has a memory range that can't be allocated in the * expected place. */ { 0x98741002, PCI_QUIRK_REALLOC_BAR, 0, 0 }, { 0 } }; /* map register information */ #define PCI_MAPMEM 0x01 /* memory map */ #define PCI_MAPMEMP 0x02 /* prefetchable memory map */ #define PCI_MAPPORT 0x04 /* port map */ struct devlist pci_devq; uint32_t pci_generation; uint32_t pci_numdevs = 0; static int pcie_chipset, pcix_chipset; /* sysctl vars */ SYSCTL_NODE(_hw, OID_AUTO, pci, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, "PCI bus tuning parameters"); static int pci_enable_io_modes = 1; SYSCTL_INT(_hw_pci, OID_AUTO, enable_io_modes, CTLFLAG_RWTUN, &pci_enable_io_modes, 1, "Enable I/O and memory bits in the config register. Some BIOSes do not" " enable these bits correctly. We'd like to do this all the time, but" " there are some peripherals that this causes problems with."); static int pci_do_realloc_bars = 1; SYSCTL_INT(_hw_pci, OID_AUTO, realloc_bars, CTLFLAG_RWTUN, &pci_do_realloc_bars, 0, "Attempt to allocate a new range for any BARs whose original " "firmware-assigned ranges fail to allocate during the initial device scan."); static int pci_do_power_nodriver = 0; SYSCTL_INT(_hw_pci, OID_AUTO, do_power_nodriver, CTLFLAG_RWTUN, &pci_do_power_nodriver, 0, "Place a function into D3 state when no driver attaches to it. 0 means" " disable. 1 means conservatively place devices into D3 state. 2 means" " aggressively place devices into D3 state. 3 means put absolutely" " everything in D3 state."); int pci_do_power_resume = 1; SYSCTL_INT(_hw_pci, OID_AUTO, do_power_resume, CTLFLAG_RWTUN, &pci_do_power_resume, 1, "Transition from D3 -> D0 on resume."); int pci_do_power_suspend = 1; SYSCTL_INT(_hw_pci, OID_AUTO, do_power_suspend, CTLFLAG_RWTUN, &pci_do_power_suspend, 1, "Transition from D0 -> D3 on suspend."); static int pci_do_msi = 1; SYSCTL_INT(_hw_pci, OID_AUTO, enable_msi, CTLFLAG_RWTUN, &pci_do_msi, 1, "Enable support for MSI interrupts"); static int pci_do_msix = 1; SYSCTL_INT(_hw_pci, OID_AUTO, enable_msix, CTLFLAG_RWTUN, &pci_do_msix, 1, "Enable support for MSI-X interrupts"); static int pci_msix_rewrite_table = 0; SYSCTL_INT(_hw_pci, OID_AUTO, msix_rewrite_table, CTLFLAG_RWTUN, &pci_msix_rewrite_table, 0, "Rewrite entire MSI-X table when updating MSI-X entries"); static int pci_honor_msi_blacklist = 1; SYSCTL_INT(_hw_pci, OID_AUTO, honor_msi_blacklist, CTLFLAG_RDTUN, &pci_honor_msi_blacklist, 1, "Honor chipset blacklist for MSI/MSI-X"); #if defined(__i386__) || defined(__amd64__) static int pci_usb_takeover = 1; #else static int pci_usb_takeover = 0; #endif SYSCTL_INT(_hw_pci, OID_AUTO, usb_early_takeover, CTLFLAG_RDTUN, &pci_usb_takeover, 1, "Enable early takeover of USB controllers. Disable this if you depend on" " BIOS emulation of USB devices, that is you use USB devices (like" " keyboard or mouse) but do not load USB drivers"); static int pci_clear_bars; SYSCTL_INT(_hw_pci, OID_AUTO, clear_bars, CTLFLAG_RDTUN, &pci_clear_bars, 0, "Ignore firmware-assigned resources for BARs."); #if defined(NEW_PCIB) && defined(PCI_RES_BUS) static int pci_clear_buses; SYSCTL_INT(_hw_pci, OID_AUTO, clear_buses, CTLFLAG_RDTUN, &pci_clear_buses, 0, "Ignore firmware-assigned bus numbers."); #endif static int pci_enable_ari = 1; SYSCTL_INT(_hw_pci, OID_AUTO, enable_ari, CTLFLAG_RDTUN, &pci_enable_ari, 0, "Enable support for PCIe Alternative RID Interpretation"); /* * Some x86 firmware only enables PCIe hotplug if we claim to support aspm, * however enabling it breaks some arm64 firmware as it powers off devices. */ #if defined(__i386__) || defined(__amd64__) int pci_enable_aspm = 1; #else int pci_enable_aspm = 0; #endif SYSCTL_INT(_hw_pci, OID_AUTO, enable_aspm, CTLFLAG_RDTUN, &pci_enable_aspm, 0, "Enable support for PCIe Active State Power Management"); static int pci_clear_aer_on_attach = 0; SYSCTL_INT(_hw_pci, OID_AUTO, clear_aer_on_attach, CTLFLAG_RWTUN, &pci_clear_aer_on_attach, 0, "Clear port and device AER state on driver attach"); static bool pci_enable_mps_tune = true; SYSCTL_BOOL(_hw_pci, OID_AUTO, enable_mps_tune, CTLFLAG_RWTUN, &pci_enable_mps_tune, 1, "Enable tuning of MPS(maximum payload size)." ); static bool pci_intx_reroute = true; SYSCTL_BOOL(_hw_pci, OID_AUTO, intx_reroute, CTLFLAG_RWTUN, &pci_intx_reroute, 0, "Re-route INTx interrupts when scanning devices"); static int pci_has_quirk(uint32_t devid, int quirk) { const struct pci_quirk *q; for (q = &pci_quirks[0]; q->devid; q++) { if (q->devid == devid && q->type == quirk) return (1); } return (0); } /* Find a device_t by bus/slot/function in domain 0 */ device_t pci_find_bsf(uint8_t bus, uint8_t slot, uint8_t func) { return (pci_find_dbsf(0, bus, slot, func)); } /* Find a device_t by domain/bus/slot/function */ device_t pci_find_dbsf(uint32_t domain, uint8_t bus, uint8_t slot, uint8_t func) { struct pci_devinfo *dinfo = NULL; STAILQ_FOREACH(dinfo, &pci_devq, pci_links) { if ((dinfo->cfg.domain == domain) && (dinfo->cfg.bus == bus) && (dinfo->cfg.slot == slot) && (dinfo->cfg.func == func)) { break; } } return (dinfo != NULL ? dinfo->cfg.dev : NULL); } /* Find a device_t by vendor/device ID */ device_t pci_find_device(uint16_t vendor, uint16_t device) { struct pci_devinfo *dinfo; STAILQ_FOREACH(dinfo, &pci_devq, pci_links) { if ((dinfo->cfg.vendor == vendor) && (dinfo->cfg.device == device)) { return (dinfo->cfg.dev); } } return (NULL); } device_t pci_find_class(uint8_t class, uint8_t subclass) { struct pci_devinfo *dinfo; STAILQ_FOREACH(dinfo, &pci_devq, pci_links) { if (dinfo->cfg.baseclass == class && dinfo->cfg.subclass == subclass) { return (dinfo->cfg.dev); } } return (NULL); } device_t pci_find_class_from(uint8_t class, uint8_t subclass, device_t from) { struct pci_devinfo *dinfo; bool found = false; STAILQ_FOREACH(dinfo, &pci_devq, pci_links) { if (from != NULL && found == false) { if (from != dinfo->cfg.dev) continue; found = true; continue; } if (dinfo->cfg.baseclass == class && dinfo->cfg.subclass == subclass) { return (dinfo->cfg.dev); } } return (NULL); } static int pci_printf(pcicfgregs *cfg, const char *fmt, ...) { va_list ap; int retval; retval = printf("pci%d:%d:%d:%d: ", cfg->domain, cfg->bus, cfg->slot, cfg->func); va_start(ap, fmt); retval += vprintf(fmt, ap); va_end(ap); return (retval); } /* return base address of memory or port map */ static pci_addr_t pci_mapbase(uint64_t mapreg) { if (PCI_BAR_MEM(mapreg)) return (mapreg & PCIM_BAR_MEM_BASE); else return (mapreg & PCIM_BAR_IO_BASE); } /* return map type of memory or port map */ static const char * pci_maptype(uint64_t mapreg) { if (PCI_BAR_IO(mapreg)) return ("I/O Port"); if (mapreg & PCIM_BAR_MEM_PREFETCH) return ("Prefetchable Memory"); return ("Memory"); } /* return log2 of map size decoded for memory or port map */ int pci_mapsize(uint64_t testval) { int ln2size; testval = pci_mapbase(testval); ln2size = 0; if (testval != 0) { while ((testval & 1) == 0) { ln2size++; testval >>= 1; } } return (ln2size); } /* return base address of device ROM */ static pci_addr_t pci_rombase(uint64_t mapreg) { return (mapreg & PCIM_BIOS_ADDR_MASK); } /* return log2 of map size decided for device ROM */ static int pci_romsize(uint64_t testval) { int ln2size; testval = pci_rombase(testval); ln2size = 0; if (testval != 0) { while ((testval & 1) == 0) { ln2size++; testval >>= 1; } } return (ln2size); } /* return log2 of address range supported by map register */ static int pci_maprange(uint64_t mapreg) { int ln2range = 0; if (PCI_BAR_IO(mapreg)) ln2range = 32; else switch (mapreg & PCIM_BAR_MEM_TYPE) { case PCIM_BAR_MEM_32: ln2range = 32; break; case PCIM_BAR_MEM_1MB: ln2range = 20; break; case PCIM_BAR_MEM_64: ln2range = 64; break; } return (ln2range); } /* adjust some values from PCI 1.0 devices to match 2.0 standards ... */ static void pci_fixancient(pcicfgregs *cfg) { if ((cfg->hdrtype & PCIM_HDRTYPE) != PCIM_HDRTYPE_NORMAL) return; /* PCI to PCI bridges use header type 1 */ if (cfg->baseclass == PCIC_BRIDGE && cfg->subclass == PCIS_BRIDGE_PCI) cfg->hdrtype = PCIM_HDRTYPE_BRIDGE; } /* extract header type specific config data */ static void pci_hdrtypedata(device_t pcib, int b, int s, int f, pcicfgregs *cfg) { #define REG(n, w) PCIB_READ_CONFIG(pcib, b, s, f, n, w) switch (cfg->hdrtype & PCIM_HDRTYPE) { case PCIM_HDRTYPE_NORMAL: cfg->subvendor = REG(PCIR_SUBVEND_0, 2); cfg->subdevice = REG(PCIR_SUBDEV_0, 2); cfg->mingnt = REG(PCIR_MINGNT, 1); cfg->maxlat = REG(PCIR_MAXLAT, 1); cfg->nummaps = PCI_MAXMAPS_0; break; case PCIM_HDRTYPE_BRIDGE: cfg->bridge.br_seclat = REG(PCIR_SECLAT_1, 1); cfg->bridge.br_subbus = REG(PCIR_SUBBUS_1, 1); cfg->bridge.br_secbus = REG(PCIR_SECBUS_1, 1); cfg->bridge.br_pribus = REG(PCIR_PRIBUS_1, 1); cfg->bridge.br_control = REG(PCIR_BRIDGECTL_1, 2); cfg->nummaps = PCI_MAXMAPS_1; break; case PCIM_HDRTYPE_CARDBUS: cfg->bridge.br_seclat = REG(PCIR_SECLAT_2, 1); cfg->bridge.br_subbus = REG(PCIR_SUBBUS_2, 1); cfg->bridge.br_secbus = REG(PCIR_SECBUS_2, 1); cfg->bridge.br_pribus = REG(PCIR_PRIBUS_2, 1); cfg->bridge.br_control = REG(PCIR_BRIDGECTL_2, 2); cfg->subvendor = REG(PCIR_SUBVEND_2, 2); cfg->subdevice = REG(PCIR_SUBDEV_2, 2); cfg->nummaps = PCI_MAXMAPS_2; break; } #undef REG } /* read configuration header into pcicfgregs structure */ struct pci_devinfo * pci_read_device(device_t pcib, device_t bus, int d, int b, int s, int f) { #define REG(n, w) PCIB_READ_CONFIG(pcib, b, s, f, n, w) uint16_t vid, did; vid = REG(PCIR_VENDOR, 2); if (vid == PCIV_INVALID) return (NULL); did = REG(PCIR_DEVICE, 2); return (pci_fill_devinfo(pcib, bus, d, b, s, f, vid, did)); } struct pci_devinfo * pci_alloc_devinfo_method(device_t dev) { return (malloc(sizeof(struct pci_devinfo), M_DEVBUF, M_WAITOK | M_ZERO)); } static struct pci_devinfo * pci_fill_devinfo(device_t pcib, device_t bus, int d, int b, int s, int f, uint16_t vid, uint16_t did) { struct pci_devinfo *devlist_entry; pcicfgregs *cfg; devlist_entry = PCI_ALLOC_DEVINFO(bus); cfg = &devlist_entry->cfg; cfg->domain = d; cfg->bus = b; cfg->slot = s; cfg->func = f; cfg->vendor = vid; cfg->device = did; cfg->cmdreg = REG(PCIR_COMMAND, 2); cfg->statreg = REG(PCIR_STATUS, 2); cfg->baseclass = REG(PCIR_CLASS, 1); cfg->subclass = REG(PCIR_SUBCLASS, 1); cfg->progif = REG(PCIR_PROGIF, 1); cfg->revid = REG(PCIR_REVID, 1); cfg->hdrtype = REG(PCIR_HDRTYPE, 1); cfg->cachelnsz = REG(PCIR_CACHELNSZ, 1); cfg->lattimer = REG(PCIR_LATTIMER, 1); cfg->intpin = REG(PCIR_INTPIN, 1); cfg->intline = REG(PCIR_INTLINE, 1); cfg->mfdev = (cfg->hdrtype & PCIM_MFDEV) != 0; cfg->hdrtype &= ~PCIM_MFDEV; STAILQ_INIT(&cfg->maps); cfg->iov = NULL; pci_fixancient(cfg); pci_hdrtypedata(pcib, b, s, f, cfg); if (REG(PCIR_STATUS, 2) & PCIM_STATUS_CAPPRESENT) pci_read_cap(pcib, cfg); STAILQ_INSERT_TAIL(&pci_devq, devlist_entry, pci_links); devlist_entry->conf.pc_sel.pc_domain = cfg->domain; devlist_entry->conf.pc_sel.pc_bus = cfg->bus; devlist_entry->conf.pc_sel.pc_dev = cfg->slot; devlist_entry->conf.pc_sel.pc_func = cfg->func; devlist_entry->conf.pc_hdr = cfg->hdrtype; devlist_entry->conf.pc_subvendor = cfg->subvendor; devlist_entry->conf.pc_subdevice = cfg->subdevice; devlist_entry->conf.pc_vendor = cfg->vendor; devlist_entry->conf.pc_device = cfg->device; devlist_entry->conf.pc_class = cfg->baseclass; devlist_entry->conf.pc_subclass = cfg->subclass; devlist_entry->conf.pc_progif = cfg->progif; devlist_entry->conf.pc_revid = cfg->revid; pci_numdevs++; pci_generation++; return (devlist_entry); } #undef REG static void pci_ea_fill_info(device_t pcib, pcicfgregs *cfg) { #define REG(n, w) PCIB_READ_CONFIG(pcib, cfg->bus, cfg->slot, cfg->func, \ cfg->ea.ea_location + (n), w) int num_ent; int ptr; int a, b; uint32_t val; int ent_size; uint32_t dw[4]; uint64_t base, max_offset; struct pci_ea_entry *eae; if (cfg->ea.ea_location == 0) return; STAILQ_INIT(&cfg->ea.ea_entries); /* Determine the number of entries */ num_ent = REG(PCIR_EA_NUM_ENT, 2); num_ent &= PCIM_EA_NUM_ENT_MASK; /* Find the first entry to care of */ ptr = PCIR_EA_FIRST_ENT; /* Skip DWORD 2 for type 1 functions */ if ((cfg->hdrtype & PCIM_HDRTYPE) == PCIM_HDRTYPE_BRIDGE) ptr += 4; for (a = 0; a < num_ent; a++) { eae = malloc(sizeof(*eae), M_DEVBUF, M_WAITOK | M_ZERO); eae->eae_cfg_offset = cfg->ea.ea_location + ptr; /* Read a number of dwords in the entry */ val = REG(ptr, 4); ptr += 4; ent_size = (val & PCIM_EA_ES); for (b = 0; b < ent_size; b++) { dw[b] = REG(ptr, 4); ptr += 4; } eae->eae_flags = val; eae->eae_bei = (PCIM_EA_BEI & val) >> PCIM_EA_BEI_OFFSET; base = dw[0] & PCIM_EA_FIELD_MASK; max_offset = dw[1] | ~PCIM_EA_FIELD_MASK; b = 2; if (((dw[0] & PCIM_EA_IS_64) != 0) && (b < ent_size)) { base |= (uint64_t)dw[b] << 32UL; b++; } if (((dw[1] & PCIM_EA_IS_64) != 0) && (b < ent_size)) { max_offset |= (uint64_t)dw[b] << 32UL; b++; } eae->eae_base = base; eae->eae_max_offset = max_offset; STAILQ_INSERT_TAIL(&cfg->ea.ea_entries, eae, eae_link); if (bootverbose) { printf("PCI(EA) dev %04x:%04x, bei %d, flags #%x, base #%jx, max_offset #%jx\n", cfg->vendor, cfg->device, eae->eae_bei, eae->eae_flags, (uintmax_t)eae->eae_base, (uintmax_t)eae->eae_max_offset); } } } #undef REG static void pci_read_cap(device_t pcib, pcicfgregs *cfg) { #define REG(n, w) PCIB_READ_CONFIG(pcib, cfg->bus, cfg->slot, cfg->func, n, w) #define WREG(n, v, w) PCIB_WRITE_CONFIG(pcib, cfg->bus, cfg->slot, cfg->func, n, v, w) #if defined(__i386__) || defined(__amd64__) || defined(__powerpc__) uint64_t addr; #endif uint32_t val; int ptr, nextptr, ptrptr; switch (cfg->hdrtype & PCIM_HDRTYPE) { case PCIM_HDRTYPE_NORMAL: case PCIM_HDRTYPE_BRIDGE: ptrptr = PCIR_CAP_PTR; break; case PCIM_HDRTYPE_CARDBUS: ptrptr = PCIR_CAP_PTR_2; /* cardbus capabilities ptr */ break; default: return; /* no extended capabilities support */ } nextptr = REG(ptrptr, 1); /* sanity check? */ /* * Read capability entries. */ while (nextptr != 0) { /* Sanity check */ if (nextptr > 255) { printf("illegal PCI extended capability offset %d\n", nextptr); return; } /* Find the next entry */ ptr = nextptr; nextptr = REG(ptr + PCICAP_NEXTPTR, 1); /* Process this entry */ switch (REG(ptr + PCICAP_ID, 1)) { case PCIY_PMG: /* PCI power management */ if (cfg->pp.pp_cap == 0) { cfg->pp.pp_cap = REG(ptr + PCIR_POWER_CAP, 2); cfg->pp.pp_status = ptr + PCIR_POWER_STATUS; cfg->pp.pp_bse = ptr + PCIR_POWER_BSE; if ((nextptr - ptr) > PCIR_POWER_DATA) cfg->pp.pp_data = ptr + PCIR_POWER_DATA; } break; case PCIY_HT: /* HyperTransport */ /* Determine HT-specific capability type. */ val = REG(ptr + PCIR_HT_COMMAND, 2); if ((val & 0xe000) == PCIM_HTCAP_SLAVE) cfg->ht.ht_slave = ptr; #if defined(__i386__) || defined(__amd64__) || defined(__powerpc__) switch (val & PCIM_HTCMD_CAP_MASK) { case PCIM_HTCAP_MSI_MAPPING: if (!(val & PCIM_HTCMD_MSI_FIXED)) { /* Sanity check the mapping window. */ addr = REG(ptr + PCIR_HTMSI_ADDRESS_HI, 4); addr <<= 32; addr |= REG(ptr + PCIR_HTMSI_ADDRESS_LO, 4); if (addr != MSI_INTEL_ADDR_BASE) device_printf(pcib, "HT device at pci%d:%d:%d:%d has non-default MSI window 0x%llx\n", cfg->domain, cfg->bus, cfg->slot, cfg->func, (long long)addr); } else addr = MSI_INTEL_ADDR_BASE; cfg->ht.ht_msimap = ptr; cfg->ht.ht_msictrl = val; cfg->ht.ht_msiaddr = addr; break; } #endif break; case PCIY_MSI: /* PCI MSI */ cfg->msi.msi_location = ptr; cfg->msi.msi_ctrl = REG(ptr + PCIR_MSI_CTRL, 2); - cfg->msi.msi_msgnum = 1 << ((cfg->msi.msi_ctrl & - PCIM_MSICTRL_MMC_MASK)>>1); break; case PCIY_MSIX: /* PCI MSI-X */ cfg->msix.msix_location = ptr; cfg->msix.msix_ctrl = REG(ptr + PCIR_MSIX_CTRL, 2); - cfg->msix.msix_msgnum = (cfg->msix.msix_ctrl & - PCIM_MSIXCTRL_TABLE_SIZE) + 1; val = REG(ptr + PCIR_MSIX_TABLE, 4); cfg->msix.msix_table_bar = PCIR_BAR(val & PCIM_MSIX_BIR_MASK); cfg->msix.msix_table_offset = val & ~PCIM_MSIX_BIR_MASK; val = REG(ptr + PCIR_MSIX_PBA, 4); cfg->msix.msix_pba_bar = PCIR_BAR(val & PCIM_MSIX_BIR_MASK); cfg->msix.msix_pba_offset = val & ~PCIM_MSIX_BIR_MASK; break; case PCIY_VPD: /* PCI Vital Product Data */ cfg->vpd.vpd_reg = ptr; break; case PCIY_SUBVENDOR: /* Should always be true. */ if ((cfg->hdrtype & PCIM_HDRTYPE) == PCIM_HDRTYPE_BRIDGE) { val = REG(ptr + PCIR_SUBVENDCAP_ID, 4); cfg->subvendor = val & 0xffff; cfg->subdevice = val >> 16; } break; case PCIY_PCIX: /* PCI-X */ /* * Assume we have a PCI-X chipset if we have * at least one PCI-PCI bridge with a PCI-X * capability. Note that some systems with * PCI-express or HT chipsets might match on * this check as well. */ if ((cfg->hdrtype & PCIM_HDRTYPE) == PCIM_HDRTYPE_BRIDGE) pcix_chipset = 1; cfg->pcix.pcix_location = ptr; break; case PCIY_EXPRESS: /* PCI-express */ /* * Assume we have a PCI-express chipset if we have * at least one PCI-express device. */ pcie_chipset = 1; cfg->pcie.pcie_location = ptr; val = REG(ptr + PCIER_FLAGS, 2); cfg->pcie.pcie_type = val & PCIEM_FLAGS_TYPE; break; case PCIY_EA: /* Enhanced Allocation */ cfg->ea.ea_location = ptr; pci_ea_fill_info(pcib, cfg); break; default: break; } } #if defined(__powerpc__) /* * Enable the MSI mapping window for all HyperTransport * slaves. PCI-PCI bridges have their windows enabled via * PCIB_MAP_MSI(). */ if (cfg->ht.ht_slave != 0 && cfg->ht.ht_msimap != 0 && !(cfg->ht.ht_msictrl & PCIM_HTCMD_MSI_ENABLE)) { device_printf(pcib, "Enabling MSI window for HyperTransport slave at pci%d:%d:%d:%d\n", cfg->domain, cfg->bus, cfg->slot, cfg->func); cfg->ht.ht_msictrl |= PCIM_HTCMD_MSI_ENABLE; WREG(cfg->ht.ht_msimap + PCIR_HT_COMMAND, cfg->ht.ht_msictrl, 2); } #endif /* REG and WREG use carry through to next functions */ } /* * PCI Vital Product Data */ #define PCI_VPD_TIMEOUT 1000000 static int pci_read_vpd_reg(device_t pcib, pcicfgregs *cfg, int reg, uint32_t *data) { int count = PCI_VPD_TIMEOUT; KASSERT((reg & 3) == 0, ("VPD register must by 4 byte aligned")); WREG(cfg->vpd.vpd_reg + PCIR_VPD_ADDR, reg, 2); while ((REG(cfg->vpd.vpd_reg + PCIR_VPD_ADDR, 2) & 0x8000) != 0x8000) { if (--count < 0) return (ENXIO); DELAY(1); /* limit looping */ } *data = (REG(cfg->vpd.vpd_reg + PCIR_VPD_DATA, 4)); return (0); } #if 0 static int pci_write_vpd_reg(device_t pcib, pcicfgregs *cfg, int reg, uint32_t data) { int count = PCI_VPD_TIMEOUT; KASSERT((reg & 3) == 0, ("VPD register must by 4 byte aligned")); WREG(cfg->vpd.vpd_reg + PCIR_VPD_DATA, data, 4); WREG(cfg->vpd.vpd_reg + PCIR_VPD_ADDR, reg | 0x8000, 2); while ((REG(cfg->vpd.vpd_reg + PCIR_VPD_ADDR, 2) & 0x8000) == 0x8000) { if (--count < 0) return (ENXIO); DELAY(1); /* limit looping */ } return (0); } #endif #undef PCI_VPD_TIMEOUT struct vpd_readstate { device_t pcib; pcicfgregs *cfg; uint32_t val; int bytesinval; int off; uint8_t cksum; }; /* return 0 and one byte in *data if no read error, -1 else */ static int vpd_nextbyte(struct vpd_readstate *vrs, uint8_t *data) { uint32_t reg; uint8_t byte; if (vrs->bytesinval == 0) { if (pci_read_vpd_reg(vrs->pcib, vrs->cfg, vrs->off, ®)) return (-1); vrs->val = le32toh(reg); vrs->off += 4; byte = vrs->val & 0xff; vrs->bytesinval = 3; } else { vrs->val = vrs->val >> 8; byte = vrs->val & 0xff; vrs->bytesinval--; } vrs->cksum += byte; *data = byte; return (0); } /* return 0 on match, -1 and "unget" byte on no match */ static int vpd_expectbyte(struct vpd_readstate *vrs, uint8_t expected) { uint8_t data; if (vpd_nextbyte(vrs, &data) != 0) return (-1); if (data == expected) return (0); vrs->cksum -= data; vrs->val = (vrs->val << 8) + data; vrs->bytesinval++; return (-1); } /* return size if tag matches, -1 on no match, -2 on read error */ static int vpd_read_tag_size(struct vpd_readstate *vrs, uint8_t vpd_tag) { uint8_t byte1, byte2; if (vpd_expectbyte(vrs, vpd_tag) != 0) return (-1); if ((vpd_tag & 0x80) == 0) return (vpd_tag & 0x07); if (vpd_nextbyte(vrs, &byte1) != 0) return (-2); if (vpd_nextbyte(vrs, &byte2) != 0) return (-2); return ((byte2 << 8) + byte1); } /* (re)allocate buffer in multiples of 8 elements */ static void* alloc_buffer(void* buffer, size_t element_size, int needed) { int alloc, new_alloc; alloc = roundup2(needed, 8); new_alloc = roundup2(needed + 1, 8); if (alloc != new_alloc) { buffer = reallocf(buffer, new_alloc * element_size, M_DEVBUF, M_WAITOK | M_ZERO); } return (buffer); } /* read VPD keyword and return element size, return -1 on read error */ static int vpd_read_elem_head(struct vpd_readstate *vrs, char keyword[2]) { uint8_t data; if (vpd_nextbyte(vrs, &keyword[0]) != 0) return (-1); if (vpd_nextbyte(vrs, &keyword[1]) != 0) return (-1); if (vpd_nextbyte(vrs, &data) != 0) return (-1); return (data); } /* read VPD data element of given size into allocated buffer */ static char * vpd_read_value(struct vpd_readstate *vrs, int size) { int i; char char1; char *value; value = malloc(size + 1, M_DEVBUF, M_WAITOK); for (i = 0; i < size; i++) { if (vpd_nextbyte(vrs, &char1) != 0) { free(value, M_DEVBUF); return (NULL); } value[i] = char1; } value[size] = '\0'; return (value); } /* read VPD into *keyword and *value, return length of data element */ static int vpd_read_elem_data(struct vpd_readstate *vrs, char keyword[2], char **value, int maxlen) { int len; len = vpd_read_elem_head(vrs, keyword); if (len < 0 || len > maxlen) return (-1); *value = vpd_read_value(vrs, len); return (len); } /* subtract all data following first byte from checksum of RV element */ static void vpd_fixup_cksum(struct vpd_readstate *vrs, char *rvstring, int len) { int i; uint8_t fixup; fixup = 0; for (i = 1; i < len; i++) fixup += rvstring[i]; vrs->cksum -= fixup; } /* fetch one read-only element and return size of heading + data */ static int next_vpd_ro_elem(struct vpd_readstate *vrs, int maxsize) { struct pcicfg_vpd *vpd; pcicfgregs *cfg; struct vpd_readonly *vpd_ros; int len; cfg = vrs->cfg; vpd = &cfg->vpd; if (maxsize < 3) return (-1); vpd->vpd_ros = alloc_buffer(vpd->vpd_ros, sizeof(*vpd->vpd_ros), vpd->vpd_rocnt); vpd_ros = &vpd->vpd_ros[vpd->vpd_rocnt]; maxsize -= 3; len = vpd_read_elem_data(vrs, vpd_ros->keyword, &vpd_ros->value, maxsize); if (vpd_ros->value == NULL) return (-1); vpd_ros->len = len; if (vpd_ros->keyword[0] == 'R' && vpd_ros->keyword[1] == 'V') { vpd_fixup_cksum(vrs, vpd_ros->value, len); if (vrs->cksum != 0) { pci_printf(cfg, "invalid VPD checksum %#hhx\n", vrs->cksum); return (-1); } } vpd->vpd_rocnt++; return (len + 3); } /* fetch one writable element and return size of heading + data */ static int next_vpd_rw_elem(struct vpd_readstate *vrs, int maxsize) { struct pcicfg_vpd *vpd; pcicfgregs *cfg; struct vpd_write *vpd_w; int len; cfg = vrs->cfg; vpd = &cfg->vpd; if (maxsize < 3) return (-1); vpd->vpd_w = alloc_buffer(vpd->vpd_w, sizeof(*vpd->vpd_w), vpd->vpd_wcnt); if (vpd->vpd_w == NULL) { pci_printf(cfg, "out of memory"); return (-1); } vpd_w = &vpd->vpd_w[vpd->vpd_wcnt]; maxsize -= 3; vpd_w->start = vrs->off + 3 - vrs->bytesinval; len = vpd_read_elem_data(vrs, vpd_w->keyword, &vpd_w->value, maxsize); if (vpd_w->value == NULL) return (-1); vpd_w->len = len; vpd->vpd_wcnt++; return (len + 3); } /* free all memory allocated for VPD data */ static void vpd_free(struct pcicfg_vpd *vpd) { int i; free(vpd->vpd_ident, M_DEVBUF); for (i = 0; i < vpd->vpd_rocnt; i++) free(vpd->vpd_ros[i].value, M_DEVBUF); free(vpd->vpd_ros, M_DEVBUF); vpd->vpd_rocnt = 0; for (i = 0; i < vpd->vpd_wcnt; i++) free(vpd->vpd_w[i].value, M_DEVBUF); free(vpd->vpd_w, M_DEVBUF); vpd->vpd_wcnt = 0; } #define VPD_TAG_END ((0x0f << 3) | 0) /* small tag, len == 0 */ #define VPD_TAG_IDENT (0x02 | 0x80) /* large tag */ #define VPD_TAG_RO (0x10 | 0x80) /* large tag */ #define VPD_TAG_RW (0x11 | 0x80) /* large tag */ static int pci_parse_vpd(device_t pcib, pcicfgregs *cfg) { struct vpd_readstate vrs; int cksumvalid; int size, elem_size; /* init vpd reader */ vrs.bytesinval = 0; vrs.off = 0; vrs.pcib = pcib; vrs.cfg = cfg; vrs.cksum = 0; /* read VPD ident element - mandatory */ size = vpd_read_tag_size(&vrs, VPD_TAG_IDENT); if (size <= 0) { pci_printf(cfg, "no VPD ident found\n"); return (0); } cfg->vpd.vpd_ident = vpd_read_value(&vrs, size); if (cfg->vpd.vpd_ident == NULL) { pci_printf(cfg, "error accessing VPD ident data\n"); return (0); } /* read VPD RO elements - mandatory */ size = vpd_read_tag_size(&vrs, VPD_TAG_RO); if (size <= 0) { pci_printf(cfg, "no read-only VPD data found\n"); return (0); } while (size > 0) { elem_size = next_vpd_ro_elem(&vrs, size); if (elem_size < 0) { pci_printf(cfg, "error accessing read-only VPD data\n"); return (-1); } size -= elem_size; } cksumvalid = (vrs.cksum == 0); if (!cksumvalid) return (-1); /* read VPD RW elements - optional */ size = vpd_read_tag_size(&vrs, VPD_TAG_RW); if (size == -2) return (-1); while (size > 0) { elem_size = next_vpd_rw_elem(&vrs, size); if (elem_size < 0) { pci_printf(cfg, "error accessing writeable VPD data\n"); return (-1); } size -= elem_size; } /* read empty END tag - mandatory */ size = vpd_read_tag_size(&vrs, VPD_TAG_END); if (size != 0) { pci_printf(cfg, "No valid VPD end tag found\n"); } return (0); } static void pci_read_vpd(device_t pcib, pcicfgregs *cfg) { int status; status = pci_parse_vpd(pcib, cfg); if (status < 0) vpd_free(&cfg->vpd); cfg->vpd.vpd_cached = 1; #undef REG #undef WREG } int pci_get_vpd_ident_method(device_t dev, device_t child, const char **identptr) { struct pci_devinfo *dinfo = device_get_ivars(child); pcicfgregs *cfg = &dinfo->cfg; if (!cfg->vpd.vpd_cached && cfg->vpd.vpd_reg != 0) pci_read_vpd(device_get_parent(dev), cfg); *identptr = cfg->vpd.vpd_ident; if (*identptr == NULL) return (ENXIO); return (0); } int pci_get_vpd_readonly_method(device_t dev, device_t child, const char *kw, const char **vptr) { struct pci_devinfo *dinfo = device_get_ivars(child); pcicfgregs *cfg = &dinfo->cfg; int i; if (!cfg->vpd.vpd_cached && cfg->vpd.vpd_reg != 0) pci_read_vpd(device_get_parent(dev), cfg); for (i = 0; i < cfg->vpd.vpd_rocnt; i++) if (memcmp(kw, cfg->vpd.vpd_ros[i].keyword, sizeof(cfg->vpd.vpd_ros[i].keyword)) == 0) { *vptr = cfg->vpd.vpd_ros[i].value; return (0); } *vptr = NULL; return (ENXIO); } struct pcicfg_vpd * pci_fetch_vpd_list(device_t dev) { struct pci_devinfo *dinfo = device_get_ivars(dev); pcicfgregs *cfg = &dinfo->cfg; if (!cfg->vpd.vpd_cached && cfg->vpd.vpd_reg != 0) pci_read_vpd(device_get_parent(device_get_parent(dev)), cfg); return (&cfg->vpd); } /* * Find the requested HyperTransport capability and return the offset * in configuration space via the pointer provided. The function * returns 0 on success and an error code otherwise. */ int pci_find_htcap_method(device_t dev, device_t child, int capability, int *capreg) { int ptr, error; uint16_t val; error = pci_find_cap(child, PCIY_HT, &ptr); if (error) return (error); /* * Traverse the capabilities list checking each HT capability * to see if it matches the requested HT capability. */ for (;;) { val = pci_read_config(child, ptr + PCIR_HT_COMMAND, 2); if (capability == PCIM_HTCAP_SLAVE || capability == PCIM_HTCAP_HOST) val &= 0xe000; else val &= PCIM_HTCMD_CAP_MASK; if (val == capability) { if (capreg != NULL) *capreg = ptr; return (0); } /* Skip to the next HT capability. */ if (pci_find_next_cap(child, PCIY_HT, ptr, &ptr) != 0) break; } return (ENOENT); } /* * Find the next requested HyperTransport capability after start and return * the offset in configuration space via the pointer provided. The function * returns 0 on success and an error code otherwise. */ int pci_find_next_htcap_method(device_t dev, device_t child, int capability, int start, int *capreg) { int ptr; uint16_t val; KASSERT(pci_read_config(child, start + PCICAP_ID, 1) == PCIY_HT, ("start capability is not HyperTransport capability")); ptr = start; /* * Traverse the capabilities list checking each HT capability * to see if it matches the requested HT capability. */ for (;;) { /* Skip to the next HT capability. */ if (pci_find_next_cap(child, PCIY_HT, ptr, &ptr) != 0) break; val = pci_read_config(child, ptr + PCIR_HT_COMMAND, 2); if (capability == PCIM_HTCAP_SLAVE || capability == PCIM_HTCAP_HOST) val &= 0xe000; else val &= PCIM_HTCMD_CAP_MASK; if (val == capability) { if (capreg != NULL) *capreg = ptr; return (0); } } return (ENOENT); } /* * Find the requested capability and return the offset in * configuration space via the pointer provided. The function returns * 0 on success and an error code otherwise. */ int pci_find_cap_method(device_t dev, device_t child, int capability, int *capreg) { struct pci_devinfo *dinfo = device_get_ivars(child); pcicfgregs *cfg = &dinfo->cfg; uint32_t status; uint8_t ptr; int cnt; /* * Check the CAP_LIST bit of the PCI status register first. */ status = pci_read_config(child, PCIR_STATUS, 2); if (!(status & PCIM_STATUS_CAPPRESENT)) return (ENXIO); /* * Determine the start pointer of the capabilities list. */ switch (cfg->hdrtype & PCIM_HDRTYPE) { case PCIM_HDRTYPE_NORMAL: case PCIM_HDRTYPE_BRIDGE: ptr = PCIR_CAP_PTR; break; case PCIM_HDRTYPE_CARDBUS: ptr = PCIR_CAP_PTR_2; break; default: /* XXX: panic? */ return (ENXIO); /* no extended capabilities support */ } ptr = pci_read_config(child, ptr, 1); /* * Traverse the capabilities list. Limit by total theoretical * maximum number of caps: capability needs at least id and * next registers, and any type X header cannot contain caps. */ for (cnt = 0; ptr != 0 && cnt < (PCIE_REGMAX - 0x40) / 2; cnt++) { if (pci_read_config(child, ptr + PCICAP_ID, 1) == capability) { if (capreg != NULL) *capreg = ptr; return (0); } ptr = pci_read_config(child, ptr + PCICAP_NEXTPTR, 1); } return (ENOENT); } /* * Find the next requested capability after start and return the offset in * configuration space via the pointer provided. The function returns * 0 on success and an error code otherwise. */ int pci_find_next_cap_method(device_t dev, device_t child, int capability, int start, int *capreg) { uint8_t ptr; KASSERT(pci_read_config(child, start + PCICAP_ID, 1) == capability, ("start capability is not expected capability")); ptr = pci_read_config(child, start + PCICAP_NEXTPTR, 1); while (ptr != 0) { if (pci_read_config(child, ptr + PCICAP_ID, 1) == capability) { if (capreg != NULL) *capreg = ptr; return (0); } ptr = pci_read_config(child, ptr + PCICAP_NEXTPTR, 1); } return (ENOENT); } /* * Find the requested extended capability and return the offset in * configuration space via the pointer provided. The function returns * 0 on success and an error code otherwise. */ int pci_find_extcap_method(device_t dev, device_t child, int capability, int *capreg) { struct pci_devinfo *dinfo = device_get_ivars(child); pcicfgregs *cfg = &dinfo->cfg; uint32_t ecap; uint16_t ptr; /* Only supported for PCI-express devices. */ if (cfg->pcie.pcie_location == 0) return (ENXIO); ptr = PCIR_EXTCAP; ecap = pci_read_config(child, ptr, 4); if (ecap == 0xffffffff || ecap == 0) return (ENOENT); for (;;) { if (PCI_EXTCAP_ID(ecap) == capability) { if (capreg != NULL) *capreg = ptr; return (0); } ptr = PCI_EXTCAP_NEXTPTR(ecap); if (ptr == 0) break; ecap = pci_read_config(child, ptr, 4); } return (ENOENT); } /* * Find the next requested extended capability after start and return the * offset in configuration space via the pointer provided. The function * returns 0 on success and an error code otherwise. */ int pci_find_next_extcap_method(device_t dev, device_t child, int capability, int start, int *capreg) { struct pci_devinfo *dinfo = device_get_ivars(child); pcicfgregs *cfg = &dinfo->cfg; uint32_t ecap; uint16_t ptr; /* Only supported for PCI-express devices. */ if (cfg->pcie.pcie_location == 0) return (ENXIO); ecap = pci_read_config(child, start, 4); KASSERT(PCI_EXTCAP_ID(ecap) == capability, ("start extended capability is not expected capability")); ptr = PCI_EXTCAP_NEXTPTR(ecap); while (ptr != 0) { ecap = pci_read_config(child, ptr, 4); if (PCI_EXTCAP_ID(ecap) == capability) { if (capreg != NULL) *capreg = ptr; return (0); } ptr = PCI_EXTCAP_NEXTPTR(ecap); } return (ENOENT); } /* * Support for MSI-X message interrupts. */ static void pci_write_msix_entry(device_t dev, u_int index, uint64_t address, uint32_t data) { struct pci_devinfo *dinfo = device_get_ivars(dev); struct pcicfg_msix *msix = &dinfo->cfg.msix; uint32_t offset; KASSERT(msix->msix_table_len > index, ("bogus index")); offset = msix->msix_table_offset + index * 16; bus_write_4(msix->msix_table_res, offset, address & 0xffffffff); bus_write_4(msix->msix_table_res, offset + 4, address >> 32); bus_write_4(msix->msix_table_res, offset + 8, data); } void pci_enable_msix_method(device_t dev, device_t child, u_int index, uint64_t address, uint32_t data) { if (pci_msix_rewrite_table) { struct pci_devinfo *dinfo = device_get_ivars(child); struct pcicfg_msix *msix = &dinfo->cfg.msix; /* * Some VM hosts require MSIX to be disabled in the * control register before updating the MSIX table * entries are allowed. It is not enough to only * disable MSIX while updating a single entry. MSIX * must be disabled while updating all entries in the * table. */ pci_write_config(child, msix->msix_location + PCIR_MSIX_CTRL, msix->msix_ctrl & ~PCIM_MSIXCTRL_MSIX_ENABLE, 2); pci_resume_msix(child); } else pci_write_msix_entry(child, index, address, data); /* Enable MSI -> HT mapping. */ pci_ht_map_msi(child, address); } void pci_mask_msix(device_t dev, u_int index) { struct pci_devinfo *dinfo = device_get_ivars(dev); struct pcicfg_msix *msix = &dinfo->cfg.msix; uint32_t offset, val; - KASSERT(msix->msix_msgnum > index, ("bogus index")); + KASSERT(PCI_MSIX_MSGNUM(msix->msix_ctrl) > index, ("bogus index")); offset = msix->msix_table_offset + index * 16 + 12; val = bus_read_4(msix->msix_table_res, offset); val |= PCIM_MSIX_VCTRL_MASK; /* * Some devices (e.g. Samsung PM961) do not support reads of this * register, so always write the new value. */ bus_write_4(msix->msix_table_res, offset, val); } void pci_unmask_msix(device_t dev, u_int index) { struct pci_devinfo *dinfo = device_get_ivars(dev); struct pcicfg_msix *msix = &dinfo->cfg.msix; uint32_t offset, val; - KASSERT(msix->msix_table_len > index, ("bogus index")); + KASSERT(PCI_MSIX_MSGNUM(msix->msix_ctrl) > index, ("bogus index")); offset = msix->msix_table_offset + index * 16 + 12; val = bus_read_4(msix->msix_table_res, offset); val &= ~PCIM_MSIX_VCTRL_MASK; /* * Some devices (e.g. Samsung PM961) do not support reads of this * register, so always write the new value. */ bus_write_4(msix->msix_table_res, offset, val); } int pci_pending_msix(device_t dev, u_int index) { struct pci_devinfo *dinfo = device_get_ivars(dev); struct pcicfg_msix *msix = &dinfo->cfg.msix; uint32_t offset, bit; KASSERT(msix->msix_table_len > index, ("bogus index")); offset = msix->msix_pba_offset + (index / 32) * 4; bit = 1 << index % 32; return (bus_read_4(msix->msix_pba_res, offset) & bit); } /* * Restore MSI-X registers and table during resume. If MSI-X is * enabled then walk the virtual table to restore the actual MSI-X * table. */ static void pci_resume_msix(device_t dev) { struct pci_devinfo *dinfo = device_get_ivars(dev); struct pcicfg_msix *msix = &dinfo->cfg.msix; struct msix_table_entry *mte; struct msix_vector *mv; - u_int i; + u_int i, msgnum; if (msix->msix_alloc > 0) { + msgnum = PCI_MSIX_MSGNUM(msix->msix_ctrl); + /* First, mask all vectors. */ - for (i = 0; i < msix->msix_msgnum; i++) + for (i = 0; i < msgnum; i++) pci_mask_msix(dev, i); /* Second, program any messages with at least one handler. */ for (i = 0; i < msix->msix_table_len; i++) { mte = &msix->msix_table[i]; if (mte->mte_vector == 0 || mte->mte_handlers == 0) continue; mv = &msix->msix_vectors[mte->mte_vector - 1]; pci_write_msix_entry(dev, i, mv->mv_address, mv->mv_data); pci_unmask_msix(dev, i); } } pci_write_config(dev, msix->msix_location + PCIR_MSIX_CTRL, msix->msix_ctrl, 2); } /* * Attempt to allocate *count MSI-X messages. The actual number allocated is * returned in *count. After this function returns, each message will be * available to the driver as SYS_RES_IRQ resources starting at rid 1. */ int pci_alloc_msix_method(device_t dev, device_t child, int *count) { struct pci_devinfo *dinfo = device_get_ivars(child); pcicfgregs *cfg = &dinfo->cfg; struct resource_list_entry *rle; u_int actual, i, max; int error, irq; + uint16_t ctrl, msgnum; /* Don't let count == 0 get us into trouble. */ if (*count < 1) return (EINVAL); /* If rid 0 is allocated, then fail. */ rle = resource_list_find(&dinfo->resources, SYS_RES_IRQ, 0); if (rle != NULL && rle->res != NULL) return (ENXIO); /* Already have allocated messages? */ if (cfg->msi.msi_alloc != 0 || cfg->msix.msix_alloc != 0) return (ENXIO); /* If MSI-X is blacklisted for this system, fail. */ if (pci_msix_blacklisted()) return (ENXIO); /* MSI-X capability present? */ if (cfg->msix.msix_location == 0 || !pci_do_msix) return (ENODEV); /* Make sure the appropriate BARs are mapped. */ rle = resource_list_find(&dinfo->resources, SYS_RES_MEMORY, cfg->msix.msix_table_bar); if (rle == NULL || rle->res == NULL || !(rman_get_flags(rle->res) & RF_ACTIVE)) return (ENXIO); cfg->msix.msix_table_res = rle->res; if (cfg->msix.msix_pba_bar != cfg->msix.msix_table_bar) { rle = resource_list_find(&dinfo->resources, SYS_RES_MEMORY, cfg->msix.msix_pba_bar); if (rle == NULL || rle->res == NULL || !(rman_get_flags(rle->res) & RF_ACTIVE)) return (ENXIO); } cfg->msix.msix_pba_res = rle->res; + ctrl = pci_read_config(child, cfg->msix.msix_location + PCIR_MSIX_CTRL, + 2); + msgnum = PCI_MSIX_MSGNUM(ctrl); if (bootverbose) device_printf(child, "attempting to allocate %d MSI-X vectors (%d supported)\n", - *count, cfg->msix.msix_msgnum); - max = min(*count, cfg->msix.msix_msgnum); + *count, msgnum); + max = min(*count, msgnum); for (i = 0; i < max; i++) { /* Allocate a message. */ error = PCIB_ALLOC_MSIX(device_get_parent(dev), child, &irq); if (error) { if (i == 0) return (error); break; } resource_list_add(&dinfo->resources, SYS_RES_IRQ, i + 1, irq, irq, 1); } actual = i; if (bootverbose) { rle = resource_list_find(&dinfo->resources, SYS_RES_IRQ, 1); if (actual == 1) device_printf(child, "using IRQ %ju for MSI-X\n", rle->start); else { bool run; /* * Be fancy and try to print contiguous runs of * IRQ values as ranges. 'irq' is the previous IRQ. * 'run' is true if we are in a range. */ device_printf(child, "using IRQs %ju", rle->start); irq = rle->start; run = false; for (i = 1; i < actual; i++) { rle = resource_list_find(&dinfo->resources, SYS_RES_IRQ, i + 1); /* Still in a run? */ if (rle->start == irq + 1) { run = true; irq++; continue; } /* Finish previous range. */ if (run) { printf("-%d", irq); run = false; } /* Start new range. */ printf(",%ju", rle->start); irq = rle->start; } /* Unfinished range? */ if (run) printf("-%d", irq); printf(" for MSI-X\n"); } } /* Mask all vectors. */ - for (i = 0; i < cfg->msix.msix_msgnum; i++) + for (i = 0; i < msgnum; i++) pci_mask_msix(child, i); /* Allocate and initialize vector data and virtual table. */ cfg->msix.msix_vectors = mallocarray(actual, sizeof(struct msix_vector), M_DEVBUF, M_WAITOK | M_ZERO); cfg->msix.msix_table = mallocarray(actual, sizeof(struct msix_table_entry), M_DEVBUF, M_WAITOK | M_ZERO); for (i = 0; i < actual; i++) { rle = resource_list_find(&dinfo->resources, SYS_RES_IRQ, i + 1); cfg->msix.msix_vectors[i].mv_irq = rle->start; cfg->msix.msix_table[i].mte_vector = i + 1; } /* Update control register to enable MSI-X. */ - cfg->msix.msix_ctrl |= PCIM_MSIXCTRL_MSIX_ENABLE; + ctrl |= PCIM_MSIXCTRL_MSIX_ENABLE; pci_write_config(child, cfg->msix.msix_location + PCIR_MSIX_CTRL, - cfg->msix.msix_ctrl, 2); + ctrl, 2); + cfg->msix.msix_ctrl = ctrl; /* Update counts of alloc'd messages. */ cfg->msix.msix_alloc = actual; cfg->msix.msix_table_len = actual; *count = actual; return (0); } /* * By default, pci_alloc_msix() will assign the allocated IRQ * resources consecutively to the first N messages in the MSI-X table. * However, device drivers may want to use different layouts if they * either receive fewer messages than they asked for, or they wish to * populate the MSI-X table sparsely. This method allows the driver * to specify what layout it wants. It must be called after a * successful pci_alloc_msix() but before any of the associated * SYS_RES_IRQ resources are allocated via bus_alloc_resource(). * * The 'vectors' array contains 'count' message vectors. The array * maps directly to the MSI-X table in that index 0 in the array * specifies the vector for the first message in the MSI-X table, etc. * The vector value in each array index can either be 0 to indicate * that no vector should be assigned to a message slot, or it can be a * number from 1 to N (where N is the count returned from a * succcessful call to pci_alloc_msix()) to indicate which message * vector (IRQ) to be used for the corresponding message. * * On successful return, each message with a non-zero vector will have * an associated SYS_RES_IRQ whose rid is equal to the array index + * 1. Additionally, if any of the IRQs allocated via the previous * call to pci_alloc_msix() are not used in the mapping, those IRQs * will be freed back to the system automatically. * * For example, suppose a driver has a MSI-X table with 6 messages and * asks for 6 messages, but pci_alloc_msix() only returns a count of * 3. Call the three vectors allocated by pci_alloc_msix() A, B, and * C. After the call to pci_alloc_msix(), the device will be setup to * have an MSI-X table of ABC--- (where - means no vector assigned). * If the driver then passes a vector array of { 1, 0, 1, 2, 0, 2 }, * then the MSI-X table will look like A-AB-B, and the 'C' vector will * be freed back to the system. This device will also have valid * SYS_RES_IRQ rids of 1, 3, 4, and 6. * * In any case, the SYS_RES_IRQ rid X will always map to the message * at MSI-X table index X - 1 and will only be valid if a vector is * assigned to that table entry. */ int pci_remap_msix_method(device_t dev, device_t child, int count, const u_int *vectors) { struct pci_devinfo *dinfo = device_get_ivars(child); struct pcicfg_msix *msix = &dinfo->cfg.msix; struct resource_list_entry *rle; u_int i, irq, j; bool *used; /* * Have to have at least one message in the table but the * table can't be bigger than the actual MSI-X table in the * device. */ - if (count < 1 || count > msix->msix_msgnum) + if (count < 1 || count > PCI_MSIX_MSGNUM(msix->msix_ctrl)) return (EINVAL); /* Sanity check the vectors. */ for (i = 0; i < count; i++) if (vectors[i] > msix->msix_alloc) return (EINVAL); /* * Make sure there aren't any holes in the vectors to be used. * It's a big pain to support it, and it doesn't really make * sense anyway. Also, at least one vector must be used. */ used = mallocarray(msix->msix_alloc, sizeof(*used), M_DEVBUF, M_WAITOK | M_ZERO); for (i = 0; i < count; i++) if (vectors[i] != 0) used[vectors[i] - 1] = true; for (i = 0; i < msix->msix_alloc - 1; i++) if (!used[i] && used[i + 1]) { free(used, M_DEVBUF); return (EINVAL); } if (!used[0]) { free(used, M_DEVBUF); return (EINVAL); } /* Make sure none of the resources are allocated. */ for (i = 0; i < msix->msix_table_len; i++) { if (msix->msix_table[i].mte_vector == 0) continue; if (msix->msix_table[i].mte_handlers > 0) { free(used, M_DEVBUF); return (EBUSY); } rle = resource_list_find(&dinfo->resources, SYS_RES_IRQ, i + 1); KASSERT(rle != NULL, ("missing resource")); if (rle->res != NULL) { free(used, M_DEVBUF); return (EBUSY); } } /* Free the existing resource list entries. */ for (i = 0; i < msix->msix_table_len; i++) { if (msix->msix_table[i].mte_vector == 0) continue; resource_list_delete(&dinfo->resources, SYS_RES_IRQ, i + 1); } /* * Build the new virtual table keeping track of which vectors are * used. */ free(msix->msix_table, M_DEVBUF); msix->msix_table = mallocarray(count, sizeof(struct msix_table_entry), M_DEVBUF, M_WAITOK | M_ZERO); for (i = 0; i < count; i++) msix->msix_table[i].mte_vector = vectors[i]; msix->msix_table_len = count; /* Free any unused IRQs and resize the vectors array if necessary. */ j = msix->msix_alloc - 1; if (!used[j]) { struct msix_vector *vec; while (!used[j]) { PCIB_RELEASE_MSIX(device_get_parent(dev), child, msix->msix_vectors[j].mv_irq); j--; } vec = mallocarray(j + 1, sizeof(struct msix_vector), M_DEVBUF, M_WAITOK); bcopy(msix->msix_vectors, vec, sizeof(struct msix_vector) * (j + 1)); free(msix->msix_vectors, M_DEVBUF); msix->msix_vectors = vec; msix->msix_alloc = j + 1; } free(used, M_DEVBUF); /* Map the IRQs onto the rids. */ for (i = 0; i < count; i++) { if (vectors[i] == 0) continue; irq = msix->msix_vectors[vectors[i] - 1].mv_irq; resource_list_add(&dinfo->resources, SYS_RES_IRQ, i + 1, irq, irq, 1); } if (bootverbose) { device_printf(child, "Remapped MSI-X IRQs as: "); for (i = 0; i < count; i++) { if (i != 0) printf(", "); if (vectors[i] == 0) printf("---"); else printf("%d", msix->msix_vectors[vectors[i] - 1].mv_irq); } printf("\n"); } return (0); } static int pci_release_msix(device_t dev, device_t child) { struct pci_devinfo *dinfo = device_get_ivars(child); struct pcicfg_msix *msix = &dinfo->cfg.msix; struct resource_list_entry *rle; u_int i; /* Do we have any messages to release? */ if (msix->msix_alloc == 0) return (ENODEV); /* Make sure none of the resources are allocated. */ for (i = 0; i < msix->msix_table_len; i++) { if (msix->msix_table[i].mte_vector == 0) continue; if (msix->msix_table[i].mte_handlers > 0) return (EBUSY); rle = resource_list_find(&dinfo->resources, SYS_RES_IRQ, i + 1); KASSERT(rle != NULL, ("missing resource")); if (rle->res != NULL) return (EBUSY); } /* Update control register to disable MSI-X. */ msix->msix_ctrl &= ~PCIM_MSIXCTRL_MSIX_ENABLE; pci_write_config(child, msix->msix_location + PCIR_MSIX_CTRL, msix->msix_ctrl, 2); /* Free the resource list entries. */ for (i = 0; i < msix->msix_table_len; i++) { if (msix->msix_table[i].mte_vector == 0) continue; resource_list_delete(&dinfo->resources, SYS_RES_IRQ, i + 1); } free(msix->msix_table, M_DEVBUF); msix->msix_table_len = 0; /* Release the IRQs. */ for (i = 0; i < msix->msix_alloc; i++) PCIB_RELEASE_MSIX(device_get_parent(dev), child, msix->msix_vectors[i].mv_irq); free(msix->msix_vectors, M_DEVBUF); msix->msix_alloc = 0; return (0); } /* * Return the max supported MSI-X messages this device supports. * Basically, assuming the MD code can alloc messages, this function * should return the maximum value that pci_alloc_msix() can return. * Thus, it is subject to the tunables, etc. */ int pci_msix_count_method(device_t dev, device_t child) { struct pci_devinfo *dinfo = device_get_ivars(child); struct pcicfg_msix *msix = &dinfo->cfg.msix; + uint16_t ctrl; - if (pci_do_msix && msix->msix_location != 0) - return (msix->msix_msgnum); + if (pci_do_msix && msix->msix_location != 0) { + ctrl = pci_read_config(child, msix->msix_location + + PCIR_MSI_CTRL, 2); + return (PCI_MSIX_MSGNUM(ctrl)); + } return (0); } int pci_msix_pba_bar_method(device_t dev, device_t child) { struct pci_devinfo *dinfo = device_get_ivars(child); struct pcicfg_msix *msix = &dinfo->cfg.msix; if (pci_do_msix && msix->msix_location != 0) return (msix->msix_pba_bar); return (-1); } int pci_msix_table_bar_method(device_t dev, device_t child) { struct pci_devinfo *dinfo = device_get_ivars(child); struct pcicfg_msix *msix = &dinfo->cfg.msix; if (pci_do_msix && msix->msix_location != 0) return (msix->msix_table_bar); return (-1); } /* * HyperTransport MSI mapping control */ void pci_ht_map_msi(device_t dev, uint64_t addr) { struct pci_devinfo *dinfo = device_get_ivars(dev); struct pcicfg_ht *ht = &dinfo->cfg.ht; if (!ht->ht_msimap) return; if (addr && !(ht->ht_msictrl & PCIM_HTCMD_MSI_ENABLE) && ht->ht_msiaddr >> 20 == addr >> 20) { /* Enable MSI -> HT mapping. */ ht->ht_msictrl |= PCIM_HTCMD_MSI_ENABLE; pci_write_config(dev, ht->ht_msimap + PCIR_HT_COMMAND, ht->ht_msictrl, 2); } if (!addr && ht->ht_msictrl & PCIM_HTCMD_MSI_ENABLE) { /* Disable MSI -> HT mapping. */ ht->ht_msictrl &= ~PCIM_HTCMD_MSI_ENABLE; pci_write_config(dev, ht->ht_msimap + PCIR_HT_COMMAND, ht->ht_msictrl, 2); } } int pci_get_relaxed_ordering_enabled(device_t dev) { struct pci_devinfo *dinfo = device_get_ivars(dev); int cap; uint16_t val; cap = dinfo->cfg.pcie.pcie_location; if (cap == 0) return (0); val = pci_read_config(dev, cap + PCIER_DEVICE_CTL, 2); val &= PCIEM_CTL_RELAXED_ORD_ENABLE; return (val != 0); } int pci_get_max_payload(device_t dev) { struct pci_devinfo *dinfo = device_get_ivars(dev); int cap; uint16_t val; cap = dinfo->cfg.pcie.pcie_location; if (cap == 0) return (0); val = pci_read_config(dev, cap + PCIER_DEVICE_CTL, 2); val &= PCIEM_CTL_MAX_PAYLOAD; val >>= 5; return (1 << (val + 7)); } int pci_get_max_read_req(device_t dev) { struct pci_devinfo *dinfo = device_get_ivars(dev); int cap; uint16_t val; cap = dinfo->cfg.pcie.pcie_location; if (cap == 0) return (0); val = pci_read_config(dev, cap + PCIER_DEVICE_CTL, 2); val &= PCIEM_CTL_MAX_READ_REQUEST; val >>= 12; return (1 << (val + 7)); } int pci_set_max_read_req(device_t dev, int size) { struct pci_devinfo *dinfo = device_get_ivars(dev); int cap; uint16_t val; cap = dinfo->cfg.pcie.pcie_location; if (cap == 0) return (0); if (size < 128) size = 128; if (size > 4096) size = 4096; size = (1 << (fls(size) - 1)); val = pci_read_config(dev, cap + PCIER_DEVICE_CTL, 2); val &= ~PCIEM_CTL_MAX_READ_REQUEST; val |= (fls(size) - 8) << 12; pci_write_config(dev, cap + PCIER_DEVICE_CTL, val, 2); return (size); } uint32_t pcie_read_config(device_t dev, int reg, int width) { struct pci_devinfo *dinfo = device_get_ivars(dev); int cap; cap = dinfo->cfg.pcie.pcie_location; if (cap == 0) { if (width == 2) return (0xffff); return (0xffffffff); } return (pci_read_config(dev, cap + reg, width)); } void pcie_write_config(device_t dev, int reg, uint32_t value, int width) { struct pci_devinfo *dinfo = device_get_ivars(dev); int cap; cap = dinfo->cfg.pcie.pcie_location; if (cap == 0) return; pci_write_config(dev, cap + reg, value, width); } /* * Adjusts a PCI-e capability register by clearing the bits in mask * and setting the bits in (value & mask). Bits not set in mask are * not adjusted. * * Returns the old value on success or all ones on failure. */ uint32_t pcie_adjust_config(device_t dev, int reg, uint32_t mask, uint32_t value, int width) { struct pci_devinfo *dinfo = device_get_ivars(dev); uint32_t old, new; int cap; cap = dinfo->cfg.pcie.pcie_location; if (cap == 0) { if (width == 2) return (0xffff); return (0xffffffff); } old = pci_read_config(dev, cap + reg, width); new = old & ~mask; new |= (value & mask); pci_write_config(dev, cap + reg, new, width); return (old); } /* * Support for MSI message signalled interrupts. */ void pci_enable_msi_method(device_t dev, device_t child, uint64_t address, uint16_t data) { struct pci_devinfo *dinfo = device_get_ivars(child); struct pcicfg_msi *msi = &dinfo->cfg.msi; /* Write data and address values. */ pci_write_config(child, msi->msi_location + PCIR_MSI_ADDR, address & 0xffffffff, 4); if (msi->msi_ctrl & PCIM_MSICTRL_64BIT) { pci_write_config(child, msi->msi_location + PCIR_MSI_ADDR_HIGH, address >> 32, 4); pci_write_config(child, msi->msi_location + PCIR_MSI_DATA_64BIT, data, 2); } else pci_write_config(child, msi->msi_location + PCIR_MSI_DATA, data, 2); /* Enable MSI in the control register. */ msi->msi_ctrl |= PCIM_MSICTRL_MSI_ENABLE; pci_write_config(child, msi->msi_location + PCIR_MSI_CTRL, msi->msi_ctrl, 2); /* Enable MSI -> HT mapping. */ pci_ht_map_msi(child, address); } void pci_disable_msi_method(device_t dev, device_t child) { struct pci_devinfo *dinfo = device_get_ivars(child); struct pcicfg_msi *msi = &dinfo->cfg.msi; /* Disable MSI -> HT mapping. */ pci_ht_map_msi(child, 0); /* Disable MSI in the control register. */ msi->msi_ctrl &= ~PCIM_MSICTRL_MSI_ENABLE; pci_write_config(child, msi->msi_location + PCIR_MSI_CTRL, msi->msi_ctrl, 2); } /* * Restore MSI registers during resume. If MSI is enabled then * restore the data and address registers in addition to the control * register. */ static void pci_resume_msi(device_t dev) { struct pci_devinfo *dinfo = device_get_ivars(dev); struct pcicfg_msi *msi = &dinfo->cfg.msi; uint64_t address; uint16_t data; if (msi->msi_ctrl & PCIM_MSICTRL_MSI_ENABLE) { address = msi->msi_addr; data = msi->msi_data; pci_write_config(dev, msi->msi_location + PCIR_MSI_ADDR, address & 0xffffffff, 4); if (msi->msi_ctrl & PCIM_MSICTRL_64BIT) { pci_write_config(dev, msi->msi_location + PCIR_MSI_ADDR_HIGH, address >> 32, 4); pci_write_config(dev, msi->msi_location + PCIR_MSI_DATA_64BIT, data, 2); } else pci_write_config(dev, msi->msi_location + PCIR_MSI_DATA, data, 2); } pci_write_config(dev, msi->msi_location + PCIR_MSI_CTRL, msi->msi_ctrl, 2); } static int pci_remap_intr_method(device_t bus, device_t dev, u_int irq) { struct pci_devinfo *dinfo = device_get_ivars(dev); pcicfgregs *cfg = &dinfo->cfg; struct resource_list_entry *rle; struct msix_table_entry *mte; struct msix_vector *mv; uint64_t addr; uint32_t data; u_int i, j; int error; /* * Handle MSI first. We try to find this IRQ among our list * of MSI IRQs. If we find it, we request updated address and * data registers and apply the results. */ if (cfg->msi.msi_alloc > 0) { /* If we don't have any active handlers, nothing to do. */ if (cfg->msi.msi_handlers == 0) return (0); for (i = 0; i < cfg->msi.msi_alloc; i++) { rle = resource_list_find(&dinfo->resources, SYS_RES_IRQ, i + 1); if (rle->start == irq) { error = PCIB_MAP_MSI(device_get_parent(bus), dev, irq, &addr, &data); if (error) return (error); pci_disable_msi(dev); dinfo->cfg.msi.msi_addr = addr; dinfo->cfg.msi.msi_data = data; pci_enable_msi(dev, addr, data); return (0); } } return (ENOENT); } /* * For MSI-X, we check to see if we have this IRQ. If we do, * we request the updated mapping info. If that works, we go * through all the slots that use this IRQ and update them. */ if (cfg->msix.msix_alloc > 0) { bool found = false; for (i = 0; i < cfg->msix.msix_alloc; i++) { mv = &cfg->msix.msix_vectors[i]; if (mv->mv_irq == irq) { error = PCIB_MAP_MSI(device_get_parent(bus), dev, irq, &addr, &data); if (error) return (error); mv->mv_address = addr; mv->mv_data = data; for (j = 0; j < cfg->msix.msix_table_len; j++) { mte = &cfg->msix.msix_table[j]; if (mte->mte_vector != i + 1) continue; if (mte->mte_handlers == 0) continue; pci_mask_msix(dev, j); pci_enable_msix(dev, j, addr, data); pci_unmask_msix(dev, j); } found = true; } } return (found ? 0 : ENOENT); } return (ENOENT); } /* * Returns true if the specified device is blacklisted because MSI * doesn't work. */ int pci_msi_device_blacklisted(device_t dev) { if (!pci_honor_msi_blacklist) return (0); return (pci_has_quirk(pci_get_devid(dev), PCI_QUIRK_DISABLE_MSI)); } /* * Determine if MSI is blacklisted globally on this system. Currently, * we just check for blacklisted chipsets as represented by the * host-PCI bridge at device 0:0:0. In the future, it may become * necessary to check other system attributes, such as the kenv values * that give the motherboard manufacturer and model number. */ static int pci_msi_blacklisted(void) { device_t dev; if (!pci_honor_msi_blacklist) return (0); /* Blacklist all non-PCI-express and non-PCI-X chipsets. */ if (!(pcie_chipset || pcix_chipset)) { if (vm_guest != VM_GUEST_NO) { /* * Whitelist older chipsets in virtual * machines known to support MSI. */ dev = pci_find_bsf(0, 0, 0); if (dev != NULL) return (!pci_has_quirk(pci_get_devid(dev), PCI_QUIRK_ENABLE_MSI_VM)); } return (1); } dev = pci_find_bsf(0, 0, 0); if (dev != NULL) return (pci_msi_device_blacklisted(dev)); return (0); } /* * Returns true if the specified device is blacklisted because MSI-X * doesn't work. Note that this assumes that if MSI doesn't work, * MSI-X doesn't either. */ int pci_msix_device_blacklisted(device_t dev) { if (!pci_honor_msi_blacklist) return (0); if (pci_has_quirk(pci_get_devid(dev), PCI_QUIRK_DISABLE_MSIX)) return (1); return (pci_msi_device_blacklisted(dev)); } /* * Determine if MSI-X is blacklisted globally on this system. If MSI * is blacklisted, assume that MSI-X is as well. Check for additional * chipsets where MSI works but MSI-X does not. */ static int pci_msix_blacklisted(void) { device_t dev; if (!pci_honor_msi_blacklist) return (0); dev = pci_find_bsf(0, 0, 0); if (dev != NULL && pci_has_quirk(pci_get_devid(dev), PCI_QUIRK_DISABLE_MSIX)) return (1); return (pci_msi_blacklisted()); } /* * Attempt to allocate *count MSI messages. The actual number allocated is * returned in *count. After this function returns, each message will be * available to the driver as SYS_RES_IRQ resources starting at a rid 1. */ int pci_alloc_msi_method(device_t dev, device_t child, int *count) { struct pci_devinfo *dinfo = device_get_ivars(child); pcicfgregs *cfg = &dinfo->cfg; struct resource_list_entry *rle; u_int actual, i; int error, irqs[32]; - uint16_t ctrl; + uint16_t ctrl, msgnum; /* Don't let count == 0 get us into trouble. */ if (*count < 1) return (EINVAL); /* If rid 0 is allocated, then fail. */ rle = resource_list_find(&dinfo->resources, SYS_RES_IRQ, 0); if (rle != NULL && rle->res != NULL) return (ENXIO); /* Already have allocated messages? */ if (cfg->msi.msi_alloc != 0 || cfg->msix.msix_alloc != 0) return (ENXIO); /* If MSI is blacklisted for this system, fail. */ if (pci_msi_blacklisted()) return (ENXIO); /* MSI capability present? */ if (cfg->msi.msi_location == 0 || !pci_do_msi) return (ENODEV); + ctrl = pci_read_config(child, cfg->msi.msi_location + PCIR_MSI_CTRL, 2); + msgnum = PCI_MSI_MSGNUM(ctrl); if (bootverbose) device_printf(child, - "attempting to allocate %d MSI vectors (%d supported)\n", - *count, cfg->msi.msi_msgnum); + "attempting to allocate %d MSI vectors (%u supported)\n", + *count, msgnum); /* Don't ask for more than the device supports. */ - actual = min(*count, cfg->msi.msi_msgnum); + actual = min(*count, msgnum); /* Don't ask for more than 32 messages. */ actual = min(actual, 32); /* MSI requires power of 2 number of messages. */ if (!powerof2(actual)) return (EINVAL); for (;;) { /* Try to allocate N messages. */ error = PCIB_ALLOC_MSI(device_get_parent(dev), child, actual, actual, irqs); if (error == 0) break; if (actual == 1) return (error); /* Try N / 2. */ actual >>= 1; } /* * We now have N actual messages mapped onto SYS_RES_IRQ * resources in the irqs[] array, so add new resources * starting at rid 1. */ for (i = 0; i < actual; i++) resource_list_add(&dinfo->resources, SYS_RES_IRQ, i + 1, irqs[i], irqs[i], 1); if (bootverbose) { if (actual == 1) device_printf(child, "using IRQ %d for MSI\n", irqs[0]); else { bool run; /* * Be fancy and try to print contiguous runs * of IRQ values as ranges. 'run' is true if * we are in a range. */ device_printf(child, "using IRQs %d", irqs[0]); run = false; for (i = 1; i < actual; i++) { /* Still in a run? */ if (irqs[i] == irqs[i - 1] + 1) { run = true; continue; } /* Finish previous range. */ if (run) { printf("-%d", irqs[i - 1]); run = false; } /* Start new range. */ printf(",%d", irqs[i]); } /* Unfinished range? */ if (run) printf("-%d", irqs[actual - 1]); printf(" for MSI\n"); } } /* Update control register with actual count. */ - ctrl = cfg->msi.msi_ctrl; ctrl &= ~PCIM_MSICTRL_MME_MASK; ctrl |= (ffs(actual) - 1) << 4; cfg->msi.msi_ctrl = ctrl; pci_write_config(child, cfg->msi.msi_location + PCIR_MSI_CTRL, ctrl, 2); /* Update counts of alloc'd messages. */ cfg->msi.msi_alloc = actual; cfg->msi.msi_handlers = 0; *count = actual; return (0); } /* Release the MSI messages associated with this device. */ int pci_release_msi_method(device_t dev, device_t child) { struct pci_devinfo *dinfo = device_get_ivars(child); struct pcicfg_msi *msi = &dinfo->cfg.msi; struct resource_list_entry *rle; u_int i, irqs[32]; int error; /* Try MSI-X first. */ error = pci_release_msix(dev, child); if (error != ENODEV) return (error); /* Do we have any messages to release? */ if (msi->msi_alloc == 0) return (ENODEV); KASSERT(msi->msi_alloc <= 32, ("more than 32 alloc'd messages")); /* Make sure none of the resources are allocated. */ if (msi->msi_handlers > 0) return (EBUSY); for (i = 0; i < msi->msi_alloc; i++) { rle = resource_list_find(&dinfo->resources, SYS_RES_IRQ, i + 1); KASSERT(rle != NULL, ("missing MSI resource")); if (rle->res != NULL) return (EBUSY); irqs[i] = rle->start; } /* Update control register with 0 count. */ KASSERT(!(msi->msi_ctrl & PCIM_MSICTRL_MSI_ENABLE), ("%s: MSI still enabled", __func__)); msi->msi_ctrl &= ~PCIM_MSICTRL_MME_MASK; pci_write_config(child, msi->msi_location + PCIR_MSI_CTRL, msi->msi_ctrl, 2); /* Release the messages. */ PCIB_RELEASE_MSI(device_get_parent(dev), child, msi->msi_alloc, irqs); for (i = 0; i < msi->msi_alloc; i++) resource_list_delete(&dinfo->resources, SYS_RES_IRQ, i + 1); /* Update alloc count. */ msi->msi_alloc = 0; msi->msi_addr = 0; msi->msi_data = 0; return (0); } /* * Return the max supported MSI messages this device supports. * Basically, assuming the MD code can alloc messages, this function * should return the maximum value that pci_alloc_msi() can return. * Thus, it is subject to the tunables, etc. */ int pci_msi_count_method(device_t dev, device_t child) { struct pci_devinfo *dinfo = device_get_ivars(child); struct pcicfg_msi *msi = &dinfo->cfg.msi; + uint16_t ctrl; - if (pci_do_msi && msi->msi_location != 0) - return (msi->msi_msgnum); + if (pci_do_msi && msi->msi_location != 0) { + ctrl = pci_read_config(child, msi->msi_location + PCIR_MSI_CTRL, + 2); + return (PCI_MSI_MSGNUM(ctrl)); + } return (0); } /* free pcicfgregs structure and all depending data structures */ int pci_freecfg(struct pci_devinfo *dinfo) { struct devlist *devlist_head; struct pci_map *pm, *next; devlist_head = &pci_devq; if (dinfo->cfg.vpd.vpd_reg) vpd_free(&dinfo->cfg.vpd); STAILQ_FOREACH_SAFE(pm, &dinfo->cfg.maps, pm_link, next) { free(pm, M_DEVBUF); } STAILQ_REMOVE(devlist_head, dinfo, pci_devinfo, pci_links); free(dinfo, M_DEVBUF); /* increment the generation count */ pci_generation++; /* we're losing one device */ pci_numdevs--; return (0); } /* * PCI power manangement */ int pci_set_powerstate_method(device_t dev, device_t child, int state) { struct pci_devinfo *dinfo = device_get_ivars(child); pcicfgregs *cfg = &dinfo->cfg; uint16_t status; int oldstate, highest, delay; if (cfg->pp.pp_cap == 0) return (EOPNOTSUPP); /* * Optimize a no state change request away. While it would be OK to * write to the hardware in theory, some devices have shown odd * behavior when going from D3 -> D3. */ oldstate = pci_get_powerstate(child); if (oldstate == state) return (0); /* * The PCI power management specification states that after a state * transition between PCI power states, system software must * guarantee a minimal delay before the function accesses the device. * Compute the worst case delay that we need to guarantee before we * access the device. Many devices will be responsive much more * quickly than this delay, but there are some that don't respond * instantly to state changes. Transitions to/from D3 state require * 10ms, while D2 requires 200us, and D0/1 require none. The delay * is done below with DELAY rather than a sleeper function because * this function can be called from contexts where we cannot sleep. */ highest = (oldstate > state) ? oldstate : state; if (highest == PCI_POWERSTATE_D3) delay = 10000; else if (highest == PCI_POWERSTATE_D2) delay = 200; else delay = 0; status = PCI_READ_CONFIG(dev, child, cfg->pp.pp_status, 2) & ~PCIM_PSTAT_DMASK; switch (state) { case PCI_POWERSTATE_D0: status |= PCIM_PSTAT_D0; break; case PCI_POWERSTATE_D1: if ((cfg->pp.pp_cap & PCIM_PCAP_D1SUPP) == 0) return (EOPNOTSUPP); status |= PCIM_PSTAT_D1; break; case PCI_POWERSTATE_D2: if ((cfg->pp.pp_cap & PCIM_PCAP_D2SUPP) == 0) return (EOPNOTSUPP); status |= PCIM_PSTAT_D2; break; case PCI_POWERSTATE_D3: status |= PCIM_PSTAT_D3; break; default: return (EINVAL); } if (bootverbose) pci_printf(cfg, "Transition from D%d to D%d\n", oldstate, state); PCI_WRITE_CONFIG(dev, child, cfg->pp.pp_status, status, 2); if (delay) DELAY(delay); return (0); } int pci_get_powerstate_method(device_t dev, device_t child) { struct pci_devinfo *dinfo = device_get_ivars(child); pcicfgregs *cfg = &dinfo->cfg; uint16_t status; int result; if (cfg->pp.pp_cap != 0) { status = PCI_READ_CONFIG(dev, child, cfg->pp.pp_status, 2); switch (status & PCIM_PSTAT_DMASK) { case PCIM_PSTAT_D0: result = PCI_POWERSTATE_D0; break; case PCIM_PSTAT_D1: result = PCI_POWERSTATE_D1; break; case PCIM_PSTAT_D2: result = PCI_POWERSTATE_D2; break; case PCIM_PSTAT_D3: result = PCI_POWERSTATE_D3; break; default: result = PCI_POWERSTATE_UNKNOWN; break; } } else { /* No support, device is always at D0 */ result = PCI_POWERSTATE_D0; } return (result); } /* * Some convenience functions for PCI device drivers. */ static __inline void pci_set_command_bit(device_t dev, device_t child, uint16_t bit) { uint16_t command; command = PCI_READ_CONFIG(dev, child, PCIR_COMMAND, 2); command |= bit; PCI_WRITE_CONFIG(dev, child, PCIR_COMMAND, command, 2); } static __inline void pci_clear_command_bit(device_t dev, device_t child, uint16_t bit) { uint16_t command; command = PCI_READ_CONFIG(dev, child, PCIR_COMMAND, 2); command &= ~bit; PCI_WRITE_CONFIG(dev, child, PCIR_COMMAND, command, 2); } int pci_enable_busmaster_method(device_t dev, device_t child) { pci_set_command_bit(dev, child, PCIM_CMD_BUSMASTEREN); return (0); } int pci_disable_busmaster_method(device_t dev, device_t child) { pci_clear_command_bit(dev, child, PCIM_CMD_BUSMASTEREN); return (0); } int pci_enable_io_method(device_t dev, device_t child, int space) { uint16_t bit; switch(space) { case SYS_RES_IOPORT: bit = PCIM_CMD_PORTEN; break; case SYS_RES_MEMORY: bit = PCIM_CMD_MEMEN; break; default: return (EINVAL); } pci_set_command_bit(dev, child, bit); return (0); } int pci_disable_io_method(device_t dev, device_t child, int space) { uint16_t bit; switch(space) { case SYS_RES_IOPORT: bit = PCIM_CMD_PORTEN; break; case SYS_RES_MEMORY: bit = PCIM_CMD_MEMEN; break; default: return (EINVAL); } pci_clear_command_bit(dev, child, bit); return (0); } /* * New style pci driver. Parent device is either a pci-host-bridge or a * pci-pci-bridge. Both kinds are represented by instances of pcib. */ void pci_print_verbose(struct pci_devinfo *dinfo) { if (bootverbose) { pcicfgregs *cfg = &dinfo->cfg; printf("found->\tvendor=0x%04x, dev=0x%04x, revid=0x%02x\n", cfg->vendor, cfg->device, cfg->revid); printf("\tdomain=%d, bus=%d, slot=%d, func=%d\n", cfg->domain, cfg->bus, cfg->slot, cfg->func); printf("\tclass=%02x-%02x-%02x, hdrtype=0x%02x, mfdev=%d\n", cfg->baseclass, cfg->subclass, cfg->progif, cfg->hdrtype, cfg->mfdev); printf("\tcmdreg=0x%04x, statreg=0x%04x, cachelnsz=%d (dwords)\n", cfg->cmdreg, cfg->statreg, cfg->cachelnsz); printf("\tlattimer=0x%02x (%d ns), mingnt=0x%02x (%d ns), maxlat=0x%02x (%d ns)\n", cfg->lattimer, cfg->lattimer * 30, cfg->mingnt, cfg->mingnt * 250, cfg->maxlat, cfg->maxlat * 250); if (cfg->intpin > 0) printf("\tintpin=%c, irq=%d\n", cfg->intpin +'a' -1, cfg->intline); if (cfg->pp.pp_cap) { uint16_t status; status = pci_read_config(cfg->dev, cfg->pp.pp_status, 2); printf("\tpowerspec %d supports D0%s%s D3 current D%d\n", cfg->pp.pp_cap & PCIM_PCAP_SPEC, cfg->pp.pp_cap & PCIM_PCAP_D1SUPP ? " D1" : "", cfg->pp.pp_cap & PCIM_PCAP_D2SUPP ? " D2" : "", status & PCIM_PSTAT_DMASK); } if (cfg->msi.msi_location) { - int ctrl; + uint16_t ctrl, msgnum; ctrl = cfg->msi.msi_ctrl; + msgnum = PCI_MSI_MSGNUM(ctrl); printf("\tMSI supports %d message%s%s%s\n", - cfg->msi.msi_msgnum, - (cfg->msi.msi_msgnum == 1) ? "" : "s", + msgnum, (msgnum == 1) ? "" : "s", (ctrl & PCIM_MSICTRL_64BIT) ? ", 64 bit" : "", (ctrl & PCIM_MSICTRL_VECTOR) ? ", vector masks":""); } if (cfg->msix.msix_location) { + uint16_t msgnum; + + msgnum = PCI_MSIX_MSGNUM(cfg->msix.msix_ctrl); printf("\tMSI-X supports %d message%s ", - cfg->msix.msix_msgnum, - (cfg->msix.msix_msgnum == 1) ? "" : "s"); + msgnum, (msgnum == 1) ? "" : "s"); if (cfg->msix.msix_table_bar == cfg->msix.msix_pba_bar) printf("in map 0x%x\n", cfg->msix.msix_table_bar); else printf("in maps 0x%x and 0x%x\n", cfg->msix.msix_table_bar, cfg->msix.msix_pba_bar); } } } static int pci_porten(device_t dev) { return (pci_read_config(dev, PCIR_COMMAND, 2) & PCIM_CMD_PORTEN) != 0; } static int pci_memen(device_t dev) { return (pci_read_config(dev, PCIR_COMMAND, 2) & PCIM_CMD_MEMEN) != 0; } void pci_read_bar(device_t dev, int reg, pci_addr_t *mapp, pci_addr_t *testvalp, int *bar64) { struct pci_devinfo *dinfo; pci_addr_t map, testval; int ln2range; uint16_t cmd; /* * The device ROM BAR is special. It is always a 32-bit * memory BAR. Bit 0 is special and should not be set when * sizing the BAR. */ dinfo = device_get_ivars(dev); if (PCIR_IS_BIOS(&dinfo->cfg, reg)) { map = pci_read_config(dev, reg, 4); pci_write_config(dev, reg, 0xfffffffe, 4); testval = pci_read_config(dev, reg, 4); pci_write_config(dev, reg, map, 4); *mapp = map; *testvalp = testval; if (bar64 != NULL) *bar64 = 0; return; } map = pci_read_config(dev, reg, 4); ln2range = pci_maprange(map); if (ln2range == 64) map |= (pci_addr_t)pci_read_config(dev, reg + 4, 4) << 32; /* * Disable decoding via the command register before * determining the BAR's length since we will be placing it in * a weird state. */ cmd = pci_read_config(dev, PCIR_COMMAND, 2); pci_write_config(dev, PCIR_COMMAND, cmd & ~(PCI_BAR_MEM(map) ? PCIM_CMD_MEMEN : PCIM_CMD_PORTEN), 2); /* * Determine the BAR's length by writing all 1's. The bottom * log_2(size) bits of the BAR will stick as 0 when we read * the value back. * * NB: according to the PCI Local Bus Specification, rev. 3.0: * "Software writes 0FFFFFFFFh to both registers, reads them back, * and combines the result into a 64-bit value." (section 6.2.5.1) * * Writes to both registers must be performed before attempting to * read back the size value. */ testval = 0; pci_write_config(dev, reg, 0xffffffff, 4); if (ln2range == 64) { pci_write_config(dev, reg + 4, 0xffffffff, 4); testval |= (pci_addr_t)pci_read_config(dev, reg + 4, 4) << 32; } testval |= pci_read_config(dev, reg, 4); /* * Restore the original value of the BAR. We may have reprogrammed * the BAR of the low-level console device and when booting verbose, * we need the console device addressable. */ pci_write_config(dev, reg, map, 4); if (ln2range == 64) pci_write_config(dev, reg + 4, map >> 32, 4); pci_write_config(dev, PCIR_COMMAND, cmd, 2); *mapp = map; *testvalp = testval; if (bar64 != NULL) *bar64 = (ln2range == 64); } static void pci_write_bar(device_t dev, struct pci_map *pm, pci_addr_t base) { struct pci_devinfo *dinfo; int ln2range; /* The device ROM BAR is always a 32-bit memory BAR. */ dinfo = device_get_ivars(dev); if (PCIR_IS_BIOS(&dinfo->cfg, pm->pm_reg)) ln2range = 32; else ln2range = pci_maprange(pm->pm_value); pci_write_config(dev, pm->pm_reg, base, 4); if (ln2range == 64) pci_write_config(dev, pm->pm_reg + 4, base >> 32, 4); pm->pm_value = pci_read_config(dev, pm->pm_reg, 4); if (ln2range == 64) pm->pm_value |= (pci_addr_t)pci_read_config(dev, pm->pm_reg + 4, 4) << 32; } struct pci_map * pci_find_bar(device_t dev, int reg) { struct pci_devinfo *dinfo; struct pci_map *pm; dinfo = device_get_ivars(dev); STAILQ_FOREACH(pm, &dinfo->cfg.maps, pm_link) { if (pm->pm_reg == reg) return (pm); } return (NULL); } struct pci_map * pci_first_bar(device_t dev) { struct pci_devinfo *dinfo; dinfo = device_get_ivars(dev); return (STAILQ_FIRST(&dinfo->cfg.maps)); } struct pci_map * pci_next_bar(struct pci_map *pm) { return (STAILQ_NEXT(pm, pm_link)); } int pci_bar_enabled(device_t dev, struct pci_map *pm) { struct pci_devinfo *dinfo; uint16_t cmd; dinfo = device_get_ivars(dev); if (PCIR_IS_BIOS(&dinfo->cfg, pm->pm_reg) && !(pm->pm_value & PCIM_BIOS_ENABLE)) return (0); #ifdef PCI_IOV if ((dinfo->cfg.flags & PCICFG_VF) != 0) { struct pcicfg_iov *iov; iov = dinfo->cfg.iov; cmd = pci_read_config(iov->iov_pf, iov->iov_pos + PCIR_SRIOV_CTL, 2); return ((cmd & PCIM_SRIOV_VF_MSE) != 0); } #endif cmd = pci_read_config(dev, PCIR_COMMAND, 2); if (PCIR_IS_BIOS(&dinfo->cfg, pm->pm_reg) || PCI_BAR_MEM(pm->pm_value)) return ((cmd & PCIM_CMD_MEMEN) != 0); else return ((cmd & PCIM_CMD_PORTEN) != 0); } struct pci_map * pci_add_bar(device_t dev, int reg, pci_addr_t value, pci_addr_t size) { struct pci_devinfo *dinfo; struct pci_map *pm, *prev; dinfo = device_get_ivars(dev); pm = malloc(sizeof(*pm), M_DEVBUF, M_WAITOK | M_ZERO); pm->pm_reg = reg; pm->pm_value = value; pm->pm_size = size; STAILQ_FOREACH(prev, &dinfo->cfg.maps, pm_link) { KASSERT(prev->pm_reg != pm->pm_reg, ("duplicate map %02x", reg)); if (STAILQ_NEXT(prev, pm_link) == NULL || STAILQ_NEXT(prev, pm_link)->pm_reg > pm->pm_reg) break; } if (prev != NULL) STAILQ_INSERT_AFTER(&dinfo->cfg.maps, prev, pm, pm_link); else STAILQ_INSERT_TAIL(&dinfo->cfg.maps, pm, pm_link); return (pm); } static void pci_restore_bars(device_t dev) { struct pci_devinfo *dinfo; struct pci_map *pm; int ln2range; dinfo = device_get_ivars(dev); STAILQ_FOREACH(pm, &dinfo->cfg.maps, pm_link) { if (PCIR_IS_BIOS(&dinfo->cfg, pm->pm_reg)) ln2range = 32; else ln2range = pci_maprange(pm->pm_value); pci_write_config(dev, pm->pm_reg, pm->pm_value, 4); if (ln2range == 64) pci_write_config(dev, pm->pm_reg + 4, pm->pm_value >> 32, 4); } } /* * Add a resource based on a pci map register. Return 1 if the map * register is a 32bit map register or 2 if it is a 64bit register. */ static int pci_add_map(device_t bus, device_t dev, int reg, struct resource_list *rl, int force, int prefetch) { struct pci_map *pm; pci_addr_t base, map, testval; pci_addr_t start, end, count; int barlen, basezero, flags, maprange, mapsize, type; uint16_t cmd; struct resource *res; /* * The BAR may already exist if the device is a CardBus card * whose CIS is stored in this BAR. */ pm = pci_find_bar(dev, reg); if (pm != NULL) { maprange = pci_maprange(pm->pm_value); barlen = maprange == 64 ? 2 : 1; return (barlen); } pci_read_bar(dev, reg, &map, &testval, NULL); if (PCI_BAR_MEM(map)) { type = SYS_RES_MEMORY; if (map & PCIM_BAR_MEM_PREFETCH) prefetch = 1; } else type = SYS_RES_IOPORT; mapsize = pci_mapsize(testval); base = pci_mapbase(map); #ifdef __PCI_BAR_ZERO_VALID basezero = 0; #else basezero = base == 0; #endif maprange = pci_maprange(map); barlen = maprange == 64 ? 2 : 1; /* * For I/O registers, if bottom bit is set, and the next bit up * isn't clear, we know we have a BAR that doesn't conform to the * spec, so ignore it. Also, sanity check the size of the data * areas to the type of memory involved. Memory must be at least * 16 bytes in size, while I/O ranges must be at least 4. */ if (PCI_BAR_IO(testval) && (testval & PCIM_BAR_IO_RESERVED) != 0) return (barlen); if ((type == SYS_RES_MEMORY && mapsize < 4) || (type == SYS_RES_IOPORT && mapsize < 2)) return (barlen); /* Save a record of this BAR. */ pm = pci_add_bar(dev, reg, map, mapsize); if (bootverbose) { printf("\tmap[%02x]: type %s, range %2d, base %#jx, size %2d", reg, pci_maptype(map), maprange, (uintmax_t)base, mapsize); if (type == SYS_RES_IOPORT && !pci_porten(dev)) printf(", port disabled\n"); else if (type == SYS_RES_MEMORY && !pci_memen(dev)) printf(", memory disabled\n"); else printf(", enabled\n"); } /* * If base is 0, then we have problems if this architecture does * not allow that. It is best to ignore such entries for the * moment. These will be allocated later if the driver specifically * requests them. However, some removable buses look better when * all resources are allocated, so allow '0' to be overridden. * * Similarly treat maps whose values is the same as the test value * read back. These maps have had all f's written to them by the * BIOS in an attempt to disable the resources. */ if (!force && (basezero || map == testval)) return (barlen); if ((u_long)base != base) { device_printf(bus, "pci%d:%d:%d:%d bar %#x too many address bits", pci_get_domain(dev), pci_get_bus(dev), pci_get_slot(dev), pci_get_function(dev), reg); return (barlen); } /* * This code theoretically does the right thing, but has * undesirable side effects in some cases where peripherals * respond oddly to having these bits enabled. Let the user * be able to turn them off (since pci_enable_io_modes is 1 by * default). */ if (pci_enable_io_modes) { /* Turn on resources that have been left off by a lazy BIOS */ if (type == SYS_RES_IOPORT && !pci_porten(dev)) { cmd = pci_read_config(dev, PCIR_COMMAND, 2); cmd |= PCIM_CMD_PORTEN; pci_write_config(dev, PCIR_COMMAND, cmd, 2); } if (type == SYS_RES_MEMORY && !pci_memen(dev)) { cmd = pci_read_config(dev, PCIR_COMMAND, 2); cmd |= PCIM_CMD_MEMEN; pci_write_config(dev, PCIR_COMMAND, cmd, 2); } } else { if (type == SYS_RES_IOPORT && !pci_porten(dev)) return (barlen); if (type == SYS_RES_MEMORY && !pci_memen(dev)) return (barlen); } count = (pci_addr_t)1 << mapsize; flags = RF_ALIGNMENT_LOG2(mapsize); if (prefetch) flags |= RF_PREFETCHABLE; if (basezero || base == pci_mapbase(testval) || pci_clear_bars) { start = 0; /* Let the parent decide. */ end = ~0; } else { start = base; end = base + count - 1; } resource_list_add(rl, type, reg, start, end, count); /* * Try to allocate the resource for this BAR from our parent * so that this resource range is already reserved. The * driver for this device will later inherit this resource in * pci_alloc_resource(). */ res = resource_list_reserve(rl, bus, dev, type, ®, start, end, count, flags); if ((pci_do_realloc_bars || pci_has_quirk(pci_get_devid(dev), PCI_QUIRK_REALLOC_BAR)) && res == NULL && (start != 0 || end != ~0)) { /* * If the allocation fails, try to allocate a resource for * this BAR using any available range. The firmware felt * it was important enough to assign a resource, so don't * disable decoding if we can help it. */ resource_list_delete(rl, type, reg); resource_list_add(rl, type, reg, 0, ~0, count); res = resource_list_reserve(rl, bus, dev, type, ®, 0, ~0, count, flags); } if (res == NULL) { /* * If the allocation fails, delete the resource list entry * and disable decoding for this device. * * If the driver requests this resource in the future, * pci_reserve_map() will try to allocate a fresh * resource range. */ resource_list_delete(rl, type, reg); pci_disable_io(dev, type); if (bootverbose) device_printf(bus, "pci%d:%d:%d:%d bar %#x failed to allocate\n", pci_get_domain(dev), pci_get_bus(dev), pci_get_slot(dev), pci_get_function(dev), reg); } else { start = rman_get_start(res); pci_write_bar(dev, pm, start); } return (barlen); } /* * For ATA devices we need to decide early what addressing mode to use. * Legacy demands that the primary and secondary ATA ports sits on the * same addresses that old ISA hardware did. This dictates that we use * those addresses and ignore the BAR's if we cannot set PCI native * addressing mode. */ static void pci_ata_maps(device_t bus, device_t dev, struct resource_list *rl, int force, uint32_t prefetchmask) { int rid, type, progif; #if 0 /* if this device supports PCI native addressing use it */ progif = pci_read_config(dev, PCIR_PROGIF, 1); if ((progif & 0x8a) == 0x8a) { if (pci_mapbase(pci_read_config(dev, PCIR_BAR(0), 4)) && pci_mapbase(pci_read_config(dev, PCIR_BAR(2), 4))) { printf("Trying ATA native PCI addressing mode\n"); pci_write_config(dev, PCIR_PROGIF, progif | 0x05, 1); } } #endif progif = pci_read_config(dev, PCIR_PROGIF, 1); type = SYS_RES_IOPORT; if (progif & PCIP_STORAGE_IDE_MODEPRIM) { pci_add_map(bus, dev, PCIR_BAR(0), rl, force, prefetchmask & (1 << 0)); pci_add_map(bus, dev, PCIR_BAR(1), rl, force, prefetchmask & (1 << 1)); } else { rid = PCIR_BAR(0); resource_list_add(rl, type, rid, 0x1f0, 0x1f7, 8); (void)resource_list_reserve(rl, bus, dev, type, &rid, 0x1f0, 0x1f7, 8, 0); rid = PCIR_BAR(1); resource_list_add(rl, type, rid, 0x3f6, 0x3f6, 1); (void)resource_list_reserve(rl, bus, dev, type, &rid, 0x3f6, 0x3f6, 1, 0); } if (progif & PCIP_STORAGE_IDE_MODESEC) { pci_add_map(bus, dev, PCIR_BAR(2), rl, force, prefetchmask & (1 << 2)); pci_add_map(bus, dev, PCIR_BAR(3), rl, force, prefetchmask & (1 << 3)); } else { rid = PCIR_BAR(2); resource_list_add(rl, type, rid, 0x170, 0x177, 8); (void)resource_list_reserve(rl, bus, dev, type, &rid, 0x170, 0x177, 8, 0); rid = PCIR_BAR(3); resource_list_add(rl, type, rid, 0x376, 0x376, 1); (void)resource_list_reserve(rl, bus, dev, type, &rid, 0x376, 0x376, 1, 0); } pci_add_map(bus, dev, PCIR_BAR(4), rl, force, prefetchmask & (1 << 4)); pci_add_map(bus, dev, PCIR_BAR(5), rl, force, prefetchmask & (1 << 5)); } static void pci_assign_interrupt(device_t bus, device_t dev, int force_route) { struct pci_devinfo *dinfo = device_get_ivars(dev); pcicfgregs *cfg = &dinfo->cfg; char tunable_name[64]; int irq; /* Has to have an intpin to have an interrupt. */ if (cfg->intpin == 0) return; /* Let the user override the IRQ with a tunable. */ irq = PCI_INVALID_IRQ; snprintf(tunable_name, sizeof(tunable_name), "hw.pci%d.%d.%d.INT%c.irq", cfg->domain, cfg->bus, cfg->slot, cfg->intpin + 'A' - 1); if (TUNABLE_INT_FETCH(tunable_name, &irq) && (irq >= 255 || irq <= 0)) irq = PCI_INVALID_IRQ; /* * If we didn't get an IRQ via the tunable, then we either use the * IRQ value in the intline register or we ask the bus to route an * interrupt for us. If force_route is true, then we only use the * value in the intline register if the bus was unable to assign an * IRQ. */ if (!PCI_INTERRUPT_VALID(irq)) { if (!PCI_INTERRUPT_VALID(cfg->intline) || force_route) irq = PCI_ASSIGN_INTERRUPT(bus, dev); if (!PCI_INTERRUPT_VALID(irq)) irq = cfg->intline; } /* If after all that we don't have an IRQ, just bail. */ if (!PCI_INTERRUPT_VALID(irq)) return; /* Update the config register if it changed. */ if (irq != cfg->intline) { cfg->intline = irq; pci_write_config(dev, PCIR_INTLINE, irq, 1); } /* Add this IRQ as rid 0 interrupt resource. */ resource_list_add(&dinfo->resources, SYS_RES_IRQ, 0, irq, irq, 1); } /* Perform early OHCI takeover from SMM. */ static void ohci_early_takeover(device_t self) { struct resource *res; uint32_t ctl; int rid; int i; rid = PCIR_BAR(0); res = bus_alloc_resource_any(self, SYS_RES_MEMORY, &rid, RF_ACTIVE); if (res == NULL) return; ctl = bus_read_4(res, OHCI_CONTROL); if (ctl & OHCI_IR) { if (bootverbose) printf("ohci early: " "SMM active, request owner change\n"); bus_write_4(res, OHCI_COMMAND_STATUS, OHCI_OCR); for (i = 0; (i < 100) && (ctl & OHCI_IR); i++) { DELAY(1000); ctl = bus_read_4(res, OHCI_CONTROL); } if (ctl & OHCI_IR) { if (bootverbose) printf("ohci early: " "SMM does not respond, resetting\n"); bus_write_4(res, OHCI_CONTROL, OHCI_HCFS_RESET); } /* Disable interrupts */ bus_write_4(res, OHCI_INTERRUPT_DISABLE, OHCI_ALL_INTRS); } bus_release_resource(self, SYS_RES_MEMORY, rid, res); } /* Perform early UHCI takeover from SMM. */ static void uhci_early_takeover(device_t self) { struct resource *res; int rid; /* * Set the PIRQD enable bit and switch off all the others. We don't * want legacy support to interfere with us XXX Does this also mean * that the BIOS won't touch the keyboard anymore if it is connected * to the ports of the root hub? */ pci_write_config(self, PCI_LEGSUP, PCI_LEGSUP_USBPIRQDEN, 2); /* Disable interrupts */ rid = PCI_UHCI_BASE_REG; res = bus_alloc_resource_any(self, SYS_RES_IOPORT, &rid, RF_ACTIVE); if (res != NULL) { bus_write_2(res, UHCI_INTR, 0); bus_release_resource(self, SYS_RES_IOPORT, rid, res); } } /* Perform early EHCI takeover from SMM. */ static void ehci_early_takeover(device_t self) { struct resource *res; uint32_t cparams; uint32_t eec; uint8_t eecp; uint8_t bios_sem; uint8_t offs; int rid; int i; rid = PCIR_BAR(0); res = bus_alloc_resource_any(self, SYS_RES_MEMORY, &rid, RF_ACTIVE); if (res == NULL) return; cparams = bus_read_4(res, EHCI_HCCPARAMS); /* Synchronise with the BIOS if it owns the controller. */ for (eecp = EHCI_HCC_EECP(cparams); eecp != 0; eecp = EHCI_EECP_NEXT(eec)) { eec = pci_read_config(self, eecp, 4); if (EHCI_EECP_ID(eec) != EHCI_EC_LEGSUP) { continue; } bios_sem = pci_read_config(self, eecp + EHCI_LEGSUP_BIOS_SEM, 1); if (bios_sem == 0) { continue; } if (bootverbose) printf("ehci early: " "SMM active, request owner change\n"); pci_write_config(self, eecp + EHCI_LEGSUP_OS_SEM, 1, 1); for (i = 0; (i < 100) && (bios_sem != 0); i++) { DELAY(1000); bios_sem = pci_read_config(self, eecp + EHCI_LEGSUP_BIOS_SEM, 1); } if (bios_sem != 0) { if (bootverbose) printf("ehci early: " "SMM does not respond\n"); } /* Disable interrupts */ offs = EHCI_CAPLENGTH(bus_read_4(res, EHCI_CAPLEN_HCIVERSION)); bus_write_4(res, offs + EHCI_USBINTR, 0); } bus_release_resource(self, SYS_RES_MEMORY, rid, res); } /* Perform early XHCI takeover from SMM. */ static void xhci_early_takeover(device_t self) { struct resource *res; uint32_t cparams; uint32_t eec; uint8_t eecp; uint8_t bios_sem; uint8_t offs; int rid; int i; rid = PCIR_BAR(0); res = bus_alloc_resource_any(self, SYS_RES_MEMORY, &rid, RF_ACTIVE); if (res == NULL) return; cparams = bus_read_4(res, XHCI_HCSPARAMS0); eec = -1; /* Synchronise with the BIOS if it owns the controller. */ for (eecp = XHCI_HCS0_XECP(cparams) << 2; eecp != 0 && XHCI_XECP_NEXT(eec); eecp += XHCI_XECP_NEXT(eec) << 2) { eec = bus_read_4(res, eecp); if (XHCI_XECP_ID(eec) != XHCI_ID_USB_LEGACY) continue; bios_sem = bus_read_1(res, eecp + XHCI_XECP_BIOS_SEM); if (bios_sem == 0) continue; if (bootverbose) printf("xhci early: " "SMM active, request owner change\n"); bus_write_1(res, eecp + XHCI_XECP_OS_SEM, 1); /* wait a maximum of 5 second */ for (i = 0; (i < 5000) && (bios_sem != 0); i++) { DELAY(1000); bios_sem = bus_read_1(res, eecp + XHCI_XECP_BIOS_SEM); } if (bios_sem != 0) { if (bootverbose) printf("xhci early: " "SMM does not respond\n"); } /* Disable interrupts */ offs = bus_read_1(res, XHCI_CAPLENGTH); bus_write_4(res, offs + XHCI_USBCMD, 0); bus_read_4(res, offs + XHCI_USBSTS); } bus_release_resource(self, SYS_RES_MEMORY, rid, res); } #if defined(NEW_PCIB) && defined(PCI_RES_BUS) static void pci_reserve_secbus(device_t bus, device_t dev, pcicfgregs *cfg, struct resource_list *rl) { struct resource *res; char *cp; rman_res_t start, end, count; int rid, sec_bus, sec_reg, sub_bus, sub_reg, sup_bus; switch (cfg->hdrtype & PCIM_HDRTYPE) { case PCIM_HDRTYPE_BRIDGE: sec_reg = PCIR_SECBUS_1; sub_reg = PCIR_SUBBUS_1; break; case PCIM_HDRTYPE_CARDBUS: sec_reg = PCIR_SECBUS_2; sub_reg = PCIR_SUBBUS_2; break; default: return; } /* * If the existing bus range is valid, attempt to reserve it * from our parent. If this fails for any reason, clear the * secbus and subbus registers. * * XXX: Should we reset sub_bus to sec_bus if it is < sec_bus? * This would at least preserve the existing sec_bus if it is * valid. */ sec_bus = PCI_READ_CONFIG(bus, dev, sec_reg, 1); sub_bus = PCI_READ_CONFIG(bus, dev, sub_reg, 1); /* Quirk handling. */ switch (pci_get_devid(dev)) { case 0x12258086: /* Intel 82454KX/GX (Orion) */ sup_bus = pci_read_config(dev, 0x41, 1); if (sup_bus != 0xff) { sec_bus = sup_bus + 1; sub_bus = sup_bus + 1; PCI_WRITE_CONFIG(bus, dev, sec_reg, sec_bus, 1); PCI_WRITE_CONFIG(bus, dev, sub_reg, sub_bus, 1); } break; case 0x00dd10de: /* Compaq R3000 BIOS sets wrong subordinate bus number. */ if ((cp = kern_getenv("smbios.planar.maker")) == NULL) break; if (strncmp(cp, "Compal", 6) != 0) { freeenv(cp); break; } freeenv(cp); if ((cp = kern_getenv("smbios.planar.product")) == NULL) break; if (strncmp(cp, "08A0", 4) != 0) { freeenv(cp); break; } freeenv(cp); if (sub_bus < 0xa) { sub_bus = 0xa; PCI_WRITE_CONFIG(bus, dev, sub_reg, sub_bus, 1); } break; } if (bootverbose) printf("\tsecbus=%d, subbus=%d\n", sec_bus, sub_bus); if (sec_bus > 0 && sub_bus >= sec_bus) { start = sec_bus; end = sub_bus; count = end - start + 1; resource_list_add(rl, PCI_RES_BUS, 0, 0, ~0, count); /* * If requested, clear secondary bus registers in * bridge devices to force a complete renumbering * rather than reserving the existing range. However, * preserve the existing size. */ if (pci_clear_buses) goto clear; rid = 0; res = resource_list_reserve(rl, bus, dev, PCI_RES_BUS, &rid, start, end, count, 0); if (res != NULL) return; if (bootverbose) device_printf(bus, "pci%d:%d:%d:%d secbus failed to allocate\n", pci_get_domain(dev), pci_get_bus(dev), pci_get_slot(dev), pci_get_function(dev)); } clear: PCI_WRITE_CONFIG(bus, dev, sec_reg, 0, 1); PCI_WRITE_CONFIG(bus, dev, sub_reg, 0, 1); } static struct resource * pci_alloc_secbus(device_t dev, device_t child, int *rid, rman_res_t start, rman_res_t end, rman_res_t count, u_int flags) { struct pci_devinfo *dinfo; pcicfgregs *cfg; struct resource_list *rl; struct resource *res; int sec_reg, sub_reg; dinfo = device_get_ivars(child); cfg = &dinfo->cfg; rl = &dinfo->resources; switch (cfg->hdrtype & PCIM_HDRTYPE) { case PCIM_HDRTYPE_BRIDGE: sec_reg = PCIR_SECBUS_1; sub_reg = PCIR_SUBBUS_1; break; case PCIM_HDRTYPE_CARDBUS: sec_reg = PCIR_SECBUS_2; sub_reg = PCIR_SUBBUS_2; break; default: return (NULL); } if (*rid != 0) return (NULL); if (resource_list_find(rl, PCI_RES_BUS, *rid) == NULL) resource_list_add(rl, PCI_RES_BUS, *rid, start, end, count); if (!resource_list_reserved(rl, PCI_RES_BUS, *rid)) { res = resource_list_reserve(rl, dev, child, PCI_RES_BUS, rid, start, end, count, flags & ~RF_ACTIVE); if (res == NULL) { resource_list_delete(rl, PCI_RES_BUS, *rid); device_printf(child, "allocating %ju bus%s failed\n", count, count == 1 ? "" : "es"); return (NULL); } if (bootverbose) device_printf(child, "Lazy allocation of %ju bus%s at %ju\n", count, count == 1 ? "" : "es", rman_get_start(res)); PCI_WRITE_CONFIG(dev, child, sec_reg, rman_get_start(res), 1); PCI_WRITE_CONFIG(dev, child, sub_reg, rman_get_end(res), 1); } return (resource_list_alloc(rl, dev, child, PCI_RES_BUS, rid, start, end, count, flags)); } #endif static int pci_ea_bei_to_rid(device_t dev, int bei) { #ifdef PCI_IOV struct pci_devinfo *dinfo; int iov_pos; struct pcicfg_iov *iov; dinfo = device_get_ivars(dev); iov = dinfo->cfg.iov; if (iov != NULL) iov_pos = iov->iov_pos; else iov_pos = 0; #endif /* Check if matches BAR */ if ((bei >= PCIM_EA_BEI_BAR_0) && (bei <= PCIM_EA_BEI_BAR_5)) return (PCIR_BAR(bei)); /* Check ROM */ if (bei == PCIM_EA_BEI_ROM) return (PCIR_BIOS); #ifdef PCI_IOV /* Check if matches VF_BAR */ if ((iov != NULL) && (bei >= PCIM_EA_BEI_VF_BAR_0) && (bei <= PCIM_EA_BEI_VF_BAR_5)) return (PCIR_SRIOV_BAR(bei - PCIM_EA_BEI_VF_BAR_0) + iov_pos); #endif return (-1); } int pci_ea_is_enabled(device_t dev, int rid) { struct pci_ea_entry *ea; struct pci_devinfo *dinfo; dinfo = device_get_ivars(dev); STAILQ_FOREACH(ea, &dinfo->cfg.ea.ea_entries, eae_link) { if (pci_ea_bei_to_rid(dev, ea->eae_bei) == rid) return ((ea->eae_flags & PCIM_EA_ENABLE) > 0); } return (0); } void pci_add_resources_ea(device_t bus, device_t dev, int alloc_iov) { struct pci_ea_entry *ea; struct pci_devinfo *dinfo; pci_addr_t start, end, count; struct resource_list *rl; int type, flags, rid; struct resource *res; uint32_t tmp; #ifdef PCI_IOV struct pcicfg_iov *iov; #endif dinfo = device_get_ivars(dev); rl = &dinfo->resources; flags = 0; #ifdef PCI_IOV iov = dinfo->cfg.iov; #endif if (dinfo->cfg.ea.ea_location == 0) return; STAILQ_FOREACH(ea, &dinfo->cfg.ea.ea_entries, eae_link) { /* * TODO: Ignore EA-BAR if is not enabled. * Currently the EA implementation supports * only situation, where EA structure contains * predefined entries. In case they are not enabled * leave them unallocated and proceed with * a legacy-BAR mechanism. */ if ((ea->eae_flags & PCIM_EA_ENABLE) == 0) continue; switch ((ea->eae_flags & PCIM_EA_PP) >> PCIM_EA_PP_OFFSET) { case PCIM_EA_P_MEM_PREFETCH: case PCIM_EA_P_VF_MEM_PREFETCH: flags = RF_PREFETCHABLE; /* FALLTHROUGH */ case PCIM_EA_P_VF_MEM: case PCIM_EA_P_MEM: type = SYS_RES_MEMORY; break; case PCIM_EA_P_IO: type = SYS_RES_IOPORT; break; default: continue; } if (alloc_iov != 0) { #ifdef PCI_IOV /* Allocating IOV, confirm BEI matches */ if ((ea->eae_bei < PCIM_EA_BEI_VF_BAR_0) || (ea->eae_bei > PCIM_EA_BEI_VF_BAR_5)) continue; #else continue; #endif } else { /* Allocating BAR, confirm BEI matches */ if (((ea->eae_bei < PCIM_EA_BEI_BAR_0) || (ea->eae_bei > PCIM_EA_BEI_BAR_5)) && (ea->eae_bei != PCIM_EA_BEI_ROM)) continue; } rid = pci_ea_bei_to_rid(dev, ea->eae_bei); if (rid < 0) continue; /* Skip resources already allocated by EA */ if ((resource_list_find(rl, SYS_RES_MEMORY, rid) != NULL) || (resource_list_find(rl, SYS_RES_IOPORT, rid) != NULL)) continue; start = ea->eae_base; count = ea->eae_max_offset + 1; #ifdef PCI_IOV if (iov != NULL) count = count * iov->iov_num_vfs; #endif end = start + count - 1; if (count == 0) continue; resource_list_add(rl, type, rid, start, end, count); res = resource_list_reserve(rl, bus, dev, type, &rid, start, end, count, flags); if (res == NULL) { resource_list_delete(rl, type, rid); /* * Failed to allocate using EA, disable entry. * Another attempt to allocation will be performed * further, but this time using legacy BAR registers */ tmp = pci_read_config(dev, ea->eae_cfg_offset, 4); tmp &= ~PCIM_EA_ENABLE; pci_write_config(dev, ea->eae_cfg_offset, tmp, 4); /* * Disabling entry might fail in case it is hardwired. * Read flags again to match current status. */ ea->eae_flags = pci_read_config(dev, ea->eae_cfg_offset, 4); continue; } /* As per specification, fill BAR with zeros */ pci_write_config(dev, rid, 0, 4); } } void pci_add_resources(device_t bus, device_t dev, int force, uint32_t prefetchmask) { struct pci_devinfo *dinfo; pcicfgregs *cfg; struct resource_list *rl; const struct pci_quirk *q; uint32_t devid; int i; dinfo = device_get_ivars(dev); cfg = &dinfo->cfg; rl = &dinfo->resources; devid = (cfg->device << 16) | cfg->vendor; /* Allocate resources using Enhanced Allocation */ pci_add_resources_ea(bus, dev, 0); /* ATA devices needs special map treatment */ if ((pci_get_class(dev) == PCIC_STORAGE) && (pci_get_subclass(dev) == PCIS_STORAGE_IDE) && ((pci_get_progif(dev) & PCIP_STORAGE_IDE_MASTERDEV) || (!pci_read_config(dev, PCIR_BAR(0), 4) && !pci_read_config(dev, PCIR_BAR(2), 4))) ) pci_ata_maps(bus, dev, rl, force, prefetchmask); else for (i = 0; i < cfg->nummaps;) { /* Skip resources already managed by EA */ if ((resource_list_find(rl, SYS_RES_MEMORY, PCIR_BAR(i)) != NULL) || (resource_list_find(rl, SYS_RES_IOPORT, PCIR_BAR(i)) != NULL) || pci_ea_is_enabled(dev, PCIR_BAR(i))) { i++; continue; } /* * Skip quirked resources. */ for (q = &pci_quirks[0]; q->devid != 0; q++) if (q->devid == devid && q->type == PCI_QUIRK_UNMAP_REG && q->arg1 == PCIR_BAR(i)) break; if (q->devid != 0) { i++; continue; } i += pci_add_map(bus, dev, PCIR_BAR(i), rl, force, prefetchmask & (1 << i)); } /* * Add additional, quirked resources. */ for (q = &pci_quirks[0]; q->devid != 0; q++) if (q->devid == devid && q->type == PCI_QUIRK_MAP_REG) pci_add_map(bus, dev, q->arg1, rl, force, 0); if (cfg->intpin > 0 && PCI_INTERRUPT_VALID(cfg->intline) && pci_intx_reroute) { #ifdef __PCI_REROUTE_INTERRUPT /* * Try to re-route interrupts. Sometimes the BIOS or * firmware may leave bogus values in these registers. * If the re-route fails, then just stick with what we * have. */ pci_assign_interrupt(bus, dev, 1); #else pci_assign_interrupt(bus, dev, 0); #endif } if (pci_usb_takeover && pci_get_class(dev) == PCIC_SERIALBUS && pci_get_subclass(dev) == PCIS_SERIALBUS_USB) { if (pci_get_progif(dev) == PCIP_SERIALBUS_USB_XHCI) xhci_early_takeover(dev); else if (pci_get_progif(dev) == PCIP_SERIALBUS_USB_EHCI) ehci_early_takeover(dev); else if (pci_get_progif(dev) == PCIP_SERIALBUS_USB_OHCI) ohci_early_takeover(dev); else if (pci_get_progif(dev) == PCIP_SERIALBUS_USB_UHCI) uhci_early_takeover(dev); } #if defined(NEW_PCIB) && defined(PCI_RES_BUS) /* * Reserve resources for secondary bus ranges behind bridge * devices. */ pci_reserve_secbus(bus, dev, cfg, rl); #endif } static struct pci_devinfo * pci_identify_function(device_t pcib, device_t dev, int domain, int busno, int slot, int func) { struct pci_devinfo *dinfo; dinfo = pci_read_device(pcib, dev, domain, busno, slot, func); if (dinfo != NULL) pci_add_child(dev, dinfo); return (dinfo); } void pci_add_children(device_t dev, int domain, int busno) { #define REG(n, w) PCIB_READ_CONFIG(pcib, busno, s, f, n, w) device_t pcib = device_get_parent(dev); struct pci_devinfo *dinfo; int maxslots; int s, f, pcifunchigh; uint8_t hdrtype; int first_func; /* * Try to detect a device at slot 0, function 0. If it exists, try to * enable ARI. We must enable ARI before detecting the rest of the * functions on this bus as ARI changes the set of slots and functions * that are legal on this bus. */ dinfo = pci_identify_function(pcib, dev, domain, busno, 0, 0); if (dinfo != NULL && pci_enable_ari) PCIB_TRY_ENABLE_ARI(pcib, dinfo->cfg.dev); /* * Start looking for new devices on slot 0 at function 1 because we * just identified the device at slot 0, function 0. */ first_func = 1; maxslots = PCIB_MAXSLOTS(pcib); for (s = 0; s <= maxslots; s++, first_func = 0) { pcifunchigh = 0; f = 0; DELAY(1); /* If function 0 is not present, skip to the next slot. */ if (REG(PCIR_VENDOR, 2) == PCIV_INVALID) continue; hdrtype = REG(PCIR_HDRTYPE, 1); if ((hdrtype & PCIM_HDRTYPE) > PCI_MAXHDRTYPE) continue; if (hdrtype & PCIM_MFDEV) pcifunchigh = PCIB_MAXFUNCS(pcib); for (f = first_func; f <= pcifunchigh; f++) pci_identify_function(pcib, dev, domain, busno, s, f); } #undef REG } int pci_rescan_method(device_t dev) { #define REG(n, w) PCIB_READ_CONFIG(pcib, busno, s, f, n, w) device_t pcib = device_get_parent(dev); device_t child, *devlist, *unchanged; int devcount, error, i, j, maxslots, oldcount; int busno, domain, s, f, pcifunchigh; uint8_t hdrtype; /* No need to check for ARI on a rescan. */ error = device_get_children(dev, &devlist, &devcount); if (error) return (error); if (devcount != 0) { unchanged = malloc(devcount * sizeof(device_t), M_TEMP, M_NOWAIT | M_ZERO); if (unchanged == NULL) { free(devlist, M_TEMP); return (ENOMEM); } } else unchanged = NULL; domain = pcib_get_domain(dev); busno = pcib_get_bus(dev); maxslots = PCIB_MAXSLOTS(pcib); for (s = 0; s <= maxslots; s++) { /* If function 0 is not present, skip to the next slot. */ f = 0; if (REG(PCIR_VENDOR, 2) == PCIV_INVALID) continue; pcifunchigh = 0; hdrtype = REG(PCIR_HDRTYPE, 1); if ((hdrtype & PCIM_HDRTYPE) > PCI_MAXHDRTYPE) continue; if (hdrtype & PCIM_MFDEV) pcifunchigh = PCIB_MAXFUNCS(pcib); for (f = 0; f <= pcifunchigh; f++) { if (REG(PCIR_VENDOR, 2) == PCIV_INVALID) continue; /* * Found a valid function. Check if a * device_t for this device already exists. */ for (i = 0; i < devcount; i++) { child = devlist[i]; if (child == NULL) continue; if (pci_get_slot(child) == s && pci_get_function(child) == f) { unchanged[i] = child; goto next_func; } } pci_identify_function(pcib, dev, domain, busno, s, f); next_func:; } } /* Remove devices that are no longer present. */ for (i = 0; i < devcount; i++) { if (unchanged[i] != NULL) continue; device_delete_child(dev, devlist[i]); } free(devlist, M_TEMP); oldcount = devcount; /* Try to attach the devices just added. */ error = device_get_children(dev, &devlist, &devcount); if (error) { free(unchanged, M_TEMP); return (error); } for (i = 0; i < devcount; i++) { for (j = 0; j < oldcount; j++) { if (devlist[i] == unchanged[j]) goto next_device; } device_probe_and_attach(devlist[i]); next_device:; } free(unchanged, M_TEMP); free(devlist, M_TEMP); return (0); #undef REG } #ifdef PCI_IOV device_t pci_add_iov_child(device_t bus, device_t pf, uint16_t rid, uint16_t vid, uint16_t did) { struct pci_devinfo *vf_dinfo; device_t pcib; int busno, slot, func; pcib = device_get_parent(bus); PCIB_DECODE_RID(pcib, rid, &busno, &slot, &func); vf_dinfo = pci_fill_devinfo(pcib, bus, pci_get_domain(pcib), busno, slot, func, vid, did); vf_dinfo->cfg.flags |= PCICFG_VF; pci_add_child(bus, vf_dinfo); return (vf_dinfo->cfg.dev); } device_t pci_create_iov_child_method(device_t bus, device_t pf, uint16_t rid, uint16_t vid, uint16_t did) { return (pci_add_iov_child(bus, pf, rid, vid, did)); } #endif /* * For PCIe device set Max_Payload_Size to match PCIe root's. */ static void pcie_setup_mps(device_t dev) { struct pci_devinfo *dinfo = device_get_ivars(dev); device_t root; uint16_t rmps, mmps, mps; if (dinfo->cfg.pcie.pcie_location == 0) return; root = pci_find_pcie_root_port(dev); if (root == NULL) return; /* Check whether the MPS is already configured. */ rmps = pcie_read_config(root, PCIER_DEVICE_CTL, 2) & PCIEM_CTL_MAX_PAYLOAD; mps = pcie_read_config(dev, PCIER_DEVICE_CTL, 2) & PCIEM_CTL_MAX_PAYLOAD; if (mps == rmps) return; /* Check whether the device is capable of the root's MPS. */ mmps = (pcie_read_config(dev, PCIER_DEVICE_CAP, 2) & PCIEM_CAP_MAX_PAYLOAD) << 5; if (rmps > mmps) { /* * The device is unable to handle root's MPS. Limit root. * XXX: We should traverse through all the tree, applying * it to all the devices. */ pcie_adjust_config(root, PCIER_DEVICE_CTL, PCIEM_CTL_MAX_PAYLOAD, mmps, 2); } else { pcie_adjust_config(dev, PCIER_DEVICE_CTL, PCIEM_CTL_MAX_PAYLOAD, rmps, 2); } } static void pci_add_child_clear_aer(device_t dev, struct pci_devinfo *dinfo) { int aer; uint32_t r; uint16_t r2; if (dinfo->cfg.pcie.pcie_location != 0 && dinfo->cfg.pcie.pcie_type == PCIEM_TYPE_ROOT_PORT) { r2 = pci_read_config(dev, dinfo->cfg.pcie.pcie_location + PCIER_ROOT_CTL, 2); r2 &= ~(PCIEM_ROOT_CTL_SERR_CORR | PCIEM_ROOT_CTL_SERR_NONFATAL | PCIEM_ROOT_CTL_SERR_FATAL); pci_write_config(dev, dinfo->cfg.pcie.pcie_location + PCIER_ROOT_CTL, r2, 2); } if (pci_find_extcap(dev, PCIZ_AER, &aer) == 0) { r = pci_read_config(dev, aer + PCIR_AER_UC_STATUS, 4); pci_write_config(dev, aer + PCIR_AER_UC_STATUS, r, 4); if (r != 0 && bootverbose) { pci_printf(&dinfo->cfg, "clearing AER UC 0x%08x -> 0x%08x\n", r, pci_read_config(dev, aer + PCIR_AER_UC_STATUS, 4)); } r = pci_read_config(dev, aer + PCIR_AER_UC_MASK, 4); r &= ~(PCIM_AER_UC_TRAINING_ERROR | PCIM_AER_UC_DL_PROTOCOL_ERROR | PCIM_AER_UC_SURPRISE_LINK_DOWN | PCIM_AER_UC_POISONED_TLP | PCIM_AER_UC_FC_PROTOCOL_ERROR | PCIM_AER_UC_COMPLETION_TIMEOUT | PCIM_AER_UC_COMPLETER_ABORT | PCIM_AER_UC_UNEXPECTED_COMPLETION | PCIM_AER_UC_RECEIVER_OVERFLOW | PCIM_AER_UC_MALFORMED_TLP | PCIM_AER_UC_ECRC_ERROR | PCIM_AER_UC_UNSUPPORTED_REQUEST | PCIM_AER_UC_ACS_VIOLATION | PCIM_AER_UC_INTERNAL_ERROR | PCIM_AER_UC_MC_BLOCKED_TLP | PCIM_AER_UC_ATOMIC_EGRESS_BLK | PCIM_AER_UC_TLP_PREFIX_BLOCKED); pci_write_config(dev, aer + PCIR_AER_UC_MASK, r, 4); r = pci_read_config(dev, aer + PCIR_AER_COR_STATUS, 4); pci_write_config(dev, aer + PCIR_AER_COR_STATUS, r, 4); if (r != 0 && bootverbose) { pci_printf(&dinfo->cfg, "clearing AER COR 0x%08x -> 0x%08x\n", r, pci_read_config(dev, aer + PCIR_AER_COR_STATUS, 4)); } r = pci_read_config(dev, aer + PCIR_AER_COR_MASK, 4); r &= ~(PCIM_AER_COR_RECEIVER_ERROR | PCIM_AER_COR_BAD_TLP | PCIM_AER_COR_BAD_DLLP | PCIM_AER_COR_REPLAY_ROLLOVER | PCIM_AER_COR_REPLAY_TIMEOUT | PCIM_AER_COR_ADVISORY_NF_ERROR | PCIM_AER_COR_INTERNAL_ERROR | PCIM_AER_COR_HEADER_LOG_OVFLOW); pci_write_config(dev, aer + PCIR_AER_COR_MASK, r, 4); r = pci_read_config(dev, dinfo->cfg.pcie.pcie_location + PCIER_DEVICE_CTL, 2); r |= PCIEM_CTL_COR_ENABLE | PCIEM_CTL_NFER_ENABLE | PCIEM_CTL_FER_ENABLE | PCIEM_CTL_URR_ENABLE; pci_write_config(dev, dinfo->cfg.pcie.pcie_location + PCIER_DEVICE_CTL, r, 2); } } void pci_add_child(device_t bus, struct pci_devinfo *dinfo) { device_t dev; dinfo->cfg.dev = dev = device_add_child(bus, NULL, -1); device_set_ivars(dev, dinfo); resource_list_init(&dinfo->resources); pci_cfg_save(dev, dinfo, 0); pci_cfg_restore(dev, dinfo); pci_print_verbose(dinfo); pci_add_resources(bus, dev, 0, 0); if (pci_enable_mps_tune) pcie_setup_mps(dev); pci_child_added(dinfo->cfg.dev); if (pci_clear_aer_on_attach) pci_add_child_clear_aer(dev, dinfo); EVENTHANDLER_INVOKE(pci_add_device, dinfo->cfg.dev); } void pci_child_added_method(device_t dev, device_t child) { } static int pci_probe(device_t dev) { device_set_desc(dev, "PCI bus"); /* Allow other subclasses to override this driver. */ return (BUS_PROBE_GENERIC); } int pci_attach_common(device_t dev) { struct pci_softc *sc; int busno, domain; #ifdef PCI_RES_BUS int rid; #endif sc = device_get_softc(dev); domain = pcib_get_domain(dev); busno = pcib_get_bus(dev); #ifdef PCI_RES_BUS rid = 0; sc->sc_bus = bus_alloc_resource(dev, PCI_RES_BUS, &rid, busno, busno, 1, 0); if (sc->sc_bus == NULL) { device_printf(dev, "failed to allocate bus number\n"); return (ENXIO); } #endif if (bootverbose) device_printf(dev, "domain=%d, physical bus=%d\n", domain, busno); sc->sc_dma_tag = bus_get_dma_tag(dev); return (0); } int pci_attach(device_t dev) { int busno, domain, error; error = pci_attach_common(dev); if (error) return (error); /* * Since there can be multiple independently numbered PCI * buses on systems with multiple PCI domains, we can't use * the unit number to decide which bus we are probing. We ask * the parent pcib what our domain and bus numbers are. */ domain = pcib_get_domain(dev); busno = pcib_get_bus(dev); pci_add_children(dev, domain, busno); return (bus_generic_attach(dev)); } int pci_detach(device_t dev) { #ifdef PCI_RES_BUS struct pci_softc *sc; #endif int error; error = bus_generic_detach(dev); if (error) return (error); #ifdef PCI_RES_BUS sc = device_get_softc(dev); error = bus_release_resource(dev, PCI_RES_BUS, 0, sc->sc_bus); if (error) return (error); #endif return (device_delete_children(dev)); } static void pci_hint_device_unit(device_t dev, device_t child, const char *name, int *unitp) { int line, unit; const char *at; char me1[24], me2[32]; uint8_t b, s, f; uint32_t d; device_location_cache_t *cache; d = pci_get_domain(child); b = pci_get_bus(child); s = pci_get_slot(child); f = pci_get_function(child); snprintf(me1, sizeof(me1), "pci%u:%u:%u", b, s, f); snprintf(me2, sizeof(me2), "pci%u:%u:%u:%u", d, b, s, f); line = 0; cache = dev_wired_cache_init(); while (resource_find_dev(&line, name, &unit, "at", NULL) == 0) { resource_string_value(name, unit, "at", &at); if (strcmp(at, me1) == 0 || strcmp(at, me2) == 0) { *unitp = unit; break; } if (dev_wired_cache_match(cache, child, at)) { *unitp = unit; break; } } dev_wired_cache_fini(cache); } static void pci_set_power_child(device_t dev, device_t child, int state) { device_t pcib; int dstate; /* * Set the device to the given state. If the firmware suggests * a different power state, use it instead. If power management * is not present, the firmware is responsible for managing * device power. Skip children who aren't attached since they * are handled separately. */ pcib = device_get_parent(dev); dstate = state; if (device_is_attached(child) && PCIB_POWER_FOR_SLEEP(pcib, child, &dstate) == 0) pci_set_powerstate(child, dstate); } int pci_suspend_child(device_t dev, device_t child) { struct pci_devinfo *dinfo; struct resource_list_entry *rle; int error; dinfo = device_get_ivars(child); /* * Save the PCI configuration space for the child and set the * device in the appropriate power state for this sleep state. */ pci_cfg_save(child, dinfo, 0); /* Suspend devices before potentially powering them down. */ error = bus_generic_suspend_child(dev, child); if (error) return (error); if (pci_do_power_suspend) { /* * Make sure this device's interrupt handler is not invoked * in the case the device uses a shared interrupt that can * be raised by some other device. * This is applicable only to regular (legacy) PCI interrupts * as MSI/MSI-X interrupts are never shared. */ rle = resource_list_find(&dinfo->resources, SYS_RES_IRQ, 0); if (rle != NULL && rle->res != NULL) (void)bus_suspend_intr(child, rle->res); pci_set_power_child(dev, child, PCI_POWERSTATE_D3); } return (0); } int pci_resume_child(device_t dev, device_t child) { struct pci_devinfo *dinfo; struct resource_list_entry *rle; if (pci_do_power_resume) pci_set_power_child(dev, child, PCI_POWERSTATE_D0); dinfo = device_get_ivars(child); pci_cfg_restore(child, dinfo); if (!device_is_attached(child)) pci_cfg_save(child, dinfo, 1); bus_generic_resume_child(dev, child); /* * Allow interrupts only after fully resuming the driver and hardware. */ if (pci_do_power_suspend) { /* See pci_suspend_child for details. */ rle = resource_list_find(&dinfo->resources, SYS_RES_IRQ, 0); if (rle != NULL && rle->res != NULL) (void)bus_resume_intr(child, rle->res); } return (0); } int pci_resume(device_t dev) { device_t child, *devlist; int error, i, numdevs; if ((error = device_get_children(dev, &devlist, &numdevs)) != 0) return (error); /* * Resume critical devices first, then everything else later. */ for (i = 0; i < numdevs; i++) { child = devlist[i]; switch (pci_get_class(child)) { case PCIC_DISPLAY: case PCIC_MEMORY: case PCIC_BRIDGE: case PCIC_BASEPERIPH: BUS_RESUME_CHILD(dev, child); break; } } for (i = 0; i < numdevs; i++) { child = devlist[i]; switch (pci_get_class(child)) { case PCIC_DISPLAY: case PCIC_MEMORY: case PCIC_BRIDGE: case PCIC_BASEPERIPH: break; default: BUS_RESUME_CHILD(dev, child); } } free(devlist, M_TEMP); return (0); } static void pci_load_vendor_data(void) { caddr_t data; void *ptr; size_t sz; data = preload_search_by_type("pci_vendor_data"); if (data != NULL) { ptr = preload_fetch_addr(data); sz = preload_fetch_size(data); if (ptr != NULL && sz != 0) { pci_vendordata = ptr; pci_vendordata_size = sz; /* terminate the database */ pci_vendordata[pci_vendordata_size] = '\n'; } } } void pci_driver_added(device_t dev, driver_t *driver) { int numdevs; device_t *devlist; device_t child; struct pci_devinfo *dinfo; int i; if (bootverbose) device_printf(dev, "driver added\n"); DEVICE_IDENTIFY(driver, dev); if (device_get_children(dev, &devlist, &numdevs) != 0) return; for (i = 0; i < numdevs; i++) { child = devlist[i]; if (device_get_state(child) != DS_NOTPRESENT) continue; dinfo = device_get_ivars(child); pci_print_verbose(dinfo); if (bootverbose) pci_printf(&dinfo->cfg, "reprobing on driver added\n"); pci_cfg_restore(child, dinfo); if (device_probe_and_attach(child) != 0) pci_child_detached(dev, child); } free(devlist, M_TEMP); } int pci_setup_intr(device_t dev, device_t child, struct resource *irq, int flags, driver_filter_t *filter, driver_intr_t *intr, void *arg, void **cookiep) { struct pci_devinfo *dinfo; struct msix_table_entry *mte; struct msix_vector *mv; uint64_t addr; uint32_t data; void *cookie; int error, rid; error = bus_generic_setup_intr(dev, child, irq, flags, filter, intr, arg, &cookie); if (error) return (error); /* If this is not a direct child, just bail out. */ if (device_get_parent(child) != dev) { *cookiep = cookie; return(0); } rid = rman_get_rid(irq); if (rid == 0) { /* Make sure that INTx is enabled */ pci_clear_command_bit(dev, child, PCIM_CMD_INTxDIS); } else { /* * Check to see if the interrupt is MSI or MSI-X. * Ask our parent to map the MSI and give * us the address and data register values. * If we fail for some reason, teardown the * interrupt handler. */ dinfo = device_get_ivars(child); if (dinfo->cfg.msi.msi_alloc > 0) { if (dinfo->cfg.msi.msi_addr == 0) { KASSERT(dinfo->cfg.msi.msi_handlers == 0, ("MSI has handlers, but vectors not mapped")); error = PCIB_MAP_MSI(device_get_parent(dev), child, rman_get_start(irq), &addr, &data); if (error) goto bad; dinfo->cfg.msi.msi_addr = addr; dinfo->cfg.msi.msi_data = data; } if (dinfo->cfg.msi.msi_handlers == 0) pci_enable_msi(child, dinfo->cfg.msi.msi_addr, dinfo->cfg.msi.msi_data); dinfo->cfg.msi.msi_handlers++; } else { KASSERT(dinfo->cfg.msix.msix_alloc > 0, ("No MSI or MSI-X interrupts allocated")); KASSERT(rid <= dinfo->cfg.msix.msix_table_len, ("MSI-X index too high")); mte = &dinfo->cfg.msix.msix_table[rid - 1]; KASSERT(mte->mte_vector != 0, ("no message vector")); mv = &dinfo->cfg.msix.msix_vectors[mte->mte_vector - 1]; KASSERT(mv->mv_irq == rman_get_start(irq), ("IRQ mismatch")); if (mv->mv_address == 0) { KASSERT(mte->mte_handlers == 0, ("MSI-X table entry has handlers, but vector not mapped")); error = PCIB_MAP_MSI(device_get_parent(dev), child, rman_get_start(irq), &addr, &data); if (error) goto bad; mv->mv_address = addr; mv->mv_data = data; } /* * The MSIX table entry must be made valid by * incrementing the mte_handlers before * calling pci_enable_msix() and * pci_resume_msix(). Else the MSIX rewrite * table quirk will not work as expected. */ mte->mte_handlers++; if (mte->mte_handlers == 1) { pci_enable_msix(child, rid - 1, mv->mv_address, mv->mv_data); pci_unmask_msix(child, rid - 1); } } /* * Make sure that INTx is disabled if we are using MSI/MSI-X, * unless the device is affected by PCI_QUIRK_MSI_INTX_BUG, * in which case we "enable" INTx so MSI/MSI-X actually works. */ if (!pci_has_quirk(pci_get_devid(child), PCI_QUIRK_MSI_INTX_BUG)) pci_set_command_bit(dev, child, PCIM_CMD_INTxDIS); else pci_clear_command_bit(dev, child, PCIM_CMD_INTxDIS); bad: if (error) { (void)bus_generic_teardown_intr(dev, child, irq, cookie); return (error); } } *cookiep = cookie; return (0); } int pci_teardown_intr(device_t dev, device_t child, struct resource *irq, void *cookie) { struct msix_table_entry *mte; struct resource_list_entry *rle; struct pci_devinfo *dinfo; int error, rid; if (irq == NULL || !(rman_get_flags(irq) & RF_ACTIVE)) return (EINVAL); /* If this isn't a direct child, just bail out */ if (device_get_parent(child) != dev) return(bus_generic_teardown_intr(dev, child, irq, cookie)); rid = rman_get_rid(irq); if (rid == 0) { /* Mask INTx */ pci_set_command_bit(dev, child, PCIM_CMD_INTxDIS); } else { /* * Check to see if the interrupt is MSI or MSI-X. If so, * decrement the appropriate handlers count and mask the * MSI-X message, or disable MSI messages if the count * drops to 0. */ dinfo = device_get_ivars(child); rle = resource_list_find(&dinfo->resources, SYS_RES_IRQ, rid); if (rle->res != irq) return (EINVAL); if (dinfo->cfg.msi.msi_alloc > 0) { KASSERT(rid <= dinfo->cfg.msi.msi_alloc, ("MSI-X index too high")); if (dinfo->cfg.msi.msi_handlers == 0) return (EINVAL); dinfo->cfg.msi.msi_handlers--; if (dinfo->cfg.msi.msi_handlers == 0) pci_disable_msi(child); } else { KASSERT(dinfo->cfg.msix.msix_alloc > 0, ("No MSI or MSI-X interrupts allocated")); KASSERT(rid <= dinfo->cfg.msix.msix_table_len, ("MSI-X index too high")); mte = &dinfo->cfg.msix.msix_table[rid - 1]; if (mte->mte_handlers == 0) return (EINVAL); mte->mte_handlers--; if (mte->mte_handlers == 0) pci_mask_msix(child, rid - 1); } } error = bus_generic_teardown_intr(dev, child, irq, cookie); if (rid > 0) KASSERT(error == 0, ("%s: generic teardown failed for MSI/MSI-X", __func__)); return (error); } int pci_print_child(device_t dev, device_t child) { struct pci_devinfo *dinfo; struct resource_list *rl; int retval = 0; dinfo = device_get_ivars(child); rl = &dinfo->resources; retval += bus_print_child_header(dev, child); retval += resource_list_print_type(rl, "port", SYS_RES_IOPORT, "%#jx"); retval += resource_list_print_type(rl, "mem", SYS_RES_MEMORY, "%#jx"); retval += resource_list_print_type(rl, "irq", SYS_RES_IRQ, "%jd"); if (device_get_flags(dev)) retval += printf(" flags %#x", device_get_flags(dev)); retval += printf(" at device %d.%d", pci_get_slot(child), pci_get_function(child)); retval += bus_print_child_domain(dev, child); retval += bus_print_child_footer(dev, child); return (retval); } static const struct { int class; int subclass; int report; /* 0 = bootverbose, 1 = always */ const char *desc; } pci_nomatch_tab[] = { {PCIC_OLD, -1, 1, "old"}, {PCIC_OLD, PCIS_OLD_NONVGA, 1, "non-VGA display device"}, {PCIC_OLD, PCIS_OLD_VGA, 1, "VGA-compatible display device"}, {PCIC_STORAGE, -1, 1, "mass storage"}, {PCIC_STORAGE, PCIS_STORAGE_SCSI, 1, "SCSI"}, {PCIC_STORAGE, PCIS_STORAGE_IDE, 1, "ATA"}, {PCIC_STORAGE, PCIS_STORAGE_FLOPPY, 1, "floppy disk"}, {PCIC_STORAGE, PCIS_STORAGE_IPI, 1, "IPI"}, {PCIC_STORAGE, PCIS_STORAGE_RAID, 1, "RAID"}, {PCIC_STORAGE, PCIS_STORAGE_ATA_ADMA, 1, "ATA (ADMA)"}, {PCIC_STORAGE, PCIS_STORAGE_SATA, 1, "SATA"}, {PCIC_STORAGE, PCIS_STORAGE_SAS, 1, "SAS"}, {PCIC_STORAGE, PCIS_STORAGE_NVM, 1, "NVM"}, {PCIC_NETWORK, -1, 1, "network"}, {PCIC_NETWORK, PCIS_NETWORK_ETHERNET, 1, "ethernet"}, {PCIC_NETWORK, PCIS_NETWORK_TOKENRING, 1, "token ring"}, {PCIC_NETWORK, PCIS_NETWORK_FDDI, 1, "fddi"}, {PCIC_NETWORK, PCIS_NETWORK_ATM, 1, "ATM"}, {PCIC_NETWORK, PCIS_NETWORK_ISDN, 1, "ISDN"}, {PCIC_DISPLAY, -1, 1, "display"}, {PCIC_DISPLAY, PCIS_DISPLAY_VGA, 1, "VGA"}, {PCIC_DISPLAY, PCIS_DISPLAY_XGA, 1, "XGA"}, {PCIC_DISPLAY, PCIS_DISPLAY_3D, 1, "3D"}, {PCIC_MULTIMEDIA, -1, 1, "multimedia"}, {PCIC_MULTIMEDIA, PCIS_MULTIMEDIA_VIDEO, 1, "video"}, {PCIC_MULTIMEDIA, PCIS_MULTIMEDIA_AUDIO, 1, "audio"}, {PCIC_MULTIMEDIA, PCIS_MULTIMEDIA_TELE, 1, "telephony"}, {PCIC_MULTIMEDIA, PCIS_MULTIMEDIA_HDA, 1, "HDA"}, {PCIC_MEMORY, -1, 1, "memory"}, {PCIC_MEMORY, PCIS_MEMORY_RAM, 1, "RAM"}, {PCIC_MEMORY, PCIS_MEMORY_FLASH, 1, "flash"}, {PCIC_BRIDGE, -1, 1, "bridge"}, {PCIC_BRIDGE, PCIS_BRIDGE_HOST, 1, "HOST-PCI"}, {PCIC_BRIDGE, PCIS_BRIDGE_ISA, 1, "PCI-ISA"}, {PCIC_BRIDGE, PCIS_BRIDGE_EISA, 1, "PCI-EISA"}, {PCIC_BRIDGE, PCIS_BRIDGE_MCA, 1, "PCI-MCA"}, {PCIC_BRIDGE, PCIS_BRIDGE_PCI, 1, "PCI-PCI"}, {PCIC_BRIDGE, PCIS_BRIDGE_PCMCIA, 1, "PCI-PCMCIA"}, {PCIC_BRIDGE, PCIS_BRIDGE_NUBUS, 1, "PCI-NuBus"}, {PCIC_BRIDGE, PCIS_BRIDGE_CARDBUS, 1, "PCI-CardBus"}, {PCIC_BRIDGE, PCIS_BRIDGE_RACEWAY, 1, "PCI-RACEway"}, {PCIC_SIMPLECOMM, -1, 1, "simple comms"}, {PCIC_SIMPLECOMM, PCIS_SIMPLECOMM_UART, 1, "UART"}, /* could detect 16550 */ {PCIC_SIMPLECOMM, PCIS_SIMPLECOMM_PAR, 1, "parallel port"}, {PCIC_SIMPLECOMM, PCIS_SIMPLECOMM_MULSER, 1, "multiport serial"}, {PCIC_SIMPLECOMM, PCIS_SIMPLECOMM_MODEM, 1, "generic modem"}, {PCIC_BASEPERIPH, -1, 0, "base peripheral"}, {PCIC_BASEPERIPH, PCIS_BASEPERIPH_PIC, 1, "interrupt controller"}, {PCIC_BASEPERIPH, PCIS_BASEPERIPH_DMA, 1, "DMA controller"}, {PCIC_BASEPERIPH, PCIS_BASEPERIPH_TIMER, 1, "timer"}, {PCIC_BASEPERIPH, PCIS_BASEPERIPH_RTC, 1, "realtime clock"}, {PCIC_BASEPERIPH, PCIS_BASEPERIPH_PCIHOT, 1, "PCI hot-plug controller"}, {PCIC_BASEPERIPH, PCIS_BASEPERIPH_SDHC, 1, "SD host controller"}, {PCIC_BASEPERIPH, PCIS_BASEPERIPH_IOMMU, 1, "IOMMU"}, {PCIC_INPUTDEV, -1, 1, "input device"}, {PCIC_INPUTDEV, PCIS_INPUTDEV_KEYBOARD, 1, "keyboard"}, {PCIC_INPUTDEV, PCIS_INPUTDEV_DIGITIZER,1, "digitizer"}, {PCIC_INPUTDEV, PCIS_INPUTDEV_MOUSE, 1, "mouse"}, {PCIC_INPUTDEV, PCIS_INPUTDEV_SCANNER, 1, "scanner"}, {PCIC_INPUTDEV, PCIS_INPUTDEV_GAMEPORT, 1, "gameport"}, {PCIC_DOCKING, -1, 1, "docking station"}, {PCIC_PROCESSOR, -1, 1, "processor"}, {PCIC_SERIALBUS, -1, 1, "serial bus"}, {PCIC_SERIALBUS, PCIS_SERIALBUS_FW, 1, "FireWire"}, {PCIC_SERIALBUS, PCIS_SERIALBUS_ACCESS, 1, "AccessBus"}, {PCIC_SERIALBUS, PCIS_SERIALBUS_SSA, 1, "SSA"}, {PCIC_SERIALBUS, PCIS_SERIALBUS_USB, 1, "USB"}, {PCIC_SERIALBUS, PCIS_SERIALBUS_FC, 1, "Fibre Channel"}, {PCIC_SERIALBUS, PCIS_SERIALBUS_SMBUS, 0, "SMBus"}, {PCIC_WIRELESS, -1, 1, "wireless controller"}, {PCIC_WIRELESS, PCIS_WIRELESS_IRDA, 1, "iRDA"}, {PCIC_WIRELESS, PCIS_WIRELESS_IR, 1, "IR"}, {PCIC_WIRELESS, PCIS_WIRELESS_RF, 1, "RF"}, {PCIC_INTELLIIO, -1, 1, "intelligent I/O controller"}, {PCIC_INTELLIIO, PCIS_INTELLIIO_I2O, 1, "I2O"}, {PCIC_SATCOM, -1, 1, "satellite communication"}, {PCIC_SATCOM, PCIS_SATCOM_TV, 1, "sat TV"}, {PCIC_SATCOM, PCIS_SATCOM_AUDIO, 1, "sat audio"}, {PCIC_SATCOM, PCIS_SATCOM_VOICE, 1, "sat voice"}, {PCIC_SATCOM, PCIS_SATCOM_DATA, 1, "sat data"}, {PCIC_CRYPTO, -1, 1, "encrypt/decrypt"}, {PCIC_CRYPTO, PCIS_CRYPTO_NETCOMP, 1, "network/computer crypto"}, {PCIC_CRYPTO, PCIS_CRYPTO_ENTERTAIN, 1, "entertainment crypto"}, {PCIC_DASP, -1, 0, "dasp"}, {PCIC_DASP, PCIS_DASP_DPIO, 1, "DPIO module"}, {PCIC_DASP, PCIS_DASP_PERFCNTRS, 1, "performance counters"}, {PCIC_DASP, PCIS_DASP_COMM_SYNC, 1, "communication synchronizer"}, {PCIC_DASP, PCIS_DASP_MGMT_CARD, 1, "signal processing management"}, {PCIC_INSTRUMENT, -1, 0, "non-essential instrumentation"}, {0, 0, 0, NULL} }; void pci_probe_nomatch(device_t dev, device_t child) { int i, report; const char *cp, *scp; char *device; /* * Look for a listing for this device in a loaded device database. */ report = 1; if ((device = pci_describe_device(child)) != NULL) { device_printf(dev, "<%s>", device); free(device, M_DEVBUF); } else { /* * Scan the class/subclass descriptions for a general * description. */ cp = "unknown"; scp = NULL; for (i = 0; pci_nomatch_tab[i].desc != NULL; i++) { if (pci_nomatch_tab[i].class == pci_get_class(child)) { if (pci_nomatch_tab[i].subclass == -1) { cp = pci_nomatch_tab[i].desc; report = pci_nomatch_tab[i].report; } else if (pci_nomatch_tab[i].subclass == pci_get_subclass(child)) { scp = pci_nomatch_tab[i].desc; report = pci_nomatch_tab[i].report; } } } if (report || bootverbose) { device_printf(dev, "<%s%s%s>", cp ? cp : "", ((cp != NULL) && (scp != NULL)) ? ", " : "", scp ? scp : ""); } } if (report || bootverbose) { printf(" at device %d.%d (no driver attached)\n", pci_get_slot(child), pci_get_function(child)); } pci_cfg_save(child, device_get_ivars(child), 1); } void pci_child_detached(device_t dev, device_t child) { struct pci_devinfo *dinfo; struct resource_list *rl; dinfo = device_get_ivars(child); rl = &dinfo->resources; /* * Have to deallocate IRQs before releasing any MSI messages and * have to release MSI messages before deallocating any memory * BARs. */ if (resource_list_release_active(rl, dev, child, SYS_RES_IRQ) != 0) pci_printf(&dinfo->cfg, "Device leaked IRQ resources\n"); if (dinfo->cfg.msi.msi_alloc != 0 || dinfo->cfg.msix.msix_alloc != 0) { if (dinfo->cfg.msi.msi_alloc != 0) pci_printf(&dinfo->cfg, "Device leaked %d MSI " "vectors\n", dinfo->cfg.msi.msi_alloc); else pci_printf(&dinfo->cfg, "Device leaked %d MSI-X " "vectors\n", dinfo->cfg.msix.msix_alloc); (void)pci_release_msi(child); } if (resource_list_release_active(rl, dev, child, SYS_RES_MEMORY) != 0) pci_printf(&dinfo->cfg, "Device leaked memory resources\n"); if (resource_list_release_active(rl, dev, child, SYS_RES_IOPORT) != 0) pci_printf(&dinfo->cfg, "Device leaked I/O resources\n"); #ifdef PCI_RES_BUS if (resource_list_release_active(rl, dev, child, PCI_RES_BUS) != 0) pci_printf(&dinfo->cfg, "Device leaked PCI bus numbers\n"); #endif pci_cfg_save(child, dinfo, 1); } /* * Parse the PCI device database, if loaded, and return a pointer to a * description of the device. * * The database is flat text formatted as follows: * * Any line not in a valid format is ignored. * Lines are terminated with newline '\n' characters. * * A VENDOR line consists of the 4 digit (hex) vendor code, a TAB, then * the vendor name. * * A DEVICE line is entered immediately below the corresponding VENDOR ID. * - devices cannot be listed without a corresponding VENDOR line. * A DEVICE line consists of a TAB, the 4 digit (hex) device code, * another TAB, then the device name. */ /* * Assuming (ptr) points to the beginning of a line in the database, * return the vendor or device and description of the next entry. * The value of (vendor) or (device) inappropriate for the entry type * is set to -1. Returns nonzero at the end of the database. * * Note that this is slightly unrobust in the face of corrupt data; * we attempt to safeguard against this by spamming the end of the * database with a newline when we initialise. */ static int pci_describe_parse_line(char **ptr, int *vendor, int *device, char **desc) { char *cp = *ptr; int left; *device = -1; *vendor = -1; **desc = '\0'; for (;;) { left = pci_vendordata_size - (cp - pci_vendordata); if (left <= 0) { *ptr = cp; return(1); } /* vendor entry? */ if (*cp != '\t' && sscanf(cp, "%x\t%80[^\n]", vendor, *desc) == 2) break; /* device entry? */ if (*cp == '\t' && sscanf(cp, "%x\t%80[^\n]", device, *desc) == 2) break; /* skip to next line */ while (*cp != '\n' && left > 0) { cp++; left--; } if (*cp == '\n') { cp++; left--; } } /* skip to next line */ while (*cp != '\n' && left > 0) { cp++; left--; } if (*cp == '\n' && left > 0) cp++; *ptr = cp; return(0); } static char * pci_describe_device(device_t dev) { int vendor, device; char *desc, *vp, *dp, *line; desc = vp = dp = NULL; /* * If we have no vendor data, we can't do anything. */ if (pci_vendordata == NULL) goto out; /* * Scan the vendor data looking for this device */ line = pci_vendordata; if ((vp = malloc(80, M_DEVBUF, M_NOWAIT)) == NULL) goto out; for (;;) { if (pci_describe_parse_line(&line, &vendor, &device, &vp)) goto out; if (vendor == pci_get_vendor(dev)) break; } if ((dp = malloc(80, M_DEVBUF, M_NOWAIT)) == NULL) goto out; for (;;) { if (pci_describe_parse_line(&line, &vendor, &device, &dp)) { *dp = 0; break; } if (vendor != -1) { *dp = 0; break; } if (device == pci_get_device(dev)) break; } if (dp[0] == '\0') snprintf(dp, 80, "0x%x", pci_get_device(dev)); if ((desc = malloc(strlen(vp) + strlen(dp) + 3, M_DEVBUF, M_NOWAIT)) != NULL) sprintf(desc, "%s, %s", vp, dp); out: if (vp != NULL) free(vp, M_DEVBUF); if (dp != NULL) free(dp, M_DEVBUF); return(desc); } int pci_read_ivar(device_t dev, device_t child, int which, uintptr_t *result) { struct pci_devinfo *dinfo; pcicfgregs *cfg; dinfo = device_get_ivars(child); cfg = &dinfo->cfg; switch (which) { case PCI_IVAR_ETHADDR: /* * The generic accessor doesn't deal with failure, so * we set the return value, then return an error. */ *((uint8_t **) result) = NULL; return (EINVAL); case PCI_IVAR_SUBVENDOR: *result = cfg->subvendor; break; case PCI_IVAR_SUBDEVICE: *result = cfg->subdevice; break; case PCI_IVAR_VENDOR: *result = cfg->vendor; break; case PCI_IVAR_DEVICE: *result = cfg->device; break; case PCI_IVAR_DEVID: *result = (cfg->device << 16) | cfg->vendor; break; case PCI_IVAR_CLASS: *result = cfg->baseclass; break; case PCI_IVAR_SUBCLASS: *result = cfg->subclass; break; case PCI_IVAR_PROGIF: *result = cfg->progif; break; case PCI_IVAR_REVID: *result = cfg->revid; break; case PCI_IVAR_INTPIN: *result = cfg->intpin; break; case PCI_IVAR_IRQ: *result = cfg->intline; break; case PCI_IVAR_DOMAIN: *result = cfg->domain; break; case PCI_IVAR_BUS: *result = cfg->bus; break; case PCI_IVAR_SLOT: *result = cfg->slot; break; case PCI_IVAR_FUNCTION: *result = cfg->func; break; case PCI_IVAR_CMDREG: *result = cfg->cmdreg; break; case PCI_IVAR_CACHELNSZ: *result = cfg->cachelnsz; break; case PCI_IVAR_MINGNT: if (cfg->hdrtype != PCIM_HDRTYPE_NORMAL) { *result = -1; return (EINVAL); } *result = cfg->mingnt; break; case PCI_IVAR_MAXLAT: if (cfg->hdrtype != PCIM_HDRTYPE_NORMAL) { *result = -1; return (EINVAL); } *result = cfg->maxlat; break; case PCI_IVAR_LATTIMER: *result = cfg->lattimer; break; default: return (ENOENT); } return (0); } int pci_write_ivar(device_t dev, device_t child, int which, uintptr_t value) { struct pci_devinfo *dinfo; dinfo = device_get_ivars(child); switch (which) { case PCI_IVAR_INTPIN: dinfo->cfg.intpin = value; return (0); case PCI_IVAR_ETHADDR: case PCI_IVAR_SUBVENDOR: case PCI_IVAR_SUBDEVICE: case PCI_IVAR_VENDOR: case PCI_IVAR_DEVICE: case PCI_IVAR_DEVID: case PCI_IVAR_CLASS: case PCI_IVAR_SUBCLASS: case PCI_IVAR_PROGIF: case PCI_IVAR_REVID: case PCI_IVAR_IRQ: case PCI_IVAR_DOMAIN: case PCI_IVAR_BUS: case PCI_IVAR_SLOT: case PCI_IVAR_FUNCTION: return (EINVAL); /* disallow for now */ default: return (ENOENT); } } #include "opt_ddb.h" #ifdef DDB #include #include /* * List resources based on pci map registers, used for within ddb */ DB_SHOW_COMMAND_FLAGS(pciregs, db_pci_dump, DB_CMD_MEMSAFE) { struct pci_devinfo *dinfo; struct devlist *devlist_head; struct pci_conf *p; const char *name; int i, error, none_count; none_count = 0; /* get the head of the device queue */ devlist_head = &pci_devq; /* * Go through the list of devices and print out devices */ for (error = 0, i = 0, dinfo = STAILQ_FIRST(devlist_head); (dinfo != NULL) && (error == 0) && (i < pci_numdevs) && !db_pager_quit; dinfo = STAILQ_NEXT(dinfo, pci_links), i++) { /* Populate pd_name and pd_unit */ name = NULL; if (dinfo->cfg.dev) name = device_get_name(dinfo->cfg.dev); p = &dinfo->conf; db_printf("%s%d@pci%d:%d:%d:%d:\tclass=0x%06x card=0x%08x " "chip=0x%08x rev=0x%02x hdr=0x%02x\n", (name && *name) ? name : "none", (name && *name) ? (int)device_get_unit(dinfo->cfg.dev) : none_count++, p->pc_sel.pc_domain, p->pc_sel.pc_bus, p->pc_sel.pc_dev, p->pc_sel.pc_func, (p->pc_class << 16) | (p->pc_subclass << 8) | p->pc_progif, (p->pc_subdevice << 16) | p->pc_subvendor, (p->pc_device << 16) | p->pc_vendor, p->pc_revid, p->pc_hdr); } } #endif /* DDB */ struct resource * pci_reserve_map(device_t dev, device_t child, int type, int *rid, rman_res_t start, rman_res_t end, rman_res_t count, u_int num, u_int flags) { struct pci_devinfo *dinfo = device_get_ivars(child); struct resource_list *rl = &dinfo->resources; struct resource *res; struct pci_map *pm; uint16_t cmd; pci_addr_t map, testval; int mapsize; res = NULL; /* If rid is managed by EA, ignore it */ if (pci_ea_is_enabled(child, *rid)) goto out; pm = pci_find_bar(child, *rid); if (pm != NULL) { /* This is a BAR that we failed to allocate earlier. */ mapsize = pm->pm_size; map = pm->pm_value; } else { /* * Weed out the bogons, and figure out how large the * BAR/map is. BARs that read back 0 here are bogus * and unimplemented. Note: atapci in legacy mode are * special and handled elsewhere in the code. If you * have a atapci device in legacy mode and it fails * here, that other code is broken. */ pci_read_bar(child, *rid, &map, &testval, NULL); /* * Determine the size of the BAR and ignore BARs with a size * of 0. Device ROM BARs use a different mask value. */ if (PCIR_IS_BIOS(&dinfo->cfg, *rid)) mapsize = pci_romsize(testval); else mapsize = pci_mapsize(testval); if (mapsize == 0) goto out; pm = pci_add_bar(child, *rid, map, mapsize); } if (PCI_BAR_MEM(map) || PCIR_IS_BIOS(&dinfo->cfg, *rid)) { if (type != SYS_RES_MEMORY) { if (bootverbose) device_printf(dev, "child %s requested type %d for rid %#x," " but the BAR says it is an memio\n", device_get_nameunit(child), type, *rid); goto out; } } else { if (type != SYS_RES_IOPORT) { if (bootverbose) device_printf(dev, "child %s requested type %d for rid %#x," " but the BAR says it is an ioport\n", device_get_nameunit(child), type, *rid); goto out; } } /* * For real BARs, we need to override the size that * the driver requests, because that's what the BAR * actually uses and we would otherwise have a * situation where we might allocate the excess to * another driver, which won't work. */ count = ((pci_addr_t)1 << mapsize) * num; if (RF_ALIGNMENT(flags) < mapsize) flags = (flags & ~RF_ALIGNMENT_MASK) | RF_ALIGNMENT_LOG2(mapsize); if (PCI_BAR_MEM(map) && (map & PCIM_BAR_MEM_PREFETCH)) flags |= RF_PREFETCHABLE; /* * Allocate enough resource, and then write back the * appropriate BAR for that resource. */ resource_list_add(rl, type, *rid, start, end, count); res = resource_list_reserve(rl, dev, child, type, rid, start, end, count, flags & ~RF_ACTIVE); if (res == NULL) { resource_list_delete(rl, type, *rid); device_printf(child, "%#jx bytes of rid %#x res %d failed (%#jx, %#jx).\n", count, *rid, type, start, end); goto out; } if (bootverbose) device_printf(child, "Lazy allocation of %#jx bytes rid %#x type %d at %#jx\n", count, *rid, type, rman_get_start(res)); /* Disable decoding via the CMD register before updating the BAR */ cmd = pci_read_config(child, PCIR_COMMAND, 2); pci_write_config(child, PCIR_COMMAND, cmd & ~(PCI_BAR_MEM(map) ? PCIM_CMD_MEMEN : PCIM_CMD_PORTEN), 2); map = rman_get_start(res); pci_write_bar(child, pm, map); /* Restore the original value of the CMD register */ pci_write_config(child, PCIR_COMMAND, cmd, 2); out: return (res); } struct resource * pci_alloc_multi_resource(device_t dev, device_t child, int type, int *rid, rman_res_t start, rman_res_t end, rman_res_t count, u_long num, u_int flags) { struct pci_devinfo *dinfo; struct resource_list *rl; struct resource_list_entry *rle; struct resource *res; pcicfgregs *cfg; /* * Perform lazy resource allocation */ dinfo = device_get_ivars(child); rl = &dinfo->resources; cfg = &dinfo->cfg; switch (type) { #if defined(NEW_PCIB) && defined(PCI_RES_BUS) case PCI_RES_BUS: return (pci_alloc_secbus(dev, child, rid, start, end, count, flags)); #endif case SYS_RES_IRQ: /* * Can't alloc legacy interrupt once MSI messages have * been allocated. */ if (*rid == 0 && (cfg->msi.msi_alloc > 0 || cfg->msix.msix_alloc > 0)) return (NULL); /* * If the child device doesn't have an interrupt * routed and is deserving of an interrupt, try to * assign it one. */ if (*rid == 0 && !PCI_INTERRUPT_VALID(cfg->intline) && (cfg->intpin != 0)) pci_assign_interrupt(dev, child, 0); break; case SYS_RES_IOPORT: case SYS_RES_MEMORY: #ifdef NEW_PCIB /* * PCI-PCI bridge I/O window resources are not BARs. * For those allocations just pass the request up the * tree. */ if (cfg->hdrtype == PCIM_HDRTYPE_BRIDGE) { switch (*rid) { case PCIR_IOBASEL_1: case PCIR_MEMBASE_1: case PCIR_PMBASEL_1: /* * XXX: Should we bother creating a resource * list entry? */ return (bus_generic_alloc_resource(dev, child, type, rid, start, end, count, flags)); } } #endif /* Reserve resources for this BAR if needed. */ rle = resource_list_find(rl, type, *rid); if (rle == NULL) { res = pci_reserve_map(dev, child, type, rid, start, end, count, num, flags); if (res == NULL) return (NULL); } } return (resource_list_alloc(rl, dev, child, type, rid, start, end, count, flags)); } struct resource * pci_alloc_resource(device_t dev, device_t child, int type, int *rid, rman_res_t start, rman_res_t end, rman_res_t count, u_int flags) { #ifdef PCI_IOV struct pci_devinfo *dinfo; #endif if (device_get_parent(child) != dev) return (BUS_ALLOC_RESOURCE(device_get_parent(dev), child, type, rid, start, end, count, flags)); #ifdef PCI_IOV dinfo = device_get_ivars(child); if (dinfo->cfg.flags & PCICFG_VF) { switch (type) { /* VFs can't have I/O BARs. */ case SYS_RES_IOPORT: return (NULL); case SYS_RES_MEMORY: return (pci_vf_alloc_mem_resource(dev, child, rid, start, end, count, flags)); } /* Fall through for other types of resource allocations. */ } #endif return (pci_alloc_multi_resource(dev, child, type, rid, start, end, count, 1, flags)); } int pci_release_resource(device_t dev, device_t child, int type, int rid, struct resource *r) { struct pci_devinfo *dinfo; struct resource_list *rl; pcicfgregs *cfg __unused; if (device_get_parent(child) != dev) return (BUS_RELEASE_RESOURCE(device_get_parent(dev), child, type, rid, r)); dinfo = device_get_ivars(child); cfg = &dinfo->cfg; #ifdef PCI_IOV if (cfg->flags & PCICFG_VF) { switch (type) { /* VFs can't have I/O BARs. */ case SYS_RES_IOPORT: return (EDOOFUS); case SYS_RES_MEMORY: return (pci_vf_release_mem_resource(dev, child, rid, r)); } /* Fall through for other types of resource allocations. */ } #endif #ifdef NEW_PCIB /* * PCI-PCI bridge I/O window resources are not BARs. For * those allocations just pass the request up the tree. */ if (cfg->hdrtype == PCIM_HDRTYPE_BRIDGE && (type == SYS_RES_IOPORT || type == SYS_RES_MEMORY)) { switch (rid) { case PCIR_IOBASEL_1: case PCIR_MEMBASE_1: case PCIR_PMBASEL_1: return (bus_generic_release_resource(dev, child, type, rid, r)); } } #endif rl = &dinfo->resources; return (resource_list_release(rl, dev, child, type, rid, r)); } int pci_activate_resource(device_t dev, device_t child, int type, int rid, struct resource *r) { struct pci_devinfo *dinfo; int error; if (device_get_parent(child) != dev) return (bus_generic_activate_resource(dev, child, type, rid, r)); dinfo = device_get_ivars(child); #ifdef PCI_IOV if (dinfo->cfg.flags & PCICFG_VF) { switch (rman_get_type(r)) { /* VFs can't have I/O BARs. */ case SYS_RES_IOPORT: error = EINVAL; break; case SYS_RES_MEMORY: error = pci_vf_activate_mem_resource(dev, child, r); break; default: error = bus_generic_activate_resource(dev, child, type, rid, r); break; } } else #endif error = bus_generic_activate_resource(dev, child, type, rid, r); if (error) return (error); /* Device ROMs need their decoding explicitly enabled. */ if (type == SYS_RES_MEMORY && PCIR_IS_BIOS(&dinfo->cfg, rid)) pci_write_bar(child, pci_find_bar(child, rid), rman_get_start(r) | PCIM_BIOS_ENABLE); /* Enable decoding in the command register when activating BARs. */ switch (type) { case SYS_RES_IOPORT: case SYS_RES_MEMORY: error = PCI_ENABLE_IO(dev, child, type); break; } return (error); } int pci_deactivate_resource(device_t dev, device_t child, int type, int rid, struct resource *r) { struct pci_devinfo *dinfo; int error; if (device_get_parent(child) != dev) return (bus_generic_deactivate_resource(dev, child, type, rid, r)); dinfo = device_get_ivars(child); #ifdef PCI_IOV if (dinfo->cfg.flags & PCICFG_VF) { switch (rman_get_type(r)) { /* VFs can't have I/O BARs. */ case SYS_RES_IOPORT: error = EINVAL; break; case SYS_RES_MEMORY: error = pci_vf_deactivate_mem_resource(dev, child, r); break; default: error = bus_generic_deactivate_resource(dev, child, type, rid, r); break; } } else #endif error = bus_generic_deactivate_resource(dev, child, type, rid, r); if (error) return (error); /* Disable decoding for device ROMs. */ if (type == SYS_RES_MEMORY && PCIR_IS_BIOS(&dinfo->cfg, rid)) pci_write_bar(child, pci_find_bar(child, rid), rman_get_start(r)); return (0); } int pci_adjust_resource(device_t dev, device_t child, int type, struct resource *r, rman_res_t start, rman_res_t end) { #ifdef PCI_IOV struct pci_devinfo *dinfo; if (device_get_parent(child) != dev) return (bus_generic_adjust_resource(dev, child, type, r, start, end)); dinfo = device_get_ivars(child); if (dinfo->cfg.flags & PCICFG_VF) { switch (rman_get_type(r)) { /* VFs can't have I/O BARs. */ case SYS_RES_IOPORT: return (EINVAL); case SYS_RES_MEMORY: return (pci_vf_adjust_mem_resource(dev, child, r, start, end)); } /* Fall through for other types of resource allocations. */ } #endif return (bus_generic_adjust_resource(dev, child, type, r, start, end)); } int pci_map_resource(device_t dev, device_t child, int type, struct resource *r, struct resource_map_request *argsp, struct resource_map *map) { #ifdef PCI_IOV struct pci_devinfo *dinfo; if (device_get_parent(child) != dev) return (bus_generic_map_resource(dev, child, type, r, argsp, map)); dinfo = device_get_ivars(child); if (dinfo->cfg.flags & PCICFG_VF) { switch (rman_get_type(r)) { /* VFs can't have I/O BARs. */ case SYS_RES_IOPORT: return (EINVAL); case SYS_RES_MEMORY: return (pci_vf_map_mem_resource(dev, child, r, argsp, map)); } /* Fall through for other types of resource allocations. */ } #endif return (bus_generic_map_resource(dev, child, type, r, argsp, map)); } int pci_unmap_resource(device_t dev, device_t child, int type, struct resource *r, struct resource_map *map) { #ifdef PCI_IOV struct pci_devinfo *dinfo; if (device_get_parent(child) != dev) return (bus_generic_unmap_resource(dev, child, type, r, map)); dinfo = device_get_ivars(child); if (dinfo->cfg.flags & PCICFG_VF) { switch (rman_get_type(r)) { /* VFs can't have I/O BARs. */ case SYS_RES_IOPORT: return (EINVAL); case SYS_RES_MEMORY: return (pci_vf_unmap_mem_resource(dev, child, r, map)); } /* Fall through for other types of resource allocations. */ } #endif return (bus_generic_unmap_resource(dev, child, type, r, map)); } void pci_child_deleted(device_t dev, device_t child) { struct resource_list_entry *rle; struct resource_list *rl; struct pci_devinfo *dinfo; dinfo = device_get_ivars(child); rl = &dinfo->resources; EVENTHANDLER_INVOKE(pci_delete_device, child); /* Turn off access to resources we're about to free */ if (bus_child_present(child) != 0) { pci_write_config(child, PCIR_COMMAND, pci_read_config(child, PCIR_COMMAND, 2) & ~(PCIM_CMD_MEMEN | PCIM_CMD_PORTEN), 2); pci_disable_busmaster(child); } /* Free all allocated resources */ STAILQ_FOREACH(rle, rl, link) { if (rle->res) { if (rman_get_flags(rle->res) & RF_ACTIVE || resource_list_busy(rl, rle->type, rle->rid)) { pci_printf(&dinfo->cfg, "Resource still owned, oops. " "(type=%d, rid=%d, addr=%lx)\n", rle->type, rle->rid, rman_get_start(rle->res)); bus_release_resource(child, rle->type, rle->rid, rle->res); } resource_list_unreserve(rl, dev, child, rle->type, rle->rid); } } resource_list_free(rl); pci_freecfg(dinfo); } void pci_delete_resource(device_t dev, device_t child, int type, int rid) { struct pci_devinfo *dinfo; struct resource_list *rl; struct resource_list_entry *rle; if (device_get_parent(child) != dev) return; dinfo = device_get_ivars(child); rl = &dinfo->resources; rle = resource_list_find(rl, type, rid); if (rle == NULL) return; if (rle->res) { if (rman_get_flags(rle->res) & RF_ACTIVE || resource_list_busy(rl, type, rid)) { device_printf(dev, "delete_resource: " "Resource still owned by child, oops. " "(type=%d, rid=%d, addr=%jx)\n", type, rid, rman_get_start(rle->res)); return; } resource_list_unreserve(rl, dev, child, type, rid); } resource_list_delete(rl, type, rid); } struct resource_list * pci_get_resource_list (device_t dev, device_t child) { struct pci_devinfo *dinfo = device_get_ivars(child); return (&dinfo->resources); } #ifdef IOMMU bus_dma_tag_t pci_get_dma_tag(device_t bus, device_t dev) { bus_dma_tag_t tag; struct pci_softc *sc; if (device_get_parent(dev) == bus) { /* try iommu and return if it works */ tag = iommu_get_dma_tag(bus, dev); } else tag = NULL; if (tag == NULL) { sc = device_get_softc(bus); tag = sc->sc_dma_tag; } return (tag); } #else bus_dma_tag_t pci_get_dma_tag(device_t bus, device_t dev) { struct pci_softc *sc = device_get_softc(bus); return (sc->sc_dma_tag); } #endif uint32_t pci_read_config_method(device_t dev, device_t child, int reg, int width) { struct pci_devinfo *dinfo = device_get_ivars(child); pcicfgregs *cfg = &dinfo->cfg; #ifdef PCI_IOV /* * SR-IOV VFs don't implement the VID or DID registers, so we have to * emulate them here. */ if (cfg->flags & PCICFG_VF) { if (reg == PCIR_VENDOR) { switch (width) { case 4: return (cfg->device << 16 | cfg->vendor); case 2: return (cfg->vendor); case 1: return (cfg->vendor & 0xff); default: return (0xffffffff); } } else if (reg == PCIR_DEVICE) { switch (width) { /* Note that an unaligned 4-byte read is an error. */ case 2: return (cfg->device); case 1: return (cfg->device & 0xff); default: return (0xffffffff); } } } #endif return (PCIB_READ_CONFIG(device_get_parent(dev), cfg->bus, cfg->slot, cfg->func, reg, width)); } void pci_write_config_method(device_t dev, device_t child, int reg, uint32_t val, int width) { struct pci_devinfo *dinfo = device_get_ivars(child); pcicfgregs *cfg = &dinfo->cfg; PCIB_WRITE_CONFIG(device_get_parent(dev), cfg->bus, cfg->slot, cfg->func, reg, val, width); } int pci_child_location_method(device_t dev, device_t child, struct sbuf *sb) { sbuf_printf(sb, "slot=%d function=%d dbsf=pci%d:%d:%d:%d", pci_get_slot(child), pci_get_function(child), pci_get_domain(child), pci_get_bus(child), pci_get_slot(child), pci_get_function(child)); return (0); } int pci_child_pnpinfo_method(device_t dev, device_t child, struct sbuf *sb) { struct pci_devinfo *dinfo; pcicfgregs *cfg; dinfo = device_get_ivars(child); cfg = &dinfo->cfg; sbuf_printf(sb, "vendor=0x%04x device=0x%04x subvendor=0x%04x " "subdevice=0x%04x class=0x%02x%02x%02x", cfg->vendor, cfg->device, cfg->subvendor, cfg->subdevice, cfg->baseclass, cfg->subclass, cfg->progif); return (0); } int pci_get_device_path_method(device_t bus, device_t child, const char *locator, struct sbuf *sb) { device_t parent = device_get_parent(bus); int rv; if (strcmp(locator, BUS_LOCATOR_UEFI) == 0) { rv = bus_generic_get_device_path(parent, bus, locator, sb); if (rv == 0) { sbuf_printf(sb, "/Pci(0x%x,0x%x)", pci_get_slot(child), pci_get_function(child)); } return (0); } return (bus_generic_get_device_path(bus, child, locator, sb)); } int pci_assign_interrupt_method(device_t dev, device_t child) { struct pci_devinfo *dinfo = device_get_ivars(child); pcicfgregs *cfg = &dinfo->cfg; return (PCIB_ROUTE_INTERRUPT(device_get_parent(dev), child, cfg->intpin)); } static void pci_lookup(void *arg, const char *name, device_t *dev) { long val; char *end; int domain, bus, slot, func; if (*dev != NULL) return; /* * Accept pciconf-style selectors of either pciD:B:S:F or * pciB:S:F. In the latter case, the domain is assumed to * be zero. */ if (strncmp(name, "pci", 3) != 0) return; val = strtol(name + 3, &end, 10); if (val < 0 || val > INT_MAX || *end != ':') return; domain = val; val = strtol(end + 1, &end, 10); if (val < 0 || val > INT_MAX || *end != ':') return; bus = val; val = strtol(end + 1, &end, 10); if (val < 0 || val > INT_MAX) return; slot = val; if (*end == ':') { val = strtol(end + 1, &end, 10); if (val < 0 || val > INT_MAX || *end != '\0') return; func = val; } else if (*end == '\0') { func = slot; slot = bus; bus = domain; domain = 0; } else return; if (domain > PCI_DOMAINMAX || bus > PCI_BUSMAX || slot > PCI_SLOTMAX || func > PCIE_ARI_FUNCMAX || (slot != 0 && func > PCI_FUNCMAX)) return; *dev = pci_find_dbsf(domain, bus, slot, func); } static int pci_modevent(module_t mod, int what, void *arg) { static struct cdev *pci_cdev; static eventhandler_tag tag; switch (what) { case MOD_LOAD: STAILQ_INIT(&pci_devq); pci_generation = 0; pci_cdev = make_dev(&pcicdev, 0, UID_ROOT, GID_WHEEL, 0644, "pci"); pci_load_vendor_data(); tag = EVENTHANDLER_REGISTER(dev_lookup, pci_lookup, NULL, 1000); break; case MOD_UNLOAD: if (tag != NULL) EVENTHANDLER_DEREGISTER(dev_lookup, tag); destroy_dev(pci_cdev); break; } return (0); } static void pci_cfg_restore_pcie(device_t dev, struct pci_devinfo *dinfo) { #define WREG(n, v) pci_write_config(dev, pos + (n), (v), 2) struct pcicfg_pcie *cfg; int version, pos; cfg = &dinfo->cfg.pcie; pos = cfg->pcie_location; version = cfg->pcie_flags & PCIEM_FLAGS_VERSION; WREG(PCIER_DEVICE_CTL, cfg->pcie_device_ctl); if (version > 1 || cfg->pcie_type == PCIEM_TYPE_ROOT_PORT || cfg->pcie_type == PCIEM_TYPE_ENDPOINT || cfg->pcie_type == PCIEM_TYPE_LEGACY_ENDPOINT) WREG(PCIER_LINK_CTL, cfg->pcie_link_ctl); if (version > 1 || (cfg->pcie_type == PCIEM_TYPE_ROOT_PORT || (cfg->pcie_type == PCIEM_TYPE_DOWNSTREAM_PORT && (cfg->pcie_flags & PCIEM_FLAGS_SLOT)))) WREG(PCIER_SLOT_CTL, cfg->pcie_slot_ctl); if (version > 1 || cfg->pcie_type == PCIEM_TYPE_ROOT_PORT || cfg->pcie_type == PCIEM_TYPE_ROOT_EC) WREG(PCIER_ROOT_CTL, cfg->pcie_root_ctl); if (version > 1) { WREG(PCIER_DEVICE_CTL2, cfg->pcie_device_ctl2); WREG(PCIER_LINK_CTL2, cfg->pcie_link_ctl2); WREG(PCIER_SLOT_CTL2, cfg->pcie_slot_ctl2); } #undef WREG } static void pci_cfg_restore_pcix(device_t dev, struct pci_devinfo *dinfo) { pci_write_config(dev, dinfo->cfg.pcix.pcix_location + PCIXR_COMMAND, dinfo->cfg.pcix.pcix_command, 2); } void pci_cfg_restore(device_t dev, struct pci_devinfo *dinfo) { /* * Restore the device to full power mode. We must do this * before we restore the registers because moving from D3 to * D0 will cause the chip's BARs and some other registers to * be reset to some unknown power on reset values. Cut down * the noise on boot by doing nothing if we are already in * state D0. */ if (pci_get_powerstate(dev) != PCI_POWERSTATE_D0) pci_set_powerstate(dev, PCI_POWERSTATE_D0); pci_write_config(dev, PCIR_INTLINE, dinfo->cfg.intline, 1); pci_write_config(dev, PCIR_INTPIN, dinfo->cfg.intpin, 1); pci_write_config(dev, PCIR_CACHELNSZ, dinfo->cfg.cachelnsz, 1); pci_write_config(dev, PCIR_LATTIMER, dinfo->cfg.lattimer, 1); pci_write_config(dev, PCIR_PROGIF, dinfo->cfg.progif, 1); pci_write_config(dev, PCIR_REVID, dinfo->cfg.revid, 1); switch (dinfo->cfg.hdrtype & PCIM_HDRTYPE) { case PCIM_HDRTYPE_NORMAL: pci_write_config(dev, PCIR_MINGNT, dinfo->cfg.mingnt, 1); pci_write_config(dev, PCIR_MAXLAT, dinfo->cfg.maxlat, 1); break; case PCIM_HDRTYPE_BRIDGE: pci_write_config(dev, PCIR_SECLAT_1, dinfo->cfg.bridge.br_seclat, 1); pci_write_config(dev, PCIR_SUBBUS_1, dinfo->cfg.bridge.br_subbus, 1); pci_write_config(dev, PCIR_SECBUS_1, dinfo->cfg.bridge.br_secbus, 1); pci_write_config(dev, PCIR_PRIBUS_1, dinfo->cfg.bridge.br_pribus, 1); pci_write_config(dev, PCIR_BRIDGECTL_1, dinfo->cfg.bridge.br_control, 2); break; case PCIM_HDRTYPE_CARDBUS: pci_write_config(dev, PCIR_SECLAT_2, dinfo->cfg.bridge.br_seclat, 1); pci_write_config(dev, PCIR_SUBBUS_2, dinfo->cfg.bridge.br_subbus, 1); pci_write_config(dev, PCIR_SECBUS_2, dinfo->cfg.bridge.br_secbus, 1); pci_write_config(dev, PCIR_PRIBUS_2, dinfo->cfg.bridge.br_pribus, 1); pci_write_config(dev, PCIR_BRIDGECTL_2, dinfo->cfg.bridge.br_control, 2); break; } pci_restore_bars(dev); if ((dinfo->cfg.hdrtype & PCIM_HDRTYPE) != PCIM_HDRTYPE_BRIDGE) pci_write_config(dev, PCIR_COMMAND, dinfo->cfg.cmdreg, 2); /* * Restore extended capabilities for PCI-Express and PCI-X */ if (dinfo->cfg.pcie.pcie_location != 0) pci_cfg_restore_pcie(dev, dinfo); if (dinfo->cfg.pcix.pcix_location != 0) pci_cfg_restore_pcix(dev, dinfo); /* Restore MSI and MSI-X configurations if they are present. */ if (dinfo->cfg.msi.msi_location != 0) pci_resume_msi(dev); if (dinfo->cfg.msix.msix_location != 0) pci_resume_msix(dev); #ifdef PCI_IOV if (dinfo->cfg.iov != NULL) pci_iov_cfg_restore(dev, dinfo); #endif } static void pci_cfg_save_pcie(device_t dev, struct pci_devinfo *dinfo) { #define RREG(n) pci_read_config(dev, pos + (n), 2) struct pcicfg_pcie *cfg; int version, pos; cfg = &dinfo->cfg.pcie; pos = cfg->pcie_location; cfg->pcie_flags = RREG(PCIER_FLAGS); version = cfg->pcie_flags & PCIEM_FLAGS_VERSION; cfg->pcie_device_ctl = RREG(PCIER_DEVICE_CTL); if (version > 1 || cfg->pcie_type == PCIEM_TYPE_ROOT_PORT || cfg->pcie_type == PCIEM_TYPE_ENDPOINT || cfg->pcie_type == PCIEM_TYPE_LEGACY_ENDPOINT) cfg->pcie_link_ctl = RREG(PCIER_LINK_CTL); if (version > 1 || (cfg->pcie_type == PCIEM_TYPE_ROOT_PORT || (cfg->pcie_type == PCIEM_TYPE_DOWNSTREAM_PORT && (cfg->pcie_flags & PCIEM_FLAGS_SLOT)))) cfg->pcie_slot_ctl = RREG(PCIER_SLOT_CTL); if (version > 1 || cfg->pcie_type == PCIEM_TYPE_ROOT_PORT || cfg->pcie_type == PCIEM_TYPE_ROOT_EC) cfg->pcie_root_ctl = RREG(PCIER_ROOT_CTL); if (version > 1) { cfg->pcie_device_ctl2 = RREG(PCIER_DEVICE_CTL2); cfg->pcie_link_ctl2 = RREG(PCIER_LINK_CTL2); cfg->pcie_slot_ctl2 = RREG(PCIER_SLOT_CTL2); } #undef RREG } static void pci_cfg_save_pcix(device_t dev, struct pci_devinfo *dinfo) { dinfo->cfg.pcix.pcix_command = pci_read_config(dev, dinfo->cfg.pcix.pcix_location + PCIXR_COMMAND, 2); } void pci_cfg_save(device_t dev, struct pci_devinfo *dinfo, int setstate) { uint32_t cls; int ps; /* * Some drivers apparently write to these registers w/o updating our * cached copy. No harm happens if we update the copy, so do so here * so we can restore them. The COMMAND register is modified by the * bus w/o updating the cache. This should represent the normally * writable portion of the 'defined' part of type 0/1/2 headers. */ dinfo->cfg.vendor = pci_read_config(dev, PCIR_VENDOR, 2); dinfo->cfg.device = pci_read_config(dev, PCIR_DEVICE, 2); dinfo->cfg.cmdreg = pci_read_config(dev, PCIR_COMMAND, 2); dinfo->cfg.intline = pci_read_config(dev, PCIR_INTLINE, 1); dinfo->cfg.intpin = pci_read_config(dev, PCIR_INTPIN, 1); dinfo->cfg.cachelnsz = pci_read_config(dev, PCIR_CACHELNSZ, 1); dinfo->cfg.lattimer = pci_read_config(dev, PCIR_LATTIMER, 1); dinfo->cfg.baseclass = pci_read_config(dev, PCIR_CLASS, 1); dinfo->cfg.subclass = pci_read_config(dev, PCIR_SUBCLASS, 1); dinfo->cfg.progif = pci_read_config(dev, PCIR_PROGIF, 1); dinfo->cfg.revid = pci_read_config(dev, PCIR_REVID, 1); switch (dinfo->cfg.hdrtype & PCIM_HDRTYPE) { case PCIM_HDRTYPE_NORMAL: dinfo->cfg.subvendor = pci_read_config(dev, PCIR_SUBVEND_0, 2); dinfo->cfg.subdevice = pci_read_config(dev, PCIR_SUBDEV_0, 2); dinfo->cfg.mingnt = pci_read_config(dev, PCIR_MINGNT, 1); dinfo->cfg.maxlat = pci_read_config(dev, PCIR_MAXLAT, 1); break; case PCIM_HDRTYPE_BRIDGE: dinfo->cfg.bridge.br_seclat = pci_read_config(dev, PCIR_SECLAT_1, 1); dinfo->cfg.bridge.br_subbus = pci_read_config(dev, PCIR_SUBBUS_1, 1); dinfo->cfg.bridge.br_secbus = pci_read_config(dev, PCIR_SECBUS_1, 1); dinfo->cfg.bridge.br_pribus = pci_read_config(dev, PCIR_PRIBUS_1, 1); dinfo->cfg.bridge.br_control = pci_read_config(dev, PCIR_BRIDGECTL_1, 2); break; case PCIM_HDRTYPE_CARDBUS: dinfo->cfg.bridge.br_seclat = pci_read_config(dev, PCIR_SECLAT_2, 1); dinfo->cfg.bridge.br_subbus = pci_read_config(dev, PCIR_SUBBUS_2, 1); dinfo->cfg.bridge.br_secbus = pci_read_config(dev, PCIR_SECBUS_2, 1); dinfo->cfg.bridge.br_pribus = pci_read_config(dev, PCIR_PRIBUS_2, 1); dinfo->cfg.bridge.br_control = pci_read_config(dev, PCIR_BRIDGECTL_2, 2); dinfo->cfg.subvendor = pci_read_config(dev, PCIR_SUBVEND_2, 2); dinfo->cfg.subdevice = pci_read_config(dev, PCIR_SUBDEV_2, 2); break; } if (dinfo->cfg.pcie.pcie_location != 0) pci_cfg_save_pcie(dev, dinfo); if (dinfo->cfg.pcix.pcix_location != 0) pci_cfg_save_pcix(dev, dinfo); #ifdef PCI_IOV if (dinfo->cfg.iov != NULL) pci_iov_cfg_save(dev, dinfo); #endif /* * don't set the state for display devices, base peripherals and * memory devices since bad things happen when they are powered down. * We should (a) have drivers that can easily detach and (b) use * generic drivers for these devices so that some device actually * attaches. We need to make sure that when we implement (a) we don't * power the device down on a reattach. */ cls = pci_get_class(dev); if (!setstate) return; switch (pci_do_power_nodriver) { case 0: /* NO powerdown at all */ return; case 1: /* Conservative about what to power down */ if (cls == PCIC_STORAGE) return; /*FALLTHROUGH*/ case 2: /* Aggressive about what to power down */ if (cls == PCIC_DISPLAY || cls == PCIC_MEMORY || cls == PCIC_BASEPERIPH) return; /*FALLTHROUGH*/ case 3: /* Power down everything */ break; } /* * PCI spec says we can only go into D3 state from D0 state. * Transition from D[12] into D0 before going to D3 state. */ ps = pci_get_powerstate(dev); if (ps != PCI_POWERSTATE_D0 && ps != PCI_POWERSTATE_D3) pci_set_powerstate(dev, PCI_POWERSTATE_D0); if (pci_get_powerstate(dev) != PCI_POWERSTATE_D3) pci_set_powerstate(dev, PCI_POWERSTATE_D3); } /* Wrapper APIs suitable for device driver use. */ void pci_save_state(device_t dev) { struct pci_devinfo *dinfo; dinfo = device_get_ivars(dev); pci_cfg_save(dev, dinfo, 0); } void pci_restore_state(device_t dev) { struct pci_devinfo *dinfo; dinfo = device_get_ivars(dev); pci_cfg_restore(dev, dinfo); } static int pci_get_id_method(device_t dev, device_t child, enum pci_id_type type, uintptr_t *id) { return (PCIB_GET_ID(device_get_parent(dev), child, type, id)); } /* Find the upstream port of a given PCI device in a root complex. */ device_t pci_find_pcie_root_port(device_t dev) { struct pci_devinfo *dinfo; devclass_t pci_class; device_t pcib, bus; pci_class = devclass_find("pci"); KASSERT(device_get_devclass(device_get_parent(dev)) == pci_class, ("%s: non-pci device %s", __func__, device_get_nameunit(dev))); /* * Walk the bridge hierarchy until we find a PCI-e root * port or a non-PCI device. */ for (;;) { bus = device_get_parent(dev); KASSERT(bus != NULL, ("%s: null parent of %s", __func__, device_get_nameunit(dev))); pcib = device_get_parent(bus); KASSERT(pcib != NULL, ("%s: null bridge of %s", __func__, device_get_nameunit(bus))); /* * pcib's parent must be a PCI bus for this to be a * PCI-PCI bridge. */ if (device_get_devclass(device_get_parent(pcib)) != pci_class) return (NULL); dinfo = device_get_ivars(pcib); if (dinfo->cfg.pcie.pcie_location != 0 && dinfo->cfg.pcie.pcie_type == PCIEM_TYPE_ROOT_PORT) return (pcib); dev = pcib; } } /* * Wait for pending transactions to complete on a PCI-express function. * * The maximum delay is specified in milliseconds in max_delay. Note * that this function may sleep. * * Returns true if the function is idle and false if the timeout is * exceeded. If dev is not a PCI-express function, this returns true. */ bool pcie_wait_for_pending_transactions(device_t dev, u_int max_delay) { struct pci_devinfo *dinfo = device_get_ivars(dev); uint16_t sta; int cap; cap = dinfo->cfg.pcie.pcie_location; if (cap == 0) return (true); sta = pci_read_config(dev, cap + PCIER_DEVICE_STA, 2); while (sta & PCIEM_STA_TRANSACTION_PND) { if (max_delay == 0) return (false); /* Poll once every 100 milliseconds up to the timeout. */ if (max_delay > 100) { pause_sbt("pcietp", 100 * SBT_1MS, 0, C_HARDCLOCK); max_delay -= 100; } else { pause_sbt("pcietp", max_delay * SBT_1MS, 0, C_HARDCLOCK); max_delay = 0; } sta = pci_read_config(dev, cap + PCIER_DEVICE_STA, 2); } return (true); } /* * Determine the maximum Completion Timeout in microseconds. * * For non-PCI-express functions this returns 0. */ int pcie_get_max_completion_timeout(device_t dev) { struct pci_devinfo *dinfo = device_get_ivars(dev); int cap; cap = dinfo->cfg.pcie.pcie_location; if (cap == 0) return (0); /* * Functions using the 1.x spec use the default timeout range of * 50 microseconds to 50 milliseconds. Functions that do not * support programmable timeouts also use this range. */ if ((dinfo->cfg.pcie.pcie_flags & PCIEM_FLAGS_VERSION) < 2 || (pci_read_config(dev, cap + PCIER_DEVICE_CAP2, 4) & PCIEM_CAP2_COMP_TIMO_RANGES) == 0) return (50 * 1000); switch (pci_read_config(dev, cap + PCIER_DEVICE_CTL2, 2) & PCIEM_CTL2_COMP_TIMO_VAL) { case PCIEM_CTL2_COMP_TIMO_100US: return (100); case PCIEM_CTL2_COMP_TIMO_10MS: return (10 * 1000); case PCIEM_CTL2_COMP_TIMO_55MS: return (55 * 1000); case PCIEM_CTL2_COMP_TIMO_210MS: return (210 * 1000); case PCIEM_CTL2_COMP_TIMO_900MS: return (900 * 1000); case PCIEM_CTL2_COMP_TIMO_3500MS: return (3500 * 1000); case PCIEM_CTL2_COMP_TIMO_13S: return (13 * 1000 * 1000); case PCIEM_CTL2_COMP_TIMO_64S: return (64 * 1000 * 1000); default: return (50 * 1000); } } void pcie_apei_error(device_t dev, int sev, uint8_t *aerp) { struct pci_devinfo *dinfo = device_get_ivars(dev); const char *s; int aer; uint32_t r, r1; uint16_t rs; if (sev == PCIEM_STA_CORRECTABLE_ERROR) s = "Correctable"; else if (sev == PCIEM_STA_NON_FATAL_ERROR) s = "Uncorrectable (Non-Fatal)"; else s = "Uncorrectable (Fatal)"; device_printf(dev, "%s PCIe error reported by APEI\n", s); if (aerp) { if (sev == PCIEM_STA_CORRECTABLE_ERROR) { r = le32dec(aerp + PCIR_AER_COR_STATUS); r1 = le32dec(aerp + PCIR_AER_COR_MASK); } else { r = le32dec(aerp + PCIR_AER_UC_STATUS); r1 = le32dec(aerp + PCIR_AER_UC_MASK); } device_printf(dev, "status 0x%08x mask 0x%08x", r, r1); if (sev != PCIEM_STA_CORRECTABLE_ERROR) { r = le32dec(aerp + PCIR_AER_UC_SEVERITY); rs = le16dec(aerp + PCIR_AER_CAP_CONTROL); printf(" severity 0x%08x first %d\n", r, rs & 0x1f); } else printf("\n"); } /* As kind of recovery just report and clear the error statuses. */ if (pci_find_extcap(dev, PCIZ_AER, &aer) == 0) { r = pci_read_config(dev, aer + PCIR_AER_UC_STATUS, 4); if (r != 0) { pci_write_config(dev, aer + PCIR_AER_UC_STATUS, r, 4); device_printf(dev, "Clearing UC AER errors 0x%08x\n", r); } r = pci_read_config(dev, aer + PCIR_AER_COR_STATUS, 4); if (r != 0) { pci_write_config(dev, aer + PCIR_AER_COR_STATUS, r, 4); device_printf(dev, "Clearing COR AER errors 0x%08x\n", r); } } if (dinfo->cfg.pcie.pcie_location != 0) { rs = pci_read_config(dev, dinfo->cfg.pcie.pcie_location + PCIER_DEVICE_STA, 2); if ((rs & (PCIEM_STA_CORRECTABLE_ERROR | PCIEM_STA_NON_FATAL_ERROR | PCIEM_STA_FATAL_ERROR | PCIEM_STA_UNSUPPORTED_REQ)) != 0) { pci_write_config(dev, dinfo->cfg.pcie.pcie_location + PCIER_DEVICE_STA, rs, 2); device_printf(dev, "Clearing PCIe errors 0x%04x\n", rs); } } } /* * Perform a Function Level Reset (FLR) on a device. * * This function first waits for any pending transactions to complete * within the timeout specified by max_delay. If transactions are * still pending, the function will return false without attempting a * reset. * * If dev is not a PCI-express function or does not support FLR, this * function returns false. * * Note that no registers are saved or restored. The caller is * responsible for saving and restoring any registers including * PCI-standard registers via pci_save_state() and * pci_restore_state(). */ bool pcie_flr(device_t dev, u_int max_delay, bool force) { struct pci_devinfo *dinfo = device_get_ivars(dev); uint16_t cmd, ctl; int compl_delay; int cap; cap = dinfo->cfg.pcie.pcie_location; if (cap == 0) return (false); if (!(pci_read_config(dev, cap + PCIER_DEVICE_CAP, 4) & PCIEM_CAP_FLR)) return (false); /* * Disable busmastering to prevent generation of new * transactions while waiting for the device to go idle. If * the idle timeout fails, the command register is restored * which will re-enable busmastering. */ cmd = pci_read_config(dev, PCIR_COMMAND, 2); pci_write_config(dev, PCIR_COMMAND, cmd & ~(PCIM_CMD_BUSMASTEREN), 2); if (!pcie_wait_for_pending_transactions(dev, max_delay)) { if (!force) { pci_write_config(dev, PCIR_COMMAND, cmd, 2); return (false); } pci_printf(&dinfo->cfg, "Resetting with transactions pending after %d ms\n", max_delay); /* * Extend the post-FLR delay to cover the maximum * Completion Timeout delay of anything in flight * during the FLR delay. Enforce a minimum delay of * at least 10ms. */ compl_delay = pcie_get_max_completion_timeout(dev) / 1000; if (compl_delay < 10) compl_delay = 10; } else compl_delay = 0; /* Initiate the reset. */ ctl = pci_read_config(dev, cap + PCIER_DEVICE_CTL, 2); pci_write_config(dev, cap + PCIER_DEVICE_CTL, ctl | PCIEM_CTL_INITIATE_FLR, 2); /* Wait for 100ms. */ pause_sbt("pcieflr", (100 + compl_delay) * SBT_1MS, 0, C_HARDCLOCK); if (pci_read_config(dev, cap + PCIER_DEVICE_STA, 2) & PCIEM_STA_TRANSACTION_PND) pci_printf(&dinfo->cfg, "Transactions pending after FLR!\n"); return (true); } /* * Attempt a power-management reset by cycling the device in/out of D3 * state. PCI spec says we can only go into D3 state from D0 state. * Transition from D[12] into D0 before going to D3 state. */ int pci_power_reset(device_t dev) { int ps; ps = pci_get_powerstate(dev); if (ps != PCI_POWERSTATE_D0 && ps != PCI_POWERSTATE_D3) pci_set_powerstate(dev, PCI_POWERSTATE_D0); pci_set_powerstate(dev, PCI_POWERSTATE_D3); pci_set_powerstate(dev, ps); return (0); } /* * Try link drop and retrain of the downstream port of upstream * switch, for PCIe. According to the PCIe 3.0 spec 6.6.1, this must * cause Conventional Hot reset of the device in the slot. * Alternative, for PCIe, could be the secondary bus reset initiatied * on the upstream switch PCIR_BRIDGECTL_1, bit 6. */ int pcie_link_reset(device_t port, int pcie_location) { uint16_t v; v = pci_read_config(port, pcie_location + PCIER_LINK_CTL, 2); v |= PCIEM_LINK_CTL_LINK_DIS; pci_write_config(port, pcie_location + PCIER_LINK_CTL, v, 2); pause_sbt("pcier1", mstosbt(20), 0, 0); v &= ~PCIEM_LINK_CTL_LINK_DIS; v |= PCIEM_LINK_CTL_RETRAIN_LINK; pci_write_config(port, pcie_location + PCIER_LINK_CTL, v, 2); pause_sbt("pcier2", mstosbt(100), 0, 0); /* 100 ms */ v = pci_read_config(port, pcie_location + PCIER_LINK_STA, 2); return ((v & PCIEM_LINK_STA_TRAINING) != 0 ? ETIMEDOUT : 0); } static int pci_reset_post(device_t dev, device_t child) { if (dev == device_get_parent(child)) pci_restore_state(child); return (0); } static int pci_reset_prepare(device_t dev, device_t child) { if (dev == device_get_parent(child)) pci_save_state(child); return (0); } static int pci_reset_child(device_t dev, device_t child, int flags) { int error; if (dev == NULL || device_get_parent(child) != dev) return (0); if ((flags & DEVF_RESET_DETACH) != 0) { error = device_get_state(child) == DS_ATTACHED ? device_detach(child) : 0; } else { error = BUS_SUSPEND_CHILD(dev, child); } if (error == 0) { if (!pcie_flr(child, 1000, false)) { error = BUS_RESET_PREPARE(dev, child); if (error == 0) pci_power_reset(child); BUS_RESET_POST(dev, child); } if ((flags & DEVF_RESET_DETACH) != 0) device_probe_and_attach(child); else BUS_RESUME_CHILD(dev, child); } return (error); } const struct pci_device_table * pci_match_device(device_t child, const struct pci_device_table *id, size_t nelt) { bool match; uint16_t vendor, device, subvendor, subdevice, class, subclass, revid; vendor = pci_get_vendor(child); device = pci_get_device(child); subvendor = pci_get_subvendor(child); subdevice = pci_get_subdevice(child); class = pci_get_class(child); subclass = pci_get_subclass(child); revid = pci_get_revid(child); while (nelt-- > 0) { match = true; if (id->match_flag_vendor) match &= vendor == id->vendor; if (id->match_flag_device) match &= device == id->device; if (id->match_flag_subvendor) match &= subvendor == id->subvendor; if (id->match_flag_subdevice) match &= subdevice == id->subdevice; if (id->match_flag_class) match &= class == id->class_id; if (id->match_flag_subclass) match &= subclass == id->subclass; if (id->match_flag_revid) match &= revid == id->revid; if (match) return (id); id++; } return (NULL); } static void pci_print_faulted_dev_name(const struct pci_devinfo *dinfo) { const char *dev_name; device_t dev; dev = dinfo->cfg.dev; printf("pci%d:%d:%d:%d", dinfo->cfg.domain, dinfo->cfg.bus, dinfo->cfg.slot, dinfo->cfg.func); dev_name = device_get_name(dev); if (dev_name != NULL) printf(" (%s%d)", dev_name, device_get_unit(dev)); } void pci_print_faulted_dev(void) { struct pci_devinfo *dinfo; device_t dev; int aer, i; uint32_t r1, r2; uint16_t status; STAILQ_FOREACH(dinfo, &pci_devq, pci_links) { dev = dinfo->cfg.dev; status = pci_read_config(dev, PCIR_STATUS, 2); status &= PCIM_STATUS_MDPERR | PCIM_STATUS_STABORT | PCIM_STATUS_RTABORT | PCIM_STATUS_RMABORT | PCIM_STATUS_SERR | PCIM_STATUS_PERR; if (status != 0) { pci_print_faulted_dev_name(dinfo); printf(" error 0x%04x\n", status); } if (dinfo->cfg.pcie.pcie_location != 0) { status = pci_read_config(dev, dinfo->cfg.pcie.pcie_location + PCIER_DEVICE_STA, 2); if ((status & (PCIEM_STA_CORRECTABLE_ERROR | PCIEM_STA_NON_FATAL_ERROR | PCIEM_STA_FATAL_ERROR | PCIEM_STA_UNSUPPORTED_REQ)) != 0) { pci_print_faulted_dev_name(dinfo); printf(" PCIe DEVCTL 0x%04x DEVSTA 0x%04x\n", pci_read_config(dev, dinfo->cfg.pcie.pcie_location + PCIER_DEVICE_CTL, 2), status); } } if (pci_find_extcap(dev, PCIZ_AER, &aer) == 0) { r1 = pci_read_config(dev, aer + PCIR_AER_UC_STATUS, 4); r2 = pci_read_config(dev, aer + PCIR_AER_COR_STATUS, 4); if (r1 != 0 || r2 != 0) { pci_print_faulted_dev_name(dinfo); printf(" AER UC 0x%08x Mask 0x%08x Svr 0x%08x\n" " COR 0x%08x Mask 0x%08x Ctl 0x%08x\n", r1, pci_read_config(dev, aer + PCIR_AER_UC_MASK, 4), pci_read_config(dev, aer + PCIR_AER_UC_SEVERITY, 4), r2, pci_read_config(dev, aer + PCIR_AER_COR_MASK, 4), pci_read_config(dev, aer + PCIR_AER_CAP_CONTROL, 4)); for (i = 0; i < 4; i++) { r1 = pci_read_config(dev, aer + PCIR_AER_HEADER_LOG + i * 4, 4); printf(" HL%d: 0x%08x\n", i, r1); } } } } } #ifdef DDB DB_SHOW_COMMAND_FLAGS(pcierr, pci_print_faulted_dev_db, DB_CMD_MEMSAFE) { pci_print_faulted_dev(); } static void db_clear_pcie_errors(const struct pci_devinfo *dinfo) { device_t dev; int aer; uint32_t r; dev = dinfo->cfg.dev; r = pci_read_config(dev, dinfo->cfg.pcie.pcie_location + PCIER_DEVICE_STA, 2); pci_write_config(dev, dinfo->cfg.pcie.pcie_location + PCIER_DEVICE_STA, r, 2); if (pci_find_extcap(dev, PCIZ_AER, &aer) != 0) return; r = pci_read_config(dev, aer + PCIR_AER_UC_STATUS, 4); if (r != 0) pci_write_config(dev, aer + PCIR_AER_UC_STATUS, r, 4); r = pci_read_config(dev, aer + PCIR_AER_COR_STATUS, 4); if (r != 0) pci_write_config(dev, aer + PCIR_AER_COR_STATUS, r, 4); } DB_COMMAND_FLAGS(pci_clearerr, db_pci_clearerr, DB_CMD_MEMSAFE) { struct pci_devinfo *dinfo; device_t dev; uint16_t status, status1; STAILQ_FOREACH(dinfo, &pci_devq, pci_links) { dev = dinfo->cfg.dev; status1 = status = pci_read_config(dev, PCIR_STATUS, 2); status1 &= PCIM_STATUS_MDPERR | PCIM_STATUS_STABORT | PCIM_STATUS_RTABORT | PCIM_STATUS_RMABORT | PCIM_STATUS_SERR | PCIM_STATUS_PERR; if (status1 != 0) { status &= ~status1; pci_write_config(dev, PCIR_STATUS, status, 2); } if (dinfo->cfg.pcie.pcie_location != 0) db_clear_pcie_errors(dinfo); } } #endif diff --git a/sys/dev/pci/pcireg.h b/sys/dev/pci/pcireg.h index 623deb8b4505..f6aaf30611e4 100644 --- a/sys/dev/pci/pcireg.h +++ b/sys/dev/pci/pcireg.h @@ -1,1180 +1,1183 @@ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright 1997, Stefan Esser * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice unmodified, this list of conditions, and the following * disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * */ #ifndef __PCI_PCIREG_H #define __PCI_PCIREG_H /* * PCIM_xxx: mask to locate subfield in register * PCIR_xxx: config register offset * PCIC_xxx: device class * PCIS_xxx: device subclass * PCIP_xxx: device programming interface * PCIV_xxx: PCI vendor ID (only required to fixup ancient devices) * PCID_xxx: device ID * PCIY_xxx: capability identification number * PCIZ_xxx: extended capability identification number */ /* some PCI bus constants */ #define PCI_DOMAINMAX 65535 /* highest supported domain number */ #define PCI_BUSMAX 255 /* highest supported bus number */ #define PCI_SLOTMAX 31 /* highest supported slot number */ #define PCI_FUNCMAX 7 /* highest supported function number */ #define PCI_REGMAX 255 /* highest supported config register addr. */ #define PCIE_REGMAX 4095 /* highest supported config register addr. */ #define PCI_MAXHDRTYPE 2 #define PCIE_ARI_SLOTMAX 0 #define PCIE_ARI_FUNCMAX 255 #define PCI_RID_DOMAIN_SHIFT 16 #define PCI_RID_BUS_SHIFT 8 #define PCI_RID_SLOT_SHIFT 3 #define PCI_RID_FUNC_SHIFT 0 #define PCI_RID(bus, slot, func) \ ((((bus) & PCI_BUSMAX) << PCI_RID_BUS_SHIFT) | \ (((slot) & PCI_SLOTMAX) << PCI_RID_SLOT_SHIFT) | \ (((func) & PCI_FUNCMAX) << PCI_RID_FUNC_SHIFT)) #define PCI_ARI_RID(bus, func) \ ((((bus) & PCI_BUSMAX) << PCI_RID_BUS_SHIFT) | \ (((func) & PCIE_ARI_FUNCMAX) << PCI_RID_FUNC_SHIFT)) #define PCI_RID2BUS(rid) (((rid) >> PCI_RID_BUS_SHIFT) & PCI_BUSMAX) #define PCI_RID2SLOT(rid) (((rid) >> PCI_RID_SLOT_SHIFT) & PCI_SLOTMAX) #define PCI_RID2FUNC(rid) (((rid) >> PCI_RID_FUNC_SHIFT) & PCI_FUNCMAX) #define PCIE_ARI_RID2SLOT(rid) (0) #define PCIE_ARI_RID2FUNC(rid) \ (((rid) >> PCI_RID_FUNC_SHIFT) & PCIE_ARI_FUNCMAX) #define PCIE_ARI_SLOT(func) (((func) >> PCI_RID_SLOT_SHIFT) & PCI_SLOTMAX) #define PCIE_ARI_FUNC(func) (((func) >> PCI_RID_FUNC_SHIFT) & PCI_FUNCMAX) /* PCI config header registers for all devices */ #define PCIR_DEVVENDOR 0x00 #define PCIR_VENDOR 0x00 #define PCIR_DEVICE 0x02 #define PCIR_COMMAND 0x04 #define PCIM_CMD_PORTEN 0x0001 #define PCIM_CMD_MEMEN 0x0002 #define PCIM_CMD_BUSMASTEREN 0x0004 #define PCIM_CMD_SPECIALEN 0x0008 #define PCIM_CMD_MWRICEN 0x0010 #define PCIM_CMD_PERRESPEN 0x0040 #define PCIM_CMD_SERRESPEN 0x0100 #define PCIM_CMD_BACKTOBACK 0x0200 #define PCIM_CMD_INTxDIS 0x0400 #define PCIR_STATUS 0x06 #define PCIM_STATUS_INTxSTATE 0x0008 #define PCIM_STATUS_CAPPRESENT 0x0010 #define PCIM_STATUS_66CAPABLE 0x0020 #define PCIM_STATUS_BACKTOBACK 0x0080 #define PCIM_STATUS_MDPERR 0x0100 #define PCIM_STATUS_SEL_FAST 0x0000 #define PCIM_STATUS_SEL_MEDIMUM 0x0200 #define PCIM_STATUS_SEL_SLOW 0x0400 #define PCIM_STATUS_SEL_MASK 0x0600 #define PCIM_STATUS_STABORT 0x0800 #define PCIM_STATUS_RTABORT 0x1000 #define PCIM_STATUS_RMABORT 0x2000 #define PCIM_STATUS_SERR 0x4000 #define PCIM_STATUS_PERR 0x8000 #define PCIR_REVID 0x08 #define PCIR_PROGIF 0x09 #define PCIR_SUBCLASS 0x0a #define PCIR_CLASS 0x0b #define PCIR_CACHELNSZ 0x0c #define PCIR_LATTIMER 0x0d #define PCIR_HDRTYPE 0x0e #define PCIM_HDRTYPE 0x7f #define PCIM_HDRTYPE_NORMAL 0x00 #define PCIM_HDRTYPE_BRIDGE 0x01 #define PCIM_HDRTYPE_CARDBUS 0x02 #define PCIM_MFDEV 0x80 #define PCIR_BIST 0x0f /* PCI Spec rev 2.2: 0FFFFh is an invalid value for Vendor ID. */ #define PCIV_INVALID 0xffff /* Capability Register Offsets */ #define PCICAP_ID 0x0 #define PCICAP_NEXTPTR 0x1 /* Capability Identification Numbers */ #define PCIY_PMG 0x01 /* PCI Power Management */ #define PCIY_AGP 0x02 /* AGP */ #define PCIY_VPD 0x03 /* Vital Product Data */ #define PCIY_SLOTID 0x04 /* Slot Identification */ #define PCIY_MSI 0x05 /* Message Signaled Interrupts */ #define PCIY_CHSWP 0x06 /* CompactPCI Hot Swap */ #define PCIY_PCIX 0x07 /* PCI-X */ #define PCIY_HT 0x08 /* HyperTransport */ #define PCIY_VENDOR 0x09 /* Vendor Unique */ #define PCIY_DEBUG 0x0a /* Debug port */ #define PCIY_CRES 0x0b /* CompactPCI central resource control */ #define PCIY_HOTPLUG 0x0c /* PCI Hot-Plug */ #define PCIY_SUBVENDOR 0x0d /* PCI-PCI bridge subvendor ID */ #define PCIY_AGP8X 0x0e /* AGP 8x */ #define PCIY_SECDEV 0x0f /* Secure Device */ #define PCIY_EXPRESS 0x10 /* PCI Express */ #define PCIY_MSIX 0x11 /* MSI-X */ #define PCIY_SATA 0x12 /* SATA */ #define PCIY_PCIAF 0x13 /* PCI Advanced Features */ #define PCIY_EA 0x14 /* PCI Extended Allocation */ #define PCIY_FPB 0x15 /* Flattening Portal Bridge */ /* Extended Capability Register Fields */ #define PCIR_EXTCAP 0x100 #define PCIM_EXTCAP_ID 0x0000ffff #define PCIM_EXTCAP_VER 0x000f0000 #define PCIM_EXTCAP_NEXTPTR 0xfff00000 #define PCI_EXTCAP_ID(ecap) ((ecap) & PCIM_EXTCAP_ID) #define PCI_EXTCAP_VER(ecap) (((ecap) & PCIM_EXTCAP_VER) >> 16) #define PCI_EXTCAP_NEXTPTR(ecap) (((ecap) & PCIM_EXTCAP_NEXTPTR) >> 20) /* Extended Capability Identification Numbers */ #define PCIZ_AER 0x0001 /* Advanced Error Reporting */ #define PCIZ_VC 0x0002 /* Virtual Channel if MFVC Ext Cap not set */ #define PCIZ_SERNUM 0x0003 /* Device Serial Number */ #define PCIZ_PWRBDGT 0x0004 /* Power Budgeting */ #define PCIZ_RCLINK_DCL 0x0005 /* Root Complex Link Declaration */ #define PCIZ_RCLINK_CTL 0x0006 /* Root Complex Internal Link Control */ #define PCIZ_RCEC_ASSOC 0x0007 /* Root Complex Event Collector Association */ #define PCIZ_MFVC 0x0008 /* Multi-Function Virtual Channel */ #define PCIZ_VC2 0x0009 /* Virtual Channel if MFVC Ext Cap set */ #define PCIZ_RCRB 0x000a /* RCRB Header */ #define PCIZ_VENDOR 0x000b /* Vendor Unique */ #define PCIZ_CAC 0x000c /* Configuration Access Correction -- obsolete */ #define PCIZ_ACS 0x000d /* Access Control Services */ #define PCIZ_ARI 0x000e /* Alternative Routing-ID Interpretation */ #define PCIZ_ATS 0x000f /* Address Translation Services */ #define PCIZ_SRIOV 0x0010 /* Single Root IO Virtualization */ #define PCIZ_MRIOV 0x0011 /* Multiple Root IO Virtualization */ #define PCIZ_MULTICAST 0x0012 /* Multicast */ #define PCIZ_PAGE_REQ 0x0013 /* Page Request */ #define PCIZ_AMD 0x0014 /* Reserved for AMD */ #define PCIZ_RESIZE_BAR 0x0015 /* Resizable BAR */ #define PCIZ_DPA 0x0016 /* Dynamic Power Allocation */ #define PCIZ_TPH_REQ 0x0017 /* TPH Requester */ #define PCIZ_LTR 0x0018 /* Latency Tolerance Reporting */ #define PCIZ_SEC_PCIE 0x0019 /* Secondary PCI Express */ #define PCIZ_PMUX 0x001a /* Protocol Multiplexing */ #define PCIZ_PASID 0x001b /* Process Address Space ID */ #define PCIZ_LN_REQ 0x001c /* LN Requester */ #define PCIZ_DPC 0x001d /* Downstream Port Containment */ #define PCIZ_L1PM 0x001e /* L1 PM Substates */ #define PCIZ_PTM 0x001f /* Precision Time Measurement */ #define PCIZ_M_PCIE 0x0020 /* PCIe over M-PHY */ #define PCIZ_FRS 0x0021 /* FRS Queuing */ #define PCIZ_RTR 0x0022 /* Readiness Time Reporting */ #define PCIZ_DVSEC 0x0023 /* Designated Vendor-Specific */ #define PCIZ_VF_REBAR 0x0024 /* VF Resizable BAR */ #define PCIZ_DLNK 0x0025 /* Data Link Feature */ #define PCIZ_16GT 0x0026 /* Physical Layer 16.0 GT/s */ #define PCIZ_LMR 0x0027 /* Lane Margining at Receiver */ #define PCIZ_HIER_ID 0x0028 /* Hierarchy ID */ #define PCIZ_NPEM 0x0029 /* Native PCIe Enclosure Management */ #define PCIZ_PL32 0x002a /* Physical Layer 32.0 GT/s */ #define PCIZ_AP 0x002b /* Alternate Protocol */ #define PCIZ_SFI 0x002c /* System Firmware Intermediary */ /* config registers for header type 0 devices */ #define PCIR_BARS 0x10 #define PCIR_BAR(x) (PCIR_BARS + (x) * 4) #define PCIR_MAX_BAR_0 5 #define PCI_RID2BAR(rid) (((rid) - PCIR_BARS) / 4) #define PCI_BAR_IO(x) (((x) & PCIM_BAR_SPACE) == PCIM_BAR_IO_SPACE) #define PCI_BAR_MEM(x) (((x) & PCIM_BAR_SPACE) == PCIM_BAR_MEM_SPACE) #define PCIM_BAR_SPACE 0x00000001 #define PCIM_BAR_MEM_SPACE 0 #define PCIM_BAR_IO_SPACE 1 #define PCIM_BAR_MEM_TYPE 0x00000006 #define PCIM_BAR_MEM_32 0 #define PCIM_BAR_MEM_1MB 2 /* Locate below 1MB in PCI <= 2.1 */ #define PCIM_BAR_MEM_64 4 #define PCIM_BAR_MEM_PREFETCH 0x00000008 #define PCIM_BAR_MEM_BASE 0xfffffffffffffff0ULL #define PCIM_BAR_IO_RESERVED 0x00000002 #define PCIM_BAR_IO_BASE 0xfffffffc #define PCIR_CIS 0x28 #define PCIM_CIS_ASI_MASK 0x00000007 #define PCIM_CIS_ASI_CONFIG 0 #define PCIM_CIS_ASI_BAR0 1 #define PCIM_CIS_ASI_BAR1 2 #define PCIM_CIS_ASI_BAR2 3 #define PCIM_CIS_ASI_BAR3 4 #define PCIM_CIS_ASI_BAR4 5 #define PCIM_CIS_ASI_BAR5 6 #define PCIM_CIS_ASI_ROM 7 #define PCIM_CIS_ADDR_MASK 0x0ffffff8 #define PCIM_CIS_ROM_MASK 0xf0000000 #define PCIM_CIS_CONFIG_MASK 0xff #define PCIR_SUBVEND_0 0x2c #define PCIR_SUBDEV_0 0x2e #define PCIR_BIOS 0x30 #define PCIM_BIOS_ENABLE 0x01 #define PCIM_BIOS_ADDR_MASK 0xfffff800 #define PCIR_CAP_PTR 0x34 #define PCIR_INTLINE 0x3c #define PCIR_INTPIN 0x3d #define PCIR_MINGNT 0x3e #define PCIR_MAXLAT 0x3f /* config registers for header type 1 (PCI-to-PCI bridge) devices */ #define PCIR_MAX_BAR_1 1 #define PCIR_SECSTAT_1 0x1e #define PCIR_PRIBUS_1 0x18 #define PCIR_SECBUS_1 0x19 #define PCIR_SUBBUS_1 0x1a #define PCIR_SECLAT_1 0x1b #define PCIR_IOBASEL_1 0x1c #define PCIR_IOLIMITL_1 0x1d #define PCIR_IOBASEH_1 0x30 #define PCIR_IOLIMITH_1 0x32 #define PCIM_BRIO_16 0x0 #define PCIM_BRIO_32 0x1 #define PCIM_BRIO_MASK 0xf #define PCIR_MEMBASE_1 0x20 #define PCIR_MEMLIMIT_1 0x22 #define PCIR_PMBASEL_1 0x24 #define PCIR_PMLIMITL_1 0x26 #define PCIR_PMBASEH_1 0x28 #define PCIR_PMLIMITH_1 0x2c #define PCIM_BRPM_32 0x0 #define PCIM_BRPM_64 0x1 #define PCIM_BRPM_MASK 0xf #define PCIR_BIOS_1 0x38 #define PCIR_BRIDGECTL_1 0x3e #define PCI_PPBMEMBASE(h,l) ((((uint64_t)(h) << 32) + ((l)<<16)) & ~0xfffff) #define PCI_PPBMEMLIMIT(h,l) ((((uint64_t)(h) << 32) + ((l)<<16)) | 0xfffff) #define PCI_PPBIOBASE(h,l) ((((h)<<16) + ((l)<<8)) & ~0xfff) #define PCI_PPBIOLIMIT(h,l) ((((h)<<16) + ((l)<<8)) | 0xfff) /* config registers for header type 2 (CardBus) devices */ #define PCIR_MAX_BAR_2 0 #define PCIR_CAP_PTR_2 0x14 #define PCIR_SECSTAT_2 0x16 #define PCIR_PRIBUS_2 0x18 #define PCIR_SECBUS_2 0x19 #define PCIR_SUBBUS_2 0x1a #define PCIR_SECLAT_2 0x1b #define PCIR_MEMBASE0_2 0x1c #define PCIR_MEMLIMIT0_2 0x20 #define PCIR_MEMBASE1_2 0x24 #define PCIR_MEMLIMIT1_2 0x28 #define PCIR_IOBASE0_2 0x2c #define PCIR_IOLIMIT0_2 0x30 #define PCIR_IOBASE1_2 0x34 #define PCIR_IOLIMIT1_2 0x38 #define PCIM_CBBIO_16 0x0 #define PCIM_CBBIO_32 0x1 #define PCIM_CBBIO_MASK 0x3 #define PCIR_BRIDGECTL_2 0x3e #define PCIR_SUBVEND_2 0x40 #define PCIR_SUBDEV_2 0x42 #define PCIR_PCCARDIF_2 0x44 #define PCI_CBBMEMBASE(l) ((l) & ~0xfffff) #define PCI_CBBMEMLIMIT(l) ((l) | 0xfffff) #define PCI_CBBIOBASE(l) ((l) & ~0x3) #define PCI_CBBIOLIMIT(l) ((l) | 0x3) /* PCI device class, subclass and programming interface definitions */ #define PCIC_OLD 0x00 #define PCIS_OLD_NONVGA 0x00 #define PCIS_OLD_VGA 0x01 #define PCIC_STORAGE 0x01 #define PCIS_STORAGE_SCSI 0x00 #define PCIS_STORAGE_IDE 0x01 #define PCIP_STORAGE_IDE_MODEPRIM 0x01 #define PCIP_STORAGE_IDE_PROGINDPRIM 0x02 #define PCIP_STORAGE_IDE_MODESEC 0x04 #define PCIP_STORAGE_IDE_PROGINDSEC 0x08 #define PCIP_STORAGE_IDE_MASTERDEV 0x80 #define PCIS_STORAGE_FLOPPY 0x02 #define PCIS_STORAGE_IPI 0x03 #define PCIS_STORAGE_RAID 0x04 #define PCIS_STORAGE_ATA_ADMA 0x05 #define PCIS_STORAGE_SATA 0x06 #define PCIP_STORAGE_SATA_AHCI_1_0 0x01 #define PCIS_STORAGE_SAS 0x07 #define PCIS_STORAGE_NVM 0x08 #define PCIP_STORAGE_NVM_NVMHCI_1_0 0x01 #define PCIP_STORAGE_NVM_ENTERPRISE_NVMHCI_1_0 0x02 #define PCIS_STORAGE_UFS 0x09 #define PCIP_STORAGE_UFS_UFSHCI_1_0 0x01 #define PCIS_STORAGE_OTHER 0x80 #define PCIC_NETWORK 0x02 #define PCIS_NETWORK_ETHERNET 0x00 #define PCIS_NETWORK_TOKENRING 0x01 #define PCIS_NETWORK_FDDI 0x02 #define PCIS_NETWORK_ATM 0x03 #define PCIS_NETWORK_ISDN 0x04 #define PCIS_NETWORK_WORLDFIP 0x05 #define PCIS_NETWORK_PICMG 0x06 #define PCIS_NETWORK_INFINIBAND 0x07 #define PCIS_NETWORK_HFC 0x08 #define PCIS_NETWORK_OTHER 0x80 #define PCIC_DISPLAY 0x03 #define PCIS_DISPLAY_VGA 0x00 #define PCIS_DISPLAY_XGA 0x01 #define PCIS_DISPLAY_3D 0x02 #define PCIS_DISPLAY_OTHER 0x80 #define PCIC_MULTIMEDIA 0x04 #define PCIS_MULTIMEDIA_VIDEO 0x00 #define PCIS_MULTIMEDIA_AUDIO 0x01 #define PCIS_MULTIMEDIA_TELE 0x02 #define PCIS_MULTIMEDIA_HDA 0x03 #define PCIP_MULTIMEDIA_HDA_VENDOR 0x01 #define PCIS_MULTIMEDIA_OTHER 0x80 #define PCIC_MEMORY 0x05 #define PCIS_MEMORY_RAM 0x00 #define PCIS_MEMORY_FLASH 0x01 #define PCIS_MEMORY_OTHER 0x80 #define PCIC_BRIDGE 0x06 #define PCIS_BRIDGE_HOST 0x00 #define PCIS_BRIDGE_ISA 0x01 #define PCIS_BRIDGE_EISA 0x02 #define PCIS_BRIDGE_MCA 0x03 #define PCIS_BRIDGE_PCI 0x04 #define PCIP_BRIDGE_PCI_SUBTRACTIVE 0x01 #define PCIS_BRIDGE_PCMCIA 0x05 #define PCIS_BRIDGE_NUBUS 0x06 #define PCIS_BRIDGE_CARDBUS 0x07 #define PCIS_BRIDGE_RACEWAY 0x08 #define PCIS_BRIDGE_PCI_TRANSPARENT 0x09 #define PCIS_BRIDGE_INFINIBAND 0x0a #define PCIS_BRIDGE_AS_PCI 0x0b #define PCIS_BRIDGE_AS_PCI_ASI_SIG 0x01 #define PCIS_BRIDGE_OTHER 0x80 #define PCIC_SIMPLECOMM 0x07 #define PCIS_SIMPLECOMM_UART 0x00 #define PCIP_SIMPLECOMM_UART_8250 0x00 #define PCIP_SIMPLECOMM_UART_16450A 0x01 #define PCIP_SIMPLECOMM_UART_16550A 0x02 #define PCIP_SIMPLECOMM_UART_16650A 0x03 #define PCIP_SIMPLECOMM_UART_16750A 0x04 #define PCIP_SIMPLECOMM_UART_16850A 0x05 #define PCIP_SIMPLECOMM_UART_16950A 0x06 #define PCIS_SIMPLECOMM_PAR 0x01 #define PCIS_SIMPLECOMM_MULSER 0x02 #define PCIS_SIMPLECOMM_MODEM 0x03 #define PCIS_SIMPLECOMM_GPIB 0x04 #define PCIS_SIMPLECOMM_SMART_CARD 0x05 #define PCIS_SIMPLECOMM_OTHER 0x80 #define PCIC_BASEPERIPH 0x08 #define PCIS_BASEPERIPH_PIC 0x00 #define PCIP_BASEPERIPH_PIC_8259A 0x00 #define PCIP_BASEPERIPH_PIC_ISA 0x01 #define PCIP_BASEPERIPH_PIC_EISA 0x02 #define PCIP_BASEPERIPH_PIC_IO_APIC 0x10 #define PCIP_BASEPERIPH_PIC_IOX_APIC 0x20 #define PCIS_BASEPERIPH_DMA 0x01 #define PCIS_BASEPERIPH_TIMER 0x02 #define PCIS_BASEPERIPH_RTC 0x03 #define PCIS_BASEPERIPH_PCIHOT 0x04 #define PCIS_BASEPERIPH_SDHC 0x05 #define PCIS_BASEPERIPH_IOMMU 0x06 #define PCIS_BASEPERIPH_RCEC 0x07 #define PCIS_BASEPERIPH_OTHER 0x80 #define PCIC_INPUTDEV 0x09 #define PCIS_INPUTDEV_KEYBOARD 0x00 #define PCIS_INPUTDEV_DIGITIZER 0x01 #define PCIS_INPUTDEV_MOUSE 0x02 #define PCIS_INPUTDEV_SCANNER 0x03 #define PCIS_INPUTDEV_GAMEPORT 0x04 #define PCIS_INPUTDEV_OTHER 0x80 #define PCIC_DOCKING 0x0a #define PCIS_DOCKING_GENERIC 0x00 #define PCIS_DOCKING_OTHER 0x80 #define PCIC_PROCESSOR 0x0b #define PCIS_PROCESSOR_386 0x00 #define PCIS_PROCESSOR_486 0x01 #define PCIS_PROCESSOR_PENTIUM 0x02 #define PCIS_PROCESSOR_ALPHA 0x10 #define PCIS_PROCESSOR_POWERPC 0x20 #define PCIS_PROCESSOR_MIPS 0x30 #define PCIS_PROCESSOR_COPROC 0x40 #define PCIC_SERIALBUS 0x0c #define PCIS_SERIALBUS_FW 0x00 #define PCIS_SERIALBUS_ACCESS 0x01 #define PCIS_SERIALBUS_SSA 0x02 #define PCIS_SERIALBUS_USB 0x03 #define PCIP_SERIALBUS_USB_UHCI 0x00 #define PCIP_SERIALBUS_USB_OHCI 0x10 #define PCIP_SERIALBUS_USB_EHCI 0x20 #define PCIP_SERIALBUS_USB_XHCI 0x30 #define PCIP_SERIALBUS_USB_DEVICE 0xfe #define PCIS_SERIALBUS_FC 0x04 #define PCIS_SERIALBUS_SMBUS 0x05 #define PCIS_SERIALBUS_INFINIBAND 0x06 #define PCIS_SERIALBUS_IPMI 0x07 #define PCIP_SERIALBUS_IPMI_SMIC 0x00 #define PCIP_SERIALBUS_IPMI_KCS 0x01 #define PCIP_SERIALBUS_IPMI_BT 0x02 #define PCIS_SERIALBUS_SERCOS 0x08 #define PCIS_SERIALBUS_CANBUS 0x09 #define PCIS_SERIALBUS_MIPI_I3C 0x0a #define PCIC_WIRELESS 0x0d #define PCIS_WIRELESS_IRDA 0x00 #define PCIS_WIRELESS_IR 0x01 #define PCIS_WIRELESS_RF 0x10 #define PCIS_WIRELESS_BLUETOOTH 0x11 #define PCIS_WIRELESS_BROADBAND 0x12 #define PCIS_WIRELESS_80211A 0x20 #define PCIS_WIRELESS_80211B 0x21 #define PCIS_WIRELESS_CELL 0x40 #define PCIS_WIRELESS_CELL_E 0x41 #define PCIS_WIRELESS_OTHER 0x80 #define PCIC_INTELLIIO 0x0e #define PCIS_INTELLIIO_I2O 0x00 #define PCIC_SATCOM 0x0f #define PCIS_SATCOM_TV 0x01 #define PCIS_SATCOM_AUDIO 0x02 #define PCIS_SATCOM_VOICE 0x03 #define PCIS_SATCOM_DATA 0x04 #define PCIC_CRYPTO 0x10 #define PCIS_CRYPTO_NETCOMP 0x00 #define PCIS_CRYPTO_ENTERTAIN 0x10 #define PCIS_CRYPTO_OTHER 0x80 #define PCIC_DASP 0x11 #define PCIS_DASP_DPIO 0x00 #define PCIS_DASP_PERFCNTRS 0x01 #define PCIS_DASP_COMM_SYNC 0x10 #define PCIS_DASP_MGMT_CARD 0x20 #define PCIS_DASP_OTHER 0x80 #define PCIC_ACCEL 0x12 #define PCIS_ACCEL_PROCESSING 0x00 #define PCIC_INSTRUMENT 0x13 #define PCIC_OTHER 0xff /* Bridge Control Values. */ #define PCIB_BCR_PERR_ENABLE 0x0001 #define PCIB_BCR_SERR_ENABLE 0x0002 #define PCIB_BCR_ISA_ENABLE 0x0004 #define PCIB_BCR_VGA_ENABLE 0x0008 #define PCIB_BCR_MASTER_ABORT_MODE 0x0020 #define PCIB_BCR_SECBUS_RESET 0x0040 #define PCIB_BCR_SECBUS_BACKTOBACK 0x0080 #define PCIB_BCR_PRI_DISCARD_TIMEOUT 0x0100 #define PCIB_BCR_SEC_DISCARD_TIMEOUT 0x0200 #define PCIB_BCR_DISCARD_TIMER_STATUS 0x0400 #define PCIB_BCR_DISCARD_TIMER_SERREN 0x0800 #define CBB_BCR_PERR_ENABLE 0x0001 #define CBB_BCR_SERR_ENABLE 0x0002 #define CBB_BCR_ISA_ENABLE 0x0004 #define CBB_BCR_VGA_ENABLE 0x0008 #define CBB_BCR_MASTER_ABORT_MODE 0x0020 #define CBB_BCR_CARDBUS_RESET 0x0040 #define CBB_BCR_IREQ_INT_ENABLE 0x0080 #define CBB_BCR_PREFETCH_0_ENABLE 0x0100 #define CBB_BCR_PREFETCH_1_ENABLE 0x0200 #define CBB_BCR_WRITE_POSTING_ENABLE 0x0400 /* PCI power manangement */ #define PCIR_POWER_CAP 0x2 #define PCIM_PCAP_SPEC 0x0007 #define PCIM_PCAP_PMEREQCLK 0x0008 #define PCIM_PCAP_DEVSPECINIT 0x0020 #define PCIM_PCAP_AUXPWR_0 0x0000 #define PCIM_PCAP_AUXPWR_55 0x0040 #define PCIM_PCAP_AUXPWR_100 0x0080 #define PCIM_PCAP_AUXPWR_160 0x00c0 #define PCIM_PCAP_AUXPWR_220 0x0100 #define PCIM_PCAP_AUXPWR_270 0x0140 #define PCIM_PCAP_AUXPWR_320 0x0180 #define PCIM_PCAP_AUXPWR_375 0x01c0 #define PCIM_PCAP_AUXPWRMASK 0x01c0 #define PCIM_PCAP_D1SUPP 0x0200 #define PCIM_PCAP_D2SUPP 0x0400 #define PCIM_PCAP_D0PME 0x0800 #define PCIM_PCAP_D1PME 0x1000 #define PCIM_PCAP_D2PME 0x2000 #define PCIM_PCAP_D3PME_HOT 0x4000 #define PCIM_PCAP_D3PME_COLD 0x8000 #define PCIR_POWER_STATUS 0x4 #define PCIM_PSTAT_D0 0x0000 #define PCIM_PSTAT_D1 0x0001 #define PCIM_PSTAT_D2 0x0002 #define PCIM_PSTAT_D3 0x0003 #define PCIM_PSTAT_DMASK 0x0003 #define PCIM_PSTAT_NOSOFTRESET 0x0008 #define PCIM_PSTAT_PMEENABLE 0x0100 #define PCIM_PSTAT_D0POWER 0x0000 #define PCIM_PSTAT_D1POWER 0x0200 #define PCIM_PSTAT_D2POWER 0x0400 #define PCIM_PSTAT_D3POWER 0x0600 #define PCIM_PSTAT_D0HEAT 0x0800 #define PCIM_PSTAT_D1HEAT 0x0a00 #define PCIM_PSTAT_D2HEAT 0x0c00 #define PCIM_PSTAT_D3HEAT 0x0e00 #define PCIM_PSTAT_DATASELMASK 0x1e00 #define PCIM_PSTAT_DATAUNKN 0x0000 #define PCIM_PSTAT_DATADIV10 0x2000 #define PCIM_PSTAT_DATADIV100 0x4000 #define PCIM_PSTAT_DATADIV1000 0x6000 #define PCIM_PSTAT_DATADIVMASK 0x6000 #define PCIM_PSTAT_PME 0x8000 #define PCIR_POWER_BSE 0x6 #define PCIM_PMCSR_BSE_D3B3 0x00 #define PCIM_PMCSR_BSE_D3B2 0x40 #define PCIM_PMCSR_BSE_BPCCE 0x80 #define PCIR_POWER_DATA 0x7 /* VPD capability registers */ #define PCIR_VPD_ADDR 0x2 #define PCIR_VPD_DATA 0x4 /* PCI Message Signalled Interrupts (MSI) */ #define PCIR_MSI_CTRL 0x2 #define PCIM_MSICTRL_VECTOR 0x0100 #define PCIM_MSICTRL_64BIT 0x0080 #define PCIM_MSICTRL_MME_MASK 0x0070 #define PCIM_MSICTRL_MME_1 0x0000 #define PCIM_MSICTRL_MME_2 0x0010 #define PCIM_MSICTRL_MME_4 0x0020 #define PCIM_MSICTRL_MME_8 0x0030 #define PCIM_MSICTRL_MME_16 0x0040 #define PCIM_MSICTRL_MME_32 0x0050 #define PCIM_MSICTRL_MMC_MASK 0x000E #define PCIM_MSICTRL_MMC_1 0x0000 #define PCIM_MSICTRL_MMC_2 0x0002 #define PCIM_MSICTRL_MMC_4 0x0004 #define PCIM_MSICTRL_MMC_8 0x0006 #define PCIM_MSICTRL_MMC_16 0x0008 #define PCIM_MSICTRL_MMC_32 0x000A #define PCIM_MSICTRL_MSI_ENABLE 0x0001 +#define PCI_MSI_MSGNUM(ctrl) \ + (1 << (((ctrl) & PCIM_MSICTRL_MMC_MASK) >> 1)) #define PCIR_MSI_ADDR 0x4 #define PCIR_MSI_ADDR_HIGH 0x8 #define PCIR_MSI_DATA 0x8 #define PCIR_MSI_DATA_64BIT 0xc #define PCIR_MSI_MASK 0x10 #define PCIR_MSI_PENDING 0x14 /* PCI Enhanced Allocation registers */ #define PCIR_EA_NUM_ENT 2 /* Number of Capability Entries */ #define PCIM_EA_NUM_ENT_MASK 0x3f /* Num Entries Mask */ #define PCIR_EA_FIRST_ENT 4 /* First EA Entry in List */ #define PCIR_EA_FIRST_ENT_BRIDGE 8 /* First EA Entry for Bridges */ #define PCIM_EA_ES 0x00000007 /* Entry Size */ #define PCIM_EA_BEI 0x000000f0 /* BAR Equivalent Indicator */ #define PCIM_EA_BEI_OFFSET 4 /* 0-5 map to BARs 0-5 respectively */ #define PCIM_EA_BEI_BAR_0 0 #define PCIM_EA_BEI_BAR_5 5 #define PCIM_EA_BEI_BAR(x) (((x) >> PCIM_EA_BEI_OFFSET) & 0xf) #define PCIM_EA_BEI_BRIDGE 0x6 /* Resource behind bridge */ #define PCIM_EA_BEI_ENI 0x7 /* Equivalent Not Indicated */ #define PCIM_EA_BEI_ROM 0x8 /* Expansion ROM */ /* 9-14 map to VF BARs 0-5 respectively */ #define PCIM_EA_BEI_VF_BAR_0 9 #define PCIM_EA_BEI_VF_BAR_5 14 #define PCIM_EA_BEI_RESERVED 0xf /* Reserved - Treat like ENI */ #define PCIM_EA_PP 0x0000ff00 /* Primary Properties */ #define PCIM_EA_PP_OFFSET 8 #define PCIM_EA_SP_OFFSET 16 #define PCIM_EA_SP 0x00ff0000 /* Secondary Properties */ #define PCIM_EA_P_MEM 0x00 /* Non-Prefetch Memory */ #define PCIM_EA_P_MEM_PREFETCH 0x01 /* Prefetchable Memory */ #define PCIM_EA_P_IO 0x02 /* I/O Space */ #define PCIM_EA_P_VF_MEM_PREFETCH 0x03 /* VF Prefetchable Memory */ #define PCIM_EA_P_VF_MEM 0x04 /* VF Non-Prefetch Memory */ #define PCIM_EA_P_BRIDGE_MEM 0x05 /* Bridge Non-Prefetch Memory */ #define PCIM_EA_P_BRIDGE_MEM_PREFETCH 0x06 /* Bridge Prefetchable Memory */ #define PCIM_EA_P_BRIDGE_IO 0x07 /* Bridge I/O Space */ /* 0x08-0xfc reserved */ #define PCIM_EA_P_MEM_RESERVED 0xfd /* Reserved Memory */ #define PCIM_EA_P_IO_RESERVED 0xfe /* Reserved I/O Space */ #define PCIM_EA_P_UNAVAILABLE 0xff /* Entry Unavailable */ #define PCIM_EA_WRITABLE 0x40000000 /* Writable: 1 = RW, 0 = HwInit */ #define PCIM_EA_ENABLE 0x80000000 /* Enable for this entry */ #define PCIM_EA_BASE 4 /* Base Address Offset */ #define PCIM_EA_MAX_OFFSET 8 /* MaxOffset (resource length) */ /* bit 0 is reserved */ #define PCIM_EA_IS_64 0x00000002 /* 64-bit field flag */ #define PCIM_EA_FIELD_MASK 0xfffffffc /* For Base & Max Offset */ /* Bridge config register */ #define PCIM_EA_SEC_NR(reg) ((reg) & 0xff) #define PCIM_EA_SUB_NR(reg) (((reg) >> 8) & 0xff) /* PCI-X definitions */ /* For header type 0 devices */ #define PCIXR_COMMAND 0x2 #define PCIXM_COMMAND_DPERR_E 0x0001 /* Data Parity Error Recovery */ #define PCIXM_COMMAND_ERO 0x0002 /* Enable Relaxed Ordering */ #define PCIXM_COMMAND_MAX_READ 0x000c /* Maximum Burst Read Count */ #define PCIXM_COMMAND_MAX_READ_512 0x0000 #define PCIXM_COMMAND_MAX_READ_1024 0x0004 #define PCIXM_COMMAND_MAX_READ_2048 0x0008 #define PCIXM_COMMAND_MAX_READ_4096 0x000c #define PCIXM_COMMAND_MAX_SPLITS 0x0070 /* Maximum Split Transactions */ #define PCIXM_COMMAND_MAX_SPLITS_1 0x0000 #define PCIXM_COMMAND_MAX_SPLITS_2 0x0010 #define PCIXM_COMMAND_MAX_SPLITS_3 0x0020 #define PCIXM_COMMAND_MAX_SPLITS_4 0x0030 #define PCIXM_COMMAND_MAX_SPLITS_8 0x0040 #define PCIXM_COMMAND_MAX_SPLITS_12 0x0050 #define PCIXM_COMMAND_MAX_SPLITS_16 0x0060 #define PCIXM_COMMAND_MAX_SPLITS_32 0x0070 #define PCIXM_COMMAND_VERSION 0x3000 #define PCIXR_STATUS 0x4 #define PCIXM_STATUS_DEVFN 0x000000FF #define PCIXM_STATUS_BUS 0x0000FF00 #define PCIXM_STATUS_64BIT 0x00010000 #define PCIXM_STATUS_133CAP 0x00020000 #define PCIXM_STATUS_SC_DISCARDED 0x00040000 #define PCIXM_STATUS_UNEXP_SC 0x00080000 #define PCIXM_STATUS_COMPLEX_DEV 0x00100000 #define PCIXM_STATUS_MAX_READ 0x00600000 #define PCIXM_STATUS_MAX_READ_512 0x00000000 #define PCIXM_STATUS_MAX_READ_1024 0x00200000 #define PCIXM_STATUS_MAX_READ_2048 0x00400000 #define PCIXM_STATUS_MAX_READ_4096 0x00600000 #define PCIXM_STATUS_MAX_SPLITS 0x03800000 #define PCIXM_STATUS_MAX_SPLITS_1 0x00000000 #define PCIXM_STATUS_MAX_SPLITS_2 0x00800000 #define PCIXM_STATUS_MAX_SPLITS_3 0x01000000 #define PCIXM_STATUS_MAX_SPLITS_4 0x01800000 #define PCIXM_STATUS_MAX_SPLITS_8 0x02000000 #define PCIXM_STATUS_MAX_SPLITS_12 0x02800000 #define PCIXM_STATUS_MAX_SPLITS_16 0x03000000 #define PCIXM_STATUS_MAX_SPLITS_32 0x03800000 #define PCIXM_STATUS_MAX_CUM_READ 0x1C000000 #define PCIXM_STATUS_RCVD_SC_ERR 0x20000000 #define PCIXM_STATUS_266CAP 0x40000000 #define PCIXM_STATUS_533CAP 0x80000000 /* For header type 1 devices (PCI-X bridges) */ #define PCIXR_SEC_STATUS 0x2 #define PCIXM_SEC_STATUS_64BIT 0x0001 #define PCIXM_SEC_STATUS_133CAP 0x0002 #define PCIXM_SEC_STATUS_SC_DISC 0x0004 #define PCIXM_SEC_STATUS_UNEXP_SC 0x0008 #define PCIXM_SEC_STATUS_SC_OVERRUN 0x0010 #define PCIXM_SEC_STATUS_SR_DELAYED 0x0020 #define PCIXM_SEC_STATUS_BUS_MODE 0x03c0 #define PCIXM_SEC_STATUS_VERSION 0x3000 #define PCIXM_SEC_STATUS_266CAP 0x4000 #define PCIXM_SEC_STATUS_533CAP 0x8000 #define PCIXR_BRIDGE_STATUS 0x4 #define PCIXM_BRIDGE_STATUS_DEVFN 0x000000FF #define PCIXM_BRIDGE_STATUS_BUS 0x0000FF00 #define PCIXM_BRIDGE_STATUS_64BIT 0x00010000 #define PCIXM_BRIDGE_STATUS_133CAP 0x00020000 #define PCIXM_BRIDGE_STATUS_SC_DISCARDED 0x00040000 #define PCIXM_BRIDGE_STATUS_UNEXP_SC 0x00080000 #define PCIXM_BRIDGE_STATUS_SC_OVERRUN 0x00100000 #define PCIXM_BRIDGE_STATUS_SR_DELAYED 0x00200000 #define PCIXM_BRIDGE_STATUS_DEVID_MSGCAP 0x20000000 #define PCIXM_BRIDGE_STATUS_266CAP 0x40000000 #define PCIXM_BRIDGE_STATUS_533CAP 0x80000000 /* HT (HyperTransport) Capability definitions */ #define PCIR_HT_COMMAND 0x2 #define PCIM_HTCMD_CAP_MASK 0xf800 /* Capability type. */ #define PCIM_HTCAP_SLAVE 0x0000 /* 000xx */ #define PCIM_HTCAP_HOST 0x2000 /* 001xx */ #define PCIM_HTCAP_SWITCH 0x4000 /* 01000 */ #define PCIM_HTCAP_INTERRUPT 0x8000 /* 10000 */ #define PCIM_HTCAP_REVISION_ID 0x8800 /* 10001 */ #define PCIM_HTCAP_UNITID_CLUMPING 0x9000 /* 10010 */ #define PCIM_HTCAP_EXT_CONFIG_SPACE 0x9800 /* 10011 */ #define PCIM_HTCAP_ADDRESS_MAPPING 0xa000 /* 10100 */ #define PCIM_HTCAP_MSI_MAPPING 0xa800 /* 10101 */ #define PCIM_HTCAP_DIRECT_ROUTE 0xb000 /* 10110 */ #define PCIM_HTCAP_VCSET 0xb800 /* 10111 */ #define PCIM_HTCAP_RETRY_MODE 0xc000 /* 11000 */ #define PCIM_HTCAP_X86_ENCODING 0xc800 /* 11001 */ #define PCIM_HTCAP_GEN3 0xd000 /* 11010 */ #define PCIM_HTCAP_FLE 0xd800 /* 11011 */ #define PCIM_HTCAP_PM 0xe000 /* 11100 */ #define PCIM_HTCAP_HIGH_NODE_COUNT 0xe800 /* 11101 */ /* HT MSI Mapping Capability definitions. */ #define PCIM_HTCMD_MSI_ENABLE 0x0001 #define PCIM_HTCMD_MSI_FIXED 0x0002 #define PCIR_HTMSI_ADDRESS_LO 0x4 #define PCIR_HTMSI_ADDRESS_HI 0x8 /* PCI Vendor capability definitions */ #define PCIR_VENDOR_LENGTH 0x2 #define PCIR_VENDOR_DATA 0x3 /* PCI Device capability definitions */ #define PCIR_DEVICE_LENGTH 0x2 /* PCI EHCI Debug Port definitions */ #define PCIR_DEBUG_PORT 0x2 #define PCIM_DEBUG_PORT_OFFSET 0x1FFF #define PCIM_DEBUG_PORT_BAR 0xe000 /* PCI-PCI Bridge Subvendor definitions */ #define PCIR_SUBVENDCAP_ID 0x4 /* PCI Express definitions */ #define PCIER_FLAGS 0x2 #define PCIEM_FLAGS_VERSION 0x000F #define PCIEM_FLAGS_TYPE 0x00F0 #define PCIEM_TYPE_ENDPOINT 0x0000 #define PCIEM_TYPE_LEGACY_ENDPOINT 0x0010 #define PCIEM_TYPE_ROOT_PORT 0x0040 #define PCIEM_TYPE_UPSTREAM_PORT 0x0050 #define PCIEM_TYPE_DOWNSTREAM_PORT 0x0060 #define PCIEM_TYPE_PCI_BRIDGE 0x0070 #define PCIEM_TYPE_PCIE_BRIDGE 0x0080 #define PCIEM_TYPE_ROOT_INT_EP 0x0090 #define PCIEM_TYPE_ROOT_EC 0x00a0 #define PCIEM_FLAGS_SLOT 0x0100 #define PCIEM_FLAGS_IRQ 0x3e00 #define PCIER_DEVICE_CAP 0x4 #define PCIEM_CAP_MAX_PAYLOAD 0x00000007 #define PCIEM_CAP_PHANTHOM_FUNCS 0x00000018 #define PCIEM_CAP_EXT_TAG_FIELD 0x00000020 #define PCIEM_CAP_L0S_LATENCY 0x000001c0 #define PCIEM_CAP_L1_LATENCY 0x00000e00 #define PCIEM_CAP_ROLE_ERR_RPT 0x00008000 #define PCIEM_CAP_SLOT_PWR_LIM_VAL 0x03fc0000 #define PCIEM_CAP_SLOT_PWR_LIM_SCALE 0x0c000000 #define PCIEM_CAP_FLR 0x10000000 #define PCIER_DEVICE_CTL 0x8 #define PCIEM_CTL_COR_ENABLE 0x0001 #define PCIEM_CTL_NFER_ENABLE 0x0002 #define PCIEM_CTL_FER_ENABLE 0x0004 #define PCIEM_CTL_URR_ENABLE 0x0008 #define PCIEM_CTL_RELAXED_ORD_ENABLE 0x0010 #define PCIEM_CTL_MAX_PAYLOAD 0x00e0 #define PCIEM_CTL_EXT_TAG_FIELD 0x0100 #define PCIEM_CTL_PHANTHOM_FUNCS 0x0200 #define PCIEM_CTL_AUX_POWER_PM 0x0400 #define PCIEM_CTL_NOSNOOP_ENABLE 0x0800 #define PCIEM_CTL_MAX_READ_REQUEST 0x7000 #define PCIEM_CTL_BRDG_CFG_RETRY 0x8000 /* PCI-E - PCI/PCI-X bridges */ #define PCIEM_CTL_INITIATE_FLR 0x8000 /* FLR capable endpoints */ #define PCIER_DEVICE_STA 0xa #define PCIEM_STA_CORRECTABLE_ERROR 0x0001 #define PCIEM_STA_NON_FATAL_ERROR 0x0002 #define PCIEM_STA_FATAL_ERROR 0x0004 #define PCIEM_STA_UNSUPPORTED_REQ 0x0008 #define PCIEM_STA_AUX_POWER 0x0010 #define PCIEM_STA_TRANSACTION_PND 0x0020 #define PCIER_LINK_CAP 0xc #define PCIEM_LINK_CAP_MAX_SPEED 0x0000000f #define PCIEM_LINK_CAP_MAX_WIDTH 0x000003f0 #define PCIEM_LINK_CAP_ASPM 0x00000c00 #define PCIEM_LINK_CAP_L0S_EXIT 0x00007000 #define PCIEM_LINK_CAP_L1_EXIT 0x00038000 #define PCIEM_LINK_CAP_CLOCK_PM 0x00040000 #define PCIEM_LINK_CAP_SURPRISE_DOWN 0x00080000 #define PCIEM_LINK_CAP_DL_ACTIVE 0x00100000 #define PCIEM_LINK_CAP_LINK_BW_NOTIFY 0x00200000 #define PCIEM_LINK_CAP_ASPM_COMPLIANCE 0x00400000 #define PCIEM_LINK_CAP_PORT 0xff000000 #define PCIER_LINK_CTL 0x10 #define PCIEM_LINK_CTL_ASPMC_DIS 0x0000 #define PCIEM_LINK_CTL_ASPMC_L0S 0x0001 #define PCIEM_LINK_CTL_ASPMC_L1 0x0002 #define PCIEM_LINK_CTL_ASPMC 0x0003 #define PCIEM_LINK_CTL_RCB 0x0008 #define PCIEM_LINK_CTL_LINK_DIS 0x0010 #define PCIEM_LINK_CTL_RETRAIN_LINK 0x0020 #define PCIEM_LINK_CTL_COMMON_CLOCK 0x0040 #define PCIEM_LINK_CTL_EXTENDED_SYNC 0x0080 #define PCIEM_LINK_CTL_ECPM 0x0100 #define PCIEM_LINK_CTL_HAWD 0x0200 #define PCIEM_LINK_CTL_LBMIE 0x0400 #define PCIEM_LINK_CTL_LABIE 0x0800 #define PCIER_LINK_STA 0x12 #define PCIEM_LINK_STA_SPEED 0x000f #define PCIEM_LINK_STA_WIDTH 0x03f0 #define PCIEM_LINK_STA_TRAINING_ERROR 0x0400 #define PCIEM_LINK_STA_TRAINING 0x0800 #define PCIEM_LINK_STA_SLOT_CLOCK 0x1000 #define PCIEM_LINK_STA_DL_ACTIVE 0x2000 #define PCIEM_LINK_STA_LINK_BW_MGMT 0x4000 #define PCIEM_LINK_STA_LINK_AUTO_BW 0x8000 #define PCIER_SLOT_CAP 0x14 #define PCIEM_SLOT_CAP_APB 0x00000001 #define PCIEM_SLOT_CAP_PCP 0x00000002 #define PCIEM_SLOT_CAP_MRLSP 0x00000004 #define PCIEM_SLOT_CAP_AIP 0x00000008 #define PCIEM_SLOT_CAP_PIP 0x00000010 #define PCIEM_SLOT_CAP_HPS 0x00000020 #define PCIEM_SLOT_CAP_HPC 0x00000040 #define PCIEM_SLOT_CAP_SPLV 0x00007f80 #define PCIEM_SLOT_CAP_SPLS 0x00018000 #define PCIEM_SLOT_CAP_EIP 0x00020000 #define PCIEM_SLOT_CAP_NCCS 0x00040000 #define PCIEM_SLOT_CAP_PSN 0xfff80000 #define PCIER_SLOT_CTL 0x18 #define PCIEM_SLOT_CTL_ABPE 0x0001 #define PCIEM_SLOT_CTL_PFDE 0x0002 #define PCIEM_SLOT_CTL_MRLSCE 0x0004 #define PCIEM_SLOT_CTL_PDCE 0x0008 #define PCIEM_SLOT_CTL_CCIE 0x0010 #define PCIEM_SLOT_CTL_HPIE 0x0020 #define PCIEM_SLOT_CTL_AIC 0x00c0 #define PCIEM_SLOT_CTL_AI_ON 0x0040 #define PCIEM_SLOT_CTL_AI_BLINK 0x0080 #define PCIEM_SLOT_CTL_AI_OFF 0x00c0 #define PCIEM_SLOT_CTL_PIC 0x0300 #define PCIEM_SLOT_CTL_PI_ON 0x0100 #define PCIEM_SLOT_CTL_PI_BLINK 0x0200 #define PCIEM_SLOT_CTL_PI_OFF 0x0300 #define PCIEM_SLOT_CTL_PCC 0x0400 #define PCIEM_SLOT_CTL_PC_ON 0x0000 #define PCIEM_SLOT_CTL_PC_OFF 0x0400 #define PCIEM_SLOT_CTL_EIC 0x0800 #define PCIEM_SLOT_CTL_DLLSCE 0x1000 #define PCIER_SLOT_STA 0x1a #define PCIEM_SLOT_STA_ABP 0x0001 #define PCIEM_SLOT_STA_PFD 0x0002 #define PCIEM_SLOT_STA_MRLSC 0x0004 #define PCIEM_SLOT_STA_PDC 0x0008 #define PCIEM_SLOT_STA_CC 0x0010 #define PCIEM_SLOT_STA_MRLSS 0x0020 #define PCIEM_SLOT_STA_PDS 0x0040 #define PCIEM_SLOT_STA_EIS 0x0080 #define PCIEM_SLOT_STA_DLLSC 0x0100 #define PCIER_ROOT_CTL 0x1c #define PCIEM_ROOT_CTL_SERR_CORR 0x0001 #define PCIEM_ROOT_CTL_SERR_NONFATAL 0x0002 #define PCIEM_ROOT_CTL_SERR_FATAL 0x0004 #define PCIEM_ROOT_CTL_PME 0x0008 #define PCIEM_ROOT_CTL_CRS_VIS 0x0010 #define PCIER_ROOT_CAP 0x1e #define PCIEM_ROOT_CAP_CRS_VIS 0x0001 #define PCIER_ROOT_STA 0x20 #define PCIEM_ROOT_STA_PME_REQID_MASK 0x0000ffff #define PCIEM_ROOT_STA_PME_STATUS 0x00010000 #define PCIEM_ROOT_STA_PME_PEND 0x00020000 #define PCIER_DEVICE_CAP2 0x24 #define PCIEM_CAP2_COMP_TIMO_RANGES 0x0000000f #define PCIEM_CAP2_COMP_TIMO_RANGE_A 0x00000001 #define PCIEM_CAP2_COMP_TIMO_RANGE_B 0x00000002 #define PCIEM_CAP2_COMP_TIMO_RANGE_C 0x00000004 #define PCIEM_CAP2_COMP_TIMO_RANGE_D 0x00000008 #define PCIEM_CAP2_COMP_TIMO_DISABLE 0x00000010 #define PCIEM_CAP2_ARI 0x00000020 #define PCIER_DEVICE_CTL2 0x28 #define PCIEM_CTL2_COMP_TIMO_VAL 0x000f #define PCIEM_CTL2_COMP_TIMO_50MS 0x0000 #define PCIEM_CTL2_COMP_TIMO_100US 0x0001 #define PCIEM_CTL2_COMP_TIMO_10MS 0x0002 #define PCIEM_CTL2_COMP_TIMO_55MS 0x0005 #define PCIEM_CTL2_COMP_TIMO_210MS 0x0006 #define PCIEM_CTL2_COMP_TIMO_900MS 0x0009 #define PCIEM_CTL2_COMP_TIMO_3500MS 0x000a #define PCIEM_CTL2_COMP_TIMO_13S 0x000d #define PCIEM_CTL2_COMP_TIMO_64S 0x000e #define PCIEM_CTL2_COMP_TIMO_DISABLE 0x0010 #define PCIEM_CTL2_ARI 0x0020 #define PCIEM_CTL2_ATOMIC_REQ_ENABLE 0x0040 #define PCIEM_CTL2_ATOMIC_EGR_BLOCK 0x0080 #define PCIEM_CTL2_ID_ORDERED_REQ_EN 0x0100 #define PCIEM_CTL2_ID_ORDERED_CMP_EN 0x0200 #define PCIEM_CTL2_LTR_ENABLE 0x0400 #define PCIEM_CTL2_OBFF 0x6000 #define PCIEM_OBFF_DISABLE 0x0000 #define PCIEM_OBFF_MSGA_ENABLE 0x2000 #define PCIEM_OBFF_MSGB_ENABLE 0x4000 #define PCIEM_OBFF_WAKE_ENABLE 0x6000 #define PCIEM_CTL2_END2END_TLP 0x8000 #define PCIER_DEVICE_STA2 0x2a #define PCIER_LINK_CAP2 0x2c #define PCIER_LINK_CTL2 0x30 #define PCIER_LINK_STA2 0x32 #define PCIER_SLOT_CAP2 0x34 #define PCIER_SLOT_CTL2 0x38 #define PCIER_SLOT_STA2 0x3a /* MSI-X definitions */ #define PCIR_MSIX_CTRL 0x2 #define PCIM_MSIXCTRL_MSIX_ENABLE 0x8000 #define PCIM_MSIXCTRL_FUNCTION_MASK 0x4000 #define PCIM_MSIXCTRL_TABLE_SIZE 0x07FF +#define PCI_MSIX_MSGNUM(ctrl) (((ctrl) & PCIM_MSIXCTRL_TABLE_SIZE) + 1) #define PCIR_MSIX_TABLE 0x4 #define PCIR_MSIX_PBA 0x8 #define PCIM_MSIX_BIR_MASK 0x7 #define PCIM_MSIX_BIR_BAR_10 0 #define PCIM_MSIX_BIR_BAR_14 1 #define PCIM_MSIX_BIR_BAR_18 2 #define PCIM_MSIX_BIR_BAR_1C 3 #define PCIM_MSIX_BIR_BAR_20 4 #define PCIM_MSIX_BIR_BAR_24 5 #define PCIM_MSIX_VCTRL_MASK 0x1 /* PCI Advanced Features definitions */ #define PCIR_PCIAF_CAP 0x3 #define PCIM_PCIAFCAP_TP 0x01 #define PCIM_PCIAFCAP_FLR 0x02 #define PCIR_PCIAF_CTRL 0x4 #define PCIR_PCIAFCTRL_FLR 0x01 #define PCIR_PCIAF_STATUS 0x5 #define PCIR_PCIAFSTATUS_TP 0x01 /* Advanced Error Reporting */ #define PCIR_AER_UC_STATUS 0x04 #define PCIM_AER_UC_TRAINING_ERROR 0x00000001 #define PCIM_AER_UC_DL_PROTOCOL_ERROR 0x00000010 #define PCIM_AER_UC_SURPRISE_LINK_DOWN 0x00000020 #define PCIM_AER_UC_POISONED_TLP 0x00001000 #define PCIM_AER_UC_FC_PROTOCOL_ERROR 0x00002000 #define PCIM_AER_UC_COMPLETION_TIMEOUT 0x00004000 #define PCIM_AER_UC_COMPLETER_ABORT 0x00008000 #define PCIM_AER_UC_UNEXPECTED_COMPLETION 0x00010000 #define PCIM_AER_UC_RECEIVER_OVERFLOW 0x00020000 #define PCIM_AER_UC_MALFORMED_TLP 0x00040000 #define PCIM_AER_UC_ECRC_ERROR 0x00080000 #define PCIM_AER_UC_UNSUPPORTED_REQUEST 0x00100000 #define PCIM_AER_UC_ACS_VIOLATION 0x00200000 #define PCIM_AER_UC_INTERNAL_ERROR 0x00400000 #define PCIM_AER_UC_MC_BLOCKED_TLP 0x00800000 #define PCIM_AER_UC_ATOMIC_EGRESS_BLK 0x01000000 #define PCIM_AER_UC_TLP_PREFIX_BLOCKED 0x02000000 #define PCIR_AER_UC_MASK 0x08 /* Shares bits with UC_STATUS */ #define PCIR_AER_UC_SEVERITY 0x0c /* Shares bits with UC_STATUS */ #define PCIR_AER_COR_STATUS 0x10 #define PCIM_AER_COR_RECEIVER_ERROR 0x00000001 #define PCIM_AER_COR_BAD_TLP 0x00000040 #define PCIM_AER_COR_BAD_DLLP 0x00000080 #define PCIM_AER_COR_REPLAY_ROLLOVER 0x00000100 #define PCIM_AER_COR_REPLAY_TIMEOUT 0x00001000 #define PCIM_AER_COR_ADVISORY_NF_ERROR 0x00002000 #define PCIM_AER_COR_INTERNAL_ERROR 0x00004000 #define PCIM_AER_COR_HEADER_LOG_OVFLOW 0x00008000 #define PCIR_AER_COR_MASK 0x14 /* Shares bits with COR_STATUS */ #define PCIR_AER_CAP_CONTROL 0x18 #define PCIM_AER_FIRST_ERROR_PTR 0x0000001f #define PCIM_AER_ECRC_GEN_CAPABLE 0x00000020 #define PCIM_AER_ECRC_GEN_ENABLE 0x00000040 #define PCIM_AER_ECRC_CHECK_CAPABLE 0x00000080 #define PCIM_AER_ECRC_CHECK_ENABLE 0x00000100 #define PCIM_AER_MULT_HDR_CAPABLE 0x00000200 #define PCIM_AER_MULT_HDR_ENABLE 0x00000400 #define PCIM_AER_TLP_PREFIX_LOG_PRESENT 0x00000800 #define PCIR_AER_HEADER_LOG 0x1c #define PCIR_AER_ROOTERR_CMD 0x2c /* Only for root complex ports */ #define PCIM_AER_ROOTERR_COR_ENABLE 0x00000001 #define PCIM_AER_ROOTERR_NF_ENABLE 0x00000002 #define PCIM_AER_ROOTERR_F_ENABLE 0x00000004 #define PCIR_AER_ROOTERR_STATUS 0x30 /* Only for root complex ports */ #define PCIM_AER_ROOTERR_COR_ERR 0x00000001 #define PCIM_AER_ROOTERR_MULTI_COR_ERR 0x00000002 #define PCIM_AER_ROOTERR_UC_ERR 0x00000004 #define PCIM_AER_ROOTERR_MULTI_UC_ERR 0x00000008 #define PCIM_AER_ROOTERR_FIRST_UC_FATAL 0x00000010 #define PCIM_AER_ROOTERR_NF_ERR 0x00000020 #define PCIM_AER_ROOTERR_F_ERR 0x00000040 #define PCIM_AER_ROOTERR_INT_MESSAGE 0xf8000000 #define PCIR_AER_COR_SOURCE_ID 0x34 /* Only for root complex ports */ #define PCIR_AER_ERR_SOURCE_ID 0x36 /* Only for root complex ports */ #define PCIR_AER_TLP_PREFIX_LOG 0x38 /* Only for TLP prefix functions */ /* Virtual Channel definitions */ #define PCIR_VC_CAP1 0x04 #define PCIM_VC_CAP1_EXT_COUNT 0x00000007 #define PCIM_VC_CAP1_LOWPRI_EXT_COUNT 0x00000070 #define PCIR_VC_CAP2 0x08 #define PCIR_VC_CONTROL 0x0C #define PCIR_VC_STATUS 0x0E #define PCIR_VC_RESOURCE_CAP(n) (0x10 + (n) * 0x0C) #define PCIR_VC_RESOURCE_CTL(n) (0x14 + (n) * 0x0C) #define PCIR_VC_RESOURCE_STA(n) (0x18 + (n) * 0x0C) /* Serial Number definitions */ #define PCIR_SERIAL_LOW 0x04 #define PCIR_SERIAL_HIGH 0x08 /* SR-IOV definitions */ #define PCIR_SRIOV_CTL 0x08 #define PCIM_SRIOV_VF_EN 0x01 #define PCIM_SRIOV_VF_MSE 0x08 /* Memory space enable. */ #define PCIM_SRIOV_ARI_EN 0x10 #define PCIR_SRIOV_TOTAL_VFS 0x0E #define PCIR_SRIOV_NUM_VFS 0x10 #define PCIR_SRIOV_VF_OFF 0x14 #define PCIR_SRIOV_VF_STRIDE 0x16 #define PCIR_SRIOV_VF_DID 0x1A #define PCIR_SRIOV_PAGE_CAP 0x1C #define PCIR_SRIOV_PAGE_SIZE 0x20 #define PCI_SRIOV_BASE_PAGE_SHIFT 12 #define PCIR_SRIOV_BARS 0x24 #define PCIR_SRIOV_BAR(x) (PCIR_SRIOV_BARS + (x) * 4) /* Extended Capability Vendor-Specific definitions */ #define PCIR_VSEC_HEADER 0x04 #define PCIR_VSEC_ID(hdr) ((hdr) & 0xffff) #define PCIR_VSEC_REV(hdr) (((hdr) & 0xf0000) >> 16) #define PCIR_VSEC_LENGTH(hdr) (((hdr) & 0xfff00000) >> 20) #define PCIR_VSEC_DATA 0x08 /* * PCI Express Firmware Interface definitions */ #define PCI_OSC_STATUS 0 #define PCI_OSC_SUPPORT 1 #define PCIM_OSC_SUPPORT_EXT_PCI_CONF 0x01 /* Extended PCI Config Space */ #define PCIM_OSC_SUPPORT_ASPM 0x02 /* Active State Power Management */ #define PCIM_OSC_SUPPORT_CPMC 0x04 /* Clock Power Management Cap */ #define PCIM_OSC_SUPPORT_SEG_GROUP 0x08 /* PCI Segment Groups supported */ #define PCIM_OSC_SUPPORT_MSI 0x10 /* MSI signalling supported */ #define PCI_OSC_CTL 2 #define PCIM_OSC_CTL_PCIE_HP 0x01 /* PCIe Native Hot Plug */ #define PCIM_OSC_CTL_SHPC_HP 0x02 /* SHPC Native Hot Plug */ #define PCIM_OSC_CTL_PCIE_PME 0x04 /* PCIe Native Power Mgt Events */ #define PCIM_OSC_CTL_PCIE_AER 0x08 /* PCIe Advanced Error Reporting */ #define PCIM_OSC_CTL_PCIE_CAP_STRUCT 0x10 /* Various Capability Structures */ /* Access Control Services (ACS) definitions */ #define PCIR_ACS_CAP 0x4 #define PCIM_ACS_SOURCE_VALIDATION 0x0001 #define PCIM_ACS_TRANSLATION_BLOCKING 0x0002 #define PCIM_ACS_P2P_REQ_REDIRECT 0x0004 #define PCIM_ACS_P2P_CMP_REDIRECT 0x0008 #define PCIM_ACS_P2P_UPSTREAM_FORWARDING 0x0010 #define PCIM_ACS_P2P_EGRESS_CTL 0x0020 #define PCIM_ACS_P2P_DIRECT_TRANSLATED 0x0040 #define PCIM_ACS_ENHANCED_CAP 0x0080 #define PCIM_ACS_EGRESS_CTL_VECTOR_SIZE 0xff00 #define PCIR_ACS_CTL 0x6 #define PCIM_ACS_SOURCE_VALIDATION_ENABLE 0x0001 #define PCIM_ACS_TRANSLATION_BLOCKING_ENABLE 0x0002 #define PCIM_ACS_P2P_REQ_REDIRECT_ENABLE 0x0004 #define PCIM_ACS_P2P_CMP_REDIRECT_ENABLE 0x0008 #define PCIM_ACS_P2P_UPSTREAM_FORWARDING_ENABLE 0x0010 #define PCIM_ACS_P2P_EGRESS_CTL_ENABLE 0x0020 #define PCIM_ACS_P2P_DIRECT_TRANSLATED_ENABLE 0x0040 #define PCIM_ACS_IO_REQ_BLOCKING_ENABLE 0x0080 #define PCIM_ACS_DSP_MEM_TGT_ACC_CTL 0x0300 #define PCIM_ACS_USP_MEM_TGT_ACC_CTL 0x0c00 #define PCIM_ACS_UNCLAIMED_REQ_REDIRECT_CTL 0x1000 #define PCIR_ACS_EGRESS_CONTROL_VECTOR 0x8 /* * AMD IOMMU Base Capability * From AMD I/O Virtualization Technology (IOMMU) Specification * Publication # 48882 Revision: 3.09-PUB Date: October 2023 */ #define PCIR_AMDIOMMU_CAP_HEADER 0x0000 #define PCIR_AMDIOMMU_BASE_LOW 0x0004 #define PCIR_AMDIOMMU_BASE_HIGH 0x0008 #define PCIR_AMDIOMMU_RANGE 0x000c #define PCIR_AMDIOMMU_MISC0 0x0010 #define PCIR_AMDIOMMU_MISC1 0x0014 #define PCIM_AMDIOMMU_CAP_CAPEXT (1 << 28) #define PCIM_AMDIOMMU_CAP_EFR (1 << 27) #define PCIM_AMDIOMMU_CAP_NPCACHE (1 << 26) #define PCIM_AMDIOMMU_CAP_HTTUN (1 << 25) #define PCIM_AMDIOMMU_CAP_IOTLB (1 << 24) #define PCIM_AMDIOMMU_CAP_REV_MASK (0x1f << 19) #define PCIM_AMDIOMMU_CAP_REV_VAL (0x1 << 19) #define PCIM_AMDIOMMU_CAP_TYPE_MASK (7 << 16) #define PCIM_AMDIOMMU_CAP_TYPE_VAL (0x3 << 16) #define PCIM_AMDIOMMU_BASE_LOW_EN 0x00000001 #define PCIM_AMDIOMMU_BASE_LOW_ADDRM 0xffffc000 #define PCIM_AMDIOMMU_RANGE_UNITID_MASK 0x1f #define PCIM_AMDIOMMU_RANGE_RNGVALID (1 << 7) #define PCIM_AMDIOMMU_RANGE_BUSNUM_MASK (0xffffu << 8) #define PCIM_AMDIOMMU_RANGE_FIRSTDEV_MASK (0xffffu << 16) #define PCIM_AMDIOMMU_RANGE_LASTDEV_MASK (0xffffu << 24) #define PCIM_AMDIOMMU_MISC0_MSINUMPPR_MASK (0x1f << 27) #define PCIM_AMDIOMMU_MISC0_HTATSRESV (1 << 22) #define PCIM_AMDIOMMU_MISC0_VASIZE_MASK (0x7f << 15) #define PCIM_AMDIOMMU_MISC0_PASIZE_MASK (0x7f << 8) #define PCIM_AMDIOMMU_MISC0_GVASIZE_MASK (0x3 << 5) #define PCIM_AMDIOMMU_MISC0_MSINUM_MASK 0x1f #define PCIM_AMDIOMMU_MISC0_VASIZE_32 (0x20 << 15) #define PCIM_AMDIOMMU_MISC0_VASIZE_40 (0x28 << 15) #define PCIM_AMDIOMMU_MISC0_VASIZE_48 (0x30 << 15) #define PCIM_AMDIOMMU_MISC0_VASIZE_64 (0x40 << 15) #define PCIM_AMDIOMMU_MISC0_PASIZE_40 (0x28 << 8) #define PCIM_AMDIOMMU_MISC0_PASIZE_48 (0x30 << 8) #define PCIM_AMDIOMMU_MISC0_PASIZE_52 (0x34 << 8) #define PCIM_AMDIOMMU_MISC0_GVASIZE_48 (0x2 << 5) #define PCIM_AMDIOMMU_MISC0_GVASIZE_57 (0x3 << 5) #define PCIM_AMDIOMMU_MISC1_MSINUMGA_MASK 0x1f #endif /* __PCI_PCIREG_H */ diff --git a/sys/dev/pci/pcivar.h b/sys/dev/pci/pcivar.h index 37d7daff37f7..888159d59c1a 100644 --- a/sys/dev/pci/pcivar.h +++ b/sys/dev/pci/pcivar.h @@ -1,739 +1,737 @@ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright 1997, Stefan Esser * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice unmodified, this list of conditions, and the following * disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * */ #ifndef _PCIVAR_H_ #define _PCIVAR_H_ #include #include /* some PCI bus constants */ #define PCI_MAXMAPS_0 6 /* max. no. of memory/port maps */ #define PCI_MAXMAPS_1 2 /* max. no. of maps for PCI to PCI bridge */ #define PCI_MAXMAPS_2 1 /* max. no. of maps for CardBus bridge */ typedef uint64_t pci_addr_t; /* Config registers for PCI-PCI and PCI-Cardbus bridges. */ struct pcicfg_bridge { uint8_t br_seclat; uint8_t br_subbus; uint8_t br_secbus; uint8_t br_pribus; uint16_t br_control; }; /* Interesting values for PCI power management */ struct pcicfg_pp { uint16_t pp_cap; /* PCI power management capabilities */ uint8_t pp_status; /* conf. space addr. of PM control/status reg */ uint8_t pp_bse; /* conf. space addr. of PM BSE reg */ uint8_t pp_data; /* conf. space addr. of PM data reg */ }; struct pci_map { pci_addr_t pm_value; /* Raw BAR value */ pci_addr_t pm_size; uint16_t pm_reg; STAILQ_ENTRY(pci_map) pm_link; }; struct vpd_readonly { char keyword[2]; char *value; int len; }; struct vpd_write { char keyword[2]; char *value; int start; int len; }; struct pcicfg_vpd { uint8_t vpd_reg; /* base register, + 2 for addr, + 4 data */ char vpd_cached; char *vpd_ident; /* string identifier */ int vpd_rocnt; struct vpd_readonly *vpd_ros; int vpd_wcnt; struct vpd_write *vpd_w; }; /* Interesting values for PCI MSI */ struct pcicfg_msi { uint16_t msi_ctrl; /* Message Control */ uint8_t msi_location; /* Offset of MSI capability registers. */ - uint8_t msi_msgnum; /* Number of messages */ int msi_alloc; /* Number of allocated messages. */ uint64_t msi_addr; /* Contents of address register. */ uint16_t msi_data; /* Contents of data register. */ u_int msi_handlers; }; /* Interesting values for PCI MSI-X */ struct msix_vector { uint64_t mv_address; /* Contents of address register. */ uint32_t mv_data; /* Contents of data register. */ int mv_irq; }; struct msix_table_entry { u_int mte_vector; /* 1-based index into msix_vectors array. */ u_int mte_handlers; }; struct pcicfg_msix { uint16_t msix_ctrl; /* Message Control */ - uint16_t msix_msgnum; /* Number of messages */ uint8_t msix_location; /* Offset of MSI-X capability registers. */ uint8_t msix_table_bar; /* BAR containing vector table. */ uint8_t msix_pba_bar; /* BAR containing PBA. */ uint32_t msix_table_offset; uint32_t msix_pba_offset; u_int msix_alloc; /* Number of allocated vectors. */ u_int msix_table_len; /* Length of virtual table. */ struct msix_table_entry *msix_table; /* Virtual table. */ struct msix_vector *msix_vectors; /* Array of allocated vectors. */ struct resource *msix_table_res; /* Resource containing vector table. */ struct resource *msix_pba_res; /* Resource containing PBA. */ }; struct pci_id_ofw_iommu { uint32_t id; uint32_t xref; }; /* Interesting values for HyperTransport */ struct pcicfg_ht { uint8_t ht_slave; /* Non-zero if device is an HT slave. */ uint8_t ht_msimap; /* Offset of MSI mapping cap registers. */ uint16_t ht_msictrl; /* MSI mapping control */ uint64_t ht_msiaddr; /* MSI mapping base address */ }; /* Interesting values for PCI-express */ struct pcicfg_pcie { uint8_t pcie_location; /* Offset of PCI-e capability registers. */ uint8_t pcie_type; /* Device type. */ uint16_t pcie_flags; /* Device capabilities register. */ uint16_t pcie_device_ctl; /* Device control register. */ uint16_t pcie_link_ctl; /* Link control register. */ uint16_t pcie_slot_ctl; /* Slot control register. */ uint16_t pcie_root_ctl; /* Root control register. */ uint16_t pcie_device_ctl2; /* Second device control register. */ uint16_t pcie_link_ctl2; /* Second link control register. */ uint16_t pcie_slot_ctl2; /* Second slot control register. */ }; struct pcicfg_pcix { uint16_t pcix_command; uint8_t pcix_location; /* Offset of PCI-X capability registers. */ }; struct pcicfg_vf { int index; }; struct pci_ea_entry { int eae_bei; uint32_t eae_flags; uint64_t eae_base; uint64_t eae_max_offset; uint32_t eae_cfg_offset; STAILQ_ENTRY(pci_ea_entry) eae_link; }; struct pcicfg_ea { int ea_location; /* Structure offset in Configuration Header */ STAILQ_HEAD(, pci_ea_entry) ea_entries; /* EA entries */ }; #define PCICFG_VF 0x0001 /* Device is an SR-IOV Virtual Function */ /* config header information common to all header types */ typedef struct pcicfg { device_t dev; /* device which owns this */ STAILQ_HEAD(, pci_map) maps; /* BARs */ uint16_t subvendor; /* card vendor ID */ uint16_t subdevice; /* card device ID, assigned by card vendor */ uint16_t vendor; /* chip vendor ID */ uint16_t device; /* chip device ID, assigned by chip vendor */ uint16_t cmdreg; /* disable/enable chip and PCI options */ uint16_t statreg; /* supported PCI features and error state */ uint8_t baseclass; /* chip PCI class */ uint8_t subclass; /* chip PCI subclass */ uint8_t progif; /* chip PCI programming interface */ uint8_t revid; /* chip revision ID */ uint8_t hdrtype; /* chip config header type */ uint8_t cachelnsz; /* cache line size in 4byte units */ uint8_t intpin; /* PCI interrupt pin */ uint8_t intline; /* interrupt line (IRQ for PC arch) */ uint8_t mingnt; /* min. useful bus grant time in 250ns units */ uint8_t maxlat; /* max. tolerated bus grant latency in 250ns */ uint8_t lattimer; /* latency timer in units of 30ns bus cycles */ uint8_t mfdev; /* multi-function device (from hdrtype reg) */ uint8_t nummaps; /* actual number of PCI maps used */ uint32_t domain; /* PCI domain */ uint8_t bus; /* config space bus address */ uint8_t slot; /* config space slot address */ uint8_t func; /* config space function number */ uint32_t flags; /* flags defined above */ struct pcicfg_bridge bridge; /* Bridges */ struct pcicfg_pp pp; /* Power management */ struct pcicfg_vpd vpd; /* Vital product data */ struct pcicfg_msi msi; /* PCI MSI */ struct pcicfg_msix msix; /* PCI MSI-X */ struct pcicfg_ht ht; /* HyperTransport */ struct pcicfg_pcie pcie; /* PCI Express */ struct pcicfg_pcix pcix; /* PCI-X */ struct pcicfg_iov *iov; /* SR-IOV */ struct pcicfg_vf vf; /* SR-IOV Virtual Function */ struct pcicfg_ea ea; /* Enhanced Allocation */ } pcicfgregs; /* additional type 1 device config header information (PCI to PCI bridge) */ typedef struct { pci_addr_t pmembase; /* base address of prefetchable memory */ pci_addr_t pmemlimit; /* topmost address of prefetchable memory */ uint32_t membase; /* base address of memory window */ uint32_t memlimit; /* topmost address of memory window */ uint32_t iobase; /* base address of port window */ uint32_t iolimit; /* topmost address of port window */ uint16_t secstat; /* secondary bus status register */ uint16_t bridgectl; /* bridge control register */ uint8_t seclat; /* CardBus latency timer */ } pcih1cfgregs; /* additional type 2 device config header information (CardBus bridge) */ typedef struct { uint32_t membase0; /* base address of memory window */ uint32_t memlimit0; /* topmost address of memory window */ uint32_t membase1; /* base address of memory window */ uint32_t memlimit1; /* topmost address of memory window */ uint32_t iobase0; /* base address of port window */ uint32_t iolimit0; /* topmost address of port window */ uint32_t iobase1; /* base address of port window */ uint32_t iolimit1; /* topmost address of port window */ uint32_t pccardif; /* PC Card 16bit IF legacy more base addr. */ uint16_t secstat; /* secondary bus status register */ uint16_t bridgectl; /* bridge control register */ uint8_t seclat; /* CardBus latency timer */ } pcih2cfgregs; extern uint32_t pci_numdevs; extern int pci_enable_aspm; /* * The bitfield has to be stable and match the fields below (so that * match_flag_vendor must be bit 0) so we have to do the endian dance. We can't * use enums or #define constants because then the macros for subsetting matches * wouldn't work. These tables are parsed by devmatch and others to connect * modules with devices on the PCI bus. */ struct pci_device_table { #if BYTE_ORDER == LITTLE_ENDIAN uint16_t match_flag_vendor:1, match_flag_device:1, match_flag_subvendor:1, match_flag_subdevice:1, match_flag_class:1, match_flag_subclass:1, match_flag_revid:1, match_flag_unused:9; #else uint16_t match_flag_unused:9, match_flag_revid:1, match_flag_subclass:1, match_flag_class:1, match_flag_subdevice:1, match_flag_subvendor:1, match_flag_device:1, match_flag_vendor:1; #endif uint16_t vendor; uint16_t device; uint16_t subvendor; uint16_t subdevice; uint16_t class_id; uint16_t subclass; uint16_t revid; uint16_t unused; uintptr_t driver_data; char *descr; }; #define PCI_DEV(v, d) \ .match_flag_vendor = 1, .vendor = (v), \ .match_flag_device = 1, .device = (d) #define PCI_SUBDEV(sv, sd) \ .match_flag_subvendor = 1, .subvendor = (sv), \ .match_flag_subdevice = 1, .subdevice = (sd) #define PCI_CLASS(x) \ .match_flag_class = 1, .class_id = (x) #define PCI_SUBCLASS(x) \ .match_flag_subclass = 1, .subclass = (x) #define PCI_REVID(x) \ .match_flag_revid = 1, .revid = (x) #define PCI_DESCR(x) \ .descr = (x) #define PCI_PNP_STR \ "M16:mask;U16:vendor;U16:device;U16:subvendor;U16:subdevice;" \ "U16:class;U16:subclass;U16:revid;" #define PCI_PNP_INFO(table) \ MODULE_PNP_INFO(PCI_PNP_STR, pci, table, table, \ sizeof(table) / sizeof(table[0])) const struct pci_device_table *pci_match_device(device_t child, const struct pci_device_table *id, size_t nelt); #define PCI_MATCH(child, table) \ pci_match_device(child, (table), nitems(table)); /* Only if the prerequisites are present */ #if defined(_SYS_BUS_H_) && defined(_SYS_PCIIO_H_) struct pci_devinfo { STAILQ_ENTRY(pci_devinfo) pci_links; struct resource_list resources; pcicfgregs cfg; struct pci_conf conf; }; #endif #ifdef _SYS_BUS_H_ #include "pci_if.h" enum pci_device_ivars { PCI_IVAR_SUBVENDOR, PCI_IVAR_SUBDEVICE, PCI_IVAR_VENDOR, PCI_IVAR_DEVICE, PCI_IVAR_DEVID, PCI_IVAR_CLASS, PCI_IVAR_SUBCLASS, PCI_IVAR_PROGIF, PCI_IVAR_REVID, PCI_IVAR_INTPIN, PCI_IVAR_IRQ, PCI_IVAR_DOMAIN, PCI_IVAR_BUS, PCI_IVAR_SLOT, PCI_IVAR_FUNCTION, PCI_IVAR_ETHADDR, PCI_IVAR_CMDREG, PCI_IVAR_CACHELNSZ, PCI_IVAR_MINGNT, PCI_IVAR_MAXLAT, PCI_IVAR_LATTIMER }; /* * Simplified accessors for pci devices */ #define PCI_ACCESSOR(var, ivar, type) \ __BUS_ACCESSOR(pci, var, PCI, ivar, type) PCI_ACCESSOR(subvendor, SUBVENDOR, uint16_t) PCI_ACCESSOR(subdevice, SUBDEVICE, uint16_t) PCI_ACCESSOR(vendor, VENDOR, uint16_t) PCI_ACCESSOR(device, DEVICE, uint16_t) PCI_ACCESSOR(devid, DEVID, uint32_t) PCI_ACCESSOR(class, CLASS, uint8_t) PCI_ACCESSOR(subclass, SUBCLASS, uint8_t) PCI_ACCESSOR(progif, PROGIF, uint8_t) PCI_ACCESSOR(revid, REVID, uint8_t) PCI_ACCESSOR(intpin, INTPIN, uint8_t) PCI_ACCESSOR(irq, IRQ, uint8_t) PCI_ACCESSOR(domain, DOMAIN, uint32_t) PCI_ACCESSOR(bus, BUS, uint8_t) PCI_ACCESSOR(slot, SLOT, uint8_t) PCI_ACCESSOR(function, FUNCTION, uint8_t) PCI_ACCESSOR(ether, ETHADDR, uint8_t *) PCI_ACCESSOR(cmdreg, CMDREG, uint8_t) PCI_ACCESSOR(cachelnsz, CACHELNSZ, uint8_t) PCI_ACCESSOR(mingnt, MINGNT, uint8_t) PCI_ACCESSOR(maxlat, MAXLAT, uint8_t) PCI_ACCESSOR(lattimer, LATTIMER, uint8_t) #undef PCI_ACCESSOR /* * Operations on configuration space. */ static __inline uint32_t pci_read_config(device_t dev, int reg, int width) { return PCI_READ_CONFIG(device_get_parent(dev), dev, reg, width); } static __inline void pci_write_config(device_t dev, int reg, uint32_t val, int width) { PCI_WRITE_CONFIG(device_get_parent(dev), dev, reg, val, width); } /* * Ivars for pci bridges. */ /*typedef enum pci_device_ivars pcib_device_ivars;*/ enum pcib_device_ivars { PCIB_IVAR_DOMAIN, PCIB_IVAR_BUS }; #define PCIB_ACCESSOR(var, ivar, type) \ __BUS_ACCESSOR(pcib, var, PCIB, ivar, type) PCIB_ACCESSOR(domain, DOMAIN, uint32_t) PCIB_ACCESSOR(bus, BUS, uint32_t) #undef PCIB_ACCESSOR /* * PCI interrupt validation. Invalid interrupt values such as 0 or 128 * on i386 or other platforms should be mapped out in the MD pcireadconf * code and not here, since the only MI invalid IRQ is 255. */ #define PCI_INVALID_IRQ 255 #define PCI_INTERRUPT_VALID(x) ((x) != PCI_INVALID_IRQ) /* * Convenience functions. * * These should be used in preference to manually manipulating * configuration space. */ static __inline int pci_enable_busmaster(device_t dev) { return(PCI_ENABLE_BUSMASTER(device_get_parent(dev), dev)); } static __inline int pci_disable_busmaster(device_t dev) { return(PCI_DISABLE_BUSMASTER(device_get_parent(dev), dev)); } static __inline int pci_enable_io(device_t dev, int space) { return(PCI_ENABLE_IO(device_get_parent(dev), dev, space)); } static __inline int pci_disable_io(device_t dev, int space) { return(PCI_DISABLE_IO(device_get_parent(dev), dev, space)); } static __inline int pci_get_vpd_ident(device_t dev, const char **identptr) { return(PCI_GET_VPD_IDENT(device_get_parent(dev), dev, identptr)); } static __inline int pci_get_vpd_readonly(device_t dev, const char *kw, const char **vptr) { return(PCI_GET_VPD_READONLY(device_get_parent(dev), dev, kw, vptr)); } /* * Check if the address range falls within the VGA defined address range(s) */ static __inline int pci_is_vga_ioport_range(rman_res_t start, rman_res_t end) { return (((start >= 0x3b0 && end <= 0x3bb) || (start >= 0x3c0 && end <= 0x3df)) ? 1 : 0); } static __inline int pci_is_vga_memory_range(rman_res_t start, rman_res_t end) { return ((start >= 0xa0000 && end <= 0xbffff) ? 1 : 0); } /* * PCI power states are as defined by ACPI: * * D0 State in which device is on and running. It is receiving full * power from the system and delivering full functionality to the user. * D1 Class-specific low-power state in which device context may or may not * be lost. Buses in D1 cannot do anything to the bus that would force * devices on that bus to lose context. * D2 Class-specific low-power state in which device context may or may * not be lost. Attains greater power savings than D1. Buses in D2 * can cause devices on that bus to lose some context. Devices in D2 * must be prepared for the bus to be in D2 or higher. * D3 State in which the device is off and not running. Device context is * lost. Power can be removed from the device. */ #define PCI_POWERSTATE_D0 0 #define PCI_POWERSTATE_D1 1 #define PCI_POWERSTATE_D2 2 #define PCI_POWERSTATE_D3 3 #define PCI_POWERSTATE_UNKNOWN -1 static __inline int pci_set_powerstate(device_t dev, int state) { return PCI_SET_POWERSTATE(device_get_parent(dev), dev, state); } static __inline int pci_get_powerstate(device_t dev) { return PCI_GET_POWERSTATE(device_get_parent(dev), dev); } static __inline int pci_find_cap(device_t dev, int capability, int *capreg) { return (PCI_FIND_CAP(device_get_parent(dev), dev, capability, capreg)); } static __inline int pci_find_next_cap(device_t dev, int capability, int start, int *capreg) { return (PCI_FIND_NEXT_CAP(device_get_parent(dev), dev, capability, start, capreg)); } static __inline int pci_find_extcap(device_t dev, int capability, int *capreg) { return (PCI_FIND_EXTCAP(device_get_parent(dev), dev, capability, capreg)); } static __inline int pci_find_next_extcap(device_t dev, int capability, int start, int *capreg) { return (PCI_FIND_NEXT_EXTCAP(device_get_parent(dev), dev, capability, start, capreg)); } static __inline int pci_find_htcap(device_t dev, int capability, int *capreg) { return (PCI_FIND_HTCAP(device_get_parent(dev), dev, capability, capreg)); } static __inline int pci_find_next_htcap(device_t dev, int capability, int start, int *capreg) { return (PCI_FIND_NEXT_HTCAP(device_get_parent(dev), dev, capability, start, capreg)); } static __inline int pci_alloc_msi(device_t dev, int *count) { return (PCI_ALLOC_MSI(device_get_parent(dev), dev, count)); } static __inline int pci_alloc_msix(device_t dev, int *count) { return (PCI_ALLOC_MSIX(device_get_parent(dev), dev, count)); } static __inline void pci_enable_msi(device_t dev, uint64_t address, uint16_t data) { PCI_ENABLE_MSI(device_get_parent(dev), dev, address, data); } static __inline void pci_enable_msix(device_t dev, u_int index, uint64_t address, uint32_t data) { PCI_ENABLE_MSIX(device_get_parent(dev), dev, index, address, data); } static __inline void pci_disable_msi(device_t dev) { PCI_DISABLE_MSI(device_get_parent(dev), dev); } static __inline int pci_remap_msix(device_t dev, int count, const u_int *vectors) { return (PCI_REMAP_MSIX(device_get_parent(dev), dev, count, vectors)); } static __inline int pci_release_msi(device_t dev) { return (PCI_RELEASE_MSI(device_get_parent(dev), dev)); } static __inline int pci_msi_count(device_t dev) { return (PCI_MSI_COUNT(device_get_parent(dev), dev)); } static __inline int pci_msix_count(device_t dev) { return (PCI_MSIX_COUNT(device_get_parent(dev), dev)); } static __inline int pci_msix_pba_bar(device_t dev) { return (PCI_MSIX_PBA_BAR(device_get_parent(dev), dev)); } static __inline int pci_msix_table_bar(device_t dev) { return (PCI_MSIX_TABLE_BAR(device_get_parent(dev), dev)); } static __inline int pci_get_id(device_t dev, enum pci_id_type type, uintptr_t *id) { return (PCI_GET_ID(device_get_parent(dev), dev, type, id)); } /* * This is the deprecated interface, there is no way to tell the difference * between a failure and a valid value that happens to be the same as the * failure value. */ static __inline uint16_t pci_get_rid(device_t dev) { uintptr_t rid; if (pci_get_id(dev, PCI_ID_RID, &rid) != 0) return (0); return (rid); } static __inline void pci_child_added(device_t dev) { return (PCI_CHILD_ADDED(device_get_parent(dev), dev)); } device_t pci_find_bsf(uint8_t, uint8_t, uint8_t); device_t pci_find_dbsf(uint32_t, uint8_t, uint8_t, uint8_t); device_t pci_find_device(uint16_t, uint16_t); device_t pci_find_class(uint8_t class, uint8_t subclass); device_t pci_find_class_from(uint8_t class, uint8_t subclass, device_t devfrom); /* Can be used by drivers to manage the MSI-X table. */ int pci_pending_msix(device_t dev, u_int index); int pci_msi_device_blacklisted(device_t dev); int pci_msix_device_blacklisted(device_t dev); void pci_ht_map_msi(device_t dev, uint64_t addr); device_t pci_find_pcie_root_port(device_t dev); int pci_get_relaxed_ordering_enabled(device_t dev); int pci_get_max_payload(device_t dev); int pci_get_max_read_req(device_t dev); void pci_restore_state(device_t dev); void pci_save_state(device_t dev); int pci_set_max_read_req(device_t dev, int size); int pci_power_reset(device_t dev); uint32_t pcie_read_config(device_t dev, int reg, int width); void pcie_write_config(device_t dev, int reg, uint32_t value, int width); uint32_t pcie_adjust_config(device_t dev, int reg, uint32_t mask, uint32_t value, int width); void pcie_apei_error(device_t dev, int sev, uint8_t *aer); bool pcie_flr(device_t dev, u_int max_delay, bool force); int pcie_get_max_completion_timeout(device_t dev); bool pcie_wait_for_pending_transactions(device_t dev, u_int max_delay); int pcie_link_reset(device_t port, int pcie_location); void pci_print_faulted_dev(void); #endif /* _SYS_BUS_H_ */ /* * cdev switch for control device, initialised in generic PCI code */ extern struct cdevsw pcicdev; /* * List of all PCI devices, generation count for the list. */ STAILQ_HEAD(devlist, pci_devinfo); extern struct devlist pci_devq; extern uint32_t pci_generation; struct pci_map *pci_find_bar(device_t dev, int reg); struct pci_map *pci_first_bar(device_t dev); struct pci_map *pci_next_bar(struct pci_map *pm); int pci_bar_enabled(device_t dev, struct pci_map *pm); struct pcicfg_vpd *pci_fetch_vpd_list(device_t dev); #define VGA_PCI_BIOS_SHADOW_ADDR 0xC0000 #define VGA_PCI_BIOS_SHADOW_SIZE 131072 int vga_pci_is_boot_display(device_t dev); void * vga_pci_map_bios(device_t dev, size_t *size); void vga_pci_unmap_bios(device_t dev, void *bios); int vga_pci_repost(device_t dev); /** * Global eventhandlers invoked when PCI devices are added or removed * from the system. */ typedef void (*pci_event_fn)(void *arg, device_t dev); EVENTHANDLER_DECLARE(pci_add_device, pci_event_fn); EVENTHANDLER_DECLARE(pci_delete_device, pci_event_fn); #endif /* _PCIVAR_H_ */