Index: head/share/man/man9/pci.9 =================================================================== --- head/share/man/man9/pci.9 (revision 299931) +++ head/share/man/man9/pci.9 (revision 299932) @@ -1,919 +1,922 @@ .\" .\" Copyright (c) 2005 Bruce M Simpson .\" All rights reserved. .\" .\" Redistribution and use in source and binary forms, with or without .\" modification, are permitted provided that the following conditions .\" are met: .\" 1. Redistributions of source code must retain the above copyright .\" notice, this list of conditions and the following disclaimer. .\" 2. Redistributions in binary form must reproduce the above copyright .\" notice, this list of conditions and the following disclaimer in the .\" documentation and/or other materials provided with the distribution. .\" .\" THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND .\" ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE .\" IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE .\" ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE .\" FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL .\" DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS .\" OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) .\" HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT .\" LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY .\" OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF .\" SUCH DAMAGE. .\" .\" $FreeBSD$ .\" .Dd May 16, 2016 .Dt PCI 9 .Os .Sh NAME .Nm pci , .Nm pci_alloc_msi , .Nm pci_alloc_msix , .Nm pci_disable_busmaster , .Nm pci_disable_io , .Nm pci_enable_busmaster , .Nm pci_enable_io , .Nm pci_find_bsf , .Nm pci_find_cap , .Nm pci_find_dbsf , .Nm pci_find_device , .Nm pci_find_extcap , .Nm pci_find_htcap , .Nm pci_find_pcie_root_port , .Nm pci_get_id , .Nm pci_get_max_read_req , .Nm pci_get_powerstate , .Nm pci_get_vpd_ident , .Nm pci_get_vpd_readonly , .Nm pci_iov_attach , .Nm pci_iov_detach , .Nm pci_msi_count , .Nm pci_msix_count , .Nm pci_msix_pba_bar , .Nm pci_msix_table_bar , .Nm pci_pending_msix , .Nm pci_read_config , .Nm pci_release_msi , .Nm pci_remap_msix , .Nm pci_restore_state , .Nm pci_save_state , .Nm pci_set_max_read_req , .Nm pci_set_powerstate , .Nm pci_write_config , .Nm pcie_adjust_config , .Nm pcie_read_config , .Nm pcie_write_config .Nd PCI bus interface .Sh SYNOPSIS .In sys/bus.h .In dev/pci/pcireg.h .In dev/pci/pcivar.h .Ft int .Fn pci_alloc_msi "device_t dev" "int *count" .Ft int .Fn pci_alloc_msix "device_t dev" "int *count" .Ft int .Fn pci_disable_busmaster "device_t dev" .Ft int .Fn pci_disable_io "device_t dev" "int space" .Ft int .Fn pci_enable_busmaster "device_t dev" .Ft int .Fn pci_enable_io "device_t dev" "int space" .Ft device_t .Fn pci_find_bsf "uint8_t bus" "uint8_t slot" "uint8_t func" .Ft int .Fn pci_find_cap "device_t dev" "int capability" "int *capreg" .Ft device_t .Fn pci_find_dbsf "uint32_t domain" "uint8_t bus" "uint8_t slot" "uint8_t func" .Ft device_t .Fn pci_find_device "uint16_t vendor" "uint16_t device" .Ft int .Fn pci_find_extcap "device_t dev" "int capability" "int *capreg" .Ft int .Fn pci_find_htcap "device_t dev" "int capability" "int *capreg" .Ft device_t .Fn pci_find_pcie_root_port "device_t dev" .Ft int .Fn pci_get_id "device_t dev" "enum pci_id_type type" "uintptr_t *id" .Ft int .Fn pci_get_max_read_req "device_t dev" .Ft int .Fn pci_get_powerstate "device_t dev" .Ft int .Fn pci_get_vpd_ident "device_t dev" "const char **identptr" .Ft int .Fn pci_get_vpd_readonly "device_t dev" "const char *kw" "const char **vptr" .Ft int .Fn pci_msi_count "device_t dev" .Ft int .Fn pci_msix_count "device_t dev" .Ft int .Fn pci_msix_pba_bar "device_t dev" .Ft int .Fn pci_msix_table_bar "device_t dev" .Ft int .Fn pci_pending_msix "device_t dev" "u_int index" .Ft uint32_t .Fn pci_read_config "device_t dev" "int reg" "int width" .Ft int .Fn pci_release_msi "device_t dev" .Ft int .Fn pci_remap_msix "device_t dev" "int count" "const u_int *vectors" .Ft void .Fn pci_restore_state "device_t dev" .Ft void .Fn pci_save_state "device_t dev" .Ft int .Fn pci_set_max_read_req "device_t dev" "int size" .Ft int .Fn pci_set_powerstate "device_t dev" "int state" .Ft void .Fn pci_write_config "device_t dev" "int reg" "uint32_t val" "int width" .Ft uint32_t .Fo pcie_adjust_config .Fa "device_t dev" .Fa "int reg" .Fa "uint32_t mask" .Fa "uint32_t val" .Fa "int width" .Fc .Ft uint32_t .Fn pcie_read_config "device_t dev" "int reg" "int width" .Ft void .Fn pcie_write_config "device_t dev" "int reg" "uint32_t val" "int width" .In dev/pci/pci_iov.h .Ft int .Fn pci_iov_attach "device_t dev" "nvlist_t *pf_schema" "nvlist_t *vf_schema" .Ft int .Fn pci_iov_detach "device_t dev" .Sh DESCRIPTION The .Nm set of functions are used for managing PCI devices. The functions are split into several groups: raw configuration access, locating devices, device information, device configuration, and message signaled interrupts. .Ss Raw Configuration Access The .Fn pci_read_config function is used to read data from the PCI configuration space of the device .Fa dev , at offset .Fa reg , with .Fa width specifying the size of the access. .Pp The .Fn pci_write_config function is used to write the value .Fa val to the PCI configuration space of the device .Fa dev , at offset .Fa reg , with .Fa width specifying the size of the access. .Pp The .Fn pcie_adjust_config function is used to modify the value of a register in the PCI-express capability register set of device .Fa dev . The offset .Fa reg specifies a relative offset in the register set with .Fa width specifying the size of the access. The new value of the register is computed by modifying bits set in .Fa mask to the value in .Fa val . Any bits not specified in .Fa mask are preserved. The previous value of the register is returned. .Pp The .Fn pcie_read_config function is used to read the value of a register in the PCI-express capability register set of device .Fa dev . The offset .Fa reg specifies a relative offset in the register set with .Fa width specifying the size of the access. .Pp The .Fn pcie_write_config function is used to write the value .Fa val to a register in the PCI-express capability register set of device .Fa dev . The offset .Fa reg specifies a relative offset in the register set with .Fa width specifying the size of the access. .Pp .Em NOTE : Device drivers should only use these functions for functionality that is not available via another .Fn pci function. .Ss Locating Devices The .Fn pci_find_bsf function looks up the .Vt device_t of a PCI device, given its .Fa bus , .Fa slot , and .Fa func . The .Fa slot number actually refers to the number of the device on the bus, which does not necessarily indicate its geographic location in terms of a physical slot. Note that in case the system has multiple PCI domains, the .Fn pci_find_bsf function only searches the first one. Actually, it is equivalent to: .Bd -literal -offset indent pci_find_dbsf(0, bus, slot, func); .Ed .Pp The .Fn pci_find_dbsf function looks up the .Vt device_t of a PCI device, given its .Fa domain , .Fa bus , .Fa slot , and .Fa func . The .Fa slot number actually refers to the number of the device on the bus, which does not necessarily indicate its geographic location in terms of a physical slot. .Pp The .Fn pci_find_device function looks up the .Vt device_t of a PCI device, given its .Fa vendor and .Fa device IDs. Note that there can be multiple matches for this search; this function only returns the first matching device. .Ss Device Information The .Fn pci_find_cap function is used to locate the first instance of a PCI capability register set for the device .Fa dev . The capability to locate is specified by ID via .Fa capability . Constant macros of the form .Dv PCIY_xxx for standard capability IDs are defined in .In dev/pci/pcireg.h . If the capability is found, then .Fa *capreg is set to the offset in configuration space of the capability register set, and .Fn pci_find_cap returns zero. If the capability is not found or the device does not support capabilities, .Fn pci_find_cap returns an error. .Pp The .Fn pci_find_extcap function is used to locate the first instance of a PCI-express extended capability register set for the device .Fa dev . The extended capability to locate is specified by ID via .Fa capability . Constant macros of the form .Dv PCIZ_xxx for standard extended capability IDs are defined in .In dev/pci/pcireg.h . If the extended capability is found, then .Fa *capreg is set to the offset in configuration space of the extended capability register set, and .Fn pci_find_extcap returns zero. If the extended capability is not found or the device is not a PCI-express device, .Fn pci_find_extcap returns an error. .Pp The .Fn pci_find_htcap function is used to locate the first instance of a HyperTransport capability register set for the device .Fa dev . The capability to locate is specified by type via .Fa capability . Constant macros of the form .Dv PCIM_HTCAP_xxx for standard HyperTransport capability types are defined in .In dev/pci/pcireg.h . If the capability is found, then .Fa *capreg is set to the offset in configuration space of the capability register set, and .Fn pci_find_htcap returns zero. If the capability is not found or the device is not a HyperTransport device, .Fn pci_find_htcap returns an error. .Pp The .Fn pci_find_pcie_root_port function walks up the PCI device hierarchy to locate the PCI-express root port upstream of .Fa dev . If a root port is not found, .Fn pci_find_pcie_root_port returns .Dv NULL . .Pp The .Fn pci_get_id function is used to read an identifier from a device. The .Fa type flag is used to specify which identifier to read. The following flags are supported: .Bl -hang -width ".Dv PCI_ID_RID" .It Dv PCI_ID_RID Read the routing identifier for the device. +.It Dv PCI_ID_MSI +Read the MSI routing ID. +This is needed by some interrupt controllers to route MSI and MSI-X interrupts. .El .Pp The .Fn pci_get_vpd_ident function is used to fetch a device's Vital Product Data .Pq VPD identifier string. If the device .Fa dev supports VPD and provides an identifier string, then .Fa *identptr is set to point at a read-only, null-terminated copy of the identifier string, and .Fn pci_get_vpd_ident returns zero. If the device does not support VPD or does not provide an identifier string, then .Fn pci_get_vpd_ident returns an error. .Pp The .Fn pci_get_vpd_readonly function is used to fetch the value of a single VPD read-only keyword for the device .Fa dev . The keyword to fetch is identified by the two character string .Fa kw . If the device supports VPD and provides a read-only value for the requested keyword, then .Fa *vptr is set to point at a read-only, null-terminated copy of the value, and .Fn pci_get_vpd_readonly returns zero. If the device does not support VPD or does not provide the requested keyword, then .Fn pci_get_vpd_readonly returns an error. .Ss Device Configuration The .Fn pci_enable_busmaster function enables PCI bus mastering for the device .Fa dev , by setting the .Dv PCIM_CMD_BUSMASTEREN bit in the .Dv PCIR_COMMAND register. The .Fn pci_disable_busmaster function clears this bit. .Pp The .Fn pci_enable_io function enables memory or I/O port address decoding for the device .Fa dev , by setting the .Dv PCIM_CMD_MEMEN or .Dv PCIM_CMD_PORTEN bit in the .Dv PCIR_COMMAND register appropriately. The .Fn pci_disable_io function clears the appropriate bit. The .Fa space argument specifies which resource is affected; this can be either .Dv SYS_RES_MEMORY or .Dv SYS_RES_IOPORT as appropriate. Device drivers should generally not use these routines directly. The PCI bus will enable decoding automatically when a .Dv SYS_RES_MEMORY or .Dv SYS_RES_IOPORT resource is activated via .Xr bus_alloc_resource 9 or .Xr bus_activate_resource 9 . .Pp The .Fn pci_get_max_read_req function returns the current maximum read request size in bytes for a PCI-express device. If the .Fa dev device is not a PCI-express device, .Fn pci_get_max_read_req returns zero. .Pp The .Fn pci_set_max_read_req sets the PCI-express maximum read request size for .Fa dev . The requested .Fa size may be adjusted, and .Fn pci_set_max_read_req returns the actual size set in bytes. If the .Fa dev device is not a PCI-express device, .Fn pci_set_max_read_req returns zero. .Pp The .Fn pci_get_powerstate function returns the current power state of the device .Fa dev . If the device does not support power management capabilities, then the default state of .Dv PCI_POWERSTATE_D0 is returned. The following power states are defined by PCI: .Bl -hang -width ".Dv PCI_POWERSTATE_UNKNOWN" .It Dv PCI_POWERSTATE_D0 State in which device is on and running. It is receiving full power from the system and delivering full functionality to the user. .It Dv PCI_POWERSTATE_D1 Class-specific low-power state in which device context may or may not be lost. Busses in this state cannot do anything to the bus, to force devices to lose context. .It Dv PCI_POWERSTATE_D2 Class-specific low-power state in which device context may or may not be lost. Attains greater power savings than .Dv PCI_POWERSTATE_D1 . Busses in this state can cause devices to lose some context. Devices .Em must be prepared for the bus to be in this state or higher. .It Dv PCI_POWERSTATE_D3 State in which the device is off and not running. Device context is lost, and power from the device can be removed. .It Dv PCI_POWERSTATE_UNKNOWN State of the device is unknown. .El .Pp The .Fn pci_set_powerstate function is used to transition the device .Fa dev to the PCI power state .Fa state . If the device does not support power management capabilities or it does not support the specific power state .Fa state , then the function will fail with .Er EOPNOTSUPP . .Pp The .Fn pci_iov_attach function is used to advertise that the given device .Pq and associated device driver supports PCI Single-Root I/O Virtualization .Pq SR-IOV . A driver that supports SR-IOV must implement the .Xr PCI_IOV_INIT 9 , .Xr PCI_IOV_ADD_VF 9 and .Xr PCI_IOV_UNINIT 9 methods. This function should be called during the .Xr DEVICE_ATTACH 9 method. If this function returns an error, it is recommended that the device driver still successfully attaches, but runs with SR-IOV disabled. The .Fa pf_schema and .Fa vf_schema parameters are used to define what device-specific configuration parameters the device driver accepts when SR-IOV is enabled for the Physical Function .Pq PF and for individual Virtual Functions .Pq VFs respectively. See .Xr pci_iov_schema 9 for details on how to construct the schema. If either the .Pa pf_schema or .Pa vf_schema is invalid or specifies parameter names that conflict with parameter names that are already in use, .Fn pci_iov_attach will return an error and SR-IOV will not be available on the PF device. If a driver does not accept configuration parameters for either the PF device or the VF devices, the driver must pass an empty schema for that device. The SR-IOV infrastructure takes ownership of the .Fa pf_schema and .Fa vf_schema and is responsible for freeing them. The driver must never free the schemas itself. .Pp The .Fn pci_iov_detach function is used to advise the SR-IOV infrastructure that the driver for the given device is attempting to detach and that all SR-IOV resources for the device must be released. This function must be called during the .Xr DEVICE_DETACH 9 method if .Fn pci_iov_attach was successfully called on the device and .Fn pci_iov_detach has not subsequently been called on the device and returned no error. If this function returns an error, the .Xr DEVICE_DETACH 9 method must fail and return an error, as detaching the PF driver while VF devices are active would cause system instability. This function is safe to call and will always succeed if .Fn pci_iov_attach previously failed with an error on the given device, or if .Fn pci_iov_attach was never called on the device. .Pp The .Fn pci_save_state and .Fn pci_restore_state functions can be used by a device driver to save and restore standard PCI config registers. The .Fn pci_save_state function must be invoked while the device has valid state before .Fn pci_restore_state can be used. If the device is not in the fully-powered state .Pq Dv PCI_POWERSTATE_D0 when .Fn pci_restore_state is invoked, then the device will be transitioned to .Dv PCI_POWERSTATE_D0 before any config registers are restored. .Ss Message Signaled Interrupts Message Signaled Interrupts .Pq MSI and Enhanced Message Signaled Interrupts .Pq MSI-X are PCI capabilities that provide an alternate method for PCI devices to signal interrupts. The legacy INTx interrupt is available to PCI devices as a .Dv SYS_RES_IRQ resource with a resource ID of zero. MSI and MSI-X interrupts are available to PCI devices as one or more .Dv SYS_RES_IRQ resources with resource IDs greater than zero. A driver must ask the PCI bus to allocate MSI or MSI-X interrupts using .Fn pci_alloc_msi or .Fn pci_alloc_msix before it can use MSI or MSI-X .Dv SYS_RES_IRQ resources. A driver is not allowed to use the legacy INTx .Dv SYS_RES_IRQ resource if MSI or MSI-X interrupts have been allocated, and attempts to allocate MSI or MSI-X interrupts will fail if the driver is currently using the legacy INTx .Dv SYS_RES_IRQ resource. A driver is only allowed to use either MSI or MSI-X, but not both. .Pp The .Fn pci_msi_count function returns the maximum number of MSI messages supported by the device .Fa dev . If the device does not support MSI, then .Fn pci_msi_count returns zero. .Pp The .Fn pci_alloc_msi function attempts to allocate .Fa *count MSI messages for the device .Fa dev . The .Fn pci_alloc_msi function may allocate fewer messages than requested for various reasons including requests for more messages than the device .Fa dev supports, or if the system has a shortage of available MSI messages. On success, .Fa *count is set to the number of messages allocated and .Fn pci_alloc_msi returns zero. The .Dv SYS_RES_IRQ resources for the allocated messages will be available at consecutive resource IDs beginning with one. If .Fn pci_alloc_msi is not able to allocate any messages, it returns an error. Note that MSI only supports message counts that are powers of two; requests to allocate a non-power of two count of messages will fail. .Pp The .Fn pci_release_msi function is used to release any allocated MSI or MSI-X messages back to the system. If any MSI or MSI-X .Dv SYS_RES_IRQ resources are allocated by the driver or have a configured interrupt handler, this function will fail with .Er EBUSY . The .Fn pci_release_msi function returns zero on success and an error on failure. .Pp The .Fn pci_msix_count function returns the maximum number of MSI-X messages supported by the device .Fa dev . If the device does not support MSI-X, then .Fn pci_msix_count returns zero. .Pp The .Fn pci_msix_pba_bar function returns the offset in configuration space of the Base Address Register .Pq BAR containing the MSI-X Pending Bit Array (PBA) for device .Fa dev . The returned value can be used as the resource ID with .Xr bus_alloc_resource 9 and .Xr bus_release_resource 9 to allocate the BAR. If the device does not support MSI-X, then .Fn pci_msix_pba_bar returns -1. .Pp The .Fn pci_msix_table_bar function returns the offset in configuration space of the BAR containing the MSI-X vector table for device .Fa dev . The returned value can be used as the resource ID with .Xr bus_alloc_resource 9 and .Xr bus_release_resource 9 to allocate the BAR. If the device does not support MSI-X, then .Fn pci_msix_table_bar returns -1. .Pp The .Fn pci_alloc_msix function attempts to allocate .Fa *count MSI-X messages for the device .Fa dev . The .Fn pci_alloc_msix function may allocate fewer messages than requested for various reasons including requests for more messages than the device .Fa dev supports, or if the system has a shortage of available MSI-X messages. On success, .Fa *count is set to the number of messages allocated and .Fn pci_alloc_msix returns zero. For MSI-X messages, the resource ID for each .Dv SYS_RES_IRQ resource identifies the index in the MSI-X table of the corresponding message. A resource ID of one maps to the first index of the MSI-X table; a resource ID two identifies the second index in the table, etc. The .Fn pci_alloc_msix function assigns the .Fa *count messages allocated to the first .Fa *count table indices. If .Fn pci_alloc_msix is not able to allocate any messages, it returns an error. Unlike MSI, MSI-X does not require message counts that are powers of two. .Pp The BARs containing the MSI-X vector table and PBA must be allocated via .Xr bus_alloc_resource 9 before calling .Fn pci_alloc_msix and must not be released until after calling .Fn pci_release_msi . Note that the vector table and PBA may be stored in the same BAR or in different BARs. .Pp The .Fn pci_pending_msix function examines the .Fa dev device's PBA to determine the pending status of the MSI-X message at table index .Fa index . If the indicated message is pending, this function returns a non-zero value; otherwise, it returns zero. Passing an invalid .Fa index to this function will result in undefined behavior. .Pp As mentioned in the description of .Fn pci_alloc_msix , MSI-X messages are initially assigned to the first N table entries. A driver may use a different distribution of available messages to table entries via the .Fn pci_remap_msix function. Note that this function must be called after a successful call to .Fn pci_alloc_msix but before any of the .Dv SYS_RES_IRQ resources are allocated. The .Fn pci_remap_msix function returns zero on success, or an error on failure. .Pp The .Fa vectors array should contain .Fa count message vectors. The array maps directly to the MSI-X table in that the first entry in the array specifies the message used for the first entry in the MSI-X table, the second entry in the array corresponds to the second entry in the MSI-X table, etc. The vector value in each array index can either be zero to indicate that no message should be assigned to the corresponding MSI-X table entry, or it can be a number from one to N .Po where N is the count returned from the previous call to .Fn pci_alloc_msix .Pc to indicate which of the allocated messages should be assigned to the corresponding MSI-X table entry. .Pp If .Fn pci_remap_msix succeeds, each MSI-X table entry with a non-zero vector will have an associated .Dv SYS_RES_IRQ resource whose resource ID corresponds to the table index as described above for .Fn pci_alloc_msix . MSI-X table entries that with a vector of zero will not have an associated .Dv SYS_RES_IRQ resource. Additionally, if any of the original messages allocated by .Fn pci_alloc_msix are not used in the new distribution of messages in the MSI-X table, they will be released automatically. Note that if a driver wishes to use fewer messages than were allocated by .Fn pci_alloc_msix , the driver must use a single, contiguous range of messages beginning with one in the new distribution. The .Fn pci_remap_msix function will fail if this condition is not met. .Sh IMPLEMENTATION NOTES The .Vt pci_addr_t type varies according to the size of the PCI bus address space on the target architecture. .Sh SEE ALSO .Xr pci 4 , .Xr pciconf 8 , .Xr bus_alloc_resource 9 , .Xr bus_dma 9 , .Xr bus_release_resource 9 , .Xr bus_setup_intr 9 , .Xr bus_teardown_intr 9 , .Xr devclass 9 , .Xr device 9 , .Xr driver 9 , .Xr rman 9 .Rs .%B FreeBSD Developers' Handbook .%T NewBus .%U http://www.FreeBSD.org/doc/en_US.ISO8859-1/books/developers-handbook/ .Re .Rs .%A Shanley .%A Anderson .%B PCI System Architecture .%N 2nd Edition .%I Addison-Wesley .%O ISBN 0-201-30974-2 .Re .Sh AUTHORS .An -nosplit This manual page was written by .An Bruce M Simpson Aq Mt bms@FreeBSD.org and .An John Baldwin Aq Mt jhb@FreeBSD.org . .Sh BUGS The kernel PCI code has a number of references to .Dq "slot numbers" . These do not refer to the geographic location of PCI devices, but to the device number assigned by the combination of the PCI IDSEL mechanism and the platform firmware. This should be taken note of when working with the kernel PCI code. .Pp The PCI bus driver should allocate the MSI-X vector table and PBA internally as necessary rather than requiring the caller to do so. Index: head/sys/arm64/arm64/gic_v3_its.c =================================================================== --- head/sys/arm64/arm64/gic_v3_its.c (revision 299931) +++ head/sys/arm64/arm64/gic_v3_its.c (revision 299932) @@ -1,1785 +1,1731 @@ /*- * Copyright (c) 2015 The FreeBSD Foundation * All rights reserved. * * This software was developed by Semihalf under * the sponsorship of the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "gic_v3_reg.h" #include "gic_v3_var.h" #define GIC_V3_ITS_QUIRK_THUNDERX_PEM_BUS_OFFSET 88 #include "pic_if.h" +#include "pcib_if.h" /* Device and PIC methods */ static int gic_v3_its_attach(device_t); static device_method_t gic_v3_its_methods[] = { /* Device interface */ DEVMETHOD(device_attach, gic_v3_its_attach), /* * PIC interface */ /* MSI-X */ DEVMETHOD(pic_alloc_msix, gic_v3_its_alloc_msix), /* MSI */ DEVMETHOD(pic_alloc_msi, gic_v3_its_alloc_msi), DEVMETHOD(pic_map_msi, gic_v3_its_map_msi), /* End */ DEVMETHOD_END }; DEFINE_CLASS_0(its, gic_v3_its_driver, gic_v3_its_methods, sizeof(struct gic_v3_its_softc)); MALLOC_DEFINE(M_GIC_V3_ITS, "GICv3 ITS", GIC_V3_ITS_DEVSTR); static int its_alloc_tables(struct gic_v3_its_softc *); static void its_free_tables(struct gic_v3_its_softc *); static void its_init_commandq(struct gic_v3_its_softc *); static void its_init_cpu_collection(struct gic_v3_its_softc *); static uint32_t its_get_devid(device_t); static struct its_dev * its_device_find_locked(struct gic_v3_its_softc *, device_t, uint32_t); static int its_cmd_send(struct gic_v3_its_softc *, struct its_cmd_desc *); static void its_cmd_movi(struct gic_v3_its_softc *, struct its_dev *, struct its_col *, uint32_t); static void its_cmd_mapc(struct gic_v3_its_softc *, struct its_col *, uint8_t); static void its_cmd_mapvi(struct gic_v3_its_softc *, struct its_dev *, uint32_t, uint32_t); static void its_cmd_mapi(struct gic_v3_its_softc *, struct its_dev *, uint32_t); static void its_cmd_inv(struct gic_v3_its_softc *, struct its_dev *, uint32_t); static void its_cmd_invall(struct gic_v3_its_softc *, struct its_col *); static uint32_t its_get_devbits(device_t); static void lpi_init_conftable(struct gic_v3_its_softc *); static void lpi_bitmap_init(struct gic_v3_its_softc *); static int lpi_config_cpu(struct gic_v3_its_softc *); static void lpi_alloc_cpu_pendtables(struct gic_v3_its_softc *); const char *its_ptab_cache[] = { [GITS_BASER_CACHE_NCNB] = "(NC,NB)", [GITS_BASER_CACHE_NC] = "(NC)", [GITS_BASER_CACHE_RAWT] = "(RA,WT)", [GITS_BASER_CACHE_RAWB] = "(RA,WB)", [GITS_BASER_CACHE_WAWT] = "(WA,WT)", [GITS_BASER_CACHE_WAWB] = "(WA,WB)", [GITS_BASER_CACHE_RAWAWT] = "(RAWA,WT)", [GITS_BASER_CACHE_RAWAWB] = "(RAWA,WB)", }; const char *its_ptab_share[] = { [GITS_BASER_SHARE_NS] = "none", [GITS_BASER_SHARE_IS] = "inner", [GITS_BASER_SHARE_OS] = "outer", [GITS_BASER_SHARE_RES] = "none", }; const char *its_ptab_type[] = { [GITS_BASER_TYPE_UNIMPL] = "Unimplemented", [GITS_BASER_TYPE_DEV] = "Devices", [GITS_BASER_TYPE_VP] = "Virtual Processors", [GITS_BASER_TYPE_PP] = "Physical Processors", [GITS_BASER_TYPE_IC] = "Interrupt Collections", [GITS_BASER_TYPE_RES5] = "Reserved (5)", [GITS_BASER_TYPE_RES6] = "Reserved (6)", [GITS_BASER_TYPE_RES7] = "Reserved (7)", }; /* * Vendor specific quirks. * One needs to add appropriate entry to its_quirks[] * table if the imlementation varies from the generic ARM ITS. */ /* Cavium ThunderX PCI devid acquire function */ static uint32_t its_get_devbits_thunder(device_t); -static uint32_t its_get_devid_thunder(device_t); static const struct its_quirks its_quirks[] = { { /* * Hardware: Cavium ThunderX * Chip revision: Pass 1.0, Pass 1.1 */ .cpuid = CPU_ID_RAW(CPU_IMPL_CAVIUM, CPU_PART_THUNDER, 0, 0), .cpuid_mask = CPU_IMPL_MASK | CPU_PART_MASK, - .devid_func = its_get_devid_thunder, .devbits_func = its_get_devbits_thunder, }, }; static struct gic_v3_its_softc *its_sc; #define gic_its_read(sc, len, reg) \ bus_read_##len(&sc->its_res[0], reg) #define gic_its_write(sc, len, reg, val) \ bus_write_##len(&sc->its_res[0], reg, val) static int gic_v3_its_attach(device_t dev) { struct gic_v3_its_softc *sc; uint64_t gits_tmp; uint32_t gits_pidr2; int rid; int ret; sc = device_get_softc(dev); /* * XXX ARM64TODO: Avoid configuration of more than one ITS * device. To be removed when multi-PIC support is added * to FreeBSD (or at least multi-ITS is implemented). Limit * supported ITS sockets to '0' only. */ if (device_get_unit(dev) != 0) { device_printf(dev, "Only single instance of ITS is supported, exiting...\n"); return (ENXIO); } sc->its_socket = 0; /* * Initialize sleep & spin mutex for ITS */ /* Protects ITS device list and assigned LPIs bitmaps. */ mtx_init(&sc->its_dev_lock, "ITS dev lock", NULL, MTX_SPIN); /* Protects access to ITS command circular buffer. */ mtx_init(&sc->its_cmd_lock, "ITS cmd lock", NULL, MTX_SPIN); rid = 0; sc->its_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, RF_ACTIVE); if (sc->its_res == NULL) { device_printf(dev, "Could not allocate memory\n"); return (ENXIO); } sc->dev = dev; gits_pidr2 = gic_its_read(sc, 4, GITS_PIDR2); switch (gits_pidr2 & GITS_PIDR2_ARCH_MASK) { case GITS_PIDR2_ARCH_GICv3: /* fall through */ case GITS_PIDR2_ARCH_GICv4: if (bootverbose) { device_printf(dev, "ITS found. Architecture rev. %u\n", (u_int)(gits_pidr2 & GITS_PIDR2_ARCH_MASK) >> 4); } break; default: device_printf(dev, "No ITS found in the system\n"); gic_v3_its_detach(dev); return (ENODEV); } /* 1. Initialize commands queue */ its_init_commandq(sc); /* 2. Provide memory for any private ITS tables */ ret = its_alloc_tables(sc); if (ret != 0) { gic_v3_its_detach(dev); return (ret); } /* 3. Allocate collections. One per-CPU */ for (int cpu = 0; cpu < mp_ncpus; cpu++) if (CPU_ISSET(cpu, &all_cpus) != 0) sc->its_cols[cpu] = malloc(sizeof(*sc->its_cols[0]), M_GIC_V3_ITS, (M_WAITOK | M_ZERO)); else sc->its_cols[cpu] = NULL; /* 4. Enable ITS in GITS_CTLR */ gits_tmp = gic_its_read(sc, 4, GITS_CTLR); gic_its_write(sc, 4, GITS_CTLR, gits_tmp | GITS_CTLR_EN); /* 5. Initialize LPIs configuration table */ lpi_init_conftable(sc); /* 6. LPIs bitmap init */ lpi_bitmap_init(sc); /* 7. Allocate pending tables for all CPUs */ lpi_alloc_cpu_pendtables(sc); /* 8. CPU init */ (void)its_init_cpu(sc); /* 9. Init ITS devices list */ TAILQ_INIT(&sc->its_dev_list); arm_register_msi_pic(dev); /* * XXX ARM64TODO: We need to have ITS software context * when being called by the interrupt code (mask/unmask). * This may be used only when one ITS is present in * the system and eventually should be removed. */ KASSERT(its_sc == NULL, ("Trying to assign its_sc that is already set")); its_sc = sc; return (0); } /* Will not detach but use it for convenience */ int gic_v3_its_detach(device_t dev) { device_t parent; struct gic_v3_softc *gic_sc; struct gic_v3_its_softc *sc; u_int cpuid; int rid = 0; sc = device_get_softc(dev); cpuid = PCPU_GET(cpuid); /* Release what's possible */ /* Command queue */ if ((void *)sc->its_cmdq_base != NULL) { contigfree((void *)sc->its_cmdq_base, ITS_CMDQ_SIZE, M_GIC_V3_ITS); } /* ITTs */ its_free_tables(sc); /* Collections */ for (cpuid = 0; cpuid < mp_ncpus; cpuid++) free(sc->its_cols[cpuid], M_GIC_V3_ITS); /* LPI config table */ parent = device_get_parent(sc->dev); gic_sc = device_get_softc(parent); if ((void *)gic_sc->gic_redists.lpis.conf_base != NULL) { contigfree((void *)gic_sc->gic_redists.lpis.conf_base, LPI_CONFTAB_SIZE, M_GIC_V3_ITS); } for (cpuid = 0; cpuid < mp_ncpus; cpuid++) if ((void *)gic_sc->gic_redists.lpis.pend_base[cpuid] != NULL) { contigfree( (void *)gic_sc->gic_redists.lpis.pend_base[cpuid], roundup2(LPI_PENDTAB_SIZE, PAGE_SIZE_64K), M_GIC_V3_ITS); } /* Resource... */ bus_release_resource(dev, SYS_RES_MEMORY, rid, sc->its_res); /* XXX ARM64TODO: Reset global pointer to ITS software context */ its_sc = NULL; return (0); } static int its_alloc_tables(struct gic_v3_its_softc *sc) { uint64_t gits_baser, gits_tmp; uint64_t type, esize, cache, share, psz; size_t page_size, npages, nitspages, nidents, tn; size_t its_tbl_size; vm_offset_t ptab_vaddr; vm_paddr_t ptab_paddr; boolean_t first = TRUE; page_size = PAGE_SIZE_64K; for (tn = 0; tn < GITS_BASER_NUM; tn++) { gits_baser = gic_its_read(sc, 8, GITS_BASER(tn)); type = GITS_BASER_TYPE(gits_baser); /* Get the Table Entry size */ esize = GITS_BASER_ESIZE(gits_baser); switch (type) { case GITS_BASER_TYPE_UNIMPL: /* fall through */ case GITS_BASER_TYPE_RES5: case GITS_BASER_TYPE_RES6: case GITS_BASER_TYPE_RES7: continue; case GITS_BASER_TYPE_DEV: nidents = (1 << its_get_devbits(sc->dev)); its_tbl_size = esize * nidents; its_tbl_size = roundup2(its_tbl_size, page_size); npages = howmany(its_tbl_size, PAGE_SIZE); break; default: npages = howmany(page_size, PAGE_SIZE); break; } /* Allocate required space */ ptab_vaddr = (vm_offset_t)contigmalloc(npages * PAGE_SIZE, M_GIC_V3_ITS, (M_WAITOK | M_ZERO), 0, ~0UL, PAGE_SIZE, 0); sc->its_ptabs[tn].ptab_vaddr = ptab_vaddr; sc->its_ptabs[tn].ptab_pgsz = PAGE_SIZE; sc->its_ptabs[tn].ptab_npages = npages; ptab_paddr = vtophys(ptab_vaddr); KASSERT((ptab_paddr & GITS_BASER_PA_MASK) == ptab_paddr, ("%s: Unaligned PA for Interrupt Translation Table", device_get_name(sc->dev))); /* Set defaults: WAWB, IS */ cache = GITS_BASER_CACHE_WAWB; share = GITS_BASER_SHARE_IS; for (;;) { nitspages = howmany(its_tbl_size, page_size); switch (page_size) { case PAGE_SIZE: /* 4KB */ psz = GITS_BASER_PSZ_4K; break; case PAGE_SIZE_16K: /* 16KB */ psz = GITS_BASER_PSZ_4K; break; case PAGE_SIZE_64K: /* 64KB */ psz = GITS_BASER_PSZ_64K; break; default: device_printf(sc->dev, "Unsupported page size: %zuKB\n", (page_size / 1024)); its_free_tables(sc); return (ENXIO); } /* Clear fields under modification first */ gits_baser &= ~(GITS_BASER_VALID | GITS_BASER_CACHE_MASK | GITS_BASER_TYPE_MASK | GITS_BASER_ESIZE_MASK | GITS_BASER_PA_MASK | GITS_BASER_SHARE_MASK | GITS_BASER_PSZ_MASK | GITS_BASER_SIZE_MASK); /* Construct register value */ gits_baser |= (type << GITS_BASER_TYPE_SHIFT) | ((esize - 1) << GITS_BASER_ESIZE_SHIFT) | (cache << GITS_BASER_CACHE_SHIFT) | (share << GITS_BASER_SHARE_SHIFT) | (psz << GITS_BASER_PSZ_SHIFT) | ptab_paddr | (nitspages - 1) | GITS_BASER_VALID; gic_its_write(sc, 8, GITS_BASER(tn), gits_baser); /* * Verify. * Depending on implementation we may encounter * shareability and page size mismatch. */ gits_tmp = gic_its_read(sc, 8, GITS_BASER(tn)); if (((gits_tmp ^ gits_baser) & GITS_BASER_SHARE_MASK) != 0) { share = gits_tmp & GITS_BASER_SHARE_MASK; share >>= GITS_BASER_SHARE_SHIFT; continue; } if (((gits_tmp ^ gits_baser) & GITS_BASER_PSZ_MASK) != 0) { switch (page_size) { case PAGE_SIZE_16K: /* Drop to 4KB page */ page_size = PAGE_SIZE; continue; case PAGE_SIZE_64K: /* Drop to 16KB page */ page_size = PAGE_SIZE_16K; continue; } } /* * All possible adjustments should * be applied by now so just break the loop. */ break; } /* * Do not compare Cacheability field since * it is implementation defined. */ gits_tmp &= ~GITS_BASER_CACHE_MASK; gits_baser &= ~GITS_BASER_CACHE_MASK; if (gits_tmp != gits_baser) { device_printf(sc->dev, "Could not allocate ITS tables\n"); its_free_tables(sc); return (ENXIO); } if (bootverbose) { if (first) { device_printf(sc->dev, "Allocated ITS private tables:\n"); first = FALSE; } device_printf(sc->dev, "\tPTAB%zu for %s: PA 0x%lx," " %lu entries," " cache policy %s, %s shareable," " page size %zuKB\n", tn, its_ptab_type[type], ptab_paddr, (page_size * nitspages) / esize, its_ptab_cache[cache], its_ptab_share[share], page_size / 1024); } } return (0); } static void its_free_tables(struct gic_v3_its_softc *sc) { vm_offset_t ptab_vaddr; size_t size; size_t tn; for (tn = 0; tn < GITS_BASER_NUM; tn++) { ptab_vaddr = sc->its_ptabs[tn].ptab_vaddr; if (ptab_vaddr == 0) continue; size = sc->its_ptabs[tn].ptab_pgsz; size *= sc->its_ptabs[tn].ptab_npages; if ((void *)ptab_vaddr != NULL) contigfree((void *)ptab_vaddr, size, M_GIC_V3_ITS); /* Clear the table description */ memset(&sc->its_ptabs[tn], 0, sizeof(sc->its_ptabs[tn])); } } static void its_init_commandq(struct gic_v3_its_softc *sc) { uint64_t gits_cbaser, gits_tmp; uint64_t cache, share; vm_paddr_t cmdq_paddr; device_t dev; dev = sc->dev; /* Allocate memory for command queue */ sc->its_cmdq_base = contigmalloc(ITS_CMDQ_SIZE, M_GIC_V3_ITS, (M_WAITOK | M_ZERO), 0, ~0UL, ITS_CMDQ_SIZE, 0); /* Set command queue write pointer (command queue empty) */ sc->its_cmdq_write = sc->its_cmdq_base; /* Save command queue pointer and attributes */ cmdq_paddr = vtophys(sc->its_cmdq_base); /* Set defaults: Normal Inner WAWB, IS */ cache = GITS_CBASER_CACHE_NIWAWB; share = GITS_CBASER_SHARE_IS; gits_cbaser = (cmdq_paddr | (cache << GITS_CBASER_CACHE_SHIFT) | (share << GITS_CBASER_SHARE_SHIFT) | /* Number of 4KB pages - 1 */ ((ITS_CMDQ_SIZE / PAGE_SIZE) - 1) | /* Valid bit */ GITS_CBASER_VALID); gic_its_write(sc, 8, GITS_CBASER, gits_cbaser); gits_tmp = gic_its_read(sc, 8, GITS_CBASER); if (((gits_tmp ^ gits_cbaser) & GITS_CBASER_SHARE_MASK) != 0) { if (bootverbose) { device_printf(dev, "Will use cache flushing for commands queue\n"); } /* Command queue needs cache flushing */ sc->its_flags |= ITS_FLAGS_CMDQ_FLUSH; } gic_its_write(sc, 8, GITS_CWRITER, 0x0); } int its_init_cpu(struct gic_v3_its_softc *sc) { device_t parent; struct gic_v3_softc *gic_sc; /* * NULL in place of the softc pointer means that * this function was called during GICv3 secondary initialization. */ if (sc == NULL) { if (its_sc != NULL && device_is_attached(its_sc->dev)) { /* * XXX ARM64TODO: This is part of the workaround that * saves ITS software context for further use in * mask/unmask and here. This should be removed as soon * as the upper layer is capable of passing the ITS * context to this function. */ sc = its_sc; } else return (ENXIO); /* Skip if running secondary init on a wrong socket */ if (sc->its_socket != CPU_CURRENT_SOCKET) return (ENXIO); } /* * Check for LPIs support on this Re-Distributor. */ parent = device_get_parent(sc->dev); gic_sc = device_get_softc(parent); if ((gic_r_read(gic_sc, 4, GICR_TYPER) & GICR_TYPER_PLPIS) == 0) { if (bootverbose) { device_printf(sc->dev, "LPIs not supported on CPU%u\n", PCPU_GET(cpuid)); } return (ENXIO); } /* Configure LPIs for this CPU */ lpi_config_cpu(sc); /* Initialize collections */ its_init_cpu_collection(sc); return (0); } static void its_init_cpu_collection(struct gic_v3_its_softc *sc) { device_t parent; struct gic_v3_softc *gic_sc; uint64_t typer; uint64_t target; vm_offset_t redist_base; u_int cpuid; cpuid = PCPU_GET(cpuid); parent = device_get_parent(sc->dev); gic_sc = device_get_softc(parent); typer = gic_its_read(sc, 8, GITS_TYPER); if ((typer & GITS_TYPER_PTA) != 0) { redist_base = rman_get_bushandle(gic_sc->gic_redists.pcpu[cpuid]); /* * Target Address correspond to the base physical * address of Re-Distributors. */ target = vtophys(redist_base); } else { /* Target Address correspond to unique processor numbers */ typer = gic_r_read(gic_sc, 8, GICR_TYPER); target = GICR_TYPER_CPUNUM(typer); } sc->its_cols[cpuid]->col_target = target; sc->its_cols[cpuid]->col_id = cpuid; its_cmd_mapc(sc, sc->its_cols[cpuid], 1); its_cmd_invall(sc, sc->its_cols[cpuid]); } static void lpi_init_conftable(struct gic_v3_its_softc *sc) { device_t parent; struct gic_v3_softc *gic_sc; vm_offset_t conf_base; uint8_t prio_default; parent = device_get_parent(sc->dev); gic_sc = device_get_softc(parent); /* * LPI Configuration Table settings. * Notice that Configuration Table is shared among all * Re-Distributors, so this is going to be created just once. */ conf_base = (vm_offset_t)contigmalloc(LPI_CONFTAB_SIZE, M_GIC_V3_ITS, (M_WAITOK | M_ZERO), 0, ~0UL, PAGE_SIZE_64K, 0); if (bootverbose) { device_printf(sc->dev, "LPI Configuration Table at PA: 0x%lx\n", vtophys(conf_base)); } /* * Let the default priority be aligned with all other * interrupts assuming that each interrupt is assigned * MAX priority at startup. MAX priority on the other * hand cannot be higher than 0xFC for LPIs. */ prio_default = GIC_PRIORITY_MAX; /* Write each settings byte to LPI configuration table */ memset((void *)conf_base, (prio_default & LPI_CONF_PRIO_MASK) | LPI_CONF_GROUP1, LPI_CONFTAB_SIZE); cpu_dcache_wb_range((vm_offset_t)conf_base, roundup2(LPI_CONFTAB_SIZE, PAGE_SIZE_64K)); gic_sc->gic_redists.lpis.conf_base = conf_base; } static void lpi_alloc_cpu_pendtables(struct gic_v3_its_softc *sc) { device_t parent; struct gic_v3_softc *gic_sc; vm_offset_t pend_base; u_int cpuid; parent = device_get_parent(sc->dev); gic_sc = device_get_softc(parent); /* * LPI Pending Table settings. * This has to be done for each Re-Distributor, hence for each CPU. */ for (cpuid = 0; cpuid < mp_ncpus; cpuid++) { /* Limit allocation to active CPUs only */ if (CPU_ISSET(cpuid, &all_cpus) == 0) continue; pend_base = (vm_offset_t)contigmalloc( roundup2(LPI_PENDTAB_SIZE, PAGE_SIZE_64K), M_GIC_V3_ITS, (M_WAITOK | M_ZERO), 0, ~0UL, PAGE_SIZE_64K, 0); /* Clean D-cache so that ITS can see zeroed pages */ cpu_dcache_wb_range((vm_offset_t)pend_base, roundup2(LPI_PENDTAB_SIZE, PAGE_SIZE_64K)); if (bootverbose) { device_printf(sc->dev, "LPI Pending Table for CPU%u at PA: 0x%lx\n", cpuid, vtophys(pend_base)); } gic_sc->gic_redists.lpis.pend_base[cpuid] = pend_base; } /* Ensure visibility of pend_base addresses on other CPUs */ wmb(); } static int lpi_config_cpu(struct gic_v3_its_softc *sc) { device_t parent; struct gic_v3_softc *gic_sc; vm_offset_t conf_base, pend_base; uint64_t gicr_xbaser, gicr_temp; uint64_t cache, share, idbits; uint32_t gicr_ctlr; u_int cpuid; parent = device_get_parent(sc->dev); gic_sc = device_get_softc(parent); cpuid = PCPU_GET(cpuid); /* Ensure data observability on a current CPU */ rmb(); conf_base = gic_sc->gic_redists.lpis.conf_base; pend_base = gic_sc->gic_redists.lpis.pend_base[cpuid]; /* Disable LPIs */ gicr_ctlr = gic_r_read(gic_sc, 4, GICR_CTLR); gicr_ctlr &= ~GICR_CTLR_LPI_ENABLE; gic_r_write(gic_sc, 4, GICR_CTLR, gicr_ctlr); /* Perform full system barrier */ dsb(sy); /* * Set GICR_PROPBASER */ /* * Find out how many bits do we need for LPI identifiers. * Remark 1.: Even though we have (LPI_CONFTAB_SIZE / 8) LPIs * the notified LPI ID still starts from 8192 * (GIC_FIRST_LPI). * Remark 2.: This could be done on compilation time but there * seems to be no sufficient macro. */ idbits = flsl(LPI_CONFTAB_SIZE + GIC_FIRST_LPI) - 1; /* Set defaults: Normal Inner WAWB, IS */ cache = GICR_PROPBASER_CACHE_NIWAWB; share = GICR_PROPBASER_SHARE_IS; gicr_xbaser = vtophys(conf_base) | ((idbits - 1) & GICR_PROPBASER_IDBITS_MASK) | (cache << GICR_PROPBASER_CACHE_SHIFT) | (share << GICR_PROPBASER_SHARE_SHIFT); gic_r_write(gic_sc, 8, GICR_PROPBASER, gicr_xbaser); gicr_temp = gic_r_read(gic_sc, 8, GICR_PROPBASER); if (((gicr_xbaser ^ gicr_temp) & GICR_PROPBASER_SHARE_MASK) != 0) { if (bootverbose) { device_printf(sc->dev, "Will use cache flushing for LPI " "Configuration Table\n"); } gic_sc->gic_redists.lpis.flags |= LPI_FLAGS_CONF_FLUSH; } /* * Set GICR_PENDBASER */ /* Set defaults: Normal Inner WAWB, IS */ cache = GICR_PENDBASER_CACHE_NIWAWB; share = GICR_PENDBASER_SHARE_IS; gicr_xbaser = vtophys(pend_base) | (cache << GICR_PENDBASER_CACHE_SHIFT) | (share << GICR_PENDBASER_SHARE_SHIFT); gic_r_write(gic_sc, 8, GICR_PENDBASER, gicr_xbaser); /* Enable LPIs */ gicr_ctlr = gic_r_read(gic_sc, 4, GICR_CTLR); gicr_ctlr |= GICR_CTLR_LPI_ENABLE; gic_r_write(gic_sc, 4, GICR_CTLR, gicr_ctlr); dsb(sy); return (0); } static void lpi_bitmap_init(struct gic_v3_its_softc *sc) { device_t parent; struct gic_v3_softc *gic_sc; uint32_t lpi_id_num; size_t lpi_chunks_num; size_t bits_in_chunk; parent = device_get_parent(sc->dev); gic_sc = device_get_softc(parent); lpi_id_num = (1 << gic_sc->gic_idbits) - 1; /* Substract IDs dedicated for SGIs, PPIs and SPIs */ lpi_id_num -= GIC_FIRST_LPI; sc->its_lpi_maxid = lpi_id_num; bits_in_chunk = sizeof(*sc->its_lpi_bitmap) * NBBY; /* * Round up to the number of bits in chunk. * We will need to take care to avoid using invalid LPI IDs later. */ lpi_id_num = roundup2(lpi_id_num, bits_in_chunk); lpi_chunks_num = lpi_id_num / bits_in_chunk; sc->its_lpi_bitmap = contigmalloc((lpi_chunks_num * sizeof(*sc->its_lpi_bitmap)), M_GIC_V3_ITS, (M_WAITOK | M_ZERO), 0, ~0UL, sizeof(*sc->its_lpi_bitmap), 0); } static int lpi_alloc_chunk(struct gic_v3_its_softc *sc, struct lpi_chunk *lpic, u_int nvecs) { u_int *col_ids; int fclr; /* First cleared bit */ bitstr_t *bitmap; size_t nb, i; col_ids = malloc(sizeof(*col_ids) * nvecs, M_GIC_V3_ITS, (M_NOWAIT | M_ZERO)); if (col_ids == NULL) return (ENOMEM); mtx_lock_spin(&sc->its_dev_lock); bitmap = sc->its_lpi_bitmap; fclr = 0; retry: /* Check other bits - sloooow */ for (i = 0, nb = fclr; i < nvecs; i++, nb++) { if (nb > sc->its_lpi_maxid) { mtx_unlock_spin(&sc->its_dev_lock); free(col_ids, M_GIC_V3_ITS); return (EINVAL); } if (isset(bitmap, nb)) { /* To little free bits in this area. Move on. */ fclr = nb + 1; goto retry; } } /* This area is free. Take it. */ bit_nset(bitmap, fclr, fclr + nvecs - 1); lpic->lpi_base = fclr + GIC_FIRST_LPI; lpic->lpi_num = nvecs; lpic->lpi_free = lpic->lpi_num; lpic->lpi_col_ids = col_ids; for (i = 0; i < lpic->lpi_num; i++) { /* * Initially all interrupts go to CPU0 but can be moved * to another CPU by bus_bind_intr() or interrupts shuffling. */ lpic->lpi_col_ids[i] = 0; } mtx_unlock_spin(&sc->its_dev_lock); return (0); } static void lpi_free_chunk(struct gic_v3_its_softc *sc, struct lpi_chunk *lpic) { int start, end; KASSERT((lpic->lpi_free == lpic->lpi_num), ("Trying to free LPI chunk that is still in use.\n")); mtx_lock_spin(&sc->its_dev_lock); /* First bit of this chunk in a global bitmap */ start = lpic->lpi_base - GIC_FIRST_LPI; /* and last bit of this chunk... */ end = start + lpic->lpi_num - 1; /* Finally free this chunk */ bit_nclear(sc->its_lpi_bitmap, start, end); mtx_unlock_spin(&sc->its_dev_lock); free(lpic->lpi_col_ids, M_GIC_V3_ITS); lpic->lpi_col_ids = NULL; } static void lpi_configure(struct gic_v3_its_softc *sc, struct its_dev *its_dev, uint32_t lpinum, boolean_t unmask) { device_t parent; struct gic_v3_softc *gic_sc; uint8_t *conf_byte; parent = device_get_parent(sc->dev); gic_sc = device_get_softc(parent); conf_byte = (uint8_t *)gic_sc->gic_redists.lpis.conf_base; conf_byte += (lpinum - GIC_FIRST_LPI); if (unmask) *conf_byte |= LPI_CONF_ENABLE; else *conf_byte &= ~LPI_CONF_ENABLE; if ((gic_sc->gic_redists.lpis.flags & LPI_FLAGS_CONF_FLUSH) != 0) { /* Clean D-cache under configuration byte */ cpu_dcache_wb_range((vm_offset_t)conf_byte, sizeof(*conf_byte)); } else { /* DSB inner shareable, store */ dsb(ishst); } its_cmd_inv(sc, its_dev, lpinum); } static void lpi_map_to_device(struct gic_v3_its_softc *sc, struct its_dev *its_dev, uint32_t id, uint32_t pid) { if ((pid < its_dev->lpis.lpi_base) || (pid >= (its_dev->lpis.lpi_base + its_dev->lpis.lpi_num))) panic("Trying to map ivalid LPI %u for the device\n", pid); its_cmd_mapvi(sc, its_dev, id, pid); } static void lpi_xmask_irq(device_t parent, uint32_t irq, boolean_t unmask) { struct its_dev *its_dev; TAILQ_FOREACH(its_dev, &its_sc->its_dev_list, entry) { if (irq >= its_dev->lpis.lpi_base && irq < (its_dev->lpis.lpi_base + its_dev->lpis.lpi_num)) { lpi_configure(its_sc, its_dev, irq, unmask); return; } } panic("Trying to %s not existing LPI: %u\n", (unmask == TRUE) ? "unmask" : "mask", irq); } int lpi_migrate(device_t parent, uint32_t irq, u_int cpuid) { struct gic_v3_its_softc *sc; struct its_dev *its_dev; struct its_col *col; sc = its_sc; mtx_lock_spin(&sc->its_dev_lock); its_dev = its_device_find_locked(sc, NULL, irq); mtx_unlock_spin(&sc->its_dev_lock); if (its_dev == NULL) { /* Cannot migrate not configured LPI */ return (ENXIO); } /* Find local device's interrupt identifier */ irq = irq - its_dev->lpis.lpi_base; /* Move interrupt to another collection */ col = sc->its_cols[cpuid]; its_cmd_movi(sc, its_dev, col, irq); its_dev->lpis.lpi_col_ids[irq] = cpuid; return (0); } void lpi_unmask_irq(device_t parent, uint32_t irq) { lpi_xmask_irq(parent, irq, 1); } void lpi_mask_irq(device_t parent, uint32_t irq) { lpi_xmask_irq(parent, irq, 0); } /* * Commands handling. */ static __inline void cmd_format_command(struct its_cmd *cmd, uint8_t cmd_type) { /* Command field: DW0 [7:0] */ cmd->cmd_dword[0] &= ~CMD_COMMAND_MASK; cmd->cmd_dword[0] |= cmd_type; } static __inline void cmd_format_devid(struct its_cmd *cmd, uint32_t devid) { /* Device ID field: DW0 [63:32] */ cmd->cmd_dword[0] &= ~CMD_DEVID_MASK; cmd->cmd_dword[0] |= ((uint64_t)devid << CMD_DEVID_SHIFT); } static __inline void cmd_format_size(struct its_cmd *cmd, uint16_t size) { /* Size field: DW1 [4:0] */ cmd->cmd_dword[1] &= ~CMD_SIZE_MASK; cmd->cmd_dword[1] |= (size & CMD_SIZE_MASK); } static __inline void cmd_format_id(struct its_cmd *cmd, uint32_t id) { /* ID field: DW1 [31:0] */ cmd->cmd_dword[1] &= ~CMD_ID_MASK; cmd->cmd_dword[1] |= id; } static __inline void cmd_format_pid(struct its_cmd *cmd, uint32_t pid) { /* Physical ID field: DW1 [63:32] */ cmd->cmd_dword[1] &= ~CMD_PID_MASK; cmd->cmd_dword[1] |= ((uint64_t)pid << CMD_PID_SHIFT); } static __inline void cmd_format_col(struct its_cmd *cmd, uint16_t col_id) { /* Collection field: DW2 [16:0] */ cmd->cmd_dword[2] &= ~CMD_COL_MASK; cmd->cmd_dword[2] |= col_id; } static __inline void cmd_format_target(struct its_cmd *cmd, uint64_t target) { /* Target Address field: DW2 [47:16] */ cmd->cmd_dword[2] &= ~CMD_TARGET_MASK; cmd->cmd_dword[2] |= (target & CMD_TARGET_MASK); } static __inline void cmd_format_itt(struct its_cmd *cmd, uint64_t itt) { /* ITT Address field: DW2 [47:8] */ cmd->cmd_dword[2] &= ~CMD_ITT_MASK; cmd->cmd_dword[2] |= (itt & CMD_ITT_MASK); } static __inline void cmd_format_valid(struct its_cmd *cmd, uint8_t valid) { /* Valid field: DW2 [63] */ cmd->cmd_dword[2] &= ~CMD_VALID_MASK; cmd->cmd_dword[2] |= ((uint64_t)valid << CMD_VALID_SHIFT); } static __inline void cmd_fix_endian(struct its_cmd *cmd) { size_t i; for (i = 0; i < nitems(cmd->cmd_dword); i++) cmd->cmd_dword[i] = htole64(cmd->cmd_dword[i]); } static void its_cmd_movi(struct gic_v3_its_softc *sc, struct its_dev *its_dev, struct its_col *col, uint32_t id) { struct its_cmd_desc desc; desc.cmd_type = ITS_CMD_MOVI; desc.cmd_desc_movi.its_dev = its_dev; desc.cmd_desc_movi.col = col; desc.cmd_desc_movi.id = id; its_cmd_send(sc, &desc); } static void its_cmd_mapc(struct gic_v3_its_softc *sc, struct its_col *col, uint8_t valid) { struct its_cmd_desc desc; desc.cmd_type = ITS_CMD_MAPC; desc.cmd_desc_mapc.col = col; /* * Valid bit set - map the collection. * Valid bit cleared - unmap the collection. */ desc.cmd_desc_mapc.valid = valid; its_cmd_send(sc, &desc); } static void its_cmd_mapvi(struct gic_v3_its_softc *sc, struct its_dev *its_dev, uint32_t id, uint32_t pid) { struct its_cmd_desc desc; struct its_col *col; u_int col_id; col_id = its_dev->lpis.lpi_col_ids[id]; col = sc->its_cols[col_id]; desc.cmd_type = ITS_CMD_MAPVI; desc.cmd_desc_mapvi.its_dev = its_dev; desc.cmd_desc_mapvi.col = col; desc.cmd_desc_mapvi.id = id; desc.cmd_desc_mapvi.pid = pid; its_cmd_send(sc, &desc); } static void __unused its_cmd_mapi(struct gic_v3_its_softc *sc, struct its_dev *its_dev, uint32_t pid) { struct its_cmd_desc desc; struct its_col *col; u_int col_id; uint32_t id; KASSERT(pid >= its_dev->lpis.lpi_base, ("%s: invalid pid: %d for the ITS device", __func__, pid)); id = pid - its_dev->lpis.lpi_base; col_id = its_dev->lpis.lpi_col_ids[id]; col = sc->its_cols[col_id]; desc.cmd_type = ITS_CMD_MAPI; desc.cmd_desc_mapi.its_dev = its_dev; desc.cmd_desc_mapi.col = col; desc.cmd_desc_mapi.pid = pid; its_cmd_send(sc, &desc); } static void its_cmd_mapd(struct gic_v3_its_softc *sc, struct its_dev *its_dev, uint8_t valid) { struct its_cmd_desc desc; desc.cmd_type = ITS_CMD_MAPD; desc.cmd_desc_mapd.its_dev = its_dev; desc.cmd_desc_mapd.valid = valid; its_cmd_send(sc, &desc); } static void its_cmd_inv(struct gic_v3_its_softc *sc, struct its_dev *its_dev, uint32_t pid) { struct its_cmd_desc desc; struct its_col *col; u_int col_id; uint32_t id; KASSERT(pid >= its_dev->lpis.lpi_base, ("%s: invalid pid: %d for the ITS device", __func__, pid)); id = pid - its_dev->lpis.lpi_base; col_id = its_dev->lpis.lpi_col_ids[id]; col = sc->its_cols[col_id]; desc.cmd_type = ITS_CMD_INV; desc.cmd_desc_inv.pid = pid - its_dev->lpis.lpi_base; desc.cmd_desc_inv.its_dev = its_dev; desc.cmd_desc_inv.col = col; its_cmd_send(sc, &desc); } static void its_cmd_invall(struct gic_v3_its_softc *sc, struct its_col *col) { struct its_cmd_desc desc; desc.cmd_type = ITS_CMD_INVALL; desc.cmd_desc_invall.col = col; its_cmd_send(sc, &desc); } /* * Helper routines for commands processing. */ static __inline boolean_t its_cmd_queue_full(struct gic_v3_its_softc *sc) { size_t read_idx, write_idx; write_idx = (size_t)(sc->its_cmdq_write - sc->its_cmdq_base); read_idx = gic_its_read(sc, 4, GITS_CREADR) / sizeof(struct its_cmd); /* * The queue is full when the write offset points * at the command before the current read offset. */ if (((write_idx + 1) % ITS_CMDQ_NENTRIES) == read_idx) return (TRUE); return (FALSE); } static __inline void its_cmd_sync(struct gic_v3_its_softc *sc, struct its_cmd *cmd) { if ((sc->its_flags & ITS_FLAGS_CMDQ_FLUSH) != 0) { /* Clean D-cache under command. */ cpu_dcache_wb_range((vm_offset_t)cmd, sizeof(*cmd)); } else { /* DSB inner shareable, store */ dsb(ishst); } } static struct its_cmd * its_cmd_alloc_locked(struct gic_v3_its_softc *sc) { struct its_cmd *cmd; size_t us_left; /* * XXX ARM64TODO: This is obviously a significant delay. * The reason for that is that currently the time frames for * the command to complete (and therefore free the descriptor) * are not known. */ us_left = 1000000; mtx_assert(&sc->its_cmd_lock, MA_OWNED); while (its_cmd_queue_full(sc)) { if (us_left-- == 0) { /* Timeout while waiting for free command */ device_printf(sc->dev, "Timeout while waiting for free command\n"); return (NULL); } DELAY(1); } cmd = sc->its_cmdq_write; sc->its_cmdq_write++; if (sc->its_cmdq_write == (sc->its_cmdq_base + ITS_CMDQ_NENTRIES)) { /* Wrap the queue */ sc->its_cmdq_write = sc->its_cmdq_base; } return (cmd); } static uint64_t its_cmd_prepare(struct its_cmd *cmd, struct its_cmd_desc *desc) { uint64_t target; uint8_t cmd_type; u_int size; boolean_t error; error = FALSE; cmd_type = desc->cmd_type; target = ITS_TARGET_NONE; switch (cmd_type) { case ITS_CMD_MOVI: /* Move interrupt ID to another collection */ target = desc->cmd_desc_movi.col->col_target; cmd_format_command(cmd, ITS_CMD_MOVI); cmd_format_id(cmd, desc->cmd_desc_movi.id); cmd_format_col(cmd, desc->cmd_desc_movi.col->col_id); cmd_format_devid(cmd, desc->cmd_desc_movi.its_dev->devid); break; case ITS_CMD_SYNC: /* Wait for previous commands completion */ target = desc->cmd_desc_sync.col->col_target; cmd_format_command(cmd, ITS_CMD_SYNC); cmd_format_target(cmd, target); break; case ITS_CMD_MAPD: /* Assign ITT to device */ cmd_format_command(cmd, ITS_CMD_MAPD); cmd_format_itt(cmd, vtophys(desc->cmd_desc_mapd.its_dev->itt)); /* * Size describes number of bits to encode interrupt IDs * supported by the device minus one. * When V (valid) bit is zero, this field should be written * as zero. */ if (desc->cmd_desc_mapd.valid != 0) { size = fls(desc->cmd_desc_mapd.its_dev->lpis.lpi_num); size = MAX(1, size) - 1; } else size = 0; cmd_format_size(cmd, size); cmd_format_devid(cmd, desc->cmd_desc_mapd.its_dev->devid); cmd_format_valid(cmd, desc->cmd_desc_mapd.valid); break; case ITS_CMD_MAPC: /* Map collection to Re-Distributor */ target = desc->cmd_desc_mapc.col->col_target; cmd_format_command(cmd, ITS_CMD_MAPC); cmd_format_col(cmd, desc->cmd_desc_mapc.col->col_id); cmd_format_valid(cmd, desc->cmd_desc_mapc.valid); cmd_format_target(cmd, target); break; case ITS_CMD_MAPVI: target = desc->cmd_desc_mapvi.col->col_target; cmd_format_command(cmd, ITS_CMD_MAPVI); cmd_format_devid(cmd, desc->cmd_desc_mapvi.its_dev->devid); cmd_format_id(cmd, desc->cmd_desc_mapvi.id); cmd_format_pid(cmd, desc->cmd_desc_mapvi.pid); cmd_format_col(cmd, desc->cmd_desc_mapvi.col->col_id); break; case ITS_CMD_MAPI: target = desc->cmd_desc_mapi.col->col_target; cmd_format_command(cmd, ITS_CMD_MAPI); cmd_format_devid(cmd, desc->cmd_desc_mapi.its_dev->devid); cmd_format_id(cmd, desc->cmd_desc_mapi.pid); cmd_format_col(cmd, desc->cmd_desc_mapi.col->col_id); break; case ITS_CMD_INV: target = desc->cmd_desc_inv.col->col_target; cmd_format_command(cmd, ITS_CMD_INV); cmd_format_devid(cmd, desc->cmd_desc_inv.its_dev->devid); cmd_format_id(cmd, desc->cmd_desc_inv.pid); break; case ITS_CMD_INVALL: cmd_format_command(cmd, ITS_CMD_INVALL); cmd_format_col(cmd, desc->cmd_desc_invall.col->col_id); break; default: error = TRUE; break; } if (!error) cmd_fix_endian(cmd); return (target); } static __inline uint64_t its_cmd_cwriter_offset(struct gic_v3_its_softc *sc, struct its_cmd *cmd) { uint64_t off; off = (cmd - sc->its_cmdq_base) * sizeof(*cmd); return (off); } static void its_cmd_wait_completion(struct gic_v3_its_softc *sc, struct its_cmd *cmd_first, struct its_cmd *cmd_last) { uint64_t first, last, read; size_t us_left; /* * XXX ARM64TODO: This is obviously a significant delay. * The reason for that is that currently the time frames for * the command to complete are not known. */ us_left = 1000000; first = its_cmd_cwriter_offset(sc, cmd_first); last = its_cmd_cwriter_offset(sc, cmd_last); for (;;) { read = gic_its_read(sc, 8, GITS_CREADR); if (read < first || read >= last) break; if (us_left-- == 0) { /* This means timeout */ device_printf(sc->dev, "Timeout while waiting for CMD completion.\n"); return; } DELAY(1); } } static int its_cmd_send(struct gic_v3_its_softc *sc, struct its_cmd_desc *desc) { struct its_cmd *cmd, *cmd_sync, *cmd_write; struct its_col col_sync; struct its_cmd_desc desc_sync; uint64_t target, cwriter; mtx_lock_spin(&sc->its_cmd_lock); cmd = its_cmd_alloc_locked(sc); if (cmd == NULL) { device_printf(sc->dev, "could not allocate ITS command\n"); mtx_unlock_spin(&sc->its_cmd_lock); return (EBUSY); } target = its_cmd_prepare(cmd, desc); its_cmd_sync(sc, cmd); if (target != ITS_TARGET_NONE) { cmd_sync = its_cmd_alloc_locked(sc); if (cmd_sync == NULL) goto end; desc_sync.cmd_type = ITS_CMD_SYNC; col_sync.col_target = target; desc_sync.cmd_desc_sync.col = &col_sync; its_cmd_prepare(cmd_sync, &desc_sync); its_cmd_sync(sc, cmd_sync); } end: /* Update GITS_CWRITER */ cwriter = its_cmd_cwriter_offset(sc, sc->its_cmdq_write); gic_its_write(sc, 8, GITS_CWRITER, cwriter); cmd_write = sc->its_cmdq_write; mtx_unlock_spin(&sc->its_cmd_lock); its_cmd_wait_completion(sc, cmd, cmd_write); return (0); } /* Find ITS device descriptor by pci_dev or irq number */ static struct its_dev * its_device_find_locked(struct gic_v3_its_softc *sc, device_t pci_dev, uint32_t irq) { struct its_dev *its_dev; struct lpi_chunk *lpis; mtx_assert(&sc->its_dev_lock, MA_OWNED); KASSERT((pci_dev == NULL || irq == 0), ("%s: Can't search by both pci_dev and irq number", __func__)); /* Find existing device if any */ TAILQ_FOREACH(its_dev, &sc->its_dev_list, entry) { if (pci_dev != NULL) { if (its_dev->pci_dev == pci_dev) return (its_dev); } else if (irq != 0) { lpis = &its_dev->lpis; if ((irq >= lpis->lpi_base) && (irq < (lpis->lpi_base + lpis->lpi_num))) return (its_dev); } } return (NULL); } static struct its_dev * its_device_alloc(struct gic_v3_its_softc *sc, device_t pci_dev, u_int nvecs) { struct its_dev *newdev; uint64_t typer; uint32_t devid; size_t esize; int err; mtx_lock_spin(&sc->its_dev_lock); /* Find existing device if any */ newdev = its_device_find_locked(sc, pci_dev, 0); mtx_unlock_spin(&sc->its_dev_lock); if (newdev != NULL) return (newdev); devid = its_get_devid(pci_dev); /* There was no previously created device. Create one now */ newdev = malloc(sizeof(*newdev), M_GIC_V3_ITS, (M_NOWAIT | M_ZERO)); if (newdev == NULL) return (NULL); newdev->pci_dev = pci_dev; newdev->devid = devid; err = lpi_alloc_chunk(sc, &newdev->lpis, nvecs); if (err != 0) { free(newdev, M_GIC_V3_ITS); return (NULL); } /* Get ITT entry size */ typer = gic_its_read(sc, 8, GITS_TYPER); esize = GITS_TYPER_ITTES(typer); /* * Allocate ITT for this device. * PA has to be 256 B aligned. At least two entries for device. */ newdev->itt = (vm_offset_t)contigmalloc( roundup2(roundup2(nvecs, 2) * esize, 0x100), M_GIC_V3_ITS, (M_NOWAIT | M_ZERO), 0, ~0UL, 0x100, 0); if (newdev->itt == 0) { lpi_free_chunk(sc, &newdev->lpis); free(newdev, M_GIC_V3_ITS); return (NULL); } mtx_lock_spin(&sc->its_dev_lock); TAILQ_INSERT_TAIL(&sc->its_dev_list, newdev, entry); mtx_unlock_spin(&sc->its_dev_lock); /* Map device to its ITT */ its_cmd_mapd(sc, newdev, 1); return (newdev); } static __inline void its_device_asign_lpi_locked(struct gic_v3_its_softc *sc, struct its_dev *its_dev, u_int *irq) { mtx_assert(&sc->its_dev_lock, MA_OWNED); if (its_dev->lpis.lpi_free == 0) { panic("Requesting more LPIs than allocated for this device. " "LPI num: %u, free %u", its_dev->lpis.lpi_num, its_dev->lpis.lpi_free); } *irq = its_dev->lpis.lpi_base + (its_dev->lpis.lpi_num - its_dev->lpis.lpi_free); its_dev->lpis.lpi_free--; } /* * ITS quirks. * Add vendor specific PCI devid function here. */ static uint32_t -its_get_devid_thunder(device_t pci_dev) -{ - int bsf; - int pem; - uint32_t bus; - - bus = pci_get_bus(pci_dev); - bsf = pci_get_rid(pci_dev); - - /* Check if accessing internal PCIe (low bus numbers) */ - if (bus < GIC_V3_ITS_QUIRK_THUNDERX_PEM_BUS_OFFSET) { - return ((pci_get_domain(pci_dev) << PCI_RID_DOMAIN_SHIFT) | - bsf); - /* PEM otherwise */ - } else { - /* PEM (PCIe MAC/root complex) number is equal to domain */ - pem = pci_get_domain(pci_dev); - - /* - * Set appropriate device ID (passed by the HW along with - * the transaction to memory) for different root complex - * numbers using hard-coded domain portion for each group. - */ - if (pem < 3) - return ((0x1 << PCI_RID_DOMAIN_SHIFT) | bsf); - - if (pem < 6) - return ((0x3 << PCI_RID_DOMAIN_SHIFT) | bsf); - - if (pem < 9) - return ((0x9 << PCI_RID_DOMAIN_SHIFT) | bsf); - - if (pem < 12) - return ((0xB << PCI_RID_DOMAIN_SHIFT) | bsf); - } - - return (0); -} - -static uint32_t its_get_devbits_thunder(device_t dev) { uint32_t devid_bits; /* * GITS_TYPER[17:13] of ThunderX reports that device IDs * are to be 21 bits in length. * The entry size of the ITS table can be read from GITS_BASERn[52:48] * and on ThunderX is supposed to be 8 bytes in length (for device * table). Finally the page size that is to be used by ITS to access * this table will be set to 64KB. * * This gives 0x200000 entries of size 0x8 bytes covered by 256 pages * each of which 64KB in size. The number of pages (minus 1) should * then be written to GITS_BASERn[7:0]. In that case this value would * be 0xFF but on ThunderX the maximum value that HW accepts is 0xFD. * * Set arbitrary number of device ID bits to 20 in order to limit * the number of entries in ITS device table to 0x100000 and hence * the table size to 8MB. */ devid_bits = 20; if (bootverbose) { device_printf(dev, "Limiting number of Device ID bits implemented to %d\n", devid_bits); } return (devid_bits); } static __inline uint32_t its_get_devbits_default(device_t dev) { uint64_t gits_typer; struct gic_v3_its_softc *sc; sc = device_get_softc(dev); gits_typer = gic_its_read(sc, 8, GITS_TYPER); return (GITS_TYPER_DEVB(gits_typer)); } static uint32_t its_get_devbits(device_t dev) { const struct its_quirks *quirk; size_t i; for (i = 0; i < nitems(its_quirks); i++) { quirk = &its_quirks[i]; if (CPU_MATCH_RAW(quirk->cpuid_mask, quirk->cpuid)) { if (quirk->devbits_func != NULL) return ((*quirk->devbits_func)(dev)); } } return (its_get_devbits_default(dev)); } -static __inline uint32_t -its_get_devid_default(device_t pci_dev) -{ - - return (PCI_DEVID_GENERIC(pci_dev)); -} - static uint32_t its_get_devid(device_t pci_dev) { - const struct its_quirks *quirk; - size_t i; + uintptr_t id; - for (i = 0; i < nitems(its_quirks); i++) { - quirk = &its_quirks[i]; - if (CPU_MATCH_RAW(quirk->cpuid_mask, quirk->cpuid)) { - if (quirk->devid_func != NULL) - return ((*quirk->devid_func)(pci_dev)); - } - } + if (pci_get_id(pci_dev, PCI_ID_MSI, &id) != 0) + panic("its_get_devid: Unable to get the MSI DeviceID"); - return (its_get_devid_default(pci_dev)); + return (id); } /* * Message signalled interrupts handling. */ /* * XXX ARM64TODO: Watch out for "irq" type. * * In theory GIC can handle up to (2^32 - 1) interrupt IDs whereas * we pass "irq" pointer of type integer. This is obviously wrong but * is determined by the way as PCI layer wants it to be done. */ int gic_v3_its_alloc_msix(device_t dev, device_t pci_dev, int *irq) { struct gic_v3_its_softc *sc; struct its_dev *its_dev; u_int nvecs; sc = device_get_softc(dev); nvecs = PCI_MSIX_NUM(pci_dev); /* * Allocate device as seen by ITS if not already available. * Notice that MSI-X interrupts are allocated on one-by-one basis. */ its_dev = its_device_alloc(sc, pci_dev, nvecs); if (its_dev == NULL) return (ENOMEM); mtx_lock_spin(&sc->its_dev_lock); its_device_asign_lpi_locked(sc, its_dev, irq); mtx_unlock_spin(&sc->its_dev_lock); return (0); } int gic_v3_its_alloc_msi(device_t dev, device_t pci_dev, int count, int *irqs) { struct gic_v3_its_softc *sc; struct its_dev *its_dev; sc = device_get_softc(dev); /* Allocate device as seen by ITS if not already available. */ its_dev = its_device_alloc(sc, pci_dev, count); if (its_dev == NULL) return (ENOMEM); mtx_lock_spin(&sc->its_dev_lock); for (; count > 0; count--) { its_device_asign_lpi_locked(sc, its_dev, irqs); irqs++; } mtx_unlock_spin(&sc->its_dev_lock); return (0); } int gic_v3_its_map_msi(device_t dev, device_t pci_dev, int irq, uint64_t *addr, uint32_t *data) { struct gic_v3_its_softc *sc; bus_space_handle_t its_bsh; struct its_dev *its_dev; uint64_t its_pa; uint32_t id; sc = device_get_softc(dev); /* Verify that this device is allocated and owns this LPI */ mtx_lock_spin(&sc->its_dev_lock); its_dev = its_device_find_locked(sc, pci_dev, 0); mtx_unlock_spin(&sc->its_dev_lock); if (its_dev == NULL) return (EINVAL); id = irq - its_dev->lpis.lpi_base; lpi_map_to_device(sc, its_dev, id, irq); its_bsh = rman_get_bushandle(&sc->its_res[0]); its_pa = vtophys(its_bsh); *addr = (its_pa + GITS_TRANSLATER); *data = id; return (0); } Index: head/sys/arm64/arm64/gic_v3_var.h =================================================================== --- head/sys/arm64/arm64/gic_v3_var.h (revision 299931) +++ head/sys/arm64/arm64/gic_v3_var.h (revision 299932) @@ -1,328 +1,326 @@ /*- * Copyright (c) 2015 The FreeBSD Foundation * All rights reserved. * * This software was developed by Semihalf under * the sponsorship of the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #ifndef _GIC_V3_VAR_H_ #define _GIC_V3_VAR_H_ #define GIC_V3_DEVSTR "ARM Generic Interrupt Controller v3.0" DECLARE_CLASS(gic_v3_driver); #define LPI_FLAGS_CONF_FLUSH (1UL << 0) #define LPI_CONFTAB_SIZE PAGE_SIZE_64K /* 1 bit per LPI + 1 KB more for the obligatory PPI, SGI, SPI stuff */ #define LPI_PENDTAB_SIZE ((LPI_CONFTAB_SIZE / 8) + 0x400) struct redist_lpis { vm_offset_t conf_base; vm_offset_t pend_base[MAXCPU]; uint64_t flags; }; struct gic_redists { /* * Re-Distributor region description. * We will have few of those depending * on the #redistributor-regions property in FDT. */ struct resource ** regions; /* Number of Re-Distributor regions */ u_int nregions; /* Per-CPU Re-Distributor handler */ struct resource * pcpu[MAXCPU]; /* LPIs data */ struct redist_lpis lpis; }; struct gic_v3_softc { device_t dev; struct resource ** gic_res; struct mtx gic_mtx; /* Distributor */ struct resource * gic_dist; /* Re-Distributors */ struct gic_redists gic_redists; u_int gic_nirqs; u_int gic_idbits; boolean_t gic_registered; }; MALLOC_DECLARE(M_GIC_V3); /* Device methods */ int gic_v3_attach(device_t dev); int gic_v3_detach(device_t dev); /* * ITS */ #define GIC_V3_ITS_DEVSTR "ARM GIC Interrupt Translation Service" #define GIC_V3_ITS_COMPSTR "arm,gic-v3-its" DECLARE_CLASS(gic_v3_its_driver); /* LPI chunk owned by ITS device */ struct lpi_chunk { u_int lpi_base; u_int lpi_num; u_int lpi_free; /* First free LPI in set */ u_int *lpi_col_ids; }; /* ITS device */ struct its_dev { TAILQ_ENTRY(its_dev) entry; /* PCI device */ device_t pci_dev; /* Device ID (i.e. PCI device ID) */ uint32_t devid; /* List of assigned LPIs */ struct lpi_chunk lpis; /* Virtual address of ITT */ vm_offset_t itt; }; TAILQ_HEAD(its_dev_list, its_dev); /* ITS private table description */ struct its_ptab { vm_offset_t ptab_vaddr; /* Virtual Address of table */ size_t ptab_pgsz; /* Page size */ size_t ptab_npages; /* Number of pages */ }; /* ITS collection description. */ struct its_col { uint64_t col_target; /* Target Re-Distributor */ uint64_t col_id; /* Collection ID */ }; /* ITS command. Each command is 32 bytes long */ struct its_cmd { uint64_t cmd_dword[4]; /* ITS command double word */ }; /* ITS commands encoding */ #define ITS_CMD_MOVI (0x01) #define ITS_CMD_SYNC (0x05) #define ITS_CMD_MAPD (0x08) #define ITS_CMD_MAPC (0x09) #define ITS_CMD_MAPVI (0x0a) #define ITS_CMD_MAPI (0x0b) #define ITS_CMD_INV (0x0c) #define ITS_CMD_INVALL (0x0d) /* Command */ #define CMD_COMMAND_MASK (0xFFUL) /* PCI device ID */ #define CMD_DEVID_SHIFT (32) #define CMD_DEVID_MASK (0xFFFFFFFFUL << CMD_DEVID_SHIFT) /* Size of IRQ ID bitfield */ #define CMD_SIZE_MASK (0xFFUL) /* Virtual LPI ID */ #define CMD_ID_MASK (0xFFFFFFFFUL) /* Physical LPI ID */ #define CMD_PID_SHIFT (32) #define CMD_PID_MASK (0xFFFFFFFFUL << CMD_PID_SHIFT) /* Collection */ #define CMD_COL_MASK (0xFFFFUL) /* Target (CPU or Re-Distributor) */ #define CMD_TARGET_SHIFT (16) #define CMD_TARGET_MASK (0xFFFFFFFFUL << CMD_TARGET_SHIFT) /* Interrupt Translation Table address */ #define CMD_ITT_MASK (0xFFFFFFFFFF00UL) /* Valid command bit */ #define CMD_VALID_SHIFT (63) #define CMD_VALID_MASK (1UL << CMD_VALID_SHIFT) /* * ITS command descriptor. * Idea for command description passing taken from Linux. */ struct its_cmd_desc { uint8_t cmd_type; union { struct { struct its_dev *its_dev; struct its_col *col; uint32_t id; } cmd_desc_movi; struct { struct its_col *col; } cmd_desc_sync; struct { struct its_col *col; uint8_t valid; } cmd_desc_mapc; struct { struct its_dev *its_dev; struct its_col *col; uint32_t pid; uint32_t id; } cmd_desc_mapvi; struct { struct its_dev *its_dev; struct its_col *col; uint32_t pid; } cmd_desc_mapi; struct { struct its_dev *its_dev; uint8_t valid; } cmd_desc_mapd; struct { struct its_dev *its_dev; struct its_col *col; uint32_t pid; } cmd_desc_inv; struct { struct its_col *col; } cmd_desc_invall; }; }; #define ITS_CMDQ_SIZE PAGE_SIZE_64K #define ITS_CMDQ_NENTRIES (ITS_CMDQ_SIZE / sizeof(struct its_cmd)) #define ITS_FLAGS_CMDQ_FLUSH (1UL << 0) #define ITS_TARGET_NONE 0xFBADBEEF struct gic_v3_its_softc { device_t dev; struct resource * its_res; struct its_cmd * its_cmdq_base; /* ITS command queue base */ struct its_cmd * its_cmdq_write; /* ITS command queue write ptr */ struct its_ptab its_ptabs[GITS_BASER_NUM];/* ITS private tables */ struct its_col * its_cols[MAXCPU];/* Per-CPU collections */ uint64_t its_flags; struct its_dev_list its_dev_list; bitstr_t * its_lpi_bitmap; uint32_t its_lpi_maxid; struct mtx its_dev_lock; struct mtx its_cmd_lock; uint32_t its_socket; /* Socket number ITS is attached to */ }; /* Stuff that is specific to the vendor's implementation */ typedef uint32_t (*its_devbits_func_t)(device_t); -typedef uint32_t (*its_devid_func_t)(device_t); struct its_quirks { uint64_t cpuid; uint64_t cpuid_mask; - its_devid_func_t devid_func; its_devbits_func_t devbits_func; }; extern devclass_t gic_v3_its_devclass; int gic_v3_its_detach(device_t); int gic_v3_its_alloc_msix(device_t, device_t, int *); int gic_v3_its_alloc_msi(device_t, device_t, int, int *); int gic_v3_its_map_msi(device_t, device_t, int, uint64_t *, uint32_t *); int its_init_cpu(struct gic_v3_its_softc *); int lpi_migrate(device_t, uint32_t, u_int); void lpi_unmask_irq(device_t, uint32_t); void lpi_mask_irq(device_t, uint32_t); /* * GIC Distributor accessors. * Notice that only GIC sofc can be passed. */ #define gic_d_read(sc, len, reg) \ ({ \ bus_read_##len(sc->gic_dist, reg); \ }) #define gic_d_write(sc, len, reg, val) \ ({ \ bus_write_##len(sc->gic_dist, reg, val);\ }) /* GIC Re-Distributor accessors (per-CPU) */ #define gic_r_read(sc, len, reg) \ ({ \ u_int cpu = PCPU_GET(cpuid); \ \ bus_read_##len( \ sc->gic_redists.pcpu[cpu], \ reg); \ }) #define gic_r_write(sc, len, reg, val) \ ({ \ u_int cpu = PCPU_GET(cpuid); \ \ bus_write_##len( \ sc->gic_redists.pcpu[cpu], \ reg, val); \ }) #define PCI_DEVID_GENERIC(pci_dev) \ ({ \ ((pci_get_domain(pci_dev) << PCI_RID_DOMAIN_SHIFT) | \ (pci_get_bus(pci_dev) << PCI_RID_BUS_SHIFT) | \ (pci_get_slot(pci_dev) << PCI_RID_SLOT_SHIFT) | \ (pci_get_function(pci_dev) << PCI_RID_FUNC_SHIFT)); \ }) /* * Request number of maximum MSI-X vectors for this device. * Device can ask for less vectors than maximum supported but not more. */ #define PCI_MSIX_NUM(pci_dev) \ ({ \ struct pci_devinfo *dinfo; \ pcicfgregs *cfg; \ \ dinfo = device_get_ivars(pci_dev); \ cfg = &dinfo->cfg; \ \ cfg->msix.msix_msgnum; \ }) #endif /* _GIC_V3_VAR_H_ */ Index: head/sys/arm64/cavium/thunder_pcie_fdt.c =================================================================== --- head/sys/arm64/cavium/thunder_pcie_fdt.c (revision 299931) +++ head/sys/arm64/cavium/thunder_pcie_fdt.c (revision 299932) @@ -1,128 +1,159 @@ /* * Copyright (C) 2016 Cavium Inc. * All rights reserved. * * Developed by Semihalf. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include "opt_platform.h" #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include + +#include +#include #include +#include #include "thunder_pcie_common.h" +#include "pcib_if.h" + #ifdef THUNDERX_PASS_1_1_ERRATA static struct resource * thunder_pcie_fdt_alloc_resource(device_t, device_t, int, int *, rman_res_t, rman_res_t, rman_res_t, u_int); #endif static int thunder_pcie_fdt_attach(device_t); static int thunder_pcie_fdt_probe(device_t); +static int thunder_pcie_fdt_get_id(device_t, device_t, enum pci_id_type, + uintptr_t *); static device_method_t thunder_pcie_fdt_methods[] = { /* Device interface */ DEVMETHOD(device_probe, thunder_pcie_fdt_probe), DEVMETHOD(device_attach, thunder_pcie_fdt_attach), #ifdef THUNDERX_PASS_1_1_ERRATA DEVMETHOD(bus_alloc_resource, thunder_pcie_fdt_alloc_resource), #endif + /* pcib interface */ + DEVMETHOD(pcib_get_id, thunder_pcie_fdt_get_id), + /* End */ DEVMETHOD_END }; DEFINE_CLASS_1(pcib, thunder_pcie_fdt_driver, thunder_pcie_fdt_methods, sizeof(struct generic_pcie_softc), generic_pcie_driver); static devclass_t thunder_pcie_fdt_devclass; DRIVER_MODULE(thunder_pcib, simplebus, thunder_pcie_fdt_driver, thunder_pcie_fdt_devclass, 0, 0); DRIVER_MODULE(thunder_pcib, ofwbus, thunder_pcie_fdt_driver, thunder_pcie_fdt_devclass, 0, 0); static int thunder_pcie_fdt_probe(device_t dev) { /* Check if we're running on Cavium ThunderX */ if (!CPU_MATCH(CPU_IMPL_MASK | CPU_PART_MASK, CPU_IMPL_CAVIUM, CPU_PART_THUNDER, 0, 0)) return (ENXIO); if (!ofw_bus_status_okay(dev)) return (ENXIO); if (ofw_bus_is_compatible(dev, "pci-host-ecam-generic") || ofw_bus_is_compatible(dev, "cavium,thunder-pcie") || ofw_bus_is_compatible(dev, "cavium,pci-host-thunder-ecam")) { device_set_desc(dev, "Cavium Integrated PCI/PCI-E Controller"); return (BUS_PROBE_DEFAULT); } return (ENXIO); } static int thunder_pcie_fdt_attach(device_t dev) { struct generic_pcie_softc *sc; sc = device_get_softc(dev); thunder_pcie_identify_ecam(dev, &sc->ecam); return (pci_host_generic_attach(dev)); +} + +static int +thunder_pcie_fdt_get_id(device_t pci, device_t child, enum pci_id_type type, + uintptr_t *id) +{ + phandle_t node; + int bsf; + + if (type != PCI_ID_MSI) + return (pcib_get_id(pci, child, type, id)); + + node = ofw_bus_get_node(pci); + if (OF_hasprop(node, "msi-map")) + return (generic_pcie_get_id(pci, child, type, id)); + + bsf = pci_get_rid(child); + *id = (pci_get_domain(child) << PCI_RID_DOMAIN_SHIFT) | bsf; + + return (0); } #ifdef THUNDERX_PASS_1_1_ERRATA static struct resource * thunder_pcie_fdt_alloc_resource(device_t dev, device_t child, int type, int *rid, rman_res_t start, rman_res_t end, rman_res_t count, u_int flags) { if ((int)ofw_bus_get_node(child) > 0) return (pci_host_generic_alloc_resource(dev, child, type, rid, start, end, count, flags)); return (thunder_pcie_alloc_resource(dev, child, type, rid, start, end, count, flags)); } #endif Index: head/sys/arm64/cavium/thunder_pcie_pem.c =================================================================== --- head/sys/arm64/cavium/thunder_pcie_pem.c (revision 299931) +++ head/sys/arm64/cavium/thunder_pcie_pem.c (revision 299932) @@ -1,844 +1,882 @@ /*- * Copyright (c) 2015 The FreeBSD Foundation * All rights reserved. * * This software was developed by Semihalf under * the sponsorship of the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* PCIe external MAC root complex driver (PEM) for Cavium Thunder SOC */ #include __FBSDID("$FreeBSD$"); #include "opt_platform.h" #include #include #include #include #include #include #include #include #ifdef FDT #include #include #include #include #endif #include #include #include +#include #include #include #include #include #include #include #include "pcib_if.h" #define THUNDER_PEM_DEVICE_ID 0xa020 #define THUNDER_PEM_VENDOR_ID 0x177d /* ThunderX specific defines */ #define THUNDER_PEMn_REG_BASE(unit) (0x87e0c0000000UL | ((unit) << 24)) #define PCIERC_CFG002 0x08 #define PCIERC_CFG006 0x18 #define PCIERC_CFG032 0x80 #define PCIERC_CFG006_SEC_BUS(reg) (((reg) >> 8) & 0xFF) #define PEM_CFG_RD_REG_ALIGN(reg) ((reg) & ~0x3) #define PEM_CFG_RD_REG_DATA(val) (((val) >> 32) & 0xFFFFFFFF) #define PEM_CFG_RD 0x30 #define PEM_CFG_LINK_MASK 0x3 #define PEM_CFG_LINK_RDY 0x3 #define PEM_CFG_SLIX_TO_REG(slix) ((slix) << 4) #define SBNUM_OFFSET 0x8 #define SBNUM_MASK 0xFF #define PEM_ON_REG 0x420 #define PEM_CTL_STATUS 0x0 #define PEM_LINK_ENABLE (1 << 4) #define PEM_LINK_DLLA (1 << 29) #define PEM_LINK_LT (1 << 27) #define PEM_BUS_SHIFT (24) #define PEM_SLOT_SHIFT (19) #define PEM_FUNC_SHIFT (16) #define SLIX_S2M_REGX_ACC 0x874001000000UL #define SLIX_S2M_REGX_ACC_SIZE 0x1000 #define SLIX_S2M_REGX_ACC_SPACING 0x001000000000UL #define SLI_BASE 0x880000000000UL #define SLI_WINDOW_SPACING 0x004000000000UL #define SLI_PCI_OFFSET 0x001000000000UL #define SLI_NODE_SHIFT (44) #define SLI_NODE_MASK (3) #define SLI_GROUP_SHIFT (40) #define SLI_ID_SHIFT (24) #define SLI_ID_MASK (7) #define SLI_PEMS_PER_GROUP (3) #define SLI_GROUPS_PER_NODE (2) #define SLI_PEMS_PER_NODE (SLI_PEMS_PER_GROUP * SLI_GROUPS_PER_NODE) #define SLI_ACC_REG_CNT (256) /* * Each PEM device creates its own bus with * own address translation, so we can adjust bus addresses * as we want. To support 32-bit cards let's assume * PCI window assignment looks as following: * * 0x00000000 - 0x000FFFFF IO * 0x00100000 - 0xFFFFFFFF Memory */ #define PCI_IO_BASE 0x00000000UL #define PCI_IO_SIZE 0x00100000UL #define PCI_MEMORY_BASE PCI_IO_SIZE #define PCI_MEMORY_SIZE 0xFFF00000UL #define RID_PEM_SPACE 1 static int thunder_pem_activate_resource(device_t, device_t, int, int, struct resource *); static int thunder_pem_adjust_resource(device_t, device_t, int, struct resource *, rman_res_t, rman_res_t); static struct resource * thunder_pem_alloc_resource(device_t, device_t, int, int *, rman_res_t, rman_res_t, rman_res_t, u_int); static int thunder_pem_alloc_msi(device_t, device_t, int, int, int *); static int thunder_pem_release_msi(device_t, device_t, int, int *); static int thunder_pem_alloc_msix(device_t, device_t, int *); static int thunder_pem_release_msix(device_t, device_t, int); static int thunder_pem_map_msi(device_t, device_t, int, uint64_t *, uint32_t *); +static int thunder_pem_get_id(device_t, device_t, enum pci_id_type, + uintptr_t *); static int thunder_pem_attach(device_t); static int thunder_pem_deactivate_resource(device_t, device_t, int, int, struct resource *); static int thunder_pem_detach(device_t); static uint64_t thunder_pem_config_reg_read(struct thunder_pem_softc *, int); static int thunder_pem_link_init(struct thunder_pem_softc *); static int thunder_pem_maxslots(device_t); static int thunder_pem_probe(device_t); static uint32_t thunder_pem_read_config(device_t, u_int, u_int, u_int, u_int, int); static int thunder_pem_read_ivar(device_t, device_t, int, uintptr_t *); static void thunder_pem_release_all(device_t); static int thunder_pem_release_resource(device_t, device_t, int, int, struct resource *); static struct rman * thunder_pem_rman(struct thunder_pem_softc *, int); static void thunder_pem_slix_s2m_regx_acc_modify(struct thunder_pem_softc *, int, int); static void thunder_pem_write_config(device_t, u_int, u_int, u_int, u_int, uint32_t, int); static int thunder_pem_write_ivar(device_t, device_t, int, uintptr_t); /* Global handlers for SLI interface */ static bus_space_handle_t sli0_s2m_regx_base = 0; static bus_space_handle_t sli1_s2m_regx_base = 0; static device_method_t thunder_pem_methods[] = { /* Device interface */ DEVMETHOD(device_probe, thunder_pem_probe), DEVMETHOD(device_attach, thunder_pem_attach), DEVMETHOD(device_detach, thunder_pem_detach), /* Bus interface */ DEVMETHOD(bus_read_ivar, thunder_pem_read_ivar), DEVMETHOD(bus_write_ivar, thunder_pem_write_ivar), DEVMETHOD(bus_alloc_resource, thunder_pem_alloc_resource), DEVMETHOD(bus_release_resource, thunder_pem_release_resource), DEVMETHOD(bus_adjust_resource, thunder_pem_adjust_resource), DEVMETHOD(bus_activate_resource, thunder_pem_activate_resource), DEVMETHOD(bus_deactivate_resource, thunder_pem_deactivate_resource), DEVMETHOD(bus_setup_intr, bus_generic_setup_intr), DEVMETHOD(bus_teardown_intr, bus_generic_teardown_intr), /* pcib interface */ DEVMETHOD(pcib_maxslots, thunder_pem_maxslots), DEVMETHOD(pcib_read_config, thunder_pem_read_config), DEVMETHOD(pcib_write_config, thunder_pem_write_config), DEVMETHOD(pcib_alloc_msix, thunder_pem_alloc_msix), DEVMETHOD(pcib_release_msix, thunder_pem_release_msix), DEVMETHOD(pcib_alloc_msi, thunder_pem_alloc_msi), DEVMETHOD(pcib_release_msi, thunder_pem_release_msi), DEVMETHOD(pcib_map_msi, thunder_pem_map_msi), + DEVMETHOD(pcib_get_id, thunder_pem_get_id), DEVMETHOD_END }; DEFINE_CLASS_0(pcib, thunder_pem_driver, thunder_pem_methods, sizeof(struct thunder_pem_softc)); static devclass_t thunder_pem_devclass; extern struct bus_space memmap_bus; DRIVER_MODULE(thunder_pem, pci, thunder_pem_driver, thunder_pem_devclass, 0, 0); MODULE_DEPEND(thunder_pem, pci, 1, 1, 1); static int thunder_pem_maxslots(device_t dev) { #if 0 /* max slots per bus acc. to standard */ return (PCI_SLOTMAX); #else /* * ARM64TODO Workaround - otherwise an em(4) interface appears to be * present on every PCI function on the bus to which it is connected */ return (0); #endif } static int thunder_pem_read_ivar(device_t dev, device_t child, int index, uintptr_t *result) { struct thunder_pem_softc *sc; int secondary_bus = 0; sc = device_get_softc(dev); if (index == PCIB_IVAR_BUS) { secondary_bus = thunder_pem_config_reg_read(sc, PCIERC_CFG006); *result = PCIERC_CFG006_SEC_BUS(secondary_bus); return (0); } if (index == PCIB_IVAR_DOMAIN) { *result = sc->id; return (0); } return (ENOENT); } static int thunder_pem_write_ivar(device_t dev, device_t child, int index, uintptr_t value) { return (ENOENT); } static int thunder_pem_activate_resource(device_t dev, device_t child, int type, int rid, struct resource *r) { int err; bus_addr_t paddr; bus_size_t psize; bus_space_handle_t vaddr; struct thunder_pem_softc *sc; if ((err = rman_activate_resource(r)) != 0) return (err); sc = device_get_softc(dev); /* * If this is a memory resource, map it into the kernel. */ if (type == SYS_RES_MEMORY || type == SYS_RES_IOPORT) { paddr = (bus_addr_t)rman_get_start(r); psize = (bus_size_t)rman_get_size(r); paddr = range_addr_pci_to_phys(sc->ranges, paddr); err = bus_space_map(&memmap_bus, paddr, psize, 0, &vaddr); if (err != 0) { rman_deactivate_resource(r); return (err); } rman_set_bustag(r, &memmap_bus); rman_set_virtual(r, (void *)vaddr); rman_set_bushandle(r, vaddr); } return (0); } /* * This function is an exact copy of nexus_deactivate_resource() * Keep it up-to-date with all changes in nexus. To be removed * once bus-mapping interface is developed. */ static int thunder_pem_deactivate_resource(device_t bus, device_t child, int type, int rid, struct resource *r) { bus_size_t psize; bus_space_handle_t vaddr; psize = (bus_size_t)rman_get_size(r); vaddr = rman_get_bushandle(r); if (vaddr != 0) { bus_space_unmap(&memmap_bus, vaddr, psize); rman_set_virtual(r, NULL); rman_set_bushandle(r, 0); } return (rman_deactivate_resource(r)); } static int thunder_pem_adjust_resource(device_t dev, device_t child, int type, struct resource *res, rman_res_t start, rman_res_t end) { struct thunder_pem_softc *sc; struct rman *rm; sc = device_get_softc(dev); rm = thunder_pem_rman(sc, type); if (rm == NULL) return (bus_generic_adjust_resource(dev, child, type, res, start, end)); if (!rman_is_region_manager(res, rm)) /* * This means a child device has a memory or I/O * resource not from you which shouldn't happen. */ return (EINVAL); return (rman_adjust_resource(res, start, end)); } static int thunder_pem_alloc_msi(device_t pci, device_t child, int count, int maxcount, int *irqs) { device_t bus; bus = device_get_parent(pci); return (PCIB_ALLOC_MSI(device_get_parent(bus), child, count, maxcount, irqs)); } static int thunder_pem_release_msi(device_t pci, device_t child, int count, int *irqs) { device_t bus; bus = device_get_parent(pci); return (PCIB_RELEASE_MSI(device_get_parent(bus), child, count, irqs)); } static int thunder_pem_alloc_msix(device_t pci, device_t child, int *irq) { device_t bus; bus = device_get_parent(pci); return (PCIB_ALLOC_MSIX(device_get_parent(bus), child, irq)); } static int thunder_pem_release_msix(device_t pci, device_t child, int irq) { device_t bus; bus = device_get_parent(pci); return (PCIB_RELEASE_MSIX(device_get_parent(bus), child, irq)); } static int thunder_pem_map_msi(device_t pci, device_t child, int irq, uint64_t *addr, uint32_t *data) { device_t bus; bus = device_get_parent(pci); return (PCIB_MAP_MSI(device_get_parent(bus), child, irq, addr, data)); +} + +static int +thunder_pem_get_id(device_t pci, device_t child, enum pci_id_type type, + uintptr_t *id) +{ + int bsf; + int pem; + + if (type != PCI_ID_MSI) + return (pcib_get_id(pci, child, type, id)); + + bsf = pci_get_rid(child); + + /* PEM (PCIe MAC/root complex) number is equal to domain */ + pem = pci_get_domain(child); + + /* + * Set appropriate device ID (passed by the HW along with + * the transaction to memory) for different root complex + * numbers using hard-coded domain portion for each group. + */ + if (pem < 3) + *id = (0x1 << PCI_RID_DOMAIN_SHIFT) | bsf; + else if (pem < 6) + *id = (0x3 << PCI_RID_DOMAIN_SHIFT) | bsf; + else if (pem < 9) + *id = (0x9 << PCI_RID_DOMAIN_SHIFT) | bsf; + else if (pem < 12) + *id = (0xB << PCI_RID_DOMAIN_SHIFT) | bsf; + else + return (ENXIO); + + return (0); } static int thunder_pem_identify(device_t dev) { struct thunder_pem_softc *sc; rman_res_t start; sc = device_get_softc(dev); start = rman_get_start(sc->reg); /* Calculate PEM designations from its address */ sc->node = (start >> SLI_NODE_SHIFT) & SLI_NODE_MASK; sc->id = ((start >> SLI_ID_SHIFT) & SLI_ID_MASK) + (SLI_PEMS_PER_NODE * sc->node); sc->sli = sc->id % SLI_PEMS_PER_GROUP; sc->sli_group = (sc->id / SLI_PEMS_PER_GROUP) % SLI_GROUPS_PER_NODE; sc->sli_window_base = SLI_BASE | (((uint64_t)sc->node) << SLI_NODE_SHIFT) | ((uint64_t)sc->sli_group << SLI_GROUP_SHIFT); sc->sli_window_base += SLI_WINDOW_SPACING * sc->sli; return (0); } static void thunder_pem_slix_s2m_regx_acc_modify(struct thunder_pem_softc *sc, int sli_group, int slix) { uint64_t regval; bus_space_handle_t handle = 0; KASSERT(slix >= 0 && slix <= SLI_ACC_REG_CNT, ("Invalid SLI index")); if (sli_group == 0) handle = sli0_s2m_regx_base; else if (sli_group == 1) handle = sli1_s2m_regx_base; else device_printf(sc->dev, "SLI group is not correct\n"); if (handle) { /* Clear lower 32-bits of the SLIx register */ regval = bus_space_read_8(sc->reg_bst, handle, PEM_CFG_SLIX_TO_REG(slix)); regval &= ~(0xFFFFFFFFUL); bus_space_write_8(sc->reg_bst, handle, PEM_CFG_SLIX_TO_REG(slix), regval); } } static int thunder_pem_link_init(struct thunder_pem_softc *sc) { uint64_t regval; /* check whether PEM is safe to access. */ regval = bus_space_read_8(sc->reg_bst, sc->reg_bsh, PEM_ON_REG); if ((regval & PEM_CFG_LINK_MASK) != PEM_CFG_LINK_RDY) { device_printf(sc->dev, "PEM%d is not ON\n", sc->id); return (ENXIO); } regval = bus_space_read_8(sc->reg_bst, sc->reg_bsh, PEM_CTL_STATUS); regval |= PEM_LINK_ENABLE; bus_space_write_8(sc->reg_bst, sc->reg_bsh, PEM_CTL_STATUS, regval); /* Wait 1ms as per Cavium specification */ DELAY(1000); regval = thunder_pem_config_reg_read(sc, PCIERC_CFG032); if (((regval & PEM_LINK_DLLA) == 0) || ((regval & PEM_LINK_LT) != 0)) { device_printf(sc->dev, "PCIe RC: Port %d Link Timeout\n", sc->id); return (ENXIO); } return (0); } static int thunder_pem_init(struct thunder_pem_softc *sc) { int i, retval = 0; retval = thunder_pem_link_init(sc); if (retval) { device_printf(sc->dev, "%s failed\n", __func__); return retval; } /* To support 32-bit PCIe devices, set S2M_REGx_ACC[BA]=0x0 */ for (i = 0; i < SLI_ACC_REG_CNT; i++) { thunder_pem_slix_s2m_regx_acc_modify(sc, sc->sli_group, i); } return (retval); } static uint64_t thunder_pem_config_reg_read(struct thunder_pem_softc *sc, int reg) { uint64_t data; /* Write to ADDR register */ bus_space_write_8(sc->reg_bst, sc->reg_bsh, PEM_CFG_RD, PEM_CFG_RD_REG_ALIGN(reg)); bus_space_barrier(sc->reg_bst, sc->reg_bsh, PEM_CFG_RD, 8, BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE); /* Read from DATA register */ data = PEM_CFG_RD_REG_DATA(bus_space_read_8(sc->reg_bst, sc->reg_bsh, PEM_CFG_RD)); return (data); } static uint32_t thunder_pem_read_config(device_t dev, u_int bus, u_int slot, u_int func, u_int reg, int bytes) { uint64_t offset; uint32_t data; struct thunder_pem_softc *sc; bus_space_tag_t t; bus_space_handle_t h; if ((bus > PCI_BUSMAX) || (slot > PCI_SLOTMAX) || (func > PCI_FUNCMAX) || (reg > PCIE_REGMAX)) return (~0U); sc = device_get_softc(dev); /* Calculate offset */ offset = (bus << PEM_BUS_SHIFT) | (slot << PEM_SLOT_SHIFT) | (func << PEM_FUNC_SHIFT); t = sc->reg_bst; h = sc->pem_sli_base; bus_space_map(sc->reg_bst, sc->sli_window_base + offset, PCIE_REGMAX, 0, &h); switch (bytes) { case 1: data = bus_space_read_1(t, h, reg); break; case 2: data = le16toh(bus_space_read_2(t, h, reg)); break; case 4: data = le32toh(bus_space_read_4(t, h, reg)); break; default: data = ~0U; break; } bus_space_unmap(sc->reg_bst, h, PCIE_REGMAX); return (data); } static void thunder_pem_write_config(device_t dev, u_int bus, u_int slot, u_int func, u_int reg, uint32_t val, int bytes) { uint64_t offset; struct thunder_pem_softc *sc; bus_space_tag_t t; bus_space_handle_t h; if ((bus > PCI_BUSMAX) || (slot > PCI_SLOTMAX) || (func > PCI_FUNCMAX) || (reg > PCIE_REGMAX)) return; sc = device_get_softc(dev); /* Calculate offset */ offset = (bus << PEM_BUS_SHIFT) | (slot << PEM_SLOT_SHIFT) | (func << PEM_FUNC_SHIFT); t = sc->reg_bst; h = sc->pem_sli_base; bus_space_map(sc->reg_bst, sc->sli_window_base + offset, PCIE_REGMAX, 0, &h); switch (bytes) { case 1: bus_space_write_1(t, h, reg, val); break; case 2: bus_space_write_2(t, h, reg, htole16(val)); break; case 4: bus_space_write_4(t, h, reg, htole32(val)); break; default: break; } bus_space_unmap(sc->reg_bst, h, PCIE_REGMAX); } static struct resource * thunder_pem_alloc_resource(device_t dev, device_t child, int type, int *rid, rman_res_t start, rman_res_t end, rman_res_t count, u_int flags) { struct thunder_pem_softc *sc = device_get_softc(dev); struct rman *rm = NULL; struct resource *res; device_t parent_dev; rm = thunder_pem_rman(sc, type); if (rm == NULL) { /* Find parent device. On ThunderX we know an exact path. */ parent_dev = device_get_parent(device_get_parent(dev)); return (BUS_ALLOC_RESOURCE(parent_dev, dev, type, rid, start, end, count, flags)); } if (!RMAN_IS_DEFAULT_RANGE(start, end)) { /* * We might get PHYS addresses here inherited from EFI. * Convert to PCI if necessary. */ if (range_addr_is_phys(sc->ranges, start, count)) { start = range_addr_phys_to_pci(sc->ranges, start); end = start + count - 1; } } if (bootverbose) { device_printf(dev, "thunder_pem_alloc_resource: start=%#lx, end=%#lx, count=%#lx\n", start, end, count); } res = rman_reserve_resource(rm, start, end, count, flags, child); if (res == NULL) goto fail; rman_set_rid(res, *rid); if (flags & RF_ACTIVE) if (bus_activate_resource(child, type, *rid, res)) { rman_release_resource(res); goto fail; } return (res); fail: if (bootverbose) { device_printf(dev, "%s FAIL: type=%d, rid=%d, " "start=%016lx, end=%016lx, count=%016lx, flags=%x\n", __func__, type, *rid, start, end, count, flags); } return (NULL); } static int thunder_pem_release_resource(device_t dev, device_t child, int type, int rid, struct resource *res) { device_t parent_dev; /* Find parent device. On ThunderX we know an exact path. */ parent_dev = device_get_parent(device_get_parent(dev)); if ((type != SYS_RES_MEMORY) && (type != SYS_RES_IOPORT)) return (BUS_RELEASE_RESOURCE(parent_dev, child, type, rid, res)); return (rman_release_resource(res)); } static struct rman * thunder_pem_rman(struct thunder_pem_softc *sc, int type) { switch (type) { case SYS_RES_IOPORT: return (&sc->io_rman); case SYS_RES_MEMORY: return (&sc->mem_rman); default: break; } return (NULL); } static int thunder_pem_probe(device_t dev) { uint16_t pci_vendor_id; uint16_t pci_device_id; pci_vendor_id = pci_get_vendor(dev); pci_device_id = pci_get_device(dev); if ((pci_vendor_id == THUNDER_PEM_VENDOR_ID) && (pci_device_id == THUNDER_PEM_DEVICE_ID)) { device_set_desc_copy(dev, THUNDER_PEM_DESC); return (0); } return (ENXIO); } static int thunder_pem_attach(device_t dev) { devclass_t pci_class; device_t parent; struct thunder_pem_softc *sc; int error; int rid; int tuple; uint64_t base, size; struct rman *rman; sc = device_get_softc(dev); sc->dev = dev; /* Allocate memory for resource */ pci_class = devclass_find("pci"); parent = device_get_parent(dev); if (device_get_devclass(parent) == pci_class) rid = PCIR_BAR(0); else rid = RID_PEM_SPACE; sc->reg = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, RF_ACTIVE); if (sc->reg == NULL) { device_printf(dev, "Failed to allocate resource\n"); return (ENXIO); } sc->reg_bst = rman_get_bustag(sc->reg); sc->reg_bsh = rman_get_bushandle(sc->reg); /* Map SLI, do it only once */ if (!sli0_s2m_regx_base) { bus_space_map(sc->reg_bst, SLIX_S2M_REGX_ACC, SLIX_S2M_REGX_ACC_SIZE, 0, &sli0_s2m_regx_base); } if (!sli1_s2m_regx_base) { bus_space_map(sc->reg_bst, SLIX_S2M_REGX_ACC + SLIX_S2M_REGX_ACC_SPACING, SLIX_S2M_REGX_ACC_SIZE, 0, &sli1_s2m_regx_base); } if ((sli0_s2m_regx_base == 0) || (sli1_s2m_regx_base == 0)) { device_printf(dev, "bus_space_map failed to map slix_s2m_regx_base\n"); goto fail; } /* Identify PEM */ if (thunder_pem_identify(dev) != 0) goto fail; /* Initialize rman and allocate regions */ sc->mem_rman.rm_type = RMAN_ARRAY; sc->mem_rman.rm_descr = "PEM PCIe Memory"; error = rman_init(&sc->mem_rman); if (error != 0) { device_printf(dev, "memory rman_init() failed. error = %d\n", error); goto fail; } sc->io_rman.rm_type = RMAN_ARRAY; sc->io_rman.rm_descr = "PEM PCIe IO"; error = rman_init(&sc->io_rman); if (error != 0) { device_printf(dev, "IO rman_init() failed. error = %d\n", error); goto fail_mem; } /* * We ignore the values that may have been provided in FDT * and configure ranges according to the below formula * for all types of devices. This is because some DTBs provided * by EFI do not have proper ranges property or don't have them * at all. */ /* Fill memory window */ sc->ranges[0].pci_base = PCI_MEMORY_BASE; sc->ranges[0].size = PCI_MEMORY_SIZE; sc->ranges[0].phys_base = sc->sli_window_base + SLI_PCI_OFFSET + sc->ranges[0].pci_base; sc->ranges[0].flags = SYS_RES_MEMORY; /* Fill IO window */ sc->ranges[1].pci_base = PCI_IO_BASE; sc->ranges[1].size = PCI_IO_SIZE; sc->ranges[1].phys_base = sc->sli_window_base + SLI_PCI_OFFSET + sc->ranges[1].pci_base; sc->ranges[1].flags = SYS_RES_IOPORT; for (tuple = 0; tuple < MAX_RANGES_TUPLES; tuple++) { base = sc->ranges[tuple].pci_base; size = sc->ranges[tuple].size; if (size == 0) continue; /* empty range element */ rman = thunder_pem_rman(sc, sc->ranges[tuple].flags); if (rman != NULL) error = rman_manage_region(rman, base, base + size - 1); else error = EINVAL; if (error) { device_printf(dev, "rman_manage_region() failed. error = %d\n", error); rman_fini(&sc->mem_rman); return (error); } if (bootverbose) { device_printf(dev, "\tPCI addr: 0x%jx, CPU addr: 0x%jx, Size: 0x%jx, Flags:0x%jx\n", sc->ranges[tuple].pci_base, sc->ranges[tuple].phys_base, sc->ranges[tuple].size, sc->ranges[tuple].flags); } } if (thunder_pem_init(sc)) { device_printf(dev, "Failure during PEM init\n"); goto fail_io; } device_add_child(dev, "pci", -1); return (bus_generic_attach(dev)); fail_io: rman_fini(&sc->io_rman); fail_mem: rman_fini(&sc->mem_rman); fail: bus_free_resource(dev, SYS_RES_MEMORY, sc->reg); return (ENXIO); } static void thunder_pem_release_all(device_t dev) { struct thunder_pem_softc *sc; sc = device_get_softc(dev); rman_fini(&sc->io_rman); rman_fini(&sc->mem_rman); if (sc->reg != NULL) bus_free_resource(dev, SYS_RES_MEMORY, sc->reg); } static int thunder_pem_detach(device_t dev) { thunder_pem_release_all(dev); return (0); } Index: head/sys/arm64/cavium/thunder_pcie_pem_fdt.c =================================================================== --- head/sys/arm64/cavium/thunder_pcie_pem_fdt.c (revision 299931) +++ head/sys/arm64/cavium/thunder_pcie_pem_fdt.c (revision 299932) @@ -1,144 +1,167 @@ /* * Copyright (C) 2016 Cavium Inc. * All rights reserved. * * Developed by Semihalf. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include "opt_platform.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "thunder_pcie_common.h" #include "thunder_pcie_pem.h" #include "pcib_if.h" static int thunder_pem_fdt_probe(device_t); static int thunder_pem_fdt_alloc_msix(device_t, device_t, int *); static int thunder_pem_fdt_release_msix(device_t, device_t, int); static int thunder_pem_fdt_alloc_msi(device_t, device_t, int, int, int *); static int thunder_pem_fdt_release_msi(device_t, device_t, int, int *); static int thunder_pem_fdt_map_msi(device_t, device_t, int, uint64_t *, uint32_t *); +static int thunder_pem_fdt_get_id(device_t, device_t, enum pci_id_type, + uintptr_t *); static device_method_t thunder_pem_fdt_methods[] = { /* Device interface */ DEVMETHOD(device_probe, thunder_pem_fdt_probe), /* pcib interface */ DEVMETHOD(pcib_alloc_msix, thunder_pem_fdt_alloc_msix), DEVMETHOD(pcib_release_msix, thunder_pem_fdt_release_msix), DEVMETHOD(pcib_alloc_msi, thunder_pem_fdt_alloc_msi), DEVMETHOD(pcib_release_msi, thunder_pem_fdt_release_msi), DEVMETHOD(pcib_map_msi, thunder_pem_fdt_map_msi), + DEVMETHOD(pcib_get_id, thunder_pem_fdt_get_id), /* End */ DEVMETHOD_END }; DEFINE_CLASS_1(pcib, thunder_pem_fdt_driver, thunder_pem_fdt_methods, sizeof(struct thunder_pem_softc), thunder_pem_driver); static devclass_t thunder_pem_fdt_devclass; DRIVER_MODULE(thunder_pem, simplebus, thunder_pem_fdt_driver, thunder_pem_fdt_devclass, 0, 0); DRIVER_MODULE(thunder_pem, ofwbus, thunder_pem_fdt_driver, thunder_pem_fdt_devclass, 0, 0); static int thunder_pem_fdt_probe(device_t dev) { if (!ofw_bus_status_okay(dev)) return (ENXIO); if (ofw_bus_is_compatible(dev, "cavium,pci-host-thunder-pem")) { device_set_desc(dev, THUNDER_PEM_DESC); return (BUS_PROBE_DEFAULT); } return (ENXIO); } static int thunder_pem_fdt_alloc_msi(device_t pci, device_t child, int count, int maxcount, int *irqs) { return (arm_alloc_msi(pci, child, count, maxcount, irqs)); } static int thunder_pem_fdt_release_msi(device_t pci, device_t child, int count, int *irqs) { return (arm_release_msi(pci, child, count, irqs)); } static int thunder_pem_fdt_alloc_msix(device_t pci, device_t child, int *irq) { return (arm_alloc_msix(pci, child, irq)); } static int thunder_pem_fdt_release_msix(device_t pci, device_t child, int irq) { return (arm_release_msix(pci, child, irq)); } static int thunder_pem_fdt_map_msi(device_t pci, device_t child, int irq, uint64_t *addr, uint32_t *data) { return (arm_map_msi(pci, child, irq, addr, data)); +} + +static int +thunder_pem_fdt_get_id(device_t dev, device_t child, enum pci_id_type type, + uintptr_t *id) +{ + phandle_t node; + uint32_t rid; + uint16_t pci_rid; + + if (type != PCI_ID_MSI) + return (pcib_get_id(dev, child, type, id)); + + node = ofw_bus_get_node(dev); + pci_rid = pci_get_rid(child); + + ofw_bus_msimap(node, pci_rid, NULL, &rid); + *id = rid; + + return (0); } Index: head/sys/dev/ofw/ofw_bus_subr.c =================================================================== --- head/sys/dev/ofw/ofw_bus_subr.c (revision 299931) +++ head/sys/dev/ofw/ofw_bus_subr.c (revision 299932) @@ -1,807 +1,858 @@ /*- * Copyright (c) 2001 - 2003 by Thomas Moestl . * Copyright (c) 2005 Marius Strobl * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions, and the following disclaimer, * without modification, immediately at the beginning of the file. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in * the documentation and/or other materials provided with the * distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include "opt_platform.h" #include #include #include #include #include #include #include #include #include #include "ofw_bus_if.h" int ofw_bus_gen_setup_devinfo(struct ofw_bus_devinfo *obd, phandle_t node) { if (obd == NULL) return (ENOMEM); /* The 'name' property is considered mandatory. */ if ((OF_getprop_alloc(node, "name", 1, (void **)&obd->obd_name)) == -1) return (EINVAL); OF_getprop_alloc(node, "compatible", 1, (void **)&obd->obd_compat); OF_getprop_alloc(node, "device_type", 1, (void **)&obd->obd_type); OF_getprop_alloc(node, "model", 1, (void **)&obd->obd_model); OF_getprop_alloc(node, "status", 1, (void **)&obd->obd_status); obd->obd_node = node; return (0); } void ofw_bus_gen_destroy_devinfo(struct ofw_bus_devinfo *obd) { if (obd == NULL) return; if (obd->obd_compat != NULL) free(obd->obd_compat, M_OFWPROP); if (obd->obd_model != NULL) free(obd->obd_model, M_OFWPROP); if (obd->obd_name != NULL) free(obd->obd_name, M_OFWPROP); if (obd->obd_type != NULL) free(obd->obd_type, M_OFWPROP); if (obd->obd_status != NULL) free(obd->obd_status, M_OFWPROP); } int ofw_bus_gen_child_pnpinfo_str(device_t cbdev, device_t child, char *buf, size_t buflen) { if (ofw_bus_get_name(child) != NULL) { strlcat(buf, "name=", buflen); strlcat(buf, ofw_bus_get_name(child), buflen); } if (ofw_bus_get_compat(child) != NULL) { strlcat(buf, " compat=", buflen); strlcat(buf, ofw_bus_get_compat(child), buflen); } return (0); }; const char * ofw_bus_gen_get_compat(device_t bus, device_t dev) { const struct ofw_bus_devinfo *obd; obd = OFW_BUS_GET_DEVINFO(bus, dev); if (obd == NULL) return (NULL); return (obd->obd_compat); } const char * ofw_bus_gen_get_model(device_t bus, device_t dev) { const struct ofw_bus_devinfo *obd; obd = OFW_BUS_GET_DEVINFO(bus, dev); if (obd == NULL) return (NULL); return (obd->obd_model); } const char * ofw_bus_gen_get_name(device_t bus, device_t dev) { const struct ofw_bus_devinfo *obd; obd = OFW_BUS_GET_DEVINFO(bus, dev); if (obd == NULL) return (NULL); return (obd->obd_name); } phandle_t ofw_bus_gen_get_node(device_t bus, device_t dev) { const struct ofw_bus_devinfo *obd; obd = OFW_BUS_GET_DEVINFO(bus, dev); if (obd == NULL) return (0); return (obd->obd_node); } const char * ofw_bus_gen_get_type(device_t bus, device_t dev) { const struct ofw_bus_devinfo *obd; obd = OFW_BUS_GET_DEVINFO(bus, dev); if (obd == NULL) return (NULL); return (obd->obd_type); } const char * ofw_bus_get_status(device_t dev) { const struct ofw_bus_devinfo *obd; obd = OFW_BUS_GET_DEVINFO(device_get_parent(dev), dev); if (obd == NULL) return (NULL); return (obd->obd_status); } int ofw_bus_status_okay(device_t dev) { const char *status; status = ofw_bus_get_status(dev); if (status == NULL || strcmp(status, "okay") == 0 || strcmp(status, "ok") == 0) return (1); return (0); } static int ofw_bus_node_is_compatible(const char *compat, int len, const char *onecompat) { int onelen, l, ret; onelen = strlen(onecompat); ret = 0; while (len > 0) { if (strlen(compat) == onelen && strncasecmp(compat, onecompat, onelen) == 0) { /* Found it. */ ret = 1; break; } /* Slide to the next sub-string. */ l = strlen(compat) + 1; compat += l; len -= l; } return (ret); } int ofw_bus_is_compatible(device_t dev, const char *onecompat) { phandle_t node; const char *compat; int len; if ((compat = ofw_bus_get_compat(dev)) == NULL) return (0); if ((node = ofw_bus_get_node(dev)) == -1) return (0); /* Get total 'compatible' prop len */ if ((len = OF_getproplen(node, "compatible")) <= 0) return (0); return (ofw_bus_node_is_compatible(compat, len, onecompat)); } int ofw_bus_is_compatible_strict(device_t dev, const char *compatible) { const char *compat; size_t len; if ((compat = ofw_bus_get_compat(dev)) == NULL) return (0); len = strlen(compatible); if (strlen(compat) == len && strncasecmp(compat, compatible, len) == 0) return (1); return (0); } const struct ofw_compat_data * ofw_bus_search_compatible(device_t dev, const struct ofw_compat_data *compat) { if (compat == NULL) return NULL; for (; compat->ocd_str != NULL; ++compat) { if (ofw_bus_is_compatible(dev, compat->ocd_str)) break; } return (compat); } int ofw_bus_has_prop(device_t dev, const char *propname) { phandle_t node; if ((node = ofw_bus_get_node(dev)) == -1) return (0); return (OF_hasprop(node, propname)); } void ofw_bus_setup_iinfo(phandle_t node, struct ofw_bus_iinfo *ii, int intrsz) { pcell_t addrc; int msksz; if (OF_getencprop(node, "#address-cells", &addrc, sizeof(addrc)) == -1) addrc = 2; ii->opi_addrc = addrc * sizeof(pcell_t); ii->opi_imapsz = OF_getencprop_alloc(node, "interrupt-map", 1, (void **)&ii->opi_imap); if (ii->opi_imapsz > 0) { msksz = OF_getencprop_alloc(node, "interrupt-map-mask", 1, (void **)&ii->opi_imapmsk); /* * Failure to get the mask is ignored; a full mask is used * then. We barf on bad mask sizes, however. */ if (msksz != -1 && msksz != ii->opi_addrc + intrsz) panic("ofw_bus_setup_iinfo: bad interrupt-map-mask " "property!"); } } int ofw_bus_lookup_imap(phandle_t node, struct ofw_bus_iinfo *ii, void *reg, int regsz, void *pintr, int pintrsz, void *mintr, int mintrsz, phandle_t *iparent) { uint8_t maskbuf[regsz + pintrsz]; int rv; if (ii->opi_imapsz <= 0) return (0); KASSERT(regsz >= ii->opi_addrc, ("ofw_bus_lookup_imap: register size too small: %d < %d", regsz, ii->opi_addrc)); if (node != -1) { rv = OF_getencprop(node, "reg", reg, regsz); if (rv < regsz) panic("ofw_bus_lookup_imap: cannot get reg property"); } return (ofw_bus_search_intrmap(pintr, pintrsz, reg, ii->opi_addrc, ii->opi_imap, ii->opi_imapsz, ii->opi_imapmsk, maskbuf, mintr, mintrsz, iparent)); } /* * Map an interrupt using the firmware reg, interrupt-map and * interrupt-map-mask properties. * The interrupt property to be mapped must be of size intrsz, and pointed to * by intr. The regs property of the node for which the mapping is done must * be passed as regs. This property is an array of register specifications; * the size of the address part of such a specification must be passed as * physsz. Only the first element of the property is used. * imap and imapsz hold the interrupt mask and it's size. * imapmsk is a pointer to the interrupt-map-mask property, which must have * a size of physsz + intrsz; it may be NULL, in which case a full mask is * assumed. * maskbuf must point to a buffer of length physsz + intrsz. * The interrupt is returned in result, which must point to a buffer of length * rintrsz (which gives the expected size of the mapped interrupt). * Returns number of cells in the interrupt if a mapping was found, 0 otherwise. */ int ofw_bus_search_intrmap(void *intr, int intrsz, void *regs, int physsz, void *imap, int imapsz, void *imapmsk, void *maskbuf, void *result, int rintrsz, phandle_t *iparent) { phandle_t parent; uint8_t *ref = maskbuf; uint8_t *uiintr = intr; uint8_t *uiregs = regs; uint8_t *uiimapmsk = imapmsk; uint8_t *mptr; pcell_t paddrsz; pcell_t pintrsz; int i, rsz, tsz; rsz = -1; if (imapmsk != NULL) { for (i = 0; i < physsz; i++) ref[i] = uiregs[i] & uiimapmsk[i]; for (i = 0; i < intrsz; i++) ref[physsz + i] = uiintr[i] & uiimapmsk[physsz + i]; } else { bcopy(regs, ref, physsz); bcopy(intr, ref + physsz, intrsz); } mptr = imap; i = imapsz; paddrsz = 0; while (i > 0) { bcopy(mptr + physsz + intrsz, &parent, sizeof(parent)); #ifndef OFW_IMAP_NO_IPARENT_ADDR_CELLS /* * Find if we need to read the parent address data. * CHRP-derived OF bindings, including ePAPR-compliant FDTs, * use this as an optional part of the specifier. */ if (OF_getencprop(OF_node_from_xref(parent), "#address-cells", &paddrsz, sizeof(paddrsz)) == -1) paddrsz = 0; /* default */ paddrsz *= sizeof(pcell_t); #endif if (OF_searchencprop(OF_node_from_xref(parent), "#interrupt-cells", &pintrsz, sizeof(pintrsz)) == -1) pintrsz = 1; /* default */ pintrsz *= sizeof(pcell_t); /* Compute the map stride size. */ tsz = physsz + intrsz + sizeof(phandle_t) + paddrsz + pintrsz; KASSERT(i >= tsz, ("ofw_bus_search_intrmap: truncated map")); if (bcmp(ref, mptr, physsz + intrsz) == 0) { bcopy(mptr + physsz + intrsz + sizeof(parent) + paddrsz, result, MIN(rintrsz, pintrsz)); if (iparent != NULL) *iparent = parent; return (pintrsz/sizeof(pcell_t)); } mptr += tsz; i -= tsz; } return (0); } int +ofw_bus_msimap(phandle_t node, uint16_t pci_rid, phandle_t *msi_parent, + uint32_t *msi_rid) +{ + pcell_t *map, mask, msi_base, rid_base, rid_length; + ssize_t len; + uint32_t masked_rid, rid; + int err, i; + + /* TODO: This should be OF_searchprop_alloc if we had it */ + len = OF_getencprop_alloc(node, "msi-map", sizeof(*map), (void **)&map); + if (len < 0) { + if (msi_parent != NULL) { + *msi_parent = 0; + OF_getencprop(node, "msi-parent", msi_parent, + sizeof(*msi_parent)); + } + if (msi_rid != NULL) + *msi_rid = pci_rid; + return (0); + } + + err = ENOENT; + rid = 0; + mask = 0xffffffff; + OF_getencprop(node, "msi-map-mask", &mask, sizeof(mask)); + + masked_rid = pci_rid & mask; + for (i = 0; i < len; i += 4) { + rid_base = map[i + 0]; + rid_length = map[i + 3]; + + if (masked_rid < rid_base || + masked_rid >= (rid_base + rid_length)) + continue; + + msi_base = map[i + 2]; + + if (msi_parent != NULL) + *msi_parent = map[i + 1]; + if (msi_rid != NULL) + *msi_rid = masked_rid - rid_base + msi_base; + err = 0; + break; + } + + free(map, M_OFWPROP); + + return (err); +} + +int ofw_bus_reg_to_rl(device_t dev, phandle_t node, pcell_t acells, pcell_t scells, struct resource_list *rl) { uint64_t phys, size; ssize_t i, j, rid, nreg, ret; uint32_t *reg; char *name; /* * This may be just redundant when having ofw_bus_devinfo * but makes this routine independent of it. */ ret = OF_getprop_alloc(node, "name", sizeof(*name), (void **)&name); if (ret == -1) name = NULL; ret = OF_getencprop_alloc(node, "reg", sizeof(*reg), (void **)®); nreg = (ret == -1) ? 0 : ret; if (nreg % (acells + scells) != 0) { if (bootverbose) device_printf(dev, "Malformed reg property on <%s>\n", (name == NULL) ? "unknown" : name); nreg = 0; } for (i = 0, rid = 0; i < nreg; i += acells + scells, rid++) { phys = size = 0; for (j = 0; j < acells; j++) { phys <<= 32; phys |= reg[i + j]; } for (j = 0; j < scells; j++) { size <<= 32; size |= reg[i + acells + j]; } /* Skip the dummy reg property of glue devices like ssm(4). */ if (size != 0) resource_list_add(rl, SYS_RES_MEMORY, rid, phys, phys + size - 1, size); } free(name, M_OFWPROP); free(reg, M_OFWPROP); return (0); } /* * Get interrupt parent for given node. * Returns 0 if interrupt parent doesn't exist. */ phandle_t ofw_bus_find_iparent(phandle_t node) { phandle_t iparent; if (OF_searchencprop(node, "interrupt-parent", &iparent, sizeof(iparent)) == -1) { for (iparent = node; iparent != 0; iparent = OF_parent(iparent)) { if (OF_hasprop(iparent, "interrupt-controller")) break; } iparent = OF_xref_from_node(iparent); } return (iparent); } int ofw_bus_intr_to_rl(device_t dev, phandle_t node, struct resource_list *rl, int *rlen) { phandle_t iparent; uint32_t icells, *intr; int err, i, irqnum, nintr, rid; boolean_t extended; nintr = OF_getencprop_alloc(node, "interrupts", sizeof(*intr), (void **)&intr); if (nintr > 0) { iparent = ofw_bus_find_iparent(node); if (iparent == 0) { device_printf(dev, "No interrupt-parent found, " "assuming direct parent\n"); iparent = OF_parent(node); iparent = OF_xref_from_node(iparent); } if (OF_searchencprop(OF_node_from_xref(iparent), "#interrupt-cells", &icells, sizeof(icells)) == -1) { device_printf(dev, "Missing #interrupt-cells " "property, assuming <1>\n"); icells = 1; } if (icells < 1 || icells > nintr) { device_printf(dev, "Invalid #interrupt-cells property " "value <%d>, assuming <1>\n", icells); icells = 1; } extended = false; } else { nintr = OF_getencprop_alloc(node, "interrupts-extended", sizeof(*intr), (void **)&intr); if (nintr <= 0) return (0); extended = true; } err = 0; rid = 0; for (i = 0; i < nintr; i += icells) { if (extended) { iparent = intr[i++]; if (OF_searchencprop(OF_node_from_xref(iparent), "#interrupt-cells", &icells, sizeof(icells)) == -1) { device_printf(dev, "Missing #interrupt-cells " "property\n"); err = ENOENT; break; } if (icells < 1 || (i + icells) > nintr) { device_printf(dev, "Invalid #interrupt-cells " "property value <%d>\n", icells); err = ERANGE; break; } } irqnum = ofw_bus_map_intr(dev, iparent, icells, &intr[i]); resource_list_add(rl, SYS_RES_IRQ, rid++, irqnum, irqnum, 1); } if (rlen != NULL) *rlen = rid; free(intr, M_OFWPROP); return (err); } phandle_t ofw_bus_find_child(phandle_t start, const char *child_name) { char *name; int ret; phandle_t child; for (child = OF_child(start); child != 0; child = OF_peer(child)) { ret = OF_getprop_alloc(child, "name", sizeof(*name), (void **)&name); if (ret == -1) continue; if (strcmp(name, child_name) == 0) { free(name, M_OFWPROP); return (child); } free(name, M_OFWPROP); } return (0); } phandle_t ofw_bus_find_compatible(phandle_t node, const char *onecompat) { phandle_t child, ret; void *compat; int len; /* * Traverse all children of 'start' node, and find first with * matching 'compatible' property. */ for (child = OF_child(node); child != 0; child = OF_peer(child)) { len = OF_getprop_alloc(child, "compatible", 1, &compat); if (len >= 0) { ret = ofw_bus_node_is_compatible(compat, len, onecompat); free(compat, M_OFWPROP); if (ret != 0) return (child); } ret = ofw_bus_find_compatible(child, onecompat); if (ret != 0) return (ret); } return (0); } /** * @brief Return child of bus whose phandle is node * * A direct child of @p will be returned if it its phandle in the * OFW tree is @p node. Otherwise, NULL is returned. * * @param bus The bus to examine * @param node The phandle_t to look for. */ device_t ofw_bus_find_child_device_by_phandle(device_t bus, phandle_t node) { device_t *children, retval, child; int nkid, i; /* * Nothing can match the flag value for no node. */ if (node == -1) return (NULL); /* * Search the children for a match. We microoptimize * a bit by not using ofw_bus_get since we already know * the parent. We do not recurse. */ if (device_get_children(bus, &children, &nkid) != 0) return (NULL); retval = NULL; for (i = 0; i < nkid; i++) { child = children[i]; if (OFW_BUS_GET_NODE(bus, child) == node) { retval = child; break; } } free(children, M_TEMP); return (retval); } /* * Parse property that contain list of xrefs and values * (like standard "clocks" and "resets" properties) * Input arguments: * node - consumers device node * list_name - name of parsed list - "clocks" * cells_name - name of size property - "#clock-cells" * idx - the index of the requested list entry, or, if -1, an indication * to return the number of entries in the parsed list. * Output arguments: * producer - handle of producer * ncells - number of cells in result or the number of items in the list when * idx == -1. * cells - array of decoded cells */ static int ofw_bus_parse_xref_list_internal(phandle_t node, const char *list_name, const char *cells_name, int idx, phandle_t *producer, int *ncells, pcell_t **cells) { phandle_t pnode; phandle_t *elems; uint32_t pcells; int rv, i, j, nelems, cnt; elems = NULL; nelems = OF_getencprop_alloc(node, list_name, sizeof(*elems), (void **)&elems); if (nelems <= 0) return (ENOENT); rv = (idx == -1) ? 0 : ENOENT; for (i = 0, cnt = 0; i < nelems; i += pcells, cnt++) { pnode = elems[i++]; if (OF_getencprop(OF_node_from_xref(pnode), cells_name, &pcells, sizeof(pcells)) == -1) { printf("Missing %s property\n", cells_name); rv = ENOENT; break; } if ((i + pcells) > nelems) { printf("Invalid %s property value <%d>\n", cells_name, pcells); rv = ERANGE; break; } if (cnt == idx) { *cells= malloc(pcells * sizeof(**cells), M_OFWPROP, M_WAITOK); *producer = pnode; *ncells = pcells; for (j = 0; j < pcells; j++) (*cells)[j] = elems[i + j]; rv = 0; break; } } if (elems != NULL) free(elems, M_OFWPROP); if (idx == -1 && rv == 0) *ncells = cnt; return (rv); } /* * Parse property that contain list of xrefs and values * (like standard "clocks" and "resets" properties) * Input arguments: * node - consumers device node * list_name - name of parsed list - "clocks" * cells_name - name of size property - "#clock-cells" * idx - the index of the requested list entry (>= 0) * Output arguments: * producer - handle of producer * ncells - number of cells in result * cells - array of decoded cells */ int ofw_bus_parse_xref_list_alloc(phandle_t node, const char *list_name, const char *cells_name, int idx, phandle_t *producer, int *ncells, pcell_t **cells) { KASSERT(idx >= 0, ("ofw_bus_parse_xref_list_alloc: negative index supplied")); return (ofw_bus_parse_xref_list_internal(node, list_name, cells_name, idx, producer, ncells, cells)); } /* * Parse property that contain list of xrefs and values * (like standard "clocks" and "resets" properties) * and determine the number of items in the list * Input arguments: * node - consumers device node * list_name - name of parsed list - "clocks" * cells_name - name of size property - "#clock-cells" * Output arguments: * count - number of items in list */ int ofw_bus_parse_xref_list_get_length(phandle_t node, const char *list_name, const char *cells_name, int *count) { return (ofw_bus_parse_xref_list_internal(node, list_name, cells_name, -1, NULL, count, NULL)); } /* * Find index of string in string list property (case sensitive). */ int ofw_bus_find_string_index(phandle_t node, const char *list_name, const char *name, int *idx) { char *elems; int rv, i, cnt, nelems; elems = NULL; nelems = OF_getprop_alloc(node, list_name, 1, (void **)&elems); if (nelems <= 0) return (ENOENT); rv = ENOENT; for (i = 0, cnt = 0; i < nelems; cnt++) { if (strcmp(elems + i, name) == 0) { *idx = cnt; rv = 0; break; } i += strlen(elems + i) + 1; } if (elems != NULL) free(elems, M_OFWPROP); return (rv); } /* * Create zero terminated array of strings from string list property. */ int ofw_bus_string_list_to_array(phandle_t node, const char *list_name, const char ***out_array) { char *elems, *tptr; const char **array; int i, cnt, nelems, len; elems = NULL; nelems = OF_getprop_alloc(node, list_name, 1, (void **)&elems); if (nelems <= 0) return (nelems); /* Count number of strings. */ for (i = 0, cnt = 0; i < nelems; cnt++) i += strlen(elems + i) + 1; /* Allocate space for arrays and all strings. */ array = malloc((cnt + 1) * sizeof(char *) + nelems, M_OFWPROP, M_WAITOK); /* Get address of first string. */ tptr = (char *)(array + cnt + 1); /* Copy strings. */ memcpy(tptr, elems, nelems); free(elems, M_OFWPROP); /* Fill string pointers. */ for (i = 0, cnt = 0; i < nelems; cnt++) { len = strlen(tptr) + 1; array[cnt] = tptr; i += len; tptr += len; } array[cnt] = 0; *out_array = array; return (cnt); } Index: head/sys/dev/ofw/ofw_bus_subr.h =================================================================== --- head/sys/dev/ofw/ofw_bus_subr.h (revision 299931) +++ head/sys/dev/ofw/ofw_bus_subr.h (revision 299932) @@ -1,128 +1,131 @@ /*- * Copyright (c) 2005 Marius Strobl * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions, and the following disclaimer, * without modification, immediately at the beginning of the file. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in * the documentation and/or other materials provided with the * distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #ifndef _DEV_OFW_OFW_BUS_SUBR_H_ #define _DEV_OFW_OFW_BUS_SUBR_H_ #include #include #include "ofw_bus_if.h" #define ORIP_NOINT -1 #define ORIR_NOTFOUND 0xffffffff struct ofw_bus_iinfo { uint8_t *opi_imap; uint8_t *opi_imapmsk; int opi_imapsz; pcell_t opi_addrc; }; struct ofw_compat_data { const char *ocd_str; uintptr_t ocd_data; }; #define SIMPLEBUS_PNP_DESCR "Z:compat;P:private;" #define SIMPLEBUS_PNP_INFO(t) \ MODULE_PNP_INFO(SIMPLEBUS_PNP_DESCR, simplebus, t, t, sizeof(t[0]), sizeof(t) / sizeof(t[0])); /* Generic implementation of ofw_bus_if.m methods and helper routines */ int ofw_bus_gen_setup_devinfo(struct ofw_bus_devinfo *, phandle_t); void ofw_bus_gen_destroy_devinfo(struct ofw_bus_devinfo *); ofw_bus_get_compat_t ofw_bus_gen_get_compat; ofw_bus_get_model_t ofw_bus_gen_get_model; ofw_bus_get_name_t ofw_bus_gen_get_name; ofw_bus_get_node_t ofw_bus_gen_get_node; ofw_bus_get_type_t ofw_bus_gen_get_type; /* Helper method to report interesting OF properties in pnpinfo */ bus_child_pnpinfo_str_t ofw_bus_gen_child_pnpinfo_str; /* Routines for processing firmware interrupt maps */ void ofw_bus_setup_iinfo(phandle_t, struct ofw_bus_iinfo *, int); int ofw_bus_lookup_imap(phandle_t, struct ofw_bus_iinfo *, void *, int, void *, int, void *, int, phandle_t *); int ofw_bus_search_intrmap(void *, int, void *, int, void *, int, void *, void *, void *, int, phandle_t *); +/* Routines for processing msi maps */ +int ofw_bus_msimap(phandle_t, uint16_t, phandle_t *, uint32_t *); + /* Routines for parsing device-tree data into resource lists. */ int ofw_bus_reg_to_rl(device_t, phandle_t, pcell_t, pcell_t, struct resource_list *); int ofw_bus_intr_to_rl(device_t, phandle_t, struct resource_list *, int *); /* Helper to get device status property */ const char *ofw_bus_get_status(device_t dev); int ofw_bus_status_okay(device_t dev); /* Helper to get node's interrupt parent */ phandle_t ofw_bus_find_iparent(phandle_t); /* Helper routine for checking compat prop */ int ofw_bus_is_compatible(device_t, const char *); int ofw_bus_is_compatible_strict(device_t, const char *); /* * Helper routine to search a list of compat properties. The table is * terminated by an entry with a NULL compat-string pointer; a pointer to that * table entry is returned if none of the compat strings match for the device, * giving you control over the not-found value. Will not return NULL unless the * provided table pointer is NULL. */ const struct ofw_compat_data * ofw_bus_search_compatible(device_t, const struct ofw_compat_data *); /* Helper routine for checking existence of a prop */ int ofw_bus_has_prop(device_t, const char *); /* Helper to search for a child with a given compat prop */ phandle_t ofw_bus_find_compatible(phandle_t, const char *); /* Helper to search for a child with a given name */ phandle_t ofw_bus_find_child(phandle_t, const char *); /* Helper routine to find a device_t child matchig a given phandle_t */ device_t ofw_bus_find_child_device_by_phandle(device_t bus, phandle_t node); /* Helper routines for parsing lists */ int ofw_bus_parse_xref_list_alloc(phandle_t node, const char *list_name, const char *cells_name, int idx, phandle_t *producer, int *ncells, pcell_t **cells); int ofw_bus_parse_xref_list_get_length(phandle_t node, const char *list_name, const char *cells_name, int *count); int ofw_bus_find_string_index(phandle_t node, const char *list_name, const char *name, int *idx); int ofw_bus_string_list_to_array(phandle_t node, const char *list_name, const char ***array); #endif /* !_DEV_OFW_OFW_BUS_SUBR_H_ */ Index: head/sys/dev/pci/pci_host_generic.c =================================================================== --- head/sys/dev/pci/pci_host_generic.c (revision 299931) +++ head/sys/dev/pci/pci_host_generic.c (revision 299932) @@ -1,904 +1,925 @@ /*- * Copyright (c) 2015 Ruslan Bukin * Copyright (c) 2014 The FreeBSD Foundation * All rights reserved. * * This software was developed by Semihalf under * the sponsorship of the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* Generic ECAM PCIe driver */ #include __FBSDID("$FreeBSD$"); #include "opt_platform.h" #include #include #include #include #include #include #include #include #include #include #if defined(INTRNG) #include #endif #include #include #include #include #include #include #include #include #include #include #include #include #include "pcib_if.h" /* Assembling ECAM Configuration Address */ #define PCIE_BUS_SHIFT 20 #define PCIE_SLOT_SHIFT 15 #define PCIE_FUNC_SHIFT 12 #define PCIE_BUS_MASK 0xFF #define PCIE_SLOT_MASK 0x1F #define PCIE_FUNC_MASK 0x07 #define PCIE_REG_MASK 0xFFF #define PCIE_ADDR_OFFSET(bus, slot, func, reg) \ ((((bus) & PCIE_BUS_MASK) << PCIE_BUS_SHIFT) | \ (((slot) & PCIE_SLOT_MASK) << PCIE_SLOT_SHIFT) | \ (((func) & PCIE_FUNC_MASK) << PCIE_FUNC_SHIFT) | \ ((reg) & PCIE_REG_MASK)) #define PCI_IO_WINDOW_OFFSET 0x1000 #define SPACE_CODE_SHIFT 24 #define SPACE_CODE_MASK 0x3 #define SPACE_CODE_IO_SPACE 0x1 #define PROPS_CELL_SIZE 1 #define PCI_ADDR_CELL_SIZE 2 /* OFW bus interface */ struct generic_pcie_ofw_devinfo { struct ofw_bus_devinfo di_dinfo; struct resource_list di_rl; }; /* Forward prototypes */ static int generic_pcie_probe(device_t dev); static int parse_pci_mem_ranges(struct generic_pcie_softc *sc); static uint32_t generic_pcie_read_config(device_t dev, u_int bus, u_int slot, u_int func, u_int reg, int bytes); static void generic_pcie_write_config(device_t dev, u_int bus, u_int slot, u_int func, u_int reg, uint32_t val, int bytes); static int generic_pcie_maxslots(device_t dev); static int generic_pcie_read_ivar(device_t dev, device_t child, int index, uintptr_t *result); static int generic_pcie_write_ivar(device_t dev, device_t child, int index, uintptr_t value); static struct resource *generic_pcie_alloc_resource_ofw(device_t, device_t, int, int *, rman_res_t, rman_res_t, rman_res_t, u_int); static struct resource *generic_pcie_alloc_resource_pcie(device_t dev, device_t child, int type, int *rid, rman_res_t start, rman_res_t end, rman_res_t count, u_int flags); static int generic_pcie_release_resource(device_t dev, device_t child, int type, int rid, struct resource *res); static int generic_pcie_release_resource_ofw(device_t, device_t, int, int, struct resource *); static int generic_pcie_release_resource_pcie(device_t, device_t, int, int, struct resource *); static int generic_pcie_ofw_bus_attach(device_t); static const struct ofw_bus_devinfo *generic_pcie_ofw_get_devinfo(device_t, device_t); static __inline void get_addr_size_cells(phandle_t node, pcell_t *addr_cells, pcell_t *size_cells) { *addr_cells = 2; /* Find address cells if present */ OF_getencprop(node, "#address-cells", addr_cells, sizeof(*addr_cells)); *size_cells = 2; /* Find size cells if present */ OF_getencprop(node, "#size-cells", size_cells, sizeof(*size_cells)); } static int generic_pcie_probe(device_t dev) { if (!ofw_bus_status_okay(dev)) return (ENXIO); if (ofw_bus_is_compatible(dev, "pci-host-ecam-generic")) { device_set_desc(dev, "Generic PCI host controller"); return (BUS_PROBE_GENERIC); } if (ofw_bus_is_compatible(dev, "arm,gem5_pcie")) { device_set_desc(dev, "GEM5 PCIe host controller"); return (BUS_PROBE_DEFAULT); } return (ENXIO); } int pci_host_generic_attach(device_t dev) { struct generic_pcie_softc *sc; uint64_t phys_base; uint64_t pci_base; uint64_t size; phandle_t node; int error; int tuple; int rid; sc = device_get_softc(dev); sc->dev = dev; /* Retrieve 'ranges' property from FDT */ if (bootverbose) device_printf(dev, "parsing FDT for ECAM%d:\n", sc->ecam); if (parse_pci_mem_ranges(sc)) return (ENXIO); /* Attach OFW bus */ if (generic_pcie_ofw_bus_attach(dev) != 0) return (ENXIO); rid = 0; sc->res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, RF_ACTIVE); if (sc->res == NULL) { device_printf(dev, "could not map memory.\n"); return (ENXIO); } sc->bst = rman_get_bustag(sc->res); sc->bsh = rman_get_bushandle(sc->res); sc->mem_rman.rm_type = RMAN_ARRAY; sc->mem_rman.rm_descr = "PCIe Memory"; sc->io_rman.rm_type = RMAN_ARRAY; sc->io_rman.rm_descr = "PCIe IO window"; /* Initialize rman and allocate memory regions */ error = rman_init(&sc->mem_rman); if (error) { device_printf(dev, "rman_init() failed. error = %d\n", error); return (error); } error = rman_init(&sc->io_rman); if (error) { device_printf(dev, "rman_init() failed. error = %d\n", error); return (error); } for (tuple = 0; tuple < MAX_RANGES_TUPLES; tuple++) { phys_base = sc->ranges[tuple].phys_base; pci_base = sc->ranges[tuple].pci_base; size = sc->ranges[tuple].size; if (phys_base == 0 || size == 0) continue; /* empty range element */ if (sc->ranges[tuple].flags & FLAG_MEM) { error = rman_manage_region(&sc->mem_rman, phys_base, phys_base + size - 1); } else if (sc->ranges[tuple].flags & FLAG_IO) { error = rman_manage_region(&sc->io_rman, pci_base + PCI_IO_WINDOW_OFFSET, pci_base + PCI_IO_WINDOW_OFFSET + size - 1); } else continue; if (error) { device_printf(dev, "rman_manage_region() failed." "error = %d\n", error); rman_fini(&sc->mem_rman); return (error); } } node = ofw_bus_get_node(dev); ofw_bus_setup_iinfo(node, &sc->pci_iinfo, sizeof(cell_t)); /* Find the MSI interrupt handler */ OF_searchencprop(node, "msi-parent", &sc->msi_parent, sizeof(sc->msi_parent)); device_add_child(dev, "pci", -1); return (bus_generic_attach(dev)); } static int parse_pci_mem_ranges(struct generic_pcie_softc *sc) { pcell_t pci_addr_cells, parent_addr_cells; pcell_t attributes, size_cells; cell_t *base_ranges; int nbase_ranges; phandle_t node; int i, j, k; int tuple; node = ofw_bus_get_node(sc->dev); OF_getencprop(node, "#address-cells", &pci_addr_cells, sizeof(pci_addr_cells)); OF_getencprop(node, "#size-cells", &size_cells, sizeof(size_cells)); OF_getencprop(OF_parent(node), "#address-cells", &parent_addr_cells, sizeof(parent_addr_cells)); if (parent_addr_cells != 2 || pci_addr_cells != 3 || size_cells != 2) { device_printf(sc->dev, "Unexpected number of address or size cells in FDT\n"); return (ENXIO); } nbase_ranges = OF_getproplen(node, "ranges"); sc->nranges = nbase_ranges / sizeof(cell_t) / (parent_addr_cells + pci_addr_cells + size_cells); base_ranges = malloc(nbase_ranges, M_DEVBUF, M_WAITOK); OF_getencprop(node, "ranges", base_ranges, nbase_ranges); for (i = 0, j = 0; i < sc->nranges; i++) { attributes = (base_ranges[j++] >> SPACE_CODE_SHIFT) & \ SPACE_CODE_MASK; if (attributes == SPACE_CODE_IO_SPACE) { sc->ranges[i].flags |= FLAG_IO; } else { sc->ranges[i].flags |= FLAG_MEM; } sc->ranges[i].pci_base = 0; for (k = 0; k < (pci_addr_cells - 1); k++) { sc->ranges[i].pci_base <<= 32; sc->ranges[i].pci_base |= base_ranges[j++]; } sc->ranges[i].phys_base = 0; for (k = 0; k < parent_addr_cells; k++) { sc->ranges[i].phys_base <<= 32; sc->ranges[i].phys_base |= base_ranges[j++]; } sc->ranges[i].size = 0; for (k = 0; k < size_cells; k++) { sc->ranges[i].size <<= 32; sc->ranges[i].size |= base_ranges[j++]; } } for (; i < MAX_RANGES_TUPLES; i++) { /* zero-fill remaining tuples to mark empty elements in array */ sc->ranges[i].pci_base = 0; sc->ranges[i].phys_base = 0; sc->ranges[i].size = 0; } if (bootverbose) { for (tuple = 0; tuple < MAX_RANGES_TUPLES; tuple++) { device_printf(sc->dev, "\tPCI addr: 0x%jx, CPU addr: 0x%jx, Size: 0x%jx\n", sc->ranges[tuple].pci_base, sc->ranges[tuple].phys_base, sc->ranges[tuple].size); } } free(base_ranges, M_DEVBUF); return (0); } static uint32_t generic_pcie_read_config(device_t dev, u_int bus, u_int slot, u_int func, u_int reg, int bytes) { struct generic_pcie_softc *sc; bus_space_handle_t h; bus_space_tag_t t; uint64_t offset; uint32_t data; if ((bus > PCI_BUSMAX) || (slot > PCI_SLOTMAX) || (func > PCI_FUNCMAX) || (reg > PCIE_REGMAX)) return (~0U); sc = device_get_softc(dev); offset = PCIE_ADDR_OFFSET(bus, slot, func, reg); t = sc->bst; h = sc->bsh; switch (bytes) { case 1: data = bus_space_read_1(t, h, offset); break; case 2: data = le16toh(bus_space_read_2(t, h, offset)); break; case 4: data = le32toh(bus_space_read_4(t, h, offset)); break; default: return (~0U); } return (data); } static void generic_pcie_write_config(device_t dev, u_int bus, u_int slot, u_int func, u_int reg, uint32_t val, int bytes) { struct generic_pcie_softc *sc; bus_space_handle_t h; bus_space_tag_t t; uint64_t offset; if ((bus > PCI_BUSMAX) || (slot > PCI_SLOTMAX) || (func > PCI_FUNCMAX) || (reg > PCIE_REGMAX)) return; sc = device_get_softc(dev); offset = PCIE_ADDR_OFFSET(bus, slot, func, reg); t = sc->bst; h = sc->bsh; switch (bytes) { case 1: bus_space_write_1(t, h, offset, val); break; case 2: bus_space_write_2(t, h, offset, htole16(val)); break; case 4: bus_space_write_4(t, h, offset, htole32(val)); break; default: return; } } static int generic_pcie_maxslots(device_t dev) { return (31); /* max slots per bus acc. to standard */ } static int generic_pcie_route_interrupt(device_t bus, device_t dev, int pin) { struct generic_pcie_softc *sc; struct ofw_pci_register reg; uint32_t pintr, mintr[2]; phandle_t iparent; int intrcells; sc = device_get_softc(bus); pintr = pin; bzero(®, sizeof(reg)); reg.phys_hi = (pci_get_bus(dev) << OFW_PCI_PHYS_HI_BUSSHIFT) | (pci_get_slot(dev) << OFW_PCI_PHYS_HI_DEVICESHIFT) | (pci_get_function(dev) << OFW_PCI_PHYS_HI_FUNCTIONSHIFT); intrcells = ofw_bus_lookup_imap(ofw_bus_get_node(dev), &sc->pci_iinfo, ®, sizeof(reg), &pintr, sizeof(pintr), mintr, sizeof(mintr), &iparent); if (intrcells) { pintr = ofw_bus_map_intr(dev, iparent, intrcells, mintr); return (pintr); } device_printf(bus, "could not route pin %d for device %d.%d\n", pin, pci_get_slot(dev), pci_get_function(dev)); return (PCI_INVALID_IRQ); } static int generic_pcie_read_ivar(device_t dev, device_t child, int index, uintptr_t *result) { struct generic_pcie_softc *sc; int secondary_bus; sc = device_get_softc(dev); if (index == PCIB_IVAR_BUS) { /* this pcib adds only pci bus 0 as child */ secondary_bus = 0; *result = secondary_bus; return (0); } if (index == PCIB_IVAR_DOMAIN) { *result = sc->ecam; return (0); } if (bootverbose) device_printf(dev, "ERROR: Unknown index %d.\n", index); return (ENOENT); } static int generic_pcie_write_ivar(device_t dev, device_t child, int index, uintptr_t value) { return (ENOENT); } static struct rman * generic_pcie_rman(struct generic_pcie_softc *sc, int type) { switch (type) { case SYS_RES_IOPORT: return (&sc->io_rman); case SYS_RES_MEMORY: return (&sc->mem_rman); default: break; } return (NULL); } static int generic_pcie_release_resource_pcie(device_t dev, device_t child, int type, int rid, struct resource *res) { struct generic_pcie_softc *sc; struct rman *rm; sc = device_get_softc(dev); rm = generic_pcie_rman(sc, type); if (rm != NULL) { KASSERT(rman_is_region_manager(res, rm), ("rman mismatch")); rman_release_resource(res); } return (bus_generic_release_resource(dev, child, type, rid, res)); } static int generic_pcie_release_resource(device_t dev, device_t child, int type, int rid, struct resource *res) { /* For PCIe devices that do not have FDT nodes, use PCIB method */ if ((int)ofw_bus_get_node(child) <= 0) { return (generic_pcie_release_resource_pcie(dev, child, type, rid, res)); } /* For other devices use OFW method */ return (generic_pcie_release_resource_ofw(dev, child, type, rid, res)); } struct resource * pci_host_generic_alloc_resource(device_t dev, device_t child, int type, int *rid, rman_res_t start, rman_res_t end, rman_res_t count, u_int flags) { /* For PCIe devices that do not have FDT nodes, use PCIB method */ if ((int)ofw_bus_get_node(child) <= 0) return (generic_pcie_alloc_resource_pcie(dev, child, type, rid, start, end, count, flags)); /* For other devices use OFW method */ return (generic_pcie_alloc_resource_ofw(dev, child, type, rid, start, end, count, flags)); } static struct resource * generic_pcie_alloc_resource_pcie(device_t dev, device_t child, int type, int *rid, rman_res_t start, rman_res_t end, rman_res_t count, u_int flags) { struct generic_pcie_softc *sc; struct resource *res; struct rman *rm; sc = device_get_softc(dev); rm = generic_pcie_rman(sc, type); if (rm == NULL) return (BUS_ALLOC_RESOURCE(device_get_parent(dev), dev, type, rid, start, end, count, flags)); if (bootverbose) { device_printf(dev, "rman_reserve_resource: start=%#jx, end=%#jx, count=%#jx\n", start, end, count); } res = rman_reserve_resource(rm, start, end, count, flags, child); if (res == NULL) goto fail; rman_set_rid(res, *rid); if (flags & RF_ACTIVE) if (bus_activate_resource(child, type, *rid, res)) { rman_release_resource(res); goto fail; } return (res); fail: device_printf(dev, "%s FAIL: type=%d, rid=%d, " "start=%016jx, end=%016jx, count=%016jx, flags=%x\n", __func__, type, *rid, start, end, count, flags); return (NULL); } static int generic_pcie_adjust_resource(device_t dev, device_t child, int type, struct resource *res, rman_res_t start, rman_res_t end) { struct generic_pcie_softc *sc; struct rman *rm; sc = device_get_softc(dev); rm = generic_pcie_rman(sc, type); if (rm != NULL) return (rman_adjust_resource(res, start, end)); return (bus_generic_adjust_resource(dev, child, type, res, start, end)); } static int generic_pcie_activate_resource(device_t dev, device_t child, int type, int rid, struct resource *r) { struct generic_pcie_softc *sc; uint64_t phys_base; uint64_t pci_base; uint64_t size; int found; int res; int i; sc = device_get_softc(dev); if ((res = rman_activate_resource(r)) != 0) return (res); switch(type) { case SYS_RES_IOPORT: found = 0; for (i = 0; i < MAX_RANGES_TUPLES; i++) { pci_base = sc->ranges[i].pci_base; phys_base = sc->ranges[i].phys_base; size = sc->ranges[i].size; if ((rid > pci_base) && (rid < (pci_base + size))) { found = 1; break; } } if (found) { rman_set_start(r, rman_get_start(r) + phys_base); rman_set_end(r, rman_get_end(r) + phys_base); BUS_ACTIVATE_RESOURCE(device_get_parent(dev), child, type, rid, r); } else { device_printf(dev, "Failed to activate IOPORT resource\n"); res = 0; } break; case SYS_RES_MEMORY: BUS_ACTIVATE_RESOURCE(device_get_parent(dev), child, type, rid, r); break; default: break; } return (res); } static int generic_pcie_deactivate_resource(device_t dev, device_t child, int type, int rid, struct resource *r) { struct generic_pcie_softc *sc; vm_offset_t vaddr; int res; sc = device_get_softc(dev); if ((res = rman_deactivate_resource(r)) != 0) return (res); switch(type) { case SYS_RES_IOPORT: case SYS_RES_MEMORY: vaddr = (vm_offset_t)rman_get_virtual(r); pmap_unmapdev(vaddr, rman_get_size(r)); break; default: break; } return (res); } static int generic_pcie_alloc_msi(device_t pci, device_t child, int count, int maxcount, int *irqs) { #if defined(INTRNG) struct generic_pcie_softc *sc; sc = device_get_softc(pci); return (intr_alloc_msi(pci, child, sc->msi_parent, count, maxcount, irqs)); #elif defined(__aarch64__) return (arm_alloc_msi(pci, child, count, maxcount, irqs)); #else return (ENXIO); #endif } static int generic_pcie_release_msi(device_t pci, device_t child, int count, int *irqs) { #if defined(INTRNG) struct generic_pcie_softc *sc; sc = device_get_softc(pci); return (intr_release_msi(pci, child, sc->msi_parent, count, irqs)); #elif defined(__aarch64__) return (arm_release_msi(pci, child, count, irqs)); #else return (ENXIO); #endif } static int generic_pcie_map_msi(device_t pci, device_t child, int irq, uint64_t *addr, uint32_t *data) { #if defined(INTRNG) struct generic_pcie_softc *sc; sc = device_get_softc(pci); return (intr_map_msi(pci, child, sc->msi_parent, irq, addr, data)); #elif defined(__aarch64__) return (arm_map_msi(pci, child, irq, addr, data)); #else return (ENXIO); #endif } static int generic_pcie_alloc_msix(device_t pci, device_t child, int *irq) { #if defined(INTRNG) struct generic_pcie_softc *sc; sc = device_get_softc(pci); return (intr_alloc_msix(pci, child, sc->msi_parent, irq)); #elif defined(__aarch64__) return (arm_alloc_msix(pci, child, irq)); #else return (ENXIO); #endif } static int generic_pcie_release_msix(device_t pci, device_t child, int irq) { #if defined(INTRNG) struct generic_pcie_softc *sc; sc = device_get_softc(pci); return (intr_release_msix(pci, child, sc->msi_parent, irq)); #elif defined(__aarch64__) return (arm_release_msix(pci, child, irq)); #else return (ENXIO); #endif } +int +generic_pcie_get_id(device_t pci, device_t child, enum pci_id_type type, + uintptr_t *id) +{ + phandle_t node; + uint32_t rid; + uint16_t pci_rid; + + if (type != PCI_ID_MSI) + return (pcib_get_id(pci, child, type, id)); + + node = ofw_bus_get_node(pci); + pci_rid = pci_get_rid(child); + + ofw_bus_msimap(node, pci_rid, NULL, &rid); + *id = rid; + + return (0); +} + static device_method_t generic_pcie_methods[] = { DEVMETHOD(device_probe, generic_pcie_probe), DEVMETHOD(device_attach, pci_host_generic_attach), DEVMETHOD(bus_read_ivar, generic_pcie_read_ivar), DEVMETHOD(bus_write_ivar, generic_pcie_write_ivar), DEVMETHOD(bus_alloc_resource, pci_host_generic_alloc_resource), DEVMETHOD(bus_adjust_resource, generic_pcie_adjust_resource), DEVMETHOD(bus_release_resource, generic_pcie_release_resource), DEVMETHOD(bus_activate_resource, generic_pcie_activate_resource), DEVMETHOD(bus_deactivate_resource, generic_pcie_deactivate_resource), DEVMETHOD(bus_setup_intr, bus_generic_setup_intr), DEVMETHOD(bus_teardown_intr, bus_generic_teardown_intr), /* pcib interface */ DEVMETHOD(pcib_maxslots, generic_pcie_maxslots), DEVMETHOD(pcib_route_interrupt, generic_pcie_route_interrupt), DEVMETHOD(pcib_read_config, generic_pcie_read_config), DEVMETHOD(pcib_write_config, generic_pcie_write_config), DEVMETHOD(pcib_alloc_msi, generic_pcie_alloc_msi), DEVMETHOD(pcib_release_msi, generic_pcie_release_msi), DEVMETHOD(pcib_alloc_msix, generic_pcie_alloc_msix), DEVMETHOD(pcib_release_msix, generic_pcie_release_msix), DEVMETHOD(pcib_map_msi, generic_pcie_map_msi), + DEVMETHOD(pcib_get_id, generic_pcie_get_id), /* ofw_bus interface */ DEVMETHOD(ofw_bus_get_devinfo, generic_pcie_ofw_get_devinfo), DEVMETHOD(ofw_bus_get_compat, ofw_bus_gen_get_compat), DEVMETHOD(ofw_bus_get_model, ofw_bus_gen_get_model), DEVMETHOD(ofw_bus_get_name, ofw_bus_gen_get_name), DEVMETHOD(ofw_bus_get_node, ofw_bus_gen_get_node), DEVMETHOD(ofw_bus_get_type, ofw_bus_gen_get_type), DEVMETHOD_END }; static const struct ofw_bus_devinfo * generic_pcie_ofw_get_devinfo(device_t bus __unused, device_t child) { struct generic_pcie_ofw_devinfo *di; di = device_get_ivars(child); return (&di->di_dinfo); } static struct resource * generic_pcie_alloc_resource_ofw(device_t bus, device_t child, int type, int *rid, rman_res_t start, rman_res_t end, rman_res_t count, u_int flags) { struct generic_pcie_softc *sc; struct generic_pcie_ofw_devinfo *di; struct resource_list_entry *rle; int i; sc = device_get_softc(bus); if (RMAN_IS_DEFAULT_RANGE(start, end)) { if ((di = device_get_ivars(child)) == NULL) return (NULL); if (type == SYS_RES_IOPORT) type = SYS_RES_MEMORY; /* Find defaults for this rid */ rle = resource_list_find(&di->di_rl, type, *rid); if (rle == NULL) return (NULL); start = rle->start; end = rle->end; count = rle->count; } if (type == SYS_RES_MEMORY) { /* Remap through ranges property */ for (i = 0; i < MAX_RANGES_TUPLES; i++) { if (start >= sc->ranges[i].phys_base && end < sc->ranges[i].pci_base + sc->ranges[i].size) { start -= sc->ranges[i].phys_base; start += sc->ranges[i].pci_base; end -= sc->ranges[i].phys_base; end += sc->ranges[i].pci_base; break; } } if (i == MAX_RANGES_TUPLES) { device_printf(bus, "Could not map resource " "%#jx-%#jx\n", start, end); return (NULL); } } return (bus_generic_alloc_resource(bus, child, type, rid, start, end, count, flags)); } static int generic_pcie_release_resource_ofw(device_t bus, device_t child, int type, int rid, struct resource *res) { return (bus_generic_release_resource(bus, child, type, rid, res)); } /* Helper functions */ static int generic_pcie_ofw_bus_attach(device_t dev) { struct generic_pcie_ofw_devinfo *di; device_t child; phandle_t parent, node; pcell_t addr_cells, size_cells; parent = ofw_bus_get_node(dev); if (parent > 0) { get_addr_size_cells(parent, &addr_cells, &size_cells); /* Iterate through all bus subordinates */ for (node = OF_child(parent); node > 0; node = OF_peer(node)) { /* Allocate and populate devinfo. */ di = malloc(sizeof(*di), M_DEVBUF, M_WAITOK | M_ZERO); if (ofw_bus_gen_setup_devinfo(&di->di_dinfo, node) != 0) { free(di, M_DEVBUF); continue; } /* Initialize and populate resource list. */ resource_list_init(&di->di_rl); ofw_bus_reg_to_rl(dev, node, addr_cells, size_cells, &di->di_rl); ofw_bus_intr_to_rl(dev, node, &di->di_rl, NULL); /* Add newbus device for this FDT node */ child = device_add_child(dev, NULL, -1); if (child == NULL) { resource_list_free(&di->di_rl); ofw_bus_gen_destroy_devinfo(&di->di_dinfo); free(di, M_DEVBUF); continue; } device_set_ivars(child, di); } } return (0); } DEFINE_CLASS_0(pcib, generic_pcie_driver, generic_pcie_methods, sizeof(struct generic_pcie_softc)); devclass_t generic_pcie_devclass; DRIVER_MODULE(pcib, simplebus, generic_pcie_driver, generic_pcie_devclass, 0, 0); DRIVER_MODULE(pcib, ofwbus, generic_pcie_driver, generic_pcie_devclass, 0, 0); Index: head/sys/dev/pci/pci_host_generic.h =================================================================== --- head/sys/dev/pci/pci_host_generic.h (revision 299931) +++ head/sys/dev/pci/pci_host_generic.h (revision 299932) @@ -1,74 +1,75 @@ /* * Copyright (c) 2015 Ruslan Bukin * Copyright (c) 2015 The FreeBSD Foundation * All rights reserved. * * This software was developed by Semihalf. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * * $FreeBSD$ * */ #ifndef __PCI_HOST_GENERIC_H_ #define __PCI_HOST_GENERIC_H_ #define MAX_RANGES_TUPLES 16 #define MIN_RANGES_TUPLES 2 struct pcie_range { uint64_t pci_base; uint64_t phys_base; uint64_t size; uint64_t flags; #define FLAG_IO (1 << 0) #define FLAG_MEM (1 << 1) }; struct generic_pcie_softc { struct pcie_range ranges[MAX_RANGES_TUPLES]; int nranges; struct rman mem_rman; struct rman io_rman; struct resource *res; struct resource *res1; int ecam; bus_space_tag_t bst; bus_space_handle_t bsh; device_t dev; bus_space_handle_t ioh; #ifdef FDT struct ofw_bus_iinfo pci_iinfo; phandle_t msi_parent; #endif }; extern devclass_t generic_pcie_devclass; DECLARE_CLASS(generic_pcie_driver); struct resource *pci_host_generic_alloc_resource(device_t, device_t, int, int *, rman_res_t, rman_res_t, rman_res_t, u_int); int pci_host_generic_attach(device_t); +int generic_pcie_get_id(device_t, device_t, enum pci_id_type, uintptr_t *); #endif /* __PCI_HOST_GENERIC_H_ */ Index: head/sys/dev/pci/pci_if.m =================================================================== --- head/sys/dev/pci/pci_if.m (revision 299931) +++ head/sys/dev/pci/pci_if.m (revision 299932) @@ -1,250 +1,251 @@ #- # Copyright (c) 1998 Doug Rabson # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # 1. Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND # ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE # ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE # FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS # OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) # HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY # OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF # SUCH DAMAGE. # # $FreeBSD$ # #include #include INTERFACE pci; CODE { static int null_msi_count(device_t dev, device_t child) { return (0); } static int null_msix_bar(device_t dev, device_t child) { return (-1); } static device_t null_create_iov_child(device_t bus, device_t pf, uint16_t rid, uint16_t vid, uint16_t did) { device_printf(bus, "PCI_IOV not implemented on this bus.\n"); return (NULL); } }; HEADER { struct nvlist; enum pci_id_type { PCI_ID_RID, + PCI_ID_MSI, }; } METHOD u_int32_t read_config { device_t dev; device_t child; int reg; int width; }; METHOD void write_config { device_t dev; device_t child; int reg; u_int32_t val; int width; }; METHOD int get_powerstate { device_t dev; device_t child; }; METHOD int set_powerstate { device_t dev; device_t child; int state; }; METHOD int get_vpd_ident { device_t dev; device_t child; const char **identptr; }; METHOD int get_vpd_readonly { device_t dev; device_t child; const char *kw; const char **vptr; }; METHOD int enable_busmaster { device_t dev; device_t child; }; METHOD int disable_busmaster { device_t dev; device_t child; }; METHOD int enable_io { device_t dev; device_t child; int space; }; METHOD int disable_io { device_t dev; device_t child; int space; }; METHOD int assign_interrupt { device_t dev; device_t child; }; METHOD int find_cap { device_t dev; device_t child; int capability; int *capreg; }; METHOD int find_extcap { device_t dev; device_t child; int capability; int *capreg; }; METHOD int find_htcap { device_t dev; device_t child; int capability; int *capreg; }; METHOD int alloc_msi { device_t dev; device_t child; int *count; }; METHOD int alloc_msix { device_t dev; device_t child; int *count; }; METHOD void enable_msi { device_t dev; device_t child; uint64_t address; uint16_t data; }; METHOD void enable_msix { device_t dev; device_t child; u_int index; uint64_t address; uint32_t data; }; METHOD void disable_msi { device_t dev; device_t child; }; METHOD int remap_msix { device_t dev; device_t child; int count; const u_int *vectors; }; METHOD int release_msi { device_t dev; device_t child; }; METHOD int msi_count { device_t dev; device_t child; } DEFAULT null_msi_count; METHOD int msix_count { device_t dev; device_t child; } DEFAULT null_msi_count; METHOD int msix_pba_bar { device_t dev; device_t child; } DEFAULT null_msix_bar; METHOD int msix_table_bar { device_t dev; device_t child; } DEFAULT null_msix_bar; METHOD int get_id { device_t dev; device_t child; enum pci_id_type type; uintptr_t *id; }; METHOD struct pci_devinfo * alloc_devinfo { device_t dev; }; METHOD void child_added { device_t dev; device_t child; }; METHOD int iov_attach { device_t dev; device_t child; struct nvlist *pf_schema; struct nvlist *vf_schema; }; METHOD int iov_detach { device_t dev; device_t child; }; METHOD device_t create_iov_child { device_t bus; device_t pf; uint16_t rid; uint16_t vid; uint16_t did; } DEFAULT null_create_iov_child; Index: head/sys/dev/pci/pci_pci.c =================================================================== --- head/sys/dev/pci/pci_pci.c (revision 299931) +++ head/sys/dev/pci/pci_pci.c (revision 299932) @@ -1,2661 +1,2664 @@ /*- * Copyright (c) 1994,1995 Stefan Esser, Wolfgang StanglMeier * Copyright (c) 2000 Michael Smith * Copyright (c) 2000 BSDi * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. The name of the author may not be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); /* * PCI:PCI bridge support. */ #include "opt_pci.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include "pcib_if.h" static int pcib_probe(device_t dev); static int pcib_suspend(device_t dev); static int pcib_resume(device_t dev); static int pcib_power_for_sleep(device_t pcib, device_t dev, int *pstate); static int pcib_ari_get_id(device_t pcib, device_t dev, enum pci_id_type type, uintptr_t *id); static uint32_t pcib_read_config(device_t dev, u_int b, u_int s, u_int f, u_int reg, int width); static void pcib_write_config(device_t dev, u_int b, u_int s, u_int f, u_int reg, uint32_t val, int width); static int pcib_ari_maxslots(device_t dev); static int pcib_ari_maxfuncs(device_t dev); static int pcib_try_enable_ari(device_t pcib, device_t dev); static int pcib_ari_enabled(device_t pcib); static void pcib_ari_decode_rid(device_t pcib, uint16_t rid, int *bus, int *slot, int *func); #ifdef PCI_HP static void pcib_pcie_ab_timeout(void *arg); static void pcib_pcie_cc_timeout(void *arg); static void pcib_pcie_dll_timeout(void *arg); #endif static device_method_t pcib_methods[] = { /* Device interface */ DEVMETHOD(device_probe, pcib_probe), DEVMETHOD(device_attach, pcib_attach), DEVMETHOD(device_detach, bus_generic_detach), DEVMETHOD(device_shutdown, bus_generic_shutdown), DEVMETHOD(device_suspend, pcib_suspend), DEVMETHOD(device_resume, pcib_resume), /* Bus interface */ DEVMETHOD(bus_child_present, pcib_child_present), DEVMETHOD(bus_read_ivar, pcib_read_ivar), DEVMETHOD(bus_write_ivar, pcib_write_ivar), DEVMETHOD(bus_alloc_resource, pcib_alloc_resource), #ifdef NEW_PCIB DEVMETHOD(bus_adjust_resource, pcib_adjust_resource), DEVMETHOD(bus_release_resource, pcib_release_resource), #else DEVMETHOD(bus_adjust_resource, bus_generic_adjust_resource), DEVMETHOD(bus_release_resource, bus_generic_release_resource), #endif DEVMETHOD(bus_activate_resource, bus_generic_activate_resource), DEVMETHOD(bus_deactivate_resource, bus_generic_deactivate_resource), DEVMETHOD(bus_setup_intr, bus_generic_setup_intr), DEVMETHOD(bus_teardown_intr, bus_generic_teardown_intr), /* pcib interface */ DEVMETHOD(pcib_maxslots, pcib_ari_maxslots), DEVMETHOD(pcib_maxfuncs, pcib_ari_maxfuncs), DEVMETHOD(pcib_read_config, pcib_read_config), DEVMETHOD(pcib_write_config, pcib_write_config), DEVMETHOD(pcib_route_interrupt, pcib_route_interrupt), DEVMETHOD(pcib_alloc_msi, pcib_alloc_msi), DEVMETHOD(pcib_release_msi, pcib_release_msi), DEVMETHOD(pcib_alloc_msix, pcib_alloc_msix), DEVMETHOD(pcib_release_msix, pcib_release_msix), DEVMETHOD(pcib_map_msi, pcib_map_msi), DEVMETHOD(pcib_power_for_sleep, pcib_power_for_sleep), DEVMETHOD(pcib_get_id, pcib_ari_get_id), DEVMETHOD(pcib_try_enable_ari, pcib_try_enable_ari), DEVMETHOD(pcib_ari_enabled, pcib_ari_enabled), DEVMETHOD(pcib_decode_rid, pcib_ari_decode_rid), DEVMETHOD_END }; static devclass_t pcib_devclass; DEFINE_CLASS_0(pcib, pcib_driver, pcib_methods, sizeof(struct pcib_softc)); DRIVER_MODULE(pcib, pci, pcib_driver, pcib_devclass, NULL, NULL); #ifdef NEW_PCIB SYSCTL_DECL(_hw_pci); static int pci_clear_pcib; SYSCTL_INT(_hw_pci, OID_AUTO, clear_pcib, CTLFLAG_RDTUN, &pci_clear_pcib, 0, "Clear firmware-assigned resources for PCI-PCI bridge I/O windows."); /* * Is a resource from a child device sub-allocated from one of our * resource managers? */ static int pcib_is_resource_managed(struct pcib_softc *sc, int type, struct resource *r) { switch (type) { #ifdef PCI_RES_BUS case PCI_RES_BUS: return (rman_is_region_manager(r, &sc->bus.rman)); #endif case SYS_RES_IOPORT: return (rman_is_region_manager(r, &sc->io.rman)); case SYS_RES_MEMORY: /* Prefetchable resources may live in either memory rman. */ if (rman_get_flags(r) & RF_PREFETCHABLE && rman_is_region_manager(r, &sc->pmem.rman)) return (1); return (rman_is_region_manager(r, &sc->mem.rman)); } return (0); } static int pcib_is_window_open(struct pcib_window *pw) { return (pw->valid && pw->base < pw->limit); } /* * XXX: If RF_ACTIVE did not also imply allocating a bus space tag and * handle for the resource, we could pass RF_ACTIVE up to the PCI bus * when allocating the resource windows and rely on the PCI bus driver * to do this for us. */ static void pcib_activate_window(struct pcib_softc *sc, int type) { PCI_ENABLE_IO(device_get_parent(sc->dev), sc->dev, type); } static void pcib_write_windows(struct pcib_softc *sc, int mask) { device_t dev; uint32_t val; dev = sc->dev; if (sc->io.valid && mask & WIN_IO) { val = pci_read_config(dev, PCIR_IOBASEL_1, 1); if ((val & PCIM_BRIO_MASK) == PCIM_BRIO_32) { pci_write_config(dev, PCIR_IOBASEH_1, sc->io.base >> 16, 2); pci_write_config(dev, PCIR_IOLIMITH_1, sc->io.limit >> 16, 2); } pci_write_config(dev, PCIR_IOBASEL_1, sc->io.base >> 8, 1); pci_write_config(dev, PCIR_IOLIMITL_1, sc->io.limit >> 8, 1); } if (mask & WIN_MEM) { pci_write_config(dev, PCIR_MEMBASE_1, sc->mem.base >> 16, 2); pci_write_config(dev, PCIR_MEMLIMIT_1, sc->mem.limit >> 16, 2); } if (sc->pmem.valid && mask & WIN_PMEM) { val = pci_read_config(dev, PCIR_PMBASEL_1, 2); if ((val & PCIM_BRPM_MASK) == PCIM_BRPM_64) { pci_write_config(dev, PCIR_PMBASEH_1, sc->pmem.base >> 32, 4); pci_write_config(dev, PCIR_PMLIMITH_1, sc->pmem.limit >> 32, 4); } pci_write_config(dev, PCIR_PMBASEL_1, sc->pmem.base >> 16, 2); pci_write_config(dev, PCIR_PMLIMITL_1, sc->pmem.limit >> 16, 2); } } /* * This is used to reject I/O port allocations that conflict with an * ISA alias range. */ static int pcib_is_isa_range(struct pcib_softc *sc, rman_res_t start, rman_res_t end, rman_res_t count) { rman_res_t next_alias; if (!(sc->bridgectl & PCIB_BCR_ISA_ENABLE)) return (0); /* Only check fixed ranges for overlap. */ if (start + count - 1 != end) return (0); /* ISA aliases are only in the lower 64KB of I/O space. */ if (start >= 65536) return (0); /* Check for overlap with 0x000 - 0x0ff as a special case. */ if (start < 0x100) goto alias; /* * If the start address is an alias, the range is an alias. * Otherwise, compute the start of the next alias range and * check if it is before the end of the candidate range. */ if ((start & 0x300) != 0) goto alias; next_alias = (start & ~0x3fful) | 0x100; if (next_alias <= end) goto alias; return (0); alias: if (bootverbose) device_printf(sc->dev, "I/O range %#jx-%#jx overlaps with an ISA alias\n", start, end); return (1); } static void pcib_add_window_resources(struct pcib_window *w, struct resource **res, int count) { struct resource **newarray; int error, i; newarray = malloc(sizeof(struct resource *) * (w->count + count), M_DEVBUF, M_WAITOK); if (w->res != NULL) bcopy(w->res, newarray, sizeof(struct resource *) * w->count); bcopy(res, newarray + w->count, sizeof(struct resource *) * count); free(w->res, M_DEVBUF); w->res = newarray; w->count += count; for (i = 0; i < count; i++) { error = rman_manage_region(&w->rman, rman_get_start(res[i]), rman_get_end(res[i])); if (error) panic("Failed to add resource to rman"); } } typedef void (nonisa_callback)(rman_res_t start, rman_res_t end, void *arg); static void pcib_walk_nonisa_ranges(rman_res_t start, rman_res_t end, nonisa_callback *cb, void *arg) { rman_res_t next_end; /* * If start is within an ISA alias range, move up to the start * of the next non-alias range. As a special case, addresses * in the range 0x000 - 0x0ff should also be skipped since * those are used for various system I/O devices in ISA * systems. */ if (start <= 65535) { if (start < 0x100 || (start & 0x300) != 0) { start &= ~0x3ff; start += 0x400; } } /* ISA aliases are only in the lower 64KB of I/O space. */ while (start <= MIN(end, 65535)) { next_end = MIN(start | 0xff, end); cb(start, next_end, arg); start += 0x400; } if (start <= end) cb(start, end, arg); } static void count_ranges(rman_res_t start, rman_res_t end, void *arg) { int *countp; countp = arg; (*countp)++; } struct alloc_state { struct resource **res; struct pcib_softc *sc; int count, error; }; static void alloc_ranges(rman_res_t start, rman_res_t end, void *arg) { struct alloc_state *as; struct pcib_window *w; int rid; as = arg; if (as->error != 0) return; w = &as->sc->io; rid = w->reg; if (bootverbose) device_printf(as->sc->dev, "allocating non-ISA range %#jx-%#jx\n", start, end); as->res[as->count] = bus_alloc_resource(as->sc->dev, SYS_RES_IOPORT, &rid, start, end, end - start + 1, 0); if (as->res[as->count] == NULL) as->error = ENXIO; else as->count++; } static int pcib_alloc_nonisa_ranges(struct pcib_softc *sc, rman_res_t start, rman_res_t end) { struct alloc_state as; int i, new_count; /* First, see how many ranges we need. */ new_count = 0; pcib_walk_nonisa_ranges(start, end, count_ranges, &new_count); /* Second, allocate the ranges. */ as.res = malloc(sizeof(struct resource *) * new_count, M_DEVBUF, M_WAITOK); as.sc = sc; as.count = 0; as.error = 0; pcib_walk_nonisa_ranges(start, end, alloc_ranges, &as); if (as.error != 0) { for (i = 0; i < as.count; i++) bus_release_resource(sc->dev, SYS_RES_IOPORT, sc->io.reg, as.res[i]); free(as.res, M_DEVBUF); return (as.error); } KASSERT(as.count == new_count, ("%s: count mismatch", __func__)); /* Third, add the ranges to the window. */ pcib_add_window_resources(&sc->io, as.res, as.count); free(as.res, M_DEVBUF); return (0); } static void pcib_alloc_window(struct pcib_softc *sc, struct pcib_window *w, int type, int flags, pci_addr_t max_address) { struct resource *res; char buf[64]; int error, rid; if (max_address != (rman_res_t)max_address) max_address = ~0; w->rman.rm_start = 0; w->rman.rm_end = max_address; w->rman.rm_type = RMAN_ARRAY; snprintf(buf, sizeof(buf), "%s %s window", device_get_nameunit(sc->dev), w->name); w->rman.rm_descr = strdup(buf, M_DEVBUF); error = rman_init(&w->rman); if (error) panic("Failed to initialize %s %s rman", device_get_nameunit(sc->dev), w->name); if (!pcib_is_window_open(w)) return; if (w->base > max_address || w->limit > max_address) { device_printf(sc->dev, "initial %s window has too many bits, ignoring\n", w->name); return; } if (type == SYS_RES_IOPORT && sc->bridgectl & PCIB_BCR_ISA_ENABLE) (void)pcib_alloc_nonisa_ranges(sc, w->base, w->limit); else { rid = w->reg; res = bus_alloc_resource(sc->dev, type, &rid, w->base, w->limit, w->limit - w->base + 1, flags); if (res != NULL) pcib_add_window_resources(w, &res, 1); } if (w->res == NULL) { device_printf(sc->dev, "failed to allocate initial %s window: %#jx-%#jx\n", w->name, (uintmax_t)w->base, (uintmax_t)w->limit); w->base = max_address; w->limit = 0; pcib_write_windows(sc, w->mask); return; } pcib_activate_window(sc, type); } /* * Initialize I/O windows. */ static void pcib_probe_windows(struct pcib_softc *sc) { pci_addr_t max; device_t dev; uint32_t val; dev = sc->dev; if (pci_clear_pcib) { pcib_bridge_init(dev); } /* Determine if the I/O port window is implemented. */ val = pci_read_config(dev, PCIR_IOBASEL_1, 1); if (val == 0) { /* * If 'val' is zero, then only 16-bits of I/O space * are supported. */ pci_write_config(dev, PCIR_IOBASEL_1, 0xff, 1); if (pci_read_config(dev, PCIR_IOBASEL_1, 1) != 0) { sc->io.valid = 1; pci_write_config(dev, PCIR_IOBASEL_1, 0, 1); } } else sc->io.valid = 1; /* Read the existing I/O port window. */ if (sc->io.valid) { sc->io.reg = PCIR_IOBASEL_1; sc->io.step = 12; sc->io.mask = WIN_IO; sc->io.name = "I/O port"; if ((val & PCIM_BRIO_MASK) == PCIM_BRIO_32) { sc->io.base = PCI_PPBIOBASE( pci_read_config(dev, PCIR_IOBASEH_1, 2), val); sc->io.limit = PCI_PPBIOLIMIT( pci_read_config(dev, PCIR_IOLIMITH_1, 2), pci_read_config(dev, PCIR_IOLIMITL_1, 1)); max = 0xffffffff; } else { sc->io.base = PCI_PPBIOBASE(0, val); sc->io.limit = PCI_PPBIOLIMIT(0, pci_read_config(dev, PCIR_IOLIMITL_1, 1)); max = 0xffff; } pcib_alloc_window(sc, &sc->io, SYS_RES_IOPORT, 0, max); } /* Read the existing memory window. */ sc->mem.valid = 1; sc->mem.reg = PCIR_MEMBASE_1; sc->mem.step = 20; sc->mem.mask = WIN_MEM; sc->mem.name = "memory"; sc->mem.base = PCI_PPBMEMBASE(0, pci_read_config(dev, PCIR_MEMBASE_1, 2)); sc->mem.limit = PCI_PPBMEMLIMIT(0, pci_read_config(dev, PCIR_MEMLIMIT_1, 2)); pcib_alloc_window(sc, &sc->mem, SYS_RES_MEMORY, 0, 0xffffffff); /* Determine if the prefetchable memory window is implemented. */ val = pci_read_config(dev, PCIR_PMBASEL_1, 2); if (val == 0) { /* * If 'val' is zero, then only 32-bits of memory space * are supported. */ pci_write_config(dev, PCIR_PMBASEL_1, 0xffff, 2); if (pci_read_config(dev, PCIR_PMBASEL_1, 2) != 0) { sc->pmem.valid = 1; pci_write_config(dev, PCIR_PMBASEL_1, 0, 2); } } else sc->pmem.valid = 1; /* Read the existing prefetchable memory window. */ if (sc->pmem.valid) { sc->pmem.reg = PCIR_PMBASEL_1; sc->pmem.step = 20; sc->pmem.mask = WIN_PMEM; sc->pmem.name = "prefetch"; if ((val & PCIM_BRPM_MASK) == PCIM_BRPM_64) { sc->pmem.base = PCI_PPBMEMBASE( pci_read_config(dev, PCIR_PMBASEH_1, 4), val); sc->pmem.limit = PCI_PPBMEMLIMIT( pci_read_config(dev, PCIR_PMLIMITH_1, 4), pci_read_config(dev, PCIR_PMLIMITL_1, 2)); max = 0xffffffffffffffff; } else { sc->pmem.base = PCI_PPBMEMBASE(0, val); sc->pmem.limit = PCI_PPBMEMLIMIT(0, pci_read_config(dev, PCIR_PMLIMITL_1, 2)); max = 0xffffffff; } pcib_alloc_window(sc, &sc->pmem, SYS_RES_MEMORY, RF_PREFETCHABLE, max); } } #ifdef PCI_RES_BUS /* * Allocate a suitable secondary bus for this bridge if needed and * initialize the resource manager for the secondary bus range. Note * that the minimum count is a desired value and this may allocate a * smaller range. */ void pcib_setup_secbus(device_t dev, struct pcib_secbus *bus, int min_count) { char buf[64]; int error, rid, sec_reg; switch (pci_read_config(dev, PCIR_HDRTYPE, 1) & PCIM_HDRTYPE) { case PCIM_HDRTYPE_BRIDGE: sec_reg = PCIR_SECBUS_1; bus->sub_reg = PCIR_SUBBUS_1; break; case PCIM_HDRTYPE_CARDBUS: sec_reg = PCIR_SECBUS_2; bus->sub_reg = PCIR_SUBBUS_2; break; default: panic("not a PCI bridge"); } bus->sec = pci_read_config(dev, sec_reg, 1); bus->sub = pci_read_config(dev, bus->sub_reg, 1); bus->dev = dev; bus->rman.rm_start = 0; bus->rman.rm_end = PCI_BUSMAX; bus->rman.rm_type = RMAN_ARRAY; snprintf(buf, sizeof(buf), "%s bus numbers", device_get_nameunit(dev)); bus->rman.rm_descr = strdup(buf, M_DEVBUF); error = rman_init(&bus->rman); if (error) panic("Failed to initialize %s bus number rman", device_get_nameunit(dev)); /* * Allocate a bus range. This will return an existing bus range * if one exists, or a new bus range if one does not. */ rid = 0; bus->res = bus_alloc_resource_anywhere(dev, PCI_RES_BUS, &rid, min_count, 0); if (bus->res == NULL) { /* * Fall back to just allocating a range of a single bus * number. */ bus->res = bus_alloc_resource_anywhere(dev, PCI_RES_BUS, &rid, 1, 0); } else if (rman_get_size(bus->res) < min_count) /* * Attempt to grow the existing range to satisfy the * minimum desired count. */ (void)bus_adjust_resource(dev, PCI_RES_BUS, bus->res, rman_get_start(bus->res), rman_get_start(bus->res) + min_count - 1); /* * Add the initial resource to the rman. */ if (bus->res != NULL) { error = rman_manage_region(&bus->rman, rman_get_start(bus->res), rman_get_end(bus->res)); if (error) panic("Failed to add resource to rman"); bus->sec = rman_get_start(bus->res); bus->sub = rman_get_end(bus->res); } } static struct resource * pcib_suballoc_bus(struct pcib_secbus *bus, device_t child, int *rid, rman_res_t start, rman_res_t end, rman_res_t count, u_int flags) { struct resource *res; res = rman_reserve_resource(&bus->rman, start, end, count, flags, child); if (res == NULL) return (NULL); if (bootverbose) device_printf(bus->dev, "allocated bus range (%ju-%ju) for rid %d of %s\n", rman_get_start(res), rman_get_end(res), *rid, pcib_child_name(child)); rman_set_rid(res, *rid); return (res); } /* * Attempt to grow the secondary bus range. This is much simpler than * for I/O windows as the range can only be grown by increasing * subbus. */ static int pcib_grow_subbus(struct pcib_secbus *bus, rman_res_t new_end) { rman_res_t old_end; int error; old_end = rman_get_end(bus->res); KASSERT(new_end > old_end, ("attempt to shrink subbus")); error = bus_adjust_resource(bus->dev, PCI_RES_BUS, bus->res, rman_get_start(bus->res), new_end); if (error) return (error); if (bootverbose) device_printf(bus->dev, "grew bus range to %ju-%ju\n", rman_get_start(bus->res), rman_get_end(bus->res)); error = rman_manage_region(&bus->rman, old_end + 1, rman_get_end(bus->res)); if (error) panic("Failed to add resource to rman"); bus->sub = rman_get_end(bus->res); pci_write_config(bus->dev, bus->sub_reg, bus->sub, 1); return (0); } struct resource * pcib_alloc_subbus(struct pcib_secbus *bus, device_t child, int *rid, rman_res_t start, rman_res_t end, rman_res_t count, u_int flags) { struct resource *res; rman_res_t start_free, end_free, new_end; /* * First, see if the request can be satisified by the existing * bus range. */ res = pcib_suballoc_bus(bus, child, rid, start, end, count, flags); if (res != NULL) return (res); /* * Figure out a range to grow the bus range. First, find the * first bus number after the last allocated bus in the rman and * enforce that as a minimum starting point for the range. */ if (rman_last_free_region(&bus->rman, &start_free, &end_free) != 0 || end_free != bus->sub) start_free = bus->sub + 1; if (start_free < start) start_free = start; new_end = start_free + count - 1; /* * See if this new range would satisfy the request if it * succeeds. */ if (new_end > end) return (NULL); /* Finally, attempt to grow the existing resource. */ if (bootverbose) { device_printf(bus->dev, "attempting to grow bus range for %ju buses\n", count); printf("\tback candidate range: %ju-%ju\n", start_free, new_end); } if (pcib_grow_subbus(bus, new_end) == 0) return (pcib_suballoc_bus(bus, child, rid, start, end, count, flags)); return (NULL); } #endif #else /* * Is the prefetch window open (eg, can we allocate memory in it?) */ static int pcib_is_prefetch_open(struct pcib_softc *sc) { return (sc->pmembase > 0 && sc->pmembase < sc->pmemlimit); } /* * Is the nonprefetch window open (eg, can we allocate memory in it?) */ static int pcib_is_nonprefetch_open(struct pcib_softc *sc) { return (sc->membase > 0 && sc->membase < sc->memlimit); } /* * Is the io window open (eg, can we allocate ports in it?) */ static int pcib_is_io_open(struct pcib_softc *sc) { return (sc->iobase > 0 && sc->iobase < sc->iolimit); } /* * Get current I/O decode. */ static void pcib_get_io_decode(struct pcib_softc *sc) { device_t dev; uint32_t iolow; dev = sc->dev; iolow = pci_read_config(dev, PCIR_IOBASEL_1, 1); if ((iolow & PCIM_BRIO_MASK) == PCIM_BRIO_32) sc->iobase = PCI_PPBIOBASE( pci_read_config(dev, PCIR_IOBASEH_1, 2), iolow); else sc->iobase = PCI_PPBIOBASE(0, iolow); iolow = pci_read_config(dev, PCIR_IOLIMITL_1, 1); if ((iolow & PCIM_BRIO_MASK) == PCIM_BRIO_32) sc->iolimit = PCI_PPBIOLIMIT( pci_read_config(dev, PCIR_IOLIMITH_1, 2), iolow); else sc->iolimit = PCI_PPBIOLIMIT(0, iolow); } /* * Get current memory decode. */ static void pcib_get_mem_decode(struct pcib_softc *sc) { device_t dev; pci_addr_t pmemlow; dev = sc->dev; sc->membase = PCI_PPBMEMBASE(0, pci_read_config(dev, PCIR_MEMBASE_1, 2)); sc->memlimit = PCI_PPBMEMLIMIT(0, pci_read_config(dev, PCIR_MEMLIMIT_1, 2)); pmemlow = pci_read_config(dev, PCIR_PMBASEL_1, 2); if ((pmemlow & PCIM_BRPM_MASK) == PCIM_BRPM_64) sc->pmembase = PCI_PPBMEMBASE( pci_read_config(dev, PCIR_PMBASEH_1, 4), pmemlow); else sc->pmembase = PCI_PPBMEMBASE(0, pmemlow); pmemlow = pci_read_config(dev, PCIR_PMLIMITL_1, 2); if ((pmemlow & PCIM_BRPM_MASK) == PCIM_BRPM_64) sc->pmemlimit = PCI_PPBMEMLIMIT( pci_read_config(dev, PCIR_PMLIMITH_1, 4), pmemlow); else sc->pmemlimit = PCI_PPBMEMLIMIT(0, pmemlow); } /* * Restore previous I/O decode. */ static void pcib_set_io_decode(struct pcib_softc *sc) { device_t dev; uint32_t iohi; dev = sc->dev; iohi = sc->iobase >> 16; if (iohi > 0) pci_write_config(dev, PCIR_IOBASEH_1, iohi, 2); pci_write_config(dev, PCIR_IOBASEL_1, sc->iobase >> 8, 1); iohi = sc->iolimit >> 16; if (iohi > 0) pci_write_config(dev, PCIR_IOLIMITH_1, iohi, 2); pci_write_config(dev, PCIR_IOLIMITL_1, sc->iolimit >> 8, 1); } /* * Restore previous memory decode. */ static void pcib_set_mem_decode(struct pcib_softc *sc) { device_t dev; pci_addr_t pmemhi; dev = sc->dev; pci_write_config(dev, PCIR_MEMBASE_1, sc->membase >> 16, 2); pci_write_config(dev, PCIR_MEMLIMIT_1, sc->memlimit >> 16, 2); pmemhi = sc->pmembase >> 32; if (pmemhi > 0) pci_write_config(dev, PCIR_PMBASEH_1, pmemhi, 4); pci_write_config(dev, PCIR_PMBASEL_1, sc->pmembase >> 16, 2); pmemhi = sc->pmemlimit >> 32; if (pmemhi > 0) pci_write_config(dev, PCIR_PMLIMITH_1, pmemhi, 4); pci_write_config(dev, PCIR_PMLIMITL_1, sc->pmemlimit >> 16, 2); } #endif #ifdef PCI_HP /* * PCI-express HotPlug support. */ static void pcib_probe_hotplug(struct pcib_softc *sc) { device_t dev; dev = sc->dev; if (pci_find_cap(dev, PCIY_EXPRESS, NULL) != 0) return; if (!(pcie_read_config(dev, PCIER_FLAGS, 2) & PCIEM_FLAGS_SLOT)) return; sc->pcie_link_cap = pcie_read_config(dev, PCIER_LINK_CAP, 4); sc->pcie_slot_cap = pcie_read_config(dev, PCIER_SLOT_CAP, 4); if (sc->pcie_slot_cap & PCIEM_SLOT_CAP_HPC) sc->flags |= PCIB_HOTPLUG; } /* * Send a HotPlug command to the slot control register. If this slot * uses command completion interrupts, these updates will be buffered * while a previous command is completing. */ static void pcib_pcie_hotplug_command(struct pcib_softc *sc, uint16_t val, uint16_t mask) { device_t dev; uint16_t ctl, new; dev = sc->dev; if (sc->pcie_slot_cap & PCIEM_SLOT_CAP_NCCS) { ctl = pcie_read_config(dev, PCIER_SLOT_CTL, 2); new = (ctl & ~mask) | val; if (new != ctl) pcie_write_config(dev, PCIER_SLOT_CTL, new, 2); return; } if (sc->flags & PCIB_HOTPLUG_CMD_PENDING) { sc->pcie_pending_link_ctl_val &= ~mask; sc->pcie_pending_link_ctl_val |= val; sc->pcie_pending_link_ctl_mask |= mask; } else { ctl = pcie_read_config(dev, PCIER_SLOT_CTL, 2); new = (ctl & ~mask) | val; if (new != ctl) { pcie_write_config(dev, PCIER_SLOT_CTL, ctl, 2); sc->flags |= PCIB_HOTPLUG_CMD_PENDING; if (!cold) callout_reset(&sc->pcie_cc_timer, hz, pcib_pcie_cc_timeout, sc); } } } static void pcib_pcie_hotplug_command_completed(struct pcib_softc *sc) { device_t dev; uint16_t ctl, new; dev = sc->dev; if (bootverbose) device_printf(dev, "Command Completed\n"); if (!(sc->flags & PCIB_HOTPLUG_CMD_PENDING)) return; if (sc->pcie_pending_link_ctl_mask != 0) { ctl = pcie_read_config(dev, PCIER_SLOT_CTL, 2); new = ctl & ~sc->pcie_pending_link_ctl_mask; new |= sc->pcie_pending_link_ctl_val; if (new != ctl) { pcie_write_config(dev, PCIER_SLOT_CTL, ctl, 2); if (!cold) callout_reset(&sc->pcie_cc_timer, hz, pcib_pcie_cc_timeout, sc); } else sc->flags &= ~PCIB_HOTPLUG_CMD_PENDING; sc->pcie_pending_link_ctl_mask = 0; sc->pcie_pending_link_ctl_val = 0; } else { callout_stop(&sc->pcie_cc_timer); sc->flags &= ~PCIB_HOTPLUG_CMD_PENDING; } } /* * Returns true if a card is fully inserted from the user's * perspective. It may not yet be ready for access, but the driver * can now start enabling access if necessary. */ static bool pcib_hotplug_inserted(struct pcib_softc *sc) { /* Pretend the card isn't present if a detach is forced. */ if (sc->flags & PCIB_DETACHING) return (false); /* Card must be present in the slot. */ if ((sc->pcie_slot_sta & PCIEM_SLOT_STA_PDS) == 0) return (false); /* A power fault implicitly turns off power to the slot. */ if (sc->pcie_slot_sta & PCIEM_SLOT_STA_PFD) return (false); /* If the MRL is disengaged, the slot is powered off. */ if (sc->pcie_slot_cap & PCIEM_SLOT_CAP_MRLSP && (sc->pcie_slot_sta & PCIEM_SLOT_STA_MRLSS) != 0) return (false); return (true); } /* * Returns -1 if the card is fully inserted, powered, and ready for * access. Otherwise, returns 0. */ static int pcib_hotplug_present(struct pcib_softc *sc) { device_t dev; dev = sc->dev; /* Card must be inserted. */ if (!pcib_hotplug_inserted(sc)) return (0); /* * Require the Electromechanical Interlock to be engaged if * present. */ if (sc->pcie_slot_cap & PCIEM_SLOT_CAP_EIP && (sc->pcie_slot_sta & PCIEM_SLOT_STA_EIS) == 0) return (0); /* Require the Data Link Layer to be active. */ if (sc->pcie_link_cap & PCIEM_LINK_CAP_DL_ACTIVE) { if (!(sc->pcie_link_sta & PCIEM_LINK_STA_DL_ACTIVE)) return (0); } return (-1); } static void pcib_pcie_hotplug_update(struct pcib_softc *sc, uint16_t val, uint16_t mask, bool schedule_task) { bool card_inserted; /* Clear DETACHING if Present Detect has cleared. */ if ((sc->pcie_slot_sta & (PCIEM_SLOT_STA_PDC | PCIEM_SLOT_STA_PDS)) == PCIEM_SLOT_STA_PDC) sc->flags &= ~PCIB_DETACHING; card_inserted = pcib_hotplug_inserted(sc); /* Turn the power indicator on if a card is inserted. */ if (sc->pcie_slot_cap & PCIEM_SLOT_CAP_PIP) { mask |= PCIEM_SLOT_CTL_PIC; if (card_inserted) val |= PCIEM_SLOT_CTL_PI_ON; else if (sc->flags & PCIB_DETACH_PENDING) val |= PCIEM_SLOT_CTL_PI_BLINK; else val |= PCIEM_SLOT_CTL_PI_OFF; } /* Turn the power on via the Power Controller if a card is inserted. */ if (sc->pcie_slot_cap & PCIEM_SLOT_CAP_PCP) { mask |= PCIEM_SLOT_CTL_PCC; if (card_inserted) val |= PCIEM_SLOT_CTL_PC_ON; else val |= PCIEM_SLOT_CTL_PC_OFF; } /* * If a card is inserted, enable the Electromechanical * Interlock. If a card is not inserted (or we are in the * process of detaching), disable the Electromechanical * Interlock. */ if (sc->pcie_slot_cap & PCIEM_SLOT_CAP_EIP) { if (card_inserted != !(sc->pcie_slot_sta & PCIEM_SLOT_STA_EIS)) { mask |= PCIEM_SLOT_CTL_EIC; val |= PCIEM_SLOT_CTL_EIC; } } /* * Start a timer to see if the Data Link Layer times out. * Note that we only start the timer if Presence Detect * changed on this interrupt. Stop any scheduled timer if * the Data Link Layer is active. */ if (sc->pcie_link_cap & PCIEM_LINK_CAP_DL_ACTIVE) { if (card_inserted && !(sc->pcie_link_sta & PCIEM_LINK_STA_DL_ACTIVE) && sc->pcie_slot_sta & PCIEM_SLOT_STA_PDC) { if (cold) device_printf(sc->dev, "Data Link Layer inactive\n"); else callout_reset(&sc->pcie_dll_timer, hz, pcib_pcie_dll_timeout, sc); } else if (sc->pcie_link_sta & PCIEM_LINK_STA_DL_ACTIVE) callout_stop(&sc->pcie_dll_timer); } pcib_pcie_hotplug_command(sc, val, mask); /* * During attach the child "pci" device is added sychronously; * otherwise, the task is scheduled to manage the child * device. */ if (schedule_task && (pcib_hotplug_present(sc) != 0) != (sc->child != NULL)) taskqueue_enqueue(taskqueue_thread, &sc->pcie_hp_task); } static void pcib_pcie_intr(void *arg) { struct pcib_softc *sc; device_t dev; sc = arg; dev = sc->dev; sc->pcie_slot_sta = pcie_read_config(dev, PCIER_SLOT_STA, 2); /* Clear the events just reported. */ pcie_write_config(dev, PCIER_SLOT_STA, sc->pcie_slot_sta, 2); if (sc->pcie_slot_sta & PCIEM_SLOT_STA_ABP) { if (sc->flags & PCIB_DETACH_PENDING) { device_printf(dev, "Attention Button Pressed: Detach Cancelled\n"); sc->flags &= ~PCIB_DETACH_PENDING; callout_stop(&sc->pcie_ab_timer); } else { device_printf(dev, "Attention Button Pressed: Detaching in 5 seconds\n"); sc->flags |= PCIB_DETACH_PENDING; callout_reset(&sc->pcie_ab_timer, 5 * hz, pcib_pcie_ab_timeout, sc); } } if (sc->pcie_slot_sta & PCIEM_SLOT_STA_PFD) device_printf(dev, "Power Fault Detected\n"); if (sc->pcie_slot_sta & PCIEM_SLOT_STA_MRLSC) device_printf(dev, "MRL Sensor Changed to %s\n", sc->pcie_slot_sta & PCIEM_SLOT_STA_MRLSS ? "open" : "closed"); if (bootverbose && sc->pcie_slot_sta & PCIEM_SLOT_STA_PDC) device_printf(dev, "Present Detect Changed to %s\n", sc->pcie_slot_sta & PCIEM_SLOT_STA_PDS ? "card present" : "empty"); if (sc->pcie_slot_sta & PCIEM_SLOT_STA_CC) pcib_pcie_hotplug_command_completed(sc); if (sc->pcie_slot_sta & PCIEM_SLOT_STA_DLLSC) { sc->pcie_link_sta = pcie_read_config(dev, PCIER_LINK_STA, 2); if (bootverbose) device_printf(dev, "Data Link Layer State Changed to %s\n", sc->pcie_link_sta & PCIEM_LINK_STA_DL_ACTIVE ? "active" : "inactive"); } pcib_pcie_hotplug_update(sc, 0, 0, true); } static void pcib_pcie_hotplug_task(void *context, int pending) { struct pcib_softc *sc; device_t dev; sc = context; mtx_lock(&Giant); dev = sc->dev; if (pcib_hotplug_present(sc) != 0) { if (sc->child == NULL) { sc->child = device_add_child(dev, "pci", -1); bus_generic_attach(dev); } } else { if (sc->child != NULL) { if (device_delete_child(dev, sc->child) == 0) sc->child = NULL; } } mtx_unlock(&Giant); } static void pcib_pcie_ab_timeout(void *arg) { struct pcib_softc *sc; device_t dev; sc = arg; dev = sc->dev; mtx_assert(&Giant, MA_OWNED); if (sc->flags & PCIB_DETACH_PENDING) { sc->flags |= PCIB_DETACHING; sc->flags &= ~PCIB_DETACH_PENDING; pcib_pcie_hotplug_update(sc, 0, 0, true); } } static void pcib_pcie_cc_timeout(void *arg) { struct pcib_softc *sc; device_t dev; sc = arg; dev = sc->dev; mtx_assert(&Giant, MA_OWNED); if (sc->flags & PCIB_HOTPLUG_CMD_PENDING) { device_printf(dev, "Hotplug Command Timed Out - forcing detach\n"); sc->flags &= ~(PCIB_HOTPLUG_CMD_PENDING | PCIB_DETACH_PENDING); sc->flags |= PCIB_DETACHING; pcib_pcie_hotplug_update(sc, 0, 0, true); } } static void pcib_pcie_dll_timeout(void *arg) { struct pcib_softc *sc; device_t dev; uint16_t sta; sc = arg; dev = sc->dev; mtx_assert(&Giant, MA_OWNED); sta = pcie_read_config(dev, PCIER_LINK_STA, 2); if (!(sta & PCIEM_LINK_STA_DL_ACTIVE)) { device_printf(dev, "Timed out waiting for Data Link Layer Active\n"); sc->flags |= PCIB_DETACHING; pcib_pcie_hotplug_update(sc, 0, 0, true); } else if (sta != sc->pcie_link_sta) { device_printf(dev, "Missed HotPlug interrupt waiting for DLL Active\n"); pcib_pcie_intr(sc); } } static int pcib_alloc_pcie_irq(struct pcib_softc *sc) { device_t dev; int count, error, rid; rid = -1; dev = sc->dev; /* * For simplicity, only use MSI-X if there is a single message. * To support a device with multiple messages we would have to * use remap intr if the MSI number is not 0. */ count = pci_msix_count(dev); if (count == 1) { error = pci_alloc_msix(dev, &count); if (error == 0) rid = 1; } if (rid < 0 && pci_msi_count(dev) > 0) { count = 1; error = pci_alloc_msi(dev, &count); if (error == 0) rid = 1; } if (rid < 0) rid = 0; sc->pcie_irq = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid, RF_ACTIVE); if (sc->pcie_irq == NULL) { device_printf(dev, "Failed to allocate interrupt for PCI-e events\n"); if (rid > 0) pci_release_msi(dev); return (ENXIO); } error = bus_setup_intr(dev, sc->pcie_irq, INTR_TYPE_MISC, NULL, pcib_pcie_intr, sc, &sc->pcie_ihand); if (error) { device_printf(dev, "Failed to setup PCI-e interrupt handler\n"); bus_release_resource(dev, SYS_RES_IRQ, rid, sc->pcie_irq); if (rid > 0) pci_release_msi(dev); return (error); } return (0); } static void pcib_setup_hotplug(struct pcib_softc *sc) { device_t dev; uint16_t mask, val; dev = sc->dev; callout_init(&sc->pcie_ab_timer, 0); callout_init(&sc->pcie_cc_timer, 0); callout_init(&sc->pcie_dll_timer, 0); TASK_INIT(&sc->pcie_hp_task, 0, pcib_pcie_hotplug_task, sc); /* Allocate IRQ. */ if (pcib_alloc_pcie_irq(sc) != 0) return; sc->pcie_link_sta = pcie_read_config(dev, PCIER_LINK_STA, 2); sc->pcie_slot_sta = pcie_read_config(dev, PCIER_SLOT_STA, 2); /* Enable HotPlug events. */ mask = PCIEM_SLOT_CTL_DLLSCE | PCIEM_SLOT_CTL_HPIE | PCIEM_SLOT_CTL_CCIE | PCIEM_SLOT_CTL_PDCE | PCIEM_SLOT_CTL_MRLSCE | PCIEM_SLOT_CTL_PFDE | PCIEM_SLOT_CTL_ABPE; val = PCIEM_SLOT_CTL_PDCE | PCIEM_SLOT_CTL_HPIE; if (sc->pcie_slot_cap & PCIEM_SLOT_CAP_APB) val |= PCIEM_SLOT_CTL_ABPE; if (sc->pcie_slot_cap & PCIEM_SLOT_CAP_PCP) val |= PCIEM_SLOT_CTL_PFDE; if (sc->pcie_slot_cap & PCIEM_SLOT_CAP_MRLSP) val |= PCIEM_SLOT_CTL_MRLSCE; if (!(sc->pcie_slot_cap & PCIEM_SLOT_CAP_NCCS)) val |= PCIEM_SLOT_CTL_CCIE; if (sc->pcie_link_cap & PCIEM_LINK_CAP_DL_ACTIVE) val |= PCIEM_SLOT_CTL_DLLSCE; /* Turn the attention indicator off. */ if (sc->pcie_slot_cap & PCIEM_SLOT_CAP_AIP) { mask |= PCIEM_SLOT_CTL_AIC; val |= PCIEM_SLOT_CTL_AI_OFF; } pcib_pcie_hotplug_update(sc, val, mask, false); } #endif /* * Get current bridge configuration. */ static void pcib_cfg_save(struct pcib_softc *sc) { #ifndef NEW_PCIB device_t dev; uint16_t command; dev = sc->dev; command = pci_read_config(dev, PCIR_COMMAND, 2); if (command & PCIM_CMD_PORTEN) pcib_get_io_decode(sc); if (command & PCIM_CMD_MEMEN) pcib_get_mem_decode(sc); #endif } /* * Restore previous bridge configuration. */ static void pcib_cfg_restore(struct pcib_softc *sc) { device_t dev; #ifndef NEW_PCIB uint16_t command; #endif dev = sc->dev; #ifdef NEW_PCIB pcib_write_windows(sc, WIN_IO | WIN_MEM | WIN_PMEM); #else command = pci_read_config(dev, PCIR_COMMAND, 2); if (command & PCIM_CMD_PORTEN) pcib_set_io_decode(sc); if (command & PCIM_CMD_MEMEN) pcib_set_mem_decode(sc); #endif } /* * Generic device interface */ static int pcib_probe(device_t dev) { if ((pci_get_class(dev) == PCIC_BRIDGE) && (pci_get_subclass(dev) == PCIS_BRIDGE_PCI)) { device_set_desc(dev, "PCI-PCI bridge"); return(-10000); } return(ENXIO); } void pcib_attach_common(device_t dev) { struct pcib_softc *sc; struct sysctl_ctx_list *sctx; struct sysctl_oid *soid; int comma; sc = device_get_softc(dev); sc->dev = dev; /* * Get current bridge configuration. */ sc->domain = pci_get_domain(dev); #if !(defined(NEW_PCIB) && defined(PCI_RES_BUS)) sc->bus.sec = pci_read_config(dev, PCIR_SECBUS_1, 1); sc->bus.sub = pci_read_config(dev, PCIR_SUBBUS_1, 1); #endif sc->bridgectl = pci_read_config(dev, PCIR_BRIDGECTL_1, 2); pcib_cfg_save(sc); /* * The primary bus register should always be the bus of the * parent. */ sc->pribus = pci_get_bus(dev); pci_write_config(dev, PCIR_PRIBUS_1, sc->pribus, 1); /* * Setup sysctl reporting nodes */ sctx = device_get_sysctl_ctx(dev); soid = device_get_sysctl_tree(dev); SYSCTL_ADD_UINT(sctx, SYSCTL_CHILDREN(soid), OID_AUTO, "domain", CTLFLAG_RD, &sc->domain, 0, "Domain number"); SYSCTL_ADD_UINT(sctx, SYSCTL_CHILDREN(soid), OID_AUTO, "pribus", CTLFLAG_RD, &sc->pribus, 0, "Primary bus number"); SYSCTL_ADD_UINT(sctx, SYSCTL_CHILDREN(soid), OID_AUTO, "secbus", CTLFLAG_RD, &sc->bus.sec, 0, "Secondary bus number"); SYSCTL_ADD_UINT(sctx, SYSCTL_CHILDREN(soid), OID_AUTO, "subbus", CTLFLAG_RD, &sc->bus.sub, 0, "Subordinate bus number"); /* * Quirk handling. */ switch (pci_get_devid(dev)) { #if !(defined(NEW_PCIB) && defined(PCI_RES_BUS)) case 0x12258086: /* Intel 82454KX/GX (Orion) */ { uint8_t supbus; supbus = pci_read_config(dev, 0x41, 1); if (supbus != 0xff) { sc->bus.sec = supbus + 1; sc->bus.sub = supbus + 1; } break; } #endif /* * The i82380FB mobile docking controller is a PCI-PCI bridge, * and it is a subtractive bridge. However, the ProgIf is wrong * so the normal setting of PCIB_SUBTRACTIVE bit doesn't * happen. There are also Toshiba and Cavium ThunderX bridges * that behave this way. */ case 0xa002177d: /* Cavium ThunderX */ case 0x124b8086: /* Intel 82380FB Mobile */ case 0x060513d7: /* Toshiba ???? */ sc->flags |= PCIB_SUBTRACTIVE; break; #if !(defined(NEW_PCIB) && defined(PCI_RES_BUS)) /* Compaq R3000 BIOS sets wrong subordinate bus number. */ case 0x00dd10de: { char *cp; if ((cp = kern_getenv("smbios.planar.maker")) == NULL) break; if (strncmp(cp, "Compal", 6) != 0) { freeenv(cp); break; } freeenv(cp); if ((cp = kern_getenv("smbios.planar.product")) == NULL) break; if (strncmp(cp, "08A0", 4) != 0) { freeenv(cp); break; } freeenv(cp); if (sc->bus.sub < 0xa) { pci_write_config(dev, PCIR_SUBBUS_1, 0xa, 1); sc->bus.sub = pci_read_config(dev, PCIR_SUBBUS_1, 1); } break; } #endif } if (pci_msi_device_blacklisted(dev)) sc->flags |= PCIB_DISABLE_MSI; if (pci_msix_device_blacklisted(dev)) sc->flags |= PCIB_DISABLE_MSIX; /* * Intel 815, 845 and other chipsets say they are PCI-PCI bridges, * but have a ProgIF of 0x80. The 82801 family (AA, AB, BAM/CAM, * BA/CA/DB and E) PCI bridges are HUB-PCI bridges, in Intelese. * This means they act as if they were subtractively decoding * bridges and pass all transactions. Mark them and real ProgIf 1 * parts as subtractive. */ if ((pci_get_devid(dev) & 0xff00ffff) == 0x24008086 || pci_read_config(dev, PCIR_PROGIF, 1) == PCIP_BRIDGE_PCI_SUBTRACTIVE) sc->flags |= PCIB_SUBTRACTIVE; #ifdef PCI_HP pcib_probe_hotplug(sc); #endif #ifdef NEW_PCIB #ifdef PCI_RES_BUS pcib_setup_secbus(dev, &sc->bus, 1); #endif pcib_probe_windows(sc); #endif #ifdef PCI_HP if (sc->flags & PCIB_HOTPLUG) pcib_setup_hotplug(sc); #endif if (bootverbose) { device_printf(dev, " domain %d\n", sc->domain); device_printf(dev, " secondary bus %d\n", sc->bus.sec); device_printf(dev, " subordinate bus %d\n", sc->bus.sub); #ifdef NEW_PCIB if (pcib_is_window_open(&sc->io)) device_printf(dev, " I/O decode 0x%jx-0x%jx\n", (uintmax_t)sc->io.base, (uintmax_t)sc->io.limit); if (pcib_is_window_open(&sc->mem)) device_printf(dev, " memory decode 0x%jx-0x%jx\n", (uintmax_t)sc->mem.base, (uintmax_t)sc->mem.limit); if (pcib_is_window_open(&sc->pmem)) device_printf(dev, " prefetched decode 0x%jx-0x%jx\n", (uintmax_t)sc->pmem.base, (uintmax_t)sc->pmem.limit); #else if (pcib_is_io_open(sc)) device_printf(dev, " I/O decode 0x%x-0x%x\n", sc->iobase, sc->iolimit); if (pcib_is_nonprefetch_open(sc)) device_printf(dev, " memory decode 0x%jx-0x%jx\n", (uintmax_t)sc->membase, (uintmax_t)sc->memlimit); if (pcib_is_prefetch_open(sc)) device_printf(dev, " prefetched decode 0x%jx-0x%jx\n", (uintmax_t)sc->pmembase, (uintmax_t)sc->pmemlimit); #endif if (sc->bridgectl & (PCIB_BCR_ISA_ENABLE | PCIB_BCR_VGA_ENABLE) || sc->flags & PCIB_SUBTRACTIVE) { device_printf(dev, " special decode "); comma = 0; if (sc->bridgectl & PCIB_BCR_ISA_ENABLE) { printf("ISA"); comma = 1; } if (sc->bridgectl & PCIB_BCR_VGA_ENABLE) { printf("%sVGA", comma ? ", " : ""); comma = 1; } if (sc->flags & PCIB_SUBTRACTIVE) printf("%ssubtractive", comma ? ", " : ""); printf("\n"); } } /* * Always enable busmastering on bridges so that transactions * initiated on the secondary bus are passed through to the * primary bus. */ pci_enable_busmaster(dev); } #ifdef PCI_HP static int pcib_present(struct pcib_softc *sc) { if (sc->flags & PCIB_HOTPLUG) return (pcib_hotplug_present(sc) != 0); return (1); } #endif int pcib_attach_child(device_t dev) { struct pcib_softc *sc; sc = device_get_softc(dev); if (sc->bus.sec == 0) { /* no secondary bus; we should have fixed this */ return(0); } #ifdef PCI_HP if (!pcib_present(sc)) { /* An empty HotPlug slot, so don't add a PCI bus yet. */ return (0); } #endif sc->child = device_add_child(dev, "pci", -1); return (bus_generic_attach(dev)); } int pcib_attach(device_t dev) { pcib_attach_common(dev); return (pcib_attach_child(dev)); } int pcib_suspend(device_t dev) { pcib_cfg_save(device_get_softc(dev)); return (bus_generic_suspend(dev)); } int pcib_resume(device_t dev) { pcib_cfg_restore(device_get_softc(dev)); return (bus_generic_resume(dev)); } void pcib_bridge_init(device_t dev) { pci_write_config(dev, PCIR_IOBASEL_1, 0xff, 1); pci_write_config(dev, PCIR_IOBASEH_1, 0xffff, 2); pci_write_config(dev, PCIR_IOLIMITL_1, 0, 1); pci_write_config(dev, PCIR_IOLIMITH_1, 0, 2); pci_write_config(dev, PCIR_MEMBASE_1, 0xffff, 2); pci_write_config(dev, PCIR_MEMLIMIT_1, 0, 2); pci_write_config(dev, PCIR_PMBASEL_1, 0xffff, 2); pci_write_config(dev, PCIR_PMBASEH_1, 0xffffffff, 4); pci_write_config(dev, PCIR_PMLIMITL_1, 0, 2); pci_write_config(dev, PCIR_PMLIMITH_1, 0, 4); } int pcib_child_present(device_t dev, device_t child) { #ifdef PCI_HP struct pcib_softc *sc = device_get_softc(dev); int retval; retval = bus_child_present(dev); if (retval != 0 && sc->flags & PCIB_HOTPLUG) retval = pcib_hotplug_present(sc); return (retval); #else return (bus_child_present(dev)); #endif } int pcib_read_ivar(device_t dev, device_t child, int which, uintptr_t *result) { struct pcib_softc *sc = device_get_softc(dev); switch (which) { case PCIB_IVAR_DOMAIN: *result = sc->domain; return(0); case PCIB_IVAR_BUS: *result = sc->bus.sec; return(0); } return(ENOENT); } int pcib_write_ivar(device_t dev, device_t child, int which, uintptr_t value) { switch (which) { case PCIB_IVAR_DOMAIN: return(EINVAL); case PCIB_IVAR_BUS: return(EINVAL); } return(ENOENT); } #ifdef NEW_PCIB /* * Attempt to allocate a resource from the existing resources assigned * to a window. */ static struct resource * pcib_suballoc_resource(struct pcib_softc *sc, struct pcib_window *w, device_t child, int type, int *rid, rman_res_t start, rman_res_t end, rman_res_t count, u_int flags) { struct resource *res; if (!pcib_is_window_open(w)) return (NULL); res = rman_reserve_resource(&w->rman, start, end, count, flags & ~RF_ACTIVE, child); if (res == NULL) return (NULL); if (bootverbose) device_printf(sc->dev, "allocated %s range (%#jx-%#jx) for rid %x of %s\n", w->name, rman_get_start(res), rman_get_end(res), *rid, pcib_child_name(child)); rman_set_rid(res, *rid); /* * If the resource should be active, pass that request up the * tree. This assumes the parent drivers can handle * activating sub-allocated resources. */ if (flags & RF_ACTIVE) { if (bus_activate_resource(child, type, *rid, res) != 0) { rman_release_resource(res); return (NULL); } } return (res); } /* Allocate a fresh resource range for an unconfigured window. */ static int pcib_alloc_new_window(struct pcib_softc *sc, struct pcib_window *w, int type, rman_res_t start, rman_res_t end, rman_res_t count, u_int flags) { struct resource *res; rman_res_t base, limit, wmask; int rid; /* * If this is an I/O window on a bridge with ISA enable set * and the start address is below 64k, then try to allocate an * initial window of 0x1000 bytes long starting at address * 0xf000 and walking down. Note that if the original request * was larger than the non-aliased range size of 0x100 our * caller would have raised the start address up to 64k * already. */ if (type == SYS_RES_IOPORT && sc->bridgectl & PCIB_BCR_ISA_ENABLE && start < 65536) { for (base = 0xf000; (long)base >= 0; base -= 0x1000) { limit = base + 0xfff; /* * Skip ranges that wouldn't work for the * original request. Note that the actual * window that overlaps are the non-alias * ranges within [base, limit], so this isn't * quite a simple comparison. */ if (start + count > limit - 0x400) continue; if (base == 0) { /* * The first open region for the window at * 0 is 0x400-0x4ff. */ if (end - count + 1 < 0x400) continue; } else { if (end - count + 1 < base) continue; } if (pcib_alloc_nonisa_ranges(sc, base, limit) == 0) { w->base = base; w->limit = limit; return (0); } } return (ENOSPC); } wmask = ((rman_res_t)1 << w->step) - 1; if (RF_ALIGNMENT(flags) < w->step) { flags &= ~RF_ALIGNMENT_MASK; flags |= RF_ALIGNMENT_LOG2(w->step); } start &= ~wmask; end |= wmask; count = roundup2(count, (rman_res_t)1 << w->step); rid = w->reg; res = bus_alloc_resource(sc->dev, type, &rid, start, end, count, flags & ~RF_ACTIVE); if (res == NULL) return (ENOSPC); pcib_add_window_resources(w, &res, 1); pcib_activate_window(sc, type); w->base = rman_get_start(res); w->limit = rman_get_end(res); return (0); } /* Try to expand an existing window to the requested base and limit. */ static int pcib_expand_window(struct pcib_softc *sc, struct pcib_window *w, int type, rman_res_t base, rman_res_t limit) { struct resource *res; int error, i, force_64k_base; KASSERT(base <= w->base && limit >= w->limit, ("attempting to shrink window")); /* * XXX: pcib_grow_window() doesn't try to do this anyway and * the error handling for all the edge cases would be tedious. */ KASSERT(limit == w->limit || base == w->base, ("attempting to grow both ends of a window")); /* * Yet more special handling for requests to expand an I/O * window behind an ISA-enabled bridge. Since I/O windows * have to grow in 0x1000 increments and the end of the 0xffff * range is an alias, growing a window below 64k will always * result in allocating new resources and never adjusting an * existing resource. */ if (type == SYS_RES_IOPORT && sc->bridgectl & PCIB_BCR_ISA_ENABLE && (limit <= 65535 || (base <= 65535 && base != w->base))) { KASSERT(limit == w->limit || limit <= 65535, ("attempting to grow both ends across 64k ISA alias")); if (base != w->base) error = pcib_alloc_nonisa_ranges(sc, base, w->base - 1); else error = pcib_alloc_nonisa_ranges(sc, w->limit + 1, limit); if (error == 0) { w->base = base; w->limit = limit; } return (error); } /* * Find the existing resource to adjust. Usually there is only one, * but for an ISA-enabled bridge we might be growing the I/O window * above 64k and need to find the existing resource that maps all * of the area above 64k. */ for (i = 0; i < w->count; i++) { if (rman_get_end(w->res[i]) == w->limit) break; } KASSERT(i != w->count, ("did not find existing resource")); res = w->res[i]; /* * Usually the resource we found should match the window's * existing range. The one exception is the ISA-enabled case * mentioned above in which case the resource should start at * 64k. */ if (type == SYS_RES_IOPORT && sc->bridgectl & PCIB_BCR_ISA_ENABLE && w->base <= 65535) { KASSERT(rman_get_start(res) == 65536, ("existing resource mismatch")); force_64k_base = 1; } else { KASSERT(w->base == rman_get_start(res), ("existing resource mismatch")); force_64k_base = 0; } error = bus_adjust_resource(sc->dev, type, res, force_64k_base ? rman_get_start(res) : base, limit); if (error) return (error); /* Add the newly allocated region to the resource manager. */ if (w->base != base) { error = rman_manage_region(&w->rman, base, w->base - 1); w->base = base; } else { error = rman_manage_region(&w->rman, w->limit + 1, limit); w->limit = limit; } if (error) { if (bootverbose) device_printf(sc->dev, "failed to expand %s resource manager\n", w->name); (void)bus_adjust_resource(sc->dev, type, res, force_64k_base ? rman_get_start(res) : w->base, w->limit); } return (error); } /* * Attempt to grow a window to make room for a given resource request. */ static int pcib_grow_window(struct pcib_softc *sc, struct pcib_window *w, int type, rman_res_t start, rman_res_t end, rman_res_t count, u_int flags) { rman_res_t align, start_free, end_free, front, back, wmask; int error; /* * Clamp the desired resource range to the maximum address * this window supports. Reject impossible requests. * * For I/O port requests behind a bridge with the ISA enable * bit set, force large allocations to start above 64k. */ if (!w->valid) return (EINVAL); if (sc->bridgectl & PCIB_BCR_ISA_ENABLE && count > 0x100 && start < 65536) start = 65536; if (end > w->rman.rm_end) end = w->rman.rm_end; if (start + count - 1 > end || start + count < start) return (EINVAL); wmask = ((rman_res_t)1 << w->step) - 1; /* * If there is no resource at all, just try to allocate enough * aligned space for this resource. */ if (w->res == NULL) { error = pcib_alloc_new_window(sc, w, type, start, end, count, flags); if (error) { if (bootverbose) device_printf(sc->dev, "failed to allocate initial %s window (%#jx-%#jx,%#jx)\n", w->name, start, end, count); return (error); } if (bootverbose) device_printf(sc->dev, "allocated initial %s window of %#jx-%#jx\n", w->name, (uintmax_t)w->base, (uintmax_t)w->limit); goto updatewin; } /* * See if growing the window would help. Compute the minimum * amount of address space needed on both the front and back * ends of the existing window to satisfy the allocation. * * For each end, build a candidate region adjusting for the * required alignment, etc. If there is a free region at the * edge of the window, grow from the inner edge of the free * region. Otherwise grow from the window boundary. * * Growing an I/O window below 64k for a bridge with the ISA * enable bit doesn't require any special magic as the step * size of an I/O window (1k) always includes multiple * non-alias ranges when it is grown in either direction. * * XXX: Special case: if w->res is completely empty and the * request size is larger than w->res, we should find the * optimal aligned buffer containing w->res and allocate that. */ if (bootverbose) device_printf(sc->dev, "attempting to grow %s window for (%#jx-%#jx,%#jx)\n", w->name, start, end, count); align = (rman_res_t)1 << RF_ALIGNMENT(flags); if (start < w->base) { if (rman_first_free_region(&w->rman, &start_free, &end_free) != 0 || start_free != w->base) end_free = w->base; if (end_free > end) end_free = end + 1; /* Move end_free down until it is properly aligned. */ end_free &= ~(align - 1); end_free--; front = end_free - (count - 1); /* * The resource would now be allocated at (front, * end_free). Ensure that fits in the (start, end) * bounds. end_free is checked above. If 'front' is * ok, ensure it is properly aligned for this window. * Also check for underflow. */ if (front >= start && front <= end_free) { if (bootverbose) printf("\tfront candidate range: %#jx-%#jx\n", front, end_free); front &= ~wmask; front = w->base - front; } else front = 0; } else front = 0; if (end > w->limit) { if (rman_last_free_region(&w->rman, &start_free, &end_free) != 0 || end_free != w->limit) start_free = w->limit + 1; if (start_free < start) start_free = start; /* Move start_free up until it is properly aligned. */ start_free = roundup2(start_free, align); back = start_free + count - 1; /* * The resource would now be allocated at (start_free, * back). Ensure that fits in the (start, end) * bounds. start_free is checked above. If 'back' is * ok, ensure it is properly aligned for this window. * Also check for overflow. */ if (back <= end && start_free <= back) { if (bootverbose) printf("\tback candidate range: %#jx-%#jx\n", start_free, back); back |= wmask; back -= w->limit; } else back = 0; } else back = 0; /* * Try to allocate the smallest needed region first. * If that fails, fall back to the other region. */ error = ENOSPC; while (front != 0 || back != 0) { if (front != 0 && (front <= back || back == 0)) { error = pcib_expand_window(sc, w, type, w->base - front, w->limit); if (error == 0) break; front = 0; } else { error = pcib_expand_window(sc, w, type, w->base, w->limit + back); if (error == 0) break; back = 0; } } if (error) return (error); if (bootverbose) device_printf(sc->dev, "grew %s window to %#jx-%#jx\n", w->name, (uintmax_t)w->base, (uintmax_t)w->limit); updatewin: /* Write the new window. */ KASSERT((w->base & wmask) == 0, ("start address is not aligned")); KASSERT((w->limit & wmask) == wmask, ("end address is not aligned")); pcib_write_windows(sc, w->mask); return (0); } /* * We have to trap resource allocation requests and ensure that the bridge * is set up to, or capable of handling them. */ struct resource * pcib_alloc_resource(device_t dev, device_t child, int type, int *rid, rman_res_t start, rman_res_t end, rman_res_t count, u_int flags) { struct pcib_softc *sc; struct resource *r; sc = device_get_softc(dev); /* * VGA resources are decoded iff the VGA enable bit is set in * the bridge control register. VGA resources do not fall into * the resource windows and are passed up to the parent. */ if ((type == SYS_RES_IOPORT && pci_is_vga_ioport_range(start, end)) || (type == SYS_RES_MEMORY && pci_is_vga_memory_range(start, end))) { if (sc->bridgectl & PCIB_BCR_VGA_ENABLE) return (bus_generic_alloc_resource(dev, child, type, rid, start, end, count, flags)); else return (NULL); } switch (type) { #ifdef PCI_RES_BUS case PCI_RES_BUS: return (pcib_alloc_subbus(&sc->bus, child, rid, start, end, count, flags)); #endif case SYS_RES_IOPORT: if (pcib_is_isa_range(sc, start, end, count)) return (NULL); r = pcib_suballoc_resource(sc, &sc->io, child, type, rid, start, end, count, flags); if (r != NULL || (sc->flags & PCIB_SUBTRACTIVE) != 0) break; if (pcib_grow_window(sc, &sc->io, type, start, end, count, flags) == 0) r = pcib_suballoc_resource(sc, &sc->io, child, type, rid, start, end, count, flags); break; case SYS_RES_MEMORY: /* * For prefetchable resources, prefer the prefetchable * memory window, but fall back to the regular memory * window if that fails. Try both windows before * attempting to grow a window in case the firmware * has used a range in the regular memory window to * map a prefetchable BAR. */ if (flags & RF_PREFETCHABLE) { r = pcib_suballoc_resource(sc, &sc->pmem, child, type, rid, start, end, count, flags); if (r != NULL) break; } r = pcib_suballoc_resource(sc, &sc->mem, child, type, rid, start, end, count, flags); if (r != NULL || (sc->flags & PCIB_SUBTRACTIVE) != 0) break; if (flags & RF_PREFETCHABLE) { if (pcib_grow_window(sc, &sc->pmem, type, start, end, count, flags) == 0) { r = pcib_suballoc_resource(sc, &sc->pmem, child, type, rid, start, end, count, flags); if (r != NULL) break; } } if (pcib_grow_window(sc, &sc->mem, type, start, end, count, flags & ~RF_PREFETCHABLE) == 0) r = pcib_suballoc_resource(sc, &sc->mem, child, type, rid, start, end, count, flags); break; default: return (bus_generic_alloc_resource(dev, child, type, rid, start, end, count, flags)); } /* * If attempts to suballocate from the window fail but this is a * subtractive bridge, pass the request up the tree. */ if (sc->flags & PCIB_SUBTRACTIVE && r == NULL) return (bus_generic_alloc_resource(dev, child, type, rid, start, end, count, flags)); return (r); } int pcib_adjust_resource(device_t bus, device_t child, int type, struct resource *r, rman_res_t start, rman_res_t end) { struct pcib_softc *sc; sc = device_get_softc(bus); if (pcib_is_resource_managed(sc, type, r)) return (rman_adjust_resource(r, start, end)); return (bus_generic_adjust_resource(bus, child, type, r, start, end)); } int pcib_release_resource(device_t dev, device_t child, int type, int rid, struct resource *r) { struct pcib_softc *sc; int error; sc = device_get_softc(dev); if (pcib_is_resource_managed(sc, type, r)) { if (rman_get_flags(r) & RF_ACTIVE) { error = bus_deactivate_resource(child, type, rid, r); if (error) return (error); } return (rman_release_resource(r)); } return (bus_generic_release_resource(dev, child, type, rid, r)); } #else /* * We have to trap resource allocation requests and ensure that the bridge * is set up to, or capable of handling them. */ struct resource * pcib_alloc_resource(device_t dev, device_t child, int type, int *rid, rman_res_t start, rman_res_t end, rman_res_t count, u_int flags) { struct pcib_softc *sc = device_get_softc(dev); const char *name, *suffix; int ok; /* * Fail the allocation for this range if it's not supported. */ name = device_get_nameunit(child); if (name == NULL) { name = ""; suffix = ""; } else suffix = " "; switch (type) { case SYS_RES_IOPORT: ok = 0; if (!pcib_is_io_open(sc)) break; ok = (start >= sc->iobase && end <= sc->iolimit); /* * Make sure we allow access to VGA I/O addresses when the * bridge has the "VGA Enable" bit set. */ if (!ok && pci_is_vga_ioport_range(start, end)) ok = (sc->bridgectl & PCIB_BCR_VGA_ENABLE) ? 1 : 0; if ((sc->flags & PCIB_SUBTRACTIVE) == 0) { if (!ok) { if (start < sc->iobase) start = sc->iobase; if (end > sc->iolimit) end = sc->iolimit; if (start < end) ok = 1; } } else { ok = 1; #if 0 /* * If we overlap with the subtractive range, then * pick the upper range to use. */ if (start < sc->iolimit && end > sc->iobase) start = sc->iolimit + 1; #endif } if (end < start) { device_printf(dev, "ioport: end (%jx) < start (%jx)\n", end, start); start = 0; end = 0; ok = 0; } if (!ok) { device_printf(dev, "%s%srequested unsupported I/O " "range 0x%jx-0x%jx (decoding 0x%x-0x%x)\n", name, suffix, start, end, sc->iobase, sc->iolimit); return (NULL); } if (bootverbose) device_printf(dev, "%s%srequested I/O range 0x%jx-0x%jx: in range\n", name, suffix, start, end); break; case SYS_RES_MEMORY: ok = 0; if (pcib_is_nonprefetch_open(sc)) ok = ok || (start >= sc->membase && end <= sc->memlimit); if (pcib_is_prefetch_open(sc)) ok = ok || (start >= sc->pmembase && end <= sc->pmemlimit); /* * Make sure we allow access to VGA memory addresses when the * bridge has the "VGA Enable" bit set. */ if (!ok && pci_is_vga_memory_range(start, end)) ok = (sc->bridgectl & PCIB_BCR_VGA_ENABLE) ? 1 : 0; if ((sc->flags & PCIB_SUBTRACTIVE) == 0) { if (!ok) { ok = 1; if (flags & RF_PREFETCHABLE) { if (pcib_is_prefetch_open(sc)) { if (start < sc->pmembase) start = sc->pmembase; if (end > sc->pmemlimit) end = sc->pmemlimit; } else { ok = 0; } } else { /* non-prefetchable */ if (pcib_is_nonprefetch_open(sc)) { if (start < sc->membase) start = sc->membase; if (end > sc->memlimit) end = sc->memlimit; } else { ok = 0; } } } } else if (!ok) { ok = 1; /* subtractive bridge: always ok */ #if 0 if (pcib_is_nonprefetch_open(sc)) { if (start < sc->memlimit && end > sc->membase) start = sc->memlimit + 1; } if (pcib_is_prefetch_open(sc)) { if (start < sc->pmemlimit && end > sc->pmembase) start = sc->pmemlimit + 1; } #endif } if (end < start) { device_printf(dev, "memory: end (%jx) < start (%jx)\n", end, start); start = 0; end = 0; ok = 0; } if (!ok && bootverbose) device_printf(dev, "%s%srequested unsupported memory range %#jx-%#jx " "(decoding %#jx-%#jx, %#jx-%#jx)\n", name, suffix, start, end, (uintmax_t)sc->membase, (uintmax_t)sc->memlimit, (uintmax_t)sc->pmembase, (uintmax_t)sc->pmemlimit); if (!ok) return (NULL); if (bootverbose) device_printf(dev,"%s%srequested memory range " "0x%jx-0x%jx: good\n", name, suffix, start, end); break; default: break; } /* * Bridge is OK decoding this resource, so pass it up. */ return (bus_generic_alloc_resource(dev, child, type, rid, start, end, count, flags)); } #endif /* * If ARI is enabled on this downstream port, translate the function number * to the non-ARI slot/function. The downstream port will convert it back in * hardware. If ARI is not enabled slot and func are not modified. */ static __inline void pcib_xlate_ari(device_t pcib, int bus, int *slot, int *func) { struct pcib_softc *sc; int ari_func; sc = device_get_softc(pcib); ari_func = *func; if (sc->flags & PCIB_ENABLE_ARI) { KASSERT(*slot == 0, ("Non-zero slot number with ARI enabled!")); *slot = PCIE_ARI_SLOT(ari_func); *func = PCIE_ARI_FUNC(ari_func); } } static void pcib_enable_ari(struct pcib_softc *sc, uint32_t pcie_pos) { uint32_t ctl2; ctl2 = pci_read_config(sc->dev, pcie_pos + PCIER_DEVICE_CTL2, 4); ctl2 |= PCIEM_CTL2_ARI; pci_write_config(sc->dev, pcie_pos + PCIER_DEVICE_CTL2, ctl2, 4); sc->flags |= PCIB_ENABLE_ARI; } /* * PCIB interface. */ int pcib_maxslots(device_t dev) { return (PCI_SLOTMAX); } static int pcib_ari_maxslots(device_t dev) { struct pcib_softc *sc; sc = device_get_softc(dev); if (sc->flags & PCIB_ENABLE_ARI) return (PCIE_ARI_SLOTMAX); else return (PCI_SLOTMAX); } static int pcib_ari_maxfuncs(device_t dev) { struct pcib_softc *sc; sc = device_get_softc(dev); if (sc->flags & PCIB_ENABLE_ARI) return (PCIE_ARI_FUNCMAX); else return (PCI_FUNCMAX); } static void pcib_ari_decode_rid(device_t pcib, uint16_t rid, int *bus, int *slot, int *func) { struct pcib_softc *sc; sc = device_get_softc(pcib); *bus = PCI_RID2BUS(rid); if (sc->flags & PCIB_ENABLE_ARI) { *slot = PCIE_ARI_RID2SLOT(rid); *func = PCIE_ARI_RID2FUNC(rid); } else { *slot = PCI_RID2SLOT(rid); *func = PCI_RID2FUNC(rid); } } /* * Since we are a child of a PCI bus, its parent must support the pcib interface. */ static uint32_t pcib_read_config(device_t dev, u_int b, u_int s, u_int f, u_int reg, int width) { #ifdef PCI_HP struct pcib_softc *sc; sc = device_get_softc(dev); if (!pcib_present(sc)) { switch (width) { case 2: return (0xffff); case 1: return (0xff); default: return (0xffffffff); } } #endif pcib_xlate_ari(dev, b, &s, &f); return(PCIB_READ_CONFIG(device_get_parent(device_get_parent(dev)), b, s, f, reg, width)); } static void pcib_write_config(device_t dev, u_int b, u_int s, u_int f, u_int reg, uint32_t val, int width) { #ifdef PCI_HP struct pcib_softc *sc; sc = device_get_softc(dev); if (!pcib_present(sc)) return; #endif pcib_xlate_ari(dev, b, &s, &f); PCIB_WRITE_CONFIG(device_get_parent(device_get_parent(dev)), b, s, f, reg, val, width); } /* * Route an interrupt across a PCI bridge. */ int pcib_route_interrupt(device_t pcib, device_t dev, int pin) { device_t bus; int parent_intpin; int intnum; /* * * The PCI standard defines a swizzle of the child-side device/intpin to * the parent-side intpin as follows. * * device = device on child bus * child_intpin = intpin on child bus slot (0-3) * parent_intpin = intpin on parent bus slot (0-3) * * parent_intpin = (device + child_intpin) % 4 */ parent_intpin = (pci_get_slot(dev) + (pin - 1)) % 4; /* * Our parent is a PCI bus. Its parent must export the pcib interface * which includes the ability to route interrupts. */ bus = device_get_parent(pcib); intnum = PCIB_ROUTE_INTERRUPT(device_get_parent(bus), pcib, parent_intpin + 1); if (PCI_INTERRUPT_VALID(intnum) && bootverbose) { device_printf(pcib, "slot %d INT%c is routed to irq %d\n", pci_get_slot(dev), 'A' + pin - 1, intnum); } return(intnum); } /* Pass request to alloc MSI/MSI-X messages up to the parent bridge. */ int pcib_alloc_msi(device_t pcib, device_t dev, int count, int maxcount, int *irqs) { struct pcib_softc *sc = device_get_softc(pcib); device_t bus; if (sc->flags & PCIB_DISABLE_MSI) return (ENXIO); bus = device_get_parent(pcib); return (PCIB_ALLOC_MSI(device_get_parent(bus), dev, count, maxcount, irqs)); } /* Pass request to release MSI/MSI-X messages up to the parent bridge. */ int pcib_release_msi(device_t pcib, device_t dev, int count, int *irqs) { device_t bus; bus = device_get_parent(pcib); return (PCIB_RELEASE_MSI(device_get_parent(bus), dev, count, irqs)); } /* Pass request to alloc an MSI-X message up to the parent bridge. */ int pcib_alloc_msix(device_t pcib, device_t dev, int *irq) { struct pcib_softc *sc = device_get_softc(pcib); device_t bus; if (sc->flags & PCIB_DISABLE_MSIX) return (ENXIO); bus = device_get_parent(pcib); return (PCIB_ALLOC_MSIX(device_get_parent(bus), dev, irq)); } /* Pass request to release an MSI-X message up to the parent bridge. */ int pcib_release_msix(device_t pcib, device_t dev, int irq) { device_t bus; bus = device_get_parent(pcib); return (PCIB_RELEASE_MSIX(device_get_parent(bus), dev, irq)); } /* Pass request to map MSI/MSI-X message up to parent bridge. */ int pcib_map_msi(device_t pcib, device_t dev, int irq, uint64_t *addr, uint32_t *data) { device_t bus; int error; bus = device_get_parent(pcib); error = PCIB_MAP_MSI(device_get_parent(bus), dev, irq, addr, data); if (error) return (error); pci_ht_map_msi(pcib, *addr); return (0); } /* Pass request for device power state up to parent bridge. */ int pcib_power_for_sleep(device_t pcib, device_t dev, int *pstate) { device_t bus; bus = device_get_parent(pcib); return (PCIB_POWER_FOR_SLEEP(bus, dev, pstate)); } static int pcib_ari_enabled(device_t pcib) { struct pcib_softc *sc; sc = device_get_softc(pcib); return ((sc->flags & PCIB_ENABLE_ARI) != 0); } static int pcib_ari_get_id(device_t pcib, device_t dev, enum pci_id_type type, uintptr_t *id) { struct pcib_softc *sc; + device_t bus_dev; uint8_t bus, slot, func; - if (type != PCI_ID_RID) - return (ENXIO); + if (type != PCI_ID_RID) { + bus_dev = device_get_parent(pcib); + return (PCIB_GET_ID(device_get_parent(bus_dev), dev, type, id)); + } sc = device_get_softc(pcib); if (sc->flags & PCIB_ENABLE_ARI) { bus = pci_get_bus(dev); func = pci_get_function(dev); *id = (PCI_ARI_RID(bus, func)); } else { bus = pci_get_bus(dev); slot = pci_get_slot(dev); func = pci_get_function(dev); *id = (PCI_RID(bus, slot, func)); } return (0); } /* * Check that the downstream port (pcib) and the endpoint device (dev) both * support ARI. If so, enable it and return 0, otherwise return an error. */ static int pcib_try_enable_ari(device_t pcib, device_t dev) { struct pcib_softc *sc; int error; uint32_t cap2; int ari_cap_off; uint32_t ari_ver; uint32_t pcie_pos; sc = device_get_softc(pcib); /* * ARI is controlled in a register in the PCIe capability structure. * If the downstream port does not have the PCIe capability structure * then it does not support ARI. */ error = pci_find_cap(pcib, PCIY_EXPRESS, &pcie_pos); if (error != 0) return (ENODEV); /* Check that the PCIe port advertises ARI support. */ cap2 = pci_read_config(pcib, pcie_pos + PCIER_DEVICE_CAP2, 4); if (!(cap2 & PCIEM_CAP2_ARI)) return (ENODEV); /* * Check that the endpoint device advertises ARI support via the ARI * extended capability structure. */ error = pci_find_extcap(dev, PCIZ_ARI, &ari_cap_off); if (error != 0) return (ENODEV); /* * Finally, check that the endpoint device supports the same version * of ARI that we do. */ ari_ver = pci_read_config(dev, ari_cap_off, 4); if (PCI_EXTCAP_VER(ari_ver) != PCIB_SUPPORTED_ARI_VER) { if (bootverbose) device_printf(pcib, "Unsupported version of ARI (%d) detected\n", PCI_EXTCAP_VER(ari_ver)); return (ENXIO); } pcib_enable_ari(sc, pcie_pos); return (0); }