diff --git a/share/man/man9/bus_alloc_resource.9 b/share/man/man9/bus_alloc_resource.9 index 210abb3074e1..d69917d17ffd 100644 --- a/share/man/man9/bus_alloc_resource.9 +++ b/share/man/man9/bus_alloc_resource.9 @@ -1,191 +1,187 @@ .\" -*- nroff -*- .\" .\" Copyright (c) 2000 Alexander Langer .\" .\" All rights reserved. .\" .\" This program is free software. .\" .\" Redistribution and use in source and binary forms, with or without .\" modification, are permitted provided that the following conditions .\" are met: .\" 1. Redistributions of source code must retain the above copyright .\" notice, this list of conditions and the following disclaimer. .\" 2. Redistributions in binary form must reproduce the above copyright .\" notice, this list of conditions and the following disclaimer in the .\" documentation and/or other materials provided with the distribution. .\" .\" THIS SOFTWARE IS PROVIDED BY THE DEVELOPERS ``AS IS'' AND ANY EXPRESS OR .\" IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES .\" OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. .\" IN NO EVENT SHALL THE DEVELOPERS BE LIABLE FOR ANY DIRECT, INDIRECT, .\" INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT .\" NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, .\" DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY .\" THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT .\" (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF .\" THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. .\" -.Dd October 30, 2025 +.Dd March 6, 2026 .Dt BUS_ALLOC_RESOURCE 9 .Os .Sh NAME .Nm bus_alloc_resource , .Nm bus_alloc_resource_any , .Nm bus_alloc_resource_anywhere .Nd allocate resources from a parent bus .Sh SYNOPSIS .In sys/param.h .In sys/bus.h .Pp .In machine/bus.h .In sys/rman.h .In machine/resource.h .Ft struct resource * .Fo bus_alloc_resource .Fa "device_t dev" "int type" "int rid" "rman_res_t start" "rman_res_t end" .Fa "rman_res_t count" "u_int flags" .Fc .Ft struct resource * .Fn bus_alloc_resource_any "device_t dev" "int type" "int rid" "u_int flags" .Ft struct resource * .Fo bus_alloc_resource_anywhere .Fa "device_t dev" "int type" "int rid" "rman_res_t count" "u_int flags" .Fc .Sh DESCRIPTION This is an easy interface to the resource-management functions. It hides the indirection through the parent's method table. This function generally should be called in attach, but (except in some rare cases) never earlier. .Pp The .Fn bus_alloc_resource_any and .Fn bus_alloc_resource_anywhere functions are convenience wrappers for .Fn bus_alloc_resource . .Fn bus_alloc_resource_any sets .Fa start , .Fa end , and .Fa count to the default resource (see description of .Fa start below). .Fn bus_alloc_resource_anywhere sets .Fa start and .Fa end to the default resource and uses the provided .Fa count argument. .Pp The arguments are as follows: .Bl -item .It .Fa dev is the device that requests ownership of the resource. Before allocation, the resource is owned by the parent bus. .It .Fa type is the type of resource you want to allocate. It is one of: .Bl -tag -width SYS_RES_MEMORY .It Dv PCI_RES_BUS for PCI bus numbers .It Dv SYS_RES_IRQ for IRQs .It Dv SYS_RES_DRQ for ISA DMA lines .It Dv SYS_RES_IOPORT for I/O ports .It Dv SYS_RES_MEMORY for I/O memory .El .It .Fa rid is a bus specific handle that identifies the resource being allocated. For ISA this is an index into an array of resources that have been setup for this device by either the PnP mechanism, or via the hints mechanism. For PCCARD, this is an index into the array of resources described by the PC Card's CIS entry. For PCI, the offset into PCI config space which has the BAR to use to access the resource. .It .Fa start and .Fa end are the start/end addresses of the resource. If you specify values of 0ul for .Fa start and ~0ul for .Fa end and 1 for .Fa count , the default values for the bus are calculated. .It .Fa count is the size of the resource. For example, the size of an I/O port is usually 1 byte (but some devices override this). If you specified the default values for .Fa start and .Fa end , then the default value of the bus is used if .Fa count is smaller than the default value and .Fa count is used, if it is bigger than the default value. .It .Fa flags sets the flags for the resource. -You can set one or more of these flags: -.Bl -tag -width RF_SHAREABLE -.It Dv RF_ALLOCATED -resource has been reserved. -The resource still needs to be activated with -.Xr bus_activate_resource 9 . +You can set zero or more of these flags: +.Bl -tag -width RF_PREFETCHABLE .It Dv RF_ACTIVE activate resource atomically. .It Dv RF_PREFETCHABLE resource is prefetchable. .It Dv RF_SHAREABLE resource permits contemporaneous sharing. It should always be set unless you know that the resource cannot be shared. It is the bus driver's task to filter out the flag if the bus does not support sharing. .It Dv RF_UNMAPPED do not establish implicit mapping when activated via .Xr bus_activate_resource 9 . .El .El .Sh RETURN VALUES A pointer to .Va struct resource is returned on success, a null pointer otherwise. .Sh EXAMPLES This is some example code that allocates a 32 byte I/O port range and an IRQ. .Bd -literal struct resource *portres, *irqres; portres = bus_alloc_resource(dev, SYS_RES_IOPORT, 0, 0ul, ~0ul, 32, RF_ACTIVE); irqres = bus_alloc_resource_any(dev, SYS_RES_IRQ, 0, RF_ACTIVE | RF_SHAREABLE); .Ed .Sh SEE ALSO .Xr bus_activate_resource 9 , .Xr bus_adjust_resource 9 , .Xr bus_map_resource 9 , .Xr bus_release_resource 9 , .Xr device 9 , .Xr driver 9 .Sh AUTHORS .An -nosplit This manual page was written by .An Alexander Langer Aq Mt alex@big.endian.de with parts by .An Warner Losh Aq Mt imp@FreeBSD.org . diff --git a/sys/arm/nvidia/tegra_pcie.c b/sys/arm/nvidia/tegra_pcie.c index ec272523df1b..c7a30965a5e5 100644 --- a/sys/arm/nvidia/tegra_pcie.c +++ b/sys/arm/nvidia/tegra_pcie.c @@ -1,1624 +1,1623 @@ /*- * Copyright (c) 2016 Michal Meloun * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include /* * Nvidia Integrated PCI/PCI-Express controller driver. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "ofw_bus_if.h" #include "msi_if.h" #include "pcib_if.h" #include "pic_if.h" #define AFI_AXI_BAR0_SZ 0x000 #define AFI_AXI_BAR1_SZ 0x004 #define AFI_AXI_BAR2_SZ 0x008 #define AFI_AXI_BAR3_SZ 0x00c #define AFI_AXI_BAR4_SZ 0x010 #define AFI_AXI_BAR5_SZ 0x014 #define AFI_AXI_BAR0_START 0x018 #define AFI_AXI_BAR1_START 0x01c #define AFI_AXI_BAR2_START 0x020 #define AFI_AXI_BAR3_START 0x024 #define AFI_AXI_BAR4_START 0x028 #define AFI_AXI_BAR5_START 0x02c #define AFI_FPCI_BAR0 0x030 #define AFI_FPCI_BAR1 0x034 #define AFI_FPCI_BAR2 0x038 #define AFI_FPCI_BAR3 0x03c #define AFI_FPCI_BAR4 0x040 #define AFI_FPCI_BAR5 0x044 #define AFI_MSI_BAR_SZ 0x060 #define AFI_MSI_FPCI_BAR_ST 0x064 #define AFI_MSI_AXI_BAR_ST 0x068 #define AFI_MSI_VEC(x) (0x06c + 4 * (x)) #define AFI_MSI_EN_VEC(x) (0x08c + 4 * (x)) #define AFI_MSI_INTR_IN_REG 32 #define AFI_MSI_REGS 8 #define AFI_CONFIGURATION 0x0ac #define AFI_CONFIGURATION_EN_FPCI (1 << 0) #define AFI_FPCI_ERROR_MASKS 0x0b0 #define AFI_INTR_MASK 0x0b4 #define AFI_INTR_MASK_MSI_MASK (1 << 8) #define AFI_INTR_MASK_INT_MASK (1 << 0) #define AFI_INTR_CODE 0x0b8 #define AFI_INTR_CODE_MASK 0xf #define AFI_INTR_CODE_INT_CODE_INI_SLVERR 1 #define AFI_INTR_CODE_INT_CODE_INI_DECERR 2 #define AFI_INTR_CODE_INT_CODE_TGT_SLVERR 3 #define AFI_INTR_CODE_INT_CODE_TGT_DECERR 4 #define AFI_INTR_CODE_INT_CODE_TGT_WRERR 5 #define AFI_INTR_CODE_INT_CODE_SM_MSG 6 #define AFI_INTR_CODE_INT_CODE_DFPCI_DECERR 7 #define AFI_INTR_CODE_INT_CODE_AXI_DECERR 8 #define AFI_INTR_CODE_INT_CODE_FPCI_TIMEOUT 9 #define AFI_INTR_CODE_INT_CODE_PE_PRSNT_SENSE 10 #define AFI_INTR_CODE_INT_CODE_PE_CLKREQ_SENSE 11 #define AFI_INTR_CODE_INT_CODE_CLKCLAMP_SENSE 12 #define AFI_INTR_CODE_INT_CODE_RDY4PD_SENSE 13 #define AFI_INTR_CODE_INT_CODE_P2P_ERROR 14 #define AFI_INTR_SIGNATURE 0x0bc #define AFI_UPPER_FPCI_ADDRESS 0x0c0 #define AFI_SM_INTR_ENABLE 0x0c4 #define AFI_SM_INTR_RP_DEASSERT (1 << 14) #define AFI_SM_INTR_RP_ASSERT (1 << 13) #define AFI_SM_INTR_HOTPLUG (1 << 12) #define AFI_SM_INTR_PME (1 << 11) #define AFI_SM_INTR_FATAL_ERROR (1 << 10) #define AFI_SM_INTR_UNCORR_ERROR (1 << 9) #define AFI_SM_INTR_CORR_ERROR (1 << 8) #define AFI_SM_INTR_INTD_DEASSERT (1 << 7) #define AFI_SM_INTR_INTC_DEASSERT (1 << 6) #define AFI_SM_INTR_INTB_DEASSERT (1 << 5) #define AFI_SM_INTR_INTA_DEASSERT (1 << 4) #define AFI_SM_INTR_INTD_ASSERT (1 << 3) #define AFI_SM_INTR_INTC_ASSERT (1 << 2) #define AFI_SM_INTR_INTB_ASSERT (1 << 1) #define AFI_SM_INTR_INTA_ASSERT (1 << 0) #define AFI_AFI_INTR_ENABLE 0x0c8 #define AFI_AFI_INTR_ENABLE_CODE(code) (1 << (code)) #define AFI_PCIE_CONFIG 0x0f8 #define AFI_PCIE_CONFIG_PCIE_DISABLE(x) (1 << ((x) + 1)) #define AFI_PCIE_CONFIG_PCIE_DISABLE_ALL 0x6 #define AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_MASK (0xf << 20) #define AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_XBAR2_1 (0x0 << 20) #define AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_XBAR4_1 (0x1 << 20) #define AFI_FUSE 0x104 #define AFI_FUSE_PCIE_T0_GEN2_DIS (1 << 2) #define AFI_PEX0_CTRL 0x110 #define AFI_PEX1_CTRL 0x118 #define AFI_PEX2_CTRL 0x128 #define AFI_PEX_CTRL_OVERRIDE_EN (1 << 4) #define AFI_PEX_CTRL_REFCLK_EN (1 << 3) #define AFI_PEX_CTRL_CLKREQ_EN (1 << 1) #define AFI_PEX_CTRL_RST_L (1 << 0) #define AFI_AXI_BAR6_SZ 0x134 #define AFI_AXI_BAR7_SZ 0x138 #define AFI_AXI_BAR8_SZ 0x13c #define AFI_AXI_BAR6_START 0x140 #define AFI_AXI_BAR7_START 0x144 #define AFI_AXI_BAR8_START 0x148 #define AFI_FPCI_BAR6 0x14c #define AFI_FPCI_BAR7 0x150 #define AFI_FPCI_BAR8 0x154 #define AFI_PLLE_CONTROL 0x160 #define AFI_PLLE_CONTROL_BYPASS_PADS2PLLE_CONTROL (1 << 9) #define AFI_PLLE_CONTROL_BYPASS_PCIE2PLLE_CONTROL (1 << 8) #define AFI_PLLE_CONTROL_PADS2PLLE_CONTROL_EN (1 << 1) #define AFI_PLLE_CONTROL_PCIE2PLLE_CONTROL_EN (1 << 0) #define AFI_PEXBIAS_CTRL 0x168 /* Configuration space */ #define RP_VEND_XP 0x0F00 #define RP_VEND_XP_DL_UP (1 << 30) #define RP_VEND_CTL2 0x0fa8 #define RP_VEND_CTL2_PCA_ENABLE (1 << 7) #define RP_PRIV_MISC 0x0FE0 #define RP_PRIV_MISC_PRSNT_MAP_EP_PRSNT (0xE << 0) #define RP_PRIV_MISC_PRSNT_MAP_EP_ABSNT (0xF << 0) #define RP_LINK_CONTROL_STATUS 0x0090 #define RP_LINK_CONTROL_STATUS_DL_LINK_ACTIVE 0x20000000 #define RP_LINK_CONTROL_STATUS_LINKSTAT_MASK 0x3fff0000 /* PADS space */ #define PADS_REFCLK_CFG0 0x000c8 #define PADS_REFCLK_CFG1 0x000cc /* Wait 50 ms (per port) for link. */ #define TEGRA_PCIE_LINKUP_TIMEOUT 50000 /* FPCI Address space */ #define FPCI_MAP_IO 0xFDFC000000ULL #define FPCI_MAP_TYPE0_CONFIG 0xFDFC000000ULL #define FPCI_MAP_TYPE1_CONFIG 0xFDFF000000ULL #define FPCI_MAP_EXT_TYPE0_CONFIG 0xFE00000000ULL #define FPCI_MAP_EXT_TYPE1_CONFIG 0xFE10000000ULL #define TEGRA_PCIB_MSI_ENABLE #define DEBUG #ifdef DEBUG #define debugf(fmt, args...) do { printf(fmt,##args); } while (0) #else #define debugf(fmt, args...) #endif /* * Configuration space format: * [27:24] extended register * [23:16] bus * [15:11] slot (device) * [10: 8] function * [ 7: 0] register */ #define PCI_CFG_EXT_REG(reg) ((((reg) >> 8) & 0x0f) << 24) #define PCI_CFG_BUS(bus) (((bus) & 0xff) << 16) #define PCI_CFG_DEV(dev) (((dev) & 0x1f) << 11) #define PCI_CFG_FUN(fun) (((fun) & 0x07) << 8) #define PCI_CFG_BASE_REG(reg) ((reg) & 0xff) #define PADS_WR4(_sc, _r, _v) bus_write_4((_sc)->pads_mem_res, (_r), (_v)) #define PADS_RD4(_sc, _r) bus_read_4((_sc)->pads_mem_res, (_r)) #define AFI_WR4(_sc, _r, _v) bus_write_4((_sc)->afi_mem_res, (_r), (_v)) #define AFI_RD4(_sc, _r) bus_read_4((_sc)->afi_mem_res, (_r)) static struct { bus_size_t axi_start; bus_size_t fpci_start; bus_size_t size; } bars[] = { {AFI_AXI_BAR0_START, AFI_FPCI_BAR0, AFI_AXI_BAR0_SZ}, /* BAR 0 */ {AFI_AXI_BAR1_START, AFI_FPCI_BAR1, AFI_AXI_BAR1_SZ}, /* BAR 1 */ {AFI_AXI_BAR2_START, AFI_FPCI_BAR2, AFI_AXI_BAR2_SZ}, /* BAR 2 */ {AFI_AXI_BAR3_START, AFI_FPCI_BAR3, AFI_AXI_BAR3_SZ}, /* BAR 3 */ {AFI_AXI_BAR4_START, AFI_FPCI_BAR4, AFI_AXI_BAR4_SZ}, /* BAR 4 */ {AFI_AXI_BAR5_START, AFI_FPCI_BAR5, AFI_AXI_BAR5_SZ}, /* BAR 5 */ {AFI_AXI_BAR6_START, AFI_FPCI_BAR6, AFI_AXI_BAR6_SZ}, /* BAR 6 */ {AFI_AXI_BAR7_START, AFI_FPCI_BAR7, AFI_AXI_BAR7_SZ}, /* BAR 7 */ {AFI_AXI_BAR8_START, AFI_FPCI_BAR8, AFI_AXI_BAR8_SZ}, /* BAR 8 */ {AFI_MSI_AXI_BAR_ST, AFI_MSI_FPCI_BAR_ST, AFI_MSI_BAR_SZ}, /* MSI 9 */ }; struct pcie_soc { char **regulator_names; bool cml_clk; bool pca_enable; uint32_t pads_refclk_cfg0; uint32_t pads_refclk_cfg1; }; /* Tegra 124 config. */ static char *tegra124_reg_names[] = { "avddio-pex-supply", "dvddio-pex-supply", "avdd-pex-pll-supply", "hvdd-pex-supply", "hvdd-pex-pll-e-supply", "vddio-pex-ctl-supply", "avdd-pll-erefe-supply", NULL }; static struct pcie_soc tegra124_soc = { .regulator_names = tegra124_reg_names, .cml_clk = true, .pca_enable = false, .pads_refclk_cfg0 = 0x44ac44ac, }; /* Tegra 210 config. */ static char *tegra210_reg_names[] = { "avdd-pll-uerefe-supply", "hvddio-pex-supply", "dvddio-pex-supply", "dvdd-pex-pll-supply", "hvdd-pex-pll-e-supply", "vddio-pex-ctl-supply", NULL }; static struct pcie_soc tegra210_soc = { .regulator_names = tegra210_reg_names, .cml_clk = true, .pca_enable = true, .pads_refclk_cfg0 = 0x90b890b8, }; /* Compatible devices. */ static struct ofw_compat_data compat_data[] = { {"nvidia,tegra124-pcie", (uintptr_t)&tegra124_soc}, {"nvidia,tegra210-pcie", (uintptr_t)&tegra210_soc}, {NULL, 0}, }; #define TEGRA_FLAG_MSI_USED 0x0001 struct tegra_pcib_irqsrc { struct intr_irqsrc isrc; u_int irq; u_int flags; }; struct tegra_pcib_port { int enabled; int port_idx; /* chip port index */ int num_lanes; /* number of lanes */ bus_size_t afi_pex_ctrl; /* offset of afi_pex_ctrl */ phy_t phy; /* port phy */ /* Config space properties. */ bus_addr_t rp_base_addr; /* PA of config window */ bus_size_t rp_size; /* size of config window */ bus_space_handle_t cfg_handle; /* handle of config window */ }; #define TEGRA_PCIB_MAX_PORTS 3 #define TEGRA_PCIB_MAX_MSI AFI_MSI_INTR_IN_REG * AFI_MSI_REGS struct tegra_pcib_softc { struct ofw_pci_softc ofw_pci; device_t dev; struct pcie_soc *soc; struct mtx mtx; struct resource *pads_mem_res; struct resource *afi_mem_res; struct resource *cfg_mem_res; struct resource *irq_res; struct resource *msi_irq_res; void *intr_cookie; void *msi_intr_cookie; struct ofw_pci_range mem_range; struct ofw_pci_range pref_mem_range; struct ofw_pci_range io_range; clk_t clk_pex; clk_t clk_afi; clk_t clk_pll_e; clk_t clk_cml; hwreset_t hwreset_pex; hwreset_t hwreset_afi; hwreset_t hwreset_pcie_x; regulator_t regulators[16]; /* Safe maximum */ vm_offset_t msi_page; /* VA of MSI page */ bus_addr_t cfg_base_addr; /* base address of config */ bus_size_t cfg_cur_offs; /* currently mapped window */ bus_space_handle_t cfg_handle; /* handle of config window */ bus_space_tag_t bus_tag; /* tag of config window */ int lanes_cfg; int num_ports; struct tegra_pcib_port *ports[TEGRA_PCIB_MAX_PORTS]; struct tegra_pcib_irqsrc *isrcs; }; static int tegra_pcib_maxslots(device_t dev) { return (16); } static int tegra_pcib_route_interrupt(device_t bus, device_t dev, int pin) { struct tegra_pcib_softc *sc; u_int irq; sc = device_get_softc(bus); irq = intr_map_clone_irq(rman_get_start(sc->irq_res)); device_printf(bus, "route pin %d for device %d.%d to %u\n", pin, pci_get_slot(dev), pci_get_function(dev), irq); return (irq); } static int tegra_pcbib_map_cfg(struct tegra_pcib_softc *sc, u_int bus, u_int slot, u_int func, u_int reg) { bus_size_t offs; int flags, rv; offs = sc->cfg_base_addr; offs |= PCI_CFG_BUS(bus) | PCI_CFG_DEV(slot) | PCI_CFG_FUN(func) | PCI_CFG_EXT_REG(reg); if ((sc->cfg_handle != 0) && (sc->cfg_cur_offs == offs)) return (0); if (sc->cfg_handle != 0) bus_space_unmap(sc->bus_tag, sc->cfg_handle, 0x800); #if defined(BUS_SPACE_MAP_NONPOSTED) flags = BUS_SPACE_MAP_NONPOSTED; #else flags = 0; #endif rv = bus_space_map(sc->bus_tag, offs, 0x800, flags, &sc->cfg_handle); if (rv != 0) device_printf(sc->dev, "Cannot map config space\n"); else sc->cfg_cur_offs = offs; return (rv); } static uint32_t tegra_pcib_read_config(device_t dev, u_int bus, u_int slot, u_int func, u_int reg, int bytes) { struct tegra_pcib_softc *sc; bus_space_handle_t hndl; uint32_t off; uint32_t val; int rv, i; sc = device_get_softc(dev); if (bus == 0) { if (func != 0) return (0xFFFFFFFF); for (i = 0; i < TEGRA_PCIB_MAX_PORTS; i++) { if ((sc->ports[i] != NULL) && (sc->ports[i]->port_idx == slot)) { hndl = sc->ports[i]->cfg_handle; off = reg & 0xFFF; break; } } if (i >= TEGRA_PCIB_MAX_PORTS) return (0xFFFFFFFF); } else { rv = tegra_pcbib_map_cfg(sc, bus, slot, func, reg); if (rv != 0) return (0xFFFFFFFF); hndl = sc->cfg_handle; off = PCI_CFG_BASE_REG(reg); } val = bus_space_read_4(sc->bus_tag, hndl, off & ~3); switch (bytes) { case 4: break; case 2: if (off & 3) val >>= 16; val &= 0xffff; break; case 1: val >>= ((off & 3) << 3); val &= 0xff; break; } return val; } static void tegra_pcib_write_config(device_t dev, u_int bus, u_int slot, u_int func, u_int reg, uint32_t val, int bytes) { struct tegra_pcib_softc *sc; bus_space_handle_t hndl; uint32_t off; uint32_t val2; int rv, i; sc = device_get_softc(dev); if (bus == 0) { if (func != 0) return; for (i = 0; i < TEGRA_PCIB_MAX_PORTS; i++) { if ((sc->ports[i] != NULL) && (sc->ports[i]->port_idx == slot)) { hndl = sc->ports[i]->cfg_handle; off = reg & 0xFFF; break; } } if (i >= TEGRA_PCIB_MAX_PORTS) return; } else { rv = tegra_pcbib_map_cfg(sc, bus, slot, func, reg); if (rv != 0) return; hndl = sc->cfg_handle; off = PCI_CFG_BASE_REG(reg); } switch (bytes) { case 4: bus_space_write_4(sc->bus_tag, hndl, off, val); break; case 2: val2 = bus_space_read_4(sc->bus_tag, hndl, off & ~3); val2 &= ~(0xffff << ((off & 3) << 3)); val2 |= ((val & 0xffff) << ((off & 3) << 3)); bus_space_write_4(sc->bus_tag, hndl, off & ~3, val2); break; case 1: val2 = bus_space_read_4(sc->bus_tag, hndl, off & ~3); val2 &= ~(0xff << ((off & 3) << 3)); val2 |= ((val & 0xff) << ((off & 3) << 3)); bus_space_write_4(sc->bus_tag, hndl, off & ~3, val2); break; } } static int tegra_pci_intr(void *arg) { struct tegra_pcib_softc *sc = arg; uint32_t code, signature; code = bus_read_4(sc->afi_mem_res, AFI_INTR_CODE) & AFI_INTR_CODE_MASK; signature = bus_read_4(sc->afi_mem_res, AFI_INTR_SIGNATURE); bus_write_4(sc->afi_mem_res, AFI_INTR_CODE, 0); if (code == AFI_INTR_CODE_INT_CODE_SM_MSG) return(FILTER_STRAY); printf("tegra_pci_intr: code %x sig %x\n", code, signature); return (FILTER_HANDLED); } /* ----------------------------------------------------------------------- * * PCI MSI interface */ static int tegra_pcib_alloc_msi(device_t pci, device_t child, int count, int maxcount, int *irqs) { phandle_t msi_parent; /* XXXX ofw_bus_msimap() don't works for Tegra DT. ofw_bus_msimap(ofw_bus_get_node(pci), pci_get_rid(child), &msi_parent, NULL); */ msi_parent = OF_xref_from_node(ofw_bus_get_node(pci)); return (intr_alloc_msi(pci, child, msi_parent, count, maxcount, irqs)); } static int tegra_pcib_release_msi(device_t pci, device_t child, int count, int *irqs) { phandle_t msi_parent; /* XXXX ofw_bus_msimap() don't works for Tegra DT. ofw_bus_msimap(ofw_bus_get_node(pci), pci_get_rid(child), &msi_parent, NULL); */ msi_parent = OF_xref_from_node(ofw_bus_get_node(pci)); return (intr_release_msi(pci, child, msi_parent, count, irqs)); } static int tegra_pcib_map_msi(device_t pci, device_t child, int irq, uint64_t *addr, uint32_t *data) { phandle_t msi_parent; /* XXXX ofw_bus_msimap() don't works for Tegra DT. ofw_bus_msimap(ofw_bus_get_node(pci), pci_get_rid(child), &msi_parent, NULL); */ msi_parent = OF_xref_from_node(ofw_bus_get_node(pci)); return (intr_map_msi(pci, child, msi_parent, irq, addr, data)); } #ifdef TEGRA_PCIB_MSI_ENABLE /* -------------------------------------------------------------------------- * * Interrupts * */ static inline void tegra_pcib_isrc_mask(struct tegra_pcib_softc *sc, struct tegra_pcib_irqsrc *tgi, uint32_t val) { uint32_t reg; int offs, bit; offs = tgi->irq / AFI_MSI_INTR_IN_REG; bit = 1 << (tgi->irq % AFI_MSI_INTR_IN_REG); if (val != 0) AFI_WR4(sc, AFI_MSI_VEC(offs), bit); reg = AFI_RD4(sc, AFI_MSI_EN_VEC(offs)); if (val != 0) reg |= bit; else reg &= ~bit; AFI_WR4(sc, AFI_MSI_EN_VEC(offs), reg); } static int tegra_pcib_msi_intr(void *arg) { u_int irq, i, bit, reg; struct tegra_pcib_softc *sc; struct trapframe *tf; struct tegra_pcib_irqsrc *tgi; sc = (struct tegra_pcib_softc *)arg; tf = curthread->td_intr_frame; for (i = 0; i < AFI_MSI_REGS; i++) { reg = AFI_RD4(sc, AFI_MSI_VEC(i)); /* Handle one vector. */ while (reg != 0) { bit = ffs(reg) - 1; /* Send EOI */ AFI_WR4(sc, AFI_MSI_VEC(i), 1 << bit); irq = i * AFI_MSI_INTR_IN_REG + bit; tgi = &sc->isrcs[irq]; if (intr_isrc_dispatch(&tgi->isrc, tf) != 0) { /* Disable stray. */ tegra_pcib_isrc_mask(sc, tgi, 0); device_printf(sc->dev, "Stray irq %u disabled\n", irq); } reg = AFI_RD4(sc, AFI_MSI_VEC(i)); } } return (FILTER_HANDLED); } static int tegra_pcib_msi_attach(struct tegra_pcib_softc *sc) { int error; uint32_t irq; const char *name; sc->isrcs = malloc(sizeof(*sc->isrcs) * TEGRA_PCIB_MAX_MSI, M_DEVBUF, M_WAITOK | M_ZERO); name = device_get_nameunit(sc->dev); for (irq = 0; irq < TEGRA_PCIB_MAX_MSI; irq++) { sc->isrcs[irq].irq = irq; error = intr_isrc_register(&sc->isrcs[irq].isrc, sc->dev, 0, "%s,%u", name, irq); if (error != 0) return (error); /* XXX deregister ISRCs */ } if (intr_msi_register(sc->dev, OF_xref_from_node(ofw_bus_get_node(sc->dev))) != 0) return (ENXIO); return (0); } static int tegra_pcib_msi_detach(struct tegra_pcib_softc *sc) { /* * There has not been established any procedure yet * how to detach PIC from living system correctly. */ device_printf(sc->dev, "%s: not implemented yet\n", __func__); return (EBUSY); } static void tegra_pcib_msi_disable_intr(device_t dev, struct intr_irqsrc *isrc) { struct tegra_pcib_softc *sc; struct tegra_pcib_irqsrc *tgi; sc = device_get_softc(dev); tgi = (struct tegra_pcib_irqsrc *)isrc; tegra_pcib_isrc_mask(sc, tgi, 0); } static void tegra_pcib_msi_enable_intr(device_t dev, struct intr_irqsrc *isrc) { struct tegra_pcib_softc *sc; struct tegra_pcib_irqsrc *tgi; sc = device_get_softc(dev); tgi = (struct tegra_pcib_irqsrc *)isrc; tegra_pcib_isrc_mask(sc, tgi, 1); } /* MSI interrupts are edge trigered -> do nothing */ static void tegra_pcib_msi_post_filter(device_t dev, struct intr_irqsrc *isrc) { } static void tegra_pcib_msi_post_ithread(device_t dev, struct intr_irqsrc *isrc) { } static void tegra_pcib_msi_pre_ithread(device_t dev, struct intr_irqsrc *isrc) { } static int tegra_pcib_msi_setup_intr(device_t dev, struct intr_irqsrc *isrc, struct resource *res, struct intr_map_data *data) { if (data == NULL || data->type != INTR_MAP_DATA_MSI) return (ENOTSUP); if (isrc->isrc_handlers == 0) tegra_pcib_msi_enable_intr(dev, isrc); return (0); } static int tegra_pcib_msi_teardown_intr(device_t dev, struct intr_irqsrc *isrc, struct resource *res, struct intr_map_data *data) { struct tegra_pcib_softc *sc; struct tegra_pcib_irqsrc *tgi; sc = device_get_softc(dev); tgi = (struct tegra_pcib_irqsrc *)isrc; if (isrc->isrc_handlers == 0) tegra_pcib_isrc_mask(sc, tgi, 0); return (0); } static int tegra_pcib_msi_alloc_msi(device_t dev, device_t child, int count, int maxcount, device_t *pic, struct intr_irqsrc **srcs) { struct tegra_pcib_softc *sc; int i, irq, end_irq; bool found; KASSERT(powerof2(count), ("%s: bad count", __func__)); KASSERT(powerof2(maxcount), ("%s: bad maxcount", __func__)); sc = device_get_softc(dev); mtx_lock(&sc->mtx); found = false; for (irq = 0; (irq + count - 1) < TEGRA_PCIB_MAX_MSI; irq++) { /* Start on an aligned interrupt */ if ((irq & (maxcount - 1)) != 0) continue; /* Assume we found a valid range until shown otherwise */ found = true; /* Check this range is valid */ for (end_irq = irq; end_irq < irq + count; end_irq++) { /* This is already used */ if ((sc->isrcs[end_irq].flags & TEGRA_FLAG_MSI_USED) == TEGRA_FLAG_MSI_USED) { found = false; break; } } if (found) break; } /* Not enough interrupts were found */ if (!found || irq == (TEGRA_PCIB_MAX_MSI - 1)) { mtx_unlock(&sc->mtx); return (ENXIO); } for (i = 0; i < count; i++) { /* Mark the interrupt as used */ sc->isrcs[irq + i].flags |= TEGRA_FLAG_MSI_USED; } mtx_unlock(&sc->mtx); for (i = 0; i < count; i++) srcs[i] = (struct intr_irqsrc *)&sc->isrcs[irq + i]; *pic = device_get_parent(dev); return (0); } static int tegra_pcib_msi_release_msi(device_t dev, device_t child, int count, struct intr_irqsrc **isrc) { struct tegra_pcib_softc *sc; struct tegra_pcib_irqsrc *ti; int i; sc = device_get_softc(dev); mtx_lock(&sc->mtx); for (i = 0; i < count; i++) { ti = (struct tegra_pcib_irqsrc *)isrc[i]; KASSERT((ti->flags & TEGRA_FLAG_MSI_USED) == TEGRA_FLAG_MSI_USED, ("%s: Trying to release an unused MSI-X interrupt", __func__)); ti->flags &= ~TEGRA_FLAG_MSI_USED; } mtx_unlock(&sc->mtx); return (0); } static int tegra_pcib_msi_map_msi(device_t dev, device_t child, struct intr_irqsrc *isrc, uint64_t *addr, uint32_t *data) { struct tegra_pcib_softc *sc = device_get_softc(dev); struct tegra_pcib_irqsrc *ti = (struct tegra_pcib_irqsrc *)isrc; *addr = vtophys(sc->msi_page); *data = ti->irq; return (0); } #endif /* ------------------------------------------------------------------- */ static bus_size_t tegra_pcib_pex_ctrl(struct tegra_pcib_softc *sc, int port) { switch (port) { case 0: return (AFI_PEX0_CTRL); case 1: return (AFI_PEX1_CTRL); case 2: return (AFI_PEX2_CTRL); default: panic("invalid port number: %d\n", port); } } static int tegra_pcib_enable_fdt_resources(struct tegra_pcib_softc *sc) { int i, rv; rv = hwreset_assert(sc->hwreset_pcie_x); if (rv != 0) { device_printf(sc->dev, "Cannot assert 'pcie_x' reset\n"); return (rv); } rv = hwreset_assert(sc->hwreset_afi); if (rv != 0) { device_printf(sc->dev, "Cannot assert 'afi' reset\n"); return (rv); } rv = hwreset_assert(sc->hwreset_pex); if (rv != 0) { device_printf(sc->dev, "Cannot assert 'pex' reset\n"); return (rv); } tegra_powergate_power_off(TEGRA_POWERGATE_PCX); /* Regulators. */ for (i = 0; i < nitems(sc->regulators); i++) { if (sc->regulators[i] == NULL) continue; rv = regulator_enable(sc->regulators[i]); if (rv != 0) { device_printf(sc->dev, "Cannot enable '%s' regulator\n", sc->soc->regulator_names[i]); return (rv); } } rv = tegra_powergate_sequence_power_up(TEGRA_POWERGATE_PCX, sc->clk_pex, sc->hwreset_pex); if (rv != 0) { device_printf(sc->dev, "Cannot enable 'PCX' powergate\n"); return (rv); } rv = hwreset_deassert(sc->hwreset_afi); if (rv != 0) { device_printf(sc->dev, "Cannot unreset 'afi' reset\n"); return (rv); } rv = clk_enable(sc->clk_afi); if (rv != 0) { device_printf(sc->dev, "Cannot enable 'afi' clock\n"); return (rv); } if (sc->soc->cml_clk) { rv = clk_enable(sc->clk_cml); if (rv != 0) { device_printf(sc->dev, "Cannot enable 'cml' clock\n"); return (rv); } } rv = clk_enable(sc->clk_pll_e); if (rv != 0) { device_printf(sc->dev, "Cannot enable 'pll_e' clock\n"); return (rv); } return (0); } static struct tegra_pcib_port * tegra_pcib_parse_port(struct tegra_pcib_softc *sc, phandle_t node) { struct tegra_pcib_port *port; uint32_t tmp[5]; char tmpstr[6]; int rv; port = malloc(sizeof(struct tegra_pcib_port), M_DEVBUF, M_WAITOK); rv = OF_getprop(node, "status", tmpstr, sizeof(tmpstr)); if (rv <= 0 || strcmp(tmpstr, "okay") == 0 || strcmp(tmpstr, "ok") == 0) port->enabled = 1; else port->enabled = 0; rv = OF_getencprop(node, "assigned-addresses", tmp, sizeof(tmp)); if (rv != sizeof(tmp)) { device_printf(sc->dev, "Cannot parse assigned-address: %d\n", rv); goto fail; } port->rp_base_addr = tmp[2]; port->rp_size = tmp[4]; port->port_idx = OFW_PCI_PHYS_HI_DEVICE(tmp[0]) - 1; if (port->port_idx >= TEGRA_PCIB_MAX_PORTS) { device_printf(sc->dev, "Invalid port index: %d\n", port->port_idx); goto fail; } /* XXX - TODO: * Implement proper function for parsing pci "reg" property: * - it have PCI bus format * - its relative to matching "assigned-addresses" */ rv = OF_getencprop(node, "reg", tmp, sizeof(tmp)); if (rv != sizeof(tmp)) { device_printf(sc->dev, "Cannot parse reg: %d\n", rv); goto fail; } port->rp_base_addr += tmp[2]; rv = OF_getencprop(node, "nvidia,num-lanes", &port->num_lanes, sizeof(port->num_lanes)); if (rv != sizeof(port->num_lanes)) { device_printf(sc->dev, "Cannot parse nvidia,num-lanes: %d\n", rv); goto fail; } if (port->num_lanes > 4) { device_printf(sc->dev, "Invalid nvidia,num-lanes: %d\n", port->num_lanes); goto fail; } port->afi_pex_ctrl = tegra_pcib_pex_ctrl(sc, port->port_idx); sc->lanes_cfg |= port->num_lanes << (4 * port->port_idx); /* Phy. */ rv = phy_get_by_ofw_name(sc->dev, node, "pcie-0", &port->phy); if (rv != 0) { device_printf(sc->dev, "Cannot get 'pcie-0' phy for port %d\n", port->port_idx); goto fail; } return (port); fail: free(port, M_DEVBUF); return (NULL); } static int tegra_pcib_parse_fdt_resources(struct tegra_pcib_softc *sc, phandle_t node) { phandle_t child; struct tegra_pcib_port *port; int i, rv; /* Regulators. */ for (i = 0; sc->soc->regulator_names[i] != NULL; i++) { if (i >= nitems(sc->regulators)) { device_printf(sc->dev, "Too many regulators present in DT.\n"); return (EOVERFLOW); } rv = regulator_get_by_ofw_property(sc->dev, 0, sc->soc->regulator_names[i], sc->regulators + i); if (rv != 0) { device_printf(sc->dev, "Cannot get '%s' regulator\n", sc->soc->regulator_names[i]); return (ENXIO); } } /* Resets. */ rv = hwreset_get_by_ofw_name(sc->dev, 0, "pex", &sc->hwreset_pex); if (rv != 0) { device_printf(sc->dev, "Cannot get 'pex' reset\n"); return (ENXIO); } rv = hwreset_get_by_ofw_name(sc->dev, 0, "afi", &sc->hwreset_afi); if (rv != 0) { device_printf(sc->dev, "Cannot get 'afi' reset\n"); return (ENXIO); } rv = hwreset_get_by_ofw_name(sc->dev, 0, "pcie_x", &sc->hwreset_pcie_x); if (rv != 0) { device_printf(sc->dev, "Cannot get 'pcie_x' reset\n"); return (ENXIO); } /* Clocks. */ rv = clk_get_by_ofw_name(sc->dev, 0, "pex", &sc->clk_pex); if (rv != 0) { device_printf(sc->dev, "Cannot get 'pex' clock\n"); return (ENXIO); } rv = clk_get_by_ofw_name(sc->dev, 0, "afi", &sc->clk_afi); if (rv != 0) { device_printf(sc->dev, "Cannot get 'afi' clock\n"); return (ENXIO); } rv = clk_get_by_ofw_name(sc->dev, 0, "pll_e", &sc->clk_pll_e); if (rv != 0) { device_printf(sc->dev, "Cannot get 'pll_e' clock\n"); return (ENXIO); } if (sc->soc->cml_clk) { rv = clk_get_by_ofw_name(sc->dev, 0, "cml", &sc->clk_cml); if (rv != 0) { device_printf(sc->dev, "Cannot get 'cml' clock\n"); return (ENXIO); } } /* Ports */ sc->num_ports = 0; for (child = OF_child(node); child != 0; child = OF_peer(child)) { port = tegra_pcib_parse_port(sc, child); if (port == NULL) { device_printf(sc->dev, "Cannot parse PCIe port node\n"); return (ENXIO); } sc->ports[sc->num_ports++] = port; } return (0); } static int tegra_pcib_decode_ranges(struct tegra_pcib_softc *sc, struct ofw_pci_range *ranges, int nranges) { int i; for (i = 2; i < nranges; i++) { if ((ranges[i].pci_hi & OFW_PCI_PHYS_HI_SPACEMASK) == OFW_PCI_PHYS_HI_SPACE_IO) { if (sc->io_range.size != 0) { device_printf(sc->dev, "Duplicated IO range found in DT\n"); return (ENXIO); } sc->io_range = ranges[i]; } if (((ranges[i].pci_hi & OFW_PCI_PHYS_HI_SPACEMASK) == OFW_PCI_PHYS_HI_SPACE_MEM32)) { if (ranges[i].pci_hi & OFW_PCI_PHYS_HI_PREFETCHABLE) { if (sc->pref_mem_range.size != 0) { device_printf(sc->dev, "Duplicated memory range found " "in DT\n"); return (ENXIO); } sc->pref_mem_range = ranges[i]; } else { if (sc->mem_range.size != 0) { device_printf(sc->dev, "Duplicated memory range found " "in DT\n"); return (ENXIO); } sc->mem_range = ranges[i]; } } } if ((sc->io_range.size == 0) || (sc->mem_range.size == 0) || (sc->pref_mem_range.size == 0)) { device_printf(sc->dev, " Not all required ranges are found in DT\n"); return (ENXIO); } return (0); } /* * Hardware config. */ static int tegra_pcib_wait_for_link(struct tegra_pcib_softc *sc, struct tegra_pcib_port *port) { uint32_t reg; int i; /* Setup link detection. */ reg = tegra_pcib_read_config(sc->dev, 0, port->port_idx, 0, RP_PRIV_MISC, 4); reg &= ~RP_PRIV_MISC_PRSNT_MAP_EP_ABSNT; reg |= RP_PRIV_MISC_PRSNT_MAP_EP_PRSNT; tegra_pcib_write_config(sc->dev, 0, port->port_idx, 0, RP_PRIV_MISC, reg, 4); for (i = TEGRA_PCIE_LINKUP_TIMEOUT; i > 0; i--) { reg = tegra_pcib_read_config(sc->dev, 0, port->port_idx, 0, RP_VEND_XP, 4); if (reg & RP_VEND_XP_DL_UP) break; DELAY(1); } if (i <= 0) return (ETIMEDOUT); for (i = TEGRA_PCIE_LINKUP_TIMEOUT; i > 0; i--) { reg = tegra_pcib_read_config(sc->dev, 0, port->port_idx, 0, RP_LINK_CONTROL_STATUS, 4); if (reg & RP_LINK_CONTROL_STATUS_DL_LINK_ACTIVE) break; DELAY(1); } if (i <= 0) return (ETIMEDOUT); return (0); } static void tegra_pcib_port_enable(struct tegra_pcib_softc *sc, int port_num) { struct tegra_pcib_port *port; uint32_t reg; int rv; port = sc->ports[port_num]; /* Put port to reset. */ reg = AFI_RD4(sc, port->afi_pex_ctrl); reg &= ~AFI_PEX_CTRL_RST_L; AFI_WR4(sc, port->afi_pex_ctrl, reg); AFI_RD4(sc, port->afi_pex_ctrl); DELAY(10); /* Enable clocks. */ reg |= AFI_PEX_CTRL_REFCLK_EN; reg |= AFI_PEX_CTRL_CLKREQ_EN; reg |= AFI_PEX_CTRL_OVERRIDE_EN; AFI_WR4(sc, port->afi_pex_ctrl, reg); AFI_RD4(sc, port->afi_pex_ctrl); DELAY(100); /* Release reset. */ reg |= AFI_PEX_CTRL_RST_L; AFI_WR4(sc, port->afi_pex_ctrl, reg); if (sc->soc->pca_enable) { reg = tegra_pcib_read_config(sc->dev, 0, port->port_idx, 0, RP_VEND_CTL2, 4); reg |= RP_VEND_CTL2_PCA_ENABLE; tegra_pcib_write_config(sc->dev, 0, port->port_idx, 0, RP_VEND_CTL2, reg, 4); } rv = tegra_pcib_wait_for_link(sc, port); if (bootverbose) device_printf(sc->dev, " port %d (%d lane%s): Link is %s\n", port->port_idx, port->num_lanes, port->num_lanes > 1 ? "s": "", rv == 0 ? "up": "down"); } static void tegra_pcib_port_disable(struct tegra_pcib_softc *sc, uint32_t port_num) { struct tegra_pcib_port *port; uint32_t reg; port = sc->ports[port_num]; /* Put port to reset. */ reg = AFI_RD4(sc, port->afi_pex_ctrl); reg &= ~AFI_PEX_CTRL_RST_L; AFI_WR4(sc, port->afi_pex_ctrl, reg); AFI_RD4(sc, port->afi_pex_ctrl); DELAY(10); /* Disable clocks. */ reg &= ~AFI_PEX_CTRL_CLKREQ_EN; reg &= ~AFI_PEX_CTRL_REFCLK_EN; AFI_WR4(sc, port->afi_pex_ctrl, reg); if (bootverbose) device_printf(sc->dev, " port %d (%d lane%s): Disabled\n", port->port_idx, port->num_lanes, port->num_lanes > 1 ? "s": ""); } static void tegra_pcib_set_bar(struct tegra_pcib_softc *sc, int bar, uint32_t axi, uint64_t fpci, uint32_t size, int is_memory) { uint32_t fpci_reg; uint32_t axi_reg; uint32_t size_reg; axi_reg = axi & ~0xFFF; size_reg = size >> 12; fpci_reg = (uint32_t)(fpci >> 8) & ~0xF; fpci_reg |= is_memory ? 0x1 : 0x0; AFI_WR4(sc, bars[bar].axi_start, axi_reg); AFI_WR4(sc, bars[bar].size, size_reg); AFI_WR4(sc, bars[bar].fpci_start, fpci_reg); } static int tegra_pcib_enable(struct tegra_pcib_softc *sc) { int rv; int i; uint32_t reg; rv = tegra_pcib_enable_fdt_resources(sc); if (rv != 0) { device_printf(sc->dev, "Cannot enable FDT resources\n"); return (rv); } /* Enable PLLE control. */ reg = AFI_RD4(sc, AFI_PLLE_CONTROL); reg &= ~AFI_PLLE_CONTROL_BYPASS_PADS2PLLE_CONTROL; reg |= AFI_PLLE_CONTROL_PADS2PLLE_CONTROL_EN; AFI_WR4(sc, AFI_PLLE_CONTROL, reg); /* Set bias pad. */ AFI_WR4(sc, AFI_PEXBIAS_CTRL, 0); /* Configure mode and ports. */ reg = AFI_RD4(sc, AFI_PCIE_CONFIG); reg &= ~AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_MASK; if (sc->lanes_cfg == 0x14) { if (bootverbose) device_printf(sc->dev, "Using x1,x4 configuration\n"); reg |= AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_XBAR4_1; } else if (sc->lanes_cfg == 0x12) { if (bootverbose) device_printf(sc->dev, "Using x1,x2 configuration\n"); reg |= AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_XBAR2_1; } else { device_printf(sc->dev, "Unsupported lanes configuration: 0x%X\n", sc->lanes_cfg); } reg |= AFI_PCIE_CONFIG_PCIE_DISABLE_ALL; for (i = 0; i < TEGRA_PCIB_MAX_PORTS; i++) { if ((sc->ports[i] != NULL)) reg &= ~AFI_PCIE_CONFIG_PCIE_DISABLE(sc->ports[i]->port_idx); } AFI_WR4(sc, AFI_PCIE_CONFIG, reg); /* Enable Gen2 support. */ reg = AFI_RD4(sc, AFI_FUSE); reg &= ~AFI_FUSE_PCIE_T0_GEN2_DIS; AFI_WR4(sc, AFI_FUSE, reg); for (i = 0; i < TEGRA_PCIB_MAX_PORTS; i++) { if (sc->ports[i] != NULL) { rv = phy_enable(sc->ports[i]->phy); if (rv != 0) { device_printf(sc->dev, "Cannot enable phy for port %d\n", sc->ports[i]->port_idx); return (rv); } } } /* Configure PCIe reference clock */ PADS_WR4(sc, PADS_REFCLK_CFG0, sc->soc->pads_refclk_cfg0); if (sc->num_ports > 2) PADS_WR4(sc, PADS_REFCLK_CFG1, sc->soc->pads_refclk_cfg1); rv = hwreset_deassert(sc->hwreset_pcie_x); if (rv != 0) { device_printf(sc->dev, "Cannot unreset 'pci_x' reset\n"); return (rv); } /* Enable config space. */ reg = AFI_RD4(sc, AFI_CONFIGURATION); reg |= AFI_CONFIGURATION_EN_FPCI; AFI_WR4(sc, AFI_CONFIGURATION, reg); /* Enable AFI errors. */ reg = 0; reg |= AFI_AFI_INTR_ENABLE_CODE(AFI_INTR_CODE_INT_CODE_INI_SLVERR); reg |= AFI_AFI_INTR_ENABLE_CODE(AFI_INTR_CODE_INT_CODE_INI_DECERR); reg |= AFI_AFI_INTR_ENABLE_CODE(AFI_INTR_CODE_INT_CODE_TGT_SLVERR); reg |= AFI_AFI_INTR_ENABLE_CODE(AFI_INTR_CODE_INT_CODE_TGT_DECERR); reg |= AFI_AFI_INTR_ENABLE_CODE(AFI_INTR_CODE_INT_CODE_TGT_WRERR); reg |= AFI_AFI_INTR_ENABLE_CODE(AFI_INTR_CODE_INT_CODE_SM_MSG); reg |= AFI_AFI_INTR_ENABLE_CODE(AFI_INTR_CODE_INT_CODE_DFPCI_DECERR); reg |= AFI_AFI_INTR_ENABLE_CODE(AFI_INTR_CODE_INT_CODE_AXI_DECERR); reg |= AFI_AFI_INTR_ENABLE_CODE(AFI_INTR_CODE_INT_CODE_FPCI_TIMEOUT); reg |= AFI_AFI_INTR_ENABLE_CODE(AFI_INTR_CODE_INT_CODE_PE_PRSNT_SENSE); reg |= AFI_AFI_INTR_ENABLE_CODE(AFI_INTR_CODE_INT_CODE_PE_CLKREQ_SENSE); reg |= AFI_AFI_INTR_ENABLE_CODE(AFI_INTR_CODE_INT_CODE_CLKCLAMP_SENSE); reg |= AFI_AFI_INTR_ENABLE_CODE(AFI_INTR_CODE_INT_CODE_RDY4PD_SENSE); reg |= AFI_AFI_INTR_ENABLE_CODE(AFI_INTR_CODE_INT_CODE_P2P_ERROR); AFI_WR4(sc, AFI_AFI_INTR_ENABLE, reg); AFI_WR4(sc, AFI_SM_INTR_ENABLE, 0xffffffff); /* Enable INT, disable MSI. */ AFI_WR4(sc, AFI_INTR_MASK, AFI_INTR_MASK_INT_MASK); /* Mask all FPCI errors. */ AFI_WR4(sc, AFI_FPCI_ERROR_MASKS, 0); /* Setup AFI translation windows. */ /* BAR 0 - type 1 extended configuration. */ tegra_pcib_set_bar(sc, 0, rman_get_start(sc->cfg_mem_res), FPCI_MAP_EXT_TYPE1_CONFIG, rman_get_size(sc->cfg_mem_res), 0); /* BAR 1 - downstream I/O. */ tegra_pcib_set_bar(sc, 1, sc->io_range.host, FPCI_MAP_IO, sc->io_range.size, 0); /* BAR 2 - downstream prefetchable memory 1:1. */ tegra_pcib_set_bar(sc, 2, sc->pref_mem_range.host, sc->pref_mem_range.host, sc->pref_mem_range.size, 1); /* BAR 3 - downstream not prefetchable memory 1:1 .*/ tegra_pcib_set_bar(sc, 3, sc->mem_range.host, sc->mem_range.host, sc->mem_range.size, 1); /* BAR 3-8 clear. */ tegra_pcib_set_bar(sc, 4, 0, 0, 0, 0); tegra_pcib_set_bar(sc, 5, 0, 0, 0, 0); tegra_pcib_set_bar(sc, 6, 0, 0, 0, 0); tegra_pcib_set_bar(sc, 7, 0, 0, 0, 0); tegra_pcib_set_bar(sc, 8, 0, 0, 0, 0); /* MSI BAR - clear. */ tegra_pcib_set_bar(sc, 9, 0, 0, 0, 0); return(0); } #ifdef TEGRA_PCIB_MSI_ENABLE static int tegra_pcib_attach_msi(device_t dev) { struct tegra_pcib_softc *sc; uint32_t reg; int i, rv; sc = device_get_softc(dev); sc->msi_page = (uintptr_t)kmem_alloc_contig(PAGE_SIZE, M_WAITOK, 0, BUS_SPACE_MAXADDR, PAGE_SIZE, 0, VM_MEMATTR_DEFAULT); /* MSI BAR */ tegra_pcib_set_bar(sc, 9, vtophys(sc->msi_page), vtophys(sc->msi_page), PAGE_SIZE, 0); /* Disable and clear all interrupts. */ for (i = 0; i < AFI_MSI_REGS; i++) { AFI_WR4(sc, AFI_MSI_EN_VEC(i), 0); AFI_WR4(sc, AFI_MSI_VEC(i), 0xFFFFFFFF); } rv = bus_setup_intr(dev, sc->msi_irq_res, INTR_TYPE_BIO | INTR_MPSAFE, tegra_pcib_msi_intr, NULL, sc, &sc->msi_intr_cookie); if (rv != 0) { device_printf(dev, "cannot setup MSI interrupt handler\n"); rv = ENXIO; goto out; } if (tegra_pcib_msi_attach(sc) != 0) { device_printf(dev, "WARNING: unable to attach PIC\n"); tegra_pcib_msi_detach(sc); goto out; } /* Unmask MSI interrupt. */ reg = AFI_RD4(sc, AFI_INTR_MASK); reg |= AFI_INTR_MASK_MSI_MASK; AFI_WR4(sc, AFI_INTR_MASK, reg); out: return (rv); } #endif static int tegra_pcib_probe(device_t dev) { if (!ofw_bus_status_okay(dev)) return (ENXIO); if (ofw_bus_search_compatible(dev, compat_data)->ocd_data != 0) { device_set_desc(dev, "Nvidia Integrated PCI/PCI-E Controller"); return (BUS_PROBE_DEFAULT); } return (ENXIO); } static int tegra_pcib_attach(device_t dev) { struct tegra_pcib_softc *sc; phandle_t node; int rv; int rid; struct tegra_pcib_port *port; int i; sc = device_get_softc(dev); sc->dev = dev; mtx_init(&sc->mtx, "msi_mtx", NULL, MTX_DEF); node = ofw_bus_get_node(dev); sc->soc = (struct pcie_soc *)ofw_bus_search_compatible(dev, compat_data)->ocd_data; rv = tegra_pcib_parse_fdt_resources(sc, node); if (rv != 0) { device_printf(dev, "Cannot get FDT resources\n"); return (rv); } /* Allocate bus_space resources. */ rid = 0; sc->pads_mem_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, RF_ACTIVE); if (sc->pads_mem_res == NULL) { device_printf(dev, "Cannot allocate PADS register\n"); rv = ENXIO; goto out; } /* * XXX - FIXME - * tag for config space is not filled when RF_ALLOCATED flag is used. + * tag for config space is not filled when RF_ACTIVE flag is not used. */ sc->bus_tag = rman_get_bustag(sc->pads_mem_res); rid = 1; sc->afi_mem_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, RF_ACTIVE); if (sc->afi_mem_res == NULL) { device_printf(dev, "Cannot allocate AFI register\n"); rv = ENXIO; goto out; } rid = 2; - sc->cfg_mem_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, - RF_ALLOCATED); + sc->cfg_mem_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, 0); if (sc->cfg_mem_res == NULL) { device_printf(dev, "Cannot allocate config space memory\n"); rv = ENXIO; goto out; } sc->cfg_base_addr = rman_get_start(sc->cfg_mem_res); /* Map RP slots */ for (i = 0; i < TEGRA_PCIB_MAX_PORTS; i++) { if (sc->ports[i] == NULL) continue; port = sc->ports[i]; rv = bus_space_map(sc->bus_tag, port->rp_base_addr, port->rp_size, 0, &port->cfg_handle); if (rv != 0) { device_printf(sc->dev, "Cannot allocate memory for " "port: %d\n", i); rv = ENXIO; goto out; } } /* * Get PCI interrupt */ rid = 0; sc->irq_res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid, RF_ACTIVE | RF_SHAREABLE); if (sc->irq_res == NULL) { device_printf(dev, "Cannot allocate IRQ resources\n"); rv = ENXIO; goto out; } rid = 1; sc->msi_irq_res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid, RF_ACTIVE); if (sc->irq_res == NULL) { device_printf(dev, "Cannot allocate MSI IRQ resources\n"); rv = ENXIO; goto out; } sc->ofw_pci.sc_range_mask = 0x3; rv = ofw_pcib_init(dev); if (rv != 0) goto out; rv = tegra_pcib_decode_ranges(sc, sc->ofw_pci.sc_range, sc->ofw_pci.sc_nrange); if (rv != 0) goto out; if (bus_setup_intr(dev, sc->irq_res, INTR_TYPE_BIO | INTR_MPSAFE, tegra_pci_intr, NULL, sc, &sc->intr_cookie)) { device_printf(dev, "cannot setup interrupt handler\n"); rv = ENXIO; goto out; } /* * Enable PCIE device. */ rv = tegra_pcib_enable(sc); if (rv != 0) goto out; for (i = 0; i < TEGRA_PCIB_MAX_PORTS; i++) { if (sc->ports[i] == NULL) continue; if (sc->ports[i]->enabled) tegra_pcib_port_enable(sc, i); else tegra_pcib_port_disable(sc, i); } #ifdef TEGRA_PCIB_MSI_ENABLE rv = tegra_pcib_attach_msi(dev); if (rv != 0) goto out; #endif device_add_child(dev, "pci", DEVICE_UNIT_ANY); bus_attach_children(dev); return (0); out: return (rv); } static device_method_t tegra_pcib_methods[] = { /* Device interface */ DEVMETHOD(device_probe, tegra_pcib_probe), DEVMETHOD(device_attach, tegra_pcib_attach), /* Bus interface */ DEVMETHOD(bus_setup_intr, bus_generic_setup_intr), DEVMETHOD(bus_teardown_intr, bus_generic_teardown_intr), /* pcib interface */ DEVMETHOD(pcib_maxslots, tegra_pcib_maxslots), DEVMETHOD(pcib_read_config, tegra_pcib_read_config), DEVMETHOD(pcib_write_config, tegra_pcib_write_config), DEVMETHOD(pcib_route_interrupt, tegra_pcib_route_interrupt), DEVMETHOD(pcib_alloc_msi, tegra_pcib_alloc_msi), DEVMETHOD(pcib_release_msi, tegra_pcib_release_msi), DEVMETHOD(pcib_map_msi, tegra_pcib_map_msi), DEVMETHOD(pcib_request_feature, pcib_request_feature_allow), #ifdef TEGRA_PCIB_MSI_ENABLE /* MSI/MSI-X */ DEVMETHOD(msi_alloc_msi, tegra_pcib_msi_alloc_msi), DEVMETHOD(msi_release_msi, tegra_pcib_msi_release_msi), DEVMETHOD(msi_map_msi, tegra_pcib_msi_map_msi), /* Interrupt controller interface */ DEVMETHOD(pic_disable_intr, tegra_pcib_msi_disable_intr), DEVMETHOD(pic_enable_intr, tegra_pcib_msi_enable_intr), DEVMETHOD(pic_setup_intr, tegra_pcib_msi_setup_intr), DEVMETHOD(pic_teardown_intr, tegra_pcib_msi_teardown_intr), DEVMETHOD(pic_post_filter, tegra_pcib_msi_post_filter), DEVMETHOD(pic_post_ithread, tegra_pcib_msi_post_ithread), DEVMETHOD(pic_pre_ithread, tegra_pcib_msi_pre_ithread), #endif /* OFW bus interface */ DEVMETHOD(ofw_bus_get_compat, ofw_bus_gen_get_compat), DEVMETHOD(ofw_bus_get_model, ofw_bus_gen_get_model), DEVMETHOD(ofw_bus_get_name, ofw_bus_gen_get_name), DEVMETHOD(ofw_bus_get_node, ofw_bus_gen_get_node), DEVMETHOD(ofw_bus_get_type, ofw_bus_gen_get_type), DEVMETHOD_END }; DEFINE_CLASS_1(pcib, tegra_pcib_driver, tegra_pcib_methods, sizeof(struct tegra_pcib_softc), ofw_pcib_driver); DRIVER_MODULE(tegra_pcib, simplebus, tegra_pcib_driver, NULL, NULL); diff --git a/sys/x86/iommu/amd_drv.c b/sys/x86/iommu/amd_drv.c index bf7ac4b157c4..3a7b2819cbdd 100644 --- a/sys/x86/iommu/amd_drv.c +++ b/sys/x86/iommu/amd_drv.c @@ -1,1224 +1,1224 @@ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2024 The FreeBSD Foundation * * This software was developed by Konstantin Belousov * under sponsorship from the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include "opt_acpi.h" #include "opt_ddb.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "pcib_if.h" #include #include #include #include #include #include #include #include #include static int amdiommu_enable = 0; static bool amdiommu_running = false; /* * All enumerated AMD IOMMU units. * Access is unlocked, the list is not modified after early * single-threaded startup. */ static TAILQ_HEAD(, amdiommu_unit) amdiommu_units = TAILQ_HEAD_INITIALIZER(amdiommu_units); typedef bool (*amdiommu_itercc_t)(void *, void *); typedef bool (*amdiommu_iter40_t)(ACPI_IVRS_HARDWARE2 *, void *); typedef bool (*amdiommu_iter11_t)(ACPI_IVRS_HARDWARE2 *, void *); typedef bool (*amdiommu_iter10_t)(ACPI_IVRS_HARDWARE1 *, void *); static bool amdiommu_ivrs_iterate_tbl_typed(amdiommu_itercc_t iter, void *arg, int type, ACPI_TABLE_IVRS *ivrs_tbl) { char *ptr, *ptrend; bool done; done = false; ptr = (char *)ivrs_tbl + sizeof(*ivrs_tbl); ptrend = (char *)ivrs_tbl + ivrs_tbl->Header.Length; for (;;) { ACPI_IVRS_HEADER *ivrsh; if (ptr >= ptrend) break; ivrsh = (ACPI_IVRS_HEADER *)ptr; if (ivrsh->Length <= 0) { printf("amdiommu_iterate_tbl: corrupted IVRS table, " "length %d\n", ivrsh->Length); break; } ptr += ivrsh->Length; if (ivrsh->Type == type) { done = iter((void *)ivrsh, arg); if (done) break; } } return (done); } /* * Walk over IVRS, calling callback iterators following priority: * 0x40, then 0x11, then 0x10 subtable. First iterator returning true * ends the walk. * Returns true if any iterator returned true, otherwise false. */ static bool amdiommu_ivrs_iterate_tbl(amdiommu_iter40_t iter40, amdiommu_iter11_t iter11, amdiommu_iter10_t iter10, void *arg) { ACPI_TABLE_IVRS *ivrs_tbl; ACPI_STATUS status; bool done; status = AcpiGetTable(ACPI_SIG_IVRS, 1, (ACPI_TABLE_HEADER **)&ivrs_tbl); if (ACPI_FAILURE(status)) return (false); done = false; if (iter40 != NULL) done = amdiommu_ivrs_iterate_tbl_typed( (amdiommu_itercc_t)iter40, arg, ACPI_IVRS_TYPE_HARDWARE3, ivrs_tbl); if (!done && iter11 != NULL) done = amdiommu_ivrs_iterate_tbl_typed( (amdiommu_itercc_t)iter11, arg, ACPI_IVRS_TYPE_HARDWARE2, ivrs_tbl); if (!done && iter10 != NULL) done = amdiommu_ivrs_iterate_tbl_typed( (amdiommu_itercc_t)iter10, arg, ACPI_IVRS_TYPE_HARDWARE1, ivrs_tbl); AcpiPutTable((ACPI_TABLE_HEADER *)ivrs_tbl); return (done); } struct ivhd_lookup_data { struct amdiommu_unit *sc; uint16_t devid; }; static bool ivrs_lookup_ivhd_0x40(ACPI_IVRS_HARDWARE2 *h2, void *arg) { struct ivhd_lookup_data *ildp; KASSERT(h2->Header.Type == ACPI_IVRS_TYPE_HARDWARE2 || h2->Header.Type == ACPI_IVRS_TYPE_HARDWARE3, ("Misparsed IVHD, h2 type %#x", h2->Header.Type)); ildp = arg; if (h2->Header.DeviceId != ildp->devid) return (false); ildp->sc->unit_dom = h2->PciSegmentGroup; ildp->sc->efr = h2->EfrRegisterImage; return (true); } static bool ivrs_lookup_ivhd_0x10(ACPI_IVRS_HARDWARE1 *h1, void *arg) { struct ivhd_lookup_data *ildp; KASSERT(h1->Header.Type == ACPI_IVRS_TYPE_HARDWARE1, ("Misparsed IVHD, h1 type %#x", h1->Header.Type)); ildp = arg; if (h1->Header.DeviceId != ildp->devid) return (false); ildp->sc->unit_dom = h1->PciSegmentGroup; return (true); } static u_int amdiommu_devtbl_sz(struct amdiommu_unit *sc __unused) { return (sizeof(struct amdiommu_dte) * (1 << 16)); } static void amdiommu_free_dev_tbl(struct amdiommu_unit *sc) { u_int devtbl_sz; devtbl_sz = amdiommu_devtbl_sz(sc); pmap_qremove((vm_offset_t)sc->dev_tbl, atop(devtbl_sz)); kva_free((vm_offset_t)sc->dev_tbl, devtbl_sz); sc->dev_tbl = NULL; vm_object_deallocate(sc->devtbl_obj); sc->devtbl_obj = NULL; } static int amdiommu_create_dev_tbl(struct amdiommu_unit *sc) { vm_offset_t seg_vaddr; u_int devtbl_sz, dom, i, reclaimno, segnum_log, segnum, seg_sz; int error; static const int devtab_base_regs[] = { AMDIOMMU_DEVTAB_BASE, AMDIOMMU_DEVTAB_S1_BASE, AMDIOMMU_DEVTAB_S2_BASE, AMDIOMMU_DEVTAB_S3_BASE, AMDIOMMU_DEVTAB_S4_BASE, AMDIOMMU_DEVTAB_S5_BASE, AMDIOMMU_DEVTAB_S6_BASE, AMDIOMMU_DEVTAB_S7_BASE }; segnum_log = (sc->efr & AMDIOMMU_EFR_DEVTBLSEG_MASK) >> AMDIOMMU_EFR_DEVTBLSEG_SHIFT; segnum = 1 << segnum_log; KASSERT(segnum <= nitems(devtab_base_regs), ("%s: unsupported devtab segment count %u", __func__, segnum)); devtbl_sz = amdiommu_devtbl_sz(sc); seg_sz = devtbl_sz / segnum; sc->devtbl_obj = vm_pager_allocate(OBJT_PHYS, NULL, atop(devtbl_sz), VM_PROT_ALL, 0, NULL); if (bus_get_domain(sc->iommu.dev, &dom) == 0) sc->devtbl_obj->domain.dr_policy = DOMAINSET_PREF(dom); sc->hw_ctrl &= ~AMDIOMMU_CTRL_DEVTABSEG_MASK; sc->hw_ctrl |= (uint64_t)segnum_log << ilog2(AMDIOMMU_CTRL_DEVTABSEG_2); sc->hw_ctrl |= AMDIOMMU_CTRL_COHERENT; amdiommu_write8(sc, AMDIOMMU_CTRL, sc->hw_ctrl); seg_vaddr = kva_alloc(devtbl_sz); if (seg_vaddr == 0) return (ENOMEM); sc->dev_tbl = (void *)seg_vaddr; for (i = 0; i < segnum; i++) { vm_page_t m; uint64_t rval; for (reclaimno = 0; reclaimno < 3; reclaimno++) { VM_OBJECT_WLOCK(sc->devtbl_obj); m = vm_page_alloc_contig(sc->devtbl_obj, i * atop(seg_sz), VM_ALLOC_NORMAL | VM_ALLOC_NOBUSY, atop(seg_sz), 0, ~0ul, IOMMU_PAGE_SIZE, 0, VM_MEMATTR_DEFAULT); VM_OBJECT_WUNLOCK(sc->devtbl_obj); if (m != NULL) break; error = vm_page_reclaim_contig(VM_ALLOC_NORMAL, atop(seg_sz), 0, ~0ul, IOMMU_PAGE_SIZE, 0); if (error != 0) vm_wait(sc->devtbl_obj); } if (m == NULL) { amdiommu_free_dev_tbl(sc); return (ENOMEM); } rval = VM_PAGE_TO_PHYS(m) | (atop(seg_sz) - 1); for (u_int j = 0; j < atop(seg_sz); j++, seg_vaddr += PAGE_SIZE, m++) { pmap_zero_page(m); pmap_qenter(seg_vaddr, &m, 1); } amdiommu_write8(sc, devtab_base_regs[i], rval); } return (0); } static int amdiommu_cmd_event_intr(void *arg) { struct amdiommu_unit *unit; uint64_t status; unit = arg; status = amdiommu_read8(unit, AMDIOMMU_CMDEV_STATUS); if ((status & AMDIOMMU_CMDEVS_COMWAITINT) != 0) { amdiommu_write8(unit, AMDIOMMU_CMDEV_STATUS, AMDIOMMU_CMDEVS_COMWAITINT); taskqueue_enqueue(unit->x86c.qi_taskqueue, &unit->x86c.qi_task); } if ((status & (AMDIOMMU_CMDEVS_EVLOGINT | AMDIOMMU_CMDEVS_EVOVRFLW)) != 0) amdiommu_event_intr(unit, status); return (FILTER_HANDLED); } static int amdiommu_setup_intr(struct amdiommu_unit *sc) { int error, msi_count, msix_count; msi_count = pci_msi_count(sc->iommu.dev); msix_count = pci_msix_count(sc->iommu.dev); if (msi_count == 0 && msix_count == 0) { device_printf(sc->iommu.dev, "needs MSI-class intr\n"); return (ENXIO); } #if 0 /* * XXXKIB how MSI-X is supposed to be organized for BAR-less * function? Practically available hardware implements only * one IOMMU unit per function, and uses MSI. */ if (msix_count > 0) { sc->msix_table = bus_alloc_resource_any(sc->iommu.dev, SYS_RES_MEMORY, &sc->msix_tab_rid, RF_ACTIVE); if (sc->msix_table == NULL) return (ENXIO); if (sc->msix_pba_rid != sc->msix_tab_rid) { /* Separate BAR for PBA */ sc->msix_pba = bus_alloc_resource_any(sc->iommu.dev, SYS_RES_MEMORY, &sc->msix_pba_rid, RF_ACTIVE); if (sc->msix_pba == NULL) { bus_release_resource(sc->iommu.dev, SYS_RES_MEMORY, &sc->msix_tab_rid, sc->msix_table); return (ENXIO); } } } #endif error = ENXIO; if (msix_count > 0) { error = pci_alloc_msix(sc->iommu.dev, &msix_count); if (error == 0) sc->numirqs = msix_count; } if (error != 0 && msi_count > 0) { error = pci_alloc_msi(sc->iommu.dev, &msi_count); if (error == 0) sc->numirqs = msi_count; } if (error != 0) { device_printf(sc->iommu.dev, "Failed to allocate MSI/MSI-x (%d)\n", error); return (ENXIO); } /* * XXXKIB Spec states that MISC0.MsiNum must be zero for IOMMU * using MSI interrupts. But at least one BIOS programmed '2' * there, making driver use wrong rid and causing * command/event interrupt ignored as stray. Try to fix it * with dirty force by assuming MsiNum is zero for MSI. */ sc->irq_cmdev_rid = 1; if (msix_count > 0) { sc->irq_cmdev_rid += pci_read_config(sc->iommu.dev, sc->seccap_reg + PCIR_AMDIOMMU_MISC0, 4) & PCIM_AMDIOMMU_MISC0_MSINUM_MASK; } sc->irq_cmdev = bus_alloc_resource_any(sc->iommu.dev, SYS_RES_IRQ, &sc->irq_cmdev_rid, RF_SHAREABLE | RF_ACTIVE); if (sc->irq_cmdev == NULL) { device_printf(sc->iommu.dev, "unable to map CMD/EV interrupt\n"); return (ENXIO); } error = bus_setup_intr(sc->iommu.dev, sc->irq_cmdev, INTR_TYPE_MISC, amdiommu_cmd_event_intr, NULL, sc, &sc->irq_cmdev_cookie); if (error != 0) { device_printf(sc->iommu.dev, "unable to setup interrupt (%d)\n", error); return (ENXIO); } bus_describe_intr(sc->iommu.dev, sc->irq_cmdev, sc->irq_cmdev_cookie, "cmdev"); if (x2apic_mode) { AMDIOMMU_LOCK(sc); sc->hw_ctrl |= AMDIOMMU_CTRL_GA_EN | AMDIOMMU_CTRL_XT_EN; amdiommu_write8(sc, AMDIOMMU_CTRL, sc->hw_ctrl); // XXXKIB AMDIOMMU_CTRL_INTCAPXT_EN and program x2APIC_CTRL AMDIOMMU_UNLOCK(sc); } return (0); } static int amdiommu_probe(device_t dev) { int seccap_reg; int error; uint32_t cap_h, cap_type, cap_rev; if (acpi_disabled("amdiommu")) return (ENXIO); TUNABLE_INT_FETCH("hw.amdiommu.enable", &amdiommu_enable); if (!amdiommu_enable) return (ENXIO); if (pci_get_class(dev) != PCIC_BASEPERIPH || pci_get_subclass(dev) != PCIS_BASEPERIPH_IOMMU) return (ENXIO); error = pci_find_cap(dev, PCIY_SECDEV, &seccap_reg); if (error != 0 || seccap_reg == 0) return (ENXIO); cap_h = pci_read_config(dev, seccap_reg + PCIR_AMDIOMMU_CAP_HEADER, 4); cap_type = cap_h & PCIM_AMDIOMMU_CAP_TYPE_MASK; cap_rev = cap_h & PCIM_AMDIOMMU_CAP_REV_MASK; if (cap_type != PCIM_AMDIOMMU_CAP_TYPE_VAL && cap_rev != PCIM_AMDIOMMU_CAP_REV_VAL) return (ENXIO); device_set_desc(dev, "DMA remap"); return (BUS_PROBE_SPECIFIC); } static int amdiommu_attach(device_t dev) { struct amdiommu_unit *sc; struct ivhd_lookup_data ild; int error; uint32_t base_low, base_high; bool res; sc = device_get_softc(dev); sc->iommu.unit = device_get_unit(dev); sc->iommu.dev = dev; error = pci_find_cap(dev, PCIY_SECDEV, &sc->seccap_reg); if (error != 0 || sc->seccap_reg == 0) return (ENXIO); base_low = pci_read_config(dev, sc->seccap_reg + PCIR_AMDIOMMU_BASE_LOW, 4); base_high = pci_read_config(dev, sc->seccap_reg + PCIR_AMDIOMMU_BASE_HIGH, 4); sc->mmio_base = (base_low & PCIM_AMDIOMMU_BASE_LOW_ADDRM) | ((uint64_t)base_high << 32); sc->device_id = pci_get_rid(dev); ild.sc = sc; ild.devid = sc->device_id; res = amdiommu_ivrs_iterate_tbl(ivrs_lookup_ivhd_0x40, ivrs_lookup_ivhd_0x40, ivrs_lookup_ivhd_0x10, &ild); if (!res) { device_printf(dev, "Cannot find IVHD\n"); return (ENXIO); } mtx_init(&sc->iommu.lock, "amdihw", NULL, MTX_DEF); sc->domids = new_unrhdr(0, 0xffff, &sc->iommu.lock); LIST_INIT(&sc->domains); sysctl_ctx_init(&sc->iommu.sysctl_ctx); sc->mmio_sz = ((sc->efr & AMDIOMMU_EFR_PC_SUP) != 0 ? 512 : 16) * 1024; sc->mmio_rid = AMDIOMMU_RID; error = bus_set_resource(dev, SYS_RES_MEMORY, AMDIOMMU_RID, sc->mmio_base, sc->mmio_sz); if (error != 0) { device_printf(dev, "bus_set_resource %#jx-%#jx failed, error %d\n", (uintmax_t)sc->mmio_base, (uintmax_t)sc->mmio_base + sc->mmio_sz, error); error = ENXIO; goto errout1; } sc->mmio_res = bus_alloc_resource(dev, SYS_RES_MEMORY, &sc->mmio_rid, sc->mmio_base, sc->mmio_base + sc->mmio_sz - 1, sc->mmio_sz, - RF_ALLOCATED | RF_ACTIVE | RF_SHAREABLE); + RF_ACTIVE | RF_SHAREABLE); if (sc->mmio_res == NULL) { device_printf(dev, "bus_alloc_resource %#jx-%#jx failed\n", (uintmax_t)sc->mmio_base, (uintmax_t)sc->mmio_base + sc->mmio_sz); error = ENXIO; goto errout2; } sc->hw_ctrl = amdiommu_read8(sc, AMDIOMMU_CTRL); if (bootverbose) device_printf(dev, "ctrl reg %#jx\n", (uintmax_t)sc->hw_ctrl); if ((sc->hw_ctrl & AMDIOMMU_CTRL_EN) != 0) { device_printf(dev, "CTRL_EN is set, bailing out\n"); error = EBUSY; goto errout2; } iommu_high = BUS_SPACE_MAXADDR; error = amdiommu_create_dev_tbl(sc); if (error != 0) goto errout3; error = amdiommu_init_cmd(sc); if (error != 0) goto errout4; error = amdiommu_init_event(sc); if (error != 0) goto errout5; error = amdiommu_setup_intr(sc); if (error != 0) goto errout6; error = iommu_init_busdma(AMD2IOMMU(sc)); if (error != 0) goto errout7; error = amdiommu_init_irt(sc); if (error != 0) goto errout8; /* * Unlike DMAR, AMD IOMMU does not process command queue * unless IOMMU is enabled. But since non-present devtab * entry makes IOMMU ignore transactions from corresponding * initiator, de-facto IOMMU operations are disabled for the * DMA and intr remapping. */ AMDIOMMU_LOCK(sc); sc->hw_ctrl |= AMDIOMMU_CTRL_EN; amdiommu_write8(sc, AMDIOMMU_CTRL, sc->hw_ctrl); if (bootverbose) { printf("amdiommu%d: enabled translation\n", AMD2IOMMU(sc)->unit); } AMDIOMMU_UNLOCK(sc); TAILQ_INSERT_TAIL(&amdiommu_units, sc, unit_next); amdiommu_running = true; return (0); errout8: iommu_fini_busdma(&sc->iommu); errout7: pci_release_msi(dev); errout6: amdiommu_fini_event(sc); errout5: amdiommu_fini_cmd(sc); errout4: amdiommu_free_dev_tbl(sc); errout3: bus_release_resource(dev, SYS_RES_MEMORY, sc->mmio_rid, sc->mmio_res); errout2: bus_delete_resource(dev, SYS_RES_MEMORY, sc->mmio_rid); errout1: sysctl_ctx_free(&sc->iommu.sysctl_ctx); delete_unrhdr(sc->domids); mtx_destroy(&sc->iommu.lock); return (error); } static int amdiommu_detach(device_t dev) { return (EBUSY); } static int amdiommu_suspend(device_t dev) { /* XXXKIB */ return (0); } static int amdiommu_resume(device_t dev) { /* XXXKIB */ return (0); } static device_method_t amdiommu_methods[] = { DEVMETHOD(device_probe, amdiommu_probe), DEVMETHOD(device_attach, amdiommu_attach), DEVMETHOD(device_detach, amdiommu_detach), DEVMETHOD(device_suspend, amdiommu_suspend), DEVMETHOD(device_resume, amdiommu_resume), DEVMETHOD_END }; static driver_t amdiommu_driver = { "amdiommu", amdiommu_methods, sizeof(struct amdiommu_unit), }; EARLY_DRIVER_MODULE(amdiommu, pci, amdiommu_driver, 0, 0, BUS_PASS_SUPPORTDEV); MODULE_DEPEND(amdiommu, pci, 1, 1, 1); int amdiommu_is_running(void) { return (amdiommu_running ? 0 : ENXIO); } static struct amdiommu_unit * amdiommu_unit_by_device_id(u_int pci_seg, u_int device_id) { struct amdiommu_unit *unit; TAILQ_FOREACH(unit, &amdiommu_units, unit_next) { if (unit->unit_dom == pci_seg && unit->device_id == device_id) return (unit); } return (NULL); } struct ivhd_find_unit { u_int domain; uintptr_t rid; int devno; enum { IFU_DEV_PCI, IFU_DEV_IOAPIC, IFU_DEV_HPET, } type; u_int device_id; uint16_t rid_real; uint8_t dte; uint32_t edte; }; static bool amdiommu_find_unit_scan_ivrs(ACPI_IVRS_DE_HEADER *d, size_t tlen, struct ivhd_find_unit *ifu) { char *db, *de; size_t len; for (de = (char *)d + tlen; (char *)d < de; d = (ACPI_IVRS_DE_HEADER *)(db + len)) { db = (char *)d; if (d->Type == ACPI_IVRS_TYPE_PAD4) { len = sizeof(ACPI_IVRS_DEVICE4); } else if (d->Type == ACPI_IVRS_TYPE_ALL) { ACPI_IVRS_DEVICE4 *d4; d4 = (ACPI_IVRS_DEVICE4 *)db; len = sizeof(*d4); ifu->dte = d4->Header.DataSetting; } else if (d->Type == ACPI_IVRS_TYPE_SELECT) { ACPI_IVRS_DEVICE4 *d4; d4 = (ACPI_IVRS_DEVICE4 *)db; if (d4->Header.Id == ifu->rid) { ifu->dte = d4->Header.DataSetting; ifu->rid_real = ifu->rid; return (true); } len = sizeof(*d4); } else if (d->Type == ACPI_IVRS_TYPE_START) { ACPI_IVRS_DEVICE4 *d4, *d4n; d4 = (ACPI_IVRS_DEVICE4 *)db; d4n = d4 + 1; if (d4n->Header.Type != ACPI_IVRS_TYPE_END) { printf("IVRS dev4 start not followed by END " "(%#x)\n", d4n->Header.Type); return (false); } if (d4->Header.Id <= ifu->rid && ifu->rid <= d4n->Header.Id) { ifu->dte = d4->Header.DataSetting; ifu->rid_real = ifu->rid; return (true); } len = 2 * sizeof(*d4); } else if (d->Type == ACPI_IVRS_TYPE_PAD8) { len = sizeof(ACPI_IVRS_DEVICE8A); } else if (d->Type == ACPI_IVRS_TYPE_ALIAS_SELECT) { ACPI_IVRS_DEVICE8A *d8a; d8a = (ACPI_IVRS_DEVICE8A *)db; if (d8a->Header.Id == ifu->rid) { ifu->dte = d8a->Header.DataSetting; ifu->rid_real = d8a->UsedId; return (true); } len = sizeof(*d8a); } else if (d->Type == ACPI_IVRS_TYPE_ALIAS_START) { ACPI_IVRS_DEVICE8A *d8a; ACPI_IVRS_DEVICE4 *d4; d8a = (ACPI_IVRS_DEVICE8A *)db; d4 = (ACPI_IVRS_DEVICE4 *)(d8a + 1); if (d4->Header.Type != ACPI_IVRS_TYPE_END) { printf("IVRS alias start not followed by END " "(%#x)\n", d4->Header.Type); return (false); } if (d8a->Header.Id <= ifu->rid && ifu->rid <= d4->Header.Id) { ifu->dte = d8a->Header.DataSetting; ifu->rid_real = d8a->UsedId; return (true); } len = sizeof(*d8a) + sizeof(*d4); } else if (d->Type == ACPI_IVRS_TYPE_EXT_SELECT) { ACPI_IVRS_DEVICE8B *d8b; d8b = (ACPI_IVRS_DEVICE8B *)db; if (d8b->Header.Id == ifu->rid) { ifu->dte = d8b->Header.DataSetting; ifu->rid_real = ifu->rid; ifu->edte = d8b->ExtendedData; return (true); } len = sizeof(*d8b); } else if (d->Type == ACPI_IVRS_TYPE_EXT_START) { ACPI_IVRS_DEVICE8B *d8b; ACPI_IVRS_DEVICE4 *d4; d8b = (ACPI_IVRS_DEVICE8B *)db; d4 = (ACPI_IVRS_DEVICE4 *)(db + sizeof(*d8b)); if (d4->Header.Type != ACPI_IVRS_TYPE_END) { printf("IVRS ext start not followed by END " "(%#x)\n", d4->Header.Type); return (false); } if (d8b->Header.Id >= ifu->rid && ifu->rid <= d4->Header.Id) { ifu->dte = d8b->Header.DataSetting; ifu->rid_real = ifu->rid; ifu->edte = d8b->ExtendedData; return (true); } len = sizeof(*d8b) + sizeof(*d4); } else if (d->Type == ACPI_IVRS_TYPE_SPECIAL) { ACPI_IVRS_DEVICE8C *d8c; d8c = (ACPI_IVRS_DEVICE8C *)db; if (((ifu->type == IFU_DEV_IOAPIC && d8c->Variety == ACPI_IVHD_IOAPIC) || (ifu->type == IFU_DEV_HPET && d8c->Variety == ACPI_IVHD_HPET)) && ifu->devno == d8c->Handle) { ifu->dte = d8c->Header.DataSetting; ifu->rid_real = d8c->UsedId; return (true); } len = sizeof(*d8c); } else if (d->Type == ACPI_IVRS_TYPE_HID) { ACPI_IVRS_DEVICE_HID *dh; dh = (ACPI_IVRS_DEVICE_HID *)db; len = sizeof(*dh) + dh->UidLength; /* XXXKIB */ } else { #if 0 printf("amdiommu: unknown IVRS device entry type %#x\n", d->Type); #endif if (d->Type <= 63) len = sizeof(ACPI_IVRS_DEVICE4); else if (d->Type <= 127) len = sizeof(ACPI_IVRS_DEVICE8A); else { printf("amdiommu: abort, cannot " "advance iterator, item type %#x\n", d->Type); return (false); } } } return (false); } static bool amdiommu_find_unit_scan_0x11(ACPI_IVRS_HARDWARE2 *ivrs, void *arg) { struct ivhd_find_unit *ifu = arg; ACPI_IVRS_DE_HEADER *d; bool res; KASSERT(ivrs->Header.Type == ACPI_IVRS_TYPE_HARDWARE2 || ivrs->Header.Type == ACPI_IVRS_TYPE_HARDWARE3, ("Misparsed IVHD h2, ivrs type %#x", ivrs->Header.Type)); if (ifu->domain != ivrs->PciSegmentGroup) return (false); d = (ACPI_IVRS_DE_HEADER *)(ivrs + 1); res = amdiommu_find_unit_scan_ivrs(d, ivrs->Header.Length, ifu); if (res) ifu->device_id = ivrs->Header.DeviceId; return (res); } static bool amdiommu_find_unit_scan_0x10(ACPI_IVRS_HARDWARE1 *ivrs, void *arg) { struct ivhd_find_unit *ifu = arg; ACPI_IVRS_DE_HEADER *d; bool res; KASSERT(ivrs->Header.Type == ACPI_IVRS_TYPE_HARDWARE1, ("Misparsed IVHD h1, ivrs type %#x", ivrs->Header.Type)); if (ifu->domain != ivrs->PciSegmentGroup) return (false); d = (ACPI_IVRS_DE_HEADER *)(ivrs + 1); res = amdiommu_find_unit_scan_ivrs(d, ivrs->Header.Length, ifu); if (res) ifu->device_id = ivrs->Header.DeviceId; return (res); } static void amdiommu_dev_prop_dtr(device_t dev, const char *name, void *val, void *dtr_ctx) { free(val, M_DEVBUF); } static int * amdiommu_dev_fetch_flagsp(struct amdiommu_unit *unit, device_t dev) { int *flagsp, error; bus_topo_assert(); error = device_get_prop(dev, device_get_nameunit(unit->iommu.dev), (void **)&flagsp); if (error == ENOENT) { flagsp = malloc(sizeof(int), M_DEVBUF, M_WAITOK | M_ZERO); device_set_prop(dev, device_get_nameunit(unit->iommu.dev), flagsp, amdiommu_dev_prop_dtr, unit); } return (flagsp); } static int amdiommu_get_dev_prop_flags(struct amdiommu_unit *unit, device_t dev) { int *flagsp, flags; bus_topo_lock(); flagsp = amdiommu_dev_fetch_flagsp(unit, dev); flags = *flagsp; bus_topo_unlock(); return (flags); } static void amdiommu_set_dev_prop_flags(struct amdiommu_unit *unit, device_t dev, int flag) { int *flagsp; bus_topo_lock(); flagsp = amdiommu_dev_fetch_flagsp(unit, dev); *flagsp |= flag; bus_topo_unlock(); } int amdiommu_find_unit(device_t dev, struct amdiommu_unit **unitp, uint16_t *ridp, uint8_t *dtep, uint32_t *edtep, bool verbose) { struct ivhd_find_unit ifu; struct amdiommu_unit *unit; int error, flags; bool res; if (!amdiommu_enable) return (ENXIO); if (device_get_devclass(device_get_parent(dev)) != devclass_find("pci")) return (ENXIO); bzero(&ifu, sizeof(ifu)); ifu.type = IFU_DEV_PCI; error = pci_get_id(dev, PCI_ID_RID, &ifu.rid); if (error != 0) { if (verbose) device_printf(dev, "amdiommu cannot get rid, error %d\n", error); return (ENXIO); } ifu.domain = pci_get_domain(dev); res = amdiommu_ivrs_iterate_tbl(amdiommu_find_unit_scan_0x11, amdiommu_find_unit_scan_0x11, amdiommu_find_unit_scan_0x10, &ifu); if (!res) { if (verbose) device_printf(dev, "(%#06x:%#06x) amdiommu cannot match rid in IVHD\n", ifu.domain, (unsigned)ifu.rid); return (ENXIO); } unit = amdiommu_unit_by_device_id(ifu.domain, ifu.device_id); if (unit == NULL) { if (verbose) device_printf(dev, "(%#06x:%#06x) amdiommu cannot find unit\n", ifu.domain, (unsigned)ifu.rid); return (ENXIO); } *unitp = unit; iommu_device_set_iommu_prop(dev, unit->iommu.dev); if (ridp != NULL) *ridp = ifu.rid_real; if (dtep != NULL) *dtep = ifu.dte; if (edtep != NULL) *edtep = ifu.edte; if (verbose) { flags = amdiommu_get_dev_prop_flags(unit, dev); if ((flags & AMDIOMMU_DEV_REPORTED) == 0) { amdiommu_set_dev_prop_flags(unit, dev, AMDIOMMU_DEV_REPORTED); device_printf(dev, "amdiommu%d " "initiator rid %#06x dte %#x edte %#x\n", unit->iommu.unit, ifu.rid_real, ifu.dte, ifu.edte); } } return (0); } int amdiommu_find_unit_for_ioapic(int apic_id, struct amdiommu_unit **unitp, uint16_t *ridp, uint8_t *dtep, uint32_t *edtep, bool verbose) { struct ivhd_find_unit ifu; struct amdiommu_unit *unit; device_t apic_dev; bool res; if (!amdiommu_enable) return (ENXIO); bzero(&ifu, sizeof(ifu)); ifu.type = IFU_DEV_IOAPIC; ifu.devno = apic_id; ifu.rid = -1; res = amdiommu_ivrs_iterate_tbl(amdiommu_find_unit_scan_0x11, amdiommu_find_unit_scan_0x11, amdiommu_find_unit_scan_0x10, &ifu); if (!res) { if (verbose) printf("amdiommu cannot match ioapic no %d in IVHD\n", apic_id); return (ENXIO); } unit = amdiommu_unit_by_device_id(0, ifu.device_id); apic_dev = ioapic_get_dev(apic_id); if (apic_dev != NULL) iommu_device_set_iommu_prop(apic_dev, unit->iommu.dev); if (unit == NULL) { if (verbose) printf("amdiommu cannot find unit by dev id %#x\n", ifu.device_id); return (ENXIO); } *unitp = unit; if (ridp != NULL) *ridp = ifu.rid_real; if (dtep != NULL) *dtep = ifu.dte; if (edtep != NULL) *edtep = ifu.edte; if (verbose) { printf("amdiommu%d IOAPIC %d " "initiator rid %#06x dte %#x edte %#x\n", unit->iommu.unit, apic_id, ifu.rid_real, ifu.dte, ifu.edte); } return (0); } int amdiommu_find_unit_for_hpet(device_t hpet, struct amdiommu_unit **unitp, uint16_t *ridp, uint8_t *dtep, uint32_t *edtep, bool verbose) { struct ivhd_find_unit ifu; struct amdiommu_unit *unit; int hpet_no; bool res; if (!amdiommu_enable) return (ENXIO); hpet_no = hpet_get_uid(hpet); bzero(&ifu, sizeof(ifu)); ifu.type = IFU_DEV_HPET; ifu.devno = hpet_no; ifu.rid = -1; res = amdiommu_ivrs_iterate_tbl(amdiommu_find_unit_scan_0x11, amdiommu_find_unit_scan_0x11, amdiommu_find_unit_scan_0x10, &ifu); if (!res) { if (verbose) printf("amdiommu cannot match hpet no %d in IVHD\n", hpet_no); return (ENXIO); } unit = amdiommu_unit_by_device_id(0, ifu.device_id); if (unit == NULL) { if (verbose) printf("amdiommu cannot find unit id %d\n", hpet_no); return (ENXIO); } *unitp = unit; iommu_device_set_iommu_prop(hpet, unit->iommu.dev); if (ridp != NULL) *ridp = ifu.rid_real; if (dtep != NULL) *dtep = ifu.dte; if (edtep != NULL) *edtep = ifu.edte; if (verbose) { printf("amdiommu%d HPET no %d " "initiator rid %#06x dte %#x edte %#x\n", unit->iommu.unit, hpet_no, ifu.rid_real, ifu.dte, ifu.edte); } return (0); } static struct iommu_unit * amdiommu_find_method(device_t dev, bool verbose) { struct amdiommu_unit *unit; int error; uint32_t edte; uint16_t rid; uint8_t dte; error = amdiommu_find_unit(dev, &unit, &rid, &dte, &edte, verbose); if (error != 0) { if (verbose && amdiommu_enable) device_printf(dev, "cannot find amdiommu unit, error %d\n", error); return (NULL); } return (&unit->iommu); } static struct x86_unit_common * amdiommu_get_x86_common(struct iommu_unit *unit) { struct amdiommu_unit *iommu; iommu = IOMMU2AMD(unit); return (&iommu->x86c); } static void amdiommu_unit_pre_instantiate_ctx(struct iommu_unit *unit) { } static struct x86_iommu amd_x86_iommu = { .get_x86_common = amdiommu_get_x86_common, .unit_pre_instantiate_ctx = amdiommu_unit_pre_instantiate_ctx, .find = amdiommu_find_method, .domain_unload_entry = amdiommu_domain_unload_entry, .domain_unload = amdiommu_domain_unload, .get_ctx = amdiommu_get_ctx, .free_ctx_locked = amdiommu_free_ctx_locked_method, .alloc_msi_intr = amdiommu_alloc_msi_intr, .map_msi_intr = amdiommu_map_msi_intr, .unmap_msi_intr = amdiommu_unmap_msi_intr, .map_ioapic_intr = amdiommu_map_ioapic_intr, .unmap_ioapic_intr = amdiommu_unmap_ioapic_intr, }; static void x86_iommu_set_amd(void *arg __unused) { if (cpu_vendor_id == CPU_VENDOR_AMD) set_x86_iommu(&amd_x86_iommu); } SYSINIT(x86_iommu, SI_SUB_TUNABLES, SI_ORDER_ANY, x86_iommu_set_amd, NULL); #ifdef DDB #include #include static void amdiommu_print_domain(struct amdiommu_domain *domain, bool show_mappings) { struct iommu_domain *iodom; iodom = DOM2IODOM(domain); db_printf( " @%p dom %d pglvl %d end %jx refs %d\n" " ctx_cnt %d flags %x pgobj %p map_ents %u\n", domain, domain->domain, domain->pglvl, (uintmax_t)domain->iodom.end, domain->refs, domain->ctx_cnt, domain->iodom.flags, domain->pgtbl_obj, domain->iodom.entries_cnt); iommu_db_domain_print_contexts(iodom); if (show_mappings) iommu_db_domain_print_mappings(iodom); } static void amdiommu_print_one(struct amdiommu_unit *unit, bool show_domains, bool show_mappings, bool show_cmdq) { struct amdiommu_domain *domain; struct amdiommu_cmd_generic *cp; u_int cmd_head, cmd_tail, ci; cmd_head = amdiommu_read4(unit, AMDIOMMU_CMDBUF_HEAD); cmd_tail = amdiommu_read4(unit, AMDIOMMU_CMDBUF_TAIL); db_printf("amdiommu%d at %p, mmio at %#jx/sz %#jx\n", unit->iommu.unit, unit, (uintmax_t)unit->mmio_base, (uintmax_t)unit->mmio_sz); db_printf(" hw ctrl %#018jx cmdevst %#018jx\n", (uintmax_t)amdiommu_read8(unit, AMDIOMMU_CTRL), (uintmax_t)amdiommu_read8(unit, AMDIOMMU_CMDEV_STATUS)); db_printf(" devtbl at %p\n", unit->dev_tbl); db_printf(" hwseq at %p phys %#jx val %#jx\n", &unit->x86c.inv_waitd_seq_hw, pmap_kextract((vm_offset_t)&unit->x86c.inv_waitd_seq_hw), unit->x86c.inv_waitd_seq_hw); db_printf(" invq at %p base %#jx hw head/tail %#x/%#x\n", unit->x86c.inv_queue, (uintmax_t)amdiommu_read8(unit, AMDIOMMU_CMDBUF_BASE), cmd_head, cmd_tail); if (show_cmdq) { db_printf(" cmd q:\n"); for (ci = cmd_head; ci != cmd_tail;) { cp = (struct amdiommu_cmd_generic *)(unit-> x86c.inv_queue + ci); db_printf( " idx %#x op %#x %#010x %#010x %#010x %#010x\n", ci >> AMDIOMMU_CMD_SZ_SHIFT, cp->op, cp->w0, cp->ww1, cp->w2, cp->w3); ci += AMDIOMMU_CMD_SZ; if (ci == unit->x86c.inv_queue_size) ci = 0; } } if (show_domains) { db_printf(" domains:\n"); LIST_FOREACH(domain, &unit->domains, link) { amdiommu_print_domain(domain, show_mappings); if (db_pager_quit) break; } } } DB_SHOW_COMMAND(amdiommu, db_amdiommu_print) { struct amdiommu_unit *unit; bool show_domains, show_mappings, show_cmdq; show_domains = strchr(modif, 'd') != NULL; show_mappings = strchr(modif, 'm') != NULL; show_cmdq = strchr(modif, 'q') != NULL; if (!have_addr) { db_printf("usage: show amdiommu [/d] [/m] [/q] index\n"); return; } if ((vm_offset_t)addr < 0x10000) unit = amdiommu_unit_by_device_id(0, (u_int)addr); else unit = (struct amdiommu_unit *)addr; amdiommu_print_one(unit, show_domains, show_mappings, show_cmdq); } DB_SHOW_ALL_COMMAND(amdiommus, db_show_all_amdiommus) { struct amdiommu_unit *unit; bool show_domains, show_mappings, show_cmdq; show_domains = strchr(modif, 'd') != NULL; show_mappings = strchr(modif, 'm') != NULL; show_cmdq = strchr(modif, 'q') != NULL; TAILQ_FOREACH(unit, &amdiommu_units, unit_next) { amdiommu_print_one(unit, show_domains, show_mappings, show_cmdq); if (db_pager_quit) break; } } #endif