Index: head/sys/mips/nlm/xlp_pci.c =================================================================== --- head/sys/mips/nlm/xlp_pci.c (revision 314316) +++ head/sys/mips/nlm/xlp_pci.c (revision 314317) @@ -1,576 +1,577 @@ /*- * Copyright (c) 2003-2012 Broadcom Corporation * All Rights Reserved * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in * the documentation and/or other materials provided with the * distribution. * * THIS SOFTWARE IS PROVIDED BY BROADCOM ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL BROADCOM OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "pcib_if.h" +#include #include "pci_if.h" static int xlp_pci_attach(device_t dev) { struct pci_devinfo *dinfo; device_t pcib; int maxslots, s, f, pcifunchigh, irq; int busno, node, devoffset; uint16_t devid; uint8_t hdrtype; /* * The on-chip devices are on a bus that is almost, but not * quite, completely like PCI. Add those things by hand. */ pcib = device_get_parent(dev); busno = pcib_get_bus(dev); maxslots = PCIB_MAXSLOTS(pcib); for (s = 0; s <= maxslots; s++) { pcifunchigh = 0; f = 0; hdrtype = PCIB_READ_CONFIG(pcib, busno, s, f, PCIR_HDRTYPE, 1); if ((hdrtype & PCIM_HDRTYPE) > PCI_MAXHDRTYPE) continue; if (hdrtype & PCIM_MFDEV) pcifunchigh = PCI_FUNCMAX; node = s / 8; for (f = 0; f <= pcifunchigh; f++) { devoffset = XLP_HDR_OFFSET(node, 0, s % 8, f); if (!nlm_dev_exists(devoffset)) continue; /* Find if there is a desc for the SoC device */ devid = PCIB_READ_CONFIG(pcib, busno, s, f, PCIR_DEVICE, 2); /* Skip devices that don't have a proper PCI header */ switch (devid) { case PCI_DEVICE_ID_NLM_ICI: case PCI_DEVICE_ID_NLM_PIC: case PCI_DEVICE_ID_NLM_FMN: case PCI_DEVICE_ID_NLM_UART: case PCI_DEVICE_ID_NLM_I2C: case PCI_DEVICE_ID_NLM_NOR: case PCI_DEVICE_ID_NLM_MMC: continue; case PCI_DEVICE_ID_NLM_EHCI: irq = PIC_USB_IRQ(f); PCIB_WRITE_CONFIG(pcib, busno, s, f, XLP_PCI_DEVSCRATCH_REG0 << 2, (1 << 8) | irq, 4); } dinfo = pci_read_device(pcib, dev, pcib_get_domain(dev), busno, s, f); pci_add_child(dev, dinfo); } } return (bus_generic_attach(dev)); } static int xlp_pci_probe(device_t dev) { device_t pcib; pcib = device_get_parent(dev); /* * Only the top level bus has SoC devices, leave the rest to * Generic PCI code */ if (strcmp(device_get_nameunit(pcib), "pcib0") != 0) return (ENXIO); device_set_desc(dev, "XLP SoCbus"); return (BUS_PROBE_DEFAULT); } static devclass_t pci_devclass; static device_method_t xlp_pci_methods[] = { /* Device interface */ DEVMETHOD(device_probe, xlp_pci_probe), DEVMETHOD(device_attach, xlp_pci_attach), DEVMETHOD(bus_rescan, bus_null_rescan), DEVMETHOD_END }; DEFINE_CLASS_1(pci, xlp_pci_driver, xlp_pci_methods, sizeof(struct pci_softc), pci_driver); DRIVER_MODULE(xlp_pci, pcib, xlp_pci_driver, pci_devclass, 0, 0); static int xlp_pcib_probe(device_t dev) { if (ofw_bus_is_compatible(dev, "netlogic,xlp-pci")) { device_set_desc(dev, "XLP PCI bus"); return (BUS_PROBE_DEFAULT); } return (ENXIO); } static int xlp_pcib_read_ivar(device_t dev, device_t child, int which, uintptr_t *result) { switch (which) { case PCIB_IVAR_DOMAIN: *result = 0; return (0); case PCIB_IVAR_BUS: *result = 0; return (0); } return (ENOENT); } static int xlp_pcib_write_ivar(device_t dev, device_t child, int which, uintptr_t result) { switch (which) { case PCIB_IVAR_DOMAIN: return (EINVAL); case PCIB_IVAR_BUS: return (EINVAL); } return (ENOENT); } static int xlp_pcib_maxslots(device_t dev) { return (PCI_SLOTMAX); } static u_int32_t xlp_pcib_read_config(device_t dev, u_int b, u_int s, u_int f, u_int reg, int width) { uint32_t data = 0; uint64_t cfgaddr; int regindex = reg/sizeof(uint32_t); cfgaddr = nlm_pcicfg_base(XLP_HDR_OFFSET(0, b, s, f)); if ((width == 2) && (reg & 1)) return 0xFFFFFFFF; else if ((width == 4) && (reg & 3)) return 0xFFFFFFFF; /* * The intline and int pin of SoC devices are DOA, except * for bridges (slot %8 == 1). * use the values we stashed in a writable PCI scratch reg. */ if (b == 0 && regindex == 0xf && s % 8 > 1) regindex = XLP_PCI_DEVSCRATCH_REG0; data = nlm_read_pci_reg(cfgaddr, regindex); if (width == 1) return ((data >> ((reg & 3) << 3)) & 0xff); else if (width == 2) return ((data >> ((reg & 3) << 3)) & 0xffff); else return (data); } static void xlp_pcib_write_config(device_t dev, u_int b, u_int s, u_int f, u_int reg, u_int32_t val, int width) { uint64_t cfgaddr; uint32_t data = 0; int regindex = reg / sizeof(uint32_t); cfgaddr = nlm_pcicfg_base(XLP_HDR_OFFSET(0, b, s, f)); if ((width == 2) && (reg & 1)) return; else if ((width == 4) && (reg & 3)) return; if (width == 1) { data = nlm_read_pci_reg(cfgaddr, regindex); data = (data & ~(0xff << ((reg & 3) << 3))) | (val << ((reg & 3) << 3)); } else if (width == 2) { data = nlm_read_pci_reg(cfgaddr, regindex); data = (data & ~(0xffff << ((reg & 3) << 3))) | (val << ((reg & 3) << 3)); } else { data = val; } /* * use shadow reg for intpin/intline which are dead */ if (b == 0 && regindex == 0xf && s % 8 > 1) regindex = XLP_PCI_DEVSCRATCH_REG0; nlm_write_pci_reg(cfgaddr, regindex, data); } /* * Enable byte swap in hardware when compiled big-endian. * Programs a link's PCIe SWAP regions from the link's IO and MEM address * ranges. */ static void xlp_pcib_hardware_swap_enable(int node, int link) { #if BYTE_ORDER == BIG_ENDIAN uint64_t bbase, linkpcibase; uint32_t bar; int pcieoffset; pcieoffset = XLP_IO_PCIE_OFFSET(node, link); if (!nlm_dev_exists(pcieoffset)) return; bbase = nlm_get_bridge_regbase(node); linkpcibase = nlm_pcicfg_base(pcieoffset); bar = nlm_read_bridge_reg(bbase, BRIDGE_PCIEMEM_BASE0 + link); nlm_write_pci_reg(linkpcibase, PCIE_BYTE_SWAP_MEM_BASE, bar); bar = nlm_read_bridge_reg(bbase, BRIDGE_PCIEMEM_LIMIT0 + link); nlm_write_pci_reg(linkpcibase, PCIE_BYTE_SWAP_MEM_LIM, bar | 0xFFF); bar = nlm_read_bridge_reg(bbase, BRIDGE_PCIEIO_BASE0 + link); nlm_write_pci_reg(linkpcibase, PCIE_BYTE_SWAP_IO_BASE, bar); bar = nlm_read_bridge_reg(bbase, BRIDGE_PCIEIO_LIMIT0 + link); nlm_write_pci_reg(linkpcibase, PCIE_BYTE_SWAP_IO_LIM, bar | 0xFFF); #endif } static int xlp_pcib_attach(device_t dev) { int node, link; /* enable hardware swap on all nodes/links */ for (node = 0; node < XLP_MAX_NODES; node++) for (link = 0; link < 4; link++) xlp_pcib_hardware_swap_enable(node, link); device_add_child(dev, "pci", -1); bus_generic_attach(dev); return (0); } /* * XLS PCIe can have upto 4 links, and each link has its on IRQ * Find the link on which the device is on */ static int xlp_pcie_link(device_t pcib, device_t dev) { device_t parent, tmp; /* find the lane on which the slot is connected to */ tmp = dev; while (1) { parent = device_get_parent(tmp); if (parent == NULL || parent == pcib) { device_printf(dev, "Cannot find parent bus\n"); return (-1); } if (strcmp(device_get_nameunit(parent), "pci0") == 0) break; tmp = parent; } return (pci_get_function(tmp)); } static int xlp_alloc_msi(device_t pcib, device_t dev, int count, int maxcount, int *irqs) { int i, link; /* * Each link has 32 MSIs that can be allocated, but for now * we only support one device per link. * msi_alloc() equivalent is needed when we start supporting * bridges on the PCIe link. */ link = xlp_pcie_link(pcib, dev); if (link == -1) return (ENXIO); /* * encode the irq so that we know it is a MSI interrupt when we * setup interrupts */ for (i = 0; i < count; i++) irqs[i] = 64 + link * 32 + i; return (0); } static int xlp_release_msi(device_t pcib, device_t dev, int count, int *irqs) { return (0); } static int xlp_map_msi(device_t pcib, device_t dev, int irq, uint64_t *addr, uint32_t *data) { int link; if (irq < 64) { device_printf(dev, "%s: map_msi for irq %d - ignored", device_get_nameunit(pcib), irq); return (ENXIO); } link = (irq - 64) / 32; *addr = MIPS_MSI_ADDR(0); *data = MIPS_MSI_DATA(PIC_PCIE_IRQ(link)); return (0); } static void bridge_pcie_ack(int irq, void *arg) { uint32_t node,reg; uint64_t base; node = nlm_nodeid(); reg = PCIE_MSI_STATUS; switch (irq) { case PIC_PCIE_0_IRQ: base = nlm_pcicfg_base(XLP_IO_PCIE0_OFFSET(node)); break; case PIC_PCIE_1_IRQ: base = nlm_pcicfg_base(XLP_IO_PCIE1_OFFSET(node)); break; case PIC_PCIE_2_IRQ: base = nlm_pcicfg_base(XLP_IO_PCIE2_OFFSET(node)); break; case PIC_PCIE_3_IRQ: base = nlm_pcicfg_base(XLP_IO_PCIE3_OFFSET(node)); break; default: return; } nlm_write_pci_reg(base, reg, 0xFFFFFFFF); return; } static int mips_platform_pcib_setup_intr(device_t dev, device_t child, struct resource *irq, int flags, driver_filter_t *filt, driver_intr_t *intr, void *arg, void **cookiep) { int error = 0; int xlpirq; error = rman_activate_resource(irq); if (error) return error; if (rman_get_start(irq) != rman_get_end(irq)) { device_printf(dev, "Interrupt allocation %ju != %ju\n", rman_get_start(irq), rman_get_end(irq)); return (EINVAL); } xlpirq = rman_get_start(irq); if (xlpirq == 0) return (0); if (strcmp(device_get_name(dev), "pcib") != 0) return (0); /* * temporary hack for MSI, we support just one device per * link, and assign the link interrupt to the device interrupt */ if (xlpirq >= 64) { int node, val, link; uint64_t base; xlpirq -= 64; if (xlpirq % 32 != 0) return (0); node = nlm_nodeid(); link = xlpirq / 32; base = nlm_pcicfg_base(XLP_IO_PCIE_OFFSET(node,link)); /* MSI Interrupt Vector enable at bridge's configuration */ nlm_write_pci_reg(base, PCIE_MSI_EN, PCIE_MSI_VECTOR_INT_EN); val = nlm_read_pci_reg(base, PCIE_INT_EN0); /* MSI Interrupt enable at bridge's configuration */ nlm_write_pci_reg(base, PCIE_INT_EN0, (val | PCIE_MSI_INT_EN)); /* legacy interrupt disable at bridge */ val = nlm_read_pci_reg(base, PCIE_BRIDGE_CMD); nlm_write_pci_reg(base, PCIE_BRIDGE_CMD, (val | PCIM_CMD_INTxDIS)); /* MSI address update at bridge */ nlm_write_pci_reg(base, PCIE_BRIDGE_MSI_ADDRL, MSI_MIPS_ADDR_BASE); nlm_write_pci_reg(base, PCIE_BRIDGE_MSI_ADDRH, 0); val = nlm_read_pci_reg(base, PCIE_BRIDGE_MSI_CAP); /* MSI capability enable at bridge */ nlm_write_pci_reg(base, PCIE_BRIDGE_MSI_CAP, (val | (PCIM_MSICTRL_MSI_ENABLE << 16) | (PCIM_MSICTRL_MMC_32 << 16))); xlpirq = PIC_PCIE_IRQ(link); } /* if it is for real PCIe, we need to ack at bridge too */ if (xlpirq >= PIC_PCIE_IRQ(0) && xlpirq <= PIC_PCIE_IRQ(3)) xlp_set_bus_ack(xlpirq, bridge_pcie_ack, NULL); cpu_establish_hardintr(device_get_name(child), filt, intr, arg, xlpirq, flags, cookiep); return (0); } static int mips_platform_pcib_teardown_intr(device_t dev, device_t child, struct resource *irq, void *cookie) { if (strcmp(device_get_name(child), "pci") == 0) { /* if needed reprogram the pic to clear pcix related entry */ device_printf(dev, "teardown intr\n"); } return (bus_generic_teardown_intr(dev, child, irq, cookie)); } static int mips_pcib_route_interrupt(device_t bus, device_t dev, int pin) { int f, d; /* * Validate requested pin number. */ if ((pin < 1) || (pin > 4)) return (255); if (pci_get_bus(dev) == 0 && pci_get_vendor(dev) == PCI_VENDOR_NETLOGIC) { f = pci_get_function(dev); d = pci_get_slot(dev) % 8; /* * For PCIe links, return link IRT, for other SoC devices * get the IRT from its PCIe header */ if (d == 1) return (PIC_PCIE_IRQ(f)); else return (255); /* use intline, don't reroute */ } else { /* Regular PCI devices */ return (PIC_PCIE_IRQ(xlp_pcie_link(bus, dev))); } } static device_method_t xlp_pcib_methods[] = { /* Device interface */ DEVMETHOD(device_probe, xlp_pcib_probe), DEVMETHOD(device_attach, xlp_pcib_attach), /* Bus interface */ DEVMETHOD(bus_read_ivar, xlp_pcib_read_ivar), DEVMETHOD(bus_write_ivar, xlp_pcib_write_ivar), DEVMETHOD(bus_alloc_resource, bus_generic_alloc_resource), DEVMETHOD(bus_release_resource, bus_generic_release_resource), DEVMETHOD(bus_activate_resource, bus_generic_activate_resource), DEVMETHOD(bus_deactivate_resource, bus_generic_deactivate_resource), DEVMETHOD(bus_setup_intr, mips_platform_pcib_setup_intr), DEVMETHOD(bus_teardown_intr, mips_platform_pcib_teardown_intr), /* pcib interface */ DEVMETHOD(pcib_maxslots, xlp_pcib_maxslots), DEVMETHOD(pcib_read_config, xlp_pcib_read_config), DEVMETHOD(pcib_write_config, xlp_pcib_write_config), DEVMETHOD(pcib_route_interrupt, mips_pcib_route_interrupt), DEVMETHOD(pcib_request_feature, pcib_request_feature_allow), DEVMETHOD(pcib_alloc_msi, xlp_alloc_msi), DEVMETHOD(pcib_release_msi, xlp_release_msi), DEVMETHOD(pcib_map_msi, xlp_map_msi), DEVMETHOD_END }; static driver_t xlp_pcib_driver = { "pcib", xlp_pcib_methods, 1, /* no softc */ }; static devclass_t pcib_devclass; DRIVER_MODULE(xlp_pcib, simplebus, xlp_pcib_driver, pcib_devclass, 0, 0); Index: head/sys/mips/rmi/xlr_pci.c =================================================================== --- head/sys/mips/rmi/xlr_pci.c (revision 314316) +++ head/sys/mips/rmi/xlr_pci.c (revision 314317) @@ -1,656 +1,657 @@ /*- * Copyright (c) 2003-2009 RMI Corporation * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of RMI Corporation, nor the names of its contributors, * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * RMI_BSD */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "pcib_if.h" +#include #define pci_cfg_offset(bus,slot,devfn,where) (((bus)<<16) + ((slot) << 11)+((devfn)<<8)+(where)) #define PCIE_LINK_STATE 0x4000 #define LSU_CFG0_REGID 0 #define LSU_CERRLOG_REGID 9 #define LSU_CERROVF_REGID 10 #define LSU_CERRINT_REGID 11 /* MSI support */ #define MSI_MIPS_ADDR_DEST 0x000ff000 #define MSI_MIPS_ADDR_RH 0x00000008 #define MSI_MIPS_ADDR_RH_OFF 0x00000000 #define MSI_MIPS_ADDR_RH_ON 0x00000008 #define MSI_MIPS_ADDR_DM 0x00000004 #define MSI_MIPS_ADDR_DM_PHYSICAL 0x00000000 #define MSI_MIPS_ADDR_DM_LOGICAL 0x00000004 /* Fields in data for Intel MSI messages. */ #define MSI_MIPS_DATA_TRGRMOD 0x00008000 /* Trigger mode */ #define MSI_MIPS_DATA_TRGREDG 0x00000000 /* edge */ #define MSI_MIPS_DATA_TRGRLVL 0x00008000 /* level */ #define MSI_MIPS_DATA_LEVEL 0x00004000 /* Polarity. */ #define MSI_MIPS_DATA_DEASSERT 0x00000000 #define MSI_MIPS_DATA_ASSERT 0x00004000 #define MSI_MIPS_DATA_DELMOD 0x00000700 /* Delivery Mode */ #define MSI_MIPS_DATA_DELFIXED 0x00000000 /* fixed */ #define MSI_MIPS_DATA_DELLOPRI 0x00000100 /* lowest priority */ #define MSI_MIPS_DATA_INTVEC 0x000000ff /* * Build Intel MSI message and data values from a source. AMD64 systems * seem to be compatible, so we use the same function for both. */ #define MIPS_MSI_ADDR(cpu) \ (MSI_MIPS_ADDR_BASE | (cpu) << 12 | \ MSI_MIPS_ADDR_RH_OFF | MSI_MIPS_ADDR_DM_PHYSICAL) #define MIPS_MSI_DATA(irq) \ (MSI_MIPS_DATA_TRGRLVL | MSI_MIPS_DATA_DELFIXED | \ MSI_MIPS_DATA_ASSERT | (irq)) struct xlr_pcib_softc { bus_dma_tag_t sc_pci_dmat; /* PCI DMA tag pointer */ }; static devclass_t pcib_devclass; static void *xlr_pci_config_base; static struct rman irq_rman, port_rman, mem_rman; static void xlr_pci_init_resources(void) { irq_rman.rm_start = 0; irq_rman.rm_end = 255; irq_rman.rm_type = RMAN_ARRAY; irq_rman.rm_descr = "PCI Mapped Interrupts"; if (rman_init(&irq_rman) || rman_manage_region(&irq_rman, 0, 255)) panic("pci_init_resources irq_rman"); port_rman.rm_type = RMAN_ARRAY; port_rman.rm_descr = "I/O ports"; if (rman_init(&port_rman) || rman_manage_region(&port_rman, 0x10000000, 0x1fffffff)) panic("pci_init_resources port_rman"); mem_rman.rm_type = RMAN_ARRAY; mem_rman.rm_descr = "I/O memory"; if (rman_init(&mem_rman) || rman_manage_region(&mem_rman, 0xd0000000, 0xdfffffff)) panic("pci_init_resources mem_rman"); } static int xlr_pcib_probe(device_t dev) { if (xlr_board_info.is_xls) device_set_desc(dev, "XLS PCIe bus"); else device_set_desc(dev, "XLR PCI bus"); xlr_pci_init_resources(); xlr_pci_config_base = (void *)MIPS_PHYS_TO_KSEG1(DEFAULT_PCI_CONFIG_BASE); return (0); } static int xlr_pcib_read_ivar(device_t dev, device_t child, int which, uintptr_t *result) { switch (which) { case PCIB_IVAR_DOMAIN: *result = 0; return (0); case PCIB_IVAR_BUS: *result = 0; return (0); } return (ENOENT); } static int xlr_pcib_write_ivar(device_t dev, device_t child, int which, uintptr_t result) { switch (which) { case PCIB_IVAR_DOMAIN: return (EINVAL); case PCIB_IVAR_BUS: return (EINVAL); } return (ENOENT); } static int xlr_pcib_maxslots(device_t dev) { return (PCI_SLOTMAX); } static __inline__ void disable_and_clear_cache_error(void) { uint64_t lsu_cfg0; lsu_cfg0 = read_xlr_ctrl_register(CPU_BLOCKID_LSU, LSU_CFG0_REGID); lsu_cfg0 = lsu_cfg0 & ~0x2e; write_xlr_ctrl_register(CPU_BLOCKID_LSU, LSU_CFG0_REGID, lsu_cfg0); /* Clear cache error log */ write_xlr_ctrl_register(CPU_BLOCKID_LSU, LSU_CERRLOG_REGID, 0); } static __inline__ void clear_and_enable_cache_error(void) { uint64_t lsu_cfg0 = 0; /* first clear the cache error logging register */ write_xlr_ctrl_register(CPU_BLOCKID_LSU, LSU_CERRLOG_REGID, 0); write_xlr_ctrl_register(CPU_BLOCKID_LSU, LSU_CERROVF_REGID, 0); write_xlr_ctrl_register(CPU_BLOCKID_LSU, LSU_CERRINT_REGID, 0); lsu_cfg0 = read_xlr_ctrl_register(CPU_BLOCKID_LSU, LSU_CFG0_REGID); lsu_cfg0 = lsu_cfg0 | 0x2e; write_xlr_ctrl_register(CPU_BLOCKID_LSU, LSU_CFG0_REGID, lsu_cfg0); } static uint32_t pci_cfg_read_32bit(uint32_t addr) { uint32_t temp = 0; uint32_t *p = (uint32_t *)xlr_pci_config_base + addr / sizeof(uint32_t); uint64_t cerr_cpu_log = 0; disable_and_clear_cache_error(); temp = bswap32(*p); /* Read cache err log */ cerr_cpu_log = read_xlr_ctrl_register(CPU_BLOCKID_LSU, LSU_CERRLOG_REGID); if (cerr_cpu_log) { /* Device don't exist. */ temp = ~0x0; } clear_and_enable_cache_error(); return (temp); } static u_int32_t xlr_pcib_read_config(device_t dev, u_int b, u_int s, u_int f, u_int reg, int width) { uint32_t data = 0; if ((width == 2) && (reg & 1)) return 0xFFFFFFFF; else if ((width == 4) && (reg & 3)) return 0xFFFFFFFF; data = pci_cfg_read_32bit(pci_cfg_offset(b, s, f, reg)); if (width == 1) return ((data >> ((reg & 3) << 3)) & 0xff); else if (width == 2) return ((data >> ((reg & 3) << 3)) & 0xffff); else return (data); } static void xlr_pcib_write_config(device_t dev, u_int b, u_int s, u_int f, u_int reg, u_int32_t val, int width) { uint32_t cfgaddr = pci_cfg_offset(b, s, f, reg); uint32_t data = 0, *p; if ((width == 2) && (reg & 1)) return; else if ((width == 4) && (reg & 3)) return; if (width == 1) { data = pci_cfg_read_32bit(cfgaddr); data = (data & ~(0xff << ((reg & 3) << 3))) | (val << ((reg & 3) << 3)); } else if (width == 2) { data = pci_cfg_read_32bit(cfgaddr); data = (data & ~(0xffff << ((reg & 3) << 3))) | (val << ((reg & 3) << 3)); } else { data = val; } p = (uint32_t *)xlr_pci_config_base + cfgaddr / sizeof(uint32_t); *p = bswap32(data); return; } static int xlr_pcib_attach(device_t dev) { struct xlr_pcib_softc *sc; sc = device_get_softc(dev); /* * XLR C revision chips cannot do DMA above 2G physical address * create a parent tag with this lowaddr */ if (xlr_is_c_revision()) { if (bus_dma_tag_create(bus_get_dma_tag(dev), 1, 0, 0x7fffffff, ~0, NULL, NULL, 0x7fffffff, 0xff, 0x7fffffff, 0, NULL, NULL, &sc->sc_pci_dmat) != 0) panic("%s: bus_dma_tag_create failed", __func__); } device_add_child(dev, "pci", -1); bus_generic_attach(dev); return (0); } static void xlr_pcib_identify(driver_t * driver, device_t parent) { BUS_ADD_CHILD(parent, 0, "pcib", 0); } /* * XLS PCIe can have upto 4 links, and each link has its on IRQ * Find the link on which the device is on */ static int xls_pcie_link(device_t pcib, device_t dev) { device_t parent, tmp; /* find the lane on which the slot is connected to */ printf("xls_pcie_link : bus %s dev %s\n", device_get_nameunit(pcib), device_get_nameunit(dev)); tmp = dev; while (1) { parent = device_get_parent(tmp); if (parent == NULL || parent == pcib) { device_printf(dev, "Cannot find parent bus\n"); return (-1); } if (strcmp(device_get_nameunit(parent), "pci0") == 0) break; tmp = parent; } return (pci_get_slot(tmp)); } /* * Find the IRQ for the link, each link has a different interrupt * at the XLS pic */ static int xls_pcie_link_irq(int link) { switch (link) { case 0: return (PIC_PCIE_LINK0_IRQ); case 1: return (PIC_PCIE_LINK1_IRQ); case 2: if (xlr_is_xls_b0()) return (PIC_PCIE_B0_LINK2_IRQ); else return (PIC_PCIE_LINK2_IRQ); case 3: if (xlr_is_xls_b0()) return (PIC_PCIE_B0_LINK3_IRQ); else return (PIC_PCIE_LINK3_IRQ); } return (-1); } static int xlr_alloc_msi(device_t pcib, device_t dev, int count, int maxcount, int *irqs) { int i, link; /* * Each link has 32 MSIs that can be allocated, but for now * we only support one device per link. * msi_alloc() equivalent is needed when we start supporting * bridges on the PCIe link. */ link = xls_pcie_link(pcib, dev); if (link == -1) return (ENXIO); /* * encode the irq so that we know it is a MSI interrupt when we * setup interrupts */ for (i = 0; i < count; i++) irqs[i] = 64 + link * 32 + i; return (0); } static int xlr_release_msi(device_t pcib, device_t dev, int count, int *irqs) { device_printf(dev, "%s: msi release %d\n", device_get_nameunit(pcib), count); return (0); } static int xlr_map_msi(device_t pcib, device_t dev, int irq, uint64_t *addr, uint32_t *data) { int msi; if (irq >= 64) { msi = irq - 64; *addr = MIPS_MSI_ADDR(0); *data = MIPS_MSI_DATA(msi); return (0); } else { device_printf(dev, "%s: map_msi for irq %d - ignored", device_get_nameunit(pcib), irq); return (ENXIO); } } static void bridge_pcix_ack(int irq) { (void)xlr_read_reg(xlr_io_mmio(XLR_IO_PCIX_OFFSET), 0x140 >> 2); } static void bridge_pcie_ack(int irq) { uint32_t reg; xlr_reg_t *pcie_mmio_le = xlr_io_mmio(XLR_IO_PCIE_1_OFFSET); switch (irq) { case PIC_PCIE_LINK0_IRQ: reg = PCIE_LINK0_MSI_STATUS; break; case PIC_PCIE_LINK1_IRQ: reg = PCIE_LINK1_MSI_STATUS; break; case PIC_PCIE_LINK2_IRQ: case PIC_PCIE_B0_LINK2_IRQ: reg = PCIE_LINK2_MSI_STATUS; break; case PIC_PCIE_LINK3_IRQ: case PIC_PCIE_B0_LINK3_IRQ: reg = PCIE_LINK3_MSI_STATUS; break; default: return; } xlr_write_reg(pcie_mmio_le, reg>>2, 0xffffffff); } static int mips_platform_pci_setup_intr(device_t dev, device_t child, struct resource *irq, int flags, driver_filter_t *filt, driver_intr_t *intr, void *arg, void **cookiep) { int error = 0; int xlrirq; error = rman_activate_resource(irq); if (error) return error; if (rman_get_start(irq) != rman_get_end(irq)) { device_printf(dev, "Interrupt allocation %ju != %ju\n", rman_get_start(irq), rman_get_end(irq)); return (EINVAL); } xlrirq = rman_get_start(irq); if (strcmp(device_get_name(dev), "pcib") != 0) return (0); if (xlr_board_info.is_xls == 0) { xlr_establish_intr(device_get_name(child), filt, intr, arg, PIC_PCIX_IRQ, flags, cookiep, bridge_pcix_ack); pic_setup_intr(PIC_IRT_PCIX_INDEX, PIC_PCIX_IRQ, 0x1, 1); } else { /* * temporary hack for MSI, we support just one device per * link, and assign the link interrupt to the device interrupt */ if (xlrirq >= 64) { xlrirq -= 64; if (xlrirq % 32 != 0) return (0); xlrirq = xls_pcie_link_irq(xlrirq / 32); if (xlrirq == -1) return (EINVAL); } xlr_establish_intr(device_get_name(child), filt, intr, arg, xlrirq, flags, cookiep, bridge_pcie_ack); pic_setup_intr(xlrirq - PIC_IRQ_BASE, xlrirq, 0x1, 1); } return (bus_generic_setup_intr(dev, child, irq, flags, filt, intr, arg, cookiep)); } static int mips_platform_pci_teardown_intr(device_t dev, device_t child, struct resource *irq, void *cookie) { if (strcmp(device_get_name(child), "pci") == 0) { /* if needed reprogram the pic to clear pcix related entry */ device_printf(dev, "teardown intr\n"); } return (bus_generic_teardown_intr(dev, child, irq, cookie)); } static struct resource * xlr_pci_alloc_resource(device_t bus, device_t child, int type, int *rid, rman_res_t start, rman_res_t end, rman_res_t count, u_int flags) { struct rman *rm; struct resource *rv; vm_offset_t va; int needactivate = flags & RF_ACTIVE; switch (type) { case SYS_RES_IRQ: rm = &irq_rman; break; case SYS_RES_IOPORT: rm = &port_rman; break; case SYS_RES_MEMORY: rm = &mem_rman; break; default: return (0); } rv = rman_reserve_resource(rm, start, end, count, flags, child); if (rv == NULL) return (0); rman_set_rid(rv, *rid); if (type == SYS_RES_MEMORY || type == SYS_RES_IOPORT) { va = (vm_offset_t)pmap_mapdev(start, count); rman_set_bushandle(rv, va); /* bushandle is same as virtual addr */ rman_set_virtual(rv, (void *)va); rman_set_bustag(rv, rmi_pci_bus_space); } if (needactivate) { if (bus_activate_resource(child, type, *rid, rv)) { rman_release_resource(rv); return (NULL); } } return (rv); } static int xlr_pci_release_resource(device_t bus, device_t child, int type, int rid, struct resource *r) { return (rman_release_resource(r)); } static bus_dma_tag_t xlr_pci_get_dma_tag(device_t bus, device_t child) { struct xlr_pcib_softc *sc; sc = device_get_softc(bus); return (sc->sc_pci_dmat); } static int xlr_pci_activate_resource(device_t bus, device_t child, int type, int rid, struct resource *r) { return (rman_activate_resource(r)); } static int xlr_pci_deactivate_resource(device_t bus, device_t child, int type, int rid, struct resource *r) { return (rman_deactivate_resource(r)); } static int mips_pci_route_interrupt(device_t bus, device_t dev, int pin) { int irq, link; /* * Validate requested pin number. */ if ((pin < 1) || (pin > 4)) return (255); if (xlr_board_info.is_xls) { link = xls_pcie_link(bus, dev); irq = xls_pcie_link_irq(link); if (irq != -1) return (irq); } else { if (pin == 1) return (PIC_PCIX_IRQ); } return (255); } static device_method_t xlr_pcib_methods[] = { /* Device interface */ DEVMETHOD(device_identify, xlr_pcib_identify), DEVMETHOD(device_probe, xlr_pcib_probe), DEVMETHOD(device_attach, xlr_pcib_attach), /* Bus interface */ DEVMETHOD(bus_read_ivar, xlr_pcib_read_ivar), DEVMETHOD(bus_write_ivar, xlr_pcib_write_ivar), DEVMETHOD(bus_alloc_resource, xlr_pci_alloc_resource), DEVMETHOD(bus_release_resource, xlr_pci_release_resource), DEVMETHOD(bus_get_dma_tag, xlr_pci_get_dma_tag), DEVMETHOD(bus_activate_resource, xlr_pci_activate_resource), DEVMETHOD(bus_deactivate_resource, xlr_pci_deactivate_resource), DEVMETHOD(bus_setup_intr, mips_platform_pci_setup_intr), DEVMETHOD(bus_teardown_intr, mips_platform_pci_teardown_intr), /* pcib interface */ DEVMETHOD(pcib_maxslots, xlr_pcib_maxslots), DEVMETHOD(pcib_read_config, xlr_pcib_read_config), DEVMETHOD(pcib_write_config, xlr_pcib_write_config), DEVMETHOD(pcib_route_interrupt, mips_pci_route_interrupt), DEVMETHOD(pcib_request_feature, pcib_request_feature_allow), DEVMETHOD(pcib_alloc_msi, xlr_alloc_msi), DEVMETHOD(pcib_release_msi, xlr_release_msi), DEVMETHOD(pcib_map_msi, xlr_map_msi), DEVMETHOD_END }; static driver_t xlr_pcib_driver = { "pcib", xlr_pcib_methods, sizeof(struct xlr_pcib_softc), }; DRIVER_MODULE(pcib, iodi, xlr_pcib_driver, pcib_devclass, 0, 0); Index: head/sys/powerpc/powermac/cpcht.c =================================================================== --- head/sys/powerpc/powermac/cpcht.c (revision 314316) +++ head/sys/powerpc/powermac/cpcht.c (revision 314317) @@ -1,736 +1,737 @@ /*- * Copyright (C) 2008-2010 Nathan Whitehorn * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL TOOLS GMBH BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "pcib_if.h" +#include #include "pic_if.h" /* * IBM CPC9X5 Hypertransport Device interface. */ static int cpcht_probe(device_t); static int cpcht_attach(device_t); static void cpcht_configure_htbridge(device_t, phandle_t); /* * pcib interface. */ static u_int32_t cpcht_read_config(device_t, u_int, u_int, u_int, u_int, int); static void cpcht_write_config(device_t, u_int, u_int, u_int, u_int, u_int32_t, int); static int cpcht_route_interrupt(device_t, device_t, int); static int cpcht_alloc_msi(device_t dev, device_t child, int count, int maxcount, int *irqs); static int cpcht_release_msi(device_t dev, device_t child, int count, int *irqs); static int cpcht_alloc_msix(device_t dev, device_t child, int *irq); static int cpcht_release_msix(device_t dev, device_t child, int irq); static int cpcht_map_msi(device_t dev, device_t child, int irq, uint64_t *addr, uint32_t *data); /* * Driver methods. */ static device_method_t cpcht_methods[] = { /* Device interface */ DEVMETHOD(device_probe, cpcht_probe), DEVMETHOD(device_attach, cpcht_attach), /* pcib interface */ DEVMETHOD(pcib_read_config, cpcht_read_config), DEVMETHOD(pcib_write_config, cpcht_write_config), DEVMETHOD(pcib_route_interrupt, cpcht_route_interrupt), DEVMETHOD(pcib_alloc_msi, cpcht_alloc_msi), DEVMETHOD(pcib_release_msi, cpcht_release_msi), DEVMETHOD(pcib_alloc_msix, cpcht_alloc_msix), DEVMETHOD(pcib_release_msix, cpcht_release_msix), DEVMETHOD(pcib_map_msi, cpcht_map_msi), DEVMETHOD(pcib_request_feature, pcib_request_feature_allow), DEVMETHOD_END }; struct cpcht_irq { enum { IRQ_NONE, IRQ_HT, IRQ_MSI, IRQ_INTERNAL } irq_type; int ht_source; vm_offset_t ht_base; vm_offset_t apple_eoi; uint32_t eoi_data; int edge; }; static struct cpcht_irq *cpcht_irqmap = NULL; uint32_t cpcht_msipic = 0; struct cpcht_softc { struct ofw_pci_softc pci_sc; vm_offset_t sc_data; uint64_t sc_populated_slots; struct cpcht_irq htirq_map[128]; struct mtx htirq_mtx; }; static devclass_t cpcht_devclass; DEFINE_CLASS_1(pcib, cpcht_driver, cpcht_methods, sizeof(struct cpcht_softc), ofw_pci_driver); DRIVER_MODULE(cpcht, ofwbus, cpcht_driver, cpcht_devclass, 0, 0); #define CPCHT_IOPORT_BASE 0xf4000000UL /* Hardwired */ #define CPCHT_IOPORT_SIZE 0x00400000UL #define HTAPIC_REQUEST_EOI 0x20 #define HTAPIC_TRIGGER_LEVEL 0x02 #define HTAPIC_MASK 0x01 static int cpcht_probe(device_t dev) { const char *type, *compatible; type = ofw_bus_get_type(dev); compatible = ofw_bus_get_compat(dev); if (type == NULL || compatible == NULL) return (ENXIO); if (strcmp(type, "ht") != 0) return (ENXIO); if (strcmp(compatible, "u3-ht") != 0) return (ENXIO); device_set_desc(dev, "IBM CPC9X5 HyperTransport Tunnel"); return (0); } static int cpcht_attach(device_t dev) { struct cpcht_softc *sc; phandle_t node, child; u_int32_t reg[3]; int i; node = ofw_bus_get_node(dev); sc = device_get_softc(dev); if (OF_getencprop(node, "reg", reg, sizeof(reg)) < 12) return (ENXIO); if (OF_getproplen(node, "ranges") <= 0) sc->pci_sc.sc_quirks = OFW_PCI_QUIRK_RANGES_ON_CHILDREN; sc->sc_populated_slots = 0; sc->sc_data = (vm_offset_t)pmap_mapdev(reg[1], reg[2]); /* * Set up the resource manager and the HT->MPIC mapping. For cpcht, * the ranges are properties of the child bridges, and this is also * where we get the HT interrupts properties. */ #if 0 /* I/O port mappings are usually not in the device tree */ rman_manage_region(&sc->pci_sc.sc_io_rman, 0, CPCHT_IOPORT_SIZE - 1); #endif bzero(sc->htirq_map, sizeof(sc->htirq_map)); mtx_init(&sc->htirq_mtx, "cpcht irq", NULL, MTX_DEF); for (i = 0; i < 8; i++) sc->htirq_map[i].irq_type = IRQ_INTERNAL; for (child = OF_child(node); child != 0; child = OF_peer(child)) cpcht_configure_htbridge(dev, child); /* Now make the mapping table available to the MPIC */ cpcht_irqmap = sc->htirq_map; return (ofw_pci_attach(dev)); } static void cpcht_configure_htbridge(device_t dev, phandle_t child) { struct cpcht_softc *sc; struct ofw_pci_register pcir; int ptr, nextptr; uint32_t vend, val; int i, nirq, irq; u_int b, f, s; sc = device_get_softc(dev); if (OF_getencprop(child, "reg", (pcell_t *)&pcir, sizeof(pcir)) == -1) return; b = OFW_PCI_PHYS_HI_BUS(pcir.phys_hi); s = OFW_PCI_PHYS_HI_DEVICE(pcir.phys_hi); f = OFW_PCI_PHYS_HI_FUNCTION(pcir.phys_hi); /* * Mark this slot is populated. The remote south bridge does * not like us talking to unpopulated slots on the root bus. */ sc->sc_populated_slots |= (1 << s); /* * Next build up any HT->MPIC mappings for this sub-bus. One would * naively hope that enabling, disabling, and EOIing interrupts would * cause the appropriate HT bus transactions to that effect. This is * not the case. * * Instead, we have to muck about on the HT peer's root PCI bridges, * figure out what interrupts they send, enable them, and cache * the location of their WaitForEOI registers so that we can * send EOIs later. */ /* All the devices we are interested in have caps */ if (!(PCIB_READ_CONFIG(dev, b, s, f, PCIR_STATUS, 2) & PCIM_STATUS_CAPPRESENT)) return; nextptr = PCIB_READ_CONFIG(dev, b, s, f, PCIR_CAP_PTR, 1); while (nextptr != 0) { ptr = nextptr; nextptr = PCIB_READ_CONFIG(dev, b, s, f, ptr + PCICAP_NEXTPTR, 1); /* Find the HT IRQ capabilities */ if (PCIB_READ_CONFIG(dev, b, s, f, ptr + PCICAP_ID, 1) != PCIY_HT) continue; val = PCIB_READ_CONFIG(dev, b, s, f, ptr + PCIR_HT_COMMAND, 2); if ((val & PCIM_HTCMD_CAP_MASK) != PCIM_HTCAP_INTERRUPT) continue; /* Ask for the IRQ count */ PCIB_WRITE_CONFIG(dev, b, s, f, ptr + PCIR_HT_COMMAND, 0x1, 1); nirq = PCIB_READ_CONFIG(dev, b, s, f, ptr + 4, 4); nirq = ((nirq >> 16) & 0xff) + 1; device_printf(dev, "%d HT IRQs on device %d.%d\n", nirq, s, f); for (i = 0; i < nirq; i++) { PCIB_WRITE_CONFIG(dev, b, s, f, ptr + PCIR_HT_COMMAND, 0x10 + (i << 1), 1); irq = PCIB_READ_CONFIG(dev, b, s, f, ptr + 4, 4); /* * Mask this interrupt for now. */ PCIB_WRITE_CONFIG(dev, b, s, f, ptr + 4, irq | HTAPIC_MASK, 4); irq = (irq >> 16) & 0xff; sc->htirq_map[irq].irq_type = IRQ_HT; sc->htirq_map[irq].ht_source = i; sc->htirq_map[irq].ht_base = sc->sc_data + (((((s & 0x1f) << 3) | (f & 0x07)) << 8) | (ptr)); PCIB_WRITE_CONFIG(dev, b, s, f, ptr + PCIR_HT_COMMAND, 0x11 + (i << 1), 1); sc->htirq_map[irq].eoi_data = PCIB_READ_CONFIG(dev, b, s, f, ptr + 4, 4) | 0x80000000; /* * Apple uses a non-compliant IO/APIC that differs * in how we signal EOIs. Check if this device was * made by Apple, and act accordingly. */ vend = PCIB_READ_CONFIG(dev, b, s, f, PCIR_DEVVENDOR, 4); if ((vend & 0xffff) == 0x106b) sc->htirq_map[irq].apple_eoi = (sc->htirq_map[irq].ht_base - ptr) + 0x60; } } } static u_int32_t cpcht_read_config(device_t dev, u_int bus, u_int slot, u_int func, u_int reg, int width) { struct cpcht_softc *sc; vm_offset_t caoff; sc = device_get_softc(dev); caoff = sc->sc_data + (((((slot & 0x1f) << 3) | (func & 0x07)) << 8) | reg); if (bus == 0 && (!(sc->sc_populated_slots & (1 << slot)) || func > 0)) return (0xffffffff); if (bus > 0) caoff += 0x01000000UL + (bus << 16); switch (width) { case 1: return (in8rb(caoff)); break; case 2: return (in16rb(caoff)); break; case 4: return (in32rb(caoff)); break; } return (0xffffffff); } static void cpcht_write_config(device_t dev, u_int bus, u_int slot, u_int func, u_int reg, u_int32_t val, int width) { struct cpcht_softc *sc; vm_offset_t caoff; sc = device_get_softc(dev); caoff = sc->sc_data + (((((slot & 0x1f) << 3) | (func & 0x07)) << 8) | reg); if (bus == 0 && (!(sc->sc_populated_slots & (1 << slot)) || func > 0)) return; if (bus > 0) caoff += 0x01000000UL + (bus << 16); switch (width) { case 1: out8rb(caoff, val); break; case 2: out16rb(caoff, val); break; case 4: out32rb(caoff, val); break; } } static int cpcht_route_interrupt(device_t bus, device_t dev, int pin) { return (pin); } static int cpcht_alloc_msi(device_t dev, device_t child, int count, int maxcount, int *irqs) { struct cpcht_softc *sc; int i, j; sc = device_get_softc(dev); j = 0; /* Bail if no MSI PIC yet */ if (cpcht_msipic == 0) return (ENXIO); mtx_lock(&sc->htirq_mtx); for (i = 8; i < 124 - count; i++) { for (j = 0; j < count; j++) { if (sc->htirq_map[i+j].irq_type != IRQ_NONE) break; } if (j == count) break; i += j; /* We know there isn't a large enough run */ } if (j != count) { mtx_unlock(&sc->htirq_mtx); return (ENXIO); } for (j = 0; j < count; j++) { irqs[j] = MAP_IRQ(cpcht_msipic, i+j); sc->htirq_map[i+j].irq_type = IRQ_MSI; } mtx_unlock(&sc->htirq_mtx); return (0); } static int cpcht_release_msi(device_t dev, device_t child, int count, int *irqs) { struct cpcht_softc *sc; int i; sc = device_get_softc(dev); mtx_lock(&sc->htirq_mtx); for (i = 0; i < count; i++) sc->htirq_map[irqs[i] & 0xff].irq_type = IRQ_NONE; mtx_unlock(&sc->htirq_mtx); return (0); } static int cpcht_alloc_msix(device_t dev, device_t child, int *irq) { struct cpcht_softc *sc; int i; sc = device_get_softc(dev); /* Bail if no MSI PIC yet */ if (cpcht_msipic == 0) return (ENXIO); mtx_lock(&sc->htirq_mtx); for (i = 8; i < 124; i++) { if (sc->htirq_map[i].irq_type == IRQ_NONE) { sc->htirq_map[i].irq_type = IRQ_MSI; *irq = MAP_IRQ(cpcht_msipic, i); mtx_unlock(&sc->htirq_mtx); return (0); } } mtx_unlock(&sc->htirq_mtx); return (ENXIO); } static int cpcht_release_msix(device_t dev, device_t child, int irq) { struct cpcht_softc *sc; sc = device_get_softc(dev); mtx_lock(&sc->htirq_mtx); sc->htirq_map[irq & 0xff].irq_type = IRQ_NONE; mtx_unlock(&sc->htirq_mtx); return (0); } static int cpcht_map_msi(device_t dev, device_t child, int irq, uint64_t *addr, uint32_t *data) { device_t pcib; struct pci_devinfo *dinfo; struct pcicfg_ht *ht = NULL; for (pcib = child; pcib != dev; pcib = device_get_parent(device_get_parent(pcib))) { dinfo = device_get_ivars(pcib); ht = &dinfo->cfg.ht; if (ht == NULL) continue; } if (ht == NULL) return (ENXIO); *addr = ht->ht_msiaddr; *data = irq & 0xff; return (0); } /* * Driver for the integrated MPIC on U3/U4 (CPC925/CPC945) */ static int openpic_cpcht_probe(device_t); static int openpic_cpcht_attach(device_t); static void openpic_cpcht_config(device_t, u_int irq, enum intr_trigger trig, enum intr_polarity pol); static void openpic_cpcht_enable(device_t, u_int irq, u_int vector); static void openpic_cpcht_unmask(device_t, u_int irq); static void openpic_cpcht_eoi(device_t, u_int irq); static device_method_t openpic_cpcht_methods[] = { /* Device interface */ DEVMETHOD(device_probe, openpic_cpcht_probe), DEVMETHOD(device_attach, openpic_cpcht_attach), /* PIC interface */ DEVMETHOD(pic_bind, openpic_bind), DEVMETHOD(pic_config, openpic_cpcht_config), DEVMETHOD(pic_dispatch, openpic_dispatch), DEVMETHOD(pic_enable, openpic_cpcht_enable), DEVMETHOD(pic_eoi, openpic_cpcht_eoi), DEVMETHOD(pic_ipi, openpic_ipi), DEVMETHOD(pic_mask, openpic_mask), DEVMETHOD(pic_unmask, openpic_cpcht_unmask), { 0, 0 }, }; struct openpic_cpcht_softc { struct openpic_softc sc_openpic; struct mtx sc_ht_mtx; }; static driver_t openpic_cpcht_driver = { "htpic", openpic_cpcht_methods, sizeof(struct openpic_cpcht_softc), }; DRIVER_MODULE(openpic, unin, openpic_cpcht_driver, openpic_devclass, 0, 0); static int openpic_cpcht_probe(device_t dev) { const char *type = ofw_bus_get_type(dev); if (strcmp(type, "open-pic") != 0) return (ENXIO); device_set_desc(dev, OPENPIC_DEVSTR); return (0); } static int openpic_cpcht_attach(device_t dev) { struct openpic_cpcht_softc *sc; phandle_t node; int err, irq; node = ofw_bus_get_node(dev); err = openpic_common_attach(dev, node); if (err != 0) return (err); /* * The HT APIC stuff is not thread-safe, so we need a mutex to * protect it. */ sc = device_get_softc(dev); mtx_init(&sc->sc_ht_mtx, "htpic", NULL, MTX_SPIN); /* * Interrupts 0-3 are internally sourced and are level triggered * active low. Interrupts 4-123 are connected to a pulse generator * and should be programmed as edge triggered low-to-high. * * IBM CPC945 Manual, Section 9.3. */ for (irq = 0; irq < 4; irq++) openpic_config(dev, irq, INTR_TRIGGER_LEVEL, INTR_POLARITY_LOW); for (irq = 4; irq < 124; irq++) openpic_config(dev, irq, INTR_TRIGGER_EDGE, INTR_POLARITY_LOW); /* * Use this PIC for MSI only if it is the root PIC. This may not * be necessary, but Linux does it, and I cannot find any U3 machines * with MSI devices to test. */ if (dev == root_pic) cpcht_msipic = node; return (0); } static void openpic_cpcht_config(device_t dev, u_int irq, enum intr_trigger trig, enum intr_polarity pol) { struct openpic_cpcht_softc *sc; uint32_t ht_irq; /* * The interrupt settings for the MPIC are completely determined * by the internal wiring in the northbridge. Real changes to these * settings need to be negotiated with the remote IO-APIC on the HT * link. */ sc = device_get_softc(dev); if (cpcht_irqmap != NULL && irq < 128 && cpcht_irqmap[irq].ht_base > 0 && !cpcht_irqmap[irq].edge) { mtx_lock_spin(&sc->sc_ht_mtx); /* Program the data port */ out8rb(cpcht_irqmap[irq].ht_base + PCIR_HT_COMMAND, 0x10 + (cpcht_irqmap[irq].ht_source << 1)); /* Grab the IRQ config register */ ht_irq = in32rb(cpcht_irqmap[irq].ht_base + 4); /* Mask the IRQ while we fiddle settings */ out32rb(cpcht_irqmap[irq].ht_base + 4, ht_irq | HTAPIC_MASK); /* Program the interrupt sense */ ht_irq &= ~(HTAPIC_TRIGGER_LEVEL | HTAPIC_REQUEST_EOI); if (trig == INTR_TRIGGER_EDGE) { cpcht_irqmap[irq].edge = 1; } else { cpcht_irqmap[irq].edge = 0; ht_irq |= HTAPIC_TRIGGER_LEVEL | HTAPIC_REQUEST_EOI; } out32rb(cpcht_irqmap[irq].ht_base + 4, ht_irq); mtx_unlock_spin(&sc->sc_ht_mtx); } } static void openpic_cpcht_enable(device_t dev, u_int irq, u_int vec) { struct openpic_cpcht_softc *sc; uint32_t ht_irq; openpic_enable(dev, irq, vec); sc = device_get_softc(dev); if (cpcht_irqmap != NULL && irq < 128 && cpcht_irqmap[irq].ht_base > 0) { mtx_lock_spin(&sc->sc_ht_mtx); /* Program the data port */ out8rb(cpcht_irqmap[irq].ht_base + PCIR_HT_COMMAND, 0x10 + (cpcht_irqmap[irq].ht_source << 1)); /* Unmask the interrupt */ ht_irq = in32rb(cpcht_irqmap[irq].ht_base + 4); ht_irq &= ~HTAPIC_MASK; out32rb(cpcht_irqmap[irq].ht_base + 4, ht_irq); mtx_unlock_spin(&sc->sc_ht_mtx); } openpic_cpcht_eoi(dev, irq); } static void openpic_cpcht_unmask(device_t dev, u_int irq) { struct openpic_cpcht_softc *sc; uint32_t ht_irq; openpic_unmask(dev, irq); sc = device_get_softc(dev); if (cpcht_irqmap != NULL && irq < 128 && cpcht_irqmap[irq].ht_base > 0) { mtx_lock_spin(&sc->sc_ht_mtx); /* Program the data port */ out8rb(cpcht_irqmap[irq].ht_base + PCIR_HT_COMMAND, 0x10 + (cpcht_irqmap[irq].ht_source << 1)); /* Unmask the interrupt */ ht_irq = in32rb(cpcht_irqmap[irq].ht_base + 4); ht_irq &= ~HTAPIC_MASK; out32rb(cpcht_irqmap[irq].ht_base + 4, ht_irq); mtx_unlock_spin(&sc->sc_ht_mtx); } openpic_cpcht_eoi(dev, irq); } static void openpic_cpcht_eoi(device_t dev, u_int irq) { struct openpic_cpcht_softc *sc; uint32_t off, mask; if (irq == 255) return; sc = device_get_softc(dev); if (cpcht_irqmap != NULL && irq < 128 && cpcht_irqmap[irq].ht_base > 0 && !cpcht_irqmap[irq].edge) { /* If this is an HT IRQ, acknowledge it at the remote APIC */ if (cpcht_irqmap[irq].apple_eoi) { off = (cpcht_irqmap[irq].ht_source >> 3) & ~3; mask = 1 << (cpcht_irqmap[irq].ht_source & 0x1f); out32rb(cpcht_irqmap[irq].apple_eoi + off, mask); } else { mtx_lock_spin(&sc->sc_ht_mtx); out8rb(cpcht_irqmap[irq].ht_base + PCIR_HT_COMMAND, 0x11 + (cpcht_irqmap[irq].ht_source << 1)); out32rb(cpcht_irqmap[irq].ht_base + 4, cpcht_irqmap[irq].eoi_data); mtx_unlock_spin(&sc->sc_ht_mtx); } } openpic_eoi(dev, irq); } Index: head/sys/sparc64/pci/fire.c =================================================================== --- head/sys/sparc64/pci/fire.c (revision 314316) +++ head/sys/sparc64/pci/fire.c (revision 314317) @@ -1,1874 +1,1875 @@ /*- * Copyright (c) 1999, 2000 Matthew R. Green * Copyright (c) 2001 - 2003 by Thomas Moestl * Copyright (c) 2009 by Marius Strobl * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. The name of the author may not be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * from: NetBSD: psycho.c,v 1.39 2001/10/07 20:30:41 eeh Exp * from: FreeBSD: psycho.c 183152 2008-09-18 19:45:22Z marius */ #include __FBSDID("$FreeBSD$"); /* * Driver for `Fire' JBus to PCI Express and `Oberon' Uranus to PCI Express * bridges */ #include "opt_fire.h" #include "opt_ofw_pci.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include +#include #include #include #include #include "pcib_if.h" struct fire_msiqarg; static const struct fire_desc *fire_get_desc(device_t dev); static void fire_dmamap_sync(bus_dma_tag_t dt __unused, bus_dmamap_t map, bus_dmasync_op_t op); static int fire_get_intrmap(struct fire_softc *sc, u_int ino, bus_addr_t *intrmapptr, bus_addr_t *intrclrptr); static void fire_intr_assign(void *arg); static void fire_intr_clear(void *arg); static void fire_intr_disable(void *arg); static void fire_intr_enable(void *arg); static int fire_intr_register(struct fire_softc *sc, u_int ino); static inline void fire_msiq_common(struct intr_vector *iv, struct fire_msiqarg *fmqa); static void fire_msiq_filter(void *cookie); static void fire_msiq_handler(void *cookie); static void fire_set_intr(struct fire_softc *sc, u_int index, u_int ino, driver_filter_t handler, void *arg); static timecounter_get_t fire_get_timecount; /* Interrupt handlers */ static driver_filter_t fire_dmc_pec; static driver_filter_t fire_pcie; static driver_filter_t fire_xcb; /* * Methods */ static pcib_alloc_msi_t fire_alloc_msi; static pcib_alloc_msix_t fire_alloc_msix; static bus_alloc_resource_t fire_alloc_resource; static device_attach_t fire_attach; static pcib_map_msi_t fire_map_msi; static pcib_maxslots_t fire_maxslots; static device_probe_t fire_probe; static pcib_read_config_t fire_read_config; static pcib_release_msi_t fire_release_msi; static pcib_release_msix_t fire_release_msix; static pcib_route_interrupt_t fire_route_interrupt; static bus_setup_intr_t fire_setup_intr; static bus_teardown_intr_t fire_teardown_intr; static pcib_write_config_t fire_write_config; static device_method_t fire_methods[] = { /* Device interface */ DEVMETHOD(device_probe, fire_probe), DEVMETHOD(device_attach, fire_attach), DEVMETHOD(device_shutdown, bus_generic_shutdown), DEVMETHOD(device_suspend, bus_generic_suspend), DEVMETHOD(device_resume, bus_generic_resume), /* Bus interface */ DEVMETHOD(bus_read_ivar, ofw_pci_read_ivar), DEVMETHOD(bus_setup_intr, fire_setup_intr), DEVMETHOD(bus_teardown_intr, fire_teardown_intr), DEVMETHOD(bus_alloc_resource, fire_alloc_resource), DEVMETHOD(bus_activate_resource, ofw_pci_activate_resource), DEVMETHOD(bus_deactivate_resource, bus_generic_deactivate_resource), DEVMETHOD(bus_adjust_resource, ofw_pci_adjust_resource), DEVMETHOD(bus_release_resource, bus_generic_release_resource), DEVMETHOD(bus_get_dma_tag, ofw_pci_get_dma_tag), /* pcib interface */ DEVMETHOD(pcib_maxslots, fire_maxslots), DEVMETHOD(pcib_read_config, fire_read_config), DEVMETHOD(pcib_write_config, fire_write_config), DEVMETHOD(pcib_route_interrupt, fire_route_interrupt), DEVMETHOD(pcib_alloc_msi, fire_alloc_msi), DEVMETHOD(pcib_release_msi, fire_release_msi), DEVMETHOD(pcib_alloc_msix, fire_alloc_msix), DEVMETHOD(pcib_release_msix, fire_release_msix), DEVMETHOD(pcib_map_msi, fire_map_msi), DEVMETHOD(pcib_request_feature, pcib_request_feature_allow), /* ofw_bus interface */ DEVMETHOD(ofw_bus_get_node, ofw_pci_get_node), DEVMETHOD_END }; static devclass_t fire_devclass; DEFINE_CLASS_0(pcib, fire_driver, fire_methods, sizeof(struct fire_softc)); EARLY_DRIVER_MODULE(fire, nexus, fire_driver, fire_devclass, 0, 0, BUS_PASS_BUS); MODULE_DEPEND(fire, nexus, 1, 1, 1); static const struct intr_controller fire_ic = { fire_intr_enable, fire_intr_disable, fire_intr_assign, fire_intr_clear }; struct fire_icarg { struct fire_softc *fica_sc; bus_addr_t fica_map; bus_addr_t fica_clr; }; static const struct intr_controller fire_msiqc_filter = { fire_intr_enable, fire_intr_disable, fire_intr_assign, NULL }; struct fire_msiqarg { struct fire_icarg fmqa_fica; struct mtx fmqa_mtx; struct fo_msiq_record *fmqa_base; uint64_t fmqa_head; uint64_t fmqa_tail; uint32_t fmqa_msiq; uint32_t fmqa_msi; }; #define FIRE_PERF_CNT_QLTY 100 #define FIRE_SPC_BARRIER(spc, sc, offs, len, flags) \ bus_barrier((sc)->sc_mem_res[(spc)], (offs), (len), (flags)) #define FIRE_SPC_READ_8(spc, sc, offs) \ bus_read_8((sc)->sc_mem_res[(spc)], (offs)) #define FIRE_SPC_WRITE_8(spc, sc, offs, v) \ bus_write_8((sc)->sc_mem_res[(spc)], (offs), (v)) #ifndef FIRE_DEBUG #define FIRE_SPC_SET(spc, sc, offs, reg, v) \ FIRE_SPC_WRITE_8((spc), (sc), (offs), (v)) #else #define FIRE_SPC_SET(spc, sc, offs, reg, v) do { \ device_printf((sc)->sc_dev, reg " 0x%016llx -> 0x%016llx\n", \ (unsigned long long)FIRE_SPC_READ_8((spc), (sc), (offs)), \ (unsigned long long)(v)); \ FIRE_SPC_WRITE_8((spc), (sc), (offs), (v)); \ } while (0) #endif #define FIRE_PCI_BARRIER(sc, offs, len, flags) \ FIRE_SPC_BARRIER(FIRE_PCI, (sc), (offs), len, flags) #define FIRE_PCI_READ_8(sc, offs) \ FIRE_SPC_READ_8(FIRE_PCI, (sc), (offs)) #define FIRE_PCI_WRITE_8(sc, offs, v) \ FIRE_SPC_WRITE_8(FIRE_PCI, (sc), (offs), (v)) #define FIRE_CTRL_BARRIER(sc, offs, len, flags) \ FIRE_SPC_BARRIER(FIRE_CTRL, (sc), (offs), len, flags) #define FIRE_CTRL_READ_8(sc, offs) \ FIRE_SPC_READ_8(FIRE_CTRL, (sc), (offs)) #define FIRE_CTRL_WRITE_8(sc, offs, v) \ FIRE_SPC_WRITE_8(FIRE_CTRL, (sc), (offs), (v)) #define FIRE_PCI_SET(sc, offs, v) \ FIRE_SPC_SET(FIRE_PCI, (sc), (offs), # offs, (v)) #define FIRE_CTRL_SET(sc, offs, v) \ FIRE_SPC_SET(FIRE_CTRL, (sc), (offs), # offs, (v)) struct fire_desc { const char *fd_string; int fd_mode; const char *fd_name; }; static const struct fire_desc fire_compats[] = { { "pciex108e,80f0", FIRE_MODE_FIRE, "Fire" }, #if 0 { "pciex108e,80f8", FIRE_MODE_OBERON, "Oberon" }, #endif { NULL, 0, NULL } }; static const struct fire_desc * fire_get_desc(device_t dev) { const struct fire_desc *desc; const char *compat; compat = ofw_bus_get_compat(dev); if (compat == NULL) return (NULL); for (desc = fire_compats; desc->fd_string != NULL; desc++) if (strcmp(desc->fd_string, compat) == 0) return (desc); return (NULL); } static int fire_probe(device_t dev) { const char *dtype; dtype = ofw_bus_get_type(dev); if (dtype != NULL && strcmp(dtype, OFW_TYPE_PCIE) == 0 && fire_get_desc(dev) != NULL) { device_set_desc(dev, "Sun Host-PCIe bridge"); return (BUS_PROBE_GENERIC); } return (ENXIO); } static int fire_attach(device_t dev) { struct fire_softc *sc; const struct fire_desc *desc; struct ofw_pci_msi_ranges msi_ranges; struct ofw_pci_msi_addr_ranges msi_addr_ranges; struct ofw_pci_msi_eq_to_devino msi_eq_to_devino; struct fire_msiqarg *fmqa; struct timecounter *tc; bus_dma_tag_t dmat; uint64_t ino_bitmap, val; phandle_t node; uint32_t prop, prop_array[2]; int i, j, mode; u_int lw; uint16_t mps; sc = device_get_softc(dev); node = ofw_bus_get_node(dev); desc = fire_get_desc(dev); mode = desc->fd_mode; sc->sc_dev = dev; sc->sc_mode = mode; sc->sc_flags = 0; mtx_init(&sc->sc_msi_mtx, "msi_mtx", NULL, MTX_DEF); mtx_init(&sc->sc_pcib_mtx, "pcib_mtx", NULL, MTX_SPIN); /* * Fire and Oberon have two register banks: * (0) per-PBM PCI Express configuration and status registers * (1) (shared) Fire/Oberon controller configuration and status * registers */ for (i = 0; i < FIRE_NREG; i++) { j = i; sc->sc_mem_res[i] = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &j, RF_ACTIVE); if (sc->sc_mem_res[i] == NULL) panic("%s: could not allocate register bank %d", __func__, i); } if (OF_getprop(node, "portid", &sc->sc_ign, sizeof(sc->sc_ign)) == -1) panic("%s: could not determine IGN", __func__); if (OF_getprop(node, "module-revision#", &prop, sizeof(prop)) == -1) panic("%s: could not determine module-revision", __func__); device_printf(dev, "%s, module-revision %d, IGN %#x\n", desc->fd_name, prop, sc->sc_ign); /* * Hunt through all the interrupt mapping regs and register * the interrupt controller for our interrupt vectors. We do * this early in order to be able to catch stray interrupts. */ i = OF_getprop(node, "ino-bitmap", (void *)prop_array, sizeof(prop_array)); if (i == -1) panic("%s: could not get ino-bitmap", __func__); ino_bitmap = ((uint64_t)prop_array[1] << 32) | prop_array[0]; for (i = 0; i <= FO_MAX_INO; i++) { if ((ino_bitmap & (1ULL << i)) == 0) continue; j = fire_intr_register(sc, i); if (j != 0) device_printf(dev, "could not register interrupt " "controller for INO %d (%d)\n", i, j); } /* JBC/UBC module initialization */ FIRE_CTRL_SET(sc, FO_XBC_ERR_LOG_EN, ~0ULL); FIRE_CTRL_SET(sc, FO_XBC_ERR_STAT_CLR, ~0ULL); /* not enabled by OpenSolaris */ FIRE_CTRL_SET(sc, FO_XBC_INT_EN, ~0ULL); if (sc->sc_mode == FIRE_MODE_FIRE) { FIRE_CTRL_SET(sc, FIRE_JBUS_PAR_CTRL, FIRE_JBUS_PAR_CTRL_P_EN); FIRE_CTRL_SET(sc, FIRE_JBC_FATAL_RST_EN, ((1ULL << FIRE_JBC_FATAL_RST_EN_SPARE_P_INT_SHFT) & FIRE_JBC_FATAL_RST_EN_SPARE_P_INT_MASK) | FIRE_JBC_FATAL_RST_EN_MB_PEA_P_INT | FIRE_JBC_FATAL_RST_EN_CPE_P_INT | FIRE_JBC_FATAL_RST_EN_APE_P_INT | FIRE_JBC_FATAL_RST_EN_PIO_CPE_INT | FIRE_JBC_FATAL_RST_EN_JTCEEW_P_INT | FIRE_JBC_FATAL_RST_EN_JTCEEI_P_INT | FIRE_JBC_FATAL_RST_EN_JTCEER_P_INT); FIRE_CTRL_SET(sc, FIRE_JBC_CORE_BLOCK_INT_EN, ~0ULL); } /* TLU initialization */ FIRE_PCI_SET(sc, FO_PCI_TLU_OEVENT_STAT_CLR, FO_PCI_TLU_OEVENT_S_MASK | FO_PCI_TLU_OEVENT_P_MASK); /* not enabled by OpenSolaris */ FIRE_PCI_SET(sc, FO_PCI_TLU_OEVENT_INT_EN, FO_PCI_TLU_OEVENT_S_MASK | FO_PCI_TLU_OEVENT_P_MASK); FIRE_PCI_SET(sc, FO_PCI_TLU_UERR_STAT_CLR, FO_PCI_TLU_UERR_INT_S_MASK | FO_PCI_TLU_UERR_INT_P_MASK); /* not enabled by OpenSolaris */ FIRE_PCI_SET(sc, FO_PCI_TLU_UERR_INT_EN, FO_PCI_TLU_UERR_INT_S_MASK | FO_PCI_TLU_UERR_INT_P_MASK); FIRE_PCI_SET(sc, FO_PCI_TLU_CERR_STAT_CLR, FO_PCI_TLU_CERR_INT_S_MASK | FO_PCI_TLU_CERR_INT_P_MASK); /* not enabled by OpenSolaris */ FIRE_PCI_SET(sc, FO_PCI_TLU_CERR_INT_EN, FO_PCI_TLU_CERR_INT_S_MASK | FO_PCI_TLU_CERR_INT_P_MASK); val = FIRE_PCI_READ_8(sc, FO_PCI_TLU_CTRL) | ((FO_PCI_TLU_CTRL_L0S_TIM_DFLT << FO_PCI_TLU_CTRL_L0S_TIM_SHFT) & FO_PCI_TLU_CTRL_L0S_TIM_MASK) | ((FO_PCI_TLU_CTRL_CFG_DFLT << FO_PCI_TLU_CTRL_CFG_SHFT) & FO_PCI_TLU_CTRL_CFG_MASK); if (sc->sc_mode == FIRE_MODE_OBERON) val &= ~FO_PCI_TLU_CTRL_NWPR_EN; val |= FO_PCI_TLU_CTRL_CFG_REMAIN_DETECT_QUIET; FIRE_PCI_SET(sc, FO_PCI_TLU_CTRL, val); FIRE_PCI_SET(sc, FO_PCI_TLU_DEV_CTRL, 0); FIRE_PCI_SET(sc, FO_PCI_TLU_LNK_CTRL, FO_PCI_TLU_LNK_CTRL_CLK); /* DLU/LPU initialization */ if (sc->sc_mode == FIRE_MODE_OBERON) FIRE_PCI_SET(sc, FO_PCI_LPU_INT_MASK, 0); else FIRE_PCI_SET(sc, FO_PCI_LPU_RST, 0); FIRE_PCI_SET(sc, FO_PCI_LPU_LNK_LYR_CFG, FO_PCI_LPU_LNK_LYR_CFG_VC0_EN); FIRE_PCI_SET(sc, FO_PCI_LPU_FLW_CTRL_UPDT_CTRL, FO_PCI_LPU_FLW_CTRL_UPDT_CTRL_FC0_NP_EN | FO_PCI_LPU_FLW_CTRL_UPDT_CTRL_FC0_P_EN); if (sc->sc_mode == FIRE_MODE_OBERON) FIRE_PCI_SET(sc, FO_PCI_LPU_TXLNK_RPLY_TMR_THRS, (OBERON_PCI_LPU_TXLNK_RPLY_TMR_THRS_DFLT << FO_PCI_LPU_TXLNK_RPLY_TMR_THRS_SHFT) & FO_PCI_LPU_TXLNK_RPLY_TMR_THRS_MASK); else { switch ((FIRE_PCI_READ_8(sc, FO_PCI_TLU_LNK_STAT) & FO_PCI_TLU_LNK_STAT_WDTH_MASK) >> FO_PCI_TLU_LNK_STAT_WDTH_SHFT) { case 1: lw = 0; break; case 4: lw = 1; break; case 8: lw = 2; break; case 16: lw = 3; break; default: lw = 0; } mps = (FIRE_PCI_READ_8(sc, FO_PCI_TLU_CTRL) & FO_PCI_TLU_CTRL_CFG_MPS_MASK) >> FO_PCI_TLU_CTRL_CFG_MPS_SHFT; i = sizeof(fire_freq_nak_tmr_thrs) / sizeof(*fire_freq_nak_tmr_thrs); if (mps >= i) mps = i - 1; FIRE_PCI_SET(sc, FO_PCI_LPU_TXLNK_FREQ_LAT_TMR_THRS, (fire_freq_nak_tmr_thrs[mps][lw] << FO_PCI_LPU_TXLNK_FREQ_LAT_TMR_THRS_SHFT) & FO_PCI_LPU_TXLNK_FREQ_LAT_TMR_THRS_MASK); FIRE_PCI_SET(sc, FO_PCI_LPU_TXLNK_RPLY_TMR_THRS, (fire_rply_tmr_thrs[mps][lw] << FO_PCI_LPU_TXLNK_RPLY_TMR_THRS_SHFT) & FO_PCI_LPU_TXLNK_RPLY_TMR_THRS_MASK); FIRE_PCI_SET(sc, FO_PCI_LPU_TXLNK_RTR_FIFO_PTR, ((FO_PCI_LPU_TXLNK_RTR_FIFO_PTR_TL_DFLT << FO_PCI_LPU_TXLNK_RTR_FIFO_PTR_TL_SHFT) & FO_PCI_LPU_TXLNK_RTR_FIFO_PTR_TL_MASK) | ((FO_PCI_LPU_TXLNK_RTR_FIFO_PTR_HD_DFLT << FO_PCI_LPU_TXLNK_RTR_FIFO_PTR_HD_SHFT) & FO_PCI_LPU_TXLNK_RTR_FIFO_PTR_HD_MASK)); FIRE_PCI_SET(sc, FO_PCI_LPU_LTSSM_CFG2, (FO_PCI_LPU_LTSSM_CFG2_12_TO_DFLT << FO_PCI_LPU_LTSSM_CFG2_12_TO_SHFT) & FO_PCI_LPU_LTSSM_CFG2_12_TO_MASK); FIRE_PCI_SET(sc, FO_PCI_LPU_LTSSM_CFG3, (FO_PCI_LPU_LTSSM_CFG3_2_TO_DFLT << FO_PCI_LPU_LTSSM_CFG3_2_TO_SHFT) & FO_PCI_LPU_LTSSM_CFG3_2_TO_MASK); FIRE_PCI_SET(sc, FO_PCI_LPU_LTSSM_CFG4, ((FO_PCI_LPU_LTSSM_CFG4_DATA_RATE_DFLT << FO_PCI_LPU_LTSSM_CFG4_DATA_RATE_SHFT) & FO_PCI_LPU_LTSSM_CFG4_DATA_RATE_MASK) | ((FO_PCI_LPU_LTSSM_CFG4_N_FTS_DFLT << FO_PCI_LPU_LTSSM_CFG4_N_FTS_SHFT) & FO_PCI_LPU_LTSSM_CFG4_N_FTS_MASK)); FIRE_PCI_SET(sc, FO_PCI_LPU_LTSSM_CFG5, 0); } /* ILU initialization */ FIRE_PCI_SET(sc, FO_PCI_ILU_ERR_STAT_CLR, ~0ULL); /* not enabled by OpenSolaris */ FIRE_PCI_SET(sc, FO_PCI_ILU_INT_EN, ~0ULL); /* IMU initialization */ FIRE_PCI_SET(sc, FO_PCI_IMU_ERR_STAT_CLR, ~0ULL); FIRE_PCI_SET(sc, FO_PCI_IMU_INT_EN, FIRE_PCI_READ_8(sc, FO_PCI_IMU_INT_EN) & ~(FO_PCI_IMU_ERR_INT_FATAL_MES_NOT_EN_S | FO_PCI_IMU_ERR_INT_NFATAL_MES_NOT_EN_S | FO_PCI_IMU_ERR_INT_COR_MES_NOT_EN_S | FO_PCI_IMU_ERR_INT_FATAL_MES_NOT_EN_P | FO_PCI_IMU_ERR_INT_NFATAL_MES_NOT_EN_P | FO_PCI_IMU_ERR_INT_COR_MES_NOT_EN_P)); /* MMU initialization */ FIRE_PCI_SET(sc, FO_PCI_MMU_ERR_STAT_CLR, FO_PCI_MMU_ERR_INT_S_MASK | FO_PCI_MMU_ERR_INT_P_MASK); /* not enabled by OpenSolaris */ FIRE_PCI_SET(sc, FO_PCI_MMU_INT_EN, FO_PCI_MMU_ERR_INT_S_MASK | FO_PCI_MMU_ERR_INT_P_MASK); /* DMC initialization */ FIRE_PCI_SET(sc, FO_PCI_DMC_CORE_BLOCK_INT_EN, ~0ULL); FIRE_PCI_SET(sc, FO_PCI_DMC_DBG_SEL_PORTA, 0); FIRE_PCI_SET(sc, FO_PCI_DMC_DBG_SEL_PORTB, 0); /* PEC initialization */ FIRE_PCI_SET(sc, FO_PCI_PEC_CORE_BLOCK_INT_EN, ~0ULL); /* Establish handlers for interesting interrupts. */ if ((ino_bitmap & (1ULL << FO_DMC_PEC_INO)) != 0) fire_set_intr(sc, 1, FO_DMC_PEC_INO, fire_dmc_pec, sc); if ((ino_bitmap & (1ULL << FO_XCB_INO)) != 0) fire_set_intr(sc, 0, FO_XCB_INO, fire_xcb, sc); /* MSI/MSI-X support */ if (OF_getprop(node, "#msi", &sc->sc_msi_count, sizeof(sc->sc_msi_count)) == -1) panic("%s: could not determine MSI count", __func__); if (OF_getprop(node, "msi-ranges", &msi_ranges, sizeof(msi_ranges)) == -1) sc->sc_msi_first = 0; else sc->sc_msi_first = msi_ranges.first; if (OF_getprop(node, "msi-data-mask", &sc->sc_msi_data_mask, sizeof(sc->sc_msi_data_mask)) == -1) panic("%s: could not determine MSI data mask", __func__); if (OF_getprop(node, "msix-data-width", &sc->sc_msix_data_width, sizeof(sc->sc_msix_data_width)) > 0) sc->sc_flags |= FIRE_MSIX; if (OF_getprop(node, "msi-address-ranges", &msi_addr_ranges, sizeof(msi_addr_ranges)) == -1) panic("%s: could not determine MSI address ranges", __func__); sc->sc_msi_addr32 = OFW_PCI_MSI_ADDR_RANGE_32(&msi_addr_ranges); sc->sc_msi_addr64 = OFW_PCI_MSI_ADDR_RANGE_64(&msi_addr_ranges); if (OF_getprop(node, "#msi-eqs", &sc->sc_msiq_count, sizeof(sc->sc_msiq_count)) == -1) panic("%s: could not determine MSI event queue count", __func__); if (OF_getprop(node, "msi-eq-size", &sc->sc_msiq_size, sizeof(sc->sc_msiq_size)) == -1) panic("%s: could not determine MSI event queue size", __func__); if (OF_getprop(node, "msi-eq-to-devino", &msi_eq_to_devino, sizeof(msi_eq_to_devino)) == -1 && OF_getprop(node, "msi-eq-devino", &msi_eq_to_devino, sizeof(msi_eq_to_devino)) == -1) { sc->sc_msiq_first = 0; sc->sc_msiq_ino_first = FO_EQ_FIRST_INO; } else { sc->sc_msiq_first = msi_eq_to_devino.eq_first; sc->sc_msiq_ino_first = msi_eq_to_devino.devino_first; } if (sc->sc_msiq_ino_first < FO_EQ_FIRST_INO || sc->sc_msiq_ino_first + sc->sc_msiq_count - 1 > FO_EQ_LAST_INO) panic("%s: event queues exceed INO range", __func__); sc->sc_msi_bitmap = malloc(roundup2(sc->sc_msi_count, NBBY) / NBBY, M_DEVBUF, M_NOWAIT | M_ZERO); if (sc->sc_msi_bitmap == NULL) panic("%s: could not malloc MSI bitmap", __func__); sc->sc_msi_msiq_table = malloc(sc->sc_msi_count * sizeof(*sc->sc_msi_msiq_table), M_DEVBUF, M_NOWAIT | M_ZERO); if (sc->sc_msi_msiq_table == NULL) panic("%s: could not malloc MSI-MSI event queue table", __func__); sc->sc_msiq_bitmap = malloc(roundup2(sc->sc_msiq_count, NBBY) / NBBY, M_DEVBUF, M_NOWAIT | M_ZERO); if (sc->sc_msiq_bitmap == NULL) panic("%s: could not malloc MSI event queue bitmap", __func__); j = FO_EQ_RECORD_SIZE * FO_EQ_NRECORDS * sc->sc_msiq_count; sc->sc_msiq = contigmalloc(j, M_DEVBUF, M_NOWAIT, 0, ~0UL, FO_EQ_ALIGNMENT, 0); if (sc->sc_msiq == NULL) panic("%s: could not contigmalloc MSI event queue", __func__); memset(sc->sc_msiq, 0, j); FIRE_PCI_SET(sc, FO_PCI_EQ_BASE_ADDR, FO_PCI_EQ_BASE_ADDR_BYPASS | (pmap_kextract((vm_offset_t)sc->sc_msiq) & FO_PCI_EQ_BASE_ADDR_MASK)); for (i = 0; i < sc->sc_msi_count; i++) { j = (i + sc->sc_msi_first) << 3; FIRE_PCI_WRITE_8(sc, FO_PCI_MSI_MAP_BASE + j, FIRE_PCI_READ_8(sc, FO_PCI_MSI_MAP_BASE + j) & ~FO_PCI_MSI_MAP_V); } for (i = 0; i < sc->sc_msiq_count; i++) { j = i + sc->sc_msiq_ino_first; if ((ino_bitmap & (1ULL << j)) == 0) { mtx_lock(&sc->sc_msi_mtx); setbit(sc->sc_msiq_bitmap, i); mtx_unlock(&sc->sc_msi_mtx); } fmqa = intr_vectors[INTMAP_VEC(sc->sc_ign, j)].iv_icarg; mtx_init(&fmqa->fmqa_mtx, "msiq_mtx", NULL, MTX_SPIN); fmqa->fmqa_base = (struct fo_msiq_record *)((caddr_t)sc->sc_msiq + (FO_EQ_RECORD_SIZE * FO_EQ_NRECORDS * i)); j = i + sc->sc_msiq_first; fmqa->fmqa_msiq = j; j <<= 3; fmqa->fmqa_head = FO_PCI_EQ_HD_BASE + j; fmqa->fmqa_tail = FO_PCI_EQ_TL_BASE + j; FIRE_PCI_WRITE_8(sc, FO_PCI_EQ_CTRL_CLR_BASE + j, FO_PCI_EQ_CTRL_CLR_COVERR | FO_PCI_EQ_CTRL_CLR_E2I | FO_PCI_EQ_CTRL_CLR_DIS); FIRE_PCI_WRITE_8(sc, fmqa->fmqa_tail, (0 << FO_PCI_EQ_TL_SHFT) & FO_PCI_EQ_TL_MASK); FIRE_PCI_WRITE_8(sc, fmqa->fmqa_head, (0 << FO_PCI_EQ_HD_SHFT) & FO_PCI_EQ_HD_MASK); } FIRE_PCI_SET(sc, FO_PCI_MSI_32_BIT_ADDR, sc->sc_msi_addr32 & FO_PCI_MSI_32_BIT_ADDR_MASK); FIRE_PCI_SET(sc, FO_PCI_MSI_64_BIT_ADDR, sc->sc_msi_addr64 & FO_PCI_MSI_64_BIT_ADDR_MASK); /* * Establish a handler for interesting PCIe messages and disable * unintersting ones. */ mtx_lock(&sc->sc_msi_mtx); for (i = 0; i < sc->sc_msiq_count; i++) { if (isclr(sc->sc_msiq_bitmap, i) != 0) { j = i; break; } } if (i == sc->sc_msiq_count) { mtx_unlock(&sc->sc_msi_mtx); panic("%s: no spare event queue for PCIe messages", __func__); } setbit(sc->sc_msiq_bitmap, j); mtx_unlock(&sc->sc_msi_mtx); i = INTMAP_VEC(sc->sc_ign, j + sc->sc_msiq_ino_first); if (bus_set_resource(dev, SYS_RES_IRQ, 2, i, 1) != 0) panic("%s: failed to add interrupt for PCIe messages", __func__); fire_set_intr(sc, 2, INTINO(i), fire_pcie, intr_vectors[i].iv_icarg); j += sc->sc_msiq_first; /* * "Please note that setting the EQNUM field to a value larger than * 35 will yield unpredictable results." */ if (j > 35) panic("%s: invalid queue for PCIe messages (%d)", __func__, j); FIRE_PCI_SET(sc, FO_PCI_ERR_COR, FO_PCI_ERR_PME_V | ((j << FO_PCI_ERR_PME_EQNUM_SHFT) & FO_PCI_ERR_PME_EQNUM_MASK)); FIRE_PCI_SET(sc, FO_PCI_ERR_NONFATAL, FO_PCI_ERR_PME_V | ((j << FO_PCI_ERR_PME_EQNUM_SHFT) & FO_PCI_ERR_PME_EQNUM_MASK)); FIRE_PCI_SET(sc, FO_PCI_ERR_FATAL, FO_PCI_ERR_PME_V | ((j << FO_PCI_ERR_PME_EQNUM_SHFT) & FO_PCI_ERR_PME_EQNUM_MASK)); FIRE_PCI_SET(sc, FO_PCI_PM_PME, 0); FIRE_PCI_SET(sc, FO_PCI_PME_TO_ACK, 0); FIRE_PCI_WRITE_8(sc, FO_PCI_EQ_CTRL_SET_BASE + (j << 3), FO_PCI_EQ_CTRL_SET_EN); #define TC_COUNTER_MAX_MASK 0xffffffff /* * Setup JBC/UBC performance counter 0 in bus cycle counting * mode as timecounter. */ if (device_get_unit(dev) == 0) { FIRE_CTRL_SET(sc, FO_XBC_PRF_CNT0, 0); FIRE_CTRL_SET(sc, FO_XBC_PRF_CNT1, 0); FIRE_CTRL_SET(sc, FO_XBC_PRF_CNT_SEL, (FO_XBC_PRF_CNT_NONE << FO_XBC_PRF_CNT_CNT1_SHFT) | (FO_XBC_PRF_CNT_XB_CLK << FO_XBC_PRF_CNT_CNT0_SHFT)); tc = malloc(sizeof(*tc), M_DEVBUF, M_NOWAIT | M_ZERO); if (tc == NULL) panic("%s: could not malloc timecounter", __func__); tc->tc_get_timecount = fire_get_timecount; tc->tc_counter_mask = TC_COUNTER_MAX_MASK; if (OF_getprop(OF_peer(0), "clock-frequency", &prop, sizeof(prop)) == -1) panic("%s: could not determine clock frequency", __func__); tc->tc_frequency = prop; tc->tc_name = strdup(device_get_nameunit(dev), M_DEVBUF); tc->tc_priv = sc; /* * Due to initial problems with the JBus-driven performance * counters not advancing which might be firmware dependent * ensure that it actually works. */ if (fire_get_timecount(tc) - fire_get_timecount(tc) != 0) tc->tc_quality = FIRE_PERF_CNT_QLTY; else tc->tc_quality = -FIRE_PERF_CNT_QLTY; tc_init(tc); } /* * Set up the IOMMU. Both Fire and Oberon have one per PBM, but * neither has a streaming buffer. */ memcpy(&sc->sc_dma_methods, &iommu_dma_methods, sizeof(sc->sc_dma_methods)); sc->sc_is.is_flags = IOMMU_FIRE | IOMMU_PRESERVE_PROM; if (sc->sc_mode == FIRE_MODE_OBERON) { sc->sc_is.is_flags |= IOMMU_FLUSH_CACHE; sc->sc_is.is_pmaxaddr = IOMMU_MAXADDR(OBERON_IOMMU_BITS); } else { sc->sc_dma_methods.dm_dmamap_sync = fire_dmamap_sync; sc->sc_is.is_pmaxaddr = IOMMU_MAXADDR(FIRE_IOMMU_BITS); } sc->sc_is.is_sb[0] = sc->sc_is.is_sb[1] = 0; /* Punch in our copies. */ sc->sc_is.is_bustag = rman_get_bustag(sc->sc_mem_res[FIRE_PCI]); sc->sc_is.is_bushandle = rman_get_bushandle(sc->sc_mem_res[FIRE_PCI]); sc->sc_is.is_iommu = FO_PCI_MMU; val = FIRE_PCI_READ_8(sc, FO_PCI_MMU + IMR_CTL); iommu_init(device_get_nameunit(dev), &sc->sc_is, 7, -1, 0); #ifdef FIRE_DEBUG device_printf(dev, "FO_PCI_MMU + IMR_CTL 0x%016llx -> 0x%016llx\n", (long long unsigned)val, (long long unsigned)sc->sc_is.is_cr); #endif /* Create our DMA tag. */ if (bus_dma_tag_create(bus_get_dma_tag(dev), 8, 0x100000000, sc->sc_is.is_pmaxaddr, ~0, NULL, NULL, sc->sc_is.is_pmaxaddr, 0xff, 0xffffffff, 0, NULL, NULL, &dmat) != 0) panic("%s: could not create PCI DMA tag", __func__); dmat->dt_cookie = &sc->sc_is; dmat->dt_mt = &sc->sc_dma_methods; if (ofw_pci_attach_common(dev, dmat, FO_IO_SIZE, FO_MEM_SIZE) != 0) panic("%s: ofw_pci_attach_common() failed", __func__); #define FIRE_SYSCTL_ADD_UINT(name, arg, desc) \ SYSCTL_ADD_UINT(device_get_sysctl_ctx(dev), \ SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), OID_AUTO, \ (name), CTLFLAG_RD, (arg), 0, (desc)) FIRE_SYSCTL_ADD_UINT("ilu_err", &sc->sc_stats_ilu_err, "ILU unknown errors"); FIRE_SYSCTL_ADD_UINT("jbc_ce_async", &sc->sc_stats_jbc_ce_async, "JBC correctable errors"); FIRE_SYSCTL_ADD_UINT("jbc_unsol_int", &sc->sc_stats_jbc_unsol_int, "JBC unsolicited interrupt ACK/NACK errors"); FIRE_SYSCTL_ADD_UINT("jbc_unsol_rd", &sc->sc_stats_jbc_unsol_rd, "JBC unsolicited read response errors"); FIRE_SYSCTL_ADD_UINT("mmu_err", &sc->sc_stats_mmu_err, "MMU errors"); FIRE_SYSCTL_ADD_UINT("tlu_ce", &sc->sc_stats_tlu_ce, "DLU/TLU correctable errors"); FIRE_SYSCTL_ADD_UINT("tlu_oe_non_fatal", &sc->sc_stats_tlu_oe_non_fatal, "DLU/TLU other event non-fatal errors summary"); FIRE_SYSCTL_ADD_UINT("tlu_oe_rx_err", &sc->sc_stats_tlu_oe_rx_err, "DLU/TLU receive other event errors"); FIRE_SYSCTL_ADD_UINT("tlu_oe_tx_err", &sc->sc_stats_tlu_oe_tx_err, "DLU/TLU transmit other event errors"); FIRE_SYSCTL_ADD_UINT("ubc_dmardue", &sc->sc_stats_ubc_dmardue, "UBC DMARDUE erros"); #undef FIRE_SYSCTL_ADD_UINT device_add_child(dev, "pci", -1); return (bus_generic_attach(dev)); } static void fire_set_intr(struct fire_softc *sc, u_int index, u_int ino, driver_filter_t handler, void *arg) { u_long vec; int rid; rid = index; sc->sc_irq_res[index] = bus_alloc_resource_any(sc->sc_dev, SYS_RES_IRQ, &rid, RF_ACTIVE); if (sc->sc_irq_res[index] == NULL || INTINO(vec = rman_get_start(sc->sc_irq_res[index])) != ino || INTIGN(vec) != sc->sc_ign || intr_vectors[vec].iv_ic != &fire_ic || bus_setup_intr(sc->sc_dev, sc->sc_irq_res[index], INTR_TYPE_MISC | INTR_BRIDGE, handler, NULL, arg, &sc->sc_ihand[index]) != 0) panic("%s: failed to set up interrupt %d", __func__, index); } static int fire_intr_register(struct fire_softc *sc, u_int ino) { struct fire_icarg *fica; bus_addr_t intrclr, intrmap; int error; if (fire_get_intrmap(sc, ino, &intrmap, &intrclr) == 0) return (ENXIO); fica = malloc((ino >= FO_EQ_FIRST_INO && ino <= FO_EQ_LAST_INO) ? sizeof(struct fire_msiqarg) : sizeof(struct fire_icarg), M_DEVBUF, M_NOWAIT | M_ZERO); if (fica == NULL) return (ENOMEM); fica->fica_sc = sc; fica->fica_map = intrmap; fica->fica_clr = intrclr; error = (intr_controller_register(INTMAP_VEC(sc->sc_ign, ino), &fire_ic, fica)); if (error != 0) free(fica, M_DEVBUF); return (error); } static int fire_get_intrmap(struct fire_softc *sc, u_int ino, bus_addr_t *intrmapptr, bus_addr_t *intrclrptr) { if (ino > FO_MAX_INO) { device_printf(sc->sc_dev, "out of range INO %d requested\n", ino); return (0); } ino <<= 3; if (intrmapptr != NULL) *intrmapptr = FO_PCI_INT_MAP_BASE + ino; if (intrclrptr != NULL) *intrclrptr = FO_PCI_INT_CLR_BASE + ino; return (1); } /* * Interrupt handlers */ static int fire_dmc_pec(void *arg) { struct fire_softc *sc; device_t dev; uint64_t cestat, dmcstat, ilustat, imustat, mcstat, mmustat, mmutfar; uint64_t mmutfsr, oestat, pecstat, uestat, val; u_int fatal, oenfatal; fatal = 0; sc = arg; dev = sc->sc_dev; mtx_lock_spin(&sc->sc_pcib_mtx); mcstat = FIRE_PCI_READ_8(sc, FO_PCI_MULTI_CORE_ERR_STAT); if ((mcstat & FO_PCI_MULTI_CORE_ERR_STAT_DMC) != 0) { dmcstat = FIRE_PCI_READ_8(sc, FO_PCI_DMC_CORE_BLOCK_ERR_STAT); if ((dmcstat & FO_PCI_DMC_CORE_BLOCK_INT_EN_IMU) != 0) { imustat = FIRE_PCI_READ_8(sc, FO_PCI_IMU_INT_STAT); device_printf(dev, "IMU error %#llx\n", (unsigned long long)imustat); if ((imustat & FO_PCI_IMU_ERR_INT_EQ_NOT_EN_P) != 0) { fatal = 1; val = FIRE_PCI_READ_8(sc, FO_PCI_IMU_SCS_ERR_LOG); device_printf(dev, "SCS error log %#llx\n", (unsigned long long)val); } if ((imustat & FO_PCI_IMU_ERR_INT_EQ_OVER_P) != 0) { fatal = 1; val = FIRE_PCI_READ_8(sc, FO_PCI_IMU_EQS_ERR_LOG); device_printf(dev, "EQS error log %#llx\n", (unsigned long long)val); } if ((imustat & (FO_PCI_IMU_ERR_INT_MSI_MAL_ERR_P | FO_PCI_IMU_ERR_INT_MSI_PAR_ERR_P | FO_PCI_IMU_ERR_INT_PMEACK_MES_NOT_EN_P | FO_PCI_IMU_ERR_INT_PMPME_MES_NOT_EN_P | FO_PCI_IMU_ERR_INT_FATAL_MES_NOT_EN_P | FO_PCI_IMU_ERR_INT_NFATAL_MES_NOT_EN_P | FO_PCI_IMU_ERR_INT_COR_MES_NOT_EN_P | FO_PCI_IMU_ERR_INT_MSI_NOT_EN_P)) != 0) { fatal = 1; val = FIRE_PCI_READ_8(sc, FO_PCI_IMU_RDS_ERR_LOG); device_printf(dev, "RDS error log %#llx\n", (unsigned long long)val); } } if ((dmcstat & FO_PCI_DMC_CORE_BLOCK_INT_EN_MMU) != 0) { fatal = 1; mmustat = FIRE_PCI_READ_8(sc, FO_PCI_MMU_INT_STAT); mmutfar = FIRE_PCI_READ_8(sc, FO_PCI_MMU_TRANS_FAULT_ADDR); mmutfsr = FIRE_PCI_READ_8(sc, FO_PCI_MMU_TRANS_FAULT_STAT); if ((mmustat & (FO_PCI_MMU_ERR_INT_TBW_DPE_P | FO_PCI_MMU_ERR_INT_TBW_ERR_P | FO_PCI_MMU_ERR_INT_TBW_UDE_P | FO_PCI_MMU_ERR_INT_TBW_DME_P | FO_PCI_MMU_ERR_INT_TTC_CAE_P | FIRE_PCI_MMU_ERR_INT_TTC_DPE_P | OBERON_PCI_MMU_ERR_INT_TTC_DUE_P | FO_PCI_MMU_ERR_INT_TRN_ERR_P)) != 0) fatal = 1; else { sc->sc_stats_mmu_err++; FIRE_PCI_WRITE_8(sc, FO_PCI_MMU_ERR_STAT_CLR, mmustat); } device_printf(dev, "MMU error %#llx: TFAR %#llx TFSR %#llx\n", (unsigned long long)mmustat, (unsigned long long)mmutfar, (unsigned long long)mmutfsr); } } if ((mcstat & FO_PCI_MULTI_CORE_ERR_STAT_PEC) != 0) { pecstat = FIRE_PCI_READ_8(sc, FO_PCI_PEC_CORE_BLOCK_INT_STAT); if ((pecstat & FO_PCI_PEC_CORE_BLOCK_INT_STAT_UERR) != 0) { fatal = 1; uestat = FIRE_PCI_READ_8(sc, FO_PCI_TLU_UERR_INT_STAT); device_printf(dev, "DLU/TLU uncorrectable error %#llx\n", (unsigned long long)uestat); if ((uestat & (FO_PCI_TLU_UERR_INT_UR_P | OBERON_PCI_TLU_UERR_INT_POIS_P | FO_PCI_TLU_UERR_INT_MFP_P | FO_PCI_TLU_UERR_INT_ROF_P | FO_PCI_TLU_UERR_INT_UC_P | FIRE_PCI_TLU_UERR_INT_PP_P | OBERON_PCI_TLU_UERR_INT_POIS_P)) != 0) { val = FIRE_PCI_READ_8(sc, FO_PCI_TLU_RX_UERR_HDR1_LOG); device_printf(dev, "receive header log %#llx\n", (unsigned long long)val); val = FIRE_PCI_READ_8(sc, FO_PCI_TLU_RX_UERR_HDR2_LOG); device_printf(dev, "receive header log 2 %#llx\n", (unsigned long long)val); } if ((uestat & FO_PCI_TLU_UERR_INT_CTO_P) != 0) { val = FIRE_PCI_READ_8(sc, FO_PCI_TLU_TX_UERR_HDR1_LOG); device_printf(dev, "transmit header log %#llx\n", (unsigned long long)val); val = FIRE_PCI_READ_8(sc, FO_PCI_TLU_TX_UERR_HDR2_LOG); device_printf(dev, "transmit header log 2 %#llx\n", (unsigned long long)val); } if ((uestat & FO_PCI_TLU_UERR_INT_DLP_P) != 0) { val = FIRE_PCI_READ_8(sc, FO_PCI_LPU_LNK_LYR_INT_STAT); device_printf(dev, "link layer interrupt and status %#llx\n", (unsigned long long)val); } if ((uestat & FO_PCI_TLU_UERR_INT_TE_P) != 0) { val = FIRE_PCI_READ_8(sc, FO_PCI_LPU_PHY_LYR_INT_STAT); device_printf(dev, "phy layer interrupt and status %#llx\n", (unsigned long long)val); } } if ((pecstat & FO_PCI_PEC_CORE_BLOCK_INT_STAT_CERR) != 0) { sc->sc_stats_tlu_ce++; cestat = FIRE_PCI_READ_8(sc, FO_PCI_TLU_CERR_INT_STAT); device_printf(dev, "DLU/TLU correctable error %#llx\n", (unsigned long long)cestat); val = FIRE_PCI_READ_8(sc, FO_PCI_LPU_LNK_LYR_INT_STAT); device_printf(dev, "link layer interrupt and status %#llx\n", (unsigned long long)val); if ((cestat & FO_PCI_TLU_CERR_INT_RE_P) != 0) { FIRE_PCI_WRITE_8(sc, FO_PCI_LPU_LNK_LYR_INT_STAT, val); val = FIRE_PCI_READ_8(sc, FO_PCI_LPU_PHY_LYR_INT_STAT); device_printf(dev, "phy layer interrupt and status %#llx\n", (unsigned long long)val); } FIRE_PCI_WRITE_8(sc, FO_PCI_TLU_CERR_STAT_CLR, cestat); } if ((pecstat & FO_PCI_PEC_CORE_BLOCK_INT_STAT_OEVENT) != 0) { oenfatal = 0; oestat = FIRE_PCI_READ_8(sc, FO_PCI_TLU_OEVENT_INT_STAT); device_printf(dev, "DLU/TLU other event %#llx\n", (unsigned long long)oestat); if ((oestat & (FO_PCI_TLU_OEVENT_MFC_P | FO_PCI_TLU_OEVENT_MRC_P | FO_PCI_TLU_OEVENT_WUC_P | FO_PCI_TLU_OEVENT_RUC_P | FO_PCI_TLU_OEVENT_CRS_P)) != 0) { val = FIRE_PCI_READ_8(sc, FO_PCI_TLU_RX_OEVENT_HDR1_LOG); device_printf(dev, "receive header log %#llx\n", (unsigned long long)val); val = FIRE_PCI_READ_8(sc, FO_PCI_TLU_RX_OEVENT_HDR2_LOG); device_printf(dev, "receive header log 2 %#llx\n", (unsigned long long)val); if ((oestat & (FO_PCI_TLU_OEVENT_MFC_P | FO_PCI_TLU_OEVENT_MRC_P | FO_PCI_TLU_OEVENT_WUC_P | FO_PCI_TLU_OEVENT_RUC_P)) != 0) fatal = 1; else { sc->sc_stats_tlu_oe_rx_err++; oenfatal = 1; } } if ((oestat & (FO_PCI_TLU_OEVENT_MFC_P | FO_PCI_TLU_OEVENT_CTO_P | FO_PCI_TLU_OEVENT_WUC_P | FO_PCI_TLU_OEVENT_RUC_P)) != 0) { val = FIRE_PCI_READ_8(sc, FO_PCI_TLU_TX_OEVENT_HDR1_LOG); device_printf(dev, "transmit header log %#llx\n", (unsigned long long)val); val = FIRE_PCI_READ_8(sc, FO_PCI_TLU_TX_OEVENT_HDR2_LOG); device_printf(dev, "transmit header log 2 %#llx\n", (unsigned long long)val); if ((oestat & (FO_PCI_TLU_OEVENT_MFC_P | FO_PCI_TLU_OEVENT_CTO_P | FO_PCI_TLU_OEVENT_WUC_P | FO_PCI_TLU_OEVENT_RUC_P)) != 0) fatal = 1; else { sc->sc_stats_tlu_oe_tx_err++; oenfatal = 1; } } if ((oestat & (FO_PCI_TLU_OEVENT_ERO_P | FO_PCI_TLU_OEVENT_EMP_P | FO_PCI_TLU_OEVENT_EPE_P | FIRE_PCI_TLU_OEVENT_ERP_P | OBERON_PCI_TLU_OEVENT_ERBU_P | FIRE_PCI_TLU_OEVENT_EIP_P | OBERON_PCI_TLU_OEVENT_EIUE_P)) != 0) { fatal = 1; val = FIRE_PCI_READ_8(sc, FO_PCI_LPU_LNK_LYR_INT_STAT); device_printf(dev, "link layer interrupt and status %#llx\n", (unsigned long long)val); } if ((oestat & (FO_PCI_TLU_OEVENT_IIP_P | FO_PCI_TLU_OEVENT_EDP_P | FIRE_PCI_TLU_OEVENT_EHP_P | OBERON_PCI_TLU_OEVENT_TLUEITMO_S | FO_PCI_TLU_OEVENT_ERU_P)) != 0) fatal = 1; if ((oestat & (FO_PCI_TLU_OEVENT_NFP_P | FO_PCI_TLU_OEVENT_LWC_P | FO_PCI_TLU_OEVENT_LIN_P | FO_PCI_TLU_OEVENT_LRS_P | FO_PCI_TLU_OEVENT_LDN_P | FO_PCI_TLU_OEVENT_LUP_P)) != 0) oenfatal = 1; if (oenfatal != 0) { sc->sc_stats_tlu_oe_non_fatal++; FIRE_PCI_WRITE_8(sc, FO_PCI_TLU_OEVENT_STAT_CLR, oestat); if ((oestat & FO_PCI_TLU_OEVENT_LIN_P) != 0) FIRE_PCI_WRITE_8(sc, FO_PCI_LPU_LNK_LYR_INT_STAT, FIRE_PCI_READ_8(sc, FO_PCI_LPU_LNK_LYR_INT_STAT)); } } if ((pecstat & FO_PCI_PEC_CORE_BLOCK_INT_STAT_ILU) != 0) { ilustat = FIRE_PCI_READ_8(sc, FO_PCI_ILU_INT_STAT); device_printf(dev, "ILU error %#llx\n", (unsigned long long)ilustat); if ((ilustat & (FIRE_PCI_ILU_ERR_INT_IHB_PE_P | FIRE_PCI_ILU_ERR_INT_IHB_PE_P)) != 0) fatal = 1; else { sc->sc_stats_ilu_err++; FIRE_PCI_WRITE_8(sc, FO_PCI_ILU_INT_STAT, ilustat); } } } mtx_unlock_spin(&sc->sc_pcib_mtx); if (fatal != 0) panic("%s: fatal DMC/PEC error", device_get_nameunit(sc->sc_dev)); return (FILTER_HANDLED); } static int fire_xcb(void *arg) { struct fire_softc *sc; device_t dev; uint64_t errstat, intstat, val; u_int fatal; fatal = 0; sc = arg; dev = sc->sc_dev; mtx_lock_spin(&sc->sc_pcib_mtx); if (sc->sc_mode == FIRE_MODE_OBERON) { intstat = FIRE_CTRL_READ_8(sc, FO_XBC_INT_STAT); device_printf(dev, "UBC error: interrupt status %#llx\n", (unsigned long long)intstat); if ((intstat & ~(OBERON_UBC_ERR_INT_DMARDUEB_P | OBERON_UBC_ERR_INT_DMARDUEA_P)) != 0) fatal = 1; else sc->sc_stats_ubc_dmardue++; if (fatal != 0) { mtx_unlock_spin(&sc->sc_pcib_mtx); panic("%s: fatal UBC core block error", device_get_nameunit(sc->sc_dev)); } else { FIRE_CTRL_SET(sc, FO_XBC_ERR_STAT_CLR, ~0ULL); mtx_unlock_spin(&sc->sc_pcib_mtx); } } else { errstat = FIRE_CTRL_READ_8(sc, FIRE_JBC_CORE_BLOCK_ERR_STAT); if ((errstat & (FIRE_JBC_CORE_BLOCK_ERR_STAT_MERGE | FIRE_JBC_CORE_BLOCK_ERR_STAT_JBCINT | FIRE_JBC_CORE_BLOCK_ERR_STAT_DMCINT)) != 0) { intstat = FIRE_CTRL_READ_8(sc, FO_XBC_INT_STAT); device_printf(dev, "JBC interrupt status %#llx\n", (unsigned long long)intstat); if ((intstat & FIRE_JBC_ERR_INT_EBUS_TO_P) != 0) { val = FIRE_CTRL_READ_8(sc, FIRE_JBC_CSR_ERR_LOG); device_printf(dev, "CSR error log %#llx\n", (unsigned long long)val); } if ((intstat & (FIRE_JBC_ERR_INT_UNSOL_RD_P | FIRE_JBC_ERR_INT_UNSOL_INT_P)) != 0) { if ((intstat & FIRE_JBC_ERR_INT_UNSOL_RD_P) != 0) sc->sc_stats_jbc_unsol_rd++; if ((intstat & FIRE_JBC_ERR_INT_UNSOL_INT_P) != 0) sc->sc_stats_jbc_unsol_int++; val = FIRE_CTRL_READ_8(sc, FIRE_DMCINT_IDC_ERR_LOG); device_printf(dev, "DMCINT IDC error log %#llx\n", (unsigned long long)val); } if ((intstat & (FIRE_JBC_ERR_INT_MB_PER_P | FIRE_JBC_ERR_INT_MB_PEW_P)) != 0) { fatal = 1; val = FIRE_CTRL_READ_8(sc, FIRE_MERGE_TRANS_ERR_LOG); device_printf(dev, "merge transaction error log %#llx\n", (unsigned long long)val); } if ((intstat & FIRE_JBC_ERR_INT_IJP_P) != 0) { fatal = 1; val = FIRE_CTRL_READ_8(sc, FIRE_JBCINT_OTRANS_ERR_LOG); device_printf(dev, "JBCINT out transaction error log " "%#llx\n", (unsigned long long)val); val = FIRE_CTRL_READ_8(sc, FIRE_JBCINT_OTRANS_ERR_LOG2); device_printf(dev, "JBCINT out transaction error log 2 " "%#llx\n", (unsigned long long)val); } if ((intstat & (FIRE_JBC_ERR_INT_UE_ASYN_P | FIRE_JBC_ERR_INT_CE_ASYN_P | FIRE_JBC_ERR_INT_JTE_P | FIRE_JBC_ERR_INT_JBE_P | FIRE_JBC_ERR_INT_JUE_P | FIRE_JBC_ERR_INT_ICISE_P | FIRE_JBC_ERR_INT_WR_DPE_P | FIRE_JBC_ERR_INT_RD_DPE_P | FIRE_JBC_ERR_INT_ILL_BMW_P | FIRE_JBC_ERR_INT_ILL_BMR_P | FIRE_JBC_ERR_INT_BJC_P)) != 0) { if ((intstat & (FIRE_JBC_ERR_INT_UE_ASYN_P | FIRE_JBC_ERR_INT_JTE_P | FIRE_JBC_ERR_INT_JBE_P | FIRE_JBC_ERR_INT_JUE_P | FIRE_JBC_ERR_INT_ICISE_P | FIRE_JBC_ERR_INT_WR_DPE_P | FIRE_JBC_ERR_INT_RD_DPE_P | FIRE_JBC_ERR_INT_ILL_BMW_P | FIRE_JBC_ERR_INT_ILL_BMR_P | FIRE_JBC_ERR_INT_BJC_P)) != 0) fatal = 1; else sc->sc_stats_jbc_ce_async++; val = FIRE_CTRL_READ_8(sc, FIRE_JBCINT_ITRANS_ERR_LOG); device_printf(dev, "JBCINT in transaction error log %#llx\n", (unsigned long long)val); val = FIRE_CTRL_READ_8(sc, FIRE_JBCINT_ITRANS_ERR_LOG2); device_printf(dev, "JBCINT in transaction error log 2 " "%#llx\n", (unsigned long long)val); } if ((intstat & (FIRE_JBC_ERR_INT_PIO_UNMAP_RD_P | FIRE_JBC_ERR_INT_ILL_ACC_RD_P | FIRE_JBC_ERR_INT_PIO_UNMAP_P | FIRE_JBC_ERR_INT_PIO_DPE_P | FIRE_JBC_ERR_INT_PIO_CPE_P | FIRE_JBC_ERR_INT_ILL_ACC_P)) != 0) { fatal = 1; val = FIRE_CTRL_READ_8(sc, FIRE_JBC_CSR_ERR_LOG); device_printf(dev, "DMCINT ODCD error log %#llx\n", (unsigned long long)val); } if ((intstat & (FIRE_JBC_ERR_INT_MB_PEA_P | FIRE_JBC_ERR_INT_CPE_P | FIRE_JBC_ERR_INT_APE_P | FIRE_JBC_ERR_INT_PIO_CPE_P | FIRE_JBC_ERR_INT_JTCEEW_P | FIRE_JBC_ERR_INT_JTCEEI_P | FIRE_JBC_ERR_INT_JTCEER_P)) != 0) { fatal = 1; val = FIRE_CTRL_READ_8(sc, FIRE_FATAL_ERR_LOG); device_printf(dev, "fatal error log %#llx\n", (unsigned long long)val); val = FIRE_CTRL_READ_8(sc, FIRE_FATAL_ERR_LOG2); device_printf(dev, "fatal error log 2 " "%#llx\n", (unsigned long long)val); } if (fatal != 0) { mtx_unlock_spin(&sc->sc_pcib_mtx); panic("%s: fatal JBC core block error", device_get_nameunit(sc->sc_dev)); } else { FIRE_CTRL_SET(sc, FO_XBC_ERR_STAT_CLR, ~0ULL); mtx_unlock_spin(&sc->sc_pcib_mtx); } } else { mtx_unlock_spin(&sc->sc_pcib_mtx); panic("%s: unknown JCB core block error status %#llx", device_get_nameunit(sc->sc_dev), (unsigned long long)errstat); } } return (FILTER_HANDLED); } static int fire_pcie(void *arg) { struct fire_msiqarg *fmqa; struct fire_softc *sc; struct fo_msiq_record *qrec; device_t dev; uint64_t word0; u_int head, msg, msiq; fmqa = arg; sc = fmqa->fmqa_fica.fica_sc; dev = sc->sc_dev; msiq = fmqa->fmqa_msiq; mtx_lock_spin(&fmqa->fmqa_mtx); head = (FIRE_PCI_READ_8(sc, fmqa->fmqa_head) & FO_PCI_EQ_HD_MASK) >> FO_PCI_EQ_HD_SHFT; qrec = &fmqa->fmqa_base[head]; word0 = qrec->fomqr_word0; for (;;) { KASSERT((word0 & FO_MQR_WORD0_FMT_TYPE_MSG) != 0, ("%s: received non-PCIe message in event queue %d " "(word0 %#llx)", device_get_nameunit(dev), msiq, (unsigned long long)word0)); msg = (word0 & FO_MQR_WORD0_DATA0_MASK) >> FO_MQR_WORD0_DATA0_SHFT; #define PCIE_MSG_CODE_ERR_COR 0x30 #define PCIE_MSG_CODE_ERR_NONFATAL 0x31 #define PCIE_MSG_CODE_ERR_FATAL 0x33 if (msg == PCIE_MSG_CODE_ERR_COR) device_printf(dev, "correctable PCIe error\n"); else if (msg == PCIE_MSG_CODE_ERR_NONFATAL || msg == PCIE_MSG_CODE_ERR_FATAL) panic("%s: %sfatal PCIe error", device_get_nameunit(dev), msg == PCIE_MSG_CODE_ERR_NONFATAL ? "non-" : ""); else panic("%s: received unknown PCIe message %#x", device_get_nameunit(dev), msg); qrec->fomqr_word0 &= ~FO_MQR_WORD0_FMT_TYPE_MASK; head = (head + 1) % sc->sc_msiq_size; qrec = &fmqa->fmqa_base[head]; word0 = qrec->fomqr_word0; if (__predict_true((word0 & FO_MQR_WORD0_FMT_TYPE_MASK) == 0)) break; } FIRE_PCI_WRITE_8(sc, fmqa->fmqa_head, (head & FO_PCI_EQ_HD_MASK) << FO_PCI_EQ_HD_SHFT); if ((FIRE_PCI_READ_8(sc, fmqa->fmqa_tail) & FO_PCI_EQ_TL_OVERR) != 0) { device_printf(dev, "event queue %d overflow\n", msiq); msiq <<= 3; FIRE_PCI_WRITE_8(sc, FO_PCI_EQ_CTRL_CLR_BASE + msiq, FIRE_PCI_READ_8(sc, FO_PCI_EQ_CTRL_CLR_BASE + msiq) | FO_PCI_EQ_CTRL_CLR_COVERR); } mtx_unlock_spin(&fmqa->fmqa_mtx); return (FILTER_HANDLED); } static int fire_maxslots(device_t dev) { return (1); } static uint32_t fire_read_config(device_t dev, u_int bus, u_int slot, u_int func, u_int reg, int width) { return (ofw_pci_read_config_common(dev, PCIE_REGMAX, FO_CONF_OFF(bus, slot, func, reg), bus, slot, func, reg, width)); } static void fire_write_config(device_t dev, u_int bus, u_int slot, u_int func, u_int reg, uint32_t val, int width) { ofw_pci_write_config_common(dev, PCIE_REGMAX, FO_CONF_OFF(bus, slot, func, reg), bus, slot, func, reg, val, width); } static int fire_route_interrupt(device_t bridge, device_t dev, int pin) { ofw_pci_intr_t mintr; mintr = ofw_pci_route_interrupt_common(bridge, dev, pin); if (!PCI_INTERRUPT_VALID(mintr)) device_printf(bridge, "could not route pin %d for device %d.%d\n", pin, pci_get_slot(dev), pci_get_function(dev)); return (mintr); } static void fire_dmamap_sync(bus_dma_tag_t dt __unused, bus_dmamap_t map, bus_dmasync_op_t op) { if ((map->dm_flags & DMF_LOADED) == 0) return; if ((op & BUS_DMASYNC_POSTREAD) != 0) ofw_pci_dmamap_sync_stst_order_common(); else if ((op & BUS_DMASYNC_PREWRITE) != 0) membar(Sync); } static void fire_intr_enable(void *arg) { struct intr_vector *iv; struct fire_icarg *fica; struct fire_softc *sc; struct pcpu *pc; uint64_t mr; u_int ctrl, i; iv = arg; fica = iv->iv_icarg; sc = fica->fica_sc; mr = FO_PCI_IMAP_V; if (sc->sc_mode == FIRE_MODE_OBERON) mr |= (iv->iv_mid << OBERON_PCI_IMAP_T_DESTID_SHFT) & OBERON_PCI_IMAP_T_DESTID_MASK; else mr |= (iv->iv_mid << FIRE_PCI_IMAP_T_JPID_SHFT) & FIRE_PCI_IMAP_T_JPID_MASK; /* * Given that all mondos for the same target are required to use the * same interrupt controller we just use the CPU ID for indexing the * latter. */ ctrl = 0; for (i = 0; i < mp_ncpus; ++i) { pc = pcpu_find(i); if (pc == NULL || iv->iv_mid != pc->pc_mid) continue; ctrl = pc->pc_cpuid % 4; break; } mr |= (1ULL << ctrl) << FO_PCI_IMAP_INT_CTRL_NUM_SHFT & FO_PCI_IMAP_INT_CTRL_NUM_MASK; FIRE_PCI_WRITE_8(sc, fica->fica_map, mr); } static void fire_intr_disable(void *arg) { struct intr_vector *iv; struct fire_icarg *fica; struct fire_softc *sc; iv = arg; fica = iv->iv_icarg; sc = fica->fica_sc; FIRE_PCI_WRITE_8(sc, fica->fica_map, FIRE_PCI_READ_8(sc, fica->fica_map) & ~FO_PCI_IMAP_V); } static void fire_intr_assign(void *arg) { struct intr_vector *iv; struct fire_icarg *fica; struct fire_softc *sc; uint64_t mr; iv = arg; fica = iv->iv_icarg; sc = fica->fica_sc; mr = FIRE_PCI_READ_8(sc, fica->fica_map); if ((mr & FO_PCI_IMAP_V) != 0) { FIRE_PCI_WRITE_8(sc, fica->fica_map, mr & ~FO_PCI_IMAP_V); FIRE_PCI_BARRIER(sc, fica->fica_map, 8, BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE); } while (FIRE_PCI_READ_8(sc, fica->fica_clr) != INTCLR_IDLE) ; if ((mr & FO_PCI_IMAP_V) != 0) fire_intr_enable(arg); } static void fire_intr_clear(void *arg) { struct intr_vector *iv; struct fire_icarg *fica; iv = arg; fica = iv->iv_icarg; FIRE_PCI_WRITE_8(fica->fica_sc, fica->fica_clr, INTCLR_IDLE); } /* * Given that the event queue implementation matches our current MD and MI * interrupt frameworks like square pegs fit into round holes we are generous * and use one event queue per MSI for now, which limits us to 35 MSIs/MSI-Xs * per Host-PCIe-bridge (we use one event queue for the PCIe error messages). * This seems tolerable as long as most devices just use one MSI/MSI-X anyway. * Adding knowledge about MSIs/MSI-Xs to the MD interrupt code should allow us * to decouple the 1:1 mapping at the cost of no longer being able to bind * MSIs/MSI-Xs to specific CPUs as we currently have no reliable way to * quiesce a device while we move its MSIs/MSI-Xs to another event queue. */ static int fire_alloc_msi(device_t dev, device_t child, int count, int maxcount __unused, int *irqs) { struct fire_softc *sc; u_int i, j, msiqrun; if (powerof2(count) == 0 || count > 32) return (EINVAL); sc = device_get_softc(dev); mtx_lock(&sc->sc_msi_mtx); msiqrun = 0; for (i = 0; i < sc->sc_msiq_count; i++) { for (j = i; j < i + count; j++) { if (isclr(sc->sc_msiq_bitmap, j) == 0) break; } if (j == i + count) { msiqrun = i; break; } } if (i == sc->sc_msiq_count) { mtx_unlock(&sc->sc_msi_mtx); return (ENXIO); } for (i = 0; i + count < sc->sc_msi_count; i += count) { for (j = i; j < i + count; j++) if (isclr(sc->sc_msi_bitmap, j) == 0) break; if (j == i + count) { for (j = 0; j < count; j++) { setbit(sc->sc_msiq_bitmap, msiqrun + j); setbit(sc->sc_msi_bitmap, i + j); sc->sc_msi_msiq_table[i + j] = msiqrun + j; irqs[j] = sc->sc_msi_first + i + j; } mtx_unlock(&sc->sc_msi_mtx); return (0); } } mtx_unlock(&sc->sc_msi_mtx); return (ENXIO); } static int fire_release_msi(device_t dev, device_t child, int count, int *irqs) { struct fire_softc *sc; u_int i; sc = device_get_softc(dev); mtx_lock(&sc->sc_msi_mtx); for (i = 0; i < count; i++) { clrbit(sc->sc_msiq_bitmap, sc->sc_msi_msiq_table[irqs[i] - sc->sc_msi_first]); clrbit(sc->sc_msi_bitmap, irqs[i] - sc->sc_msi_first); } mtx_unlock(&sc->sc_msi_mtx); return (0); } static int fire_alloc_msix(device_t dev, device_t child, int *irq) { struct fire_softc *sc; int i, msiq; sc = device_get_softc(dev); if ((sc->sc_flags & FIRE_MSIX) == 0) return (ENXIO); mtx_lock(&sc->sc_msi_mtx); msiq = 0; for (i = 0; i < sc->sc_msiq_count; i++) { if (isclr(sc->sc_msiq_bitmap, i) != 0) { msiq = i; break; } } if (i == sc->sc_msiq_count) { mtx_unlock(&sc->sc_msi_mtx); return (ENXIO); } for (i = sc->sc_msi_count - 1; i >= 0; i--) { if (isclr(sc->sc_msi_bitmap, i) != 0) { setbit(sc->sc_msiq_bitmap, msiq); setbit(sc->sc_msi_bitmap, i); sc->sc_msi_msiq_table[i] = msiq; *irq = sc->sc_msi_first + i; mtx_unlock(&sc->sc_msi_mtx); return (0); } } mtx_unlock(&sc->sc_msi_mtx); return (ENXIO); } static int fire_release_msix(device_t dev, device_t child, int irq) { struct fire_softc *sc; sc = device_get_softc(dev); if ((sc->sc_flags & FIRE_MSIX) == 0) return (ENXIO); mtx_lock(&sc->sc_msi_mtx); clrbit(sc->sc_msiq_bitmap, sc->sc_msi_msiq_table[irq - sc->sc_msi_first]); clrbit(sc->sc_msi_bitmap, irq - sc->sc_msi_first); mtx_unlock(&sc->sc_msi_mtx); return (0); } static int fire_map_msi(device_t dev, device_t child, int irq, uint64_t *addr, uint32_t *data) { struct fire_softc *sc; struct pci_devinfo *dinfo; sc = device_get_softc(dev); dinfo = device_get_ivars(child); if (dinfo->cfg.msi.msi_alloc > 0) { if ((irq & ~sc->sc_msi_data_mask) != 0) { device_printf(dev, "invalid MSI 0x%x\n", irq); return (EINVAL); } } else { if ((sc->sc_flags & FIRE_MSIX) == 0) return (ENXIO); if (fls(irq) > sc->sc_msix_data_width) { device_printf(dev, "invalid MSI-X 0x%x\n", irq); return (EINVAL); } } if (dinfo->cfg.msi.msi_alloc > 0 && (dinfo->cfg.msi.msi_ctrl & PCIM_MSICTRL_64BIT) == 0) *addr = sc->sc_msi_addr32; else *addr = sc->sc_msi_addr64; *data = irq; return (0); } static void fire_msiq_handler(void *cookie) { struct intr_vector *iv; struct fire_msiqarg *fmqa; iv = cookie; fmqa = iv->iv_icarg; /* * Note that since fire_intr_clear() will clear the event queue * interrupt after the handler associated with the MSI [sic] has * been executed we have to protect the access to the event queue as * otherwise nested event queue interrupts cause corruption of the * event queue on MP machines. Obviously especially when abandoning * the 1:1 mapping it would be better to not clear the event queue * interrupt after each handler invocation but only once when the * outstanding MSIs have been processed but unfortunately that * doesn't work well and leads to interrupt storms with controllers/ * drivers which don't mask interrupts while the handler is executed. * Maybe delaying clearing the MSI until after the handler has been * executed could be used to work around this but that's not the * intended usage and might in turn cause lost MSIs. */ mtx_lock_spin(&fmqa->fmqa_mtx); fire_msiq_common(iv, fmqa); mtx_unlock_spin(&fmqa->fmqa_mtx); } static void fire_msiq_filter(void *cookie) { struct intr_vector *iv; struct fire_msiqarg *fmqa; iv = cookie; fmqa = iv->iv_icarg; /* * For filters we don't use fire_intr_clear() since it would clear * the event queue interrupt while we're still processing the event * queue as filters and associated post-filter handler are executed * directly, which in turn would lead to lost MSIs. So we clear the * event queue interrupt only once after processing the event queue. * Given that this still guarantees the filters to not be executed * concurrently and no other CPU can clear the event queue interrupt * while the event queue is still processed, we don't even need to * interlock the access to the event queue in this case. */ critical_enter(); fire_msiq_common(iv, fmqa); FIRE_PCI_WRITE_8(fmqa->fmqa_fica.fica_sc, fmqa->fmqa_fica.fica_clr, INTCLR_IDLE); critical_exit(); } static inline void fire_msiq_common(struct intr_vector *iv, struct fire_msiqarg *fmqa) { struct fire_softc *sc; struct fo_msiq_record *qrec; device_t dev; uint64_t word0; u_int head, msi, msiq; sc = fmqa->fmqa_fica.fica_sc; dev = sc->sc_dev; msiq = fmqa->fmqa_msiq; head = (FIRE_PCI_READ_8(sc, fmqa->fmqa_head) & FO_PCI_EQ_HD_MASK) >> FO_PCI_EQ_HD_SHFT; qrec = &fmqa->fmqa_base[head]; word0 = qrec->fomqr_word0; for (;;) { if (__predict_false((word0 & FO_MQR_WORD0_FMT_TYPE_MASK) == 0)) break; KASSERT((word0 & FO_MQR_WORD0_FMT_TYPE_MSI64) != 0 || (word0 & FO_MQR_WORD0_FMT_TYPE_MSI32) != 0, ("%s: received non-MSI/MSI-X message in event queue %d " "(word0 %#llx)", device_get_nameunit(dev), msiq, (unsigned long long)word0)); msi = (word0 & FO_MQR_WORD0_DATA0_MASK) >> FO_MQR_WORD0_DATA0_SHFT; /* * Sanity check the MSI/MSI-X as long as we use a 1:1 mapping. */ KASSERT(msi == fmqa->fmqa_msi, ("%s: received non-matching MSI/MSI-X in event queue %d " "(%d versus %d)", device_get_nameunit(dev), msiq, msi, fmqa->fmqa_msi)); FIRE_PCI_WRITE_8(sc, FO_PCI_MSI_CLR_BASE + (msi << 3), FO_PCI_MSI_CLR_EQWR_N); if (__predict_false(intr_event_handle(iv->iv_event, NULL) != 0)) printf("stray MSI/MSI-X in event queue %d\n", msiq); qrec->fomqr_word0 &= ~FO_MQR_WORD0_FMT_TYPE_MASK; head = (head + 1) % sc->sc_msiq_size; qrec = &fmqa->fmqa_base[head]; word0 = qrec->fomqr_word0; } FIRE_PCI_WRITE_8(sc, fmqa->fmqa_head, (head & FO_PCI_EQ_HD_MASK) << FO_PCI_EQ_HD_SHFT); if (__predict_false((FIRE_PCI_READ_8(sc, fmqa->fmqa_tail) & FO_PCI_EQ_TL_OVERR) != 0)) { device_printf(dev, "event queue %d overflow\n", msiq); msiq <<= 3; FIRE_PCI_WRITE_8(sc, FO_PCI_EQ_CTRL_CLR_BASE + msiq, FIRE_PCI_READ_8(sc, FO_PCI_EQ_CTRL_CLR_BASE + msiq) | FO_PCI_EQ_CTRL_CLR_COVERR); } } static int fire_setup_intr(device_t dev, device_t child, struct resource *ires, int flags, driver_filter_t *filt, driver_intr_t *intr, void *arg, void **cookiep) { struct fire_softc *sc; struct fire_msiqarg *fmqa; u_long vec; int error; u_int msi, msiq; sc = device_get_softc(dev); /* * XXX this assumes that a device only has one INTx, while in fact * Cassini+ and Saturn can use all four the firmware has assigned * to them, but so does pci(4). */ if (rman_get_rid(ires) != 0) { msi = rman_get_start(ires); msiq = sc->sc_msi_msiq_table[msi - sc->sc_msi_first]; vec = INTMAP_VEC(sc->sc_ign, sc->sc_msiq_ino_first + msiq); msiq += sc->sc_msiq_first; if (intr_vectors[vec].iv_ic != &fire_ic) { device_printf(dev, "invalid interrupt controller for vector 0x%lx\n", vec); return (EINVAL); } /* * The MD interrupt code needs the vector rather than the MSI. */ rman_set_start(ires, vec); rman_set_end(ires, vec); error = bus_generic_setup_intr(dev, child, ires, flags, filt, intr, arg, cookiep); rman_set_start(ires, msi); rman_set_end(ires, msi); if (error != 0) return (error); fmqa = intr_vectors[vec].iv_icarg; /* * XXX inject our event queue handler. */ if (filt != NULL) { intr_vectors[vec].iv_func = fire_msiq_filter; intr_vectors[vec].iv_ic = &fire_msiqc_filter; /* * Ensure the event queue interrupt is cleared, it * might have triggered before. Given we supply NULL * as ic_clear, inthand_add() won't do this for us. */ FIRE_PCI_WRITE_8(sc, fmqa->fmqa_fica.fica_clr, INTCLR_IDLE); } else intr_vectors[vec].iv_func = fire_msiq_handler; /* Record the MSI/MSI-X as long as we we use a 1:1 mapping. */ fmqa->fmqa_msi = msi; FIRE_PCI_WRITE_8(sc, FO_PCI_EQ_CTRL_SET_BASE + (msiq << 3), FO_PCI_EQ_CTRL_SET_EN); msi <<= 3; FIRE_PCI_WRITE_8(sc, FO_PCI_MSI_MAP_BASE + msi, (FIRE_PCI_READ_8(sc, FO_PCI_MSI_MAP_BASE + msi) & ~FO_PCI_MSI_MAP_EQNUM_MASK) | ((msiq << FO_PCI_MSI_MAP_EQNUM_SHFT) & FO_PCI_MSI_MAP_EQNUM_MASK)); FIRE_PCI_WRITE_8(sc, FO_PCI_MSI_CLR_BASE + msi, FO_PCI_MSI_CLR_EQWR_N); FIRE_PCI_WRITE_8(sc, FO_PCI_MSI_MAP_BASE + msi, FIRE_PCI_READ_8(sc, FO_PCI_MSI_MAP_BASE + msi) | FO_PCI_MSI_MAP_V); return (error); } /* * Make sure the vector is fully specified and we registered * our interrupt controller for it. */ vec = rman_get_start(ires); if (INTIGN(vec) != sc->sc_ign) { device_printf(dev, "invalid interrupt vector 0x%lx\n", vec); return (EINVAL); } if (intr_vectors[vec].iv_ic != &fire_ic) { device_printf(dev, "invalid interrupt controller for vector 0x%lx\n", vec); return (EINVAL); } return (bus_generic_setup_intr(dev, child, ires, flags, filt, intr, arg, cookiep)); } static int fire_teardown_intr(device_t dev, device_t child, struct resource *ires, void *cookie) { struct fire_softc *sc; u_long vec; int error; u_int msi, msiq; sc = device_get_softc(dev); if (rman_get_rid(ires) != 0) { msi = rman_get_start(ires); msiq = sc->sc_msi_msiq_table[msi - sc->sc_msi_first]; vec = INTMAP_VEC(sc->sc_ign, msiq + sc->sc_msiq_ino_first); msiq += sc->sc_msiq_first; msi <<= 3; FIRE_PCI_WRITE_8(sc, FO_PCI_MSI_MAP_BASE + msi, FIRE_PCI_READ_8(sc, FO_PCI_MSI_MAP_BASE + msi) & ~FO_PCI_MSI_MAP_V); msiq <<= 3; FIRE_PCI_WRITE_8(sc, FO_PCI_EQ_CTRL_CLR_BASE + msiq, FO_PCI_EQ_CTRL_CLR_COVERR | FO_PCI_EQ_CTRL_CLR_E2I | FO_PCI_EQ_CTRL_CLR_DIS); FIRE_PCI_WRITE_8(sc, FO_PCI_EQ_TL_BASE + msiq, (0 << FO_PCI_EQ_TL_SHFT) & FO_PCI_EQ_TL_MASK); FIRE_PCI_WRITE_8(sc, FO_PCI_EQ_HD_BASE + msiq, (0 << FO_PCI_EQ_HD_SHFT) & FO_PCI_EQ_HD_MASK); intr_vectors[vec].iv_ic = &fire_ic; /* * The MD interrupt code needs the vector rather than the MSI. */ rman_set_start(ires, vec); rman_set_end(ires, vec); error = bus_generic_teardown_intr(dev, child, ires, cookie); msi >>= 3; rman_set_start(ires, msi); rman_set_end(ires, msi); return (error); } return (bus_generic_teardown_intr(dev, child, ires, cookie)); } static struct resource * fire_alloc_resource(device_t bus, device_t child, int type, int *rid, rman_res_t start, rman_res_t end, rman_res_t count, u_int flags) { struct fire_softc *sc; if (type == SYS_RES_IRQ && *rid == 0) { sc = device_get_softc(bus); start = end = INTMAP_VEC(sc->sc_ign, end); } return (ofw_pci_alloc_resource(bus, child, type, rid, start, end, count, flags)); } static u_int fire_get_timecount(struct timecounter *tc) { struct fire_softc *sc; sc = tc->tc_priv; return (FIRE_CTRL_READ_8(sc, FO_XBC_PRF_CNT0) & TC_COUNTER_MAX_MASK); } Index: head/sys/sparc64/pci/psycho.c =================================================================== --- head/sys/sparc64/pci/psycho.c (revision 314316) +++ head/sys/sparc64/pci/psycho.c (revision 314317) @@ -1,1071 +1,1072 @@ /*- * Copyright (c) 1999, 2000 Matthew R. Green * Copyright (c) 2001 - 2003 by Thomas Moestl * Copyright (c) 2005 - 2006 Marius Strobl * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. The name of the author may not be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * from: NetBSD: psycho.c,v 1.39 2001/10/07 20:30:41 eeh Exp */ #include __FBSDID("$FreeBSD$"); /* * Support for `Hummingbird' (UltraSPARC IIe), `Psycho' and `Psycho+' * (UltraSPARC II) and `Sabre' (UltraSPARC IIi) UPA to PCI bridges. */ #include "opt_ofw_pci.h" #include "opt_psycho.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include +#include #include #include #include #include "pcib_if.h" static const struct psycho_desc *psycho_find_desc(const struct psycho_desc *, const char *); static const struct psycho_desc *psycho_get_desc(device_t); static void psycho_set_intr(struct psycho_softc *, u_int, bus_addr_t, driver_filter_t, driver_intr_t); static int psycho_find_intrmap(struct psycho_softc *, u_int, bus_addr_t *, bus_addr_t *, u_long *); static void sabre_dmamap_sync(bus_dma_tag_t dt, bus_dmamap_t map, bus_dmasync_op_t op); static void psycho_intr_enable(void *); static void psycho_intr_disable(void *); static void psycho_intr_assign(void *); static void psycho_intr_clear(void *); /* Interrupt handlers */ static driver_filter_t psycho_ue; static driver_filter_t psycho_ce; static driver_filter_t psycho_pci_bus; static driver_filter_t psycho_powerdebug; static driver_intr_t psycho_powerdown; static driver_intr_t psycho_overtemp; #ifdef PSYCHO_MAP_WAKEUP static driver_filter_t psycho_wakeup; #endif /* IOMMU support */ static void psycho_iommu_init(struct psycho_softc *, int, uint32_t); /* * Methods */ static device_probe_t psycho_probe; static device_attach_t psycho_attach; static bus_setup_intr_t psycho_setup_intr; static bus_alloc_resource_t psycho_alloc_resource; static pcib_maxslots_t psycho_maxslots; static pcib_read_config_t psycho_read_config; static pcib_write_config_t psycho_write_config; static pcib_route_interrupt_t psycho_route_interrupt; static ofw_pci_setup_device_t psycho_setup_device; static device_method_t psycho_methods[] = { /* Device interface */ DEVMETHOD(device_probe, psycho_probe), DEVMETHOD(device_attach, psycho_attach), DEVMETHOD(device_shutdown, bus_generic_shutdown), DEVMETHOD(device_suspend, bus_generic_suspend), DEVMETHOD(device_resume, bus_generic_resume), /* Bus interface */ DEVMETHOD(bus_read_ivar, ofw_pci_read_ivar), DEVMETHOD(bus_setup_intr, psycho_setup_intr), DEVMETHOD(bus_teardown_intr, bus_generic_teardown_intr), DEVMETHOD(bus_alloc_resource, psycho_alloc_resource), DEVMETHOD(bus_activate_resource, ofw_pci_activate_resource), DEVMETHOD(bus_deactivate_resource, bus_generic_deactivate_resource), DEVMETHOD(bus_adjust_resource, ofw_pci_adjust_resource), DEVMETHOD(bus_release_resource, bus_generic_release_resource), DEVMETHOD(bus_get_dma_tag, ofw_pci_get_dma_tag), /* pcib interface */ DEVMETHOD(pcib_maxslots, psycho_maxslots), DEVMETHOD(pcib_read_config, psycho_read_config), DEVMETHOD(pcib_write_config, psycho_write_config), DEVMETHOD(pcib_route_interrupt, psycho_route_interrupt), DEVMETHOD(pcib_request_feature, pcib_request_feature_allow), /* ofw_bus interface */ DEVMETHOD(ofw_bus_get_node, ofw_pci_get_node), /* ofw_pci interface */ DEVMETHOD(ofw_pci_setup_device, psycho_setup_device), DEVMETHOD_END }; static devclass_t psycho_devclass; DEFINE_CLASS_0(pcib, psycho_driver, psycho_methods, sizeof(struct psycho_softc)); EARLY_DRIVER_MODULE(psycho, nexus, psycho_driver, psycho_devclass, NULL, NULL, BUS_PASS_BUS); static SYSCTL_NODE(_hw, OID_AUTO, psycho, CTLFLAG_RD, 0, "psycho parameters"); static u_int psycho_powerfail = 1; SYSCTL_UINT(_hw_psycho, OID_AUTO, powerfail, CTLFLAG_RDTUN, &psycho_powerfail, 0, "powerfail action (0: none, 1: shutdown (default), 2: debugger)"); static SLIST_HEAD(, psycho_softc) psycho_softcs = SLIST_HEAD_INITIALIZER(psycho_softcs); static const struct intr_controller psycho_ic = { psycho_intr_enable, psycho_intr_disable, psycho_intr_assign, psycho_intr_clear }; struct psycho_icarg { struct psycho_softc *pica_sc; bus_addr_t pica_map; bus_addr_t pica_clr; }; #define PSYCHO_READ8(sc, off) \ bus_read_8((sc)->sc_mem_res, (off)) #define PSYCHO_WRITE8(sc, off, v) \ bus_write_8((sc)->sc_mem_res, (off), (v)) #define PCICTL_READ8(sc, off) \ PSYCHO_READ8((sc), (sc)->sc_pcictl + (off)) #define PCICTL_WRITE8(sc, off, v) \ PSYCHO_WRITE8((sc), (sc)->sc_pcictl + (off), (v)) /* * "Sabre" is the UltraSPARC IIi onboard UPA to PCI bridge. It manages a * single PCI bus and does not have a streaming buffer. It often has an APB * (advanced PCI bridge) connected to it, which was designed specifically for * the IIi. The APB lets the IIi handle two independent PCI buses, and * appears as two "Simba"'s underneath the Sabre. * * "Hummingbird" is the UltraSPARC IIe onboard UPA to PCI bridge. It's * basically the same as Sabre but without an APB underneath it. * * "Psycho" and "Psycho+" are dual UPA to PCI bridges. They sit on the UPA * bus and manage two PCI buses. "Psycho" has two 64-bit 33MHz buses, while * "Psycho+" controls both a 64-bit 33Mhz and a 64-bit 66Mhz PCI bus. You * will usually find a "Psycho+" since I don't think the original "Psycho" * ever shipped, and if it did it would be in the U30. * * Each "Psycho" PCI bus appears as a separate OFW node, but since they are * both part of the same IC, they only have a single register space. As such, * they need to be configured together, even though the autoconfiguration will * attach them separately. * * On UltraIIi machines, "Sabre" itself usually takes pci0, with "Simba" often * as pci1 and pci2, although they have been implemented with other PCI bus * numbers on some machines. * * On UltraII machines, there can be any number of "Psycho+" ICs, each * providing two PCI buses. */ struct psycho_desc { const char *pd_string; int pd_mode; const char *pd_name; }; static const struct psycho_desc psycho_compats[] = { { "pci108e,8000", PSYCHO_MODE_PSYCHO, "Psycho compatible" }, { "pci108e,a000", PSYCHO_MODE_SABRE, "Sabre compatible" }, { "pci108e,a001", PSYCHO_MODE_SABRE, "Hummingbird compatible" }, { NULL, 0, NULL } }; static const struct psycho_desc psycho_models[] = { { "SUNW,psycho", PSYCHO_MODE_PSYCHO, "Psycho" }, { "SUNW,sabre", PSYCHO_MODE_SABRE, "Sabre" }, { NULL, 0, NULL } }; static const struct psycho_desc * psycho_find_desc(const struct psycho_desc *table, const char *string) { const struct psycho_desc *desc; if (string == NULL) return (NULL); for (desc = table; desc->pd_string != NULL; desc++) if (strcmp(desc->pd_string, string) == 0) return (desc); return (NULL); } static const struct psycho_desc * psycho_get_desc(device_t dev) { const struct psycho_desc *rv; rv = psycho_find_desc(psycho_models, ofw_bus_get_model(dev)); if (rv == NULL) rv = psycho_find_desc(psycho_compats, ofw_bus_get_compat(dev)); return (rv); } static int psycho_probe(device_t dev) { const char *dtype; dtype = ofw_bus_get_type(dev); if (dtype != NULL && strcmp(dtype, OFW_TYPE_PCI) == 0 && psycho_get_desc(dev) != NULL) { device_set_desc(dev, "U2P UPA-PCI bridge"); return (0); } return (ENXIO); } static int psycho_attach(device_t dev) { struct psycho_icarg *pica; struct psycho_softc *asc, *sc, *osc; const struct psycho_desc *desc; bus_addr_t intrclr, intrmap; bus_dma_tag_t dmat; uint64_t csr, dr; phandle_t node; uint32_t dvmabase, prop; u_int rerun, ver; int i, j; node = ofw_bus_get_node(dev); sc = device_get_softc(dev); desc = psycho_get_desc(dev); sc->sc_dev = dev; sc->sc_mode = desc->pd_mode; /* * The Psycho gets three register banks: * (0) per-PBM configuration and status registers * (1) per-PBM PCI configuration space, containing only the * PBM 256-byte PCI header * (2) the shared Psycho configuration registers */ if (sc->sc_mode == PSYCHO_MODE_PSYCHO) { i = 2; sc->sc_pcictl = bus_get_resource_start(dev, SYS_RES_MEMORY, 0) - bus_get_resource_start(dev, SYS_RES_MEMORY, 2); switch (sc->sc_pcictl) { case PSR_PCICTL0: sc->sc_half = 0; break; case PSR_PCICTL1: sc->sc_half = 1; break; default: panic("%s: bogus PCI control register location", __func__); /* NOTREACHED */ } } else { i = 0; sc->sc_pcictl = PSR_PCICTL0; sc->sc_half = 0; } sc->sc_mem_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &i, (sc->sc_mode == PSYCHO_MODE_PSYCHO ? RF_SHAREABLE : 0) | RF_ACTIVE); if (sc->sc_mem_res == NULL) panic("%s: could not allocate registers", __func__); /* * Match other Psychos that are already configured against * the base physical address. This will be the same for a * pair of devices that share register space. */ osc = NULL; SLIST_FOREACH(asc, &psycho_softcs, sc_link) { if (rman_get_start(asc->sc_mem_res) == rman_get_start(sc->sc_mem_res)) { /* Found partner. */ osc = asc; break; } } if (osc == NULL) { sc->sc_mtx = malloc(sizeof(*sc->sc_mtx), M_DEVBUF, M_NOWAIT | M_ZERO); if (sc->sc_mtx == NULL) panic("%s: could not malloc mutex", __func__); mtx_init(sc->sc_mtx, "pcib_mtx", NULL, MTX_SPIN); } else { if (sc->sc_mode != PSYCHO_MODE_PSYCHO) panic("%s: no partner expected", __func__); if (mtx_initialized(osc->sc_mtx) == 0) panic("%s: mutex not initialized", __func__); sc->sc_mtx = osc->sc_mtx; } SLIST_INSERT_HEAD(&psycho_softcs, sc, sc_link); csr = PSYCHO_READ8(sc, PSR_CS); ver = PSYCHO_GCSR_VERS(csr); sc->sc_ign = 0x1f; /* Hummingbird/Sabre IGN is always 0x1f. */ if (sc->sc_mode == PSYCHO_MODE_PSYCHO) sc->sc_ign = PSYCHO_GCSR_IGN(csr); if (OF_getprop(node, "clock-frequency", &prop, sizeof(prop)) == -1) prop = 33000000; device_printf(dev, "%s, impl %d, version %d, IGN %#x, bus %c, %dMHz\n", desc->pd_name, (u_int)PSYCHO_GCSR_IMPL(csr), ver, sc->sc_ign, 'A' + sc->sc_half, prop / 1000 / 1000); /* Set up the PCI control and PCI diagnostic registers. */ csr = PCICTL_READ8(sc, PCR_CS); csr &= ~PCICTL_ARB_PARK; if (OF_getproplen(node, "no-bus-parking") < 0) csr |= PCICTL_ARB_PARK; /* Workarounds for version specific bugs. */ dr = PCICTL_READ8(sc, PCR_DIAG); switch (ver) { case 0: dr |= DIAG_RTRY_DIS; dr &= ~DIAG_DWSYNC_DIS; rerun = 0; break; case 1: csr &= ~PCICTL_ARB_PARK; dr |= DIAG_RTRY_DIS | DIAG_DWSYNC_DIS; rerun = 0; break; default: dr |= DIAG_DWSYNC_DIS; dr &= ~DIAG_RTRY_DIS; rerun = 1; break; } csr |= PCICTL_ERRINTEN | PCICTL_ARB_4; csr &= ~(PCICTL_SBHINTEN | PCICTL_WAKEUPEN); #ifdef PSYCHO_DEBUG device_printf(dev, "PCI CSR 0x%016llx -> 0x%016llx\n", (unsigned long long)PCICTL_READ8(sc, PCR_CS), (unsigned long long)csr); #endif PCICTL_WRITE8(sc, PCR_CS, csr); dr &= ~DIAG_ISYNC_DIS; #ifdef PSYCHO_DEBUG device_printf(dev, "PCI DR 0x%016llx -> 0x%016llx\n", (unsigned long long)PCICTL_READ8(sc, PCR_DIAG), (unsigned long long)dr); #endif PCICTL_WRITE8(sc, PCR_DIAG, dr); if (sc->sc_mode == PSYCHO_MODE_SABRE) { /* Use the PROM preset for now. */ csr = PCICTL_READ8(sc, PCR_TAS); if (csr == 0) panic("%s: Hummingbird/Sabre TAS not initialized.", __func__); dvmabase = (ffs(csr) - 1) << PCITAS_ADDR_SHIFT; } else dvmabase = -1; /* * If we're a Hummingbird/Sabre or the first of a pair of Psychos * to arrive here, do the interrupt setup and start up the IOMMU. */ if (osc == NULL) { /* * Hunt through all the interrupt mapping regs and register * our interrupt controller for the corresponding interrupt * vectors. We do this early in order to be able to catch * stray interrupts. */ for (i = 0; i <= PSYCHO_MAX_INO; i++) { if (psycho_find_intrmap(sc, i, &intrmap, &intrclr, NULL) == 0) continue; pica = malloc(sizeof(*pica), M_DEVBUF, M_NOWAIT); if (pica == NULL) panic("%s: could not allocate interrupt " "controller argument", __func__); pica->pica_sc = sc; pica->pica_map = intrmap; pica->pica_clr = intrclr; #ifdef PSYCHO_DEBUG /* * Enable all interrupts and clear all interrupt * states. This aids the debugging of interrupt * routing problems. */ device_printf(dev, "intr map (INO %d, %s) %#lx: %#lx, clr: %#lx\n", i, intrmap <= PSR_PCIB3_INT_MAP ? "PCI" : "OBIO", (u_long)intrmap, (u_long)PSYCHO_READ8(sc, intrmap), (u_long)intrclr); PSYCHO_WRITE8(sc, intrmap, INTMAP_VEC(sc->sc_ign, i)); PSYCHO_WRITE8(sc, intrclr, INTCLR_IDLE); PSYCHO_WRITE8(sc, intrmap, INTMAP_ENABLE(INTMAP_VEC(sc->sc_ign, i), PCPU_GET(mid))); #endif j = intr_controller_register(INTMAP_VEC(sc->sc_ign, i), &psycho_ic, pica); if (j != 0) device_printf(dev, "could not register " "interrupt controller for INO %d (%d)\n", i, j); } if (sc->sc_mode == PSYCHO_MODE_PSYCHO) sparc64_counter_init(device_get_nameunit(dev), rman_get_bustag(sc->sc_mem_res), rman_get_bushandle(sc->sc_mem_res), PSR_TC0); /* * Set up IOMMU and PCI configuration if we're the first * of a pair of Psychos to arrive here or a Hummingbird * or Sabre. * * We should calculate a TSB size based on amount of RAM * and number of bus controllers and number and type of * child devices. * * For the moment, 32KB should be more than enough. */ sc->sc_is = malloc(sizeof(*sc->sc_is), M_DEVBUF, M_NOWAIT | M_ZERO); if (sc->sc_is == NULL) panic("%s: could not malloc IOMMU state", __func__); sc->sc_is->is_flags = IOMMU_PRESERVE_PROM; if (sc->sc_mode == PSYCHO_MODE_SABRE) { sc->sc_dma_methods = malloc(sizeof(*sc->sc_dma_methods), M_DEVBUF, M_NOWAIT); if (sc->sc_dma_methods == NULL) panic("%s: could not malloc DMA methods", __func__); memcpy(sc->sc_dma_methods, &iommu_dma_methods, sizeof(*sc->sc_dma_methods)); sc->sc_dma_methods->dm_dmamap_sync = sabre_dmamap_sync; sc->sc_is->is_pmaxaddr = IOMMU_MAXADDR(SABRE_IOMMU_BITS); } else { sc->sc_dma_methods = &iommu_dma_methods; sc->sc_is->is_pmaxaddr = IOMMU_MAXADDR(PSYCHO_IOMMU_BITS); } sc->sc_is->is_sb[0] = sc->sc_is->is_sb[1] = 0; if (OF_getproplen(node, "no-streaming-cache") < 0) sc->sc_is->is_sb[0] = sc->sc_pcictl + PCR_STRBUF; sc->sc_is->is_flags |= (rerun != 1) ? IOMMU_RERUN_DISABLE : 0; psycho_iommu_init(sc, 3, dvmabase); } else { /* Just copy IOMMU state, config tag and address. */ sc->sc_dma_methods = &iommu_dma_methods; sc->sc_is = osc->sc_is; if (OF_getproplen(node, "no-streaming-cache") < 0) sc->sc_is->is_sb[1] = sc->sc_pcictl + PCR_STRBUF; iommu_reset(sc->sc_is); } /* Create our DMA tag. */ if (bus_dma_tag_create(bus_get_dma_tag(dev), 8, 0, sc->sc_is->is_pmaxaddr, ~0, NULL, NULL, sc->sc_is->is_pmaxaddr, 0xff, 0xffffffff, 0, NULL, NULL, &dmat) != 0) panic("%s: could not create PCI DMA tag", __func__); dmat->dt_cookie = sc->sc_is; dmat->dt_mt = sc->sc_dma_methods; if (ofw_pci_attach_common(dev, dmat, PSYCHO_IO_SIZE, PSYCHO_MEM_SIZE) != 0) panic("%s: ofw_pci_attach_common() failed", __func__); /* Clear any pending PCI error bits. */ PCIB_WRITE_CONFIG(dev, sc->sc_ops.sc_pci_secbus, PCS_DEVICE, PCS_FUNC, PCIR_STATUS, PCIB_READ_CONFIG(dev, sc->sc_ops.sc_pci_secbus, PCS_DEVICE, PCS_FUNC, PCIR_STATUS, 2), 2); PCICTL_WRITE8(sc, PCR_CS, PCICTL_READ8(sc, PCR_CS)); PCICTL_WRITE8(sc, PCR_AFS, PCICTL_READ8(sc, PCR_AFS)); if (osc == NULL) { /* * Establish handlers for interesting interrupts... * * XXX We need to remember these and remove this to support * hotplug on the UPA/FHC bus. * * XXX Not all controllers have these, but installing them * is better than trying to sort through this mess. */ psycho_set_intr(sc, 1, PSR_UE_INT_MAP, psycho_ue, NULL); psycho_set_intr(sc, 2, PSR_CE_INT_MAP, psycho_ce, NULL); switch (psycho_powerfail) { case 0: break; case 2: psycho_set_intr(sc, 3, PSR_POWER_INT_MAP, psycho_powerdebug, NULL); break; default: psycho_set_intr(sc, 3, PSR_POWER_INT_MAP, NULL, psycho_powerdown); break; } if (sc->sc_mode == PSYCHO_MODE_PSYCHO) { /* * Hummingbirds/Sabres do not have the following two * interrupts. */ /* * The spare hardware interrupt is used for the * over-temperature interrupt. */ psycho_set_intr(sc, 4, PSR_SPARE_INT_MAP, NULL, psycho_overtemp); #ifdef PSYCHO_MAP_WAKEUP /* * psycho_wakeup() doesn't do anything useful right * now. */ psycho_set_intr(sc, 5, PSR_PWRMGT_INT_MAP, psycho_wakeup, NULL); #endif /* PSYCHO_MAP_WAKEUP */ } } /* * Register a PCI bus error interrupt handler according to which * half this is. Hummingbird/Sabre don't have a PCI bus B error * interrupt but they are also only used for PCI bus A. */ psycho_set_intr(sc, 0, sc->sc_half == 0 ? PSR_PCIAERR_INT_MAP : PSR_PCIBERR_INT_MAP, psycho_pci_bus, NULL); /* * Set the latency timer register as this isn't always done by the * firmware. */ PCIB_WRITE_CONFIG(dev, sc->sc_ops.sc_pci_secbus, PCS_DEVICE, PCS_FUNC, PCIR_LATTIMER, OFW_PCI_LATENCY, 1); for (i = PCIR_VENDOR; i < PCIR_STATUS; i += sizeof(uint16_t)) le16enc(&sc->sc_pci_hpbcfg[i], bus_space_read_2(sc->sc_ops.sc_pci_cfgt, sc->sc_ops.sc_pci_bh[OFW_PCI_CS_CONFIG], PSYCHO_CONF_OFF(sc->sc_ops.sc_pci_secbus, PCS_DEVICE, PCS_FUNC, i))); for (i = PCIR_REVID; i <= PCIR_BIST; i += sizeof(uint8_t)) sc->sc_pci_hpbcfg[i] = bus_space_read_1(sc->sc_ops.sc_pci_cfgt, sc->sc_ops.sc_pci_bh[OFW_PCI_CS_CONFIG], PSYCHO_CONF_OFF( sc->sc_ops.sc_pci_secbus, PCS_DEVICE, PCS_FUNC, i)); /* * On E250 the interrupt map entry for the EBus bridge is wrong, * causing incorrect interrupts to be assigned to some devices on * the EBus. Work around it by changing our copy of the interrupt * map mask to perform a full comparison of the INO. That way * the interrupt map entry for the EBus bridge won't match at all * and the INOs specified in the "interrupts" properties of the * EBus devices will be used directly instead. */ if (strcmp(sparc64_model, "SUNW,Ultra-250") == 0 && sc->sc_ops.sc_pci_iinfo.opi_imapmsk != NULL) *(ofw_pci_intr_t *)(&sc->sc_ops.sc_pci_iinfo.opi_imapmsk[ sc->sc_ops.sc_pci_iinfo.opi_addrc]) = INTMAP_INO_MASK; device_add_child(dev, "pci", -1); return (bus_generic_attach(dev)); } static void psycho_set_intr(struct psycho_softc *sc, u_int index, bus_addr_t intrmap, driver_filter_t filt, driver_intr_t intr) { u_long vec; int rid; rid = index; sc->sc_irq_res[index] = bus_alloc_resource_any(sc->sc_dev, SYS_RES_IRQ, &rid, RF_ACTIVE); if (sc->sc_irq_res[index] == NULL && intrmap >= PSR_POWER_INT_MAP) { /* * These interrupts aren't mandatory and not available * with all controllers (not even Psychos). */ return; } if (sc->sc_irq_res[index] == NULL || INTIGN(vec = rman_get_start(sc->sc_irq_res[index])) != sc->sc_ign || INTVEC(PSYCHO_READ8(sc, intrmap)) != vec || intr_vectors[vec].iv_ic != &psycho_ic || bus_setup_intr(sc->sc_dev, sc->sc_irq_res[index], INTR_TYPE_MISC | INTR_BRIDGE, filt, intr, sc, &sc->sc_ihand[index]) != 0) panic("%s: failed to set up interrupt %d", __func__, index); } static int psycho_find_intrmap(struct psycho_softc *sc, u_int ino, bus_addr_t *intrmapptr, bus_addr_t *intrclrptr, bus_addr_t *intrdiagptr) { bus_addr_t intrclr, intrmap; uint64_t diag; int found; /* * XXX we only compare INOs rather than INRs since the firmware may * not provide the IGN and the IGN is constant for all devices on * that PCI controller. * This could cause problems for the FFB/external interrupt which * has a full vector that can be set arbitrarily. */ if (ino > PSYCHO_MAX_INO) { device_printf(sc->sc_dev, "out of range INO %d requested\n", ino); return (0); } found = 0; /* Hunt through OBIO first. */ diag = PSYCHO_READ8(sc, PSR_OBIO_INT_DIAG); for (intrmap = PSR_SCSI_INT_MAP, intrclr = PSR_SCSI_INT_CLR; intrmap <= PSR_PWRMGT_INT_MAP; intrmap += 8, intrclr += 8, diag >>= 2) { if (sc->sc_mode == PSYCHO_MODE_SABRE && (intrmap == PSR_TIMER0_INT_MAP || intrmap == PSR_TIMER1_INT_MAP || intrmap == PSR_PCIBERR_INT_MAP || intrmap == PSR_PWRMGT_INT_MAP)) continue; if (INTINO(PSYCHO_READ8(sc, intrmap)) == ino) { diag &= 2; found = 1; break; } } if (!found) { diag = PSYCHO_READ8(sc, PSR_PCI_INT_DIAG); /* Now do PCI interrupts. */ for (intrmap = PSR_PCIA0_INT_MAP, intrclr = PSR_PCIA0_INT_CLR; intrmap <= PSR_PCIB3_INT_MAP; intrmap += 8, intrclr += 32, diag >>= 8) { if (sc->sc_mode == PSYCHO_MODE_PSYCHO && (intrmap == PSR_PCIA2_INT_MAP || intrmap == PSR_PCIA3_INT_MAP)) continue; if (((PSYCHO_READ8(sc, intrmap) ^ ino) & 0x3c) == 0) { intrclr += 8 * (ino & 3); diag = (diag >> ((ino & 3) * 2)) & 2; found = 1; break; } } } if (intrmapptr != NULL) *intrmapptr = intrmap; if (intrclrptr != NULL) *intrclrptr = intrclr; if (intrdiagptr != NULL) *intrdiagptr = diag; return (found); } /* * Interrupt handlers */ static int psycho_ue(void *arg) { struct psycho_softc *sc = arg; uint64_t afar, afsr; afar = PSYCHO_READ8(sc, PSR_UE_AFA); afsr = PSYCHO_READ8(sc, PSR_UE_AFS); /* * On the UltraSPARC-IIi/IIe, IOMMU misses/protection faults cause * the AFAR to be set to the physical address of the TTE entry that * was invalid/write protected. Call into the IOMMU code to have * them decoded to virtual I/O addresses. */ if ((afsr & UEAFSR_P_DTE) != 0) iommu_decode_fault(sc->sc_is, afar); panic("%s: uncorrectable DMA error AFAR %#lx AFSR %#lx", device_get_nameunit(sc->sc_dev), (u_long)afar, (u_long)afsr); return (FILTER_HANDLED); } static int psycho_ce(void *arg) { struct psycho_softc *sc = arg; uint64_t afar, afsr; mtx_lock_spin(sc->sc_mtx); afar = PSYCHO_READ8(sc, PSR_CE_AFA); afsr = PSYCHO_READ8(sc, PSR_CE_AFS); device_printf(sc->sc_dev, "correctable DMA error AFAR %#lx " "AFSR %#lx\n", (u_long)afar, (u_long)afsr); /* Clear the error bits that we caught. */ PSYCHO_WRITE8(sc, PSR_CE_AFS, afsr); mtx_unlock_spin(sc->sc_mtx); return (FILTER_HANDLED); } static int psycho_pci_bus(void *arg) { struct psycho_softc *sc = arg; uint64_t afar, afsr; afar = PCICTL_READ8(sc, PCR_AFA); afsr = PCICTL_READ8(sc, PCR_AFS); panic("%s: PCI bus %c error AFAR %#lx AFSR %#lx", device_get_nameunit(sc->sc_dev), 'A' + sc->sc_half, (u_long)afar, (u_long)afsr); return (FILTER_HANDLED); } static int psycho_powerdebug(void *arg __unused) { kdb_enter(KDB_WHY_POWERFAIL, "powerfail"); return (FILTER_HANDLED); } static void psycho_powerdown(void *arg __unused) { static int shutdown; /* As the interrupt is cleared we may be called multiple times. */ if (shutdown != 0) return; shutdown++; printf("Power Failure Detected: Shutting down NOW.\n"); shutdown_nice(RB_POWEROFF); } static void psycho_overtemp(void *arg __unused) { static int shutdown; /* As the interrupt is cleared we may be called multiple times. */ if (shutdown != 0) return; shutdown++; printf("DANGER: OVER TEMPERATURE detected.\nShutting down NOW.\n"); shutdown_nice(RB_POWEROFF); } #ifdef PSYCHO_MAP_WAKEUP static int psycho_wakeup(void *arg) { struct psycho_softc *sc = arg; /* We don't really have a framework to deal with this properly. */ device_printf(sc->sc_dev, "power management wakeup\n"); return (FILTER_HANDLED); } #endif /* PSYCHO_MAP_WAKEUP */ static void psycho_iommu_init(struct psycho_softc *sc, int tsbsize, uint32_t dvmabase) { struct iommu_state *is = sc->sc_is; /* Punch in our copies. */ is->is_bustag = rman_get_bustag(sc->sc_mem_res); is->is_bushandle = rman_get_bushandle(sc->sc_mem_res); is->is_iommu = PSR_IOMMU; is->is_dtag = PSR_IOMMU_TLB_TAG_DIAG; is->is_ddram = PSR_IOMMU_TLB_DATA_DIAG; is->is_dqueue = PSR_IOMMU_QUEUE_DIAG; is->is_dva = PSR_IOMMU_SVADIAG; is->is_dtcmp = PSR_IOMMU_TLB_CMP_DIAG; iommu_init(device_get_nameunit(sc->sc_dev), is, tsbsize, dvmabase, 0); } static int psycho_maxslots(device_t dev) { /* XXX: is this correct? */ return (PCI_SLOTMAX); } static uint32_t psycho_read_config(device_t dev, u_int bus, u_int slot, u_int func, u_int reg, int width) { struct psycho_softc *sc; sc = device_get_softc(dev); /* * The Hummingbird and Sabre bridges are picky in that they * only allow their config space to be accessed using the * "native" width of the respective register being accessed * and return semi-random other content of their config space * otherwise. Given that the PCI specs don't say anything * about such a (unusual) limitation and lots of stuff expects * to be able to access the contents of the config space at * any width we allow just that. We do this by using a copy * of the header of the bridge (the rest is all zero anyway) * read during attach (expect for PCIR_STATUS) in order to * simplify things. * The Psycho bridges contain a dupe of their header at 0x80 * which we nullify that way also. */ if (bus == sc->sc_ops.sc_pci_secbus && slot == PCS_DEVICE && func == PCS_FUNC) { if (reg % width != 0) return (-1); if (reg >= sizeof(sc->sc_pci_hpbcfg)) return (0); if ((reg < PCIR_STATUS && reg + width > PCIR_STATUS) || reg == PCIR_STATUS || reg == PCIR_STATUS + 1) le16enc(&sc->sc_pci_hpbcfg[PCIR_STATUS], bus_space_read_2(sc->sc_ops.sc_pci_cfgt, sc->sc_ops.sc_pci_bh[OFW_PCI_CS_CONFIG], PSYCHO_CONF_OFF(sc->sc_ops.sc_pci_secbus, PCS_DEVICE, PCS_FUNC, PCIR_STATUS))); switch (width) { case 1: return (sc->sc_pci_hpbcfg[reg]); case 2: return (le16dec(&sc->sc_pci_hpbcfg[reg])); case 4: return (le32dec(&sc->sc_pci_hpbcfg[reg])); } } return (ofw_pci_read_config_common(dev, PCI_REGMAX, PSYCHO_CONF_OFF(bus, slot, func, reg), bus, slot, func, reg, width)); } static void psycho_write_config(device_t dev, u_int bus, u_int slot, u_int func, u_int reg, uint32_t val, int width) { ofw_pci_write_config_common(dev, PCI_REGMAX, PSYCHO_CONF_OFF(bus, slot, func, reg), bus, slot, func, reg, val, width); } static int psycho_route_interrupt(device_t bridge, device_t dev, int pin) { struct psycho_softc *sc; bus_addr_t intrmap; ofw_pci_intr_t mintr; mintr = ofw_pci_route_interrupt_common(bridge, dev, pin); if (PCI_INTERRUPT_VALID(mintr)) return (mintr); /* * If this is outside of the range for an intpin, it's likely a full * INO, and no mapping is required at all; this happens on the U30, * where there's no interrupt map at the Psycho node. Fortunately, * there seem to be no INOs in the intpin range on this boxen, so * this easy heuristics will do. */ if (pin > 4) return (pin); /* * Guess the INO; we always assume that this is a non-OBIO * device, and that pin is a "real" intpin number. Determine * the mapping register to be used by the slot number. * We only need to do this on E450s, it seems; here, the slot numbers * for bus A are one-based, while those for bus B seemingly have an * offset of 2 (hence the factor of 3 below). */ sc = device_get_softc(dev); intrmap = PSR_PCIA0_INT_MAP + 8 * (pci_get_slot(dev) - 1 + 3 * sc->sc_half); mintr = INTINO(PSYCHO_READ8(sc, intrmap)) + pin - 1; device_printf(bridge, "guessing interrupt %d for device %d.%d pin %d\n", (int)mintr, pci_get_slot(dev), pci_get_function(dev), pin); return (mintr); } static void sabre_dmamap_sync(bus_dma_tag_t dt, bus_dmamap_t map, bus_dmasync_op_t op) { struct iommu_state *is = dt->dt_cookie; if ((map->dm_flags & DMF_LOADED) == 0) return; if ((op & BUS_DMASYNC_POSTREAD) != 0) (void)bus_space_read_8(is->is_bustag, is->is_bushandle, PSR_DMA_WRITE_SYNC); if ((op & BUS_DMASYNC_PREWRITE) != 0) membar(Sync); } static void psycho_intr_enable(void *arg) { struct intr_vector *iv = arg; struct psycho_icarg *pica = iv->iv_icarg; PSYCHO_WRITE8(pica->pica_sc, pica->pica_map, INTMAP_ENABLE(iv->iv_vec, iv->iv_mid)); } static void psycho_intr_disable(void *arg) { struct intr_vector *iv = arg; struct psycho_icarg *pica = iv->iv_icarg; PSYCHO_WRITE8(pica->pica_sc, pica->pica_map, iv->iv_vec); } static void psycho_intr_assign(void *arg) { struct intr_vector *iv = arg; struct psycho_icarg *pica = iv->iv_icarg; PSYCHO_WRITE8(pica->pica_sc, pica->pica_map, INTMAP_TID( PSYCHO_READ8(pica->pica_sc, pica->pica_map), iv->iv_mid)); } static void psycho_intr_clear(void *arg) { struct intr_vector *iv = arg; struct psycho_icarg *pica = iv->iv_icarg; PSYCHO_WRITE8(pica->pica_sc, pica->pica_clr, INTCLR_IDLE); } static int psycho_setup_intr(device_t dev, device_t child, struct resource *ires, int flags, driver_filter_t *filt, driver_intr_t *intr, void *arg, void **cookiep) { struct psycho_softc *sc; u_long vec; sc = device_get_softc(dev); /* * Make sure the vector is fully specified and we registered * our interrupt controller for it. */ vec = rman_get_start(ires); if (INTIGN(vec) != sc->sc_ign || intr_vectors[vec].iv_ic != &psycho_ic) { device_printf(dev, "invalid interrupt vector 0x%lx\n", vec); return (EINVAL); } return (bus_generic_setup_intr(dev, child, ires, flags, filt, intr, arg, cookiep)); } static struct resource * psycho_alloc_resource(device_t bus, device_t child, int type, int *rid, rman_res_t start, rman_res_t end, rman_res_t count, u_int flags) { struct psycho_softc *sc; if (type == SYS_RES_IRQ) { sc = device_get_softc(bus); start = end = INTMAP_VEC(sc->sc_ign, end); } return (ofw_pci_alloc_resource(bus, child, type, rid, start, end, count, flags)); } static void psycho_setup_device(device_t bus, device_t child) { struct psycho_softc *sc; uint32_t rev; sc = device_get_softc(bus); /* * Revision 0 EBus bridges have a bug which prevents them from * working when bus parking is enabled. */ if ((strcmp(ofw_bus_get_name(child), "ebus") == 0 || strcmp(ofw_bus_get_name(child), "pci108e,1000") == 0) && OF_getprop(ofw_bus_get_node(child), "revision-id", &rev, sizeof(rev)) > 0 && rev == 0) PCICTL_WRITE8(sc, PCR_CS, PCICTL_READ8(sc, PCR_CS) & ~PCICTL_ARB_PARK); } Index: head/sys/sparc64/pci/schizo.c =================================================================== --- head/sys/sparc64/pci/schizo.c (revision 314316) +++ head/sys/sparc64/pci/schizo.c (revision 314317) @@ -1,1257 +1,1258 @@ /*- * Copyright (c) 1999, 2000 Matthew R. Green * Copyright (c) 2001 - 2003 by Thomas Moestl * Copyright (c) 2005 - 2011 by Marius Strobl * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. The name of the author may not be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * from: NetBSD: psycho.c,v 1.39 2001/10/07 20:30:41 eeh Exp * from: FreeBSD: psycho.c 183152 2008-09-18 19:45:22Z marius */ #include __FBSDID("$FreeBSD$"); /* * Driver for `Schizo' Fireplane/Safari to PCI 2.1, `Tomatillo' JBus to * PCI 2.2 and `XMITS' Fireplane/Safari to PCI-X bridges */ #include "opt_ofw_pci.h" #include "opt_schizo.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include +#include #include #include #include #include "pcib_if.h" static const struct schizo_desc *schizo_get_desc(device_t); static void schizo_set_intr(struct schizo_softc *, u_int, u_int, driver_filter_t); static void schizo_dmamap_sync(bus_dma_tag_t dt, bus_dmamap_t map, bus_dmasync_op_t op); static void ichip_dmamap_sync(bus_dma_tag_t dt, bus_dmamap_t map, bus_dmasync_op_t op); static void schizo_intr_enable(void *); static void schizo_intr_disable(void *); static void schizo_intr_assign(void *); static void schizo_intr_clear(void *); static int schizo_intr_register(struct schizo_softc *sc, u_int ino); static int schizo_get_intrmap(struct schizo_softc *, u_int, bus_addr_t *, bus_addr_t *); static timecounter_get_t schizo_get_timecount; /* Interrupt handlers */ static driver_filter_t schizo_pci_bus; static driver_filter_t schizo_ue; static driver_filter_t schizo_ce; static driver_filter_t schizo_host_bus; static driver_filter_t schizo_cdma; /* IOMMU support */ static void schizo_iommu_init(struct schizo_softc *, int, uint32_t); /* * Methods */ static device_probe_t schizo_probe; static device_attach_t schizo_attach; static bus_setup_intr_t schizo_setup_intr; static bus_alloc_resource_t schizo_alloc_resource; static pcib_maxslots_t schizo_maxslots; static pcib_read_config_t schizo_read_config; static pcib_write_config_t schizo_write_config; static pcib_route_interrupt_t schizo_route_interrupt; static ofw_pci_setup_device_t schizo_setup_device; static device_method_t schizo_methods[] = { /* Device interface */ DEVMETHOD(device_probe, schizo_probe), DEVMETHOD(device_attach, schizo_attach), DEVMETHOD(device_shutdown, bus_generic_shutdown), DEVMETHOD(device_suspend, bus_generic_suspend), DEVMETHOD(device_resume, bus_generic_resume), /* Bus interface */ DEVMETHOD(bus_read_ivar, ofw_pci_read_ivar), DEVMETHOD(bus_setup_intr, schizo_setup_intr), DEVMETHOD(bus_teardown_intr, bus_generic_teardown_intr), DEVMETHOD(bus_alloc_resource, schizo_alloc_resource), DEVMETHOD(bus_activate_resource, ofw_pci_activate_resource), DEVMETHOD(bus_deactivate_resource, bus_generic_deactivate_resource), DEVMETHOD(bus_adjust_resource, ofw_pci_adjust_resource), DEVMETHOD(bus_release_resource, bus_generic_release_resource), DEVMETHOD(bus_get_dma_tag, ofw_pci_get_dma_tag), /* pcib interface */ DEVMETHOD(pcib_maxslots, schizo_maxslots), DEVMETHOD(pcib_read_config, schizo_read_config), DEVMETHOD(pcib_write_config, schizo_write_config), DEVMETHOD(pcib_route_interrupt, schizo_route_interrupt), DEVMETHOD(pcib_request_feature, pcib_request_feature_allow), /* ofw_bus interface */ DEVMETHOD(ofw_bus_get_node, ofw_pci_get_node), /* ofw_pci interface */ DEVMETHOD(ofw_pci_setup_device, schizo_setup_device), DEVMETHOD_END }; static devclass_t schizo_devclass; DEFINE_CLASS_0(pcib, schizo_driver, schizo_methods, sizeof(struct schizo_softc)); EARLY_DRIVER_MODULE(schizo, nexus, schizo_driver, schizo_devclass, 0, 0, BUS_PASS_BUS); static SLIST_HEAD(, schizo_softc) schizo_softcs = SLIST_HEAD_INITIALIZER(schizo_softcs); static const struct intr_controller schizo_ic = { schizo_intr_enable, schizo_intr_disable, schizo_intr_assign, schizo_intr_clear }; struct schizo_icarg { struct schizo_softc *sica_sc; bus_addr_t sica_map; bus_addr_t sica_clr; }; #define SCHIZO_CDMA_TIMEOUT 1 /* 1 second per try */ #define SCHIZO_CDMA_TRIES 15 #define SCHIZO_PERF_CNT_QLTY 100 #define SCHIZO_SPC_BARRIER(spc, sc, offs, len, flags) \ bus_barrier((sc)->sc_mem_res[(spc)], (offs), (len), (flags)) #define SCHIZO_SPC_READ_8(spc, sc, offs) \ bus_read_8((sc)->sc_mem_res[(spc)], (offs)) #define SCHIZO_SPC_WRITE_8(spc, sc, offs, v) \ bus_write_8((sc)->sc_mem_res[(spc)], (offs), (v)) #ifndef SCHIZO_DEBUG #define SCHIZO_SPC_SET(spc, sc, offs, reg, v) \ SCHIZO_SPC_WRITE_8((spc), (sc), (offs), (v)) #else #define SCHIZO_SPC_SET(spc, sc, offs, reg, v) do { \ device_printf((sc)->sc_dev, reg " 0x%016llx -> 0x%016llx\n", \ (unsigned long long)SCHIZO_SPC_READ_8((spc), (sc), (offs)), \ (unsigned long long)(v)); \ SCHIZO_SPC_WRITE_8((spc), (sc), (offs), (v)); \ } while (0) #endif #define SCHIZO_PCI_READ_8(sc, offs) \ SCHIZO_SPC_READ_8(STX_PCI, (sc), (offs)) #define SCHIZO_PCI_WRITE_8(sc, offs, v) \ SCHIZO_SPC_WRITE_8(STX_PCI, (sc), (offs), (v)) #define SCHIZO_CTRL_READ_8(sc, offs) \ SCHIZO_SPC_READ_8(STX_CTRL, (sc), (offs)) #define SCHIZO_CTRL_WRITE_8(sc, offs, v) \ SCHIZO_SPC_WRITE_8(STX_CTRL, (sc), (offs), (v)) #define SCHIZO_PCICFG_READ_8(sc, offs) \ SCHIZO_SPC_READ_8(STX_PCICFG, (sc), (offs)) #define SCHIZO_PCICFG_WRITE_8(sc, offs, v) \ SCHIZO_SPC_WRITE_8(STX_PCICFG, (sc), (offs), (v)) #define SCHIZO_ICON_READ_8(sc, offs) \ SCHIZO_SPC_READ_8(STX_ICON, (sc), (offs)) #define SCHIZO_ICON_WRITE_8(sc, offs, v) \ SCHIZO_SPC_WRITE_8(STX_ICON, (sc), (offs), (v)) #define SCHIZO_PCI_SET(sc, offs, v) \ SCHIZO_SPC_SET(STX_PCI, (sc), (offs), # offs, (v)) #define SCHIZO_CTRL_SET(sc, offs, v) \ SCHIZO_SPC_SET(STX_CTRL, (sc), (offs), # offs, (v)) struct schizo_desc { const char *sd_string; int sd_mode; const char *sd_name; }; static const struct schizo_desc schizo_compats[] = { { "pci108e,8001", SCHIZO_MODE_SCZ, "Schizo" }, #if 0 { "pci108e,8002", SCHIZO_MODE_XMS, "XMITS" }, #endif { "pci108e,a801", SCHIZO_MODE_TOM, "Tomatillo" }, { NULL, 0, NULL } }; static const struct schizo_desc * schizo_get_desc(device_t dev) { const struct schizo_desc *desc; const char *compat; compat = ofw_bus_get_compat(dev); if (compat == NULL) return (NULL); for (desc = schizo_compats; desc->sd_string != NULL; desc++) if (strcmp(desc->sd_string, compat) == 0) return (desc); return (NULL); } static int schizo_probe(device_t dev) { const char *dtype; dtype = ofw_bus_get_type(dev); if (dtype != NULL && strcmp(dtype, OFW_TYPE_PCI) == 0 && schizo_get_desc(dev) != NULL) { device_set_desc(dev, "Sun Host-PCI bridge"); return (0); } return (ENXIO); } static int schizo_attach(device_t dev) { const struct schizo_desc *desc; struct schizo_softc *asc, *sc, *osc; struct timecounter *tc; bus_dma_tag_t dmat; uint64_t ino_bitmap, reg; phandle_t node; uint32_t prop, prop_array[2]; int i, j, mode, rid, tsbsize; sc = device_get_softc(dev); node = ofw_bus_get_node(dev); desc = schizo_get_desc(dev); mode = desc->sd_mode; sc->sc_dev = dev; sc->sc_mode = mode; sc->sc_flags = 0; /* * The Schizo has three register banks: * (0) per-PBM PCI configuration and status registers, but for bus B * shared with the UPA64s interrupt mapping register banks * (1) shared Schizo controller configuration and status registers * (2) per-PBM PCI configuration space * * The Tomatillo has four register banks: * (0) per-PBM PCI configuration and status registers * (1) per-PBM Tomatillo controller configuration registers, but on * machines having the `jbusppm' device shared with its Estar * register bank for bus A * (2) per-PBM PCI configuration space * (3) per-PBM interrupt concentrator registers */ sc->sc_half = (bus_get_resource_start(dev, SYS_RES_MEMORY, STX_PCI) >> 20) & 1; for (i = 0; i < (mode == SCHIZO_MODE_SCZ ? SCZ_NREG : TOM_NREG); i++) { rid = i; sc->sc_mem_res[i] = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, (((mode == SCHIZO_MODE_SCZ && ((sc->sc_half == 1 && i == STX_PCI) || i == STX_CTRL)) || (mode == SCHIZO_MODE_TOM && sc->sc_half == 0 && i == STX_CTRL)) ? RF_SHAREABLE : 0) | RF_ACTIVE); if (sc->sc_mem_res[i] == NULL) panic("%s: could not allocate register bank %d", __func__, i); } /* * Match other Schizos that are already configured against * the controller base physical address. This will be the * same for a pair of devices that share register space. */ osc = NULL; SLIST_FOREACH(asc, &schizo_softcs, sc_link) { if (rman_get_start(asc->sc_mem_res[STX_CTRL]) == rman_get_start(sc->sc_mem_res[STX_CTRL])) { /* Found partner. */ osc = asc; break; } } if (osc == NULL) { sc->sc_mtx = malloc(sizeof(*sc->sc_mtx), M_DEVBUF, M_NOWAIT | M_ZERO); if (sc->sc_mtx == NULL) panic("%s: could not malloc mutex", __func__); mtx_init(sc->sc_mtx, "pcib_mtx", NULL, MTX_SPIN); } else { if (sc->sc_mode != SCHIZO_MODE_SCZ) panic("%s: no partner expected", __func__); if (mtx_initialized(osc->sc_mtx) == 0) panic("%s: mutex not initialized", __func__); sc->sc_mtx = osc->sc_mtx; } SLIST_INSERT_HEAD(&schizo_softcs, sc, sc_link); if (OF_getprop(node, "portid", &sc->sc_ign, sizeof(sc->sc_ign)) == -1) panic("%s: could not determine IGN", __func__); if (OF_getprop(node, "version#", &sc->sc_ver, sizeof(sc->sc_ver)) == -1) panic("%s: could not determine version", __func__); if (mode == SCHIZO_MODE_XMS && OF_getprop(node, "module-revision#", &sc->sc_mrev, sizeof(sc->sc_mrev)) == -1) panic("%s: could not determine module-revision", __func__); if (OF_getprop(node, "clock-frequency", &prop, sizeof(prop)) == -1) prop = 33000000; if (mode == SCHIZO_MODE_XMS && (SCHIZO_PCI_READ_8(sc, STX_PCI_CTRL) & XMS_PCI_CTRL_X_MODE) != 0) { if (sc->sc_mrev < 1) panic("PCI-X mode unsupported"); sc->sc_flags |= SCHIZO_FLAGS_XMODE; } device_printf(dev, "%s, version %d, ", desc->sd_name, sc->sc_ver); if (mode == SCHIZO_MODE_XMS) printf("module-revision %d, ", sc->sc_mrev); printf("IGN %#x, bus %c, PCI%s mode, %dMHz\n", sc->sc_ign, 'A' + sc->sc_half, (sc->sc_flags & SCHIZO_FLAGS_XMODE) != 0 ? "-X" : "", prop / 1000 / 1000); /* Set up the PCI interrupt retry timer. */ SCHIZO_PCI_SET(sc, STX_PCI_INTR_RETRY_TIM, 5); /* Set up the PCI control register. */ reg = SCHIZO_PCI_READ_8(sc, STX_PCI_CTRL); reg &= ~(TOM_PCI_CTRL_DTO_IEN | STX_PCI_CTRL_ARB_PARK | STX_PCI_CTRL_ARB_MASK); reg |= STX_PCI_CTRL_MMU_IEN | STX_PCI_CTRL_SBH_IEN | STX_PCI_CTRL_ERR_IEN; if (OF_getproplen(node, "no-bus-parking") < 0) reg |= STX_PCI_CTRL_ARB_PARK; if (mode == SCHIZO_MODE_XMS && sc->sc_mrev == 1) reg |= XMS_PCI_CTRL_XMITS10_ARB_MASK; else reg |= STX_PCI_CTRL_ARB_MASK; if (mode == SCHIZO_MODE_TOM) { reg |= TOM_PCI_CTRL_PRM | TOM_PCI_CTRL_PRO | TOM_PCI_CTRL_PRL; if (sc->sc_ver <= 1) /* revision <= 2.0 */ reg |= TOM_PCI_CTRL_DTO_IEN; else reg |= STX_PCI_CTRL_PTO; } else if (mode == SCHIZO_MODE_XMS) { SCHIZO_PCI_SET(sc, XMS_PCI_PARITY_DETECT, 0x3fff); SCHIZO_PCI_SET(sc, XMS_PCI_UPPER_RETRY_COUNTER, 0x3e8); reg |= XMS_PCI_CTRL_X_ERRINT_EN; } SCHIZO_PCI_SET(sc, STX_PCI_CTRL, reg); /* Set up the PCI diagnostic register. */ reg = SCHIZO_PCI_READ_8(sc, STX_PCI_DIAG); reg &= ~(SCZ_PCI_DIAG_RTRYARB_DIS | STX_PCI_DIAG_RETRY_DIS | STX_PCI_DIAG_INTRSYNC_DIS); SCHIZO_PCI_SET(sc, STX_PCI_DIAG, reg); /* * Enable DMA write parity error interrupts of version >= 7 (i.e. * revision >= 2.5) Schizo and XMITS (enabling it on XMITS < 3.0 has * no effect though). */ if ((mode == SCHIZO_MODE_SCZ && sc->sc_ver >= 7) || mode == SCHIZO_MODE_XMS) { reg = SCHIZO_PCI_READ_8(sc, SX_PCI_CFG_ICD); reg |= SX_PCI_CFG_ICD_DMAW_PERR_IEN; SCHIZO_PCI_SET(sc, SX_PCI_CFG_ICD, reg); } /* * On Tomatillo clear the I/O prefetch lengths (workaround for a * Jalapeno bug). */ if (mode == SCHIZO_MODE_TOM) SCHIZO_PCI_SET(sc, TOM_PCI_IOC_CSR, TOM_PCI_IOC_PW | (1 << TOM_PCI_IOC_PREF_OFF_SHIFT) | TOM_PCI_IOC_CPRM | TOM_PCI_IOC_CPRO | TOM_PCI_IOC_CPRL); /* * Hunt through all the interrupt mapping regs and register * the interrupt controller for our interrupt vectors. We do * this early in order to be able to catch stray interrupts. * This is complicated by the fact that a pair of Schizo PBMs * shares one IGN. */ i = OF_getprop(node, "ino-bitmap", (void *)prop_array, sizeof(prop_array)); if (i != -1) ino_bitmap = ((uint64_t)prop_array[1] << 32) | prop_array[0]; else { /* * If the ino-bitmap property is missing, just provide the * default set of interrupts for this controller and let * schizo_setup_intr() take care of child interrupts. */ if (sc->sc_half == 0) ino_bitmap = (1ULL << STX_UE_INO) | (1ULL << STX_CE_INO) | (1ULL << STX_PCIERR_A_INO) | (1ULL << STX_BUS_INO); else ino_bitmap = 1ULL << STX_PCIERR_B_INO; } for (i = 0; i <= STX_MAX_INO; i++) { if ((ino_bitmap & (1ULL << i)) == 0) continue; if (i == STX_FB0_INO || i == STX_FB1_INO) /* Leave for upa(4). */ continue; j = schizo_intr_register(sc, i); if (j != 0) device_printf(dev, "could not register interrupt " "controller for INO %d (%d)\n", i, j); } /* * Setup Safari/JBus performance counter 0 in bus cycle counting * mode as timecounter. Unfortunately, this is broken with at * least the version 4 Tomatillos found in Fire V120 and Blade * 1500, which apparently actually count some different event at * ~0.5 and 3MHz respectively instead (also when running in full * power mode). Besides, one counter seems to be shared by a * "pair" of Tomatillos, too. */ if (sc->sc_half == 0) { SCHIZO_CTRL_SET(sc, STX_CTRL_PERF, (STX_CTRL_PERF_DIS << STX_CTRL_PERF_CNT1_SHIFT) | (STX_CTRL_PERF_BUSCYC << STX_CTRL_PERF_CNT0_SHIFT)); tc = malloc(sizeof(*tc), M_DEVBUF, M_NOWAIT | M_ZERO); if (tc == NULL) panic("%s: could not malloc timecounter", __func__); tc->tc_get_timecount = schizo_get_timecount; tc->tc_counter_mask = STX_CTRL_PERF_CNT_MASK; if (OF_getprop(OF_peer(0), "clock-frequency", &prop, sizeof(prop)) == -1) panic("%s: could not determine clock frequency", __func__); tc->tc_frequency = prop; tc->tc_name = strdup(device_get_nameunit(dev), M_DEVBUF); if (mode == SCHIZO_MODE_SCZ) tc->tc_quality = SCHIZO_PERF_CNT_QLTY; else tc->tc_quality = -SCHIZO_PERF_CNT_QLTY; tc->tc_priv = sc; tc_init(tc); } /* * Set up the IOMMU. Schizo, Tomatillo and XMITS all have * one per PBM. Schizo and XMITS additionally have a streaming * buffer, in Schizo version < 5 (i.e. revision < 2.3) it's * affected by several errata though. However, except for context * flushes, taking advantage of it should be okay even with those. */ memcpy(&sc->sc_dma_methods, &iommu_dma_methods, sizeof(sc->sc_dma_methods)); sc->sc_is.sis_sc = sc; sc->sc_is.sis_is.is_flags = IOMMU_PRESERVE_PROM; sc->sc_is.sis_is.is_pmaxaddr = IOMMU_MAXADDR(STX_IOMMU_BITS); sc->sc_is.sis_is.is_sb[0] = sc->sc_is.sis_is.is_sb[1] = 0; if (OF_getproplen(node, "no-streaming-cache") < 0) sc->sc_is.sis_is.is_sb[0] = STX_PCI_STRBUF; #define TSBCASE(x) \ case (IOTSB_BASESZ << (x)) << (IO_PAGE_SHIFT - IOTTE_SHIFT): \ tsbsize = (x); \ break; \ i = OF_getprop(node, "virtual-dma", (void *)prop_array, sizeof(prop_array)); if (i == -1 || i != sizeof(prop_array)) schizo_iommu_init(sc, 7, -1); else { switch (prop_array[1]) { TSBCASE(1); TSBCASE(2); TSBCASE(3); TSBCASE(4); TSBCASE(5); TSBCASE(6); TSBCASE(7); TSBCASE(8); default: panic("%s: unsupported DVMA size 0x%x", __func__, prop_array[1]); /* NOTREACHED */ } schizo_iommu_init(sc, tsbsize, prop_array[0]); } #undef TSBCASE /* Create our DMA tag. */ if (bus_dma_tag_create(bus_get_dma_tag(dev), 8, 0, sc->sc_is.sis_is.is_pmaxaddr, ~0, NULL, NULL, sc->sc_is.sis_is.is_pmaxaddr, 0xff, 0xffffffff, 0, NULL, NULL, &dmat) != 0) panic("%s: could not create PCI DMA tag", __func__); dmat->dt_cookie = &sc->sc_is; dmat->dt_mt = &sc->sc_dma_methods; if (ofw_pci_attach_common(dev, dmat, STX_IO_SIZE, STX_MEM_SIZE) != 0) panic("%s: ofw_pci_attach_common() failed", __func__); /* Clear any pending PCI error bits. */ PCIB_WRITE_CONFIG(dev, sc->sc_ops.sc_pci_secbus, STX_CS_DEVICE, STX_CS_FUNC, PCIR_STATUS, PCIB_READ_CONFIG(dev, sc->sc_ops.sc_pci_secbus, STX_CS_DEVICE, STX_CS_FUNC, PCIR_STATUS, 2), 2); SCHIZO_PCI_SET(sc, STX_PCI_CTRL, SCHIZO_PCI_READ_8(sc, STX_PCI_CTRL)); SCHIZO_PCI_SET(sc, STX_PCI_AFSR, SCHIZO_PCI_READ_8(sc, STX_PCI_AFSR)); /* * Establish handlers for interesting interrupts... * Someone at Sun clearly was smoking crack; with Schizos PCI * bus error interrupts for one PBM can be routed to the other * PBM though we obviously need to use the softc of the former * as the argument for the interrupt handler and the softc of * the latter as the argument for the interrupt controller. */ if (sc->sc_half == 0) { if ((ino_bitmap & (1ULL << STX_PCIERR_A_INO)) != 0 || (osc != NULL && ((struct schizo_icarg *)intr_vectors[ INTMAP_VEC(sc->sc_ign, STX_PCIERR_A_INO)].iv_icarg)-> sica_sc == osc)) /* * We are the driver for PBM A and either also * registered the interrupt controller for us or * the driver for PBM B has probed first and * registered it for us. */ schizo_set_intr(sc, 0, STX_PCIERR_A_INO, schizo_pci_bus); if ((ino_bitmap & (1ULL << STX_PCIERR_B_INO)) != 0 && osc != NULL) /* * We are the driver for PBM A but registered * the interrupt controller for PBM B, i.e. the * driver for PBM B attached first but couldn't * set up a handler for PBM B. */ schizo_set_intr(osc, 0, STX_PCIERR_B_INO, schizo_pci_bus); } else { if ((ino_bitmap & (1ULL << STX_PCIERR_B_INO)) != 0 || (osc != NULL && ((struct schizo_icarg *)intr_vectors[ INTMAP_VEC(sc->sc_ign, STX_PCIERR_B_INO)].iv_icarg)-> sica_sc == osc)) /* * We are the driver for PBM B and either also * registered the interrupt controller for us or * the driver for PBM A has probed first and * registered it for us. */ schizo_set_intr(sc, 0, STX_PCIERR_B_INO, schizo_pci_bus); if ((ino_bitmap & (1ULL << STX_PCIERR_A_INO)) != 0 && osc != NULL) /* * We are the driver for PBM B but registered * the interrupt controller for PBM A, i.e. the * driver for PBM A attached first but couldn't * set up a handler for PBM A. */ schizo_set_intr(osc, 0, STX_PCIERR_A_INO, schizo_pci_bus); } if ((ino_bitmap & (1ULL << STX_UE_INO)) != 0) schizo_set_intr(sc, 1, STX_UE_INO, schizo_ue); if ((ino_bitmap & (1ULL << STX_CE_INO)) != 0) schizo_set_intr(sc, 2, STX_CE_INO, schizo_ce); if ((ino_bitmap & (1ULL << STX_BUS_INO)) != 0) schizo_set_intr(sc, 3, STX_BUS_INO, schizo_host_bus); /* * According to the Schizo Errata I-13, consistent DMA flushing/ * syncing is FUBAR in version < 5 (i.e. revision < 2.3) bridges, * so we can't use it and need to live with the consequences. With * Schizo version >= 5, CDMA flushing/syncing is usable but requires * the workaround described in Schizo Errata I-23. With Tomatillo * and XMITS, CDMA flushing/syncing works as expected, Tomatillo * version <= 4 (i.e. revision <= 2.3) bridges additionally require * a block store after a write to TOMXMS_PCI_DMA_SYNC_PEND though. */ if ((sc->sc_mode == SCHIZO_MODE_SCZ && sc->sc_ver >= 5) || sc->sc_mode == SCHIZO_MODE_TOM || sc->sc_mode == SCHIZO_MODE_XMS) { if (sc->sc_mode == SCHIZO_MODE_SCZ) { sc->sc_dma_methods.dm_dmamap_sync = schizo_dmamap_sync; sc->sc_cdma_state = SCHIZO_CDMA_STATE_IDLE; /* * Some firmware versions include the CDMA interrupt * at RID 4 but most don't. With the latter we add * it ourselves at the spare RID 5. */ i = INTINO(bus_get_resource_start(dev, SYS_RES_IRQ, 4)); if (i == STX_CDMA_A_INO || i == STX_CDMA_B_INO) { sc->sc_cdma_vec = INTMAP_VEC(sc->sc_ign, i); (void)schizo_get_intrmap(sc, i, &sc->sc_cdma_map, &sc->sc_cdma_clr); schizo_set_intr(sc, 4, i, schizo_cdma); } else { i = STX_CDMA_A_INO + sc->sc_half; sc->sc_cdma_vec = INTMAP_VEC(sc->sc_ign, i); if (bus_set_resource(dev, SYS_RES_IRQ, 5, sc->sc_cdma_vec, 1) != 0) panic("%s: failed to add CDMA " "interrupt", __func__); j = schizo_intr_register(sc, i); if (j != 0) panic("%s: could not register " "interrupt controller for CDMA " "(%d)", __func__, j); (void)schizo_get_intrmap(sc, i, &sc->sc_cdma_map, &sc->sc_cdma_clr); schizo_set_intr(sc, 5, i, schizo_cdma); } } else { if (sc->sc_mode == SCHIZO_MODE_XMS) mtx_init(&sc->sc_sync_mtx, "pcib_sync_mtx", NULL, MTX_SPIN); sc->sc_sync_val = 1ULL << (STX_PCIERR_A_INO + sc->sc_half); sc->sc_dma_methods.dm_dmamap_sync = ichip_dmamap_sync; } if (sc->sc_mode == SCHIZO_MODE_TOM && sc->sc_ver <= 4) sc->sc_flags |= SCHIZO_FLAGS_BSWAR; } /* * Set the latency timer register as this isn't always done by the * firmware. */ PCIB_WRITE_CONFIG(dev, sc->sc_ops.sc_pci_secbus, STX_CS_DEVICE, STX_CS_FUNC, PCIR_LATTIMER, OFW_PCI_LATENCY, 1); #define SCHIZO_SYSCTL_ADD_UINT(name, arg, desc) \ SYSCTL_ADD_UINT(device_get_sysctl_ctx(dev), \ SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), OID_AUTO, \ (name), CTLFLAG_RD, (arg), 0, (desc)) SCHIZO_SYSCTL_ADD_UINT("dma_ce", &sc->sc_stats_dma_ce, "DMA correctable errors"); SCHIZO_SYSCTL_ADD_UINT("pci_non_fatal", &sc->sc_stats_pci_non_fatal, "PCI bus non-fatal errors"); #undef SCHIZO_SYSCTL_ADD_UINT device_add_child(dev, "pci", -1); return (bus_generic_attach(dev)); } static void schizo_set_intr(struct schizo_softc *sc, u_int index, u_int ino, driver_filter_t handler) { u_long vec; int rid; rid = index; sc->sc_irq_res[index] = bus_alloc_resource_any(sc->sc_dev, SYS_RES_IRQ, &rid, RF_ACTIVE); if (sc->sc_irq_res[index] == NULL || INTINO(vec = rman_get_start(sc->sc_irq_res[index])) != ino || INTIGN(vec) != sc->sc_ign || intr_vectors[vec].iv_ic != &schizo_ic || bus_setup_intr(sc->sc_dev, sc->sc_irq_res[index], INTR_TYPE_MISC | INTR_BRIDGE, handler, NULL, sc, &sc->sc_ihand[index]) != 0) panic("%s: failed to set up interrupt %d", __func__, index); } static int schizo_intr_register(struct schizo_softc *sc, u_int ino) { struct schizo_icarg *sica; bus_addr_t intrclr, intrmap; int error; if (schizo_get_intrmap(sc, ino, &intrmap, &intrclr) == 0) return (ENXIO); sica = malloc(sizeof(*sica), M_DEVBUF, M_NOWAIT); if (sica == NULL) return (ENOMEM); sica->sica_sc = sc; sica->sica_map = intrmap; sica->sica_clr = intrclr; #ifdef SCHIZO_DEBUG device_printf(sc->sc_dev, "intr map (INO %d) %#lx: %#lx, clr: %#lx\n", ino, (u_long)intrmap, (u_long)SCHIZO_PCI_READ_8(sc, intrmap), (u_long)intrclr); #endif error = (intr_controller_register(INTMAP_VEC(sc->sc_ign, ino), &schizo_ic, sica)); if (error != 0) free(sica, M_DEVBUF); return (error); } static int schizo_get_intrmap(struct schizo_softc *sc, u_int ino, bus_addr_t *intrmapptr, bus_addr_t *intrclrptr) { bus_addr_t intrclr, intrmap; uint64_t mr; /* * XXX we only look for INOs rather than INRs since the firmware * may not provide the IGN and the IGN is constant for all devices * on that PCI controller. */ if (ino > STX_MAX_INO) { device_printf(sc->sc_dev, "out of range INO %d requested\n", ino); return (0); } intrmap = STX_PCI_IMAP_BASE + (ino << 3); intrclr = STX_PCI_ICLR_BASE + (ino << 3); mr = SCHIZO_PCI_READ_8(sc, intrmap); if (INTINO(mr) != ino) { device_printf(sc->sc_dev, "interrupt map entry does not match INO (%d != %d)\n", (int)INTINO(mr), ino); return (0); } if (intrmapptr != NULL) *intrmapptr = intrmap; if (intrclrptr != NULL) *intrclrptr = intrclr; return (1); } /* * Interrupt handlers */ static int schizo_pci_bus(void *arg) { struct schizo_softc *sc = arg; uint64_t afar, afsr, csr, iommu, xstat; uint32_t status; u_int fatal; fatal = 0; mtx_lock_spin(sc->sc_mtx); afar = SCHIZO_PCI_READ_8(sc, STX_PCI_AFAR); afsr = SCHIZO_PCI_READ_8(sc, STX_PCI_AFSR); csr = SCHIZO_PCI_READ_8(sc, STX_PCI_CTRL); iommu = SCHIZO_PCI_READ_8(sc, STX_PCI_IOMMU); if ((sc->sc_flags & SCHIZO_FLAGS_XMODE) != 0) xstat = SCHIZO_PCI_READ_8(sc, XMS_PCI_X_ERR_STAT); else xstat = 0; status = PCIB_READ_CONFIG(sc->sc_dev, sc->sc_ops.sc_pci_secbus, STX_CS_DEVICE, STX_CS_FUNC, PCIR_STATUS, 2); /* * IOMMU errors are only fatal on Tomatillo and there also only if * target abort was not signaled. */ if ((csr & STX_PCI_CTRL_MMU_ERR) != 0 && (iommu & TOM_PCI_IOMMU_ERR) != 0 && ((status & PCIM_STATUS_STABORT) == 0 || ((iommu & TOM_PCI_IOMMU_ERRMASK) != TOM_PCI_IOMMU_INVALID_ERR && (iommu & TOM_PCI_IOMMU_ERR_ILLTSBTBW) == 0 && (iommu & TOM_PCI_IOMMU_ERR_BAD_VA) == 0))) fatal = 1; else if ((status & PCIM_STATUS_STABORT) != 0) fatal = 1; if ((status & (PCIM_STATUS_PERR | PCIM_STATUS_SERR | PCIM_STATUS_RMABORT | PCIM_STATUS_RTABORT | PCIM_STATUS_MDPERR)) != 0 || (csr & (SCZ_PCI_CTRL_BUS_UNUS | TOM_PCI_CTRL_DTO_ERR | STX_PCI_CTRL_TTO_ERR | STX_PCI_CTRL_RTRY_ERR | SCZ_PCI_CTRL_SBH_ERR | STX_PCI_CTRL_SERR)) != 0 || (afsr & (STX_PCI_AFSR_P_MA | STX_PCI_AFSR_P_TA | STX_PCI_AFSR_P_RTRY | STX_PCI_AFSR_P_PERR | STX_PCI_AFSR_P_TTO | STX_PCI_AFSR_P_UNUS)) != 0) fatal = 1; if (xstat & (XMS_PCI_X_ERR_STAT_P_SC_DSCRD | XMS_PCI_X_ERR_STAT_P_SC_TTO | XMS_PCI_X_ERR_STAT_P_SDSTAT | XMS_PCI_X_ERR_STAT_P_SMMU | XMS_PCI_X_ERR_STAT_P_CDSTAT | XMS_PCI_X_ERR_STAT_P_CMMU | XMS_PCI_X_ERR_STAT_PERR_RCV)) fatal = 1; if (fatal == 0) sc->sc_stats_pci_non_fatal++; device_printf(sc->sc_dev, "PCI bus %c error AFAR %#llx AFSR %#llx " "PCI CSR %#llx IOMMU %#llx PCI-X %#llx STATUS %#x\n", 'A' + sc->sc_half, (unsigned long long)afar, (unsigned long long)afsr, (unsigned long long)csr, (unsigned long long)iommu, (unsigned long long)xstat, status); /* Clear the error bits that we caught. */ PCIB_WRITE_CONFIG(sc->sc_dev, sc->sc_ops.sc_pci_secbus, STX_CS_DEVICE, STX_CS_FUNC, PCIR_STATUS, status, 2); SCHIZO_PCI_WRITE_8(sc, STX_PCI_CTRL, csr); SCHIZO_PCI_WRITE_8(sc, STX_PCI_AFSR, afsr); SCHIZO_PCI_WRITE_8(sc, STX_PCI_IOMMU, iommu); if ((sc->sc_flags & SCHIZO_FLAGS_XMODE) != 0) SCHIZO_PCI_WRITE_8(sc, XMS_PCI_X_ERR_STAT, xstat); mtx_unlock_spin(sc->sc_mtx); if (fatal != 0) panic("%s: fatal PCI bus error", device_get_nameunit(sc->sc_dev)); return (FILTER_HANDLED); } static int schizo_ue(void *arg) { struct schizo_softc *sc = arg; uint64_t afar, afsr; int i; afar = SCHIZO_CTRL_READ_8(sc, STX_CTRL_UE_AFAR); for (i = 0; i < 1000; i++) if (((afsr = SCHIZO_CTRL_READ_8(sc, STX_CTRL_UE_AFSR)) & STX_CTRL_CE_AFSR_ERRPNDG) == 0) break; panic("%s: uncorrectable DMA error AFAR %#llx AFSR %#llx", device_get_nameunit(sc->sc_dev), (unsigned long long)afar, (unsigned long long)afsr); return (FILTER_HANDLED); } static int schizo_ce(void *arg) { struct schizo_softc *sc = arg; uint64_t afar, afsr; int i; mtx_lock_spin(sc->sc_mtx); afar = SCHIZO_CTRL_READ_8(sc, STX_CTRL_CE_AFAR); for (i = 0; i < 1000; i++) if (((afsr = SCHIZO_CTRL_READ_8(sc, STX_CTRL_UE_AFSR)) & STX_CTRL_CE_AFSR_ERRPNDG) == 0) break; sc->sc_stats_dma_ce++; device_printf(sc->sc_dev, "correctable DMA error AFAR %#llx AFSR %#llx\n", (unsigned long long)afar, (unsigned long long)afsr); /* Clear the error bits that we caught. */ SCHIZO_CTRL_WRITE_8(sc, STX_CTRL_UE_AFSR, afsr); mtx_unlock_spin(sc->sc_mtx); return (FILTER_HANDLED); } static int schizo_host_bus(void *arg) { struct schizo_softc *sc = arg; uint64_t errlog; errlog = SCHIZO_CTRL_READ_8(sc, STX_CTRL_BUS_ERRLOG); panic("%s: %s error %#llx", device_get_nameunit(sc->sc_dev), sc->sc_mode == SCHIZO_MODE_TOM ? "JBus" : "Safari", (unsigned long long)errlog); return (FILTER_HANDLED); } static int schizo_cdma(void *arg) { struct schizo_softc *sc = arg; atomic_cmpset_32(&sc->sc_cdma_state, SCHIZO_CDMA_STATE_PENDING, SCHIZO_CDMA_STATE_RECEIVED); return (FILTER_HANDLED); } static void schizo_iommu_init(struct schizo_softc *sc, int tsbsize, uint32_t dvmabase) { /* Punch in our copies. */ sc->sc_is.sis_is.is_bustag = rman_get_bustag(sc->sc_mem_res[STX_PCI]); sc->sc_is.sis_is.is_bushandle = rman_get_bushandle(sc->sc_mem_res[STX_PCI]); sc->sc_is.sis_is.is_iommu = STX_PCI_IOMMU; sc->sc_is.sis_is.is_dtag = STX_PCI_IOMMU_TLB_TAG_DIAG; sc->sc_is.sis_is.is_ddram = STX_PCI_IOMMU_TLB_DATA_DIAG; sc->sc_is.sis_is.is_dqueue = STX_PCI_IOMMU_QUEUE_DIAG; sc->sc_is.sis_is.is_dva = STX_PCI_IOMMU_SVADIAG; sc->sc_is.sis_is.is_dtcmp = STX_PCI_IOMMU_TLB_CMP_DIAG; iommu_init(device_get_nameunit(sc->sc_dev), (struct iommu_state *)&sc->sc_is, tsbsize, dvmabase, 0); } static int schizo_maxslots(device_t dev) { struct schizo_softc *sc; sc = device_get_softc(dev); if (sc->sc_mode == SCHIZO_MODE_SCZ) return (sc->sc_half == 0 ? 4 : 6); /* XXX: is this correct? */ return (PCI_SLOTMAX); } static uint32_t schizo_read_config(device_t dev, u_int bus, u_int slot, u_int func, u_int reg, int width) { struct schizo_softc *sc; sc = device_get_softc(dev); /* * The Schizo bridges contain a dupe of their header at 0x80. */ if (sc->sc_mode == SCHIZO_MODE_SCZ && bus == sc->sc_ops.sc_pci_secbus && slot == STX_CS_DEVICE && func == STX_CS_FUNC && reg + width > 0x80) return (0); return (ofw_pci_read_config_common(dev, PCI_REGMAX, STX_CONF_OFF(bus, slot, func, reg), bus, slot, func, reg, width)); } static void schizo_write_config(device_t dev, u_int bus, u_int slot, u_int func, u_int reg, uint32_t val, int width) { ofw_pci_write_config_common(dev, PCI_REGMAX, STX_CONF_OFF(bus, slot, func, reg), bus, slot, func, reg, val, width); } static int schizo_route_interrupt(device_t bridge, device_t dev, int pin) { ofw_pci_intr_t mintr; mintr = ofw_pci_route_interrupt_common(bridge, dev, pin); if (!PCI_INTERRUPT_VALID(mintr)) device_printf(bridge, "could not route pin %d for device %d.%d\n", pin, pci_get_slot(dev), pci_get_function(dev)); return (mintr); } static void schizo_dmamap_sync(bus_dma_tag_t dt, bus_dmamap_t map, bus_dmasync_op_t op) { struct timeval cur, end; struct schizo_iommu_state *sis = dt->dt_cookie; struct schizo_softc *sc = sis->sis_sc; int i, res; #ifdef INVARIANTS register_t pil; #endif if ((map->dm_flags & DMF_STREAMED) != 0) { iommu_dma_methods.dm_dmamap_sync(dt, map, op); return; } if ((map->dm_flags & DMF_LOADED) == 0) return; if ((op & BUS_DMASYNC_POSTREAD) != 0) { /* * Note that in order to allow this function to be called from * filters we would need to use a spin mutex for serialization * but given that these disable interrupts we have to emulate * one. */ critical_enter(); KASSERT((rdpr(pstate) & PSTATE_IE) != 0, ("%s: interrupts disabled", __func__)); KASSERT((pil = rdpr(pil)) <= PIL_BRIDGE, ("%s: PIL too low (%ld)", __func__, pil)); for (; atomic_cmpset_acq_32(&sc->sc_cdma_state, SCHIZO_CDMA_STATE_IDLE, SCHIZO_CDMA_STATE_PENDING) == 0;) ; SCHIZO_PCI_WRITE_8(sc, sc->sc_cdma_map, INTMAP_ENABLE(sc->sc_cdma_vec, PCPU_GET(mid))); for (i = 0; i < SCHIZO_CDMA_TRIES; i++) { if (i > 0) printf("%s: try %d\n", __func__, i); SCHIZO_PCI_WRITE_8(sc, sc->sc_cdma_clr, INTCLR_RECEIVED); microuptime(&cur); end.tv_sec = SCHIZO_CDMA_TIMEOUT; end.tv_usec = 0; timevaladd(&end, &cur); for (; (res = atomic_cmpset_rel_32(&sc->sc_cdma_state, SCHIZO_CDMA_STATE_RECEIVED, SCHIZO_CDMA_STATE_IDLE)) == 0 && timevalcmp(&cur, &end, <=);) microuptime(&cur); if (res != 0) break; } if (res == 0) panic("%s: DMA does not sync", __func__); critical_exit(); } if ((op & BUS_DMASYNC_PREWRITE) != 0) membar(Sync); } static void ichip_dmamap_sync(bus_dma_tag_t dt, bus_dmamap_t map, bus_dmasync_op_t op) { struct timeval cur, end; struct schizo_iommu_state *sis = dt->dt_cookie; struct schizo_softc *sc = sis->sis_sc; uint64_t reg; if ((map->dm_flags & DMF_STREAMED) != 0) { iommu_dma_methods.dm_dmamap_sync(dt, map, op); return; } if ((map->dm_flags & DMF_LOADED) == 0) return; if ((op & BUS_DMASYNC_POSTREAD) != 0) { if (sc->sc_mode == SCHIZO_MODE_XMS) mtx_lock_spin(&sc->sc_sync_mtx); SCHIZO_PCI_WRITE_8(sc, TOMXMS_PCI_DMA_SYNC_PEND, sc->sc_sync_val); microuptime(&cur); end.tv_sec = 1; end.tv_usec = 0; timevaladd(&end, &cur); for (; ((reg = SCHIZO_PCI_READ_8(sc, TOMXMS_PCI_DMA_SYNC_PEND)) & sc->sc_sync_val) != 0 && timevalcmp(&cur, &end, <=);) microuptime(&cur); if ((reg & sc->sc_sync_val) != 0) panic("%s: DMA does not sync", __func__); if (sc->sc_mode == SCHIZO_MODE_XMS) mtx_unlock_spin(&sc->sc_sync_mtx); else if ((sc->sc_flags & SCHIZO_FLAGS_BSWAR) != 0) { ofw_pci_dmamap_sync_stst_order_common(); return; } } if ((op & BUS_DMASYNC_PREWRITE) != 0) membar(Sync); } static void schizo_intr_enable(void *arg) { struct intr_vector *iv = arg; struct schizo_icarg *sica = iv->iv_icarg; SCHIZO_PCI_WRITE_8(sica->sica_sc, sica->sica_map, INTMAP_ENABLE(iv->iv_vec, iv->iv_mid)); } static void schizo_intr_disable(void *arg) { struct intr_vector *iv = arg; struct schizo_icarg *sica = iv->iv_icarg; SCHIZO_PCI_WRITE_8(sica->sica_sc, sica->sica_map, iv->iv_vec); } static void schizo_intr_assign(void *arg) { struct intr_vector *iv = arg; struct schizo_icarg *sica = iv->iv_icarg; SCHIZO_PCI_WRITE_8(sica->sica_sc, sica->sica_map, INTMAP_TID( SCHIZO_PCI_READ_8(sica->sica_sc, sica->sica_map), iv->iv_mid)); } static void schizo_intr_clear(void *arg) { struct intr_vector *iv = arg; struct schizo_icarg *sica = iv->iv_icarg; SCHIZO_PCI_WRITE_8(sica->sica_sc, sica->sica_clr, INTCLR_IDLE); } static int schizo_setup_intr(device_t dev, device_t child, struct resource *ires, int flags, driver_filter_t *filt, driver_intr_t *intr, void *arg, void **cookiep) { struct schizo_softc *sc; u_long vec; int error; sc = device_get_softc(dev); /* * Make sure the vector is fully specified. */ vec = rman_get_start(ires); if (INTIGN(vec) != sc->sc_ign) { device_printf(dev, "invalid interrupt vector 0x%lx\n", vec); return (EINVAL); } if (intr_vectors[vec].iv_ic == &schizo_ic) { /* * Ensure we use the right softc in case the interrupt * is routed to our companion PBM for some odd reason. */ sc = ((struct schizo_icarg *)intr_vectors[vec].iv_icarg)-> sica_sc; } else if (intr_vectors[vec].iv_ic == NULL) { /* * Work around broken firmware which misses entries in * the ino-bitmap. */ error = schizo_intr_register(sc, INTINO(vec)); if (error != 0) { device_printf(dev, "could not register interrupt " "controller for vector 0x%lx (%d)\n", vec, error); return (error); } if (bootverbose) device_printf(dev, "belatedly registered as " "interrupt controller for vector 0x%lx\n", vec); } else { device_printf(dev, "invalid interrupt controller for vector 0x%lx\n", vec); return (EINVAL); } return (bus_generic_setup_intr(dev, child, ires, flags, filt, intr, arg, cookiep)); } static struct resource * schizo_alloc_resource(device_t bus, device_t child, int type, int *rid, rman_res_t start, rman_res_t end, rman_res_t count, u_int flags) { struct schizo_softc *sc; if (type == SYS_RES_IRQ) { sc = device_get_softc(bus); start = end = INTMAP_VEC(sc->sc_ign, end); } return (ofw_pci_alloc_resource(bus, child, type, rid, start, end, count, flags)); } static void schizo_setup_device(device_t bus, device_t child) { struct schizo_softc *sc; uint64_t reg; int capreg; sc = device_get_softc(bus); /* * Disable bus parking in order to work around a bus hang caused by * Casinni/Skyhawk combinations. */ if (OF_getproplen(ofw_bus_get_node(child), "pci-req-removal") >= 0) SCHIZO_PCI_SET(sc, STX_PCI_CTRL, SCHIZO_PCI_READ_8(sc, STX_PCI_CTRL) & ~STX_PCI_CTRL_ARB_PARK); if (sc->sc_mode == SCHIZO_MODE_XMS) { /* XMITS NCPQ WAR: set outstanding split transactions to 1. */ if ((sc->sc_flags & SCHIZO_FLAGS_XMODE) != 0 && (pci_read_config(child, PCIR_HDRTYPE, 1) & PCIM_HDRTYPE) != PCIM_HDRTYPE_BRIDGE && pci_find_cap(child, PCIY_PCIX, &capreg) == 0) pci_write_config(child, capreg + PCIXR_COMMAND, pci_read_config(child, capreg + PCIXR_COMMAND, 2) & 0x7c, 2); /* XMITS 3.x WAR: set BUGCNTL iff value is unexpected. */ if (sc->sc_mrev >= 4) { reg = ((sc->sc_flags & SCHIZO_FLAGS_XMODE) != 0 ? 0xa0UL : 0xffUL) << XMS_PCI_X_DIAG_BUGCNTL_SHIFT; if ((SCHIZO_PCI_READ_8(sc, XMS_PCI_X_DIAG) & XMS_PCI_X_DIAG_BUGCNTL_MASK) != reg) SCHIZO_PCI_SET(sc, XMS_PCI_X_DIAG, reg); } } } static u_int schizo_get_timecount(struct timecounter *tc) { struct schizo_softc *sc; sc = tc->tc_priv; return ((SCHIZO_CTRL_READ_8(sc, STX_CTRL_PERF_CNT) & (STX_CTRL_PERF_CNT_MASK << STX_CTRL_PERF_CNT_CNT0_SHIFT)) >> STX_CTRL_PERF_CNT_CNT0_SHIFT); }