Index: head/sys/arm/arm/gic.c =================================================================== --- head/sys/arm/arm/gic.c (revision 299927) +++ head/sys/arm/arm/gic.c (revision 299928) @@ -1,1574 +1,1800 @@ /*- * Copyright (c) 2011 The FreeBSD Foundation * All rights reserved. * * Developed by Damjan Marion * * Based on OMAP4 GIC code by Ben Gray * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. The name of the company nor the name of the author may be used to * endorse or promote products derived from this software without specific * prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include "opt_platform.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #ifdef INTRNG #include #endif + +#include +#include + #include #include #include #include #include #include #include #ifdef INTRNG #include "pic_if.h" +#include "msi_if.h" #endif #define GIC_DEBUG_SPURIOUS /* We are using GICv2 register naming */ /* Distributor Registers */ #define GICD_CTLR 0x000 /* v1 ICDDCR */ #define GICD_TYPER 0x004 /* v1 ICDICTR */ #define GICD_IIDR 0x008 /* v1 ICDIIDR */ #define GICD_IGROUPR(n) (0x0080 + ((n) * 4)) /* v1 ICDISER */ #define GICD_ISENABLER(n) (0x0100 + ((n) * 4)) /* v1 ICDISER */ #define GICD_ICENABLER(n) (0x0180 + ((n) * 4)) /* v1 ICDICER */ #define GICD_ISPENDR(n) (0x0200 + ((n) * 4)) /* v1 ICDISPR */ #define GICD_ICPENDR(n) (0x0280 + ((n) * 4)) /* v1 ICDICPR */ #define GICD_ICACTIVER(n) (0x0380 + ((n) * 4)) /* v1 ICDABR */ #define GICD_IPRIORITYR(n) (0x0400 + ((n) * 4)) /* v1 ICDIPR */ #define GICD_ITARGETSR(n) (0x0800 + ((n) * 4)) /* v1 ICDIPTR */ #define GICD_ICFGR(n) (0x0C00 + ((n) * 4)) /* v1 ICDICFR */ #define GICD_SGIR(n) (0x0F00 + ((n) * 4)) /* v1 ICDSGIR */ #define GICD_SGI_TARGET_SHIFT 16 /* CPU Registers */ #define GICC_CTLR 0x0000 /* v1 ICCICR */ #define GICC_PMR 0x0004 /* v1 ICCPMR */ #define GICC_BPR 0x0008 /* v1 ICCBPR */ #define GICC_IAR 0x000C /* v1 ICCIAR */ #define GICC_EOIR 0x0010 /* v1 ICCEOIR */ #define GICC_RPR 0x0014 /* v1 ICCRPR */ #define GICC_HPPIR 0x0018 /* v1 ICCHPIR */ #define GICC_ABPR 0x001C /* v1 ICCABPR */ #define GICC_IIDR 0x00FC /* v1 ICCIIDR*/ #define GIC_FIRST_SGI 0 /* Irqs 0-15 are SGIs/IPIs. */ #define GIC_LAST_SGI 15 #define GIC_FIRST_PPI 16 /* Irqs 16-31 are private (per */ #define GIC_LAST_PPI 31 /* core) peripheral interrupts. */ #define GIC_FIRST_SPI 32 /* Irqs 32+ are shared peripherals. */ /* First bit is a polarity bit (0 - low, 1 - high) */ #define GICD_ICFGR_POL_LOW (0 << 0) #define GICD_ICFGR_POL_HIGH (1 << 0) #define GICD_ICFGR_POL_MASK 0x1 /* Second bit is a trigger bit (0 - level, 1 - edge) */ #define GICD_ICFGR_TRIG_LVL (0 << 1) #define GICD_ICFGR_TRIG_EDGE (1 << 1) #define GICD_ICFGR_TRIG_MASK 0x2 #ifndef GIC_DEFAULT_ICFGR_INIT #define GIC_DEFAULT_ICFGR_INIT 0x00000000 #endif #ifdef INTRNG struct gic_irqsrc { struct intr_irqsrc gi_isrc; uint32_t gi_irq; enum intr_polarity gi_pol; enum intr_trigger gi_trig; #define GI_FLAG_EARLY_EOI (1 << 0) +#define GI_FLAG_MSI (1 << 1) /* This interrupt source should only */ + /* be used for MSI/MSI-X interrupts */ +#define GI_FLAG_MSI_USED (1 << 2) /* This irq is already allocated */ + /* for a MSI/MSI-X interrupt */ u_int gi_flags; }; static u_int gic_irq_cpu; static int arm_gic_intr(void *); static int arm_gic_bind_intr(device_t dev, struct intr_irqsrc *isrc); #ifdef SMP static u_int sgi_to_ipi[GIC_LAST_SGI - GIC_FIRST_SGI + 1]; static u_int sgi_first_unused = GIC_FIRST_SGI; #endif #endif #ifdef INTRNG struct arm_gic_range { uint64_t bus; uint64_t host; uint64_t size; }; struct arm_gic_devinfo { struct ofw_bus_devinfo obdinfo; struct resource_list rl; }; #endif struct arm_gic_softc { device_t gic_dev; #ifdef INTRNG void * gic_intrhand; struct gic_irqsrc * gic_irqs; #endif struct resource * gic_res[3]; bus_space_tag_t gic_c_bst; bus_space_tag_t gic_d_bst; bus_space_handle_t gic_c_bsh; bus_space_handle_t gic_d_bsh; uint8_t ver; struct mtx mutex; uint32_t nirqs; #ifdef GIC_DEBUG_SPURIOUS uint32_t last_irq[MAXCPU]; #endif #ifdef INTRNG /* FDT child data */ pcell_t addr_cells; pcell_t size_cells; int nranges; struct arm_gic_range * ranges; #endif }; #ifdef INTRNG #define GIC_INTR_ISRC(sc, irq) (&sc->gic_irqs[irq].gi_isrc) #endif static struct resource_spec arm_gic_spec[] = { { SYS_RES_MEMORY, 0, RF_ACTIVE }, /* Distributor registers */ { SYS_RES_MEMORY, 1, RF_ACTIVE }, /* CPU Interrupt Intf. registers */ #ifdef INTRNG { SYS_RES_IRQ, 0, RF_ACTIVE | RF_OPTIONAL }, /* Parent interrupt */ #endif { -1, 0 } }; static u_int arm_gic_map[MAXCPU]; static struct arm_gic_softc *gic_sc = NULL; #define gic_c_read_4(_sc, _reg) \ bus_space_read_4((_sc)->gic_c_bst, (_sc)->gic_c_bsh, (_reg)) #define gic_c_write_4(_sc, _reg, _val) \ bus_space_write_4((_sc)->gic_c_bst, (_sc)->gic_c_bsh, (_reg), (_val)) #define gic_d_read_4(_sc, _reg) \ bus_space_read_4((_sc)->gic_d_bst, (_sc)->gic_d_bsh, (_reg)) #define gic_d_write_1(_sc, _reg, _val) \ bus_space_write_1((_sc)->gic_d_bst, (_sc)->gic_d_bsh, (_reg), (_val)) #define gic_d_write_4(_sc, _reg, _val) \ bus_space_write_4((_sc)->gic_d_bst, (_sc)->gic_d_bsh, (_reg), (_val)) #ifndef INTRNG static int gic_config_irq(int irq, enum intr_trigger trig, enum intr_polarity pol); static void gic_post_filter(void *); #endif static struct ofw_compat_data compat_data[] = { {"arm,gic", true}, /* Non-standard, used in FreeBSD dts. */ {"arm,gic-400", true}, {"arm,cortex-a15-gic", true}, {"arm,cortex-a9-gic", true}, {"arm,cortex-a7-gic", true}, {"arm,arm11mp-gic", true}, {"brcm,brahma-b15-gic", true}, {"qcom,msm-qgic2", true}, {NULL, false} }; static int arm_gic_probe(device_t dev) { if (!ofw_bus_status_okay(dev)) return (ENXIO); if (!ofw_bus_search_compatible(dev, compat_data)->ocd_data) return (ENXIO); device_set_desc(dev, "ARM Generic Interrupt Controller"); return (BUS_PROBE_DEFAULT); } #ifdef INTRNG static inline void gic_irq_unmask(struct arm_gic_softc *sc, u_int irq) { gic_d_write_4(sc, GICD_ISENABLER(irq >> 5), (1UL << (irq & 0x1F))); } static inline void gic_irq_mask(struct arm_gic_softc *sc, u_int irq) { gic_d_write_4(sc, GICD_ICENABLER(irq >> 5), (1UL << (irq & 0x1F))); } #endif static uint8_t gic_cpu_mask(struct arm_gic_softc *sc) { uint32_t mask; int i; /* Read the current cpuid mask by reading ITARGETSR{0..7} */ for (i = 0; i < 8; i++) { mask = gic_d_read_4(sc, GICD_ITARGETSR(i)); if (mask != 0) break; } /* No mask found, assume we are on CPU interface 0 */ if (mask == 0) return (1); /* Collect the mask in the lower byte */ mask |= mask >> 16; mask |= mask >> 8; return (mask); } #ifdef SMP #ifdef INTRNG static void arm_gic_init_secondary(device_t dev) { struct arm_gic_softc *sc = device_get_softc(dev); u_int irq, cpu; /* Set the mask so we can find this CPU to send it IPIs */ cpu = PCPU_GET(cpuid); arm_gic_map[cpu] = gic_cpu_mask(sc); for (irq = 0; irq < sc->nirqs; irq += 4) gic_d_write_4(sc, GICD_IPRIORITYR(irq >> 2), 0); /* Set all the interrupts to be in Group 0 (secure) */ for (irq = 0; irq < sc->nirqs; irq += 32) { gic_d_write_4(sc, GICD_IGROUPR(irq >> 5), 0); } /* Enable CPU interface */ gic_c_write_4(sc, GICC_CTLR, 1); /* Set priority mask register. */ gic_c_write_4(sc, GICC_PMR, 0xff); /* Enable interrupt distribution */ gic_d_write_4(sc, GICD_CTLR, 0x01); /* Unmask attached SGI interrupts. */ for (irq = GIC_FIRST_SGI; irq <= GIC_LAST_SGI; irq++) if (intr_isrc_init_on_cpu(GIC_INTR_ISRC(sc, irq), cpu)) gic_irq_unmask(sc, irq); /* Unmask attached PPI interrupts. */ for (irq = GIC_FIRST_PPI; irq <= GIC_LAST_PPI; irq++) if (intr_isrc_init_on_cpu(GIC_INTR_ISRC(sc, irq), cpu)) gic_irq_unmask(sc, irq); } #else static void arm_gic_init_secondary(device_t dev) { struct arm_gic_softc *sc = device_get_softc(dev); int i; /* Set the mask so we can find this CPU to send it IPIs */ arm_gic_map[PCPU_GET(cpuid)] = gic_cpu_mask(sc); for (i = 0; i < sc->nirqs; i += 4) gic_d_write_4(sc, GICD_IPRIORITYR(i >> 2), 0); /* Set all the interrupts to be in Group 0 (secure) */ for (i = 0; i < sc->nirqs; i += 32) { gic_d_write_4(sc, GICD_IGROUPR(i >> 5), 0); } /* Enable CPU interface */ gic_c_write_4(sc, GICC_CTLR, 1); /* Set priority mask register. */ gic_c_write_4(sc, GICC_PMR, 0xff); /* Enable interrupt distribution */ gic_d_write_4(sc, GICD_CTLR, 0x01); /* * Activate the timer interrupts: virtual, secure, and non-secure. */ gic_d_write_4(sc, GICD_ISENABLER(27 >> 5), (1UL << (27 & 0x1F))); gic_d_write_4(sc, GICD_ISENABLER(29 >> 5), (1UL << (29 & 0x1F))); gic_d_write_4(sc, GICD_ISENABLER(30 >> 5), (1UL << (30 & 0x1F))); } #endif /* INTRNG */ #endif /* SMP */ #ifndef INTRNG int gic_decode_fdt(phandle_t iparent, pcell_t *intr, int *interrupt, int *trig, int *pol) { static u_int num_intr_cells; static phandle_t self; struct ofw_compat_data *ocd; if (self == 0) { for (ocd = compat_data; ocd->ocd_str != NULL; ocd++) { if (fdt_is_compatible(iparent, ocd->ocd_str)) { self = iparent; break; } } } if (self != iparent) return (ENXIO); if (num_intr_cells == 0) { if (OF_searchencprop(OF_node_from_xref(iparent), "#interrupt-cells", &num_intr_cells, sizeof(num_intr_cells)) == -1) { num_intr_cells = 1; } } if (num_intr_cells == 1) { *interrupt = fdt32_to_cpu(intr[0]); *trig = INTR_TRIGGER_CONFORM; *pol = INTR_POLARITY_CONFORM; } else { if (fdt32_to_cpu(intr[0]) == 0) *interrupt = fdt32_to_cpu(intr[1]) + GIC_FIRST_SPI; else *interrupt = fdt32_to_cpu(intr[1]) + GIC_FIRST_PPI; /* * In intr[2], bits[3:0] are trigger type and level flags. * 1 = low-to-high edge triggered * 2 = high-to-low edge triggered * 4 = active high level-sensitive * 8 = active low level-sensitive * The hardware only supports active-high-level or rising-edge * for SPIs */ if (*interrupt >= GIC_FIRST_SPI && fdt32_to_cpu(intr[2]) & 0x0a) { printf("unsupported trigger/polarity configuration " "0x%02x\n", fdt32_to_cpu(intr[2]) & 0x0f); } *pol = INTR_POLARITY_CONFORM; if (fdt32_to_cpu(intr[2]) & 0x03) *trig = INTR_TRIGGER_EDGE; else *trig = INTR_TRIGGER_LEVEL; } return (0); } #endif #ifdef INTRNG static inline intptr_t gic_xref(device_t dev) { #ifdef FDT return (OF_xref_from_node(ofw_bus_get_node(dev))); #else return (0); #endif } static int arm_gic_register_isrcs(struct arm_gic_softc *sc, uint32_t num) { int error; uint32_t irq; struct gic_irqsrc *irqs; struct intr_irqsrc *isrc; const char *name; irqs = malloc(num * sizeof(struct gic_irqsrc), M_DEVBUF, M_WAITOK | M_ZERO); name = device_get_nameunit(sc->gic_dev); for (irq = 0; irq < num; irq++) { irqs[irq].gi_irq = irq; irqs[irq].gi_pol = INTR_POLARITY_CONFORM; irqs[irq].gi_trig = INTR_TRIGGER_CONFORM; isrc = &irqs[irq].gi_isrc; if (irq <= GIC_LAST_SGI) { error = intr_isrc_register(isrc, sc->gic_dev, INTR_ISRCF_IPI, "%s,i%u", name, irq - GIC_FIRST_SGI); } else if (irq <= GIC_LAST_PPI) { error = intr_isrc_register(isrc, sc->gic_dev, INTR_ISRCF_PPI, "%s,p%u", name, irq - GIC_FIRST_PPI); } else { error = intr_isrc_register(isrc, sc->gic_dev, 0, "%s,s%u", name, irq - GIC_FIRST_SPI); } if (error != 0) { /* XXX call intr_isrc_deregister() */ free(irqs, M_DEVBUF); return (error); } } sc->gic_irqs = irqs; sc->nirqs = num; return (0); } static int arm_gic_fill_ranges(phandle_t node, struct arm_gic_softc *sc) { pcell_t host_cells; cell_t *base_ranges; ssize_t nbase_ranges; int i, j, k; host_cells = 1; OF_getencprop(OF_parent(node), "#address-cells", &host_cells, sizeof(host_cells)); sc->addr_cells = 2; OF_getencprop(node, "#address-cells", &sc->addr_cells, sizeof(sc->addr_cells)); sc->size_cells = 2; OF_getencprop(node, "#size-cells", &sc->size_cells, sizeof(sc->size_cells)); nbase_ranges = OF_getproplen(node, "ranges"); if (nbase_ranges < 0) return (-1); sc->nranges = nbase_ranges / sizeof(cell_t) / (sc->addr_cells + host_cells + sc->size_cells); if (sc->nranges == 0) return (0); sc->ranges = malloc(sc->nranges * sizeof(sc->ranges[0]), M_DEVBUF, M_WAITOK); base_ranges = malloc(nbase_ranges, M_DEVBUF, M_WAITOK); OF_getencprop(node, "ranges", base_ranges, nbase_ranges); for (i = 0, j = 0; i < sc->nranges; i++) { sc->ranges[i].bus = 0; for (k = 0; k < sc->addr_cells; k++) { sc->ranges[i].bus <<= 32; sc->ranges[i].bus |= base_ranges[j++]; } sc->ranges[i].host = 0; for (k = 0; k < host_cells; k++) { sc->ranges[i].host <<= 32; sc->ranges[i].host |= base_ranges[j++]; } sc->ranges[i].size = 0; for (k = 0; k < sc->size_cells; k++) { sc->ranges[i].size <<= 32; sc->ranges[i].size |= base_ranges[j++]; } } free(base_ranges, M_DEVBUF); return (sc->nranges); } static bool arm_gic_add_children(device_t dev) { struct arm_gic_softc *sc; struct arm_gic_devinfo *dinfo; phandle_t child, node; device_t cdev; sc = device_get_softc(dev); node = ofw_bus_get_node(dev); /* If we have no children don't probe for them */ child = OF_child(node); if (child == 0) return (false); if (arm_gic_fill_ranges(node, sc) < 0) { device_printf(dev, "Have a child, but no ranges\n"); return (false); } for (; child != 0; child = OF_peer(child)) { dinfo = malloc(sizeof(*dinfo), M_DEVBUF, M_WAITOK | M_ZERO); if (ofw_bus_gen_setup_devinfo(&dinfo->obdinfo, child) != 0) { free(dinfo, M_DEVBUF); continue; } resource_list_init(&dinfo->rl); ofw_bus_reg_to_rl(dev, child, sc->addr_cells, sc->size_cells, &dinfo->rl); cdev = device_add_child(dev, NULL, -1); if (cdev == NULL) { device_printf(dev, "<%s>: device_add_child failed\n", dinfo->obdinfo.obd_name); resource_list_free(&dinfo->rl); ofw_bus_gen_destroy_devinfo(&dinfo->obdinfo); free(dinfo, M_DEVBUF); continue; } device_set_ivars(cdev, dinfo); } return (true); } + +static void +arm_gic_reserve_msi_range(device_t dev, u_int start, u_int count) +{ + struct arm_gic_softc *sc; + int i; + + sc = device_get_softc(dev); + + KASSERT((start + count) < sc->nirqs, + ("%s: Trying to allocate too many MSI IRQs: %d + %d > %d", __func__, + start, count, sc->nirqs)); + for (i = 0; i < count; i++) { + KASSERT(sc->gic_irqs[start + i].gi_isrc.isrc_handlers == 0, + ("%s: MSI interrupt %d already has a handler", __func__, + count + i)); + KASSERT(sc->gic_irqs[start + i].gi_pol == INTR_POLARITY_CONFORM, + ("%s: MSI interrupt %d already has a polarity", __func__, + count + i)); + KASSERT(sc->gic_irqs[start + i].gi_trig == INTR_TRIGGER_CONFORM, + ("%s: MSI interrupt %d already has a trigger", __func__, + count + i)); + sc->gic_irqs[start + i].gi_pol = INTR_POLARITY_HIGH; + sc->gic_irqs[start + i].gi_trig = INTR_TRIGGER_EDGE; + sc->gic_irqs[start + i].gi_flags |= GI_FLAG_MSI; + } +} #endif static int arm_gic_attach(device_t dev) { struct arm_gic_softc *sc; int i; uint32_t icciidr, mask, nirqs; #ifdef INTRNG phandle_t pxref; intptr_t xref = gic_xref(dev); #endif if (gic_sc) return (ENXIO); sc = device_get_softc(dev); if (bus_alloc_resources(dev, arm_gic_spec, sc->gic_res)) { device_printf(dev, "could not allocate resources\n"); return (ENXIO); } sc->gic_dev = dev; gic_sc = sc; /* Initialize mutex */ mtx_init(&sc->mutex, "GIC lock", "", MTX_SPIN); /* Distributor Interface */ sc->gic_d_bst = rman_get_bustag(sc->gic_res[0]); sc->gic_d_bsh = rman_get_bushandle(sc->gic_res[0]); /* CPU Interface */ sc->gic_c_bst = rman_get_bustag(sc->gic_res[1]); sc->gic_c_bsh = rman_get_bushandle(sc->gic_res[1]); /* Disable interrupt forwarding to the CPU interface */ gic_d_write_4(sc, GICD_CTLR, 0x00); /* Get the number of interrupts */ nirqs = gic_d_read_4(sc, GICD_TYPER); nirqs = 32 * ((nirqs & 0x1f) + 1); #ifdef INTRNG if (arm_gic_register_isrcs(sc, nirqs)) { device_printf(dev, "could not register irqs\n"); goto cleanup; } #else sc->nirqs = nirqs; /* Set up function pointers */ arm_post_filter = gic_post_filter; arm_config_irq = gic_config_irq; #endif icciidr = gic_c_read_4(sc, GICC_IIDR); device_printf(dev,"pn 0x%x, arch 0x%x, rev 0x%x, implementer 0x%x irqs %u\n", icciidr>>20, (icciidr>>16) & 0xF, (icciidr>>12) & 0xf, (icciidr & 0xfff), sc->nirqs); /* Set all global interrupts to be level triggered, active low. */ for (i = 32; i < sc->nirqs; i += 16) { gic_d_write_4(sc, GICD_ICFGR(i >> 4), GIC_DEFAULT_ICFGR_INIT); } /* Disable all interrupts. */ for (i = 32; i < sc->nirqs; i += 32) { gic_d_write_4(sc, GICD_ICENABLER(i >> 5), 0xFFFFFFFF); } /* Find the current cpu mask */ mask = gic_cpu_mask(sc); /* Set the mask so we can find this CPU to send it IPIs */ arm_gic_map[PCPU_GET(cpuid)] = mask; /* Set all four targets to this cpu */ mask |= mask << 8; mask |= mask << 16; for (i = 0; i < sc->nirqs; i += 4) { gic_d_write_4(sc, GICD_IPRIORITYR(i >> 2), 0); if (i > 32) { gic_d_write_4(sc, GICD_ITARGETSR(i >> 2), mask); } } /* Set all the interrupts to be in Group 0 (secure) */ for (i = 0; i < sc->nirqs; i += 32) { gic_d_write_4(sc, GICD_IGROUPR(i >> 5), 0); } /* Enable CPU interface */ gic_c_write_4(sc, GICC_CTLR, 1); /* Set priority mask register. */ gic_c_write_4(sc, GICC_PMR, 0xff); /* Enable interrupt distribution */ gic_d_write_4(sc, GICD_CTLR, 0x01); #ifndef INTRNG return (0); #else /* * Now, when everything is initialized, it's right time to * register interrupt controller to interrupt framefork. */ if (intr_pic_register(dev, xref) != 0) { device_printf(dev, "could not register PIC\n"); goto cleanup; } /* * Controller is root if: * - doesn't have interrupt parent * - his interrupt parent is this controller */ pxref = ofw_bus_find_iparent(ofw_bus_get_node(dev)); if (pxref == 0 || xref == pxref) { if (intr_pic_claim_root(dev, xref, arm_gic_intr, sc, GIC_LAST_SGI - GIC_FIRST_SGI + 1) != 0) { device_printf(dev, "could not set PIC as a root\n"); intr_pic_deregister(dev, xref); goto cleanup; } } else { if (sc->gic_res[2] == NULL) { device_printf(dev, "not root PIC must have defined interrupt\n"); intr_pic_deregister(dev, xref); goto cleanup; } if (bus_setup_intr(dev, sc->gic_res[2], INTR_TYPE_CLK, arm_gic_intr, NULL, sc, &sc->gic_intrhand)) { device_printf(dev, "could not setup irq handler\n"); intr_pic_deregister(dev, xref); goto cleanup; } } OF_device_register_xref(xref, dev); /* If we have children probe and attach them */ if (arm_gic_add_children(dev)) { bus_generic_probe(dev); return (bus_generic_attach(dev)); } return (0); cleanup: /* * XXX - not implemented arm_gic_detach() should be called ! */ if (sc->gic_irqs != NULL) free(sc->gic_irqs, M_DEVBUF); bus_release_resources(dev, arm_gic_spec, sc->gic_res); return(ENXIO); #endif } #ifdef INTRNG static struct resource * arm_gic_alloc_resource(device_t bus, device_t child, int type, int *rid, rman_res_t start, rman_res_t end, rman_res_t count, u_int flags) { struct arm_gic_softc *sc; struct arm_gic_devinfo *di; struct resource_list_entry *rle; int j; KASSERT(type == SYS_RES_MEMORY, ("Invalid resoure type %x", type)); sc = device_get_softc(bus); /* * Request for the default allocation with a given rid: use resource * list stored in the local device info. */ if (RMAN_IS_DEFAULT_RANGE(start, end)) { if ((di = device_get_ivars(child)) == NULL) return (NULL); if (type == SYS_RES_IOPORT) type = SYS_RES_MEMORY; rle = resource_list_find(&di->rl, type, *rid); if (rle == NULL) { if (bootverbose) device_printf(bus, "no default resources for " "rid = %d, type = %d\n", *rid, type); return (NULL); } start = rle->start; end = rle->end; count = rle->count; } /* Remap through ranges property */ for (j = 0; j < sc->nranges; j++) { if (start >= sc->ranges[j].bus && end < sc->ranges[j].bus + sc->ranges[j].size) { start -= sc->ranges[j].bus; start += sc->ranges[j].host; end -= sc->ranges[j].bus; end += sc->ranges[j].host; break; } } if (j == sc->nranges && sc->nranges != 0) { if (bootverbose) device_printf(bus, "Could not map resource " "%#jx-%#jx\n", (uintmax_t)start, (uintmax_t)end); return (NULL); } return (bus_generic_alloc_resource(bus, child, type, rid, start, end, count, flags)); } static const struct ofw_bus_devinfo * arm_gic_ofw_get_devinfo(device_t bus __unused, device_t child) { struct arm_gic_devinfo *di; di = device_get_ivars(child); return (&di->obdinfo); } static int arm_gic_intr(void *arg) { struct arm_gic_softc *sc = arg; struct gic_irqsrc *gi; uint32_t irq_active_reg, irq; struct trapframe *tf; irq_active_reg = gic_c_read_4(sc, GICC_IAR); irq = irq_active_reg & 0x3FF; /* * 1. We do EOI here because recent read value from active interrupt * register must be used for it. Another approach is to save this * value into associated interrupt source. * 2. EOI must be done on same CPU where interrupt has fired. Thus * we must ensure that interrupted thread does not migrate to * another CPU. * 3. EOI cannot be delayed by any preemption which could happen on * critical_exit() used in MI intr code, when interrupt thread is * scheduled. See next point. * 4. IPI_RENDEZVOUS assumes that no preemption is permitted during * an action and any use of critical_exit() could break this * assumption. See comments within smp_rendezvous_action(). * 5. We always return FILTER_HANDLED as this is an interrupt * controller dispatch function. Otherwise, in cascaded interrupt * case, the whole interrupt subtree would be masked. */ if (irq >= sc->nirqs) { #ifdef GIC_DEBUG_SPURIOUS device_printf(sc->gic_dev, "Spurious interrupt detected: last irq: %d on CPU%d\n", sc->last_irq[PCPU_GET(cpuid)], PCPU_GET(cpuid)); #endif return (FILTER_HANDLED); } tf = curthread->td_intr_frame; dispatch_irq: gi = sc->gic_irqs + irq; /* * Note that GIC_FIRST_SGI is zero and is not used in 'if' statement * as compiler complains that comparing u_int >= 0 is always true. */ if (irq <= GIC_LAST_SGI) { #ifdef SMP /* Call EOI for all IPI before dispatch. */ gic_c_write_4(sc, GICC_EOIR, irq_active_reg); intr_ipi_dispatch(sgi_to_ipi[gi->gi_irq], tf); goto next_irq; #else device_printf(sc->gic_dev, "SGI %u on UP system detected\n", irq - GIC_FIRST_SGI); gic_c_write_4(sc, GICC_EOIR, irq_active_reg); goto next_irq; #endif } #ifdef GIC_DEBUG_SPURIOUS sc->last_irq[PCPU_GET(cpuid)] = irq; #endif if ((gi->gi_flags & GI_FLAG_EARLY_EOI) == GI_FLAG_EARLY_EOI) gic_c_write_4(sc, GICC_EOIR, irq_active_reg); if (intr_isrc_dispatch(&gi->gi_isrc, tf) != 0) { gic_irq_mask(sc, irq); if ((gi->gi_flags & GI_FLAG_EARLY_EOI) != GI_FLAG_EARLY_EOI) gic_c_write_4(sc, GICC_EOIR, irq_active_reg); device_printf(sc->gic_dev, "Stray irq %u disabled\n", irq); } next_irq: arm_irq_memory_barrier(irq); irq_active_reg = gic_c_read_4(sc, GICC_IAR); irq = irq_active_reg & 0x3FF; if (irq < sc->nirqs) goto dispatch_irq; return (FILTER_HANDLED); } static void gic_config(struct arm_gic_softc *sc, u_int irq, enum intr_trigger trig, enum intr_polarity pol) { uint32_t reg; uint32_t mask; if (irq < GIC_FIRST_SPI) return; mtx_lock_spin(&sc->mutex); reg = gic_d_read_4(sc, GICD_ICFGR(irq >> 4)); mask = (reg >> 2*(irq % 16)) & 0x3; if (pol == INTR_POLARITY_LOW) { mask &= ~GICD_ICFGR_POL_MASK; mask |= GICD_ICFGR_POL_LOW; } else if (pol == INTR_POLARITY_HIGH) { mask &= ~GICD_ICFGR_POL_MASK; mask |= GICD_ICFGR_POL_HIGH; } if (trig == INTR_TRIGGER_LEVEL) { mask &= ~GICD_ICFGR_TRIG_MASK; mask |= GICD_ICFGR_TRIG_LVL; } else if (trig == INTR_TRIGGER_EDGE) { mask &= ~GICD_ICFGR_TRIG_MASK; mask |= GICD_ICFGR_TRIG_EDGE; } /* Set mask */ reg = reg & ~(0x3 << 2*(irq % 16)); reg = reg | (mask << 2*(irq % 16)); gic_d_write_4(sc, GICD_ICFGR(irq >> 4), reg); mtx_unlock_spin(&sc->mutex); } static int gic_bind(struct arm_gic_softc *sc, u_int irq, cpuset_t *cpus) { uint32_t cpu, end, mask; end = min(mp_ncpus, 8); for (cpu = end; cpu < MAXCPU; cpu++) if (CPU_ISSET(cpu, cpus)) return (EINVAL); for (mask = 0, cpu = 0; cpu < end; cpu++) if (CPU_ISSET(cpu, cpus)) mask |= 1 << cpu; gic_d_write_1(sc, GICD_ITARGETSR(0) + irq, mask); return (0); } #ifdef FDT static int gic_map_fdt(device_t dev, u_int ncells, pcell_t *cells, u_int *irqp, enum intr_polarity *polp, enum intr_trigger *trigp) { if (ncells == 1) { *irqp = cells[0]; *polp = INTR_POLARITY_CONFORM; *trigp = INTR_TRIGGER_CONFORM; return (0); } if (ncells == 3) { u_int irq, tripol; /* * The 1st cell is the interrupt type: * 0 = SPI * 1 = PPI * The 2nd cell contains the interrupt number: * [0 - 987] for SPI * [0 - 15] for PPI * The 3rd cell is the flags, encoded as follows: * bits[3:0] trigger type and level flags * 1 = low-to-high edge triggered * 2 = high-to-low edge triggered * 4 = active high level-sensitive * 8 = active low level-sensitive * bits[15:8] PPI interrupt cpu mask * Each bit corresponds to each of the 8 possible cpus * attached to the GIC. A bit set to '1' indicated * the interrupt is wired to that CPU. */ switch (cells[0]) { case 0: irq = GIC_FIRST_SPI + cells[1]; /* SPI irq is checked later. */ break; case 1: irq = GIC_FIRST_PPI + cells[1]; if (irq > GIC_LAST_PPI) { device_printf(dev, "unsupported PPI interrupt " "number %u\n", cells[1]); return (EINVAL); } break; default: device_printf(dev, "unsupported interrupt type " "configuration %u\n", cells[0]); return (EINVAL); } tripol = cells[2] & 0xff; if (tripol & 0xf0 || (tripol & 0x0a && cells[0] == 0)) device_printf(dev, "unsupported trigger/polarity " "configuration 0x%02x\n", tripol); *irqp = irq; *polp = INTR_POLARITY_CONFORM; *trigp = tripol & 0x03 ? INTR_TRIGGER_EDGE : INTR_TRIGGER_LEVEL; return (0); } return (EINVAL); } #endif static int gic_map_intr(device_t dev, struct intr_map_data *data, u_int *irqp, enum intr_polarity *polp, enum intr_trigger *trigp) { u_int irq; enum intr_polarity pol; enum intr_trigger trig; struct arm_gic_softc *sc; #ifdef FDT struct intr_map_data_fdt *daf; #endif sc = device_get_softc(dev); switch (data->type) { #ifdef FDT case INTR_MAP_DATA_FDT: daf = (struct intr_map_data_fdt *)data; if (gic_map_fdt(dev, daf->ncells, daf->cells, &irq, &pol, &trig) != 0) return (EINVAL); + KASSERT(irq >= sc->nirqs || + (sc->gic_irqs[irq].gi_flags & GI_FLAG_MSI) == 0, + ("%s: Attempting to map a MSI interrupt from FDT", + __func__)); break; #endif default: return (ENOTSUP); } if (irq >= sc->nirqs) return (EINVAL); if (pol != INTR_POLARITY_CONFORM && pol != INTR_POLARITY_LOW && pol != INTR_POLARITY_HIGH) return (EINVAL); if (trig != INTR_TRIGGER_CONFORM && trig != INTR_TRIGGER_EDGE && trig != INTR_TRIGGER_LEVEL) return (EINVAL); *irqp = irq; if (polp != NULL) *polp = pol; if (trigp != NULL) *trigp = trig; return (0); } static int arm_gic_map_intr(device_t dev, struct intr_map_data *data, struct intr_irqsrc **isrcp) { int error; u_int irq; struct arm_gic_softc *sc; error = gic_map_intr(dev, data, &irq, NULL, NULL); if (error == 0) { sc = device_get_softc(dev); *isrcp = GIC_INTR_ISRC(sc, irq); } return (error); } static int arm_gic_setup_intr(device_t dev, struct intr_irqsrc *isrc, struct resource *res, struct intr_map_data *data) { struct arm_gic_softc *sc = device_get_softc(dev); struct gic_irqsrc *gi = (struct gic_irqsrc *)isrc; u_int irq; enum intr_trigger trig; enum intr_polarity pol; - if (data == NULL) + if ((gi->gi_flags & GI_FLAG_MSI) == GI_FLAG_MSI) { + irq = gi->gi_irq; + pol = gi->gi_pol; + trig = gi->gi_trig; + KASSERT(pol == INTR_POLARITY_HIGH, + ("%s: MSI interrupts must be active-high", __func__)); + KASSERT(trig == INTR_TRIGGER_EDGE, + ("%s: MSI interrupts must be edge triggered", __func__)); + } else if (data != NULL) { + /* Get config for resource. */ + if (gic_map_intr(dev, data, &irq, &pol, &trig)) + return (EINVAL); + + if (gi->gi_irq != irq) + return (EINVAL); + } else return (ENOTSUP); - /* Get config for resource. */ - if (gic_map_intr(dev, data, &irq, &pol, &trig)) - return (EINVAL); - - if (gi->gi_irq != irq) - return (EINVAL); - /* Compare config if this is not first setup. */ if (isrc->isrc_handlers != 0) { if ((pol != INTR_POLARITY_CONFORM && pol != gi->gi_pol) || (trig != INTR_TRIGGER_CONFORM && trig != gi->gi_trig)) return (EINVAL); else return (0); } - if (pol == INTR_POLARITY_CONFORM) - pol = INTR_POLARITY_LOW; /* just pick some */ - if (trig == INTR_TRIGGER_CONFORM) - trig = INTR_TRIGGER_EDGE; /* just pick some */ + /* For MSI/MSI-X we should have already configured these */ + if ((gi->gi_flags & GI_FLAG_MSI) == 0) { + if (pol == INTR_POLARITY_CONFORM) + pol = INTR_POLARITY_LOW; /* just pick some */ + if (trig == INTR_TRIGGER_CONFORM) + trig = INTR_TRIGGER_EDGE; /* just pick some */ - gi->gi_pol = pol; - gi->gi_trig = trig; - /* Edge triggered interrupts need an early EOI sent */ - if (gi->gi_pol == INTR_TRIGGER_EDGE) - gi->gi_flags |= GI_FLAG_EARLY_EOI; + gi->gi_pol = pol; + gi->gi_trig = trig; + /* Edge triggered interrupts need an early EOI sent */ + if (gi->gi_pol == INTR_TRIGGER_EDGE) + gi->gi_flags |= GI_FLAG_EARLY_EOI; + } + /* * XXX - In case that per CPU interrupt is going to be enabled in time * when SMP is already started, we need some IPI call which * enables it on others CPUs. Further, it's more complicated as * pic_enable_source() and pic_disable_source() should act on * per CPU basis only. Thus, it should be solved here somehow. */ if (isrc->isrc_flags & INTR_ISRCF_PPI) CPU_SET(PCPU_GET(cpuid), &isrc->isrc_cpu); - gic_config(sc, gi->gi_irq, trig, pol); + gic_config(sc, gi->gi_irq, gi->gi_trig, gi->gi_pol); arm_gic_bind_intr(dev, isrc); return (0); } static int arm_gic_teardown_intr(device_t dev, struct intr_irqsrc *isrc, struct resource *res, struct intr_map_data *data) { struct gic_irqsrc *gi = (struct gic_irqsrc *)isrc; - if (isrc->isrc_handlers == 0) { + if (isrc->isrc_handlers == 0 && (gi->gi_flags & GI_FLAG_MSI) == 0) { gi->gi_pol = INTR_POLARITY_CONFORM; gi->gi_trig = INTR_TRIGGER_CONFORM; } return (0); } static void arm_gic_enable_intr(device_t dev, struct intr_irqsrc *isrc) { struct arm_gic_softc *sc = device_get_softc(dev); struct gic_irqsrc *gi = (struct gic_irqsrc *)isrc; arm_irq_memory_barrier(gi->gi_irq); gic_irq_unmask(sc, gi->gi_irq); } static void arm_gic_disable_intr(device_t dev, struct intr_irqsrc *isrc) { struct arm_gic_softc *sc = device_get_softc(dev); struct gic_irqsrc *gi = (struct gic_irqsrc *)isrc; gic_irq_mask(sc, gi->gi_irq); } static void arm_gic_pre_ithread(device_t dev, struct intr_irqsrc *isrc) { struct arm_gic_softc *sc = device_get_softc(dev); struct gic_irqsrc *gi = (struct gic_irqsrc *)isrc; arm_gic_disable_intr(dev, isrc); gic_c_write_4(sc, GICC_EOIR, gi->gi_irq); } static void arm_gic_post_ithread(device_t dev, struct intr_irqsrc *isrc) { arm_irq_memory_barrier(0); arm_gic_enable_intr(dev, isrc); } static void arm_gic_post_filter(device_t dev, struct intr_irqsrc *isrc) { struct arm_gic_softc *sc = device_get_softc(dev); struct gic_irqsrc *gi = (struct gic_irqsrc *)isrc; /* EOI for edge-triggered done earlier. */ if ((gi->gi_flags & GI_FLAG_EARLY_EOI) == GI_FLAG_EARLY_EOI) return; arm_irq_memory_barrier(0); gic_c_write_4(sc, GICC_EOIR, gi->gi_irq); } static int arm_gic_bind_intr(device_t dev, struct intr_irqsrc *isrc) { struct arm_gic_softc *sc = device_get_softc(dev); struct gic_irqsrc *gi = (struct gic_irqsrc *)isrc; if (gi->gi_irq < GIC_FIRST_SPI) return (EINVAL); if (CPU_EMPTY(&isrc->isrc_cpu)) { gic_irq_cpu = intr_irq_next_cpu(gic_irq_cpu, &all_cpus); CPU_SETOF(gic_irq_cpu, &isrc->isrc_cpu); } return (gic_bind(sc, gi->gi_irq, &isrc->isrc_cpu)); } #ifdef SMP static void arm_gic_ipi_send(device_t dev, struct intr_irqsrc *isrc, cpuset_t cpus, u_int ipi) { struct arm_gic_softc *sc = device_get_softc(dev); struct gic_irqsrc *gi = (struct gic_irqsrc *)isrc; uint32_t val = 0, i; for (i = 0; i < MAXCPU; i++) if (CPU_ISSET(i, &cpus)) val |= arm_gic_map[i] << GICD_SGI_TARGET_SHIFT; gic_d_write_4(sc, GICD_SGIR(0), val | gi->gi_irq); } static int arm_gic_ipi_setup(device_t dev, u_int ipi, struct intr_irqsrc **isrcp) { struct intr_irqsrc *isrc; struct arm_gic_softc *sc = device_get_softc(dev); if (sgi_first_unused > GIC_LAST_SGI) return (ENOSPC); isrc = GIC_INTR_ISRC(sc, sgi_first_unused); sgi_to_ipi[sgi_first_unused++] = ipi; CPU_SET(PCPU_GET(cpuid), &isrc->isrc_cpu); *isrcp = isrc; return (0); } #endif #else static int arm_gic_next_irq(struct arm_gic_softc *sc, int last_irq) { uint32_t active_irq; active_irq = gic_c_read_4(sc, GICC_IAR); /* * Immediately EOIR the SGIs, because doing so requires the other * bits (ie CPU number), not just the IRQ number, and we do not * have this information later. */ if ((active_irq & 0x3ff) <= GIC_LAST_SGI) gic_c_write_4(sc, GICC_EOIR, active_irq); active_irq &= 0x3FF; if (active_irq == 0x3FF) { if (last_irq == -1) device_printf(sc->gic_dev, "Spurious interrupt detected\n"); return -1; } return active_irq; } static int arm_gic_config(device_t dev, int irq, enum intr_trigger trig, enum intr_polarity pol) { struct arm_gic_softc *sc = device_get_softc(dev); uint32_t reg; uint32_t mask; /* Function is public-accessible, so validate input arguments */ if ((irq < 0) || (irq >= sc->nirqs)) goto invalid_args; if ((trig != INTR_TRIGGER_EDGE) && (trig != INTR_TRIGGER_LEVEL) && (trig != INTR_TRIGGER_CONFORM)) goto invalid_args; if ((pol != INTR_POLARITY_HIGH) && (pol != INTR_POLARITY_LOW) && (pol != INTR_POLARITY_CONFORM)) goto invalid_args; mtx_lock_spin(&sc->mutex); reg = gic_d_read_4(sc, GICD_ICFGR(irq >> 4)); mask = (reg >> 2*(irq % 16)) & 0x3; if (pol == INTR_POLARITY_LOW) { mask &= ~GICD_ICFGR_POL_MASK; mask |= GICD_ICFGR_POL_LOW; } else if (pol == INTR_POLARITY_HIGH) { mask &= ~GICD_ICFGR_POL_MASK; mask |= GICD_ICFGR_POL_HIGH; } if (trig == INTR_TRIGGER_LEVEL) { mask &= ~GICD_ICFGR_TRIG_MASK; mask |= GICD_ICFGR_TRIG_LVL; } else if (trig == INTR_TRIGGER_EDGE) { mask &= ~GICD_ICFGR_TRIG_MASK; mask |= GICD_ICFGR_TRIG_EDGE; } /* Set mask */ reg = reg & ~(0x3 << 2*(irq % 16)); reg = reg | (mask << 2*(irq % 16)); gic_d_write_4(sc, GICD_ICFGR(irq >> 4), reg); mtx_unlock_spin(&sc->mutex); return (0); invalid_args: device_printf(dev, "gic_config_irg, invalid parameters\n"); return (EINVAL); } static void arm_gic_mask(device_t dev, int irq) { struct arm_gic_softc *sc = device_get_softc(dev); gic_d_write_4(sc, GICD_ICENABLER(irq >> 5), (1UL << (irq & 0x1F))); gic_c_write_4(sc, GICC_EOIR, irq); /* XXX - not allowed */ } static void arm_gic_unmask(device_t dev, int irq) { struct arm_gic_softc *sc = device_get_softc(dev); if (irq > GIC_LAST_SGI) arm_irq_memory_barrier(irq); gic_d_write_4(sc, GICD_ISENABLER(irq >> 5), (1UL << (irq & 0x1F))); } #ifdef SMP static void arm_gic_ipi_send(device_t dev, cpuset_t cpus, u_int ipi) { struct arm_gic_softc *sc = device_get_softc(dev); uint32_t val = 0, i; for (i = 0; i < MAXCPU; i++) if (CPU_ISSET(i, &cpus)) val |= arm_gic_map[i] << GICD_SGI_TARGET_SHIFT; gic_d_write_4(sc, GICD_SGIR(0), val | ipi); } static int arm_gic_ipi_read(device_t dev, int i) { if (i != -1) { /* * The intr code will automagically give the frame pointer * if the interrupt argument is 0. */ if ((unsigned int)i > 16) return (0); return (i); } return (0x3ff); } static void arm_gic_ipi_clear(device_t dev, int ipi) { /* no-op */ } #endif static void gic_post_filter(void *arg) { struct arm_gic_softc *sc = gic_sc; uintptr_t irq = (uintptr_t) arg; if (irq > GIC_LAST_SGI) arm_irq_memory_barrier(irq); gic_c_write_4(sc, GICC_EOIR, irq); } static int gic_config_irq(int irq, enum intr_trigger trig, enum intr_polarity pol) { return (arm_gic_config(gic_sc->gic_dev, irq, trig, pol)); } void arm_mask_irq(uintptr_t nb) { arm_gic_mask(gic_sc->gic_dev, nb); } void arm_unmask_irq(uintptr_t nb) { arm_gic_unmask(gic_sc->gic_dev, nb); } int arm_get_next_irq(int last_irq) { return (arm_gic_next_irq(gic_sc, last_irq)); } #ifdef SMP void intr_pic_init_secondary(void) { arm_gic_init_secondary(gic_sc->gic_dev); } void pic_ipi_send(cpuset_t cpus, u_int ipi) { arm_gic_ipi_send(gic_sc->gic_dev, cpus, ipi); } int pic_ipi_read(int i) { return (arm_gic_ipi_read(gic_sc->gic_dev, i)); } void pic_ipi_clear(int ipi) { arm_gic_ipi_clear(gic_sc->gic_dev, ipi); } #endif #endif /* INTRNG */ static device_method_t arm_gic_methods[] = { /* Device interface */ DEVMETHOD(device_probe, arm_gic_probe), DEVMETHOD(device_attach, arm_gic_attach), #ifdef INTRNG /* Bus interface */ DEVMETHOD(bus_add_child, bus_generic_add_child), DEVMETHOD(bus_alloc_resource, arm_gic_alloc_resource), DEVMETHOD(bus_release_resource, bus_generic_release_resource), DEVMETHOD(bus_activate_resource,bus_generic_activate_resource), /* ofw_bus interface */ DEVMETHOD(ofw_bus_get_devinfo, arm_gic_ofw_get_devinfo), DEVMETHOD(ofw_bus_get_compat, ofw_bus_gen_get_compat), DEVMETHOD(ofw_bus_get_model, ofw_bus_gen_get_model), DEVMETHOD(ofw_bus_get_name, ofw_bus_gen_get_name), DEVMETHOD(ofw_bus_get_node, ofw_bus_gen_get_node), DEVMETHOD(ofw_bus_get_type, ofw_bus_gen_get_type), /* Interrupt controller interface */ DEVMETHOD(pic_disable_intr, arm_gic_disable_intr), DEVMETHOD(pic_enable_intr, arm_gic_enable_intr), DEVMETHOD(pic_map_intr, arm_gic_map_intr), DEVMETHOD(pic_setup_intr, arm_gic_setup_intr), DEVMETHOD(pic_teardown_intr, arm_gic_teardown_intr), DEVMETHOD(pic_post_filter, arm_gic_post_filter), DEVMETHOD(pic_post_ithread, arm_gic_post_ithread), DEVMETHOD(pic_pre_ithread, arm_gic_pre_ithread), #ifdef SMP DEVMETHOD(pic_bind_intr, arm_gic_bind_intr), DEVMETHOD(pic_init_secondary, arm_gic_init_secondary), DEVMETHOD(pic_ipi_send, arm_gic_ipi_send), DEVMETHOD(pic_ipi_setup, arm_gic_ipi_setup), #endif #endif { 0, 0 } }; static driver_t arm_gic_driver = { "gic", arm_gic_methods, sizeof(struct arm_gic_softc), }; static devclass_t arm_gic_devclass; EARLY_DRIVER_MODULE(gic, simplebus, arm_gic_driver, arm_gic_devclass, 0, 0, BUS_PASS_INTERRUPT + BUS_PASS_ORDER_MIDDLE); EARLY_DRIVER_MODULE(gic, ofwbus, arm_gic_driver, arm_gic_devclass, 0, 0, BUS_PASS_INTERRUPT + BUS_PASS_ORDER_MIDDLE); #ifdef INTRNG /* * GICv2m support -- the GICv2 MSI/MSI-X controller. */ #define GICV2M_MSI_TYPER 0x008 #define MSI_TYPER_SPI_BASE(x) (((x) >> 16) & 0x3ff) #define MSI_TYPER_SPI_COUNT(x) (((x) >> 0) & 0x3ff) #define GICv2M_MSI_SETSPI_NS 0x040 #define GICV2M_MSI_IIDR 0xFCC struct arm_gicv2m_softc { struct resource *sc_mem; struct mtx sc_mutex; u_int sc_spi_start; + u_int sc_spi_end; u_int sc_spi_count; - u_int sc_spi_offset; }; static struct ofw_compat_data gicv2m_compat_data[] = { {"arm,gic-v2m-frame", true}, {NULL, false} }; static int arm_gicv2m_probe(device_t dev) { if (!ofw_bus_status_okay(dev)) return (ENXIO); if (!ofw_bus_search_compatible(dev, gicv2m_compat_data)->ocd_data) return (ENXIO); device_set_desc(dev, "ARM Generic Interrupt Controller MSI/MSIX"); return (BUS_PROBE_DEFAULT); } static int arm_gicv2m_attach(device_t dev) { struct arm_gicv2m_softc *sc; + struct arm_gic_softc *psc; uint32_t typer; int rid; + psc = device_get_softc(device_get_parent(dev)); sc = device_get_softc(dev); rid = 0; sc->sc_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, RF_ACTIVE); if (sc->sc_mem == NULL) { device_printf(dev, "Unable to allocate resources\n"); return (ENXIO); } typer = bus_read_4(sc->sc_mem, GICV2M_MSI_TYPER); sc->sc_spi_start = MSI_TYPER_SPI_BASE(typer); sc->sc_spi_count = MSI_TYPER_SPI_COUNT(typer); + sc->sc_spi_end = sc->sc_spi_start + sc->sc_spi_count; + /* Reserve these interrupts for MSI/MSI-X use */ + arm_gic_reserve_msi_range(device_get_parent(dev), sc->sc_spi_start, + sc->sc_spi_count); + mtx_init(&sc->sc_mutex, "GICv2m lock", "", MTX_DEF); + intr_msi_register(dev, gic_xref(dev)); + if (bootverbose) device_printf(dev, "using spi %u to %u\n", sc->sc_spi_start, sc->sc_spi_start + sc->sc_spi_count - 1); return (0); } +static int +arm_gicv2m_alloc_msi(device_t dev, device_t child, int count, int maxcount, + device_t *pic, struct intr_irqsrc **srcs) +{ + struct arm_gic_softc *psc; + struct arm_gicv2m_softc *sc; + int i, irq, end_irq; + bool found; + + KASSERT(powerof2(count), ("%s: bad count", __func__)); + KASSERT(powerof2(maxcount), ("%s: bad maxcount", __func__)); + + psc = device_get_softc(device_get_parent(dev)); + sc = device_get_softc(dev); + + mtx_lock(&sc->sc_mutex); + + found = false; + for (irq = sc->sc_spi_start; irq < sc->sc_spi_end && !found; irq++) { + /* Start on an aligned interrupt */ + if ((irq & (maxcount - 1)) != 0) + continue; + + /* Assume we found a valid range until shown otherwise */ + found = true; + + /* Check this range is valid */ + for (end_irq = irq; end_irq != irq + count - 1; end_irq++) { + /* No free interrupts */ + if (end_irq == sc->sc_spi_end) { + found = false; + break; + } + + KASSERT((psc->gic_irqs[irq].gi_flags & GI_FLAG_MSI)!= 0, + ("%s: Non-MSI interrupt found", __func__)); + + /* This is already used */ + if ((psc->gic_irqs[irq].gi_flags & GI_FLAG_MSI_USED) == + GI_FLAG_MSI_USED) { + found = false; + break; + } + } + } + + /* Not enough interrupts were found */ + if (!found || irq == sc->sc_spi_end) { + mtx_unlock(&sc->sc_mutex); + return (ENXIO); + } + + for (i = 0; i < count; i++) { + /* Mark the interrupt as used */ + psc->gic_irqs[irq + i].gi_flags |= GI_FLAG_MSI_USED; + + } + mtx_unlock(&sc->sc_mutex); + + for (i = 0; i < count; i++) + srcs[i] = (struct intr_irqsrc *)&psc->gic_irqs[irq + i]; + *pic = device_get_parent(dev); + + return (0); +} + +static int +arm_gicv2m_release_msi(device_t dev, device_t child, int count, + struct intr_irqsrc **isrc) +{ + struct arm_gicv2m_softc *sc; + struct gic_irqsrc *gi; + int i; + + sc = device_get_softc(dev); + + mtx_lock(&sc->sc_mutex); + for (i = 0; i < count; i++) { + gi = (struct gic_irqsrc *)isrc; + + KASSERT((gi->gi_flags & GI_FLAG_MSI_USED) == GI_FLAG_MSI_USED, + ("%s: Trying to release an unused MSI-X interrupt", + __func__)); + + gi->gi_flags &= ~GI_FLAG_MSI_USED; + mtx_unlock(&sc->sc_mutex); + } + + return (0); +} + +static int +arm_gicv2m_alloc_msix(device_t dev, device_t child, device_t *pic, + struct intr_irqsrc **isrcp) +{ + struct arm_gicv2m_softc *sc; + struct arm_gic_softc *psc; + int irq; + + psc = device_get_softc(device_get_parent(dev)); + sc = device_get_softc(dev); + + mtx_lock(&sc->sc_mutex); + /* Find an unused interrupt */ + for (irq = sc->sc_spi_start; irq < sc->sc_spi_end; irq++) { + KASSERT((psc->gic_irqs[irq].gi_flags & GI_FLAG_MSI) != 0, + ("%s: Non-MSI interrupt found", __func__)); + if ((psc->gic_irqs[irq].gi_flags & GI_FLAG_MSI_USED) == 0) + break; + } + /* No free interrupt was found */ + if (irq == sc->sc_spi_end) { + mtx_unlock(&sc->sc_mutex); + return (ENXIO); + } + + /* Mark the interrupt as used */ + psc->gic_irqs[irq].gi_flags |= GI_FLAG_MSI_USED; + mtx_unlock(&sc->sc_mutex); + + *isrcp = (struct intr_irqsrc *)&psc->gic_irqs[irq]; + *pic = device_get_parent(dev); + + return (0); +} + +static int +arm_gicv2m_release_msix(device_t dev, device_t child, struct intr_irqsrc *isrc) +{ + struct arm_gicv2m_softc *sc; + struct gic_irqsrc *gi; + + sc = device_get_softc(dev); + gi = (struct gic_irqsrc *)isrc; + + KASSERT((gi->gi_flags & GI_FLAG_MSI_USED) == GI_FLAG_MSI_USED, + ("%s: Trying to release an unused MSI-X interrupt", __func__)); + + mtx_lock(&sc->sc_mutex); + gi->gi_flags &= ~GI_FLAG_MSI_USED; + mtx_unlock(&sc->sc_mutex); + + return (0); +} + +static int +arm_gicv2m_map_msi(device_t dev, device_t child, struct intr_irqsrc *isrc, + uint64_t *addr, uint32_t *data) +{ + struct arm_gicv2m_softc *sc = device_get_softc(dev); + struct gic_irqsrc *gi = (struct gic_irqsrc *)isrc; + + *addr = vtophys(rman_get_virtual(sc->sc_mem)) + GICv2M_MSI_SETSPI_NS; + *data = gi->gi_irq; + + return (0); +} + static device_method_t arm_gicv2m_methods[] = { /* Device interface */ DEVMETHOD(device_probe, arm_gicv2m_probe), DEVMETHOD(device_attach, arm_gicv2m_attach), + + /* MSI/MSI-X */ + DEVMETHOD(msi_alloc_msi, arm_gicv2m_alloc_msi), + DEVMETHOD(msi_release_msi, arm_gicv2m_release_msi), + DEVMETHOD(msi_alloc_msix, arm_gicv2m_alloc_msix), + DEVMETHOD(msi_release_msix, arm_gicv2m_release_msix), + DEVMETHOD(msi_map_msi, arm_gicv2m_map_msi), /* End */ DEVMETHOD_END }; DEFINE_CLASS_0(gicv2m, arm_gicv2m_driver, arm_gicv2m_methods, sizeof(struct arm_gicv2m_softc)); static devclass_t arm_gicv2m_devclass; EARLY_DRIVER_MODULE(gicv2m, gic, arm_gicv2m_driver, arm_gicv2m_devclass, 0, 0, BUS_PASS_INTERRUPT + BUS_PASS_ORDER_MIDDLE); #endif Index: head/sys/conf/files.arm =================================================================== --- head/sys/conf/files.arm (revision 299927) +++ head/sys/conf/files.arm (revision 299928) @@ -1,143 +1,144 @@ # $FreeBSD$ arm/arm/autoconf.c standard arm/arm/bcopy_page.S standard arm/arm/bcopyinout.S standard arm/arm/blockio.S standard arm/arm/bus_space_asm_generic.S standard arm/arm/bus_space_base.c optional fdt arm/arm/bus_space_generic.c standard arm/arm/busdma_machdep-v4.c optional !armv6 arm/arm/busdma_machdep-v6.c optional armv6 arm/arm/copystr.S standard arm/arm/cpufunc.c standard arm/arm/cpufunc_asm.S standard arm/arm/cpufunc_asm_arm9.S optional cpu_arm9 | cpu_arm9e arm/arm/cpufunc_asm_arm11.S optional cpu_arm1176 arm/arm/cpufunc_asm_arm11x6.S optional cpu_arm1176 arm/arm/cpufunc_asm_armv4.S optional cpu_arm9 | cpu_arm9e | cpu_fa526 | cpu_xscale_pxa2x0 | cpu_xscale_ixp425 | cpu_xscale_81342 arm/arm/cpufunc_asm_armv5_ec.S optional cpu_arm9e arm/arm/cpufunc_asm_armv6.S optional cpu_arm1176 arm/arm/cpufunc_asm_armv7.S optional cpu_cortexa | cpu_krait | cpu_mv_pj4b arm/arm/cpufunc_asm_fa526.S optional cpu_fa526 arm/arm/cpufunc_asm_pj4b.S optional cpu_mv_pj4b arm/arm/cpufunc_asm_sheeva.S optional cpu_arm9e arm/arm/cpufunc_asm_xscale.S optional cpu_xscale_pxa2x0 | cpu_xscale_ixp425 | cpu_xscale_81342 arm/arm/cpufunc_asm_xscale_c3.S optional cpu_xscale_81342 arm/arm/cpuinfo.c standard arm/arm/cpu_asm-v6.S optional armv6 arm/arm/db_disasm.c optional ddb arm/arm/db_interface.c optional ddb arm/arm/db_trace.c optional ddb arm/arm/debug_monitor.c optional ddb armv6 arm/arm/disassem.c optional ddb arm/arm/dump_machdep.c standard arm/arm/elf_machdep.c standard arm/arm/elf_note.S standard arm/arm/exception.S standard arm/arm/fiq.c standard arm/arm/fiq_subr.S standard arm/arm/fusu.S standard arm/arm/gdb_machdep.c optional gdb arm/arm/generic_timer.c optional generic_timer arm/arm/gic.c optional gic arm/arm/hdmi_if.m optional hdmi arm/arm/identcpu.c standard arm/arm/in_cksum.c optional inet | inet6 arm/arm/in_cksum_arm.S optional inet | inet6 arm/arm/intr.c optional !intrng kern/subr_intr.c optional intrng arm/arm/locore.S standard no-obj arm/arm/machdep.c standard arm/arm/machdep_intr.c standard arm/arm/mem.c optional mem arm/arm/minidump_machdep.c optional mem arm/arm/mp_machdep.c optional smp arm/arm/mpcore_timer.c optional mpcore_timer arm/arm/nexus.c standard arm/arm/ofw_machdep.c optional fdt arm/arm/physmem.c standard arm/arm/pl190.c optional pl190 arm/arm/pl310.c optional pl310 arm/arm/platform.c optional platform arm/arm/platform_if.m optional platform arm/arm/pmap-v4.c optional !armv6 arm/arm/pmap-v6.c optional armv6 arm/arm/pmu.c optional pmu | fdt hwpmc arm/arm/sc_machdep.c optional sc arm/arm/setcpsr.S standard arm/arm/setstack.s standard arm/arm/stack_machdep.c optional ddb | stack arm/arm/stdatomic.c standard \ compile-with "${NORMAL_C:N-Wmissing-prototypes}" arm/arm/support.S standard arm/arm/swtch.S standard arm/arm/swtch-v4.S optional !armv6 arm/arm/swtch-v6.S optional armv6 arm/arm/sys_machdep.c standard arm/arm/syscall.c standard arm/arm/trap-v4.c optional !armv6 arm/arm/trap-v6.c optional armv6 arm/arm/uio_machdep.c standard arm/arm/undefined.c standard arm/arm/unwind.c optional ddb | kdtrace_hooks arm/arm/vm_machdep.c standard arm/arm/vfp.c standard board_id.h standard \ dependency "$S/arm/conf/genboardid.awk $S/arm/conf/mach-types" \ compile-with "${AWK} -f $S/arm/conf/genboardid.awk $S/arm/conf/mach-types > board_id.h" \ no-obj no-implicit-rule before-depend \ clean "board_id.h" cddl/compat/opensolaris/kern/opensolaris_atomic.c optional zfs | dtrace compile-with "${CDDL_C}" cddl/dev/dtrace/arm/dtrace_asm.S optional dtrace compile-with "${DTRACE_S}" cddl/dev/dtrace/arm/dtrace_subr.c optional dtrace compile-with "${DTRACE_C}" cddl/dev/fbt/arm/fbt_isa.c optional dtrace_fbt | dtraceall compile-with "${FBT_C}" crypto/blowfish/bf_enc.c optional crypto | ipsec crypto/des/des_enc.c optional crypto | ipsec | netsmb dev/dwc/if_dwc.c optional dwc dev/dwc/if_dwc_if.m optional dwc dev/fb/fb.c optional sc dev/fdt/fdt_arm_platform.c optional platform fdt dev/hwpmc/hwpmc_arm.c optional hwpmc dev/hwpmc/hwpmc_armv7.c optional hwpmc armv6 dev/iicbus/twsi/twsi.c optional twsi dev/ofw/ofw_cpu.c optional fdt dev/ofw/ofwpci.c optional fdt pci dev/pci/pci_host_generic.c optional pci_host_generic pci fdt dev/psci/psci.c optional psci dev/psci/psci_arm.S optional psci dev/syscons/scgfbrndr.c optional sc dev/syscons/scterm-teken.c optional sc dev/syscons/scvtb.c optional sc dev/uart/uart_cpu_fdt.c optional uart fdt font.h optional sc \ compile-with "uudecode < /usr/share/syscons/fonts/${SC_DFLT_FONT}-8x16.fnt && file2c 'u_char dflt_font_16[16*256] = {' '};' < ${SC_DFLT_FONT}-8x16 > font.h && uudecode < /usr/share/syscons/fonts/${SC_DFLT_FONT}-8x14.fnt && file2c 'u_char dflt_font_14[14*256] = {' '};' < ${SC_DFLT_FONT}-8x14 >> font.h && uudecode < /usr/share/syscons/fonts/${SC_DFLT_FONT}-8x8.fnt && file2c 'u_char dflt_font_8[8*256] = {' '};' < ${SC_DFLT_FONT}-8x8 >> font.h" \ no-obj no-implicit-rule before-depend \ clean "font.h ${SC_DFLT_FONT}-8x14 ${SC_DFLT_FONT}-8x16 ${SC_DFLT_FONT}-8x8" +kern/msi_if.m optional intrng kern/pic_if.m optional intrng kern/subr_busdma_bufalloc.c standard kern/subr_devmap.c standard kern/subr_sfbuf.c standard libkern/arm/aeabi_unwind.c standard libkern/arm/divsi3.S standard libkern/arm/ffs.S standard libkern/arm/ldivmod.S standard libkern/arm/ldivmod_helper.c standard libkern/arm/memclr.S standard libkern/arm/memcpy.S standard libkern/arm/memset.S standard libkern/arm/muldi3.c standard libkern/ashldi3.c standard libkern/ashrdi3.c standard libkern/divdi3.c standard libkern/ffsl.c standard libkern/ffsll.c standard libkern/fls.c standard libkern/flsl.c standard libkern/flsll.c standard libkern/lshrdi3.c standard libkern/moddi3.c standard libkern/qdivrem.c standard libkern/ucmpdi2.c standard libkern/udivdi3.c standard libkern/umoddi3.c standard Index: head/sys/dev/pci/pci_host_generic.c =================================================================== --- head/sys/dev/pci/pci_host_generic.c (revision 299927) +++ head/sys/dev/pci/pci_host_generic.c (revision 299928) @@ -1,874 +1,904 @@ /*- * Copyright (c) 2015 Ruslan Bukin * Copyright (c) 2014 The FreeBSD Foundation * All rights reserved. * * This software was developed by Semihalf under * the sponsorship of the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* Generic ECAM PCIe driver */ #include __FBSDID("$FreeBSD$"); #include "opt_platform.h" #include #include #include #include #include #include #include #include #include #include +#if defined(INTRNG) +#include +#endif + #include #include #include #include #include #include #include #include #include #include #include #include #include "pcib_if.h" /* Assembling ECAM Configuration Address */ #define PCIE_BUS_SHIFT 20 #define PCIE_SLOT_SHIFT 15 #define PCIE_FUNC_SHIFT 12 #define PCIE_BUS_MASK 0xFF #define PCIE_SLOT_MASK 0x1F #define PCIE_FUNC_MASK 0x07 #define PCIE_REG_MASK 0xFFF #define PCIE_ADDR_OFFSET(bus, slot, func, reg) \ ((((bus) & PCIE_BUS_MASK) << PCIE_BUS_SHIFT) | \ (((slot) & PCIE_SLOT_MASK) << PCIE_SLOT_SHIFT) | \ (((func) & PCIE_FUNC_MASK) << PCIE_FUNC_SHIFT) | \ ((reg) & PCIE_REG_MASK)) #define PCI_IO_WINDOW_OFFSET 0x1000 #define SPACE_CODE_SHIFT 24 #define SPACE_CODE_MASK 0x3 #define SPACE_CODE_IO_SPACE 0x1 #define PROPS_CELL_SIZE 1 #define PCI_ADDR_CELL_SIZE 2 /* OFW bus interface */ struct generic_pcie_ofw_devinfo { struct ofw_bus_devinfo di_dinfo; struct resource_list di_rl; }; /* Forward prototypes */ static int generic_pcie_probe(device_t dev); static int parse_pci_mem_ranges(struct generic_pcie_softc *sc); static uint32_t generic_pcie_read_config(device_t dev, u_int bus, u_int slot, u_int func, u_int reg, int bytes); static void generic_pcie_write_config(device_t dev, u_int bus, u_int slot, u_int func, u_int reg, uint32_t val, int bytes); static int generic_pcie_maxslots(device_t dev); static int generic_pcie_read_ivar(device_t dev, device_t child, int index, uintptr_t *result); static int generic_pcie_write_ivar(device_t dev, device_t child, int index, uintptr_t value); static struct resource *generic_pcie_alloc_resource_ofw(device_t, device_t, int, int *, rman_res_t, rman_res_t, rman_res_t, u_int); static struct resource *generic_pcie_alloc_resource_pcie(device_t dev, device_t child, int type, int *rid, rman_res_t start, rman_res_t end, rman_res_t count, u_int flags); static int generic_pcie_release_resource(device_t dev, device_t child, int type, int rid, struct resource *res); static int generic_pcie_release_resource_ofw(device_t, device_t, int, int, struct resource *); static int generic_pcie_release_resource_pcie(device_t, device_t, int, int, struct resource *); static int generic_pcie_ofw_bus_attach(device_t); static const struct ofw_bus_devinfo *generic_pcie_ofw_get_devinfo(device_t, device_t); static __inline void get_addr_size_cells(phandle_t node, pcell_t *addr_cells, pcell_t *size_cells) { *addr_cells = 2; /* Find address cells if present */ OF_getencprop(node, "#address-cells", addr_cells, sizeof(*addr_cells)); *size_cells = 2; /* Find size cells if present */ OF_getencprop(node, "#size-cells", size_cells, sizeof(*size_cells)); } static int generic_pcie_probe(device_t dev) { if (!ofw_bus_status_okay(dev)) return (ENXIO); if (ofw_bus_is_compatible(dev, "pci-host-ecam-generic")) { device_set_desc(dev, "Generic PCI host controller"); return (BUS_PROBE_GENERIC); } if (ofw_bus_is_compatible(dev, "arm,gem5_pcie")) { device_set_desc(dev, "GEM5 PCIe host controller"); return (BUS_PROBE_DEFAULT); } return (ENXIO); } int pci_host_generic_attach(device_t dev) { struct generic_pcie_softc *sc; uint64_t phys_base; uint64_t pci_base; uint64_t size; + phandle_t node; int error; int tuple; int rid; sc = device_get_softc(dev); sc->dev = dev; /* Retrieve 'ranges' property from FDT */ if (bootverbose) device_printf(dev, "parsing FDT for ECAM%d:\n", sc->ecam); if (parse_pci_mem_ranges(sc)) return (ENXIO); /* Attach OFW bus */ if (generic_pcie_ofw_bus_attach(dev) != 0) return (ENXIO); rid = 0; sc->res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, RF_ACTIVE); if (sc->res == NULL) { device_printf(dev, "could not map memory.\n"); return (ENXIO); } sc->bst = rman_get_bustag(sc->res); sc->bsh = rman_get_bushandle(sc->res); sc->mem_rman.rm_type = RMAN_ARRAY; sc->mem_rman.rm_descr = "PCIe Memory"; sc->io_rman.rm_type = RMAN_ARRAY; sc->io_rman.rm_descr = "PCIe IO window"; /* Initialize rman and allocate memory regions */ error = rman_init(&sc->mem_rman); if (error) { device_printf(dev, "rman_init() failed. error = %d\n", error); return (error); } error = rman_init(&sc->io_rman); if (error) { device_printf(dev, "rman_init() failed. error = %d\n", error); return (error); } for (tuple = 0; tuple < MAX_RANGES_TUPLES; tuple++) { phys_base = sc->ranges[tuple].phys_base; pci_base = sc->ranges[tuple].pci_base; size = sc->ranges[tuple].size; if (phys_base == 0 || size == 0) continue; /* empty range element */ if (sc->ranges[tuple].flags & FLAG_MEM) { error = rman_manage_region(&sc->mem_rman, phys_base, phys_base + size - 1); } else if (sc->ranges[tuple].flags & FLAG_IO) { error = rman_manage_region(&sc->io_rman, pci_base + PCI_IO_WINDOW_OFFSET, pci_base + PCI_IO_WINDOW_OFFSET + size - 1); } else continue; if (error) { device_printf(dev, "rman_manage_region() failed." "error = %d\n", error); rman_fini(&sc->mem_rman); return (error); } } - ofw_bus_setup_iinfo(ofw_bus_get_node(dev), &sc->pci_iinfo, - sizeof(cell_t)); + node = ofw_bus_get_node(dev); + ofw_bus_setup_iinfo(node, &sc->pci_iinfo, sizeof(cell_t)); + /* Find the MSI interrupt handler */ + OF_searchencprop(node, "msi-parent", &sc->msi_parent, + sizeof(sc->msi_parent)); + device_add_child(dev, "pci", -1); return (bus_generic_attach(dev)); } static int parse_pci_mem_ranges(struct generic_pcie_softc *sc) { pcell_t pci_addr_cells, parent_addr_cells; pcell_t attributes, size_cells; cell_t *base_ranges; int nbase_ranges; phandle_t node; int i, j, k; int tuple; node = ofw_bus_get_node(sc->dev); OF_getencprop(node, "#address-cells", &pci_addr_cells, sizeof(pci_addr_cells)); OF_getencprop(node, "#size-cells", &size_cells, sizeof(size_cells)); OF_getencprop(OF_parent(node), "#address-cells", &parent_addr_cells, sizeof(parent_addr_cells)); if (parent_addr_cells != 2 || pci_addr_cells != 3 || size_cells != 2) { device_printf(sc->dev, "Unexpected number of address or size cells in FDT\n"); return (ENXIO); } nbase_ranges = OF_getproplen(node, "ranges"); sc->nranges = nbase_ranges / sizeof(cell_t) / (parent_addr_cells + pci_addr_cells + size_cells); base_ranges = malloc(nbase_ranges, M_DEVBUF, M_WAITOK); OF_getencprop(node, "ranges", base_ranges, nbase_ranges); for (i = 0, j = 0; i < sc->nranges; i++) { attributes = (base_ranges[j++] >> SPACE_CODE_SHIFT) & \ SPACE_CODE_MASK; if (attributes == SPACE_CODE_IO_SPACE) { sc->ranges[i].flags |= FLAG_IO; } else { sc->ranges[i].flags |= FLAG_MEM; } sc->ranges[i].pci_base = 0; for (k = 0; k < (pci_addr_cells - 1); k++) { sc->ranges[i].pci_base <<= 32; sc->ranges[i].pci_base |= base_ranges[j++]; } sc->ranges[i].phys_base = 0; for (k = 0; k < parent_addr_cells; k++) { sc->ranges[i].phys_base <<= 32; sc->ranges[i].phys_base |= base_ranges[j++]; } sc->ranges[i].size = 0; for (k = 0; k < size_cells; k++) { sc->ranges[i].size <<= 32; sc->ranges[i].size |= base_ranges[j++]; } } for (; i < MAX_RANGES_TUPLES; i++) { /* zero-fill remaining tuples to mark empty elements in array */ sc->ranges[i].pci_base = 0; sc->ranges[i].phys_base = 0; sc->ranges[i].size = 0; } if (bootverbose) { for (tuple = 0; tuple < MAX_RANGES_TUPLES; tuple++) { device_printf(sc->dev, "\tPCI addr: 0x%jx, CPU addr: 0x%jx, Size: 0x%jx\n", sc->ranges[tuple].pci_base, sc->ranges[tuple].phys_base, sc->ranges[tuple].size); } } free(base_ranges, M_DEVBUF); return (0); } static uint32_t generic_pcie_read_config(device_t dev, u_int bus, u_int slot, u_int func, u_int reg, int bytes) { struct generic_pcie_softc *sc; bus_space_handle_t h; bus_space_tag_t t; uint64_t offset; uint32_t data; if ((bus > PCI_BUSMAX) || (slot > PCI_SLOTMAX) || (func > PCI_FUNCMAX) || (reg > PCIE_REGMAX)) return (~0U); sc = device_get_softc(dev); offset = PCIE_ADDR_OFFSET(bus, slot, func, reg); t = sc->bst; h = sc->bsh; switch (bytes) { case 1: data = bus_space_read_1(t, h, offset); break; case 2: data = le16toh(bus_space_read_2(t, h, offset)); break; case 4: data = le32toh(bus_space_read_4(t, h, offset)); break; default: return (~0U); } return (data); } static void generic_pcie_write_config(device_t dev, u_int bus, u_int slot, u_int func, u_int reg, uint32_t val, int bytes) { struct generic_pcie_softc *sc; bus_space_handle_t h; bus_space_tag_t t; uint64_t offset; if ((bus > PCI_BUSMAX) || (slot > PCI_SLOTMAX) || (func > PCI_FUNCMAX) || (reg > PCIE_REGMAX)) return; sc = device_get_softc(dev); offset = PCIE_ADDR_OFFSET(bus, slot, func, reg); t = sc->bst; h = sc->bsh; switch (bytes) { case 1: bus_space_write_1(t, h, offset, val); break; case 2: bus_space_write_2(t, h, offset, htole16(val)); break; case 4: bus_space_write_4(t, h, offset, htole32(val)); break; default: return; } } static int generic_pcie_maxslots(device_t dev) { return (31); /* max slots per bus acc. to standard */ } static int generic_pcie_route_interrupt(device_t bus, device_t dev, int pin) { struct generic_pcie_softc *sc; struct ofw_pci_register reg; uint32_t pintr, mintr[2]; phandle_t iparent; int intrcells; sc = device_get_softc(bus); pintr = pin; bzero(®, sizeof(reg)); reg.phys_hi = (pci_get_bus(dev) << OFW_PCI_PHYS_HI_BUSSHIFT) | (pci_get_slot(dev) << OFW_PCI_PHYS_HI_DEVICESHIFT) | (pci_get_function(dev) << OFW_PCI_PHYS_HI_FUNCTIONSHIFT); intrcells = ofw_bus_lookup_imap(ofw_bus_get_node(dev), &sc->pci_iinfo, ®, sizeof(reg), &pintr, sizeof(pintr), mintr, sizeof(mintr), &iparent); if (intrcells) { pintr = ofw_bus_map_intr(dev, iparent, intrcells, mintr); return (pintr); } device_printf(bus, "could not route pin %d for device %d.%d\n", pin, pci_get_slot(dev), pci_get_function(dev)); return (PCI_INVALID_IRQ); } static int generic_pcie_read_ivar(device_t dev, device_t child, int index, uintptr_t *result) { struct generic_pcie_softc *sc; int secondary_bus; sc = device_get_softc(dev); if (index == PCIB_IVAR_BUS) { /* this pcib adds only pci bus 0 as child */ secondary_bus = 0; *result = secondary_bus; return (0); } if (index == PCIB_IVAR_DOMAIN) { *result = sc->ecam; return (0); } if (bootverbose) device_printf(dev, "ERROR: Unknown index %d.\n", index); return (ENOENT); } static int generic_pcie_write_ivar(device_t dev, device_t child, int index, uintptr_t value) { return (ENOENT); } static struct rman * generic_pcie_rman(struct generic_pcie_softc *sc, int type) { switch (type) { case SYS_RES_IOPORT: return (&sc->io_rman); case SYS_RES_MEMORY: return (&sc->mem_rman); default: break; } return (NULL); } static int generic_pcie_release_resource_pcie(device_t dev, device_t child, int type, int rid, struct resource *res) { struct generic_pcie_softc *sc; struct rman *rm; sc = device_get_softc(dev); rm = generic_pcie_rman(sc, type); if (rm != NULL) { KASSERT(rman_is_region_manager(res, rm), ("rman mismatch")); rman_release_resource(res); } return (bus_generic_release_resource(dev, child, type, rid, res)); } static int generic_pcie_release_resource(device_t dev, device_t child, int type, int rid, struct resource *res) { /* For PCIe devices that do not have FDT nodes, use PCIB method */ if ((int)ofw_bus_get_node(child) <= 0) { return (generic_pcie_release_resource_pcie(dev, child, type, rid, res)); } /* For other devices use OFW method */ return (generic_pcie_release_resource_ofw(dev, child, type, rid, res)); } struct resource * pci_host_generic_alloc_resource(device_t dev, device_t child, int type, int *rid, rman_res_t start, rman_res_t end, rman_res_t count, u_int flags) { /* For PCIe devices that do not have FDT nodes, use PCIB method */ if ((int)ofw_bus_get_node(child) <= 0) return (generic_pcie_alloc_resource_pcie(dev, child, type, rid, start, end, count, flags)); /* For other devices use OFW method */ return (generic_pcie_alloc_resource_ofw(dev, child, type, rid, start, end, count, flags)); } static struct resource * generic_pcie_alloc_resource_pcie(device_t dev, device_t child, int type, int *rid, rman_res_t start, rman_res_t end, rman_res_t count, u_int flags) { struct generic_pcie_softc *sc; struct resource *res; struct rman *rm; sc = device_get_softc(dev); rm = generic_pcie_rman(sc, type); if (rm == NULL) return (BUS_ALLOC_RESOURCE(device_get_parent(dev), dev, type, rid, start, end, count, flags)); if (bootverbose) { device_printf(dev, "rman_reserve_resource: start=%#jx, end=%#jx, count=%#jx\n", start, end, count); } res = rman_reserve_resource(rm, start, end, count, flags, child); if (res == NULL) goto fail; rman_set_rid(res, *rid); if (flags & RF_ACTIVE) if (bus_activate_resource(child, type, *rid, res)) { rman_release_resource(res); goto fail; } return (res); fail: device_printf(dev, "%s FAIL: type=%d, rid=%d, " "start=%016jx, end=%016jx, count=%016jx, flags=%x\n", __func__, type, *rid, start, end, count, flags); return (NULL); } static int generic_pcie_adjust_resource(device_t dev, device_t child, int type, struct resource *res, rman_res_t start, rman_res_t end) { struct generic_pcie_softc *sc; struct rman *rm; sc = device_get_softc(dev); rm = generic_pcie_rman(sc, type); if (rm != NULL) return (rman_adjust_resource(res, start, end)); return (bus_generic_adjust_resource(dev, child, type, res, start, end)); } static int generic_pcie_activate_resource(device_t dev, device_t child, int type, int rid, struct resource *r) { struct generic_pcie_softc *sc; uint64_t phys_base; uint64_t pci_base; uint64_t size; int found; int res; int i; sc = device_get_softc(dev); if ((res = rman_activate_resource(r)) != 0) return (res); switch(type) { case SYS_RES_IOPORT: found = 0; for (i = 0; i < MAX_RANGES_TUPLES; i++) { pci_base = sc->ranges[i].pci_base; phys_base = sc->ranges[i].phys_base; size = sc->ranges[i].size; if ((rid > pci_base) && (rid < (pci_base + size))) { found = 1; break; } } if (found) { rman_set_start(r, rman_get_start(r) + phys_base); rman_set_end(r, rman_get_end(r) + phys_base); BUS_ACTIVATE_RESOURCE(device_get_parent(dev), child, type, rid, r); } else { device_printf(dev, "Failed to activate IOPORT resource\n"); res = 0; } break; case SYS_RES_MEMORY: BUS_ACTIVATE_RESOURCE(device_get_parent(dev), child, type, rid, r); break; default: break; } return (res); } static int generic_pcie_deactivate_resource(device_t dev, device_t child, int type, int rid, struct resource *r) { struct generic_pcie_softc *sc; vm_offset_t vaddr; int res; sc = device_get_softc(dev); if ((res = rman_deactivate_resource(r)) != 0) return (res); switch(type) { case SYS_RES_IOPORT: case SYS_RES_MEMORY: vaddr = (vm_offset_t)rman_get_virtual(r); pmap_unmapdev(vaddr, rman_get_size(r)); break; default: break; } return (res); } static int generic_pcie_alloc_msi(device_t pci, device_t child, int count, int maxcount, int *irqs) { +#if defined(INTRNG) + struct generic_pcie_softc *sc; -#if defined(__aarch64__) + sc = device_get_softc(pci); + return (intr_alloc_msi(pci, child, sc->msi_parent, count, maxcount, + irqs)); +#elif defined(__aarch64__) return (arm_alloc_msi(pci, child, count, maxcount, irqs)); #else return (ENXIO); #endif } static int generic_pcie_release_msi(device_t pci, device_t child, int count, int *irqs) { +#if defined(INTRNG) + struct generic_pcie_softc *sc; -#if defined(__aarch64__) + sc = device_get_softc(pci); + return (intr_release_msi(pci, child, sc->msi_parent, count, irqs)); +#elif defined(__aarch64__) return (arm_release_msi(pci, child, count, irqs)); #else return (ENXIO); #endif } static int generic_pcie_map_msi(device_t pci, device_t child, int irq, uint64_t *addr, uint32_t *data) { +#if defined(INTRNG) + struct generic_pcie_softc *sc; -#if defined(__aarch64__) + sc = device_get_softc(pci); + return (intr_map_msi(pci, child, sc->msi_parent, irq, addr, data)); +#elif defined(__aarch64__) return (arm_map_msi(pci, child, irq, addr, data)); #else return (ENXIO); #endif } static int generic_pcie_alloc_msix(device_t pci, device_t child, int *irq) { +#if defined(INTRNG) + struct generic_pcie_softc *sc; -#if defined(__aarch64__) + sc = device_get_softc(pci); + return (intr_alloc_msix(pci, child, sc->msi_parent, irq)); +#elif defined(__aarch64__) return (arm_alloc_msix(pci, child, irq)); #else return (ENXIO); #endif } static int generic_pcie_release_msix(device_t pci, device_t child, int irq) { +#if defined(INTRNG) + struct generic_pcie_softc *sc; -#if defined(__aarch64__) + sc = device_get_softc(pci); + return (intr_release_msix(pci, child, sc->msi_parent, irq)); +#elif defined(__aarch64__) return (arm_release_msix(pci, child, irq)); #else return (ENXIO); #endif } static device_method_t generic_pcie_methods[] = { DEVMETHOD(device_probe, generic_pcie_probe), DEVMETHOD(device_attach, pci_host_generic_attach), DEVMETHOD(bus_read_ivar, generic_pcie_read_ivar), DEVMETHOD(bus_write_ivar, generic_pcie_write_ivar), DEVMETHOD(bus_alloc_resource, pci_host_generic_alloc_resource), DEVMETHOD(bus_adjust_resource, generic_pcie_adjust_resource), DEVMETHOD(bus_release_resource, generic_pcie_release_resource), DEVMETHOD(bus_activate_resource, generic_pcie_activate_resource), DEVMETHOD(bus_deactivate_resource, generic_pcie_deactivate_resource), DEVMETHOD(bus_setup_intr, bus_generic_setup_intr), DEVMETHOD(bus_teardown_intr, bus_generic_teardown_intr), /* pcib interface */ DEVMETHOD(pcib_maxslots, generic_pcie_maxslots), DEVMETHOD(pcib_route_interrupt, generic_pcie_route_interrupt), DEVMETHOD(pcib_read_config, generic_pcie_read_config), DEVMETHOD(pcib_write_config, generic_pcie_write_config), DEVMETHOD(pcib_alloc_msi, generic_pcie_alloc_msi), DEVMETHOD(pcib_release_msi, generic_pcie_release_msi), DEVMETHOD(pcib_alloc_msix, generic_pcie_alloc_msix), DEVMETHOD(pcib_release_msix, generic_pcie_release_msix), DEVMETHOD(pcib_map_msi, generic_pcie_map_msi), /* ofw_bus interface */ DEVMETHOD(ofw_bus_get_devinfo, generic_pcie_ofw_get_devinfo), DEVMETHOD(ofw_bus_get_compat, ofw_bus_gen_get_compat), DEVMETHOD(ofw_bus_get_model, ofw_bus_gen_get_model), DEVMETHOD(ofw_bus_get_name, ofw_bus_gen_get_name), DEVMETHOD(ofw_bus_get_node, ofw_bus_gen_get_node), DEVMETHOD(ofw_bus_get_type, ofw_bus_gen_get_type), DEVMETHOD_END }; static const struct ofw_bus_devinfo * generic_pcie_ofw_get_devinfo(device_t bus __unused, device_t child) { struct generic_pcie_ofw_devinfo *di; di = device_get_ivars(child); return (&di->di_dinfo); } static struct resource * generic_pcie_alloc_resource_ofw(device_t bus, device_t child, int type, int *rid, rman_res_t start, rman_res_t end, rman_res_t count, u_int flags) { struct generic_pcie_softc *sc; struct generic_pcie_ofw_devinfo *di; struct resource_list_entry *rle; int i; sc = device_get_softc(bus); if (RMAN_IS_DEFAULT_RANGE(start, end)) { if ((di = device_get_ivars(child)) == NULL) return (NULL); if (type == SYS_RES_IOPORT) type = SYS_RES_MEMORY; /* Find defaults for this rid */ rle = resource_list_find(&di->di_rl, type, *rid); if (rle == NULL) return (NULL); start = rle->start; end = rle->end; count = rle->count; } if (type == SYS_RES_MEMORY) { /* Remap through ranges property */ for (i = 0; i < MAX_RANGES_TUPLES; i++) { if (start >= sc->ranges[i].phys_base && end < sc->ranges[i].pci_base + sc->ranges[i].size) { start -= sc->ranges[i].phys_base; start += sc->ranges[i].pci_base; end -= sc->ranges[i].phys_base; end += sc->ranges[i].pci_base; break; } } if (i == MAX_RANGES_TUPLES) { device_printf(bus, "Could not map resource " "%#jx-%#jx\n", start, end); return (NULL); } } return (bus_generic_alloc_resource(bus, child, type, rid, start, end, count, flags)); } static int generic_pcie_release_resource_ofw(device_t bus, device_t child, int type, int rid, struct resource *res) { return (bus_generic_release_resource(bus, child, type, rid, res)); } /* Helper functions */ static int generic_pcie_ofw_bus_attach(device_t dev) { struct generic_pcie_ofw_devinfo *di; device_t child; phandle_t parent, node; pcell_t addr_cells, size_cells; parent = ofw_bus_get_node(dev); if (parent > 0) { get_addr_size_cells(parent, &addr_cells, &size_cells); /* Iterate through all bus subordinates */ for (node = OF_child(parent); node > 0; node = OF_peer(node)) { /* Allocate and populate devinfo. */ di = malloc(sizeof(*di), M_DEVBUF, M_WAITOK | M_ZERO); if (ofw_bus_gen_setup_devinfo(&di->di_dinfo, node) != 0) { free(di, M_DEVBUF); continue; } /* Initialize and populate resource list. */ resource_list_init(&di->di_rl); ofw_bus_reg_to_rl(dev, node, addr_cells, size_cells, &di->di_rl); ofw_bus_intr_to_rl(dev, node, &di->di_rl, NULL); /* Add newbus device for this FDT node */ child = device_add_child(dev, NULL, -1); if (child == NULL) { resource_list_free(&di->di_rl); ofw_bus_gen_destroy_devinfo(&di->di_dinfo); free(di, M_DEVBUF); continue; } device_set_ivars(child, di); } } return (0); } DEFINE_CLASS_0(pcib, generic_pcie_driver, generic_pcie_methods, sizeof(struct generic_pcie_softc)); devclass_t generic_pcie_devclass; DRIVER_MODULE(pcib, simplebus, generic_pcie_driver, generic_pcie_devclass, 0, 0); DRIVER_MODULE(pcib, ofwbus, generic_pcie_driver, generic_pcie_devclass, 0, 0); Index: head/sys/dev/pci/pci_host_generic.h =================================================================== --- head/sys/dev/pci/pci_host_generic.h (revision 299927) +++ head/sys/dev/pci/pci_host_generic.h (revision 299928) @@ -1,73 +1,74 @@ /* * Copyright (c) 2015 Ruslan Bukin * Copyright (c) 2015 The FreeBSD Foundation * All rights reserved. * * This software was developed by Semihalf. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * * $FreeBSD$ * */ #ifndef __PCI_HOST_GENERIC_H_ #define __PCI_HOST_GENERIC_H_ #define MAX_RANGES_TUPLES 16 #define MIN_RANGES_TUPLES 2 struct pcie_range { uint64_t pci_base; uint64_t phys_base; uint64_t size; uint64_t flags; #define FLAG_IO (1 << 0) #define FLAG_MEM (1 << 1) }; struct generic_pcie_softc { struct pcie_range ranges[MAX_RANGES_TUPLES]; int nranges; struct rman mem_rman; struct rman io_rman; struct resource *res; struct resource *res1; int ecam; bus_space_tag_t bst; bus_space_handle_t bsh; device_t dev; bus_space_handle_t ioh; #ifdef FDT struct ofw_bus_iinfo pci_iinfo; + phandle_t msi_parent; #endif }; extern devclass_t generic_pcie_devclass; DECLARE_CLASS(generic_pcie_driver); struct resource *pci_host_generic_alloc_resource(device_t, device_t, int, int *, rman_res_t, rman_res_t, rman_res_t, u_int); int pci_host_generic_attach(device_t); #endif /* __PCI_HOST_GENERIC_H_ */ Index: head/sys/kern/msi_if.m =================================================================== --- head/sys/kern/msi_if.m (nonexistent) +++ head/sys/kern/msi_if.m (revision 299928) @@ -0,0 +1,74 @@ +#- +# Copyright (c) 2016 The FreeBSD Foundation +# All rights reserved. +# +# This software was developed by Andrew Turner under +# sponsorship from the FreeBSD Foundation. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +# SUCH DAMAGE. +# +# $FreeBSD$ +# + +INTERFACE msi; + +HEADER { + struct intr_irqsrc; +}; + +METHOD int alloc_msi { + device_t dev; + device_t child; + int count; + int maxcount; + device_t *pic; + struct intr_irqsrc **srcs; +}; + +METHOD int release_msi { + device_t dev; + device_t child; + int count; + struct intr_irqsrc **srcs; +}; + +METHOD int alloc_msix { + device_t dev; + device_t child; + device_t *pic; + struct intr_irqsrc **src; +}; + +METHOD int release_msix { + device_t dev; + device_t child; + struct intr_irqsrc *src; +}; + +METHOD int map_msi { + device_t dev; + device_t child; + struct intr_irqsrc *src; + uint64_t *addr; + uint32_t *data; +}; + Property changes on: head/sys/kern/msi_if.m ___________________________________________________________________ Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Index: head/sys/kern/subr_intr.c =================================================================== --- head/sys/kern/subr_intr.c (revision 299927) +++ head/sys/kern/subr_intr.c (revision 299928) @@ -1,1307 +1,1479 @@ /*- * Copyright (c) 2015-2016 Svatopluk Kraus * Copyright (c) 2015-2016 Michal Meloun * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); /* * New-style Interrupt Framework * * TODO: - to support IPI (PPI) enabling on other CPUs if already started * - to complete things for removable PICs */ #include "opt_acpi.h" #include "opt_ddb.h" #include "opt_platform.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #ifdef FDT #include #include #include #endif #ifdef DDB #include #endif #include "pic_if.h" +#include "msi_if.h" #define INTRNAME_LEN (2*MAXCOMLEN + 1) #ifdef DEBUG #define debugf(fmt, args...) do { printf("%s(): ", __func__); \ printf(fmt,##args); } while (0) #else #define debugf(fmt, args...) #endif MALLOC_DECLARE(M_INTRNG); MALLOC_DEFINE(M_INTRNG, "intr", "intr interrupt handling"); /* Main interrupt handler called from assembler -> 'hidden' for C code. */ void intr_irq_handler(struct trapframe *tf); /* Root interrupt controller stuff. */ device_t intr_irq_root_dev; static intr_irq_filter_t *irq_root_filter; static void *irq_root_arg; static u_int irq_root_ipicount; /* Interrupt controller definition. */ struct intr_pic { SLIST_ENTRY(intr_pic) pic_next; intptr_t pic_xref; /* hardware identification */ device_t pic_dev; +#define FLAG_PIC (1 << 0) +#define FLAG_MSI (1 << 1) + u_int pic_flags; }; static struct mtx pic_list_lock; static SLIST_HEAD(, intr_pic) pic_list; static struct intr_pic *pic_lookup(device_t dev, intptr_t xref); /* Interrupt source definition. */ static struct mtx isrc_table_lock; static struct intr_irqsrc *irq_sources[NIRQ]; u_int irq_next_free; /* * XXX - All stuff around struct intr_dev_data is considered as temporary * until better place for storing struct intr_map_data will be find. * * For now, there are two global interrupt numbers spaces: * <0, NIRQ) ... interrupts without config data * managed in irq_sources[] * IRQ_DDATA_BASE + <0, 2 * NIRQ) ... interrupts with config data * managed in intr_ddata_tab[] * * Read intr_ddata_lookup() to see how these spaces are worked with. * Note that each interrupt number from second space duplicates some number * from first space at this moment. An interrupt number from first space can * be duplicated even multiple times in second space. */ struct intr_dev_data { device_t idd_dev; intptr_t idd_xref; u_int idd_irq; struct intr_map_data * idd_data; struct intr_irqsrc * idd_isrc; }; static struct intr_dev_data *intr_ddata_tab[2 * NIRQ]; static u_int intr_ddata_first_unused; #define IRQ_DDATA_BASE 10000 CTASSERT(IRQ_DDATA_BASE > nitems(irq_sources)); #ifdef SMP static boolean_t irq_assign_cpu = FALSE; #endif /* * - 2 counters for each I/O interrupt. * - MAXCPU counters for each IPI counters for SMP. */ #ifdef SMP #define INTRCNT_COUNT (NIRQ * 2 + INTR_IPI_COUNT * MAXCPU) #else #define INTRCNT_COUNT (NIRQ * 2) #endif /* Data for MI statistics reporting. */ u_long intrcnt[INTRCNT_COUNT]; char intrnames[INTRCNT_COUNT * INTRNAME_LEN]; size_t sintrcnt = sizeof(intrcnt); size_t sintrnames = sizeof(intrnames); static u_int intrcnt_index; /* * Interrupt framework initialization routine. */ static void intr_irq_init(void *dummy __unused) { SLIST_INIT(&pic_list); mtx_init(&pic_list_lock, "intr pic list", NULL, MTX_DEF); + mtx_init(&isrc_table_lock, "intr isrc table", NULL, MTX_DEF); } SYSINIT(intr_irq_init, SI_SUB_INTR, SI_ORDER_FIRST, intr_irq_init, NULL); static void intrcnt_setname(const char *name, int index) { snprintf(intrnames + INTRNAME_LEN * index, INTRNAME_LEN, "%-*s", INTRNAME_LEN - 1, name); } /* * Update name for interrupt source with interrupt event. */ static void intrcnt_updatename(struct intr_irqsrc *isrc) { /* QQQ: What about stray counter name? */ mtx_assert(&isrc_table_lock, MA_OWNED); intrcnt_setname(isrc->isrc_event->ie_fullname, isrc->isrc_index); } /* * Virtualization for interrupt source interrupt counter increment. */ static inline void isrc_increment_count(struct intr_irqsrc *isrc) { if (isrc->isrc_flags & INTR_ISRCF_PPI) atomic_add_long(&isrc->isrc_count[0], 1); else isrc->isrc_count[0]++; } /* * Virtualization for interrupt source interrupt stray counter increment. */ static inline void isrc_increment_straycount(struct intr_irqsrc *isrc) { isrc->isrc_count[1]++; } /* * Virtualization for interrupt source interrupt name update. */ static void isrc_update_name(struct intr_irqsrc *isrc, const char *name) { char str[INTRNAME_LEN]; mtx_assert(&isrc_table_lock, MA_OWNED); if (name != NULL) { snprintf(str, INTRNAME_LEN, "%s: %s", isrc->isrc_name, name); intrcnt_setname(str, isrc->isrc_index); snprintf(str, INTRNAME_LEN, "stray %s: %s", isrc->isrc_name, name); intrcnt_setname(str, isrc->isrc_index + 1); } else { snprintf(str, INTRNAME_LEN, "%s:", isrc->isrc_name); intrcnt_setname(str, isrc->isrc_index); snprintf(str, INTRNAME_LEN, "stray %s:", isrc->isrc_name); intrcnt_setname(str, isrc->isrc_index + 1); } } /* * Virtualization for interrupt source interrupt counters setup. */ static void isrc_setup_counters(struct intr_irqsrc *isrc) { u_int index; /* * XXX - it does not work well with removable controllers and * interrupt sources !!! */ index = atomic_fetchadd_int(&intrcnt_index, 2); isrc->isrc_index = index; isrc->isrc_count = &intrcnt[index]; isrc_update_name(isrc, NULL); } /* * Virtualization for interrupt source interrupt counters release. */ static void isrc_release_counters(struct intr_irqsrc *isrc) { panic("%s: not implemented", __func__); } #ifdef SMP /* * Virtualization for interrupt source IPI counters setup. */ u_long * intr_ipi_setup_counters(const char *name) { u_int index, i; char str[INTRNAME_LEN]; index = atomic_fetchadd_int(&intrcnt_index, MAXCPU); for (i = 0; i < MAXCPU; i++) { snprintf(str, INTRNAME_LEN, "cpu%d:%s", i, name); intrcnt_setname(str, index + i); } return (&intrcnt[index]); } #endif /* * Main interrupt dispatch handler. It's called straight * from the assembler, where CPU interrupt is served. */ void intr_irq_handler(struct trapframe *tf) { struct trapframe * oldframe; struct thread * td; KASSERT(irq_root_filter != NULL, ("%s: no filter", __func__)); PCPU_INC(cnt.v_intr); critical_enter(); td = curthread; oldframe = td->td_intr_frame; td->td_intr_frame = tf; irq_root_filter(irq_root_arg); td->td_intr_frame = oldframe; critical_exit(); } /* * interrupt controller dispatch function for interrupts. It should * be called straight from the interrupt controller, when associated interrupt * source is learned. */ int intr_isrc_dispatch(struct intr_irqsrc *isrc, struct trapframe *tf) { KASSERT(isrc != NULL, ("%s: no source", __func__)); isrc_increment_count(isrc); #ifdef INTR_SOLO if (isrc->isrc_filter != NULL) { int error; error = isrc->isrc_filter(isrc->isrc_arg, tf); PIC_POST_FILTER(isrc->isrc_dev, isrc); if (error == FILTER_HANDLED) return (0); } else #endif if (isrc->isrc_event != NULL) { if (intr_event_handle(isrc->isrc_event, tf) == 0) return (0); } isrc_increment_straycount(isrc); return (EINVAL); } /* * Alloc unique interrupt number (resource handle) for interrupt source. * * There could be various strategies how to allocate free interrupt number * (resource handle) for new interrupt source. * * 1. Handles are always allocated forward, so handles are not recycled * immediately. However, if only one free handle left which is reused * constantly... */ static inline int isrc_alloc_irq(struct intr_irqsrc *isrc) { u_int maxirqs, irq; mtx_assert(&isrc_table_lock, MA_OWNED); maxirqs = nitems(irq_sources); if (irq_next_free >= maxirqs) return (ENOSPC); for (irq = irq_next_free; irq < maxirqs; irq++) { if (irq_sources[irq] == NULL) goto found; } for (irq = 0; irq < irq_next_free; irq++) { if (irq_sources[irq] == NULL) goto found; } irq_next_free = maxirqs; return (ENOSPC); found: isrc->isrc_irq = irq; irq_sources[irq] = isrc; irq_next_free = irq + 1; if (irq_next_free >= maxirqs) irq_next_free = 0; return (0); } /* * Free unique interrupt number (resource handle) from interrupt source. */ static inline int isrc_free_irq(struct intr_irqsrc *isrc) { mtx_assert(&isrc_table_lock, MA_OWNED); if (isrc->isrc_irq >= nitems(irq_sources)) return (EINVAL); if (irq_sources[isrc->isrc_irq] != isrc) return (EINVAL); irq_sources[isrc->isrc_irq] = NULL; isrc->isrc_irq = INTR_IRQ_INVALID; /* just to be safe */ return (0); } /* * Lookup interrupt source by interrupt number (resource handle). */ static inline struct intr_irqsrc * isrc_lookup(u_int irq) { if (irq < nitems(irq_sources)) return (irq_sources[irq]); return (NULL); } /* * Initialize interrupt source and register it into global interrupt table. */ int intr_isrc_register(struct intr_irqsrc *isrc, device_t dev, u_int flags, const char *fmt, ...) { int error; va_list ap; bzero(isrc, sizeof(struct intr_irqsrc)); isrc->isrc_dev = dev; isrc->isrc_irq = INTR_IRQ_INVALID; /* just to be safe */ isrc->isrc_flags = flags; va_start(ap, fmt); vsnprintf(isrc->isrc_name, INTR_ISRC_NAMELEN, fmt, ap); va_end(ap); mtx_lock(&isrc_table_lock); error = isrc_alloc_irq(isrc); if (error != 0) { mtx_unlock(&isrc_table_lock); return (error); } /* * Setup interrupt counters, but not for IPI sources. Those are setup * later and only for used ones (up to INTR_IPI_COUNT) to not exhaust * our counter pool. */ if ((isrc->isrc_flags & INTR_ISRCF_IPI) == 0) isrc_setup_counters(isrc); mtx_unlock(&isrc_table_lock); return (0); } /* * Deregister interrupt source from global interrupt table. */ int intr_isrc_deregister(struct intr_irqsrc *isrc) { int error; mtx_lock(&isrc_table_lock); if ((isrc->isrc_flags & INTR_ISRCF_IPI) == 0) isrc_release_counters(isrc); error = isrc_free_irq(isrc); mtx_unlock(&isrc_table_lock); return (error); } #ifdef SMP /* * A support function for a PIC to decide if provided ISRC should be inited * on given cpu. The logic of INTR_ISRCF_BOUND flag and isrc_cpu member of * struct intr_irqsrc is the following: * * If INTR_ISRCF_BOUND is set, the ISRC should be inited only on cpus * set in isrc_cpu. If not, the ISRC should be inited on every cpu and * isrc_cpu is kept consistent with it. Thus isrc_cpu is always correct. */ bool intr_isrc_init_on_cpu(struct intr_irqsrc *isrc, u_int cpu) { if (isrc->isrc_handlers == 0) return (false); if ((isrc->isrc_flags & (INTR_ISRCF_PPI | INTR_ISRCF_IPI)) == 0) return (false); if (isrc->isrc_flags & INTR_ISRCF_BOUND) return (CPU_ISSET(cpu, &isrc->isrc_cpu)); CPU_SET(cpu, &isrc->isrc_cpu); return (true); } #endif static struct intr_dev_data * intr_ddata_alloc(u_int extsize) { struct intr_dev_data *ddata; size_t size; size = sizeof(*ddata); ddata = malloc(size + extsize, M_INTRNG, M_WAITOK | M_ZERO); mtx_lock(&isrc_table_lock); if (intr_ddata_first_unused >= nitems(intr_ddata_tab)) { mtx_unlock(&isrc_table_lock); free(ddata, M_INTRNG); return (NULL); } intr_ddata_tab[intr_ddata_first_unused] = ddata; ddata->idd_irq = IRQ_DDATA_BASE + intr_ddata_first_unused++; mtx_unlock(&isrc_table_lock); ddata->idd_data = (struct intr_map_data *)((uintptr_t)ddata + size); ddata->idd_data->size = extsize; return (ddata); } static struct intr_irqsrc * intr_ddata_lookup(u_int irq, struct intr_map_data **datap) { int error; struct intr_irqsrc *isrc; struct intr_dev_data *ddata; isrc = isrc_lookup(irq); if (isrc != NULL) { if (datap != NULL) *datap = NULL; return (isrc); } if (irq < IRQ_DDATA_BASE) return (NULL); irq -= IRQ_DDATA_BASE; if (irq >= nitems(intr_ddata_tab)) return (NULL); ddata = intr_ddata_tab[irq]; if (ddata->idd_isrc == NULL) { error = intr_map_irq(ddata->idd_dev, ddata->idd_xref, ddata->idd_data, &irq); if (error != 0) return (NULL); ddata->idd_isrc = isrc_lookup(irq); } if (datap != NULL) *datap = ddata->idd_data; return (ddata->idd_isrc); } #ifdef DEV_ACPI /* * Map interrupt source according to ACPI info into framework. If such mapping * does not exist, create it. Return unique interrupt number (resource handle) * associated with mapped interrupt source. */ u_int intr_acpi_map_irq(device_t dev, u_int irq, enum intr_polarity pol, enum intr_trigger trig) { struct intr_map_data_acpi *daa; struct intr_dev_data *ddata; ddata = intr_ddata_alloc(sizeof(struct intr_map_data_acpi)); if (ddata == NULL) return (INTR_IRQ_INVALID); /* no space left */ ddata->idd_dev = dev; ddata->idd_data->type = INTR_MAP_DATA_ACPI; daa = (struct intr_map_data_acpi *)ddata->idd_data; daa->irq = irq; daa->pol = pol; daa->trig = trig; return (ddata->idd_irq); } #endif #ifdef FDT /* * Map interrupt source according to FDT data into framework. If such mapping * does not exist, create it. Return unique interrupt number (resource handle) * associated with mapped interrupt source. */ u_int intr_fdt_map_irq(phandle_t node, pcell_t *cells, u_int ncells) { size_t cellsize; struct intr_dev_data *ddata; struct intr_map_data_fdt *daf; cellsize = ncells * sizeof(*cells); ddata = intr_ddata_alloc(sizeof(struct intr_map_data_fdt) + cellsize); if (ddata == NULL) return (INTR_IRQ_INVALID); /* no space left */ ddata->idd_xref = (intptr_t)node; ddata->idd_data->type = INTR_MAP_DATA_FDT; daf = (struct intr_map_data_fdt *)ddata->idd_data; daf->ncells = ncells; memcpy(daf->cells, cells, cellsize); return (ddata->idd_irq); } #endif /* * Store GPIO interrupt decription in framework and return unique interrupt * number (resource handle) associated with it. */ u_int intr_gpio_map_irq(device_t dev, u_int pin_num, u_int pin_flags, u_int intr_mode) { struct intr_dev_data *ddata; struct intr_map_data_gpio *dag; ddata = intr_ddata_alloc(sizeof(struct intr_map_data_gpio)); if (ddata == NULL) return (INTR_IRQ_INVALID); /* no space left */ ddata->idd_dev = dev; ddata->idd_data->type = INTR_MAP_DATA_GPIO; dag = (struct intr_map_data_gpio *)ddata->idd_data; dag->gpio_pin_num = pin_num; dag->gpio_pin_flags = pin_flags; dag->gpio_intr_mode = intr_mode; return (ddata->idd_irq); } #ifdef INTR_SOLO /* * Setup filter into interrupt source. */ static int iscr_setup_filter(struct intr_irqsrc *isrc, const char *name, intr_irq_filter_t *filter, void *arg, void **cookiep) { if (filter == NULL) return (EINVAL); mtx_lock(&isrc_table_lock); /* * Make sure that we do not mix the two ways * how we handle interrupt sources. */ if (isrc->isrc_filter != NULL || isrc->isrc_event != NULL) { mtx_unlock(&isrc_table_lock); return (EBUSY); } isrc->isrc_filter = filter; isrc->isrc_arg = arg; isrc_update_name(isrc, name); mtx_unlock(&isrc_table_lock); *cookiep = isrc; return (0); } #endif /* * Interrupt source pre_ithread method for MI interrupt framework. */ static void intr_isrc_pre_ithread(void *arg) { struct intr_irqsrc *isrc = arg; PIC_PRE_ITHREAD(isrc->isrc_dev, isrc); } /* * Interrupt source post_ithread method for MI interrupt framework. */ static void intr_isrc_post_ithread(void *arg) { struct intr_irqsrc *isrc = arg; PIC_POST_ITHREAD(isrc->isrc_dev, isrc); } /* * Interrupt source post_filter method for MI interrupt framework. */ static void intr_isrc_post_filter(void *arg) { struct intr_irqsrc *isrc = arg; PIC_POST_FILTER(isrc->isrc_dev, isrc); } /* * Interrupt source assign_cpu method for MI interrupt framework. */ static int intr_isrc_assign_cpu(void *arg, int cpu) { #ifdef SMP struct intr_irqsrc *isrc = arg; int error; if (isrc->isrc_dev != intr_irq_root_dev) return (EINVAL); mtx_lock(&isrc_table_lock); if (cpu == NOCPU) { CPU_ZERO(&isrc->isrc_cpu); isrc->isrc_flags &= ~INTR_ISRCF_BOUND; } else { CPU_SETOF(cpu, &isrc->isrc_cpu); isrc->isrc_flags |= INTR_ISRCF_BOUND; } /* * In NOCPU case, it's up to PIC to either leave ISRC on same CPU or * re-balance it to another CPU or enable it on more CPUs. However, * PIC is expected to change isrc_cpu appropriately to keep us well * informed if the call is successful. */ if (irq_assign_cpu) { error = PIC_BIND_INTR(isrc->isrc_dev, isrc); if (error) { CPU_ZERO(&isrc->isrc_cpu); mtx_unlock(&isrc_table_lock); return (error); } } mtx_unlock(&isrc_table_lock); return (0); #else return (EOPNOTSUPP); #endif } /* * Create interrupt event for interrupt source. */ static int isrc_event_create(struct intr_irqsrc *isrc) { struct intr_event *ie; int error; error = intr_event_create(&ie, isrc, 0, isrc->isrc_irq, intr_isrc_pre_ithread, intr_isrc_post_ithread, intr_isrc_post_filter, intr_isrc_assign_cpu, "%s:", isrc->isrc_name); if (error) return (error); mtx_lock(&isrc_table_lock); /* * Make sure that we do not mix the two ways * how we handle interrupt sources. Let contested event wins. */ #ifdef INTR_SOLO if (isrc->isrc_filter != NULL || isrc->isrc_event != NULL) { #else if (isrc->isrc_event != NULL) { #endif mtx_unlock(&isrc_table_lock); intr_event_destroy(ie); return (isrc->isrc_event != NULL ? EBUSY : 0); } isrc->isrc_event = ie; mtx_unlock(&isrc_table_lock); return (0); } #ifdef notyet /* * Destroy interrupt event for interrupt source. */ static void isrc_event_destroy(struct intr_irqsrc *isrc) { struct intr_event *ie; mtx_lock(&isrc_table_lock); ie = isrc->isrc_event; isrc->isrc_event = NULL; mtx_unlock(&isrc_table_lock); if (ie != NULL) intr_event_destroy(ie); } #endif /* * Add handler to interrupt source. */ static int isrc_add_handler(struct intr_irqsrc *isrc, const char *name, driver_filter_t filter, driver_intr_t handler, void *arg, enum intr_type flags, void **cookiep) { int error; if (isrc->isrc_event == NULL) { error = isrc_event_create(isrc); if (error) return (error); } error = intr_event_add_handler(isrc->isrc_event, name, filter, handler, arg, intr_priority(flags), flags, cookiep); if (error == 0) { mtx_lock(&isrc_table_lock); intrcnt_updatename(isrc); mtx_unlock(&isrc_table_lock); } return (error); } /* * Lookup interrupt controller locked. */ static inline struct intr_pic * pic_lookup_locked(device_t dev, intptr_t xref) { struct intr_pic *pic; mtx_assert(&pic_list_lock, MA_OWNED); if (dev == NULL && xref == 0) return (NULL); /* Note that pic->pic_dev is never NULL on registered PIC. */ SLIST_FOREACH(pic, &pic_list, pic_next) { if (dev == NULL) { if (xref == pic->pic_xref) return (pic); } else if (xref == 0 || pic->pic_xref == 0) { if (dev == pic->pic_dev) return (pic); } else if (xref == pic->pic_xref && dev == pic->pic_dev) return (pic); } return (NULL); } /* * Lookup interrupt controller. */ static struct intr_pic * pic_lookup(device_t dev, intptr_t xref) { struct intr_pic *pic; mtx_lock(&pic_list_lock); pic = pic_lookup_locked(dev, xref); mtx_unlock(&pic_list_lock); return (pic); } /* * Create interrupt controller. */ static struct intr_pic * pic_create(device_t dev, intptr_t xref) { struct intr_pic *pic; mtx_lock(&pic_list_lock); pic = pic_lookup_locked(dev, xref); if (pic != NULL) { mtx_unlock(&pic_list_lock); return (pic); } pic = malloc(sizeof(*pic), M_INTRNG, M_NOWAIT | M_ZERO); if (pic == NULL) { mtx_unlock(&pic_list_lock); return (NULL); } pic->pic_xref = xref; pic->pic_dev = dev; SLIST_INSERT_HEAD(&pic_list, pic, pic_next); mtx_unlock(&pic_list_lock); return (pic); } #ifdef notyet /* * Destroy interrupt controller. */ static void pic_destroy(device_t dev, intptr_t xref) { struct intr_pic *pic; mtx_lock(&pic_list_lock); pic = pic_lookup_locked(dev, xref); if (pic == NULL) { mtx_unlock(&pic_list_lock); return; } SLIST_REMOVE(&pic_list, pic, intr_pic, pic_next); mtx_unlock(&pic_list_lock); free(pic, M_INTRNG); } #endif /* * Register interrupt controller. */ int intr_pic_register(device_t dev, intptr_t xref) { struct intr_pic *pic; if (dev == NULL) return (EINVAL); pic = pic_create(dev, xref); if (pic == NULL) return (ENOMEM); + pic->pic_flags |= FLAG_PIC; + debugf("PIC %p registered for %s \n", pic, device_get_nameunit(dev), dev, xref); return (0); } /* * Unregister interrupt controller. */ int intr_pic_deregister(device_t dev, intptr_t xref) { panic("%s: not implemented", __func__); } /* * Mark interrupt controller (itself) as a root one. * * Note that only an interrupt controller can really know its position * in interrupt controller's tree. So root PIC must claim itself as a root. * * In FDT case, according to ePAPR approved version 1.1 from 08 April 2011, * page 30: * "The root of the interrupt tree is determined when traversal * of the interrupt tree reaches an interrupt controller node without * an interrupts property and thus no explicit interrupt parent." */ int intr_pic_claim_root(device_t dev, intptr_t xref, intr_irq_filter_t *filter, void *arg, u_int ipicount) { + struct intr_pic *pic; - if (pic_lookup(dev, xref) == NULL) { + pic = pic_lookup(dev, xref); + if (pic == NULL) { device_printf(dev, "not registered\n"); return (EINVAL); } + + KASSERT((pic->pic_flags & FLAG_PIC) != 0, + ("%s: Found a non-PIC controller: %s", __func__, + device_get_name(pic->pic_dev))); + if (filter == NULL) { device_printf(dev, "filter missing\n"); return (EINVAL); } /* * Only one interrupt controllers could be on the root for now. * Note that we further suppose that there is not threaded interrupt * routine (handler) on the root. See intr_irq_handler(). */ if (intr_irq_root_dev != NULL) { device_printf(dev, "another root already set\n"); return (EBUSY); } intr_irq_root_dev = dev; irq_root_filter = filter; irq_root_arg = arg; irq_root_ipicount = ipicount; debugf("irq root set to %s\n", device_get_nameunit(dev)); return (0); } int intr_map_irq(device_t dev, intptr_t xref, struct intr_map_data *data, u_int *irqp) { int error; struct intr_irqsrc *isrc; struct intr_pic *pic; if (data == NULL) return (EINVAL); pic = pic_lookup(dev, xref); if (pic == NULL) return (ESRCH); + KASSERT((pic->pic_flags & FLAG_PIC) != 0, + ("%s: Found a non-PIC controller: %s", __func__, + device_get_name(pic->pic_dev))); + error = PIC_MAP_INTR(pic->pic_dev, data, &isrc); if (error == 0) *irqp = isrc->isrc_irq; return (error); } int intr_alloc_irq(device_t dev, struct resource *res) { struct intr_map_data *data; struct intr_irqsrc *isrc; KASSERT(rman_get_start(res) == rman_get_end(res), ("%s: more interrupts in resource", __func__)); isrc = intr_ddata_lookup(rman_get_start(res), &data); if (isrc == NULL) return (EINVAL); return (PIC_ALLOC_INTR(isrc->isrc_dev, isrc, res, data)); } int intr_release_irq(device_t dev, struct resource *res) { struct intr_map_data *data; struct intr_irqsrc *isrc; KASSERT(rman_get_start(res) == rman_get_end(res), ("%s: more interrupts in resource", __func__)); isrc = intr_ddata_lookup(rman_get_start(res), &data); if (isrc == NULL) return (EINVAL); return (PIC_RELEASE_INTR(isrc->isrc_dev, isrc, res, data)); } int intr_setup_irq(device_t dev, struct resource *res, driver_filter_t filt, driver_intr_t hand, void *arg, int flags, void **cookiep) { int error; struct intr_map_data *data; struct intr_irqsrc *isrc; const char *name; KASSERT(rman_get_start(res) == rman_get_end(res), ("%s: more interrupts in resource", __func__)); isrc = intr_ddata_lookup(rman_get_start(res), &data); if (isrc == NULL) return (EINVAL); name = device_get_nameunit(dev); #ifdef INTR_SOLO /* * Standard handling is done through MI interrupt framework. However, * some interrupts could request solely own special handling. This * non standard handling can be used for interrupt controllers without * handler (filter only), so in case that interrupt controllers are * chained, MI interrupt framework is called only in leaf controller. * * Note that root interrupt controller routine is served as well, * however in intr_irq_handler(), i.e. main system dispatch routine. */ if (flags & INTR_SOLO && hand != NULL) { debugf("irq %u cannot solo on %s\n", irq, name); return (EINVAL); } if (flags & INTR_SOLO) { error = iscr_setup_filter(isrc, name, (intr_irq_filter_t *)filt, arg, cookiep); debugf("irq %u setup filter error %d on %s\n", irq, error, name); } else #endif { error = isrc_add_handler(isrc, name, filt, hand, arg, flags, cookiep); debugf("irq %u add handler error %d on %s\n", irq, error, name); } if (error != 0) return (error); mtx_lock(&isrc_table_lock); error = PIC_SETUP_INTR(isrc->isrc_dev, isrc, res, data); if (error == 0) { isrc->isrc_handlers++; if (isrc->isrc_handlers == 1) PIC_ENABLE_INTR(isrc->isrc_dev, isrc); } mtx_unlock(&isrc_table_lock); if (error != 0) intr_event_remove_handler(*cookiep); return (error); } int intr_teardown_irq(device_t dev, struct resource *res, void *cookie) { int error; struct intr_map_data *data; struct intr_irqsrc *isrc; KASSERT(rman_get_start(res) == rman_get_end(res), ("%s: more interrupts in resource", __func__)); isrc = intr_ddata_lookup(rman_get_start(res), &data); if (isrc == NULL || isrc->isrc_handlers == 0) return (EINVAL); #ifdef INTR_SOLO if (isrc->isrc_filter != NULL) { if (isrc != cookie) return (EINVAL); mtx_lock(&isrc_table_lock); isrc->isrc_filter = NULL; isrc->isrc_arg = NULL; isrc->isrc_handlers = 0; PIC_DISABLE_INTR(isrc->isrc_dev, isrc); PIC_TEARDOWN_INTR(isrc->isrc_dev, isrc, res, data); isrc_update_name(isrc, NULL); mtx_unlock(&isrc_table_lock); return (0); } #endif if (isrc != intr_handler_source(cookie)) return (EINVAL); error = intr_event_remove_handler(cookie); if (error == 0) { mtx_lock(&isrc_table_lock); isrc->isrc_handlers--; if (isrc->isrc_handlers == 0) PIC_DISABLE_INTR(isrc->isrc_dev, isrc); PIC_TEARDOWN_INTR(isrc->isrc_dev, isrc, res, data); intrcnt_updatename(isrc); mtx_unlock(&isrc_table_lock); } return (error); } int intr_describe_irq(device_t dev, struct resource *res, void *cookie, const char *descr) { int error; struct intr_irqsrc *isrc; KASSERT(rman_get_start(res) == rman_get_end(res), ("%s: more interrupts in resource", __func__)); isrc = intr_ddata_lookup(rman_get_start(res), NULL); if (isrc == NULL || isrc->isrc_handlers == 0) return (EINVAL); #ifdef INTR_SOLO if (isrc->isrc_filter != NULL) { if (isrc != cookie) return (EINVAL); mtx_lock(&isrc_table_lock); isrc_update_name(isrc, descr); mtx_unlock(&isrc_table_lock); return (0); } #endif error = intr_event_describe_handler(isrc->isrc_event, cookie, descr); if (error == 0) { mtx_lock(&isrc_table_lock); intrcnt_updatename(isrc); mtx_unlock(&isrc_table_lock); } return (error); } #ifdef SMP int intr_bind_irq(device_t dev, struct resource *res, int cpu) { struct intr_irqsrc *isrc; KASSERT(rman_get_start(res) == rman_get_end(res), ("%s: more interrupts in resource", __func__)); isrc = intr_ddata_lookup(rman_get_start(res), NULL); if (isrc == NULL || isrc->isrc_handlers == 0) return (EINVAL); #ifdef INTR_SOLO if (isrc->isrc_filter != NULL) return (intr_isrc_assign_cpu(isrc, cpu)); #endif return (intr_event_bind(isrc->isrc_event, cpu)); } /* * Return the CPU that the next interrupt source should use. * For now just returns the next CPU according to round-robin. */ u_int intr_irq_next_cpu(u_int last_cpu, cpuset_t *cpumask) { if (!irq_assign_cpu || mp_ncpus == 1) return (PCPU_GET(cpuid)); do { last_cpu++; if (last_cpu > mp_maxid) last_cpu = 0; } while (!CPU_ISSET(last_cpu, cpumask)); return (last_cpu); } /* * Distribute all the interrupt sources among the available * CPUs once the AP's have been launched. */ static void intr_irq_shuffle(void *arg __unused) { struct intr_irqsrc *isrc; u_int i; if (mp_ncpus == 1) return; mtx_lock(&isrc_table_lock); irq_assign_cpu = TRUE; for (i = 0; i < NIRQ; i++) { isrc = irq_sources[i]; if (isrc == NULL || isrc->isrc_handlers == 0 || isrc->isrc_flags & (INTR_ISRCF_PPI | INTR_ISRCF_IPI)) continue; if (isrc->isrc_event != NULL && isrc->isrc_flags & INTR_ISRCF_BOUND && isrc->isrc_event->ie_cpu != CPU_FFS(&isrc->isrc_cpu) - 1) panic("%s: CPU inconsistency", __func__); if ((isrc->isrc_flags & INTR_ISRCF_BOUND) == 0) CPU_ZERO(&isrc->isrc_cpu); /* start again */ /* * We are in wicked position here if the following call fails * for bound ISRC. The best thing we can do is to clear * isrc_cpu so inconsistency with ie_cpu will be detectable. */ if (PIC_BIND_INTR(isrc->isrc_dev, isrc) != 0) CPU_ZERO(&isrc->isrc_cpu); } mtx_unlock(&isrc_table_lock); } SYSINIT(intr_irq_shuffle, SI_SUB_SMP, SI_ORDER_SECOND, intr_irq_shuffle, NULL); #else u_int intr_irq_next_cpu(u_int current_cpu, cpuset_t *cpumask) { return (PCPU_GET(cpuid)); } #endif + +/* + * Register a MSI/MSI-X interrupt controller + */ +int +intr_msi_register(device_t dev, intptr_t xref) +{ + struct intr_pic *pic; + + if (dev == NULL) + return (EINVAL); + pic = pic_create(dev, xref); + if (pic == NULL) + return (ENOMEM); + + pic->pic_flags |= FLAG_MSI; + + debugf("PIC %p registered for %s \n", pic, + device_get_nameunit(dev), dev, (uintmax_t)xref); + return (0); +} + +int +intr_alloc_msi(device_t pci, device_t child, intptr_t xref, int count, + int maxcount, int *irqs) +{ + struct intr_irqsrc **isrc; + struct intr_pic *pic; + device_t pdev; + int err, i; + + pic = pic_lookup(NULL, xref); + if (pic == NULL) + return (ESRCH); + + KASSERT((pic->pic_flags & FLAG_MSI) != 0, + ("%s: Found a non-MSI controller: %s", __func__, + device_get_name(pic->pic_dev))); + + isrc = malloc(sizeof(*isrc) * count, M_INTRNG, M_WAITOK); + err = MSI_ALLOC_MSI(pic->pic_dev, child, count, maxcount, &pdev, isrc); + if (err == 0) { + for (i = 0; i < count; i++) { + irqs[i] = isrc[i]->isrc_irq; + } + } + + free(isrc, M_INTRNG); + + return (err); +} + +int +intr_release_msi(device_t pci, device_t child, intptr_t xref, int count, + int *irqs) +{ + struct intr_irqsrc **isrc; + struct intr_pic *pic; + int i, err; + + pic = pic_lookup(NULL, xref); + if (pic == NULL) + return (ESRCH); + + KASSERT((pic->pic_flags & FLAG_MSI) != 0, + ("%s: Found a non-MSI controller: %s", __func__, + device_get_name(pic->pic_dev))); + + isrc = malloc(sizeof(*isrc) * count, M_INTRNG, M_WAITOK); + + for (i = 0; i < count; i++) { + isrc[i] = isrc_lookup(irqs[i]); + if (isrc == NULL) { + free(isrc, M_INTRNG); + return (EINVAL); + } + } + + err = MSI_RELEASE_MSI(pic->pic_dev, child, count, isrc); + free(isrc, M_INTRNG); + return (err); +} + +int +intr_alloc_msix(device_t pci, device_t child, intptr_t xref, int *irq) +{ + struct intr_irqsrc *isrc; + struct intr_pic *pic; + device_t pdev; + int err; + + pic = pic_lookup(NULL, xref); + if (pic == NULL) + return (ESRCH); + + KASSERT((pic->pic_flags & FLAG_MSI) != 0, + ("%s: Found a non-MSI controller: %s", __func__, + device_get_name(pic->pic_dev))); + + err = MSI_ALLOC_MSIX(pic->pic_dev, child, &pdev, &isrc); + if (err != 0) + return (err); + + *irq = isrc->isrc_irq; + return (0); +} + +int +intr_release_msix(device_t pci, device_t child, intptr_t xref, int irq) +{ + struct intr_irqsrc *isrc; + struct intr_pic *pic; + int err; + + pic = pic_lookup(NULL, xref); + if (pic == NULL) + return (ESRCH); + + KASSERT((pic->pic_flags & FLAG_MSI) != 0, + ("%s: Found a non-MSI controller: %s", __func__, + device_get_name(pic->pic_dev))); + + isrc = isrc_lookup(irq); + if (isrc == NULL) + return (EINVAL); + + err = MSI_RELEASE_MSIX(pic->pic_dev, child, isrc); + return (err); +} + +int +intr_map_msi(device_t pci, device_t child, intptr_t xref, int irq, + uint64_t *addr, uint32_t *data) +{ + struct intr_irqsrc *isrc; + struct intr_pic *pic; + int err; + + pic = pic_lookup(NULL, xref); + if (pic == NULL) + return (ESRCH); + + KASSERT((pic->pic_flags & FLAG_MSI) != 0, + ("%s: Found a non-MSI controller: %s", __func__, + device_get_name(pic->pic_dev))); + + isrc = isrc_lookup(irq); + if (isrc == NULL) + return (EINVAL); + + err = MSI_MAP_MSI(pic->pic_dev, child, isrc, addr, data); + return (err); +} + void dosoftints(void); void dosoftints(void) { } #ifdef SMP /* * Init interrupt controller on another CPU. */ void intr_pic_init_secondary(void) { /* * QQQ: Only root PIC is aware of other CPUs ??? */ KASSERT(intr_irq_root_dev != NULL, ("%s: no root attached", __func__)); //mtx_lock(&isrc_table_lock); PIC_INIT_SECONDARY(intr_irq_root_dev); //mtx_unlock(&isrc_table_lock); } #endif #ifdef DDB DB_SHOW_COMMAND(irqs, db_show_irqs) { u_int i, irqsum; u_long num; struct intr_irqsrc *isrc; for (irqsum = 0, i = 0; i < NIRQ; i++) { isrc = irq_sources[i]; if (isrc == NULL) continue; num = isrc->isrc_count != NULL ? isrc->isrc_count[0] : 0; db_printf("irq%-3u <%s>: cpu %02lx%s cnt %lu\n", i, isrc->isrc_name, isrc->isrc_cpu.__bits[0], isrc->isrc_flags & INTR_ISRCF_BOUND ? " (bound)" : "", num); irqsum += num; } db_printf("irq total %u\n", irqsum); } #endif Index: head/sys/mips/mediatek/files.mediatek =================================================================== --- head/sys/mips/mediatek/files.mediatek (revision 299927) +++ head/sys/mips/mediatek/files.mediatek (revision 299928) @@ -1,41 +1,42 @@ # $FreeBSD$ mips/mediatek/mtk_machdep.c standard mips/mediatek/mtk_sysctl.c standard mips/mediatek/mtk_soc.c standard mips/mediatek/mtk_reset.c standard mips/mediatek/mtk_clock.c standard mips/mediatek/mtk_pinctrl.c standard mips/mediatek/palmbus.c standard mips/mediatek/mtk_intr_v1.c optional mtk_intr_v1 mips/mediatek/mtk_intr_v2.c optional mtk_intr_v2 mips/mediatek/mtk_intr_gic.c optional mtk_intr_gic mips/mediatek/uart_dev_mtk.c optional uart uart_dev_mtk mips/mediatek/mtk_spi_v1.c optional spibus mtk_spi_v1 mips/mediatek/mtk_spi_v2.c optional spibus mtk_spi_v2 mips/mediatek/mtk_usb_phy.c optional usb mtk_usb_phy mips/mediatek/mtk_xhci.c optional usb xhci mips/mediatek/mtk_ohci.c optional usb ohci mips/mediatek/mtk_ehci.c optional usb ehci mips/mediatek/mtk_dotg.c optional usb dwcotg mips/mediatek/mtk_pcie.c optional pci mips/mediatek/mtk_gpio_v1.c optional gpio mtk_gpio_v1 mips/mediatek/mtk_gpio_v2.c optional gpio mtk_gpio_v2 #mips/mediatek/mtk_mmc.c optional mmc # Ralink/Mediatek Ethernet driver dev/rt/if_rt.c optional rt # Hack to reuse ARM intrng code kern/subr_intr.c standard +kern/msi_if.m standard kern/pic_if.m standard # Intrng compatible MIPS32 interrupt controller mips/mips/mips_pic.c standard # Standard MIPS ticker mips/mips/tick.c standard # Temporary Reset if mips/mediatek/fdt_reset.c standard mips/mediatek/fdt_reset_if.m standard Index: head/sys/sys/intr.h =================================================================== --- head/sys/sys/intr.h (revision 299927) +++ head/sys/sys/intr.h (revision 299928) @@ -1,159 +1,167 @@ /*- * Copyright (c) 2015-2016 Svatopluk Kraus * Copyright (c) 2015-2016 Michal Meloun * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #ifndef _SYS_INTR_H_ #define _SYS_INTR_H_ #include #define INTR_IRQ_INVALID 0xFFFFFFFF enum intr_map_data_type { INTR_MAP_DATA_ACPI, INTR_MAP_DATA_FDT, INTR_MAP_DATA_GPIO, }; struct intr_map_data { enum intr_map_data_type type; size_t size; }; #ifdef DEV_ACPI struct intr_map_data_acpi { struct intr_map_data hdr; u_int irq; enum intr_polarity pol; enum intr_trigger trig; }; #endif #ifdef FDT struct intr_map_data_fdt { struct intr_map_data hdr; u_int ncells; pcell_t cells[0]; }; #endif struct intr_map_data_gpio { struct intr_map_data hdr; u_int gpio_pin_num; u_int gpio_pin_flags; u_int gpio_intr_mode; }; #ifdef notyet #define INTR_SOLO INTR_MD1 typedef int intr_irq_filter_t(void *arg, struct trapframe *tf); #else typedef int intr_irq_filter_t(void *arg); #endif #define INTR_ISRC_NAMELEN (MAXCOMLEN + 1) #define INTR_ISRCF_IPI 0x01 /* IPI interrupt */ #define INTR_ISRCF_PPI 0x02 /* PPI interrupt */ #define INTR_ISRCF_BOUND 0x04 /* bound to a CPU */ /* Interrupt source definition. */ struct intr_irqsrc { device_t isrc_dev; /* where isrc is mapped */ u_int isrc_irq; /* unique identificator */ u_int isrc_flags; char isrc_name[INTR_ISRC_NAMELEN]; cpuset_t isrc_cpu; /* on which CPUs is enabled */ u_int isrc_index; u_long * isrc_count; u_int isrc_handlers; struct intr_event * isrc_event; #ifdef INTR_SOLO intr_irq_filter_t * isrc_filter; void * isrc_arg; #endif }; /* Intr interface for PIC. */ int intr_isrc_deregister(struct intr_irqsrc *); int intr_isrc_register(struct intr_irqsrc *, device_t, u_int, const char *, ...) __printflike(4, 5); #ifdef SMP bool intr_isrc_init_on_cpu(struct intr_irqsrc *isrc, u_int cpu); #endif int intr_isrc_dispatch(struct intr_irqsrc *, struct trapframe *); u_int intr_irq_next_cpu(u_int current_cpu, cpuset_t *cpumask); int intr_pic_register(device_t, intptr_t); int intr_pic_deregister(device_t, intptr_t); int intr_pic_claim_root(device_t, intptr_t, intr_irq_filter_t *, void *, u_int); extern device_t intr_irq_root_dev; /* Intr interface for BUS. */ int intr_map_irq(device_t, intptr_t, struct intr_map_data *, u_int *); int intr_alloc_irq(device_t, struct resource *); int intr_release_irq(device_t, struct resource *); int intr_setup_irq(device_t, struct resource *, driver_filter_t, driver_intr_t, void *, int, void **); int intr_teardown_irq(device_t, struct resource *, void *); int intr_describe_irq(device_t, struct resource *, void *, const char *); +/* MSI/MSI-X handling */ +int intr_msi_register(device_t, intptr_t); +int intr_alloc_msi(device_t, device_t, intptr_t, int, int, int *); +int intr_release_msi(device_t, device_t, intptr_t, int, int *); +int intr_map_msi(device_t, device_t, intptr_t, int, uint64_t *, uint32_t *); +int intr_alloc_msix(device_t, device_t, intptr_t, int *); +int intr_release_msix(device_t, device_t, intptr_t, int); + #ifdef DEV_ACPI u_int intr_acpi_map_irq(device_t, u_int, enum intr_polarity, enum intr_trigger); #endif #ifdef FDT u_int intr_fdt_map_irq(phandle_t, pcell_t *, u_int); #endif u_int intr_gpio_map_irq(device_t dev, u_int pin_num, u_int pin_flags, u_int intr_mode); #ifdef SMP int intr_bind_irq(device_t, struct resource *, int); void intr_pic_init_secondary(void); /* Virtualization for interrupt source IPI counter increment. */ static inline void intr_ipi_increment_count(u_long *counter, u_int cpu) { KASSERT(cpu < MAXCPU, ("%s: too big cpu %u", __func__, cpu)); counter[cpu]++; } /* Virtualization for interrupt source IPI counters setup. */ u_long * intr_ipi_setup_counters(const char *name); #endif #endif /* _SYS_INTR_H */