Index: projects/arm_intrng/sys/arm/arm/gic.c =================================================================== --- projects/arm_intrng/sys/arm/arm/gic.c (revision 276093) +++ projects/arm_intrng/sys/arm/arm/gic.c (revision 276094) @@ -1,508 +1,510 @@ /*- * Copyright (c) 2011 The FreeBSD Foundation * All rights reserved. * * Developed by Damjan Marion * * Based on OMAP4 GIC code by Ben Gray * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. The name of the company nor the name of the author may be used to * endorse or promote products derived from this software without specific * prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "pic_if.h" /* We are using GICv2 register naming */ /* Distributor Registers */ #define GICD_CTLR 0x000 /* v1 ICDDCR */ #define GICD_TYPER 0x004 /* v1 ICDICTR */ #define GICD_IIDR 0x008 /* v1 ICDIIDR */ #define GICD_IGROUPR(n) (0x0080 + ((n) * 4)) /* v1 ICDISER */ #define GICD_ISENABLER(n) (0x0100 + ((n) * 4)) /* v1 ICDISER */ #define GICD_ICENABLER(n) (0x0180 + ((n) * 4)) /* v1 ICDICER */ #define GICD_ISPENDR(n) (0x0200 + ((n) * 4)) /* v1 ICDISPR */ #define GICD_ICPENDR(n) (0x0280 + ((n) * 4)) /* v1 ICDICPR */ #define GICD_ICACTIVER(n) (0x0380 + ((n) * 4)) /* v1 ICDABR */ #define GICD_IPRIORITYR(n) (0x0400 + ((n) * 4)) /* v1 ICDIPR */ #define GICD_ITARGETSR(n) (0x0800 + ((n) * 4)) /* v1 ICDIPTR */ #define GICD_ICFGR(n) (0x0C00 + ((n) * 4)) /* v1 ICDICFR */ #define GICD_SGIR(n) (0x0F00 + ((n) * 4)) /* v1 ICDSGIR */ /* CPU Registers */ #define GICC_CTLR 0x0000 /* v1 ICCICR */ #define GICC_PMR 0x0004 /* v1 ICCPMR */ #define GICC_BPR 0x0008 /* v1 ICCBPR */ #define GICC_IAR 0x000C /* v1 ICCIAR */ #define GICC_EOIR 0x0010 /* v1 ICCEOIR */ #define GICC_RPR 0x0014 /* v1 ICCRPR */ #define GICC_HPPIR 0x0018 /* v1 ICCHPIR */ #define GICC_ABPR 0x001C /* v1 ICCABPR */ #define GICC_IIDR 0x00FC /* v1 ICCIIDR*/ #define GIC_FIRST_IPI 0 /* Irqs 0-15 are SGIs/IPIs. */ #define GIC_LAST_IPI 15 #define GIC_FIRST_PPI 16 /* Irqs 16-31 are private (per */ #define GIC_LAST_PPI 31 /* core) peripheral interrupts. */ #define GIC_FIRST_SPI 32 /* Irqs 32+ are shared peripherals. */ /* First bit is a polarity bit (0 - low, 1 - high) */ #define GICD_ICFGR_POL_LOW (0 << 0) #define GICD_ICFGR_POL_HIGH (1 << 0) #define GICD_ICFGR_POL_MASK 0x1 /* Second bit is a trigger bit (0 - level, 1 - edge) */ #define GICD_ICFGR_TRIG_LVL (0 << 1) #define GICD_ICFGR_TRIG_EDGE (1 << 1) #define GICD_ICFGR_TRIG_MASK 0x2 struct arm_gic_softc { device_t gic_dev; struct resource * gic_res[3]; bus_space_tag_t gic_c_bst; bus_space_tag_t gic_d_bst; bus_space_handle_t gic_c_bsh; bus_space_handle_t gic_d_bsh; void * gic_intrhand; uint8_t ver; struct mtx mutex; uint32_t nirqs; }; static struct resource_spec arm_gic_spec[] = { { SYS_RES_MEMORY, 0, RF_ACTIVE }, /* Distributor registers */ { SYS_RES_MEMORY, 1, RF_ACTIVE }, /* CPU Interrupt Intf. registers */ { SYS_RES_IRQ, 0, RF_ACTIVE | RF_OPTIONAL }, /* Parent interrupt */ { -1, 0 } }; static struct arm_gic_softc *arm_gic_sc = NULL; static int arm_gic_probe(device_t); static int arm_gic_attach(device_t); static void arm_gic_init_secondary(device_t); static int arm_gic_intr(void *); static int arm_gic_config(device_t, int, enum intr_trigger, enum intr_polarity); static void arm_gic_eoi(device_t, int); static void arm_gic_mask(device_t, int); static void arm_gic_unmask(device_t, int); static void arm_gic_ipi_send(device_t, cpuset_t, int); static void arm_gic_ipi_clear(device_t, int); #define gic_c_read_4(_sc, _reg) \ bus_space_read_4((_sc)->gic_c_bst, (_sc)->gic_c_bsh, (_reg)) #define gic_c_write_4(_sc, _reg, _val) \ bus_space_write_4((_sc)->gic_c_bst, (_sc)->gic_c_bsh, (_reg), (_val)) #define gic_d_read_4(_sc, _reg) \ bus_space_read_4((_sc)->gic_d_bst, (_sc)->gic_d_bsh, (_reg)) #define gic_d_write_4(_sc, _reg, _val) \ bus_space_write_4((_sc)->gic_d_bst, (_sc)->gic_d_bsh, (_reg), (_val)) static struct ofw_compat_data compat_data[] = { {"arm,gic", true}, /* Non-standard, used in FreeBSD dts. */ {"arm,gic-400", true}, {"arm,cortex-a15-gic", true}, {"arm,cortex-a9-gic", true}, {"arm,cortex-a7-gic", true}, {"arm,arm11mp-gic", true}, {"brcm,brahma-b15-gic", true}, {NULL, false} }; static int arm_gic_probe(device_t dev) { if (!ofw_bus_status_okay(dev)) return (ENXIO); if (!ofw_bus_search_compatible(dev, compat_data)->ocd_data) return (ENXIO); device_set_desc(dev, "ARM Generic Interrupt Controller"); return (BUS_PROBE_DEFAULT); } static void arm_gic_init_secondary(device_t dev) { struct arm_gic_softc *sc = device_get_softc(dev); int i; for (i = 0; i < sc->nirqs; i += 4) gic_d_write_4(sc, GICD_IPRIORITYR(i >> 2), 0); /* Set all the interrupts to be in Group 0 (secure) */ for (i = 0; i < sc->nirqs; i += 32) { gic_d_write_4(sc, GICD_IGROUPR(i >> 5), 0); } /* Enable CPU interface */ gic_c_write_4(sc, GICC_CTLR, 1); /* Set priority mask register. */ gic_c_write_4(sc, GICC_PMR, 0xff); /* Enable interrupt distribution */ gic_d_write_4(sc, GICD_CTLR, 0x01); /* * Activate the timer interrupts: virtual, secure, and non-secure. */ gic_d_write_4(sc, GICD_ISENABLER(27 >> 5), (1UL << (27 & 0x1F))); gic_d_write_4(sc, GICD_ISENABLER(29 >> 5), (1UL << (29 & 0x1F))); gic_d_write_4(sc, GICD_ISENABLER(30 >> 5), (1UL << (30 & 0x1F))); } int gic_decode_fdt(uint32_t iparent, uint32_t *intr, int *interrupt, int *trig, int *pol) { static u_int num_intr_cells; if (num_intr_cells == 0) { if (OF_searchencprop(OF_node_from_xref(iparent), "#interrupt-cells", &num_intr_cells, sizeof(num_intr_cells)) == -1) { num_intr_cells = 1; } } if (num_intr_cells == 1) { *interrupt = fdt32_to_cpu(intr[0]); *trig = INTR_TRIGGER_CONFORM; *pol = INTR_POLARITY_CONFORM; } else { if (intr[0] == 0) *interrupt = fdt32_to_cpu(intr[1]) + GIC_FIRST_SPI; else *interrupt = fdt32_to_cpu(intr[1]) + GIC_FIRST_PPI; /* * In intr[2], bits[3:0] are trigger type and level flags. * 1 = low-to-high edge triggered * 2 = high-to-low edge triggered * 4 = active high level-sensitive * 8 = active low level-sensitive * The hardware only supports active-high-level or rising-edge. */ if (intr[2] & 0x0a) { printf("unsupported trigger/polarity configuration " "0x%2x\n", intr[2] & 0x0f); return (ENOTSUP); } *pol = INTR_POLARITY_CONFORM; if (intr[2] & 0x01) *trig = INTR_TRIGGER_EDGE; else *trig = INTR_TRIGGER_LEVEL; } return (0); } static int arm_gic_attach(device_t dev) { struct arm_gic_softc *sc; int i; uint32_t icciidr; if (arm_gic_sc) return (ENXIO); sc = device_get_softc(dev); if (bus_alloc_resources(dev, arm_gic_spec, sc->gic_res)) { device_printf(dev, "could not allocate resources\n"); return (ENXIO); } sc->gic_dev = dev; arm_gic_sc = sc; /* Initialize mutex */ mtx_init(&sc->mutex, "GIC lock", "", MTX_SPIN); /* Distributor Interface */ sc->gic_d_bst = rman_get_bustag(sc->gic_res[0]); sc->gic_d_bsh = rman_get_bushandle(sc->gic_res[0]); /* CPU Interface */ sc->gic_c_bst = rman_get_bustag(sc->gic_res[1]); sc->gic_c_bsh = rman_get_bushandle(sc->gic_res[1]); if (bus_setup_intr(dev, sc->gic_res[2], INTR_TYPE_MISC | INTR_CONTROLLER, arm_gic_intr, NULL, sc, &sc->gic_intrhand)) { device_printf(dev, "could not setup interrupt handler\n"); bus_release_resources(dev, arm_gic_spec, sc->gic_res); return (ENXIO); } arm_register_pic(dev, PIC_FEATURE_IPI); + for (int i = 0; i < ARM_IPI_COUNT; i++) + arm_ipi_map_irq(dev, i, i); /* Disable interrupt forwarding to the CPU interface */ gic_d_write_4(sc, GICD_CTLR, 0x00); /* Get the number of interrupts */ sc->nirqs = gic_d_read_4(sc, GICD_TYPER); sc->nirqs = 32 * ((sc->nirqs & 0x1f) + 1); icciidr = gic_c_read_4(sc, GICC_IIDR); device_printf(dev,"pn 0x%x, arch 0x%x, rev 0x%x, implementer 0x%x irqs %u\n", icciidr>>20, (icciidr>>16) & 0xF, (icciidr>>12) & 0xf, (icciidr & 0xfff), sc->nirqs); /* Set all global interrupts to be level triggered, active low. */ for (i = 32; i < sc->nirqs; i += 16) { gic_d_write_4(sc, GICD_ICFGR(i >> 4), 0x00000000); } /* Disable all interrupts. */ for (i = 32; i < sc->nirqs; i += 32) { gic_d_write_4(sc, GICD_ICENABLER(i >> 5), 0xFFFFFFFF); } for (i = 0; i < sc->nirqs; i += 4) { gic_d_write_4(sc, GICD_IPRIORITYR(i >> 2), 0); gic_d_write_4(sc, GICD_ITARGETSR(i >> 2), 1 << 0 | 1 << 8 | 1 << 16 | 1 << 24); } /* Set all the interrupts to be in Group 0 (secure) */ for (i = 0; i < sc->nirqs; i += 32) { gic_d_write_4(sc, GICD_IGROUPR(i >> 5), 0); } /* Enable CPU interface */ gic_c_write_4(sc, GICC_CTLR, 1); /* Set priority mask register. */ gic_c_write_4(sc, GICC_PMR, 0xff); /* Enable interrupt distribution */ gic_d_write_4(sc, GICD_CTLR, 0x01); return (0); } static int arm_gic_intr(void *arg) { struct arm_gic_softc *sc = (struct arm_gic_softc *)arg; uint32_t active_irq, last_irq = 0; active_irq = gic_c_read_4(sc, GICC_IAR); /* * Immediatly EOIR the SGIs, because doing so requires the other * bits (ie CPU number), not just the IRQ number, and we do not * have this information later. */ if ((active_irq & 0x3ff) <= GIC_LAST_IPI) gic_c_write_4(sc, GICC_EOIR, active_irq); active_irq &= 0x3FF; if (active_irq == 0x3FF) { if (last_irq == -1) printf("Spurious interrupt detected\n"); return (FILTER_HANDLED); } - gic_c_write_4(sc, GICC_EOIR, active_irq); + //gic_c_write_4(sc, GICC_EOIR, active_irq); arm_dispatch_irq(sc->gic_dev, NULL, active_irq); return (FILTER_HANDLED); } static int arm_gic_config(device_t dev, int irq, enum intr_trigger trig, enum intr_polarity pol) { struct arm_gic_softc *sc = device_get_softc(dev); uint32_t reg; uint32_t mask; /* Function is public-accessible, so validate input arguments */ if ((irq < 0) || (irq >= sc->nirqs)) goto invalid_args; if ((trig != INTR_TRIGGER_EDGE) && (trig != INTR_TRIGGER_LEVEL) && (trig != INTR_TRIGGER_CONFORM)) goto invalid_args; if ((pol != INTR_POLARITY_HIGH) && (pol != INTR_POLARITY_LOW) && (pol != INTR_POLARITY_CONFORM)) goto invalid_args; mtx_lock_spin(&sc->mutex); reg = gic_d_read_4(sc, GICD_ICFGR(irq >> 4)); mask = (reg >> 2*(irq % 16)) & 0x3; if (pol == INTR_POLARITY_LOW) { mask &= ~GICD_ICFGR_POL_MASK; mask |= GICD_ICFGR_POL_LOW; } else if (pol == INTR_POLARITY_HIGH) { mask &= ~GICD_ICFGR_POL_MASK; mask |= GICD_ICFGR_POL_HIGH; } if (trig == INTR_TRIGGER_LEVEL) { mask &= ~GICD_ICFGR_TRIG_MASK; mask |= GICD_ICFGR_TRIG_LVL; } else if (trig == INTR_TRIGGER_EDGE) { mask &= ~GICD_ICFGR_TRIG_MASK; mask |= GICD_ICFGR_TRIG_EDGE; } /* Set mask */ reg = reg & ~(0x3 << 2*(irq % 16)); reg = reg | (mask << 2*(irq % 16)); gic_d_write_4(sc, GICD_ICFGR(irq >> 4), reg); mtx_unlock_spin(&sc->mutex); return (0); invalid_args: device_printf(dev, "gic_config_irg, invalid parameters\n"); return (EINVAL); } static void arm_gic_eoi(device_t dev, int irq) { struct arm_gic_softc *sc = device_get_softc(dev); if (irq > GIC_LAST_IPI) arm_irq_memory_barrier(irq); gic_c_write_4(sc, GICC_EOIR, irq); } static void arm_gic_mask(device_t dev, int irq) { struct arm_gic_softc *sc = device_get_softc(dev); gic_d_write_4(sc, GICD_ICENABLER(irq >> 5), (1UL << (irq & 0x1F))); gic_c_write_4(sc, GICC_EOIR, irq); } static void arm_gic_unmask(device_t dev, int irq) { struct arm_gic_softc *sc = device_get_softc(dev); if (irq > GIC_LAST_IPI) arm_irq_memory_barrier(irq); gic_d_write_4(sc, GICD_ISENABLER(irq >> 5), (1UL << (irq & 0x1F))); } static void arm_gic_ipi_send(device_t dev, cpuset_t cpus, int ipi) { struct arm_gic_softc *sc = device_get_softc(dev); uint32_t val = 0, i; for (i = 0; i < MAXCPU; i++) if (CPU_ISSET(i, &cpus)) val |= 1 << (16 + i); gic_d_write_4(sc, GICD_SGIR(0), val | ipi); } static int arm_gic_ipi_read(device_t dev, int i) { if (i != -1) { /* * The intr code will automagically give the frame pointer * if the interrupt argument is 0. */ if ((unsigned int)i > 16) return (0); return (i); } return (0x3ff); } static void arm_gic_ipi_clear(device_t dev, int ipi) { /* no-op */ } static device_method_t arm_gic_methods[] = { /* Device interface */ DEVMETHOD(device_probe, arm_gic_probe), DEVMETHOD(device_attach, arm_gic_attach), /* Interrupt controller interface */ DEVMETHOD(pic_config, arm_gic_config), DEVMETHOD(pic_mask, arm_gic_mask), DEVMETHOD(pic_unmask, arm_gic_unmask), DEVMETHOD(pic_eoi, arm_gic_eoi), DEVMETHOD(pic_init_secondary, arm_gic_init_secondary), DEVMETHOD(pic_ipi_send, arm_gic_ipi_send), DEVMETHOD(pic_ipi_clear, arm_gic_ipi_clear), DEVMETHOD(pic_ipi_read, arm_gic_ipi_read), { 0, 0 } }; static driver_t arm_gic_driver = { "gic", arm_gic_methods, sizeof(struct arm_gic_softc), }; static devclass_t arm_gic_devclass; EARLY_DRIVER_MODULE(gic, simplebus, arm_gic_driver, arm_gic_devclass, 0, 0, BUS_PASS_INTERRUPT + BUS_PASS_ORDER_MIDDLE); EARLY_DRIVER_MODULE(gic, ofwbus, arm_gic_driver, arm_gic_devclass, 0, 0, BUS_PASS_INTERRUPT + BUS_PASS_ORDER_MIDDLE); Index: projects/arm_intrng/sys/arm/arm/intrng.c =================================================================== --- projects/arm_intrng/sys/arm/arm/intrng.c (revision 276093) +++ projects/arm_intrng/sys/arm/arm/intrng.c (revision 276094) @@ -1,599 +1,675 @@ /*- * Copyright (c) 2012-2014 Jakub Wojciech Klama . * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. All advertising materials mentioning features or use of this software * must display the following acknowledgement: * This product includes software developed by Mark Brinicombe * for the NetBSD Project. * 4. The name of the company nor the name of the author may be used to * endorse or promote products derived from this software without specific * prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * */ +#include "opt_platform.h" + #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include +#include #include #include #include #include #include #include #include #include #include #include #include "pic_if.h" #define INTRNAME_LEN (MAXCOMLEN + 1) -#define IRQ_PIC_IDX(_irq) (((_irq) >> 8) & 0xff) -#define IRQ_VECTOR_IDX(_irq) ((_irq) & 0xff) -#define IRQ_GEN(_pic, _irq) (((_pic) << 8) | ((_irq) & 0xff)) +#define MAXINTRS 1024 // XXX Need this passed in to pic registration + //#define DEBUG #ifdef DEBUG #define debugf(fmt, args...) do { printf("%s(): ", __func__); \ printf(fmt,##args); } while (0) #else #define debugf(fmt, args...) #endif typedef void (*mask_fn)(void *); struct arm_intr_controller { device_t ic_dev; phandle_t ic_node; - int ic_maxintrs; - struct arm_intr_handler *ic_intrs; + u_int ic_idx; + u_int ic_maxintrs; + struct arm_intr_handler **ic_ih_by_hwirq; + u_int ic_ih_count; + SLIST_HEAD(, arm_intr_handler) + ic_ih_list; }; struct arm_intr_handler { - device_t ih_dev; - const char * ih_ipi_name; - int ih_intrcnt_idx; - int ih_irq; - pcell_t ih_cells[8]; - int ih_ncells; + SLIST_ENTRY(arm_intr_handler) ih_next_entry; + u_int ih_intrcnt_idx; + u_int ih_resirq; + u_int ih_hwirq; + u_int ih_ncells; enum intr_trigger ih_trig; enum intr_polarity ih_pol; struct intr_event * ih_event; - struct arm_intr_controller *ih_pic; + struct arm_intr_controller *ih_ic; + pcell_t ih_cells[]; }; +static u_int resirq_encode(u_int picidx, u_int irqidx); + + static void arm_mask_irq(void *); static void arm_unmask_irq(void *); static void arm_eoi(void *); static struct arm_intr_controller arm_pics[NPIC]; static struct arm_intr_controller *arm_ipi_pic; -static int intrcnt_index = 0; -static int last_printed = 0; +static int intrcnt_index; +static int intrcnt_last_printed; MALLOC_DECLARE(M_INTRNG); MALLOC_DEFINE(M_INTRNG, "intrng", "ARM interrupt handling"); +static const char *ipi_names[] = { + "IPI:AST", + "IPI:PREEMPT", + "IPI:RENDEZVOUS", + "IPI:STOP", + "IPI:HARDCLOCK", + "IPI:TLB" +}; +CTASSERT(ARM_IPI_COUNT == nitems(ipi_names)); + +static struct arm_intr_handler * ipi_handlers[ARM_IPI_COUNT]; + /* Data for statistics reporting. */ u_long intrcnt[NIRQ]; char intrnames[NIRQ * INTRNAME_LEN]; size_t sintrcnt = sizeof(intrcnt); size_t sintrnames = sizeof(intrnames); int (*arm_config_irq)(int irq, enum intr_trigger trig, enum intr_polarity pol) = NULL; -void -arm_intrnames_init(void) +static inline struct arm_intr_controller * +ic_from_dev(device_t dev) { - /* nothing... */ + struct arm_intr_controller *ic; + u_int i; + + for (i = 0, ic = arm_pics; i < nitems(arm_pics); i++, ic++) { + if (dev == ic->ic_dev) + return (ic); + } + return (NULL); } -void -arm_dispatch_irq(device_t dev, struct trapframe *tf, int irq) +static inline struct arm_intr_controller * +ic_from_node(phandle_t node) { struct arm_intr_controller *ic; - struct arm_intr_handler *ih = NULL; - int i; + u_int i; - debugf("pic %s, tf %p, irq %d\n", device_get_nameunit(dev), tf, irq); + for (i = 0, ic = arm_pics; i < nitems(arm_pics); i++, ic++) { + if (node == ic->ic_node) + return (ic); + } + return (NULL); +} - /* - * If we got null trapframe argument, that probably means - * a call from non-root interrupt controller. In that case, - * we'll just use the saved one. - */ - if (tf == NULL) - tf = PCPU_GET(curthread)->td_intr_frame; +static struct arm_intr_controller * +ic_create(phandle_t node) +{ + struct arm_intr_controller *ic; + u_int i; - for (ic = &arm_pics[0]; ic->ic_dev != NULL; ic++) { - if (ic->ic_dev == dev) { - for (i = 0; i < ic->ic_maxintrs; i++) { - ih = &ic->ic_intrs[i]; - if (irq == ih->ih_irq) - goto done; - } - } + for (i = 0, ic = arm_pics; i < nitems(arm_pics); i++, ic++) { + if (ic->ic_node == 0) + break; } -done: + if (i == nitems(arm_pics)) + panic("no room to add interrupt controller"); - if (ic->ic_dev == NULL) - panic("arm_dispatch_irq: unknown irq"); + bzero(ic, sizeof(ic)); + ic->ic_idx = i; + ic->ic_node = node; + SLIST_INIT(&ic->ic_ih_list); - debugf("requested by %s\n", ih->ih_ipi_name != NULL - ? ih->ih_ipi_name - : device_get_nameunit(ih->ih_dev)); + debugf("allocated new interrupt controller at index %d ptr %p for node %d\n", i, ic, node); + return (ic); +} - intrcnt[ih->ih_intrcnt_idx]++; - if (intr_event_handle(ih->ih_event, tf) != 0) { - /* Stray IRQ */ - arm_mask_irq(ih); +static void +ic_setup_dev(struct arm_intr_controller *ic, device_t dev, u_int maxintrs) +{ + struct arm_intr_handler *ih; + + ic->ic_dev = dev; + ic->ic_maxintrs = maxintrs; + ic->ic_ih_by_hwirq = malloc(maxintrs * sizeof(struct arm_intr_handler *), + M_INTRNG, M_WAITOK | M_ZERO); + SLIST_FOREACH(ih, &ic->ic_ih_list, ih_next_entry) { + PIC_TRANSLATE(ic->ic_dev, ih->ih_cells, &ih->ih_hwirq, + &ih->ih_trig, &ih->ih_pol); + ic->ic_ih_by_hwirq[ih->ih_hwirq] = ih; } +} - debugf("done\n"); +static struct arm_intr_handler * +ic_add_ih(struct arm_intr_controller *ic, pcell_t *cells, u_int ncells) +{ + struct arm_intr_handler *ih; + u_int cellsize; + + cellsize = ncells * sizeof(*cells); + ih = malloc(sizeof(*ih) + cellsize, M_INTRNG, M_WAITOK | M_ZERO); + memcpy(ih->ih_cells, cells, cellsize); + ih->ih_ncells = ncells; + ih->ih_ic = ic; + ih->ih_resirq = resirq_encode(ic->ic_idx, ic->ic_ih_count++); + SLIST_INSERT_HEAD(&ic->ic_ih_list, ih, ih_next_entry); + return (ih); } +static void +ic_index_ih_by_hwirq(struct arm_intr_controller *ic, + struct arm_intr_handler *ih) +{ + + KASSERT(ih->ih_hwirq < ic->ic_maxintrs, ("%s irq %u too large", + device_get_nameunit(ic->ic_dev), ih->ih_hwirq)); + KASSERT(ic->ic_ih_by_hwirq[ih->ih_hwirq] == NULL, + ("%s irq %u already registered", device_get_nameunit(ic->ic_dev), + ih->ih_hwirq)); + ic->ic_ih_by_hwirq[ih->ih_hwirq] = ih; +} + static struct arm_intr_handler * -arm_lookup_intr_handler(device_t pic, int idx) +ih_from_fdtcells(struct arm_intr_controller *ic, pcell_t *cells, u_int ncells) { - struct arm_intr_controller *ic; + struct arm_intr_handler *ih; - for (ic = &arm_pics[0]; ic->ic_dev != NULL; ic++) { - if (ic->ic_dev != NULL && ic->ic_dev == pic) - return (&ic->ic_intrs[idx]); + SLIST_FOREACH(ih, &ic->ic_ih_list, ih_next_entry) { + if (ncells == ih->ih_ncells && memcmp(cells, ih->ih_cells, + ncells * sizeof(*cells)) == 0) + return (ih); } + return (NULL); +} +static struct arm_intr_handler * +ih_from_resirq(struct arm_intr_controller *ic, u_int resirq) +{ + struct arm_intr_handler *ih; + + SLIST_FOREACH(ih, &ic->ic_ih_list, ih_next_entry) { + if (resirq == ih->ih_resirq) + return (ih); + } return (NULL); } +static struct arm_intr_handler * +ih_from_hwirq(struct arm_intr_controller *ic, u_int hwirq) +{ -int -arm_fdt_map_irq(phandle_t ic, pcell_t *cells, int ncells) + return (ic->ic_ih_by_hwirq[hwirq]); +} + +static u_int +resirq_encode(u_int picidx, u_int irqidx) { - struct arm_intr_controller *pic; - struct arm_intr_handler *ih; - int i, j; - ic = OF_node_from_xref(ic); + return((picidx << 16) | (irqidx & 0xffff)); +} - debugf("ic %08x cells <%*D>\n", ic, ncells * sizeof(pcell_t), - (char *)cells, ","); +static u_int +resirq_decode(int resirq, struct arm_intr_controller **pic, + struct arm_intr_handler **pih) +{ + struct arm_intr_controller *ic; + u_int irqidx, picidx; - for (i = 0; arm_pics[i].ic_node != 0; i++) { - pic = &arm_pics[i]; - if (pic->ic_node == ic) { - for (j = 0; j < pic->ic_maxintrs; j++) { - ih = &pic->ic_intrs[j]; + picidx = resirq >> 16; + KASSERT(picidx < nitems(arm_pics), ("bad pic index %u", picidx)); + ic = &arm_pics[picidx]; - /* Compare pcell contents */ - if (!memcmp(cells, ih->ih_cells, ncells)) - return (IRQ_GEN(i, j)); - } + irqidx = resirq & 0xffff; + KASSERT(irqidx < ic->ic_ih_count, ("bad irq index %u for pic %u", + irqidx, picidx)); - /* Not found - new entry required */ - pic->ic_maxintrs++; - pic->ic_intrs = realloc(pic->ic_intrs, - pic->ic_maxintrs * sizeof(struct arm_intr_handler), - M_INTRNG, M_WAITOK | M_ZERO); + if (pic != NULL) + *pic = ic; + if (pih != NULL && ic != NULL) + *pih = ih_from_resirq(ic, resirq); - ih = &pic->ic_intrs[pic->ic_maxintrs - 1]; - ih->ih_pic = pic; - ih->ih_ncells = ncells; - memcpy(ih->ih_cells, cells, ncells); + return (irqidx); +} - if (pic->ic_dev != NULL) { - /* Map IRQ number */ - PIC_TRANSLATE(pic->ic_dev, cells, &ih->ih_irq, - &ih->ih_trig, &ih->ih_pol); +void +arm_intrnames_init(void) +{ + /* nothing... */ +} - debugf("translated to irq %d\n", ih->ih_irq); - } +void +arm_dispatch_irq(device_t dev, struct trapframe *tf, int irq) +{ + struct arm_intr_controller *ic; + struct arm_intr_handler *ih = NULL; - return (IRQ_GEN(i, pic->ic_maxintrs - 1)); - } - } +// debugf("pic %s, tf %p, irq %d\n", device_get_nameunit(dev), tf, irq); - /* - * Interrupt controller is not registered yet, so - * we map a stub for it. 'i' is pointing to free - * first slot in arm_pics table. + /* + * If we got null trapframe argument, that probably means + * a call from non-root interrupt controller. In that case, + * we'll just use the saved one. */ - debugf("allocating new ic at index %d\n", i); + if (tf == NULL) + tf = PCPU_GET(curthread)->td_intr_frame; - pic = &arm_pics[i]; - pic->ic_node = ic; - pic->ic_maxintrs = 1; - pic->ic_intrs = malloc(sizeof(struct arm_intr_handler), M_INTRNG, - M_WAITOK | M_ZERO); + ic = ic_from_dev(dev); + KASSERT(ic != NULL, ("%s: interrupt controller for %s not found", + __FUNCTION__, device_get_nameunit(dev))); - ih = &pic->ic_intrs[0]; - ih->ih_pic = pic; - ih->ih_ncells = ncells; - memcpy(ih->ih_cells, cells, ncells); + ih = ih_from_hwirq(ic, irq); + KASSERT(ih != NULL, ("%s: interrupt handler for %s irq %d not found", + __FUNCTION__, device_get_nameunit(dev), irq)); - return (IRQ_GEN(i, 0)); + intrcnt[ih->ih_intrcnt_idx]++; + if (intr_event_handle(ih->ih_event, tf) != 0) { + device_printf(dev, "stray irq %d; disabled", irq); + arm_mask_irq(ih); + } + +// debugf("done\n"); } -const char * -arm_describe_irq(int irq) +int +arm_fdt_map_irq(phandle_t icnode, pcell_t *cells, int ncells) { - struct arm_intr_controller *pic; + struct arm_intr_controller *ic; struct arm_intr_handler *ih; - static char buffer[INTRNAME_LEN]; - static char name[INTRNAME_LEN]; - pic = &arm_pics[IRQ_PIC_IDX(irq)]; + debugf("map icnode %08x cells <%*D>\n", icnode, ncells * sizeof(pcell_t), + (char *)cells, ","); - if (pic == NULL) - return (""); + icnode = OF_node_from_xref(icnode); - if (IRQ_VECTOR_IDX(irq) > pic->ic_maxintrs) - return (""); + ic = ic_from_node(icnode); + if (ic == NULL) + ic = ic_create(icnode); - ih = &pic->ic_intrs[IRQ_VECTOR_IDX(irq)]; + ih = ih_from_fdtcells(ic, cells, ncells); + if (ih == NULL) { + ih = ic_add_ih(ic, cells, ncells); + if (ic->ic_dev != NULL) { + PIC_TRANSLATE(ic->ic_dev, ih->ih_cells, &ih->ih_hwirq, + &ih->ih_trig, &ih->ih_pol); + ic_index_ih_by_hwirq(ic, ih); + } + } + return (ih->ih_resirq); +} - if (pic->ic_dev == NULL) { +const char * +arm_describe_irq(int resirq) +{ + struct arm_intr_controller *ic; + struct arm_intr_handler *ih; + int irqidx; + static char buffer[INTRNAME_LEN]; + + /* XXX static buffer, can this be called after APs released? */ + + irqidx = resirq_decode(resirq, &ic, &ih); + KASSERT(ic != NULL, ("%s: bad resirq 0x%08x", resirq)); + if (ic->ic_dev == NULL) { /* * Interrupt controller not attached yet. We don't know the * IC device name nor interrupt number. All we can do is to - * use FDT 'name' property. + * use its index (fdt names are unbounded length). */ - OF_getprop(ih->ih_pic->ic_node, "name", name, sizeof(name)); - snprintf(buffer, sizeof(buffer), "%s.?", name); - return (buffer); + snprintf(buffer, sizeof(buffer), "ic%d.%d", ic->ic_idx, irqidx); + } else { + KASSERT(ih != NULL, ("%s: no handler for resirq 0x%08x\n", resirq)); + snprintf(buffer, sizeof(buffer), "%s.%d", + device_get_nameunit(ih->ih_ic->ic_dev), ih->ih_hwirq); } - - snprintf(buffer, sizeof(buffer), "%s.%d", - device_get_nameunit(ih->ih_pic->ic_dev), ih->ih_irq); - return (buffer); } void arm_register_pic(device_t dev, int flags) { - struct arm_intr_controller *ic = NULL; + struct arm_intr_controller *ic; struct arm_intr_handler *ih; phandle_t node; - int i; node = ofw_bus_get_node(dev); - - /* Find room for IC */ - for (i = 0; i < NPIC; i++) { - if (arm_pics[i].ic_dev != NULL) - continue; - - if (arm_pics[i].ic_node == node) { - ic = &arm_pics[i]; - break; - } - - if (arm_pics[i].ic_node == 0) { - ic = &arm_pics[i]; - break; - } - } - + ic = ic_from_node(node); if (ic == NULL) - panic("not enough room to register interrupt controller"); + ic = ic_create(node); - ic->ic_dev = dev; - ic->ic_node = node; + ic_setup_dev(ic, dev, MAXINTRS); /* - * Normally ic_intrs is allocated by arm_fdt_map_irq(), but the nexus - * root isn't described by fdt data. If the node is -1 and the ic_intrs - * array hasn't yet been allocated, we're dealing with nexus, allocate a - * single entry for irq 0. + * The nexus root usually isn't described by fdt data. If the node is + * -1 and the number of interrupts added is zero and the device's + * name is "nexus", allocate a single entry for irq 0. */ - if (node == -1 && ic->ic_intrs == NULL) { - ic->ic_intrs = malloc(sizeof(struct arm_intr_handler), M_INTRNG, - M_WAITOK | M_ZERO); - ic->ic_maxintrs = 1; - ih = &ic->ic_intrs[0]; - ih->ih_pic = ic; - ih->ih_ncells = 0; - } + if (node == -1 && ic->ic_ih_count == 0 && + strcmp(device_get_name(dev), "nexus") == 0) { + ih = ic_add_ih(ic, NULL, 0); + ih->ih_hwirq = 0; + ic_index_ih_by_hwirq(ic, ih); + } debugf("device %s node %08x slot %d\n", device_get_nameunit(dev), - ic->ic_node, i); + ic->ic_node, ic->ic_idx); if (flags & PIC_FEATURE_IPI) { - if (arm_ipi_pic != NULL) - panic("there's already registered interrupt " - "controller for serving IPIs"); - + KASSERT(arm_ipi_pic == NULL, + ("controller for IPIs is already registered")); arm_ipi_pic = ic; } - /* Resolve IRQ numbers for interrupt handlers added earlier */ - for (i = 0; i < ic->ic_maxintrs; i++) { - ih = &ic->ic_intrs[i]; - - /* Map IRQ number */ - PIC_TRANSLATE(ic->ic_dev, ih->ih_cells, &ih->ih_irq, - &ih->ih_trig, &ih->ih_pol); - - debugf("translated to irq %d\n", ih->ih_irq); - } - - device_printf(dev, "registered as interrupt controller\n"); + /* + * arm_describe_irq() has to print fake names earlier when the device + * issn't registered yet, emit a string that has the same fake name in + * it, so that earlier output links to this device. + */ + device_printf(dev, "registered as interrupt controller ic%d\n", + ic->ic_idx); } void arm_setup_irqhandler(device_t dev, driver_filter_t *filt, - void (*hand)(void*), void *arg, int irq, int flags, void **cookiep) + void (*hand)(void*), void *arg, int resirq, int flags, void **cookiep) { - struct arm_intr_controller *pic; + struct arm_intr_controller *ic; struct arm_intr_handler *ih; const char *name; int error; - int ipi; + int irqidx; - if (irq < 0) - return; - - ipi = (flags & INTR_IPI) != 0; - KASSERT(!ipi || arm_ipi_pic != NULL, - ("No IPI pic setup when adding an IPI")); - pic = ipi ? arm_ipi_pic : &arm_pics[IRQ_PIC_IDX(irq)]; - ih = arm_lookup_intr_handler(pic->ic_dev, IRQ_VECTOR_IDX(irq)); - - if (ipi) { - name = (const char *)dev; - debugf("setup ipi %d\n", irq); + if (flags & INTR_IPI) { + ic = arm_ipi_pic; + irqidx = resirq; /* resirq is the same as hwirq for IPIs */ + KASSERT(ic != NULL, ("%s: no interrupt controller for IPIs", + __FUNCTION__)); + ih = ipi_handlers[irqidx]; + KASSERT(ih != NULL, + ("%s: interrupt handler for %s IPI %u not found", + __FUNCTION__, device_get_nameunit(ic->ic_dev), + irqidx)); + KASSERT(irqidx < ARM_IPI_COUNT, ("IPI number too big: %u", + irqidx)); + name = ipi_names[irqidx]; + debugf("setup ipi %u (%s)\n", irqidx, name); } else { + irqidx = resirq_decode(resirq, &ic, &ih); name = device_get_nameunit(dev); - debugf("setup irq %d on %s\n", IRQ_VECTOR_IDX(irq), - device_get_nameunit(pic->ic_dev)); + debugf("setup irq %s.%d on %s\n", + device_get_nameunit(ic->ic_dev), ih->ih_hwirq, name); } - debugf("pic %p, ih %p\n", pic, ih); - if (ih->ih_event == NULL) { - error = intr_event_create(&ih->ih_event, (void *)ih, 0, irq, + error = intr_event_create(&ih->ih_event, ih, 0, resirq, (mask_fn)arm_mask_irq, (mask_fn)arm_unmask_irq, - arm_eoi, NULL, "intr%d.%d:", IRQ_PIC_IDX(irq), - IRQ_VECTOR_IDX(irq)); - - if (error) + arm_eoi, NULL, "ic%d.%d:", ic->ic_idx, ih->ih_hwirq); + if (error) { + device_printf(dev, "intr_event_create() failed " + "for irq %s.%u\n", device_get_nameunit(ic->ic_dev), + ih->ih_hwirq); return; - - ih->ih_dev = dev; - ih->ih_ipi_name = ipi ? name : NULL; - ih->ih_pic = pic; - - arm_unmask_irq(ih); - - last_printed += - snprintf(intrnames + last_printed, + } + intrcnt_last_printed += 1 + + snprintf(intrnames + intrcnt_last_printed, INTRNAME_LEN, "%s:%d: %s", - device_get_nameunit(pic->ic_dev), - ih->ih_irq, name); - - last_printed++; - ih->ih_intrcnt_idx = intrcnt_index; - intrcnt_index++; - + device_get_nameunit(ic->ic_dev), irqidx, name); + + ih->ih_intrcnt_idx = intrcnt_index++; } + if (!TAILQ_EMPTY(&ih->ih_event->ie_handlers)) + arm_mask_irq(ih); intr_event_add_handler(ih->ih_event, name, filt, hand, arg, intr_priority(flags), flags, cookiep); - - /* Unmask IPIs immediately */ - if (ipi) - arm_unmask_irq(ih); + arm_unmask_irq(ih); } int -arm_remove_irqhandler(int irq, void *cookie) +arm_remove_irqhandler(int resirq, void *cookie) { - struct arm_intr_controller *pic; + struct arm_intr_controller *ic; struct arm_intr_handler *ih; int error; - if (irq < 0) - return (ENXIO); - - pic = &arm_pics[IRQ_PIC_IDX(irq)]; - ih = arm_lookup_intr_handler(pic->ic_dev, IRQ_VECTOR_IDX(irq)); - - if (ih->ih_event == NULL) - return (ENXIO); - + resirq_decode(resirq, &ic, &ih); arm_mask_irq(ih); error = intr_event_remove_handler(cookie); - if (!TAILQ_EMPTY(&ih->ih_event->ie_handlers)) arm_unmask_irq(ih); - return (error); } static void arm_mask_irq(void *arg) { struct arm_intr_handler *ih = (struct arm_intr_handler *)arg; - PIC_MASK(ih->ih_pic->ic_dev, ih->ih_irq); + PIC_MASK(ih->ih_ic->ic_dev, ih->ih_hwirq); } static void arm_unmask_irq(void *arg) { struct arm_intr_handler *ih = (struct arm_intr_handler *)arg; - PIC_UNMASK(ih->ih_pic->ic_dev, ih->ih_irq); + PIC_UNMASK(ih->ih_ic->ic_dev, ih->ih_hwirq); } static void arm_eoi(void *arg) { struct arm_intr_handler *ih = (struct arm_intr_handler *)arg; - PIC_EOI(ih->ih_pic->ic_dev, ih->ih_irq); + PIC_EOI(ih->ih_ic->ic_dev, ih->ih_hwirq); } int -arm_intrng_config_irq(int irq, enum intr_trigger trig, enum intr_polarity pol) +arm_intrng_config_irq(int resirq, enum intr_trigger trig, enum intr_polarity pol) { - struct arm_intr_controller *pic; + struct arm_intr_controller *ic; struct arm_intr_handler *ih; - pic = &arm_pics[IRQ_PIC_IDX(irq)]; - ih = arm_lookup_intr_handler(pic->ic_dev, IRQ_VECTOR_IDX(irq)); + resirq_decode(resirq, &ic, &ih); - if (ih == NULL) - return (ENXIO); - - return PIC_CONFIG(pic->ic_dev, ih->ih_irq, trig, pol); + return PIC_CONFIG(ic->ic_dev, ih->ih_hwirq, trig, pol); } #ifdef SMP + void +arm_ipi_map_irq(device_t dev, u_int ipi, u_int hwirq) +{ + struct arm_intr_controller *ic; + struct arm_intr_handler *ih; + + ic = ic_from_dev(dev); + KASSERT(ic != NULL, ("ipi controller not registered")); + + ih = ih_from_hwirq(ic, hwirq); + KASSERT(ih == NULL, ("handler already registered for IPI %u", ipi)); + + ih = ic_add_ih(ic, NULL, 0); + ih->ih_hwirq = hwirq; + ic_index_ih_by_hwirq(ic, ih); + ipi_handlers[ipi] = ih; + debugf("ipi %u mapped to %s.%u\n", ipi, device_get_nameunit(dev), hwirq); +} + +void arm_init_secondary_ic(void) { - KASSERT(arm_ipi_pic != NULL, ("no IPI PIC attached")); + KASSERT(arm_ipi_pic != NULL, ("%s: no IPI PIC attached", __FUNCTION__)); PIC_INIT_SECONDARY(arm_ipi_pic->ic_dev); } void pic_ipi_send(cpuset_t cpus, u_int ipi) { - KASSERT(arm_ipi_pic != NULL, ("no IPI PIC attached")); - PIC_IPI_SEND(arm_ipi_pic->ic_dev, cpus, ipi); + KASSERT(ipi < ARM_IPI_COUNT, ("invalid IPI %u", ipi)); + KASSERT(ipi_handlers[ipi] != NULL, ("no handler for IPI %u", ipi)); + PIC_IPI_SEND(ipi_handlers[ipi]->ih_ic->ic_dev, cpus, ipi); } void pic_ipi_clear(int ipi) { - - KASSERT(arm_ipi_pic != NULL, ("no IPI PIC attached")); - PIC_IPI_CLEAR(arm_ipi_pic->ic_dev, ipi); + + KASSERT(ipi < ARM_IPI_COUNT, ("invalid IPI %u", ipi)); + KASSERT(ipi_handlers[ipi] != NULL, ("no handler for IPI %u", ipi)); + PIC_IPI_CLEAR(ipi_handlers[ipi]->ih_ic->ic_dev, ipi); } int pic_ipi_read(int ipi) { - KASSERT(arm_ipi_pic != NULL, ("no IPI PIC attached")); + KASSERT(arm_ipi_pic != NULL, ("no IPI interrupt controller")); return (PIC_IPI_READ(arm_ipi_pic->ic_dev, ipi)); } void arm_unmask_ipi(int ipi) { - KASSERT(arm_ipi_pic != NULL, ("no IPI PIC attached")); - PIC_UNMASK(arm_ipi_pic->ic_dev, ipi); + KASSERT(ipi < ARM_IPI_COUNT, ("invalid IPI %u", ipi)); + KASSERT(ipi_handlers[ipi] != NULL, ("no handler for IPI %u", ipi)); + PIC_UNMASK(ipi_handlers[ipi]->ih_ic->ic_dev, ipi); } void arm_mask_ipi(int ipi) { - KASSERT(arm_ipi_pic != NULL, ("no IPI PIC attached")); - PIC_MASK(arm_ipi_pic->ic_dev, ipi); + KASSERT(ipi < ARM_IPI_COUNT, ("invalid IPI %u", ipi)); + KASSERT(ipi_handlers[ipi] != NULL, ("no handler for IPI %u", ipi)); + PIC_MASK(ipi_handlers[ipi]->ih_ic->ic_dev, ipi); } #endif void dosoftints(void); void dosoftints(void) { } /* * arm_irq_memory_barrier() * * Ensure all writes to device memory have reached devices before proceeding. * * This is intended to be called from the post-filter and post-thread routines * of an interrupt controller implementation. A peripheral device driver should * use bus_space_barrier() if it needs to ensure a write has reached the * hardware for some reason other than clearing interrupt conditions. * * The need for this function arises from the ARM weak memory ordering model. * Writes to locations mapped with the Device attribute bypass any caches, but * are buffered. Multiple writes to the same device will be observed by that * device in the order issued by the cpu. Writes to different devices may * appear at those devices in a different order than issued by the cpu. That * is, if the cpu writes to device A then device B, the write to device B could * complete before the write to device A. * * Consider a typical device interrupt handler which services the interrupt and * writes to a device status-acknowledge register to clear the interrupt before * returning. That write is posted to the L2 controller which "immediately" * places it in a store buffer and automatically drains that buffer. This can * be less immediate than you'd think... There may be no free slots in the store * buffers, so an existing buffer has to be drained first to make room. The * target bus may be busy with other traffic (such as DMA for various devices), * delaying the drain of the store buffer for some indeterminate time. While * all this delay is happening, execution proceeds on the CPU, unwinding its way * out of the interrupt call stack to the point where the interrupt driver code * is ready to EOI and unmask the interrupt. The interrupt controller may be * accessed via a faster bus than the hardware whose handler just ran; the write * to unmask and EOI the interrupt may complete quickly while the device write * to ack and clear the interrupt source is still lingering in a store buffer * waiting for access to a slower bus. With the interrupt unmasked at the * interrupt controller but still active at the device, as soon as interrupts * are enabled on the core the device re-interrupts immediately: now you've got * a spurious interrupt on your hands. * * The right way to fix this problem is for every device driver to use the * proper bus_space_barrier() calls in its interrupt handler. For ARM a single * barrier call at the end of the handler would work. This would have to be * done to every driver in the system, not just arm-specific drivers. * * Another potential fix is to map all device memory as Strongly-Ordered rather * than Device memory, which takes the store buffers out of the picture. This * has a pretty big impact on overall system performance, because each strongly * ordered memory access causes all L2 store buffers to be drained. * * A compromise solution is to have the interrupt controller implementation call * this function to establish a barrier between writes to the interrupt-source * device and writes to the interrupt controller device. * * This takes the interrupt number as an argument, and currently doesn't use it. * The plan is that maybe some day there is a way to flag certain interrupts as * "memory barrier safe" and we can avoid this overhead with them. */ void arm_irq_memory_barrier(uintptr_t irq) { dsb(); cpu_l2cache_drain_writebuf(); } Index: projects/arm_intrng/sys/arm/arm/mp_machdep.c =================================================================== --- projects/arm_intrng/sys/arm/arm/mp_machdep.c (revision 276093) +++ projects/arm_intrng/sys/arm/arm/mp_machdep.c (revision 276094) @@ -1,435 +1,434 @@ /*- * Copyright (c) 2011 Semihalf. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #ifdef VFP #include #endif #ifdef CPU_MV_PJ4B #include #include #endif #include "opt_smp.h" void *temp_pagetable; extern struct pcpu __pcpu[]; /* used to hold the AP's until we are ready to release them */ struct mtx ap_boot_mtx; struct pcb stoppcbs[MAXCPU]; /* # of Applications processors */ volatile int mp_naps; /* Set to 1 once we're ready to let the APs out of the pen. */ volatile int aps_ready = 0; static int ipi_handler(void *arg); void set_stackptrs(int cpu); /* Temporary variables for init_secondary() */ void *dpcpu[MAXCPU - 1]; /* Determine if we running MP machine */ int cpu_mp_probe(void) { CPU_SETOF(0, &all_cpus); return (platform_mp_probe()); } /* Start Application Processor via platform specific function */ static int check_ap(void) { uint32_t ms; for (ms = 0; ms < 2000; ++ms) { if ((mp_naps + 1) == mp_ncpus) return (0); /* success */ else DELAY(1000); } return (-2); } extern unsigned char _end[]; /* Initialize and fire up non-boot processors */ void cpu_mp_start(void) { int error, i; vm_offset_t temp_pagetable_va; vm_paddr_t addr, addr_end; mtx_init(&ap_boot_mtx, "ap boot", NULL, MTX_SPIN); /* Reserve memory for application processors */ for(i = 0; i < (mp_ncpus - 1); i++) dpcpu[i] = (void *)kmem_malloc(kernel_arena, DPCPU_SIZE, M_WAITOK | M_ZERO); temp_pagetable_va = (vm_offset_t)contigmalloc(L1_TABLE_SIZE, M_TEMP, 0, 0x0, 0xffffffff, L1_TABLE_SIZE, 0); addr = arm_physmem_kernaddr; addr_end = (vm_offset_t)&_end - KERNVIRTADDR + arm_physmem_kernaddr; addr_end &= ~L1_S_OFFSET; addr_end += L1_S_SIZE; bzero((void *)temp_pagetable_va, L1_TABLE_SIZE); for (addr = arm_physmem_kernaddr; addr <= addr_end; addr += L1_S_SIZE) { ((int *)(temp_pagetable_va))[addr >> L1_S_SHIFT] = L1_TYPE_S|L1_SHARED|L1_S_C|L1_S_B|L1_S_AP(AP_KRW)|L1_S_DOM(PMAP_DOMAIN_KERNEL)|addr; ((int *)(temp_pagetable_va))[(addr - arm_physmem_kernaddr + KERNVIRTADDR) >> L1_S_SHIFT] = L1_TYPE_S|L1_SHARED|L1_S_C|L1_S_B|L1_S_AP(AP_KRW)|L1_S_DOM(PMAP_DOMAIN_KERNEL)|addr; } #if defined(CPU_MV_PJ4B) /* Add ARMADAXP registers required for snoop filter initialization */ ((int *)(temp_pagetable_va))[MV_BASE >> L1_S_SHIFT] = L1_TYPE_S|L1_SHARED|L1_S_B|L1_S_AP(AP_KRW)|fdt_immr_pa; #endif temp_pagetable = (void*)(vtophys(temp_pagetable_va)); cpu_idcache_wbinv_all(); cpu_l2cache_wbinv_all(); /* Initialize boot code and start up processors */ platform_mp_start_ap(); /* Check if ap's started properly */ error = check_ap(); if (error) printf("WARNING: Some AP's failed to start\n"); else for (i = 1; i < mp_ncpus; i++) CPU_SET(i, &all_cpus); contigfree((void *)temp_pagetable_va, L1_TABLE_SIZE, M_TEMP); } /* Introduce rest of cores to the world */ void cpu_mp_announce(void) { } extern vm_paddr_t pmap_pa; void init_secondary(int cpu) { struct pcpu *pc; uint32_t loop_counter; int start = 0, end = 0; cpu_idcache_inv_all(); cpu_setup(NULL); setttb(pmap_pa); cpu_tlb_flushID(); pc = &__pcpu[cpu]; /* * pcpu_init() updates queue, so it should not be executed in parallel * on several cores */ while(mp_naps < (cpu - 1)) ; pcpu_init(pc, cpu, sizeof(struct pcpu)); dpcpu_init(dpcpu[cpu - 1], cpu); /* Provide stack pointers for other processor modes. */ set_stackptrs(cpu); /* Signal our startup to BSP */ atomic_add_rel_32(&mp_naps, 1); /* Spin until the BSP releases the APs */ while (!aps_ready) ; /* Initialize curthread */ KASSERT(PCPU_GET(idlethread) != NULL, ("no idle thread")); pc->pc_curthread = pc->pc_idlethread; pc->pc_curpcb = pc->pc_idlethread->td_pcb; set_curthread(pc->pc_idlethread); #ifdef VFP pc->pc_cpu = cpu; vfp_init(); #endif mtx_lock_spin(&ap_boot_mtx); atomic_add_rel_32(&smp_cpus, 1); if (smp_cpus == mp_ncpus) { /* enable IPI's, tlb shootdown, freezes etc */ atomic_store_rel_int(&smp_started, 1); } mtx_unlock_spin(&ap_boot_mtx); /* Enable ipi */ #ifdef IPI_IRQ_START start = IPI_IRQ_START; #ifdef IPI_IRQ_END end = IPI_IRQ_END; #else end = IPI_IRQ_START; #endif #endif - for (int i = start; i <= end; i++) + for (int i = 0; i < ARM_IPI_COUNT; i++) arm_unmask_ipi(i); enable_interrupts(PSR_I); loop_counter = 0; while (smp_started == 0) { DELAY(100); loop_counter++; if (loop_counter == 1000) CTR0(KTR_SMP, "AP still wait for smp_started"); } /* Start per-CPU event timers. */ cpu_initclocks_ap(); CTR0(KTR_SMP, "go into scheduler"); platform_mp_init_secondary(); /* Enter the scheduler */ sched_throw(NULL); panic("scheduler returned us to %s", __func__); /* NOTREACHED */ } static int ipi_handler(void *arg) { u_int cpu, ipi; cpu = PCPU_GET(cpuid); ipi = pic_ipi_read((int)arg); while ((ipi != 0x3ff)) { switch (ipi) { case IPI_RENDEZVOUS: CTR0(KTR_SMP, "IPI_RENDEZVOUS"); smp_rendezvous_action(); break; case IPI_AST: CTR0(KTR_SMP, "IPI_AST"); break; case IPI_STOP: /* * IPI_STOP_HARD is mapped to IPI_STOP so it is not * necessary to add it in the switch. */ CTR0(KTR_SMP, "IPI_STOP or IPI_STOP_HARD"); savectx(&stoppcbs[cpu]); /* * CPUs are stopped when entering the debugger and at * system shutdown, both events which can precede a * panic dump. For the dump to be correct, all caches * must be flushed and invalidated, but on ARM there's * no way to broadcast a wbinv_all to other cores. * Instead, we have each core do the local wbinv_all as * part of stopping the core. The core requesting the * stop will do the l2 cache flush after all other cores * have done their l1 flushes and stopped. */ cpu_idcache_wbinv_all(); /* Indicate we are stopped */ CPU_SET_ATOMIC(cpu, &stopped_cpus); /* Wait for restart */ while (!CPU_ISSET(cpu, &started_cpus)) cpu_spinwait(); CPU_CLR_ATOMIC(cpu, &started_cpus); CPU_CLR_ATOMIC(cpu, &stopped_cpus); CTR0(KTR_SMP, "IPI_STOP (restart)"); break; case IPI_PREEMPT: CTR1(KTR_SMP, "%s: IPI_PREEMPT", __func__); sched_preempt(curthread); break; case IPI_HARDCLOCK: CTR1(KTR_SMP, "%s: IPI_HARDCLOCK", __func__); hardclockintr(); break; case IPI_TLB: CTR1(KTR_SMP, "%s: IPI_TLB", __func__); cpufuncs.cf_tlb_flushID(); break; default: panic("Unknown IPI 0x%0x on cpu %d", ipi, curcpu); } pic_ipi_clear(ipi); ipi = pic_ipi_read(-1); } return (FILTER_HANDLED); } static void release_aps(void *dummy __unused) { uint32_t loop_counter; int start = 0, end = 0; if (mp_ncpus == 1) return; #ifdef IPI_IRQ_START start = IPI_IRQ_START; #ifdef IPI_IRQ_END end = IPI_IRQ_END; #else end = IPI_IRQ_START; #endif #endif - for (int i = start; i <= end; i++) { + for (int i = 0; i < ARM_IPI_COUNT; i++) { /* * IPI handler */ /* * Use 0xdeadbeef as the argument value for irq 0, * if we used 0, the intr code will give the trap frame * pointer instead. */ - arm_setup_irqhandler((device_t)"ipi", ipi_handler, NULL, (void *)i, i, + arm_setup_irqhandler(NULL, ipi_handler, NULL, (void *)i, i, INTR_TYPE_MISC | INTR_EXCL | INTR_IPI, NULL); - arm_unmask_ipi(i); } atomic_store_rel_int(&aps_ready, 1); printf("Release APs\n"); for (loop_counter = 0; loop_counter < 2000; loop_counter++) { if (smp_started) return; DELAY(1000); } printf("AP's not started\n"); } SYSINIT(start_aps, SI_SUB_SMP, SI_ORDER_FIRST, release_aps, NULL); struct cpu_group * cpu_topo(void) { return (smp_topo_1level(CG_SHARE_L2, mp_ncpus, 0)); } void cpu_mp_setmaxid(void) { platform_mp_setmaxid(); } /* Sending IPI */ void ipi_all_but_self(u_int ipi) { cpuset_t other_cpus; other_cpus = all_cpus; CPU_CLR(PCPU_GET(cpuid), &other_cpus); CTR2(KTR_SMP, "%s: ipi: %x", __func__, ipi); platform_ipi_send(other_cpus, ipi); } void ipi_cpu(int cpu, u_int ipi) { cpuset_t cpus; CPU_ZERO(&cpus); CPU_SET(cpu, &cpus); CTR3(KTR_SMP, "%s: cpu: %d, ipi: %x", __func__, cpu, ipi); platform_ipi_send(cpus, ipi); } void ipi_selected(cpuset_t cpus, u_int ipi) { CTR2(KTR_SMP, "%s: ipi: %x", __func__, ipi); platform_ipi_send(cpus, ipi); } void tlb_broadcast(int ipi) { if (smp_started) ipi_all_but_self(ipi); } Index: projects/arm_intrng/sys/arm/include/intr.h =================================================================== --- projects/arm_intrng/sys/arm/include/intr.h (revision 276093) +++ projects/arm_intrng/sys/arm/include/intr.h (revision 276094) @@ -1,130 +1,131 @@ /* $NetBSD: intr.h,v 1.7 2003/06/16 20:01:00 thorpej Exp $ */ /*- * Copyright (c) 1997 Mark Brinicombe. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. All advertising materials mentioning features or use of this software * must display the following acknowledgement: * This product includes software developed by Mark Brinicombe * for the NetBSD Project. * 4. The name of the company nor the name of the author may be used to * endorse or promote products derived from this software without specific * prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ * */ #ifndef _MACHINE_INTR_H_ #define _MACHINE_INTR_H_ #include #include #include #include "opt_global.h" #if defined(ARM_INTRNG) #define NIRQ 255 #define NPIC 16 #define INTR_CONTROLLER INTR_MD1 #define INTR_IPI INTR_MD2 #define CORE_PIC_IDX (0) #define CORE_PIC_NODE (0xffffffff) /* Interrupt controller features used in arm_register_pic(): */ #define PIC_FEATURE_IPI 0x1 void arm_register_pic(device_t dev, int features); void arm_unregister_pic(device_t dev); void arm_dispatch_irq(device_t dev, struct trapframe *tf, int irq); void arm_setup_irqhandler(device_t dev, int (*)(void*), void (*)(void*), void *, int, int, void **); int arm_remove_irqhandler(int, void *); int arm_intrng_config_irq(int, enum intr_trigger, enum intr_polarity); #ifdef SMP void arm_init_secondary_ic(void); +void arm_ipi_map_irq(device_t, u_int, u_int); void arm_unmask_ipi(int); void arm_mask_ipi(int); #endif #else /* XXX move to std.* files? */ #ifdef CPU_XSCALE_81342 #define NIRQ 128 #elif defined(CPU_XSCALE_PXA2X0) #include #define NIRQ IRQ_GPIO_MAX #elif defined(SOC_MV_DISCOVERY) #define NIRQ 96 #elif defined(CPU_ARM9) || defined(SOC_MV_KIRKWOOD) || \ defined(CPU_XSCALE_IXP435) #define NIRQ 64 #elif defined(CPU_CORTEXA) #define NIRQ 1020 #elif defined(CPU_KRAIT) #define NIRQ 288 #elif defined(CPU_ARM1136) || defined(CPU_ARM1176) #define NIRQ 128 #elif defined(SOC_MV_ARMADAXP) #define MAIN_IRQ_NUM 116 #define ERR_IRQ_NUM 32 #define ERR_IRQ (MAIN_IRQ_NUM) #define MSI_IRQ_NUM 32 #define MSI_IRQ (ERR_IRQ + ERR_IRQ_NUM) #define NIRQ (MAIN_IRQ_NUM + ERR_IRQ_NUM + MSI_IRQ_NUM) #else #define NIRQ 32 #endif int arm_get_next_irq(int); void arm_mask_irq(uintptr_t); void arm_unmask_irq(uintptr_t); void arm_setup_irqhandler(const char *, int (*)(void*), void (*)(void*), void *, int, int, void **); int arm_remove_irqhandler(int, void *); extern void (*arm_post_filter)(void *); extern int (*arm_config_irq)(int irq, enum intr_trigger trig, enum intr_polarity pol); void gic_init_secondary(void); #endif /* !ARM_INTRNG */ const char *arm_describe_irq(int irq); void arm_intrnames_init(void); void arm_irq_memory_barrier(uintptr_t); int gic_decode_fdt(uint32_t iparentnode, uint32_t *intrcells, int *interrupt, int *trig, int *pol); #ifdef FDT int arm_fdt_map_irq(phandle_t, pcell_t *, int); #endif #endif /* _MACHINE_INTR_H */ Index: projects/arm_intrng/sys/arm/include/smp.h =================================================================== --- projects/arm_intrng/sys/arm/include/smp.h (revision 276093) +++ projects/arm_intrng/sys/arm/include/smp.h (revision 276094) @@ -1,40 +1,44 @@ /* $FreeBSD$ */ #ifndef _MACHINE_SMP_H_ #define _MACHINE_SMP_H_ #include #include -#define IPI_AST 0 -#define IPI_PREEMPT 2 -#define IPI_RENDEZVOUS 3 -#define IPI_STOP 4 -#define IPI_STOP_HARD 4 -#define IPI_HARDCLOCK 6 -#define IPI_TLB 7 +enum { + IPI_AST, + IPI_PREEMPT, + IPI_RENDEZVOUS, + IPI_STOP, + IPI_STOP_HARD = IPI_STOP, /* These are synonyms on arm. */ + IPI_HARDCLOCK, + IPI_TLB, + ARM_IPI_COUNT +}; + void init_secondary(int cpu); void mpentry(void); void ipi_all_but_self(u_int ipi); void ipi_cpu(int cpu, u_int ipi); void ipi_selected(cpuset_t cpus, u_int ipi); /* PIC interface */ void pic_ipi_send(cpuset_t cpus, u_int ipi); void pic_ipi_clear(int ipi); int pic_ipi_read(int arg); /* Platform interface */ void platform_mp_setmaxid(void); int platform_mp_probe(void); void platform_mp_start_ap(void); void platform_mp_init_secondary(void); void platform_ipi_send(cpuset_t cpus, u_int ipi); /* global data in mp_machdep.c */ extern struct pcb stoppcbs[]; #endif /* !_MACHINE_SMP_H_ */