Index: sys/arm/allwinner/a10/a10_intc.c =================================================================== --- sys/arm/allwinner/a10/a10_intc.c +++ sys/arm/allwinner/a10/a10_intc.c @@ -266,7 +266,7 @@ } xref = OF_xref_from_node(ofw_bus_get_node(sc->sc_dev)); - error = intr_pic_register(sc->sc_dev, xref); + error = intr_pic_register(sc->sc_dev, xref, A10_INTR_MAX_NIRQS); if (error != 0) return (error); Index: sys/arm/allwinner/aw_nmi.c =================================================================== --- sys/arm/allwinner/aw_nmi.c +++ sys/arm/allwinner/aw_nmi.c @@ -362,7 +362,7 @@ device_get_nameunit(sc->dev), sc->intr.irq) != 0) goto error; - if (intr_pic_register(dev, (intptr_t)xref) != 0) { + if (intr_pic_register(dev, (intptr_t)xref, 1) != 0) { device_printf(dev, "could not register pic\n"); goto error; } Index: sys/arm/arm/gic.c =================================================================== --- sys/arm/arm/gic.c +++ sys/arm/arm/gic.c @@ -711,7 +711,7 @@ * Now, when everything is initialized, it's right time to * register interrupt controller to interrupt framefork. */ - if (intr_pic_register(dev, xref) != 0) { + if (intr_pic_register(dev, xref, sc->nirqs) != 0) { device_printf(dev, "could not register PIC\n"); goto cleanup; } Index: sys/arm/broadcom/bcm2835/bcm2835_gpio.c =================================================================== --- sys/arm/broadcom/bcm2835/bcm2835_gpio.c +++ sys/arm/broadcom/bcm2835/bcm2835_gpio.c @@ -1046,7 +1046,7 @@ return (error); /* XXX deregister ISRCs */ } return (intr_pic_register(sc->sc_dev, - OF_xref_from_node(ofw_bus_get_node(sc->sc_dev)))); + OF_xref_from_node(ofw_bus_get_node(sc->sc_dev)), BCM_GPIO_PINS)); } static int Index: sys/arm/broadcom/bcm2835/bcm2835_intr.c =================================================================== --- sys/arm/broadcom/bcm2835/bcm2835_intr.c +++ sys/arm/broadcom/bcm2835/bcm2835_intr.c @@ -341,7 +341,7 @@ if (error != 0) return (error); } - return (intr_pic_register(sc->sc_dev, xref)); + return (intr_pic_register(sc->sc_dev, xref, BCM_INTC_NIRQS)); } #endif Index: sys/arm/broadcom/bcm2835/bcm2836.c =================================================================== --- sys/arm/broadcom/bcm2835/bcm2836.c +++ sys/arm/broadcom/bcm2835/bcm2836.c @@ -653,7 +653,7 @@ } xref = OF_xref_from_node(ofw_bus_get_node(sc->bls_dev)); - error = intr_pic_register(sc->bls_dev, xref); + error = intr_pic_register(sc->bls_dev, xref, BCM_LINTC_NIRQS); if (error != 0) return (error); Index: sys/arm/freescale/imx/imx_gpio.c =================================================================== --- sys/arm/freescale/imx/imx_gpio.c +++ sys/arm/freescale/imx/imx_gpio.c @@ -681,7 +681,8 @@ #ifdef INTRNG gpio_pic_register_isrcs(sc); - intr_pic_register(dev, OF_xref_from_node(ofw_bus_get_node(dev))); + intr_pic_register(dev, OF_xref_from_node(ofw_bus_get_node(dev)), + sc->gpio_npins); #endif sc->sc_busdev = gpiobus_attach_bus(dev); Index: sys/arm/mv/mpic.c =================================================================== --- sys/arm/mv/mpic.c +++ sys/arm/mv/mpic.c @@ -274,7 +274,7 @@ bus_release_resources(dev, mv_mpic_spec, sc->mpic_res); return (ENXIO); } - if (intr_pic_register(dev, OF_xref_from_device(dev)) != 0) { + if (intr_pic_register(dev, OF_xref_from_device(dev), sc->nirqs) != 0) { device_printf(dev, "could not register PIC\n"); bus_release_resources(dev, mv_mpic_spec, sc->mpic_res); return (ENXIO); Index: sys/arm/nvidia/tegra_gpio.c =================================================================== --- sys/arm/nvidia/tegra_gpio.c +++ sys/arm/nvidia/tegra_gpio.c @@ -453,7 +453,7 @@ return (error); /* XXX deregister ISRCs */ } return (intr_pic_register(sc->dev, - OF_xref_from_node(ofw_bus_get_node(sc->dev)))); + OF_xref_from_node(ofw_bus_get_node(sc->dev)), sc->gpio_npins)); } static int Index: sys/arm/ti/aintc.c =================================================================== --- sys/arm/ti/aintc.c +++ sys/arm/ti/aintc.c @@ -236,7 +236,7 @@ } xref = OF_xref_from_node(ofw_bus_get_node(sc->sc_dev)); - error = intr_pic_register(sc->sc_dev, xref); + error = intr_pic_register(sc->sc_dev, xref, INTC_NIRQS); if (error != 0) return (error); Index: sys/arm/ti/omap4/omap4_wugen.c =================================================================== --- sys/arm/ti/omap4/omap4_wugen.c +++ sys/arm/ti/omap4/omap4_wugen.c @@ -185,6 +185,7 @@ phandle_t node; phandle_t parent_xref; int rid, rv; + u_int irq_count; sc = device_get_softc(dev); sc->sc_dev = dev; @@ -210,7 +211,12 @@ return (ENXIO); } - if (intr_pic_register(dev, OF_xref_from_node(node)) != 0) { + if (intr_pic_intr_count(sc->sc_parent, &irq_count) != 0) { + device_printf(dev, + "unable to find interrupt parent interrupts\n"); + goto fail; + } + if (intr_pic_register(dev, OF_xref_from_node(node), irq_count) != 0) { device_printf(dev, "can't register PIC\n"); goto fail; } Index: sys/arm/ti/ti_gpio.c =================================================================== --- sys/arm/ti/ti_gpio.c +++ sys/arm/ti/ti_gpio.c @@ -905,7 +905,7 @@ return (error); /* XXX deregister ISRCs */ } return (intr_pic_register(sc->sc_dev, - OF_xref_from_node(ofw_bus_get_node(sc->sc_dev)))); + OF_xref_from_node(ofw_bus_get_node(sc->sc_dev)), sc->sc_maxpin)); } static int Index: sys/arm64/arm64/gic_v3_fdt.c =================================================================== --- sys/arm64/arm64/gic_v3_fdt.c +++ sys/arm64/arm64/gic_v3_fdt.c @@ -139,7 +139,7 @@ #ifdef INTRNG xref = OF_xref_from_node(ofw_bus_get_node(dev)); - if (intr_pic_register(dev, xref) != 0) { + if (intr_pic_register(dev, xref, sc->gic_nirqs) != 0) { device_printf(dev, "could not register PIC\n"); goto error; } Index: sys/kern/subr_intr.c =================================================================== --- sys/kern/subr_intr.c +++ sys/kern/subr_intr.c @@ -101,17 +101,20 @@ #define FLAG_PIC (1 << 0) #define FLAG_MSI (1 << 1) u_int pic_flags; + u_int pic_irq_base; + u_int pic_irq_next; + u_int pic_irq_count; + struct intr_irqsrc ** pic_sources; }; static struct mtx pic_list_lock; static SLIST_HEAD(, intr_pic) pic_list; +static u_int pic_next_irq_base; static struct intr_pic *pic_lookup(device_t dev, intptr_t xref); /* Interrupt source definition. */ static struct mtx isrc_table_lock; -static struct intr_irqsrc *irq_sources[NIRQ]; -u_int irq_next_free; /* * XXX - All stuff around struct intr_dev_data is considered as temporary @@ -140,7 +143,6 @@ static u_int intr_ddata_first_unused; #define IRQ_DDATA_BASE 10000 -CTASSERT(IRQ_DDATA_BASE > nitems(irq_sources)); #ifdef SMP static boolean_t irq_assign_cpu = FALSE; @@ -355,35 +357,35 @@ * constantly... */ static inline int -isrc_alloc_irq(struct intr_irqsrc *isrc) +isrc_alloc_irq(struct intr_pic *pic, struct intr_irqsrc *isrc) { u_int maxirqs, irq; mtx_assert(&isrc_table_lock, MA_OWNED); - maxirqs = nitems(irq_sources); - if (irq_next_free >= maxirqs) + maxirqs = pic->pic_irq_base + pic->pic_irq_count; + if (pic->pic_irq_next >= maxirqs) return (ENOSPC); - for (irq = irq_next_free; irq < maxirqs; irq++) { - if (irq_sources[irq] == NULL) + for (irq = pic->pic_irq_next; irq < maxirqs; irq++) { + if (pic->pic_sources[irq] == NULL) goto found; } - for (irq = 0; irq < irq_next_free; irq++) { - if (irq_sources[irq] == NULL) + for (irq = pic->pic_irq_base; irq < pic->pic_irq_next; irq++) { + if (pic->pic_sources[irq] == NULL) goto found; } - irq_next_free = maxirqs; + pic->pic_irq_next = maxirqs; return (ENOSPC); found: isrc->isrc_irq = irq; - irq_sources[irq] = isrc; + pic->pic_sources[irq - pic->pic_irq_base] = isrc; - irq_next_free = irq + 1; - if (irq_next_free >= maxirqs) - irq_next_free = 0; + pic->pic_irq_next = irq + 1; + if (pic->pic_irq_next >= maxirqs) + pic->pic_irq_next = pic->pic_irq_base; return (0); } @@ -391,17 +393,19 @@ * Free unique interrupt number (resource handle) from interrupt source. */ static inline int -isrc_free_irq(struct intr_irqsrc *isrc) +isrc_free_irq(struct intr_pic *pic, struct intr_irqsrc *isrc) { + u_int irq; mtx_assert(&isrc_table_lock, MA_OWNED); - if (isrc->isrc_irq >= nitems(irq_sources)) + irq = isrc->isrc_irq - pic->pic_irq_base; + if (irq > pic->pic_irq_count) return (EINVAL); - if (irq_sources[isrc->isrc_irq] != isrc) + if (pic->pic_sources[irq] != isrc) return (EINVAL); - irq_sources[isrc->isrc_irq] = NULL; + pic->pic_sources[irq] = NULL; isrc->isrc_irq = INTR_IRQ_INVALID; /* just to be safe */ return (0); } @@ -412,9 +416,13 @@ static inline struct intr_irqsrc * isrc_lookup(u_int irq) { + struct intr_pic *pic; + + SLIST_FOREACH(pic, &pic_list, pic_next) { + if (irq >= pic->pic_irq_base && irq < pic->pic_irq_count) + return (pic->pic_sources[irq - pic->pic_irq_base]); + } - if (irq < nitems(irq_sources)) - return (irq_sources[irq]); return (NULL); } @@ -425,9 +433,14 @@ intr_isrc_register(struct intr_irqsrc *isrc, device_t dev, u_int flags, const char *fmt, ...) { + struct intr_pic *pic; int error; va_list ap; + pic = pic_lookup(isrc->isrc_dev, 0); + /* This can only happen if the pic disappears somehow */ + KASSERT(pic != NULL, ("%s: Unable to find isrc pic", __func__)); + bzero(isrc, sizeof(struct intr_irqsrc)); isrc->isrc_dev = dev; isrc->isrc_irq = INTR_IRQ_INVALID; /* just to be safe */ @@ -438,7 +451,7 @@ va_end(ap); mtx_lock(&isrc_table_lock); - error = isrc_alloc_irq(isrc); + error = isrc_alloc_irq(pic, isrc); if (error != 0) { mtx_unlock(&isrc_table_lock); return (error); @@ -460,12 +473,17 @@ int intr_isrc_deregister(struct intr_irqsrc *isrc) { + struct intr_pic *pic; int error; + pic = pic_lookup(isrc->isrc_dev, 0); + /* This can only happen if the pic disappears somehow */ + KASSERT(pic != NULL, ("%s: Unable to find isrc pic", __func__)); + mtx_lock(&isrc_table_lock); if ((isrc->isrc_flags & INTR_ISRCF_IPI) == 0) isrc_release_counters(isrc); - error = isrc_free_irq(isrc); + error = isrc_free_irq(pic, isrc); mtx_unlock(&isrc_table_lock); return (error); } @@ -912,7 +930,7 @@ * Register interrupt controller. */ int -intr_pic_register(device_t dev, intptr_t xref) +intr_pic_register(device_t dev, intptr_t xref, u_int count) { struct intr_pic *pic; @@ -922,8 +940,21 @@ if (pic == NULL) return (ENOMEM); + pic->pic_sources = malloc(sizeof(*pic->pic_sources) * count, M_INTRNG, + M_WAITOK | M_ZERO); + pic->pic_irq_base = atomic_fetchadd_int(&pic_next_irq_base, count); + pic->pic_irq_next = pic->pic_irq_base; + pic->pic_irq_count = count; + + KASSERT((pic->pic_irq_base + count) < IRQ_DDATA_BASE, + ("%s: PIC %s is attempting to handle device data irqs, " + "base: %u count: %u", __func__, device_get_nameunit(dev), + pic->pic_irq_base, count)); + + pic->pic_flags |= FLAG_PIC; + debugf("PIC %p registered for %s \n", pic, device_get_nameunit(dev), dev, xref); return (0); @@ -992,6 +1023,24 @@ } int +intr_pic_intr_count(device_t dev, u_int *count) +{ + struct intr_pic *pic; + + pic = pic_lookup(dev, 0); + if (pic == NULL) + return (ESRCH); + + KASSERT((pic->pic_flags & FLAG_PIC) != 0, + ("%s: Found a non-PIC controller: %s", __func__, + device_get_name(pic->pic_dev))); + + *count = pic->pic_irq_count; + + return (0); +} + +int intr_map_irq(device_t dev, intptr_t xref, struct intr_map_data *data, u_int *irqp) { @@ -1227,23 +1276,16 @@ return (last_cpu); } -/* - * Distribute all the interrupt sources among the available - * CPUs once the AP's have been launched. - */ static void -intr_irq_shuffle(void *arg __unused) +intr_irq_shuffle_pic(struct intr_pic *pic) { struct intr_irqsrc *isrc; u_int i; - if (mp_ncpus == 1) - return; + mtx_assert(&isrc_table_lock, MA_OWNED); - mtx_lock(&isrc_table_lock); - irq_assign_cpu = TRUE; - for (i = 0; i < NIRQ; i++) { - isrc = irq_sources[i]; + for (i = 0; i < pic->pic_irq_count; i++) { + isrc = pic->pic_sources[i]; if (isrc == NULL || isrc->isrc_handlers == 0 || isrc->isrc_flags & (INTR_ISRCF_PPI | INTR_ISRCF_IPI)) continue; @@ -1264,6 +1306,26 @@ if (PIC_BIND_INTR(isrc->isrc_dev, isrc) != 0) CPU_ZERO(&isrc->isrc_cpu); } +} + +/* + * Distribute all the interrupt sources among the available + * CPUs once the AP's have been launched. + */ +static void +intr_irq_shuffle(void *arg __unused) +{ + struct intr_pic *pic; + + if (mp_ncpus == 1) + return; + + mtx_lock(&isrc_table_lock); + irq_assign_cpu = TRUE; + + SLIST_FOREACH(pic, &pic_list, pic_next) { + intr_irq_shuffle_pic(pic); + } mtx_unlock(&isrc_table_lock); } SYSINIT(intr_irq_shuffle, SI_SUB_SMP, SI_ORDER_SECOND, intr_irq_shuffle, NULL); @@ -1459,20 +1521,26 @@ #ifdef DDB DB_SHOW_COMMAND(irqs, db_show_irqs) { + struct intr_pic *pic; u_int i, irqsum; u_long num; struct intr_irqsrc *isrc; - for (irqsum = 0, i = 0; i < NIRQ; i++) { - isrc = irq_sources[i]; - if (isrc == NULL) - continue; - - num = isrc->isrc_count != NULL ? isrc->isrc_count[0] : 0; - db_printf("irq%-3u <%s>: cpu %02lx%s cnt %lu\n", i, - isrc->isrc_name, isrc->isrc_cpu.__bits[0], - isrc->isrc_flags & INTR_ISRCF_BOUND ? " (bound)" : "", num); - irqsum += num; + irqsum = 0; + SLIST_FOREACH(pic, &pic_list, pic_next) { + for (i = 0; i < pic->pic_irq_count; i++) { + isrc = pic->pic_sources[i]; + if (isrc == NULL) + continue; + + num = + isrc->isrc_count != NULL ? isrc->isrc_count[0] : 0; + db_printf("irq%-3u <%s>: cpu %02lx%s cnt %lu\n", i, + isrc->isrc_name, isrc->isrc_cpu.__bits[0], + isrc->isrc_flags & INTR_ISRCF_BOUND ? + " (bound)" : "", num); + irqsum += num; + } } db_printf("irq total %u\n", irqsum); } Index: sys/mips/mediatek/mtk_gpio_v1.c =================================================================== --- sys/mips/mediatek/mtk_gpio_v1.c +++ sys/mips/mediatek/mtk_gpio_v1.c @@ -308,7 +308,7 @@ goto fail; } - if (intr_pic_register(dev, OF_xref_from_node(node)) != 0) { + if (intr_pic_register(dev, OF_xref_from_node(node), sc->num_pins) != 0){ device_printf(dev, "could not register PIC\n"); goto fail; } Index: sys/mips/mediatek/mtk_gpio_v2.c =================================================================== --- sys/mips/mediatek/mtk_gpio_v2.c +++ sys/mips/mediatek/mtk_gpio_v2.c @@ -299,7 +299,7 @@ goto fail; } - if (intr_pic_register(dev, OF_xref_from_node(node)) != 0) { + if (intr_pic_register(dev, OF_xref_from_node(node), sc->num_pins) != 0){ device_printf(dev, "could not register PIC\n"); goto fail; } Index: sys/mips/mediatek/mtk_intr_gic.c =================================================================== --- sys/mips/mediatek/mtk_intr_gic.c +++ sys/mips/mediatek/mtk_intr_gic.c @@ -213,7 +213,7 @@ * Now, when everything is initialized, it's right time to * register interrupt controller to interrupt framefork. */ - if (intr_pic_register(dev, xref) != 0) { + if (intr_pic_register(dev, xref, sc->nirqs) != 0) { device_printf(dev, "could not register PIC\n"); goto cleanup; } Index: sys/mips/mediatek/mtk_intr_v1.c =================================================================== --- sys/mips/mediatek/mtk_intr_v1.c +++ sys/mips/mediatek/mtk_intr_v1.c @@ -201,7 +201,7 @@ * Now, when everything is initialized, it's right time to * register interrupt controller to interrupt framefork. */ - if (intr_pic_register(dev, xref) != 0) { + if (intr_pic_register(dev, xref, sc->nirqs, sc->nirqs) != 0) { device_printf(dev, "could not register PIC\n"); goto cleanup; } Index: sys/mips/mediatek/mtk_intr_v2.c =================================================================== --- sys/mips/mediatek/mtk_intr_v2.c +++ sys/mips/mediatek/mtk_intr_v2.c @@ -196,7 +196,7 @@ * Now, when everything is initialized, it's right time to * register interrupt controller to interrupt framefork. */ - if (intr_pic_register(dev, xref) != 0) { + if (intr_pic_register(dev, xref, sc->nirqs) != 0) { device_printf(dev, "could not register PIC\n"); goto cleanup; } Index: sys/mips/mediatek/mtk_pcie.c =================================================================== --- sys/mips/mediatek/mtk_pcie.c +++ sys/mips/mediatek/mtk_pcie.c @@ -319,7 +319,7 @@ } /* Register ourselves as an interrupt controller */ - if (intr_pic_register(dev, xref) != 0) { + if (intr_pic_register(dev, xref, sc->sc_num_irq) != 0) { device_printf(dev, "could not register PIC\n"); goto cleanup_rman; } Index: sys/mips/mips/mips_pic.c =================================================================== --- sys/mips/mips/mips_pic.c +++ sys/mips/mips/mips_pic.c @@ -223,7 +223,7 @@ * Now, when everything is initialized, it's right time to * register interrupt controller to interrupt framefork. */ - if (intr_pic_register(dev, xref) != 0) { + if (intr_pic_register(dev, xref, NREAL_IRQS) != 0) { device_printf(dev, "could not register PIC\n"); goto cleanup; } Index: sys/sys/intr.h =================================================================== --- sys/sys/intr.h +++ sys/sys/intr.h @@ -110,9 +110,10 @@ int intr_isrc_dispatch(struct intr_irqsrc *, struct trapframe *); u_int intr_irq_next_cpu(u_int current_cpu, cpuset_t *cpumask); -int intr_pic_register(device_t, intptr_t); +int intr_pic_register(device_t, intptr_t, u_int); int intr_pic_deregister(device_t, intptr_t); int intr_pic_claim_root(device_t, intptr_t, intr_irq_filter_t *, void *, u_int); +int intr_pic_intr_count(device_t, u_int *); extern device_t intr_irq_root_dev;