diff --git a/sys/kern/pic_if.m b/sys/kern/pic_if.m --- a/sys/kern/pic_if.m +++ b/sys/kern/pic_if.m @@ -89,6 +89,14 @@ return (EOPNOTSUPP); } + + static int + dflt_pic_ipi_setup_per_cpu(device_t dev, u_int ipi, + struct intr_irqsrc *isrc, u_int cpu) + { + + return (EOPNOTSUPP); + } }; METHOD int activate_intr { @@ -172,3 +180,10 @@ u_int ipi; struct intr_irqsrc **isrcp; } DEFAULT dflt_pic_ipi_setup; + +METHOD int ipi_setup_per_cpu { + device_t dev; + u_int ipi; + struct intr_irqsrc **isrcp; + u_int cpu; +} DEFAULT dflt_pic_ipi_setup_per_cpu; diff --git a/sys/kern/subr_intr.c b/sys/kern/subr_intr.c --- a/sys/kern/subr_intr.c +++ b/sys/kern/subr_intr.c @@ -137,13 +137,18 @@ struct intr_ipi { intr_ipi_handler_t *ii_handler; void *ii_handler_arg; - struct intr_irqsrc *ii_isrc; + union { + struct intr_irqsrc **ii_isrc_per_cpu; + struct intr_irqsrc *ii_isrc; + }; char ii_name[INTR_IPI_NAMELEN]; u_long *ii_count; + bool ii_isrc_is_per_cpu; }; static device_t intr_ipi_dev; static u_int intr_ipi_dev_priority; +static uint32_t intr_ipi_flags; static bool intr_ipi_dev_frozen; #endif @@ -1840,7 +1845,7 @@ } int -intr_ipi_pic_register(device_t dev, u_int priority) +intr_ipi_pic_register_flags(device_t dev, u_int priority, uint32_t flags) { if (intr_ipi_dev_frozen) { device_printf(dev, "IPI device already frozen"); @@ -1850,11 +1855,18 @@ if (intr_ipi_dev == NULL || priority > intr_ipi_dev_priority) { intr_ipi_dev_priority = priority; intr_ipi_dev = dev; + intr_ipi_flags = flags; } return (0); } +int +intr_ipi_pic_register(device_t dev, u_int priority) +{ + return (intr_ipi_pic_register_flags(dev, priority, 0)); +} + /* * Setup IPI handler on interrupt controller. * @@ -1866,7 +1878,7 @@ { struct intr_irqsrc *isrc; struct intr_ipi *ii; - int error; + int cpu, error; if (!intr_ipi_dev_frozen) { if (intr_ipi_dev == NULL) @@ -1878,22 +1890,52 @@ KASSERT(hand != NULL, ("%s: ipi %u no handler", __func__, ipi)); - error = PIC_IPI_SETUP(intr_ipi_dev, ipi, &isrc); - if (error != 0) - return; - - isrc->isrc_handlers++; - ii = intr_ipi_lookup(ipi); KASSERT(ii->ii_count == NULL, ("%s: ipi %u reused", __func__, ipi)); + ii->ii_isrc_is_per_cpu = (intr_ipi_flags & INTR_IPI_PER_CPU) != 0; + + if (!ii->ii_isrc_is_per_cpu) { + error = PIC_IPI_SETUP(intr_ipi_dev, ipi, &isrc); + if (error != 0) + return; + + ii->ii_isrc = isrc; + isrc->isrc_handlers++; + } else { + ii->ii_isrc_per_cpu = mallocarray(mp_maxid + 1, + sizeof(*ii->ii_isrc_per_cpu), M_INTRNG, M_WAITOK | M_ZERO); + + /* + * all_cpus may not have been initialized when this is called + * so CPU_FOREACH won't work. + */ + for (cpu = 0; cpu <= mp_maxid; cpu++) { + error = PIC_IPI_SETUP_PER_CPU(intr_ipi_dev, ipi, &isrc, + cpu); + if (error != 0) { + free(ii->ii_isrc_per_cpu, M_INTRNG); + ii->ii_isrc_per_cpu = NULL; + return; + } + + ii->ii_isrc_per_cpu[cpu] = isrc; + isrc->isrc_handlers++; + } + } + ii->ii_handler = hand; ii->ii_handler_arg = arg; - ii->ii_isrc = isrc; strlcpy(ii->ii_name, name, INTR_IPI_NAMELEN); ii->ii_count = intr_ipi_setup_counters(name); - PIC_ENABLE_INTR(intr_ipi_dev, isrc); + if (!ii->ii_isrc_is_per_cpu) { + PIC_ENABLE_INTR(intr_ipi_dev, ii->ii_isrc); + } else { + for (cpu = 0; cpu <= mp_maxid; cpu++) { + PIC_ENABLE_INTR(intr_ipi_dev, ii->ii_isrc_per_cpu[cpu]); + } + } } void @@ -1921,7 +1963,16 @@ dsb(ishst); #endif - PIC_IPI_SEND(intr_ipi_dev, ii->ii_isrc, cpus, ipi); + if (ii->ii_isrc_is_per_cpu) { + u_int cpu; + + CPU_FOREACH_ISSET(cpu, &cpus) { + PIC_IPI_SEND(intr_ipi_dev, ii->ii_isrc_per_cpu[cpu], + cpus, ipi); + } + } else { + PIC_IPI_SEND(intr_ipi_dev, ii->ii_isrc, cpus, ipi); + } } /* diff --git a/sys/sys/intr.h b/sys/sys/intr.h --- a/sys/sys/intr.h +++ b/sys/sys/intr.h @@ -159,6 +159,10 @@ #ifdef SMP typedef void intr_ipi_handler_t(void *); +/* Flags for intr_ipi_pic_register_flags */ +#define INTR_IPI_PER_CPU (0x1u << 0) + +int intr_ipi_pic_register_flags(device_t dev, u_int priority, uint32_t flags); int intr_ipi_pic_register(device_t dev, u_int priority); void intr_ipi_setup(u_int ipi, const char *name, intr_ipi_handler_t *hand, void *arg);