diff --git a/sys/kern/kern_intr.c b/sys/kern/kern_intr.c --- a/sys/kern/kern_intr.c +++ b/sys/kern/kern_intr.c @@ -3,6 +3,7 @@ * * Copyright (c) 1997, Stefan Esser * All rights reserved. + * Copyright © 2022-2023 Elliott Mitchell * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -286,10 +287,11 @@ struct intr_event *ie; va_list ap; - /* The only valid flag during creation is IE_SOFT. */ - if ((flags & ~IE_SOFT) != 0) + /* The flags valid during creation are IE_SOFT and IE_MULTIPROC. */ + if ((flags & ~(IE_SOFT | IE_MULTIPROC)) != 0) return (EINVAL); - ie = malloc(sizeof(struct intr_event), M_ITHREAD, M_WAITOK | M_ZERO); + ie = malloc(sizeof(struct intr_event) + (flags & IE_MULTIPROC ? + sizeof(u_long) * (mp_ncpus - 1) : 0), M_ITHREAD, M_WAITOK | M_ZERO); ie->ie_source = source; ie->ie_pre_ithread = pre_ithread; ie->ie_post_ithread = post_ithread; @@ -298,6 +300,7 @@ ie->ie_flags = flags; ie->ie_irq = irq; ie->ie_cpu = NOCPU; + /* ->ie_stray and ->ie_intrcnt[] handled by M_ZERO */ CK_SLIST_INIT(&ie->ie_handlers); mtx_init(&ie->ie_lock, "intr event", NULL, MTX_DEF); @@ -1059,18 +1062,22 @@ void *arg, int pri, enum intr_type flags, void **cookiep) { struct intr_event *ie; + int eflags = IE_SOFT; int error = 0; if (flags & INTR_ENTROPY) return (EINVAL); + if (flags & INTR_MULTIPROC) + eflags |= IE_MULTIPROC; + ie = (eventp != NULL) ? *eventp : NULL; if (ie != NULL) { if (!(ie->ie_flags & IE_SOFT)) return (EINVAL); } else { - error = intr_event_create(&ie, NULL, IE_SOFT, 0, + error = intr_event_create(&ie, NULL, eflags, 0, NULL, NULL, NULL, swi_assign_cpu, "swi%d:", pri); if (error) return (error); @@ -1370,9 +1377,29 @@ intr_prof_stack_use(td, frame); #endif - /* An interrupt with no event or handlers is a stray interrupt. */ - if (ie == NULL || CK_SLIST_EMPTY(&ie->ie_handlers)) + /* An interrupt with no event is a stray interrupt. */ + if (ie == NULL) + return (EINVAL); + + /* + * Note about interrupt counters. Having per-processor counters avoids + * the need for atomic increment of counters. Whereas other interrupt + * types will be bound to a single processor and not need atomics. + * Per-processor interrupts are *never* flagged as stray, so stray + * counters don't need atomic increment. + */ + + /* Increment the interrupt counter. */ + if (__predict_false(ie->ie_flags & IE_MULTIPROC)) + ++ie->ie_intrcnt[curcpu]; + else + ++ie->ie_intrcnt[0]; + + /* An interrupt with no handlers is a stray interrupt. */ + if (CK_SLIST_EMPTY(&ie->ie_handlers)) { + ++ie->ie_stray; return (EINVAL); + } /* * Execute fast interrupt handlers directly. @@ -1473,8 +1500,10 @@ td->td_intr_nesting_level--; #ifdef notyet /* The interrupt is not aknowledged by any filter and has no ithread. */ - if (!thread && !filter) + if (!thread && !filter) { + ++ie->ie_stray; return (EINVAL); + } #endif return (0); } @@ -1625,8 +1654,11 @@ start_softintr(void *dummy) { + KASSERT(clk_intr_event == NULL, ("clk_intr_event non-NULL at %s()", + __func__)); + if (swi_add(&clk_intr_event, "clk", NULL, NULL, SWI_CLOCK, - INTR_MPSAFE, NULL)) + INTR_MPSAFE | INTR_MULTIPROC, NULL)) panic("died while creating clk swi ithread"); } SYSINIT(start_softintr, SI_SUB_SOFTINTR, SI_ORDER_FIRST, start_softintr, diff --git a/sys/kern/subr_intr.c b/sys/kern/subr_intr.c --- a/sys/kern/subr_intr.c +++ b/sys/kern/subr_intr.c @@ -2,7 +2,7 @@ * Copyright (c) 2015-2016 Svatopluk Kraus * Copyright (c) 2015-2016 Michal Meloun * All rights reserved. - * Copyright © 2022 Elliott Mitchell + * Copyright © 2022-2023 Elliott Mitchell * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -626,12 +626,15 @@ isrc_event_create(struct intr_irqsrc *isrc) { struct intr_event *ie; + u_int flags = 0; int error; if (isrc->isrc_event != NULL) return (0); - error = intr_event_create(&ie, isrc, 0, isrc->isrc_irq, + if (isrc->isrc_flags & (INTR_ISRCF_IPI|INTR_ISRCF_PPI)) + flags |= IE_MULTIPROC; + error = intr_event_create(&ie, isrc, flags, isrc->isrc_irq, intr_isrc_pre_ithread, intr_isrc_post_ithread, intr_isrc_post_filter, intr_isrc_assign_cpu, "%s:", isrc->isrc_name); if (error) @@ -751,6 +754,13 @@ if (error) return (error); + if (flags & INTR_MULTIPROC && + !(isrc->isrc_event->ie_flags & IE_MULTIPROC)) { + debugf("%s: Requested multi-processor interupt, but got " + "uniprocessor interrupt\n", name); + return (ENODEV); + } + error = intr_event_add_handler(isrc->isrc_event, name, filter, handler, arg, intr_priority(flags), flags, cookiep); if (error == 0) { diff --git a/sys/sys/bus.h b/sys/sys/bus.h --- a/sys/sys/bus.h +++ b/sys/sys/bus.h @@ -281,7 +281,8 @@ INTR_MD1 = 4096, /* flag reserved for MD use */ INTR_MD2 = 8192, /* flag reserved for MD use */ INTR_MD3 = 16384, /* flag reserved for MD use */ - INTR_MD4 = 32768 /* flag reserved for MD use */ + INTR_MD4 = 32768, /* flag reserved for MD use */ + INTR_MULTIPROC = 0x10000, /* interrupt occurs on multiple procs */ }; enum intr_trigger { diff --git a/sys/sys/interrupt.h b/sys/sys/interrupt.h --- a/sys/sys/interrupt.h +++ b/sys/sys/interrupt.h @@ -127,11 +127,14 @@ int ie_cpu; /* CPU this event is bound to. */ volatile int ie_phase; /* Switched to establish a barrier. */ volatile int ie_active[2]; /* Filters in ISR context. */ + u_long ie_stray; /* Stray interrupt counter */ + u_long ie_intrcnt[1]; /* Interrupt counter(s) */ }; /* Interrupt event flags kept in ie_flags. */ #define IE_SOFT 0x000001 /* Software interrupt. */ #define IE_ADDING_THREAD 0x000004 /* Currently building an ithread. */ +#define IE_MULTIPROC 0x000008 /* Interrupt occurs on multiple procs */ /* Flags to pass to swi_sched. */ #define SWI_FROMNMI 0x1 diff --git a/sys/xen/intrng/xen_arch_intr.c b/sys/xen/intrng/xen_arch_intr.c --- a/sys/xen/intrng/xen_arch_intr.c +++ b/sys/xen/intrng/xen_arch_intr.c @@ -179,7 +179,7 @@ xen_sc = sc; /* Setup and enable the event channel interrupt */ - if (bus_setup_intr(dev, sc->intr, INTR_TYPE_MISC|INTR_MPSAFE, + if (bus_setup_intr(dev, sc->intr, INTR_TYPE_MISC|INTR_MPSAFE|INTR_MULTIPROC, xen_intr_handle_upcall, NULL, sc, &sc->cookie)) { panic("Could not setup event channel interrupt"); return (ENXIO);