diff --git a/sys/kern/kern_intr.c b/sys/kern/kern_intr.c --- a/sys/kern/kern_intr.c +++ b/sys/kern/kern_intr.c @@ -3,6 +3,7 @@ * * Copyright (c) 1997, Stefan Esser * All rights reserved. + * Copyright © 2022-2023 Elliott Mitchell * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -286,10 +287,11 @@ struct intr_event *ie; va_list ap; - /* The only valid flag during creation is IE_SOFT. */ - if ((flags & ~IE_SOFT) != 0) + /* The flags valid during creation are IE_SOFT and IE_MULTIPROC. */ + if ((flags & ~(IE_SOFT | IE_MULTIPROC)) != 0) return (EINVAL); - ie = malloc(sizeof(struct intr_event), M_ITHREAD, M_WAITOK | M_ZERO); + ie = malloc(sizeof(struct intr_event) + (flags & IE_MULTIPROC ? + sizeof(u_long) * (mp_ncpus - 1) : 0), M_ITHREAD, M_WAITOK | M_ZERO); ie->ie_source = source; ie->ie_pre_ithread = pre_ithread; ie->ie_post_ithread = post_ithread; @@ -298,6 +300,7 @@ ie->ie_flags = flags; ie->ie_irq = irq; ie->ie_cpu = NOCPU; + /* ->ie_stray and ->ie_intrcnt[] handled by M_ZERO */ CK_SLIST_INIT(&ie->ie_handlers); mtx_init(&ie->ie_lock, "intr event", NULL, MTX_DEF); @@ -1370,10 +1373,30 @@ intr_prof_stack_use(td, frame); #endif - /* An interrupt with no event or handlers is a stray interrupt. */ - if (ie == NULL || CK_SLIST_EMPTY(&ie->ie_handlers)) + /* An interrupt with no event is a stray interrupt. */ + if (ie == NULL) return (EINVAL); + /* + * Note about interrupt counters. Having per-processor counters avoids + * the need for atomic increment of counters. Whereas other interrupt + * types will be bound to a single processor and not need atomics. + * Per-processor interrupts are *never* flagged as stray, so stray + * counters don't need atomic increment. + */ + + /* Increment the interrupt counter. */ + if (__predict_false(ie->ie_flags & IE_MULTIPROC)) + ++ie->ie_intrcnt[curcpu]; + else + ++ie->ie_intrcnt[0]; + + /* An interrupt with no handlers is a stray interrupt. */ + if (CK_SLIST_EMPTY(&ie->ie_handlers)) { + ++ie->ie_stray; + return (EINVAL); + } + /* * Execute fast interrupt handlers directly. * To support clock handlers, if a handler registers @@ -1473,8 +1496,10 @@ td->td_intr_nesting_level--; #ifdef notyet /* The interrupt is not aknowledged by any filter and has no ithread. */ - if (!thread && !filter) + if (!thread && !filter) { + ++ie->ie_stray; return (EINVAL); + } #endif return (0); } @@ -1625,9 +1650,12 @@ start_softintr(void *dummy) { - if (swi_add(&clk_intr_event, "clk", NULL, NULL, SWI_CLOCK, - INTR_MPSAFE, NULL)) - panic("died while creating clk swi ithread"); + KASSERT(clk_intr_event == NULL, ("clk_intr_event non-NULL at %s()", + __func__)); + + if (intr_event_create(&clk_intr_event, NULL, IE_SOFT | IE_MULTIPROC, + 0, NULL, NULL, NULL, swi_assign_cpu, "clk swi%d:", SWI_CLOCK)) + panic("died while creating clk swi event"); } SYSINIT(start_softintr, SI_SUB_SOFTINTR, SI_ORDER_FIRST, start_softintr, NULL); diff --git a/sys/kern/subr_intr.c b/sys/kern/subr_intr.c --- a/sys/kern/subr_intr.c +++ b/sys/kern/subr_intr.c @@ -2,7 +2,7 @@ * Copyright (c) 2015-2016 Svatopluk Kraus * Copyright (c) 2015-2016 Michal Meloun * All rights reserved. - * Copyright © 2022 Elliott Mitchell + * Copyright © 2022-2023 Elliott Mitchell * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -625,12 +625,15 @@ isrc_event_create(struct intr_irqsrc *isrc) { struct intr_event *ie; + u_int flags = 0; int error; if (isrc->isrc_event != NULL) return (0); - error = intr_event_create(&ie, isrc, 0, isrc->isrc_irq, + if (isrc->isrc_flags & (INTR_ISRCF_IPI|INTR_ISRCF_PPI)) + flags |= IE_MULTIPROC; + error = intr_event_create(&ie, isrc, flags, isrc->isrc_irq, intr_isrc_pre_ithread, intr_isrc_post_ithread, intr_isrc_post_filter, intr_isrc_assign_cpu, "%s:", isrc->isrc_name); if (error) diff --git a/sys/sys/interrupt.h b/sys/sys/interrupt.h --- a/sys/sys/interrupt.h +++ b/sys/sys/interrupt.h @@ -127,11 +127,14 @@ int ie_cpu; /* CPU this event is bound to. */ volatile int ie_phase; /* Switched to establish a barrier. */ volatile int ie_active[2]; /* Filters in ISR context. */ + u_long ie_stray; /* Stray interrupt counter */ + u_long ie_intrcnt[1]; /* Interrupt counter(s) */ }; /* Interrupt event flags kept in ie_flags. */ #define IE_SOFT 0x000001 /* Software interrupt. */ #define IE_ADDING_THREAD 0x000004 /* Currently building an ithread. */ +#define IE_MULTIPROC 0x000008 /* Interrupt occurs on multiple procs */ /* Flags to pass to swi_sched. */ #define SWI_FROMNMI 0x1