diff --git a/sys/arm/arm/machdep_intr.c b/sys/arm/arm/machdep_intr.c --- a/sys/arm/arm/machdep_intr.c +++ b/sys/arm/arm/machdep_intr.c @@ -28,40 +28,9 @@ #include "opt_platform.h" #include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include #include -#include - -#include "pic_if.h" - -#ifdef SMP -#define INTR_IPI_NAMELEN (MAXCOMLEN + 1) - -struct intr_ipi { - intr_ipi_handler_t * ii_handler; - void * ii_handler_arg; - intr_ipi_send_t * ii_send; - void * ii_send_arg; - char ii_name[INTR_IPI_NAMELEN]; - u_long * ii_count; -}; - -static struct intr_ipi ipi_sources[INTR_IPI_COUNT]; -#endif +#include /* * arm_irq_memory_barrier() @@ -125,96 +94,3 @@ dsb(); cpu_l2cache_drain_writebuf(); } - -#ifdef SMP -static inline struct intr_ipi * -intr_ipi_lookup(u_int ipi) -{ - - if (ipi >= INTR_IPI_COUNT) - panic("%s: no such IPI %u", __func__, ipi); - - return (&ipi_sources[ipi]); -} - -void -intr_ipi_dispatch(u_int ipi) -{ - struct intr_ipi *ii; - - ii = intr_ipi_lookup(ipi); - if (ii->ii_count == NULL) - panic("%s: not setup IPI %u", __func__, ipi); - - intr_ipi_increment_count(ii->ii_count, PCPU_GET(cpuid)); - - ii->ii_handler(ii->ii_handler_arg); -} - -void -intr_ipi_send(cpuset_t cpus, u_int ipi) -{ - struct intr_ipi *ii; - - ii = intr_ipi_lookup(ipi); - if (ii->ii_count == NULL) - panic("%s: not setup IPI %u", __func__, ipi); - - ii->ii_send(ii->ii_send_arg, cpus, ipi); -} - -void -intr_ipi_setup(u_int ipi, const char *name, intr_ipi_handler_t *hand, - void *h_arg, intr_ipi_send_t *send, void *s_arg) -{ - struct intr_ipi *ii; - - ii = intr_ipi_lookup(ipi); - - KASSERT(hand != NULL, ("%s: ipi %u no handler", __func__, ipi)); - KASSERT(send != NULL, ("%s: ipi %u no sender", __func__, ipi)); - KASSERT(ii->ii_count == NULL, ("%s: ipi %u reused", __func__, ipi)); - - ii->ii_handler = hand; - ii->ii_handler_arg = h_arg; - ii->ii_send = send; - ii->ii_send_arg = s_arg; - strlcpy(ii->ii_name, name, INTR_IPI_NAMELEN); - ii->ii_count = intr_ipi_setup_counters(name); -} - -/* - * Send IPI thru interrupt controller. - */ -static void -pic_ipi_send(void *arg, cpuset_t cpus, u_int ipi) -{ - - KASSERT(intr_irq_root_dev != NULL, ("%s: no root attached", __func__)); - PIC_IPI_SEND(intr_irq_root_dev, arg, cpus, ipi); -} - -/* - * Setup IPI handler on interrupt controller. - * - * Not SMP coherent. - */ -int -intr_pic_ipi_setup(u_int ipi, const char *name, intr_ipi_handler_t *hand, - void *arg) -{ - int error; - struct intr_irqsrc *isrc; - - KASSERT(intr_irq_root_dev != NULL, ("%s: no root attached", __func__)); - - error = PIC_IPI_SETUP(intr_irq_root_dev, ipi, &isrc); - if (error != 0) - return (error); - - isrc->isrc_handlers++; - intr_ipi_setup(ipi, name, hand, arg, pic_ipi_send, isrc); - PIC_ENABLE_INTR(intr_irq_root_dev, isrc); - return (0); -} -#endif diff --git a/sys/arm/arm/mp_machdep.c b/sys/arm/arm/mp_machdep.c --- a/sys/arm/arm/mp_machdep.c +++ b/sys/arm/arm/mp_machdep.c @@ -300,11 +300,11 @@ if (mp_ncpus == 1) return; - intr_pic_ipi_setup(IPI_RENDEZVOUS, "rendezvous", ipi_rendezvous, NULL); - intr_pic_ipi_setup(IPI_AST, "ast", ipi_ast, NULL); - intr_pic_ipi_setup(IPI_STOP, "stop", ipi_stop, NULL); - intr_pic_ipi_setup(IPI_PREEMPT, "preempt", ipi_preempt, NULL); - intr_pic_ipi_setup(IPI_HARDCLOCK, "hardclock", ipi_hardclock, NULL); + intr_ipi_setup(IPI_RENDEZVOUS, "rendezvous", ipi_rendezvous, NULL); + intr_ipi_setup(IPI_AST, "ast", ipi_ast, NULL); + intr_ipi_setup(IPI_STOP, "stop", ipi_stop, NULL); + intr_ipi_setup(IPI_PREEMPT, "preempt", ipi_preempt, NULL); + intr_ipi_setup(IPI_HARDCLOCK, "hardclock", ipi_hardclock, NULL); atomic_store_rel_int(&aps_ready, 1); /* Wake the other threads up */ diff --git a/sys/arm/include/intr.h b/sys/arm/include/intr.h --- a/sys/arm/include/intr.h +++ b/sys/arm/include/intr.h @@ -49,19 +49,6 @@ #include -#ifdef SMP -typedef void intr_ipi_send_t(void *, cpuset_t, u_int); -typedef void intr_ipi_handler_t(void *); - -void intr_ipi_dispatch(u_int); -void intr_ipi_send(cpuset_t, u_int); - -void intr_ipi_setup(u_int, const char *, intr_ipi_handler_t *, void *, - intr_ipi_send_t *, void *); - -int intr_pic_ipi_setup(u_int, const char *, intr_ipi_handler_t *, void *); -#endif - void arm_irq_memory_barrier(uintptr_t); #endif /* _MACHINE_INTR_H */ diff --git a/sys/arm64/arm64/mp_machdep.c b/sys/arm64/arm64/mp_machdep.c --- a/sys/arm64/arm64/mp_machdep.c +++ b/sys/arm64/arm64/mp_machdep.c @@ -77,8 +77,6 @@ #include -#include "pic_if.h" - #define MP_BOOTSTACK_SIZE (kstack_pages * PAGE_SIZE) #define MP_QUIRK_CPULIST 0x01 /* The list of cpus may be wrong, */ @@ -98,25 +96,6 @@ }; #endif -typedef void intr_ipi_send_t(void *, cpuset_t, u_int); -typedef void intr_ipi_handler_t(void *); - -#define INTR_IPI_NAMELEN (MAXCOMLEN + 1) -struct intr_ipi { - intr_ipi_handler_t * ii_handler; - void * ii_handler_arg; - intr_ipi_send_t * ii_send; - void * ii_send_arg; - char ii_name[INTR_IPI_NAMELEN]; - u_long * ii_count; -}; - -static struct intr_ipi ipi_sources[INTR_IPI_COUNT]; - -static struct intr_ipi *intr_ipi_lookup(u_int); -static void intr_pic_ipi_setup(u_int, const char *, intr_ipi_handler_t *, - void *); - static void ipi_ast(void *); static void ipi_hardclock(void *); static void ipi_preempt(void *); @@ -165,12 +144,12 @@ if (mp_ncpus == 1) return; - intr_pic_ipi_setup(IPI_AST, "ast", ipi_ast, NULL); - intr_pic_ipi_setup(IPI_PREEMPT, "preempt", ipi_preempt, NULL); - intr_pic_ipi_setup(IPI_RENDEZVOUS, "rendezvous", ipi_rendezvous, NULL); - intr_pic_ipi_setup(IPI_STOP, "stop", ipi_stop, NULL); - intr_pic_ipi_setup(IPI_STOP_HARD, "stop hard", ipi_stop, NULL); - intr_pic_ipi_setup(IPI_HARDCLOCK, "hardclock", ipi_hardclock, NULL); + intr_ipi_setup(IPI_AST, "ast", ipi_ast, NULL); + intr_ipi_setup(IPI_PREEMPT, "preempt", ipi_preempt, NULL); + intr_ipi_setup(IPI_RENDEZVOUS, "rendezvous", ipi_rendezvous, NULL); + intr_ipi_setup(IPI_STOP, "stop", ipi_stop, NULL); + intr_ipi_setup(IPI_STOP_HARD, "stop hard", ipi_stop, NULL); + intr_ipi_setup(IPI_HARDCLOCK, "hardclock", ipi_hardclock, NULL); atomic_store_rel_int(&aps_ready, 1); /* Wake up the other CPUs */ @@ -315,71 +294,6 @@ SYSINIT(smp_after_idle_runnable, SI_SUB_SMP, SI_ORDER_ANY, smp_after_idle_runnable, NULL); -/* - * Send IPI thru interrupt controller. - */ -static void -pic_ipi_send(void *arg, cpuset_t cpus, u_int ipi) -{ - - KASSERT(intr_irq_root_dev != NULL, ("%s: no root attached", __func__)); - - /* - * Ensure that this CPU's stores will be visible to IPI - * recipients before starting to send the interrupts. - */ - dsb(ishst); - - PIC_IPI_SEND(intr_irq_root_dev, arg, cpus, ipi); -} - -/* - * Setup IPI handler on interrupt controller. - * - * Not SMP coherent. - */ -static void -intr_pic_ipi_setup(u_int ipi, const char *name, intr_ipi_handler_t *hand, - void *arg) -{ - struct intr_irqsrc *isrc; - struct intr_ipi *ii; - int error; - - KASSERT(intr_irq_root_dev != NULL, ("%s: no root attached", __func__)); - KASSERT(hand != NULL, ("%s: ipi %u no handler", __func__, ipi)); - - error = PIC_IPI_SETUP(intr_irq_root_dev, ipi, &isrc); - if (error != 0) - return; - - isrc->isrc_handlers++; - - ii = intr_ipi_lookup(ipi); - KASSERT(ii->ii_count == NULL, ("%s: ipi %u reused", __func__, ipi)); - - ii->ii_handler = hand; - ii->ii_handler_arg = arg; - ii->ii_send = pic_ipi_send; - ii->ii_send_arg = isrc; - strlcpy(ii->ii_name, name, INTR_IPI_NAMELEN); - ii->ii_count = intr_ipi_setup_counters(name); - - PIC_ENABLE_INTR(intr_irq_root_dev, isrc); -} - -static void -intr_ipi_send(cpuset_t cpus, u_int ipi) -{ - struct intr_ipi *ii; - - ii = intr_ipi_lookup(ipi); - if (ii->ii_count == NULL) - panic("%s: not setup IPI %u", __func__, ipi); - - ii->ii_send(ii->ii_send_arg, cpus, ipi); -} - static void ipi_ast(void *dummy __unused) { @@ -888,112 +802,6 @@ } } -/* - * Lookup IPI source. - */ -static struct intr_ipi * -intr_ipi_lookup(u_int ipi) -{ - - if (ipi >= INTR_IPI_COUNT) - panic("%s: no such IPI %u", __func__, ipi); - - return (&ipi_sources[ipi]); -} - -/* - * interrupt controller dispatch function for IPIs. It should - * be called straight from the interrupt controller, when associated - * interrupt source is learned. Or from anybody who has an interrupt - * source mapped. - */ -void -intr_ipi_dispatch(u_int ipi) -{ - struct intr_ipi *ii; - - ii = intr_ipi_lookup(ipi); - if (ii->ii_count == NULL) - panic("%s: not setup IPI %u", __func__, ipi); - - intr_ipi_increment_count(ii->ii_count, PCPU_GET(cpuid)); - - ii->ii_handler(ii->ii_handler_arg); -} - -#ifdef notyet -/* - * Map IPI into interrupt controller. - * - * Not SMP coherent. - */ -static int -ipi_map(struct intr_irqsrc *isrc, u_int ipi) -{ - boolean_t is_percpu; - int error; - - if (ipi >= INTR_IPI_COUNT) - panic("%s: no such IPI %u", __func__, ipi); - - KASSERT(intr_irq_root_dev != NULL, ("%s: no root attached", __func__)); - - isrc->isrc_type = INTR_ISRCT_NAMESPACE; - isrc->isrc_nspc_type = INTR_IRQ_NSPC_IPI; - isrc->isrc_nspc_num = ipi_next_num; - - error = PIC_REGISTER(intr_irq_root_dev, isrc, &is_percpu); - if (error == 0) { - isrc->isrc_dev = intr_irq_root_dev; - ipi_next_num++; - } - return (error); -} - -/* - * Setup IPI handler to interrupt source. - * - * Note that there could be more ways how to send and receive IPIs - * on a platform like fast interrupts for example. In that case, - * one can call this function with ASIF_NOALLOC flag set and then - * call intr_ipi_dispatch() when appropriate. - * - * Not SMP coherent. - */ -int -intr_ipi_set_handler(u_int ipi, const char *name, intr_ipi_filter_t *filter, - void *arg, u_int flags) -{ - struct intr_irqsrc *isrc; - int error; - - if (filter == NULL) - return(EINVAL); - - isrc = intr_ipi_lookup(ipi); - if (isrc->isrc_ipifilter != NULL) - return (EEXIST); - - if ((flags & AISHF_NOALLOC) == 0) { - error = ipi_map(isrc, ipi); - if (error != 0) - return (error); - } - - isrc->isrc_ipifilter = filter; - isrc->isrc_arg = arg; - isrc->isrc_handlers = 1; - isrc->isrc_count = intr_ipi_setup_counters(name); - isrc->isrc_index = 0; /* it should not be used in IPI case */ - - if (isrc->isrc_dev != NULL) { - PIC_ENABLE_INTR(isrc->isrc_dev, isrc); - PIC_ENABLE_SOURCE(isrc->isrc_dev, isrc); - } - return (0); -} -#endif - /* Sending IPI */ void ipi_all_but_self(u_int ipi) diff --git a/sys/arm64/include/intr.h b/sys/arm64/include/intr.h --- a/sys/arm64/include/intr.h +++ b/sys/arm64/include/intr.h @@ -42,10 +42,6 @@ { } -#ifdef SMP -void intr_ipi_dispatch(u_int); -#endif - #ifdef DEV_ACPI #define ACPI_INTR_XREF 1 #define ACPI_MSI_XREF 2 diff --git a/sys/kern/subr_intr.c b/sys/kern/subr_intr.c --- a/sys/kern/subr_intr.c +++ b/sys/kern/subr_intr.c @@ -2,6 +2,11 @@ * Copyright (c) 2015-2016 Svatopluk Kraus * Copyright (c) 2015-2016 Michal Meloun * All rights reserved. + * Copyright (c) 2015-2016 The FreeBSD Foundation + * Copyright (c) 2021 Jessica Clarke + * + * Portions of this software were developed by Andrew Turner under + * sponsorship from the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -124,6 +129,18 @@ SLIST_HEAD(, intr_pic_child) pic_children; }; +#ifdef SMP +#define INTR_IPI_NAMELEN (MAXCOMLEN + 1) + +struct intr_ipi { + intr_ipi_handler_t *ii_handler; + void *ii_handler_arg; + struct intr_irqsrc *ii_isrc; + char ii_name[INTR_IPI_NAMELEN]; + u_long *ii_count; +}; +#endif + static struct mtx pic_list_lock; static SLIST_HEAD(, intr_pic) pic_list; @@ -140,6 +157,8 @@ #else static bool irq_assign_cpu = false; #endif + +static struct intr_ipi ipi_sources[INTR_IPI_COUNT]; #endif u_int intr_nirq = NIRQ; @@ -298,39 +317,6 @@ bit_nclear(intrcnt_bitmap, idx, idx + 1); } -#ifdef SMP -/* - * Virtualization for interrupt source IPI counters setup. - */ -u_long * -intr_ipi_setup_counters(const char *name) -{ - u_int index, i; - char str[INTRNAME_LEN]; - - mtx_lock(&isrc_table_lock); - - /* - * We should never have a problem finding mp_maxid + 1 contiguous - * counters, in practice. Interrupts will be allocated sequentially - * during boot, so the array should fill from low to high index. Once - * reserved, the IPI counters will never be released. Similarly, we - * will not need to allocate more IPIs once the system is running. - */ - bit_ffc_area(intrcnt_bitmap, nintrcnt, mp_maxid + 1, &index); - if (index == -1) - panic("Failed to allocate %d counters. Array exhausted?", - mp_maxid + 1); - bit_nset(intrcnt_bitmap, index, index + mp_maxid); - for (i = 0; i < mp_maxid + 1; i++) { - snprintf(str, INTRNAME_LEN, "cpu%d:%s", i, name); - intrcnt_setname(str, index + i); - } - mtx_unlock(&isrc_table_lock); - return (&intrcnt[index]); -} -#endif - /* * Main interrupt dispatch handler. It's called straight * from the assembler, where CPU interrupt is served. @@ -1774,3 +1760,139 @@ M_INTRNG, M_WAITOK | M_ZERO); } SYSINIT(intr_map_init, SI_SUB_INTR, SI_ORDER_FIRST, intr_map_init, NULL); + +#ifdef SMP +/* Virtualization for interrupt source IPI counter increment. */ +static inline void +intr_ipi_increment_count(u_long *counter, u_int cpu) +{ + + KASSERT(cpu < MAXCPU, ("%s: too big cpu %u", __func__, cpu)); + counter[cpu]++; +} + +/* + * Virtualization for interrupt source IPI counters setup. + */ +static u_long * +intr_ipi_setup_counters(const char *name) +{ + u_int index, i; + char str[INTRNAME_LEN]; + + mtx_lock(&isrc_table_lock); + + /* + * We should never have a problem finding mp_maxid + 1 contiguous + * counters, in practice. Interrupts will be allocated sequentially + * during boot, so the array should fill from low to high index. Once + * reserved, the IPI counters will never be released. Similarly, we + * will not need to allocate more IPIs once the system is running. + */ + bit_ffc_area(intrcnt_bitmap, nintrcnt, mp_maxid + 1, &index); + if (index == -1) + panic("Failed to allocate %d counters. Array exhausted?", + mp_maxid + 1); + bit_nset(intrcnt_bitmap, index, index + mp_maxid); + for (i = 0; i < mp_maxid + 1; i++) { + snprintf(str, INTRNAME_LEN, "cpu%d:%s", i, name); + intrcnt_setname(str, index + i); + } + mtx_unlock(&isrc_table_lock); + return (&intrcnt[index]); +} + +/* + * Lookup IPI source. + */ +static struct intr_ipi * +intr_ipi_lookup(u_int ipi) +{ + + if (ipi >= INTR_IPI_COUNT) + panic("%s: no such IPI %u", __func__, ipi); + + return (&ipi_sources[ipi]); +} + +/* + * Setup IPI handler on interrupt controller. + * + * Not SMP coherent. + */ +void +intr_ipi_setup(u_int ipi, const char *name, intr_ipi_handler_t *hand, + void *arg) +{ + struct intr_irqsrc *isrc; + struct intr_ipi *ii; + int error; + + KASSERT(intr_irq_root_dev != NULL, ("%s: no root attached", __func__)); + KASSERT(hand != NULL, ("%s: ipi %u no handler", __func__, ipi)); + + error = PIC_IPI_SETUP(intr_irq_root_dev, ipi, &isrc); + if (error != 0) + return; + + isrc->isrc_handlers++; + + ii = intr_ipi_lookup(ipi); + KASSERT(ii->ii_count == NULL, ("%s: ipi %u reused", __func__, ipi)); + + ii->ii_handler = hand; + ii->ii_handler_arg = arg; + ii->ii_isrc = isrc; + strlcpy(ii->ii_name, name, INTR_IPI_NAMELEN); + ii->ii_count = intr_ipi_setup_counters(name); + + PIC_ENABLE_INTR(intr_irq_root_dev, isrc); +} + +void +intr_ipi_send(cpuset_t cpus, u_int ipi) +{ + struct intr_ipi *ii; + + KASSERT(intr_irq_root_dev != NULL, ("%s: no root attached", __func__)); + + ii = intr_ipi_lookup(ipi); + if (ii->ii_count == NULL) + panic("%s: not setup IPI %u", __func__, ipi); + + /* + * XXX: Surely needed on other architectures too? Either way should be + * some kind of MI hook defined in an MD header, or the responsibility + * of the MD caller if not widespread. + */ +#ifdef __aarch64__ + /* + * Ensure that this CPU's stores will be visible to IPI + * recipients before starting to send the interrupts. + */ + dsb(ishst); +#endif + + PIC_IPI_SEND(intr_irq_root_dev, ii->ii_isrc, cpus, ipi); +} + +/* + * interrupt controller dispatch function for IPIs. It should + * be called straight from the interrupt controller, when associated + * interrupt source is learned. Or from anybody who has an interrupt + * source mapped. + */ +void +intr_ipi_dispatch(u_int ipi) +{ + struct intr_ipi *ii; + + ii = intr_ipi_lookup(ipi); + if (ii->ii_count == NULL) + panic("%s: not setup IPI %u", __func__, ipi); + + intr_ipi_increment_count(ii->ii_count, PCPU_GET(cpuid)); + + ii->ii_handler(ii->ii_handler_arg); +} +#endif diff --git a/sys/sys/intr.h b/sys/sys/intr.h --- a/sys/sys/intr.h +++ b/sys/sys/intr.h @@ -148,21 +148,18 @@ int intr_bind_irq(device_t, struct resource *, int); void intr_pic_init_secondary(void); +#endif -/* Virtualization for interrupt source IPI counter increment. */ -static inline void -intr_ipi_increment_count(u_long *counter, u_int cpu) -{ - - KASSERT(cpu < MAXCPU, ("%s: too big cpu %u", __func__, cpu)); - counter[cpu]++; -} +extern u_int intr_nirq; /* number of IRQs on intrng platforms */ -/* Virtualization for interrupt source IPI counters setup. */ -u_long * intr_ipi_setup_counters(const char *name); +/* Intr interface for IPIs. */ +#ifdef SMP +typedef void intr_ipi_handler_t(void *); +void intr_ipi_setup(u_int ipi, const char *name, intr_ipi_handler_t *hand, + void *arg); +void intr_ipi_send(cpuset_t cpus, u_int ipi); +void intr_ipi_dispatch(u_int ipi); #endif -extern u_int intr_nirq; /* number of IRQs on intrng platforms */ - #endif /* _SYS_INTR_H */