Index: sys/conf/files.riscv =================================================================== --- sys/conf/files.riscv +++ sys/conf/files.riscv @@ -47,7 +47,7 @@ riscv/riscv/exception.S standard riscv/riscv/exec_machdep.c standard riscv/riscv/gdb_machdep.c optional gdb -riscv/riscv/intr_machdep.c standard +riscv/riscv/intc.c standard riscv/riscv/identcpu.c standard riscv/riscv/locore.S standard no-obj riscv/riscv/machdep.c standard Index: sys/riscv/include/intr.h =================================================================== --- sys/riscv/include/intr.h +++ sys/riscv/include/intr.h @@ -37,32 +37,11 @@ #ifndef _MACHINE_INTR_MACHDEP_H_ #define _MACHINE_INTR_MACHDEP_H_ -#define RISCV_NIRQ 1024 - #ifndef NIRQ -#define NIRQ RISCV_NIRQ +#define NIRQ 1024 #endif -#ifdef INTRNG #include -#endif - -struct trapframe; - -int riscv_teardown_intr(void *); -int riscv_setup_intr(const char *, driver_filter_t *, driver_intr_t *, - void *, int, int, void **); -void riscv_cpu_intr(struct trapframe *); - -typedef unsigned long * riscv_intrcnt_t; - -riscv_intrcnt_t riscv_intrcnt_create(const char *); -void riscv_intrcnt_setname(riscv_intrcnt_t, const char *); - -#ifdef SMP -void riscv_setup_ipihandler(driver_filter_t *); -void riscv_unmask_ipi(void); -#endif enum { IRQ_SOFTWARE_USER, @@ -77,7 +56,6 @@ IRQ_EXTERNAL_SUPERVISOR, IRQ_EXTERNAL_HYPERVISOR, IRQ_EXTERNAL_MACHINE, - INTC_NIRQS }; #endif /* !_MACHINE_INTR_MACHDEP_H_ */ Index: sys/riscv/include/smp.h =================================================================== --- sys/riscv/include/smp.h +++ sys/riscv/include/smp.h @@ -39,14 +39,15 @@ #include -#define IPI_AST (1 << 0) -#define IPI_PREEMPT (1 << 1) -#define IPI_RENDEZVOUS (1 << 2) -#define IPI_STOP (1 << 3) -#define IPI_STOP_HARD (1 << 4) -#define IPI_HARDCLOCK (1 << 5) - -#define INTR_IPI_COUNT 1 +enum { + IPI_AST, + IPI_PREEMPT, + IPI_RENDEZVOUS, + IPI_STOP, + IPI_STOP_HARD, + IPI_HARDCLOCK, + INTR_IPI_COUNT +}; void ipi_all_but_self(u_int ipi); void ipi_cpu(int cpu, u_int ipi); Index: sys/riscv/riscv/intc.c =================================================================== --- /dev/null +++ sys/riscv/riscv/intc.c @@ -0,0 +1,311 @@ +/*- + * Copyright (c) 2015-2017 Ruslan Bukin + * All rights reserved. + * Copyright (c) 2021 Jessica Clarke + * + * Portions of this software were developed by SRI International and the + * University of Cambridge Computer Laboratory under DARPA/AFRL contract + * FA8750-10-C-0237 ("CTSRD"), as part of the DARPA CRASH research programme. + * + * Portions of this software were developed by the University of Cambridge + * Computer Laboratory as part of the CTSRD Project, with support from the + * UK Higher Education Innovation Fund (HEIF). + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include + +#include +#include +#include + +#include "pic_if.h" + +#define INTC_NIRQS 16 + +struct intc_irqsrc { + struct intr_irqsrc isrc; + u_int irq; +}; + +struct intc_softc { + device_t dev; + struct intc_irqsrc isrcs[INTC_NIRQS]; +}; + +static int intc_intr(void *arg); + +static phandle_t +intc_ofw_find(device_t dev, uint32_t hartid) +{ + phandle_t node; + pcell_t reg; + + node = OF_finddevice("/cpus"); + if (node == -1) { + device_printf(dev, "Can't find cpus node\n"); + return ((phandle_t)-1); + } + + for (node = OF_child(node); node != 0; node = OF_peer(node)) { + if (!ofw_bus_node_status_okay(node)) + continue; + + if (!ofw_bus_node_is_compatible(node, "riscv")) + continue; + + if (OF_searchencprop(node, "reg", ®, sizeof(reg)) == -1) + continue; + + if (reg == hartid) + break; + } + + if (node == 0) { + device_printf(dev, "Can't find boot cpu node\n"); + return ((phandle_t)-1); + } + + for (node = OF_child(node); node != 0; node = OF_peer(node)) { + if (!ofw_bus_node_status_okay(node)) + continue; + + if (ofw_bus_node_is_compatible(node, "riscv,cpu-intc")) + break; + } + + if (node == 0) { + device_printf(dev, + "Can't find boot cpu local interrupt controller\n"); + return ((phandle_t)-1); + } + + return (node); +} + +static void +intc_identify(driver_t *driver, device_t parent) +{ + device_t dev; + phandle_t node; + + if (device_find_child(parent, "intc", -1) != NULL) + return; + + node = intc_ofw_find(parent, PCPU_GET(hart)); + if (node == -1) + return; + + dev = simplebus_add_device(parent, node, 0, "intc", -1, NULL); + if (dev == NULL) + device_printf(parent, "Can't add intc child\n"); +} + +static int +intc_probe(device_t dev) +{ + device_set_desc(dev, "RISC-V Local Interrupt Controller"); + + return (BUS_PROBE_NOWILDCARD); +} + +static int +intc_attach(device_t dev) +{ + struct intc_irqsrc *isrcs; + struct intc_softc *sc; + struct intr_pic *pic; + const char *name; + phandle_t xref; + u_int flags; + int i, error; + + sc = device_get_softc(dev); + sc->dev = dev; + + name = device_get_nameunit(dev); + xref = OF_xref_from_node(ofw_bus_get_node(dev)); + + isrcs = sc->isrcs; + for (i = 0; i < INTC_NIRQS; i++) { + isrcs[i].irq = i; + flags = i == IRQ_SOFTWARE_SUPERVISOR + ? INTR_ISRCF_IPI : INTR_ISRCF_PPI; + error = intr_isrc_register(&isrcs[i].isrc, sc->dev, flags, + "%s,%u", name, i); + if (error != 0) { + device_printf(dev, "Can't register interrupt %d\n", i); + return (error); + } + } + + pic = intr_pic_register(sc->dev, xref); + if (pic == NULL) + return (ENXIO); + + return (intr_pic_claim_root(sc->dev, xref, intc_intr, sc)); +} + +static void +intc_disable_intr(device_t dev, struct intr_irqsrc *isrc) +{ + u_int irq; + + irq = ((struct intc_irqsrc *)isrc)->irq; + if (irq >= INTC_NIRQS) + panic("%s: Unsupported IRQ %u", __func__, irq); + + csr_clear(sie, 1ul << irq); +} + +static void +intc_enable_intr(device_t dev, struct intr_irqsrc *isrc) +{ + u_int irq; + + irq = ((struct intc_irqsrc *)isrc)->irq; + if (irq >= INTC_NIRQS) + panic("%s: Unsupported IRQ %u", __func__, irq); + + csr_set(sie, 1ul << irq); +} + +static int +intc_map_intr(device_t dev, struct intr_map_data *data, + struct intr_irqsrc **isrcp) +{ + struct intr_map_data_fdt *daf; + struct intc_softc *sc; + + sc = device_get_softc(dev); + + if (data->type != INTR_MAP_DATA_FDT) + return (ENOTSUP); + + daf = (struct intr_map_data_fdt *)data; + if (daf->ncells != 1 || daf->cells[0] >= INTC_NIRQS) + return (EINVAL); + + *isrcp = &sc->isrcs[daf->cells[0]].isrc; + + return (0); +} + +static int +intc_setup_intr(device_t dev, struct intr_irqsrc *isrc, + struct resource *res, struct intr_map_data *data) +{ + if (isrc->isrc_flags & INTR_ISRCF_PPI) + CPU_SET(PCPU_GET(cpuid), &isrc->isrc_cpu); + + return (0); +} + +#ifdef SMP +static void +intc_init_secondary(device_t dev) +{ + struct intc_softc *sc; + struct intr_irqsrc *isrc; + u_int cpu, irq; + + sc = device_get_softc(dev); + cpu = PCPU_GET(cpuid); + + /* Unmask attached interrupts */ + for (irq = 0; irq < INTC_NIRQS; irq++) { + isrc = &sc->isrcs[irq].isrc; + if (intr_isrc_init_on_cpu(isrc, cpu)) + intc_enable_intr(dev, isrc); + } +} +#endif + +static int +intc_intr(void *arg) +{ + struct trapframe *frame; + struct intc_softc *sc; + uint64_t active_irq; + struct intc_irqsrc *src; + + sc = arg; + frame = curthread->td_intr_frame; + + KASSERT((frame->tf_scause & SCAUSE_INTR) != 0, + ("%s: not an interrupt frame", __func__)); + + active_irq = frame->tf_scause & SCAUSE_CODE; + + if (active_irq >= INTC_NIRQS) + return (FILTER_HANDLED); + + src = &sc->isrcs[active_irq]; + if (intr_isrc_dispatch(&src->isrc, frame) != 0) { + intc_disable_intr(sc->dev, &src->isrc); + device_printf(sc->dev, "Stray irq %lu disabled\n", + active_irq); + } + + return (FILTER_HANDLED); +} + +static device_method_t intc_methods[] = { + /* Device interface */ + DEVMETHOD(device_identify, intc_identify), + DEVMETHOD(device_probe, intc_probe), + DEVMETHOD(device_attach, intc_attach), + + /* Interrupt controller interface */ + DEVMETHOD(pic_disable_intr, intc_disable_intr), + DEVMETHOD(pic_enable_intr, intc_enable_intr), + DEVMETHOD(pic_map_intr, intc_map_intr), + DEVMETHOD(pic_setup_intr, intc_setup_intr), +#ifdef SMP + DEVMETHOD(pic_init_secondary, intc_init_secondary), +#endif + + DEVMETHOD_END +}; + +DEFINE_CLASS_0(intc, intc_driver, intc_methods, sizeof(struct intc_softc)); +EARLY_DRIVER_MODULE(intc, ofwbus, intc_driver, 0, 0, + BUS_PASS_INTERRUPT + BUS_PASS_ORDER_FIRST); Index: sys/riscv/riscv/intr_machdep.c =================================================================== --- sys/riscv/riscv/intr_machdep.c +++ /dev/null @@ -1,274 +0,0 @@ -/*- - * Copyright (c) 2015-2017 Ruslan Bukin - * All rights reserved. - * - * Portions of this software were developed by SRI International and the - * University of Cambridge Computer Laboratory under DARPA/AFRL contract - * FA8750-10-C-0237 ("CTSRD"), as part of the DARPA CRASH research programme. - * - * Portions of this software were developed by the University of Cambridge - * Computer Laboratory as part of the CTSRD Project, with support from the - * UK Higher Education Innovation Fund (HEIF). - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ - -#include -__FBSDID("$FreeBSD$"); - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include - -#ifdef SMP -#include -#endif - -void intr_irq_handler(struct trapframe *tf); - -struct intc_irqsrc { - struct intr_irqsrc isrc; - u_int irq; -}; - -struct intc_irqsrc isrcs[INTC_NIRQS]; - -static void -riscv_mask_irq(void *source) -{ - int irq; - - irq = (int)(uintptr_t)source; - - switch (irq) { - case IRQ_TIMER_SUPERVISOR: - csr_clear(sie, SIE_STIE); - break; - case IRQ_SOFTWARE_USER: - csr_clear(sie, SIE_USIE); - break; - case IRQ_SOFTWARE_SUPERVISOR: - csr_clear(sie, SIE_SSIE); - break; - default: - panic("Unknown irq %d\n", irq); - } -} - -static void -riscv_unmask_irq(void *source) -{ - int irq; - - irq = (int)(uintptr_t)source; - - switch (irq) { - case IRQ_TIMER_SUPERVISOR: - csr_set(sie, SIE_STIE); - break; - case IRQ_SOFTWARE_USER: - csr_set(sie, SIE_USIE); - break; - case IRQ_SOFTWARE_SUPERVISOR: - csr_set(sie, SIE_SSIE); - break; - default: - panic("Unknown irq %d\n", irq); - } -} - -int -riscv_setup_intr(const char *name, driver_filter_t *filt, - void (*handler)(void*), void *arg, int irq, int flags, void **cookiep) -{ - struct intr_irqsrc *isrc; - int error; - - if (irq < 0 || irq >= INTC_NIRQS) - panic("%s: unknown intr %d", __func__, irq); - - isrc = &isrcs[irq].isrc; - if (isrc->isrc_event == NULL) { - error = intr_event_create(&isrc->isrc_event, isrc, 0, irq, - riscv_mask_irq, riscv_unmask_irq, NULL, NULL, "int%d", irq); - if (error) - return (error); - riscv_unmask_irq((void*)(uintptr_t)irq); - } - - error = intr_event_add_handler(isrc->isrc_event, name, - filt, handler, arg, intr_priority(flags), flags, cookiep); - if (error) { - printf("Failed to setup intr: %d\n", irq); - return (error); - } - - return (0); -} - -int -riscv_teardown_intr(void *ih) -{ - - /* TODO */ - - return (0); -} - -void -riscv_cpu_intr(struct trapframe *frame) -{ - struct intr_irqsrc *isrc; - int active_irq; - - KASSERT((frame->tf_scause & SCAUSE_INTR) != 0, - ("riscv_cpu_intr: wrong frame passed")); - - active_irq = frame->tf_scause & SCAUSE_CODE; - - switch (active_irq) { - case IRQ_SOFTWARE_USER: - case IRQ_SOFTWARE_SUPERVISOR: - case IRQ_TIMER_SUPERVISOR: - critical_enter(); - isrc = &isrcs[active_irq].isrc; - if (intr_isrc_dispatch(isrc, frame) != 0) - printf("stray interrupt %d\n", active_irq); - critical_exit(); - break; - case IRQ_EXTERNAL_SUPERVISOR: - intr_irq_handler(frame); - break; - } -} - -#ifdef SMP -void -riscv_setup_ipihandler(driver_filter_t *filt) -{ - - riscv_setup_intr("ipi", filt, NULL, NULL, IRQ_SOFTWARE_SUPERVISOR, - INTR_TYPE_MISC, NULL); -} - -void -riscv_unmask_ipi(void) -{ - - csr_set(sie, SIE_SSIE); -} - -/* Sending IPI */ -static void -ipi_send(struct pcpu *pc, int ipi) -{ - u_long mask; - - CTR3(KTR_SMP, "%s: cpu: %d, ipi: %x", __func__, pc->pc_cpuid, ipi); - - atomic_set_32(&pc->pc_pending_ipis, ipi); - mask = (1 << pc->pc_hart); - - sbi_send_ipi(&mask); - - CTR1(KTR_SMP, "%s: sent", __func__); -} - -void -ipi_all_but_self(u_int ipi) -{ - cpuset_t other_cpus; - - other_cpus = all_cpus; - CPU_CLR(PCPU_GET(cpuid), &other_cpus); - - CTR2(KTR_SMP, "%s: ipi: %x", __func__, ipi); - ipi_selected(other_cpus, ipi); -} - -void -ipi_cpu(int cpu, u_int ipi) -{ - cpuset_t cpus; - - CPU_ZERO(&cpus); - CPU_SET(cpu, &cpus); - - ipi_send(cpuid_to_pcpu[cpu], ipi); -} - -void -ipi_selected(cpuset_t cpus, u_int ipi) -{ - struct pcpu *pc; - u_long mask; - - CTR1(KTR_SMP, "ipi_selected: ipi: %x", ipi); - - mask = 0; - STAILQ_FOREACH(pc, &cpuhead, pc_allcpu) { - if (CPU_ISSET(pc->pc_cpuid, &cpus)) { - CTR3(KTR_SMP, "%s: pc: %p, ipi: %x\n", __func__, pc, - ipi); - atomic_set_32(&pc->pc_pending_ipis, ipi); - mask |= (1 << pc->pc_hart); - } - } - sbi_send_ipi(&mask); -} -#endif - -/* Interrupt machdep initialization routine. */ -static void -intc_init(void *dummy __unused) -{ - int error; - int i; - - for (i = 0; i < INTC_NIRQS; i++) { - isrcs[i].irq = i; - error = intr_isrc_register(&isrcs[i].isrc, NULL, - 0, "intc,%u", i); - if (error != 0) - printf("Can't register interrupt %d\n", i); - } -} - -SYSINIT(intc_init, SI_SUB_INTR, SI_ORDER_MIDDLE, intc_init, NULL); Index: sys/riscv/riscv/mp_machdep.c =================================================================== --- sys/riscv/riscv/mp_machdep.c +++ sys/riscv/riscv/mp_machdep.c @@ -86,7 +86,11 @@ static device_probe_t riscv64_cpu_probe; static device_attach_t riscv64_cpu_attach; -static int ipi_handler(void *); +static void ipi_ast(void *); +static void ipi_hardclock(void *); +static void ipi_preempt(void *); +static void ipi_rendezvous(void *); +static void ipi_stop(void *); struct pcb stoppcbs[MAXCPU]; @@ -197,7 +201,12 @@ return; /* Setup the IPI handler */ - riscv_setup_ipihandler(ipi_handler); + intr_ipi_setup(IPI_AST, "ast", ipi_ast, NULL); + intr_ipi_setup(IPI_PREEMPT, "preempt", ipi_preempt, NULL); + intr_ipi_setup(IPI_RENDEZVOUS, "rendezvous", ipi_rendezvous, NULL); + intr_ipi_setup(IPI_STOP, "stop", ipi_stop, NULL); + intr_ipi_setup(IPI_STOP_HARD, "stop hard", ipi_stop, NULL); + intr_ipi_setup(IPI_HARDCLOCK, "hardclock", ipi_hardclock, NULL); atomic_store_rel_int(&aps_ready, 1); @@ -256,17 +265,14 @@ */ identify_cpu(); - /* Enable software interrupts */ - riscv_unmask_ipi(); + /* Setup and enable interrupts */ + intr_pic_init_secondary(); #ifndef EARLY_AP_STARTUP /* Start per-CPU event timers. */ cpu_initclocks_ap(); #endif - /* Enable external (PLIC) interrupts */ - csr_set(sie, SIE_SEIE); - /* Activate this hart in the kernel pmap. */ CPU_SET_ATOMIC(hart, &kernel_pmap->pm_active); @@ -317,74 +323,59 @@ SYSINIT(smp_after_idle_runnable, SI_SUB_SMP, SI_ORDER_ANY, smp_after_idle_runnable, NULL); -static int -ipi_handler(void *arg) +static void +ipi_ast(void *dummy __unused) { - u_int ipi_bitmap; - u_int cpu, ipi; - int bit; + CTR0(KTR_SMP, "IPI_AST"); +} - csr_clear(sip, SIP_SSIP); +static void +ipi_preempt(void *dummy __unused) +{ + CTR1(KTR_SMP, "%s: IPI_PREEMPT", __func__); + sched_preempt(curthread); +} + +static void +ipi_rendezvous(void *dummy __unused) +{ + CTR0(KTR_SMP, "IPI_RENDEZVOUS"); + smp_rendezvous_action(); +} + +static void +ipi_stop(void *dummy __unused) +{ + u_int cpu; + + CTR0(KTR_SMP, "IPI_STOP"); cpu = PCPU_GET(cpuid); + savectx(&stoppcbs[cpu]); - mb(); - - ipi_bitmap = atomic_readandclear_int(PCPU_PTR(pending_ipis)); - if (ipi_bitmap == 0) - return (FILTER_HANDLED); - - while ((bit = ffs(ipi_bitmap))) { - bit = (bit - 1); - ipi = (1 << bit); - ipi_bitmap &= ~ipi; - - mb(); - - switch (ipi) { - case IPI_AST: - CTR0(KTR_SMP, "IPI_AST"); - break; - case IPI_PREEMPT: - CTR1(KTR_SMP, "%s: IPI_PREEMPT", __func__); - sched_preempt(curthread); - break; - case IPI_RENDEZVOUS: - CTR0(KTR_SMP, "IPI_RENDEZVOUS"); - smp_rendezvous_action(); - break; - case IPI_STOP: - case IPI_STOP_HARD: - CTR0(KTR_SMP, (ipi == IPI_STOP) ? "IPI_STOP" : "IPI_STOP_HARD"); - savectx(&stoppcbs[cpu]); - - /* Indicate we are stopped */ - CPU_SET_ATOMIC(cpu, &stopped_cpus); - - /* Wait for restart */ - while (!CPU_ISSET(cpu, &started_cpus)) - cpu_spinwait(); - - CPU_CLR_ATOMIC(cpu, &started_cpus); - CPU_CLR_ATOMIC(cpu, &stopped_cpus); - CTR0(KTR_SMP, "IPI_STOP (restart)"); - - /* - * The kernel debugger might have set a breakpoint, - * so flush the instruction cache. - */ - fence_i(); - break; - case IPI_HARDCLOCK: - CTR1(KTR_SMP, "%s: IPI_HARDCLOCK", __func__); - hardclockintr(); - break; - default: - panic("Unknown IPI %#0x on cpu %d", ipi, curcpu); - } - } + /* Indicate we are stopped */ + CPU_SET_ATOMIC(cpu, &stopped_cpus); + + /* Wait for restart */ + while (!CPU_ISSET(cpu, &started_cpus)) + cpu_spinwait(); + + CPU_CLR_ATOMIC(cpu, &started_cpus); + CPU_CLR_ATOMIC(cpu, &stopped_cpus); + CTR0(KTR_SMP, "IPI_STOP (restart)"); + + /* + * The kernel debugger might have set a breakpoint, + * so flush the instruction cache. + */ + fence_i(); +} - return (FILTER_HANDLED); +static void +ipi_hardclock(void *dummy __unused) +{ + CTR1(KTR_SMP, "%s: IPI_HARDCLOCK", __func__); + hardclockintr(); } struct cpu_group * @@ -562,3 +553,34 @@ } } } + +void +ipi_all_but_self(u_int ipi) +{ + cpuset_t other_cpus; + + other_cpus = all_cpus; + CPU_CLR(PCPU_GET(cpuid), &other_cpus); + + CTR2(KTR_SMP, "%s: ipi: %x", __func__, ipi); + intr_ipi_send(other_cpus, ipi); +} + +void +ipi_cpu(int cpu, u_int ipi) +{ + cpuset_t cpus; + + CPU_ZERO(&cpus); + CPU_SET(cpu, &cpus); + + CTR3(KTR_SMP, "%s: cpu: %d, ipi: %x", __func__, pc->pc_cpuid, ipi); + intr_ipi_send(cpus, ipi); +} + +void +ipi_selected(cpuset_t cpus, u_int ipi) +{ + CTR1(KTR_SMP, "ipi_selected: ipi: %x", ipi); + intr_ipi_send(cpus, ipi); +} Index: sys/riscv/riscv/plic.c =================================================================== --- sys/riscv/riscv/plic.c +++ sys/riscv/riscv/plic.c @@ -94,16 +94,18 @@ struct plic_softc { device_t dev; - struct resource * intc_res; + struct resource *mem_res; + struct resource *irq_res; + void *ih; struct plic_irqsrc isrcs[PLIC_MAX_IRQS]; struct plic_context contexts[MAXCPU]; int ndev; }; #define RD4(sc, reg) \ - bus_read_4(sc->intc_res, (reg)) + bus_read_4(sc->mem_res, (reg)) #define WR4(sc, reg, val) \ - bus_write_4(sc->intc_res, (reg), (val)) + bus_write_4(sc->mem_res, (reg), (val)) static u_int plic_irq_cpu; @@ -279,9 +281,9 @@ /* Request memory resources */ rid = 0; - sc->intc_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, + sc->mem_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, RF_ACTIVE); - if (sc->intc_res == NULL) { + if (sc->mem_res == NULL) { device_printf(dev, "Error: could not allocate memory resources\n"); return (ENXIO); @@ -305,7 +307,7 @@ * * This is tricky for a few reasons. The PLIC divides the interrupt * enable, threshold, and claim bits by "context", where each context - * routes to a Core-Local Interrupt Controller (CLIC). + * routes to a core's local interrupt controller. * * The tricky part is that the PLIC spec imposes no restrictions on how * these contexts are laid out. So for example, there is no guarantee @@ -319,10 +321,15 @@ * entries that are not for supervisor external interrupts. * * 2. Walk up the device tree to find the corresponding CPU, and grab - * it's hart ID. + * its hart ID. * * 3. Convert the hart to a cpuid, and calculate the register offsets * based on the context number. + * + * 4. Save the index for the boot hart's S-mode external interrupt in + * order to allocate and setup the corresponding resource, since the + * local interrupt controller newbus device is associated with that + * specific node. */ nintr = OF_getencprop_alloc_multi(node, "interrupts-extended", sizeof(uint32_t), (void **)&cells); @@ -332,12 +339,16 @@ } /* interrupts-extended is a list of phandles and interrupt types. */ + rid = -1; for (i = 0, context = 0; i < nintr; i += 2, context++) { /* Skip M-mode external interrupts */ if (cells[i + 1] != IRQ_EXTERNAL_SUPERVISOR) continue; - /* Get the hart ID from the CLIC's phandle. */ + /* + * Get the hart ID from the core's interrupt controller + * phandle. + */ hart = plic_get_hartid(dev, OF_node_from_xref(cells[i])); if (hart < 0) { OF_prop_free(cells); @@ -352,6 +363,9 @@ return (ENXIO); } + if (cpu == 0) + rid = i / 2; + /* Set the enable and context register offsets for the CPU. */ sc->contexts[cpu].enable_offset = PLIC_ENABLE_BASE + context * PLIC_ENABLE_STRIDE; @@ -360,6 +374,20 @@ } OF_prop_free(cells); + if (rid == -1) { + device_printf(dev, + "Could not find local interrupt controller\n"); + return (ENXIO); + } + + sc->irq_res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid, + RF_ACTIVE); + if (sc->irq_res == NULL) { + device_printf(dev, + "Error: could not allocate IRQ resources\n"); + return (ENXIO); + } + /* Set the threshold for each CPU to accept all priorities. */ CPU_FOREACH(cpu) WR4(sc, PLIC_THRESHOLD(sc, cpu), 0); @@ -369,9 +397,8 @@ if (pic == NULL) return (ENXIO); - csr_set(sie, SIE_SEIE); - - return (intr_pic_claim_root(sc->dev, xref, plic_intr, sc)); + return (bus_setup_intr(dev, sc->irq_res, INTR_TYPE_CLK | INTR_MPSAFE, + plic_intr, NULL, sc, &sc->ih)); } static void Index: sys/riscv/riscv/riscv_console.c =================================================================== --- sys/riscv/riscv/riscv_console.c +++ sys/riscv/riscv/riscv_console.c @@ -246,8 +246,6 @@ sc = device_get_softc(dev); sc->dev = dev; - csr_set(sie, SIE_SSIE); - bus_generic_attach(sc->dev); return (0); Index: sys/riscv/riscv/sbi.c =================================================================== --- sys/riscv/riscv/sbi.c +++ sys/riscv/riscv/sbi.c @@ -33,11 +33,30 @@ #include #include #include +#include +#include #include +#include +#include +#include #include +#include +#include #include +#include #include +#ifdef SMP +#include +#endif + +#include +#include +#include + +#ifdef SMP +#include "pic_if.h" +#endif /* SBI Implementation-Specific Definitions */ #define OPENSBI_VERSION_MAJOR_OFFSET 16 @@ -45,6 +64,13 @@ struct sbi_softc { device_t dev; +#ifdef SMP + struct resource *irq_res; + void *ih; + struct intr_irqsrc isrc; + bool ipi_deferred_attach; + uint32_t pending_ipis[MAXCPU]; +#endif }; static struct sbi_softc *sbi_softc = NULL; @@ -337,7 +363,158 @@ device_printf(parent, "Can't add sbi child\n"); } +#ifdef SMP +static void +sbi_ipi_send(device_t dev, struct intr_irqsrc *isrc, cpuset_t cpus, + u_int ipi) +{ + struct sbi_softc *sc; + struct pcpu *pc; + u_long mask; + u_int cpu; + + sc = device_get_softc(dev); + + KASSERT(isrc == &sc->isrc, ("%s: not the IPI isrc", __func__)); + KASSERT(ipi < INTR_IPI_COUNT, + ("%s: not a valid IPI: %u", __func__, ipi)); + + mask = 0; + STAILQ_FOREACH(pc, &cpuhead, pc_allcpu) { + cpu = pc->pc_cpuid; + if (CPU_ISSET(cpu, &cpus)) { + atomic_set_32(&sc->pending_ipis[cpu], 1u << ipi); + mask |= (1ul << pc->pc_hart); + } + } + sbi_send_ipi(&mask); +} + +static int +sbi_ipi_setup(device_t dev, u_int ipi, struct intr_irqsrc **isrcp) +{ + struct sbi_softc *sc; + + sc = device_get_softc(dev); + + KASSERT(ipi < INTR_IPI_COUNT, + ("%s: not a valid IPI: %u", __func__, ipi)); + + *isrcp = &sc->isrc; + + return (0); + +} + +static int +sbi_ipi_intr(void *arg) +{ + struct sbi_softc *sc; + struct trapframe *frame; + uint32_t ipi_bitmap; + u_int cpu, ipi; + int bit; + + sc = arg; + frame = curthread->td_intr_frame; + + csr_clear(sip, SIP_SSIP); + + cpu = PCPU_GET(cpuid); + + mb(); + + ipi_bitmap = atomic_readandclear_32(&sc->pending_ipis[cpu]); + if (ipi_bitmap == 0) + return (FILTER_HANDLED); + + mb(); + + while ((bit = ffs(ipi_bitmap))) { + ipi = (bit - 1); + ipi_bitmap &= ~(1u << ipi); + + intr_ipi_dispatch(ipi, frame); + } + + return (FILTER_HANDLED); +} + +static int +sbi_ipi_attach(device_t dev) +{ + struct sbi_softc *sc; + const char *name; + int irq, rid, error; + phandle_t iparent; + pcell_t cell; + + sc = device_get_softc(dev); + + /* Local interrupt controller attaches during BUS_PASS_ORDER_FIRST */ + if (bus_current_pass < (BUS_PASS_INTERRUPT + BUS_PASS_ORDER_EARLY)) { + sc->ipi_deferred_attach = true; + return (0); + } + + if (!sc->ipi_deferred_attach) + return (0); + + /* Don't retry on every new pass if we fail */ + sc->ipi_deferred_attach = false; + + memset(sc->pending_ipis, 0, sizeof(sc->pending_ipis)); + + name = device_get_nameunit(dev); + error = intr_isrc_register(&sc->isrc, sc->dev, INTR_ISRCF_IPI, + "%s,ipi", name); + if (error != 0) { + device_printf(dev, "Can't register interrupt: %d\n", error); + return (ENXIO); + } + + iparent = OF_xref_from_node(ofw_bus_get_node(intr_irq_root_dev)); + cell = IRQ_SOFTWARE_SUPERVISOR; + irq = ofw_bus_map_intr(dev, iparent, 1, &cell); + error = bus_set_resource(dev, SYS_RES_IRQ, 0, irq, 1); + if (error != 0) { + device_printf(dev, "Unable to register IRQ resource\n"); + return (ENXIO); + } + + rid = 0; + sc->irq_res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid, + RF_ACTIVE | RF_SHAREABLE); + if (sc->irq_res == NULL) { + device_printf(dev, "Unable to alloc IRQ resource\n"); + return (ENXIO); + } + + error = bus_setup_intr(dev, sc->irq_res, INTR_TYPE_CLK, + sbi_ipi_intr, NULL, sc, &sc->ih); + if (error != 0) { + device_printf(dev, "Unable to setup IRQ resource\n"); + return (ENXIO); + } + + /* TODO: Define a set of priorities once other IPI sources exist */ + error = intr_ipi_pic_register(dev, 0); + if (error != 0) { + device_printf(dev, "Can't register as IPI source: %d\n", error); + return (ENXIO); + } + + return (0); +} + static void +sbi_new_pass(device_t dev) +{ + (void)sbi_ipi_attach(dev); +} +#endif + +static int sbi_probe(device_t dev) { device_set_desc(dev, "RISC-V Supervisor Binary Interface"); @@ -345,21 +522,31 @@ return (BUS_PROBE_NOWILDCARD); } -static void +static int sbi_attach(device_t dev) { struct sbi_softc *sc; +#ifdef SMP + int error; +#endif if (sbi_softc != NULL) return (ENXIO); sc = device_get_softc(dev); sc->dev = dev; - sbi_softc = sc; EVENTHANDLER_REGISTER(shutdown_final, sbi_shutdown_final, NULL, SHUTDOWN_PRI_LAST); +#ifdef SMP + error = sbi_ipi_attach(dev); + if (error != 0) + return (error); +#endif + + sbi_softc = sc; + return (0); } @@ -369,6 +556,15 @@ DEVMETHOD(device_probe, sbi_probe), DEVMETHOD(device_attach, sbi_attach), +#ifdef SMP + /* Bus interface */ + DEVMETHOD(bus_new_pass, sbi_new_pass), + + /* Interrupt controller interface */ + DEVMETHOD(pic_ipi_send, sbi_ipi_send), + DEVMETHOD(pic_ipi_setup, sbi_ipi_setup), +#endif + DEVMETHOD_END }; Index: sys/riscv/riscv/timer.c =================================================================== --- sys/riscv/riscv/timer.c +++ sys/riscv/riscv/timer.c @@ -46,6 +46,7 @@ #include #include #include +#include #include #include #include @@ -55,9 +56,11 @@ #include #include +#include #include struct riscv_timer_softc { + struct resource *irq_res; void *ih; uint32_t clkfreq; struct eventtimer et; @@ -108,7 +111,6 @@ if (first != 0) { counts = ((uint32_t)et->et_frequency * first) >> 32; sbi_set_timer(get_timecount() + counts); - csr_set(sie, SIE_STIE); return (0); } @@ -180,7 +182,9 @@ riscv_timer_attach(device_t dev) { struct riscv_timer_softc *sc; - int error; + int irq, rid, error; + phandle_t iparent; + pcell_t cell; sc = device_get_softc(dev); if (riscv_timer_sc != NULL) @@ -196,11 +200,28 @@ riscv_timer_sc = sc; + iparent = OF_xref_from_node(ofw_bus_get_node(intr_irq_root_dev)); + cell = IRQ_TIMER_SUPERVISOR; + irq = ofw_bus_map_intr(dev, iparent, 1, &cell); + error = bus_set_resource(dev, SYS_RES_IRQ, 0, irq, 1); + if (error != 0) { + device_printf(dev, "Unable to register IRQ resource\n"); + return (ENXIO); + } + + rid = 0; + sc->irq_res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid, + RF_ACTIVE); + if (sc->irq_res == NULL) { + device_printf(dev, "Unable to alloc IRQ resource\n"); + return (ENXIO); + } + /* Setup IRQs handler */ - error = riscv_setup_intr(device_get_nameunit(dev), riscv_timer_intr, - NULL, sc, IRQ_TIMER_SUPERVISOR, INTR_TYPE_CLK, &sc->ih); - if (error) { - device_printf(dev, "Unable to alloc int resource.\n"); + error = bus_setup_intr(dev, sc->irq_res, INTR_TYPE_CLK, + riscv_timer_intr, NULL, sc, &sc->ih); + if (error != 0) { + device_printf(dev, "Unable to setup IRQ resource\n"); return (ENXIO); } Index: sys/riscv/riscv/trap.c =================================================================== --- sys/riscv/riscv/trap.c +++ sys/riscv/riscv/trap.c @@ -71,6 +71,8 @@ #include #endif +void intr_irq_handler(struct trapframe *tf); + int (*dtrace_invop_jump_addr)(struct trapframe *); /* Called from exception.S */ @@ -287,7 +289,7 @@ exception = frame->tf_scause & SCAUSE_CODE; if ((frame->tf_scause & SCAUSE_INTR) != 0) { /* Interrupt */ - riscv_cpu_intr(frame); + intr_irq_handler(frame); return; } @@ -358,7 +360,7 @@ exception = frame->tf_scause & SCAUSE_CODE; if ((frame->tf_scause & SCAUSE_INTR) != 0) { /* Interrupt */ - riscv_cpu_intr(frame); + intr_irq_handler(frame); return; } intr_enable();