diff --git a/sys/conf/files.riscv b/sys/conf/files.riscv index 975f9d3f576e..b6a87c63422b 100644 --- a/sys/conf/files.riscv +++ b/sys/conf/files.riscv @@ -1,73 +1,74 @@ cddl/dev/dtrace/riscv/dtrace_asm.S optional dtrace compile-with "${DTRACE_S}" cddl/dev/dtrace/riscv/dtrace_subr.c optional dtrace compile-with "${DTRACE_C}" cddl/dev/dtrace/riscv/instr_size.c optional dtrace compile-with "${DTRACE_C}" cddl/dev/fbt/riscv/fbt_isa.c optional dtrace_fbt | dtraceall compile-with "${FBT_C}" crypto/des/des_enc.c optional netsmb dev/ofw/ofw_cpu.c optional fdt dev/ofw/ofw_pcib.c optional pci fdt dev/pci/pci_dw.c optional pci fdt dev/pci/pci_dw_if.m optional pci fdt dev/pci/pci_host_generic.c optional pci dev/pci/pci_host_generic_fdt.c optional pci fdt dev/uart/uart_cpu_fdt.c optional uart fdt dev/uart/uart_dev_lowrisc.c optional uart_lowrisc dev/xilinx/axi_quad_spi.c optional xilinx_spi dev/xilinx/axidma.c optional axidma xdma dev/xilinx/if_xae.c optional xae dev/xilinx/xlnx_pcib.c optional pci fdt xlnx_pcib kern/msi_if.m standard kern/pic_if.m standard kern/subr_devmap.c standard kern/subr_dummy_vdso_tc.c standard kern/subr_intr.c standard kern/subr_physmem.c standard libkern/bcopy.c standard libkern/memcmp.c standard libkern/memset.c standard libkern/strcmp.c standard libkern/strlen.c standard libkern/strncmp.c standard riscv/riscv/autoconf.c standard riscv/riscv/bus_machdep.c standard riscv/riscv/bus_space_asm.S standard riscv/riscv/busdma_bounce.c standard riscv/riscv/busdma_machdep.c standard riscv/riscv/clock.c standard riscv/riscv/copyinout.S standard riscv/riscv/cpufunc_asm.S standard riscv/riscv/db_disasm.c optional ddb riscv/riscv/db_interface.c optional ddb riscv/riscv/db_trace.c optional ddb riscv/riscv/dump_machdep.c standard riscv/riscv/elf_machdep.c standard riscv/riscv/exception.S standard riscv/riscv/exec_machdep.c standard riscv/riscv/gdb_machdep.c optional gdb -riscv/riscv/intr_machdep.c standard +riscv/riscv/intc.c standard riscv/riscv/identcpu.c standard riscv/riscv/locore.S standard no-obj riscv/riscv/machdep.c standard riscv/riscv/minidump_machdep.c standard riscv/riscv/mp_machdep.c optional smp riscv/riscv/mem.c standard riscv/riscv/nexus.c standard riscv/riscv/ofw_machdep.c optional fdt riscv/riscv/plic.c standard riscv/riscv/pmap.c standard riscv/riscv/ptrace_machdep.c standard riscv/riscv/riscv_console.c optional rcons riscv/riscv/riscv_syscon.c optional syscon riscv_syscon fdt riscv/riscv/sbi.c standard +riscv/riscv/sbi_ipi.c optional smp riscv/riscv/stack_machdep.c optional ddb | stack riscv/riscv/support.S standard riscv/riscv/swtch.S standard riscv/riscv/sys_machdep.c standard riscv/riscv/trap.c standard riscv/riscv/timer.c standard riscv/riscv/uio_machdep.c standard riscv/riscv/uma_machdep.c standard riscv/riscv/unwind.c optional ddb | kdtrace_hooks | stack riscv/riscv/vm_machdep.c standard # Zstd contrib/zstd/lib/freebsd/zstd_kfreebsd.c optional zstdio compile-with ${ZSTD_C} diff --git a/sys/riscv/include/intr.h b/sys/riscv/include/intr.h index a4e3a7093492..ad811dcbc449 100644 --- a/sys/riscv/include/intr.h +++ b/sys/riscv/include/intr.h @@ -1,81 +1,59 @@ /*- * Copyright (c) 2015-2016 Ruslan Bukin * All rights reserved. * * Portions of this software were developed by SRI International and the * University of Cambridge Computer Laboratory under DARPA/AFRL contract * FA8750-10-C-0237 ("CTSRD"), as part of the DARPA CRASH research programme. * * Portions of this software were developed by the University of Cambridge * Computer Laboratory as part of the CTSRD Project, with support from the * UK Higher Education Innovation Fund (HEIF). * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #ifndef _MACHINE_INTR_MACHDEP_H_ #define _MACHINE_INTR_MACHDEP_H_ -#define RISCV_NIRQ 1024 - #ifndef NIRQ -#define NIRQ RISCV_NIRQ +#define NIRQ 1024 #endif -#ifdef INTRNG #include -#endif - -struct trapframe; - -int riscv_teardown_intr(void *); -int riscv_setup_intr(const char *, driver_filter_t *, driver_intr_t *, - void *, int, int, void **); -void riscv_cpu_intr(struct trapframe *); - -typedef unsigned long * riscv_intrcnt_t; - -riscv_intrcnt_t riscv_intrcnt_create(const char *); -void riscv_intrcnt_setname(riscv_intrcnt_t, const char *); - -#ifdef SMP -void riscv_setup_ipihandler(driver_filter_t *); -void riscv_unmask_ipi(void); -#endif enum { IRQ_SOFTWARE_USER, IRQ_SOFTWARE_SUPERVISOR, IRQ_SOFTWARE_HYPERVISOR, IRQ_SOFTWARE_MACHINE, IRQ_TIMER_USER, IRQ_TIMER_SUPERVISOR, IRQ_TIMER_HYPERVISOR, IRQ_TIMER_MACHINE, IRQ_EXTERNAL_USER, IRQ_EXTERNAL_SUPERVISOR, IRQ_EXTERNAL_HYPERVISOR, IRQ_EXTERNAL_MACHINE, - INTC_NIRQS }; #endif /* !_MACHINE_INTR_MACHDEP_H_ */ diff --git a/sys/riscv/include/smp.h b/sys/riscv/include/smp.h index 8bafc1ac6a3a..863f0bbd6d21 100644 --- a/sys/riscv/include/smp.h +++ b/sys/riscv/include/smp.h @@ -1,53 +1,54 @@ /*- * Copyright (c) 2016 Ruslan Bukin * All rights reserved. * * Portions of this software were developed by SRI International and the * University of Cambridge Computer Laboratory under DARPA/AFRL contract * FA8750-10-C-0237 ("CTSRD"), as part of the DARPA CRASH research programme. * * Portions of this software were developed by the University of Cambridge * Computer Laboratory as part of the CTSRD Project, with support from the * UK Higher Education Innovation Fund (HEIF). * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #ifndef _MACHINE_SMP_H_ #define _MACHINE_SMP_H_ #include -#define IPI_AST (1 << 0) -#define IPI_PREEMPT (1 << 1) -#define IPI_RENDEZVOUS (1 << 2) -#define IPI_STOP (1 << 3) -#define IPI_STOP_HARD (1 << 4) -#define IPI_HARDCLOCK (1 << 5) - -#define INTR_IPI_COUNT 1 +enum { + IPI_AST, + IPI_PREEMPT, + IPI_RENDEZVOUS, + IPI_STOP, + IPI_STOP_HARD, + IPI_HARDCLOCK, + INTR_IPI_COUNT +}; void ipi_all_but_self(u_int ipi); void ipi_cpu(int cpu, u_int ipi); void ipi_selected(cpuset_t cpus, u_int ipi); #endif /* !_MACHINE_SMP_H_ */ diff --git a/sys/riscv/riscv/intc.c b/sys/riscv/riscv/intc.c new file mode 100644 index 000000000000..399bb05bbcfe --- /dev/null +++ b/sys/riscv/riscv/intc.c @@ -0,0 +1,311 @@ +/*- + * Copyright (c) 2015-2017 Ruslan Bukin + * All rights reserved. + * Copyright (c) 2021 Jessica Clarke + * + * Portions of this software were developed by SRI International and the + * University of Cambridge Computer Laboratory under DARPA/AFRL contract + * FA8750-10-C-0237 ("CTSRD"), as part of the DARPA CRASH research programme. + * + * Portions of this software were developed by the University of Cambridge + * Computer Laboratory as part of the CTSRD Project, with support from the + * UK Higher Education Innovation Fund (HEIF). + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include + +#include +#include +#include + +#include "pic_if.h" + +#define INTC_NIRQS 16 + +struct intc_irqsrc { + struct intr_irqsrc isrc; + u_int irq; +}; + +struct intc_softc { + device_t dev; + struct intc_irqsrc isrcs[INTC_NIRQS]; +}; + +static int intc_intr(void *arg); + +static phandle_t +intc_ofw_find(device_t dev, uint32_t hartid) +{ + phandle_t node; + pcell_t reg; + + node = OF_finddevice("/cpus"); + if (node == -1) { + device_printf(dev, "Can't find cpus node\n"); + return ((phandle_t)-1); + } + + for (node = OF_child(node); node != 0; node = OF_peer(node)) { + if (!ofw_bus_node_status_okay(node)) + continue; + + if (!ofw_bus_node_is_compatible(node, "riscv")) + continue; + + if (OF_searchencprop(node, "reg", ®, sizeof(reg)) == -1) + continue; + + if (reg == hartid) + break; + } + + if (node == 0) { + device_printf(dev, "Can't find boot cpu node\n"); + return ((phandle_t)-1); + } + + for (node = OF_child(node); node != 0; node = OF_peer(node)) { + if (!ofw_bus_node_status_okay(node)) + continue; + + if (ofw_bus_node_is_compatible(node, "riscv,cpu-intc")) + break; + } + + if (node == 0) { + device_printf(dev, + "Can't find boot cpu local interrupt controller\n"); + return ((phandle_t)-1); + } + + return (node); +} + +static void +intc_identify(driver_t *driver, device_t parent) +{ + device_t dev; + phandle_t node; + + if (device_find_child(parent, "intc", -1) != NULL) + return; + + node = intc_ofw_find(parent, PCPU_GET(hart)); + if (node == -1) + return; + + dev = simplebus_add_device(parent, node, 0, "intc", -1, NULL); + if (dev == NULL) + device_printf(parent, "Can't add intc child\n"); +} + +static int +intc_probe(device_t dev) +{ + device_set_desc(dev, "RISC-V Local Interrupt Controller"); + + return (BUS_PROBE_NOWILDCARD); +} + +static int +intc_attach(device_t dev) +{ + struct intc_irqsrc *isrcs; + struct intc_softc *sc; + struct intr_pic *pic; + const char *name; + phandle_t xref; + u_int flags; + int i, error; + + sc = device_get_softc(dev); + sc->dev = dev; + + name = device_get_nameunit(dev); + xref = OF_xref_from_node(ofw_bus_get_node(dev)); + + isrcs = sc->isrcs; + for (i = 0; i < INTC_NIRQS; i++) { + isrcs[i].irq = i; + flags = i == IRQ_SOFTWARE_SUPERVISOR + ? INTR_ISRCF_IPI : INTR_ISRCF_PPI; + error = intr_isrc_register(&isrcs[i].isrc, sc->dev, flags, + "%s,%u", name, i); + if (error != 0) { + device_printf(dev, "Can't register interrupt %d\n", i); + return (error); + } + } + + pic = intr_pic_register(sc->dev, xref); + if (pic == NULL) + return (ENXIO); + + return (intr_pic_claim_root(sc->dev, xref, intc_intr, sc)); +} + +static void +intc_disable_intr(device_t dev, struct intr_irqsrc *isrc) +{ + u_int irq; + + irq = ((struct intc_irqsrc *)isrc)->irq; + if (irq >= INTC_NIRQS) + panic("%s: Unsupported IRQ %u", __func__, irq); + + csr_clear(sie, 1ul << irq); +} + +static void +intc_enable_intr(device_t dev, struct intr_irqsrc *isrc) +{ + u_int irq; + + irq = ((struct intc_irqsrc *)isrc)->irq; + if (irq >= INTC_NIRQS) + panic("%s: Unsupported IRQ %u", __func__, irq); + + csr_set(sie, 1ul << irq); +} + +static int +intc_map_intr(device_t dev, struct intr_map_data *data, + struct intr_irqsrc **isrcp) +{ + struct intr_map_data_fdt *daf; + struct intc_softc *sc; + + sc = device_get_softc(dev); + + if (data->type != INTR_MAP_DATA_FDT) + return (ENOTSUP); + + daf = (struct intr_map_data_fdt *)data; + if (daf->ncells != 1 || daf->cells[0] >= INTC_NIRQS) + return (EINVAL); + + *isrcp = &sc->isrcs[daf->cells[0]].isrc; + + return (0); +} + +static int +intc_setup_intr(device_t dev, struct intr_irqsrc *isrc, + struct resource *res, struct intr_map_data *data) +{ + if (isrc->isrc_flags & INTR_ISRCF_PPI) + CPU_SET(PCPU_GET(cpuid), &isrc->isrc_cpu); + + return (0); +} + +#ifdef SMP +static void +intc_init_secondary(device_t dev) +{ + struct intc_softc *sc; + struct intr_irqsrc *isrc; + u_int cpu, irq; + + sc = device_get_softc(dev); + cpu = PCPU_GET(cpuid); + + /* Unmask attached interrupts */ + for (irq = 0; irq < INTC_NIRQS; irq++) { + isrc = &sc->isrcs[irq].isrc; + if (intr_isrc_init_on_cpu(isrc, cpu)) + intc_enable_intr(dev, isrc); + } +} +#endif + +static int +intc_intr(void *arg) +{ + struct trapframe *frame; + struct intc_softc *sc; + uint64_t active_irq; + struct intc_irqsrc *src; + + sc = arg; + frame = curthread->td_intr_frame; + + KASSERT((frame->tf_scause & SCAUSE_INTR) != 0, + ("%s: not an interrupt frame", __func__)); + + active_irq = frame->tf_scause & SCAUSE_CODE; + + if (active_irq >= INTC_NIRQS) + return (FILTER_HANDLED); + + src = &sc->isrcs[active_irq]; + if (intr_isrc_dispatch(&src->isrc, frame) != 0) { + intc_disable_intr(sc->dev, &src->isrc); + device_printf(sc->dev, "Stray irq %lu disabled\n", + active_irq); + } + + return (FILTER_HANDLED); +} + +static device_method_t intc_methods[] = { + /* Device interface */ + DEVMETHOD(device_identify, intc_identify), + DEVMETHOD(device_probe, intc_probe), + DEVMETHOD(device_attach, intc_attach), + + /* Interrupt controller interface */ + DEVMETHOD(pic_disable_intr, intc_disable_intr), + DEVMETHOD(pic_enable_intr, intc_enable_intr), + DEVMETHOD(pic_map_intr, intc_map_intr), + DEVMETHOD(pic_setup_intr, intc_setup_intr), +#ifdef SMP + DEVMETHOD(pic_init_secondary, intc_init_secondary), +#endif + + DEVMETHOD_END +}; + +DEFINE_CLASS_0(intc, intc_driver, intc_methods, sizeof(struct intc_softc)); +EARLY_DRIVER_MODULE(intc, ofwbus, intc_driver, 0, 0, + BUS_PASS_INTERRUPT + BUS_PASS_ORDER_FIRST); diff --git a/sys/riscv/riscv/intr_machdep.c b/sys/riscv/riscv/intr_machdep.c deleted file mode 100644 index 6c00b29c120d..000000000000 --- a/sys/riscv/riscv/intr_machdep.c +++ /dev/null @@ -1,274 +0,0 @@ -/*- - * Copyright (c) 2015-2017 Ruslan Bukin - * All rights reserved. - * - * Portions of this software were developed by SRI International and the - * University of Cambridge Computer Laboratory under DARPA/AFRL contract - * FA8750-10-C-0237 ("CTSRD"), as part of the DARPA CRASH research programme. - * - * Portions of this software were developed by the University of Cambridge - * Computer Laboratory as part of the CTSRD Project, with support from the - * UK Higher Education Innovation Fund (HEIF). - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include - -#ifdef SMP -#include -#endif - -void intr_irq_handler(struct trapframe *tf); - -struct intc_irqsrc { - struct intr_irqsrc isrc; - u_int irq; -}; - -struct intc_irqsrc isrcs[INTC_NIRQS]; - -static void -riscv_mask_irq(void *source) -{ - int irq; - - irq = (int)(uintptr_t)source; - - switch (irq) { - case IRQ_TIMER_SUPERVISOR: - csr_clear(sie, SIE_STIE); - break; - case IRQ_SOFTWARE_USER: - csr_clear(sie, SIE_USIE); - break; - case IRQ_SOFTWARE_SUPERVISOR: - csr_clear(sie, SIE_SSIE); - break; - default: - panic("Unknown irq %d\n", irq); - } -} - -static void -riscv_unmask_irq(void *source) -{ - int irq; - - irq = (int)(uintptr_t)source; - - switch (irq) { - case IRQ_TIMER_SUPERVISOR: - csr_set(sie, SIE_STIE); - break; - case IRQ_SOFTWARE_USER: - csr_set(sie, SIE_USIE); - break; - case IRQ_SOFTWARE_SUPERVISOR: - csr_set(sie, SIE_SSIE); - break; - default: - panic("Unknown irq %d\n", irq); - } -} - -int -riscv_setup_intr(const char *name, driver_filter_t *filt, - void (*handler)(void*), void *arg, int irq, int flags, void **cookiep) -{ - struct intr_irqsrc *isrc; - int error; - - if (irq < 0 || irq >= INTC_NIRQS) - panic("%s: unknown intr %d", __func__, irq); - - isrc = &isrcs[irq].isrc; - if (isrc->isrc_event == NULL) { - error = intr_event_create(&isrc->isrc_event, isrc, 0, irq, - riscv_mask_irq, riscv_unmask_irq, NULL, NULL, "int%d", irq); - if (error) - return (error); - riscv_unmask_irq((void*)(uintptr_t)irq); - } - - error = intr_event_add_handler(isrc->isrc_event, name, - filt, handler, arg, intr_priority(flags), flags, cookiep); - if (error) { - printf("Failed to setup intr: %d\n", irq); - return (error); - } - - return (0); -} - -int -riscv_teardown_intr(void *ih) -{ - - /* TODO */ - - return (0); -} - -void -riscv_cpu_intr(struct trapframe *frame) -{ - struct intr_irqsrc *isrc; - int active_irq; - - KASSERT((frame->tf_scause & SCAUSE_INTR) != 0, - ("riscv_cpu_intr: wrong frame passed")); - - active_irq = frame->tf_scause & SCAUSE_CODE; - - CTR3(KTR_TRAP, "%s: irq=%d, umode=%d", __func__, active_irq, - TRAPF_USERMODE(frame)); - - switch (active_irq) { - case IRQ_SOFTWARE_USER: - case IRQ_SOFTWARE_SUPERVISOR: - case IRQ_TIMER_SUPERVISOR: - critical_enter(); - isrc = &isrcs[active_irq].isrc; - if (intr_isrc_dispatch(isrc, frame) != 0) - printf("stray interrupt %d\n", active_irq); - critical_exit(); - break; - case IRQ_EXTERNAL_SUPERVISOR: - intr_irq_handler(frame); - break; - } -} - -#ifdef SMP -void -riscv_setup_ipihandler(driver_filter_t *filt) -{ - - riscv_setup_intr("ipi", filt, NULL, NULL, IRQ_SOFTWARE_SUPERVISOR, - INTR_TYPE_MISC, NULL); -} - -void -riscv_unmask_ipi(void) -{ - - csr_set(sie, SIE_SSIE); -} - -/* Sending IPI */ -static void -ipi_send(struct pcpu *pc, int ipi) -{ - u_long mask; - - CTR3(KTR_SMP, "%s: cpu: %d, ipi: %x", __func__, pc->pc_cpuid, ipi); - - atomic_set_32(&pc->pc_pending_ipis, ipi); - mask = (1 << pc->pc_hart); - - sbi_send_ipi(&mask); - - CTR1(KTR_SMP, "%s: sent", __func__); -} - -void -ipi_all_but_self(u_int ipi) -{ - cpuset_t other_cpus; - - other_cpus = all_cpus; - CPU_CLR(PCPU_GET(cpuid), &other_cpus); - - CTR2(KTR_SMP, "%s: ipi: %x", __func__, ipi); - ipi_selected(other_cpus, ipi); -} - -void -ipi_cpu(int cpu, u_int ipi) -{ - cpuset_t cpus; - - CPU_ZERO(&cpus); - CPU_SET(cpu, &cpus); - - ipi_send(cpuid_to_pcpu[cpu], ipi); -} - -void -ipi_selected(cpuset_t cpus, u_int ipi) -{ - struct pcpu *pc; - u_long mask; - - CTR1(KTR_SMP, "ipi_selected: ipi: %x", ipi); - - mask = 0; - STAILQ_FOREACH(pc, &cpuhead, pc_allcpu) { - if (CPU_ISSET(pc->pc_cpuid, &cpus)) { - CTR3(KTR_SMP, "%s: pc: %p, ipi: %x\n", __func__, pc, - ipi); - atomic_set_32(&pc->pc_pending_ipis, ipi); - mask |= (1 << pc->pc_hart); - } - } - sbi_send_ipi(&mask); -} -#endif - -/* Interrupt machdep initialization routine. */ -static void -intc_init(void *dummy __unused) -{ - int error; - int i; - - for (i = 0; i < INTC_NIRQS; i++) { - isrcs[i].irq = i; - error = intr_isrc_register(&isrcs[i].isrc, NULL, - 0, "intc,%u", i); - if (error != 0) - printf("Can't register interrupt %d\n", i); - } -} - -SYSINIT(intc_init, SI_SUB_INTR, SI_ORDER_MIDDLE, intc_init, NULL); diff --git a/sys/riscv/riscv/mp_machdep.c b/sys/riscv/riscv/mp_machdep.c index 2c144e457b4e..be5c50485bf4 100644 --- a/sys/riscv/riscv/mp_machdep.c +++ b/sys/riscv/riscv/mp_machdep.c @@ -1,576 +1,598 @@ /*- * Copyright (c) 2015 The FreeBSD Foundation * Copyright (c) 2016 Ruslan Bukin * All rights reserved. * * Portions of this software were developed by Andrew Turner under * sponsorship from the FreeBSD Foundation. * * Portions of this software were developed by SRI International and the * University of Cambridge Computer Laboratory under DARPA/AFRL contract * FA8750-10-C-0237 ("CTSRD"), as part of the DARPA CRASH research programme. * * Portions of this software were developed by the University of Cambridge * Computer Laboratory as part of the CTSRD Project, with support from the * UK Higher Education Innovation Fund (HEIF). * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include "opt_kstack_pages.h" #include "opt_platform.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #ifdef FDT #include #include #endif #define MP_BOOTSTACK_SIZE (kstack_pages * PAGE_SIZE) uint32_t __riscv_boot_ap[MAXCPU]; static enum { CPUS_UNKNOWN, #ifdef FDT CPUS_FDT, #endif } cpu_enum_method; static device_identify_t riscv64_cpu_identify; static device_probe_t riscv64_cpu_probe; static device_attach_t riscv64_cpu_attach; -static int ipi_handler(void *); +static void ipi_ast(void *); +static void ipi_hardclock(void *); +static void ipi_preempt(void *); +static void ipi_rendezvous(void *); +static void ipi_stop(void *); extern uint32_t boot_hart; extern cpuset_t all_harts; #ifdef INVARIANTS static uint32_t cpu_reg[MAXCPU][2]; #endif static device_t cpu_list[MAXCPU]; void mpentry(u_long hartid); void init_secondary(uint64_t); static struct mtx ap_boot_mtx; /* Stacks for AP initialization, discarded once idle threads are started. */ void *bootstack; static void *bootstacks[MAXCPU]; /* Count of started APs, used to synchronize access to bootstack. */ static volatile int aps_started; /* Set to 1 once we're ready to let the APs out of the pen. */ static volatile int aps_ready; /* Temporary variables for init_secondary() */ void *dpcpu[MAXCPU - 1]; static device_method_t riscv64_cpu_methods[] = { /* Device interface */ DEVMETHOD(device_identify, riscv64_cpu_identify), DEVMETHOD(device_probe, riscv64_cpu_probe), DEVMETHOD(device_attach, riscv64_cpu_attach), DEVMETHOD_END }; static driver_t riscv64_cpu_driver = { "riscv64_cpu", riscv64_cpu_methods, 0 }; DRIVER_MODULE(riscv64_cpu, cpu, riscv64_cpu_driver, 0, 0); static void riscv64_cpu_identify(driver_t *driver, device_t parent) { if (device_find_child(parent, "riscv64_cpu", -1) != NULL) return; if (BUS_ADD_CHILD(parent, 0, "riscv64_cpu", -1) == NULL) device_printf(parent, "add child failed\n"); } static int riscv64_cpu_probe(device_t dev) { u_int cpuid; cpuid = device_get_unit(dev); if (cpuid >= MAXCPU || cpuid > mp_maxid) return (EINVAL); device_quiet(dev); return (0); } static int riscv64_cpu_attach(device_t dev) { const uint32_t *reg; size_t reg_size; u_int cpuid; int i; cpuid = device_get_unit(dev); if (cpuid >= MAXCPU || cpuid > mp_maxid) return (EINVAL); KASSERT(cpu_list[cpuid] == NULL, ("Already have cpu %u", cpuid)); reg = cpu_get_cpuid(dev, ®_size); if (reg == NULL) return (EINVAL); if (bootverbose) { device_printf(dev, "register <"); for (i = 0; i < reg_size; i++) printf("%s%x", (i == 0) ? "" : " ", reg[i]); printf(">\n"); } /* Set the device to start it later */ cpu_list[cpuid] = dev; return (0); } static void release_aps(void *dummy __unused) { cpuset_t mask; int i; if (mp_ncpus == 1) return; - /* Setup the IPI handler */ - riscv_setup_ipihandler(ipi_handler); + /* Setup the IPI handlers */ + intr_ipi_setup(IPI_AST, "ast", ipi_ast, NULL); + intr_ipi_setup(IPI_PREEMPT, "preempt", ipi_preempt, NULL); + intr_ipi_setup(IPI_RENDEZVOUS, "rendezvous", ipi_rendezvous, NULL); + intr_ipi_setup(IPI_STOP, "stop", ipi_stop, NULL); + intr_ipi_setup(IPI_STOP_HARD, "stop hard", ipi_stop, NULL); + intr_ipi_setup(IPI_HARDCLOCK, "hardclock", ipi_hardclock, NULL); atomic_store_rel_int(&aps_ready, 1); /* Wake up the other CPUs */ mask = all_harts; CPU_CLR(boot_hart, &mask); printf("Release APs\n"); sbi_send_ipi(mask.__bits); for (i = 0; i < 2000; i++) { if (atomic_load_acq_int(&smp_started)) return; DELAY(1000); } printf("APs not started\n"); } SYSINIT(start_aps, SI_SUB_SMP, SI_ORDER_FIRST, release_aps, NULL); void init_secondary(uint64_t hart) { struct pcpu *pcpup; u_int cpuid; /* Renumber this cpu */ cpuid = hart; if (cpuid < boot_hart) cpuid += mp_maxid + 1; cpuid -= boot_hart; /* Setup the pcpu pointer */ pcpup = &__pcpu[cpuid]; __asm __volatile("mv tp, %0" :: "r"(pcpup)); /* Workaround: make sure wfi doesn't halt the hart */ csr_set(sie, SIE_SSIE); csr_set(sip, SIE_SSIE); /* Signal the BSP and spin until it has released all APs. */ atomic_add_int(&aps_started, 1); while (!atomic_load_int(&aps_ready)) __asm __volatile("wfi"); /* Initialize curthread */ KASSERT(PCPU_GET(idlethread) != NULL, ("no idle thread")); pcpup->pc_curthread = pcpup->pc_idlethread; schedinit_ap(); - /* Enable software interrupts */ - riscv_unmask_ipi(); + /* Setup and enable interrupts */ + intr_pic_init_secondary(); #ifndef EARLY_AP_STARTUP /* Start per-CPU event timers. */ cpu_initclocks_ap(); #endif - /* Enable external (PLIC) interrupts */ - csr_set(sie, SIE_SEIE); - /* Activate this hart in the kernel pmap. */ CPU_SET_ATOMIC(hart, &kernel_pmap->pm_active); /* Activate process 0's pmap. */ pmap_activate_boot(vmspace_pmap(proc0.p_vmspace)); mtx_lock_spin(&ap_boot_mtx); atomic_add_rel_32(&smp_cpus, 1); if (smp_cpus == mp_ncpus) { /* enable IPI's, tlb shootdown, freezes etc */ atomic_store_rel_int(&smp_started, 1); } mtx_unlock_spin(&ap_boot_mtx); if (bootverbose) printf("Secondary CPU %u fully online\n", cpuid); /* Enter the scheduler */ sched_ap_entry(); panic("scheduler returned us to init_secondary"); /* NOTREACHED */ } static void smp_after_idle_runnable(void *arg __unused) { int cpu; if (mp_ncpus == 1) return; KASSERT(smp_started != 0, ("%s: SMP not started yet", __func__)); /* * Wait for all APs to handle an interrupt. After that, we know that * the APs have entered the scheduler at least once, so the boot stacks * are safe to free. */ smp_rendezvous(smp_no_rendezvous_barrier, NULL, smp_no_rendezvous_barrier, NULL); for (cpu = 1; cpu <= mp_maxid; cpu++) { if (bootstacks[cpu] != NULL) kmem_free(bootstacks[cpu], MP_BOOTSTACK_SIZE); } } SYSINIT(smp_after_idle_runnable, SI_SUB_SMP, SI_ORDER_ANY, smp_after_idle_runnable, NULL); -static int -ipi_handler(void *arg) +static void +ipi_ast(void *dummy __unused) +{ + CTR0(KTR_SMP, "IPI_AST"); +} + +static void +ipi_preempt(void *dummy __unused) +{ + CTR1(KTR_SMP, "%s: IPI_PREEMPT", __func__); + sched_preempt(curthread); +} + +static void +ipi_rendezvous(void *dummy __unused) { - u_int ipi_bitmap; - u_int cpu, ipi; - int bit; + CTR0(KTR_SMP, "IPI_RENDEZVOUS"); + smp_rendezvous_action(); +} + +static void +ipi_stop(void *dummy __unused) +{ + u_int cpu; - csr_clear(sip, SIP_SSIP); + CTR0(KTR_SMP, "IPI_STOP"); cpu = PCPU_GET(cpuid); + savectx(&stoppcbs[cpu]); - mb(); - - ipi_bitmap = atomic_readandclear_int(PCPU_PTR(pending_ipis)); - if (ipi_bitmap == 0) - return (FILTER_HANDLED); - - while ((bit = ffs(ipi_bitmap))) { - bit = (bit - 1); - ipi = (1 << bit); - ipi_bitmap &= ~ipi; - - mb(); - - switch (ipi) { - case IPI_AST: - CTR0(KTR_SMP, "IPI_AST"); - break; - case IPI_PREEMPT: - CTR1(KTR_SMP, "%s: IPI_PREEMPT", __func__); - sched_preempt(curthread); - break; - case IPI_RENDEZVOUS: - CTR0(KTR_SMP, "IPI_RENDEZVOUS"); - smp_rendezvous_action(); - break; - case IPI_STOP: - case IPI_STOP_HARD: - CTR0(KTR_SMP, (ipi == IPI_STOP) ? "IPI_STOP" : "IPI_STOP_HARD"); - savectx(&stoppcbs[cpu]); - - /* Indicate we are stopped */ - CPU_SET_ATOMIC(cpu, &stopped_cpus); - - /* Wait for restart */ - while (!CPU_ISSET(cpu, &started_cpus)) - cpu_spinwait(); - - CPU_CLR_ATOMIC(cpu, &started_cpus); - CPU_CLR_ATOMIC(cpu, &stopped_cpus); - CTR0(KTR_SMP, "IPI_STOP (restart)"); - - /* - * The kernel debugger might have set a breakpoint, - * so flush the instruction cache. - */ - fence_i(); - break; - case IPI_HARDCLOCK: - CTR1(KTR_SMP, "%s: IPI_HARDCLOCK", __func__); - hardclockintr(); - break; - default: - panic("Unknown IPI %#0x on cpu %d", ipi, curcpu); - } - } + /* Indicate we are stopped */ + CPU_SET_ATOMIC(cpu, &stopped_cpus); + + /* Wait for restart */ + while (!CPU_ISSET(cpu, &started_cpus)) + cpu_spinwait(); - return (FILTER_HANDLED); + CPU_CLR_ATOMIC(cpu, &started_cpus); + CPU_CLR_ATOMIC(cpu, &stopped_cpus); + CTR0(KTR_SMP, "IPI_STOP (restart)"); + + /* + * The kernel debugger might have set a breakpoint, + * so flush the instruction cache. + */ + fence_i(); +} + +static void +ipi_hardclock(void *dummy __unused) +{ + CTR1(KTR_SMP, "%s: IPI_HARDCLOCK", __func__); + hardclockintr(); } struct cpu_group * cpu_topo(void) { return (smp_topo_none()); } /* Determine if we running MP machine */ int cpu_mp_probe(void) { return (mp_ncpus > 1); } #ifdef FDT static bool cpu_check_mmu(u_int id __unused, phandle_t node, u_int addr_size __unused, pcell_t *reg __unused) { char type[32]; /* Check if this hart supports MMU. */ if (OF_getprop(node, "mmu-type", (void *)type, sizeof(type)) == -1 || strncmp(type, "riscv,none", 10) == 0) return (false); return (true); } static bool cpu_init_fdt(u_int id, phandle_t node, u_int addr_size, pcell_t *reg) { struct pcpu *pcpup; vm_paddr_t start_addr; uint64_t hart; u_int cpuid; int naps; int error; if (!cpu_check_mmu(id, node, addr_size, reg)) return (false); KASSERT(id < MAXCPU, ("Too many CPUs")); KASSERT(addr_size == 1 || addr_size == 2, ("Invalid register size")); #ifdef INVARIANTS cpu_reg[id][0] = reg[0]; if (addr_size == 2) cpu_reg[id][1] = reg[1]; #endif hart = reg[0]; if (addr_size == 2) { hart <<= 32; hart |= reg[1]; } KASSERT(hart < MAXCPU, ("Too many harts.")); /* We are already running on this cpu */ if (hart == boot_hart) return (true); /* * Rotate the CPU IDs to put the boot CPU as CPU 0. * We keep the other CPUs ordered. */ cpuid = hart; if (cpuid < boot_hart) cpuid += mp_maxid + 1; cpuid -= boot_hart; /* Check if we are able to start this cpu */ if (cpuid > mp_maxid) return (false); /* * Depending on the SBI implementation, APs are waiting either in * locore.S or to be activated explicitly, via SBI call. */ if (sbi_probe_extension(SBI_EXT_ID_HSM) != 0) { start_addr = pmap_kextract((vm_offset_t)mpentry); error = sbi_hsm_hart_start(hart, start_addr, 0); if (error != 0) { mp_ncpus--; /* Send a warning to the user and continue. */ printf("AP %u (hart %lu) failed to start, error %d\n", cpuid, hart, error); return (false); } } pcpup = &__pcpu[cpuid]; pcpu_init(pcpup, cpuid, sizeof(struct pcpu)); pcpup->pc_hart = hart; dpcpu[cpuid - 1] = kmem_malloc(DPCPU_SIZE, M_WAITOK | M_ZERO); dpcpu_init(dpcpu[cpuid - 1], cpuid); bootstacks[cpuid] = kmem_malloc(MP_BOOTSTACK_SIZE, M_WAITOK | M_ZERO); naps = atomic_load_int(&aps_started); bootstack = (char *)bootstacks[cpuid] + MP_BOOTSTACK_SIZE; if (bootverbose) printf("Starting CPU %u (hart %lx)\n", cpuid, hart); atomic_store_32(&__riscv_boot_ap[hart], 1); /* Wait for the AP to switch to its boot stack. */ while (atomic_load_int(&aps_started) < naps + 1) cpu_spinwait(); CPU_SET(cpuid, &all_cpus); CPU_SET(hart, &all_harts); return (true); } #endif /* Initialize and fire up non-boot processors */ void cpu_mp_start(void) { u_int cpu; mtx_init(&ap_boot_mtx, "ap boot", NULL, MTX_SPIN); CPU_SET(0, &all_cpus); CPU_SET(boot_hart, &all_harts); switch(cpu_enum_method) { #ifdef FDT case CPUS_FDT: ofw_cpu_early_foreach(cpu_init_fdt, true); break; #endif case CPUS_UNKNOWN: break; } CPU_FOREACH(cpu) { /* Already identified. */ if (cpu == 0) continue; identify_cpu(cpu); } } /* Introduce rest of cores to the world */ void cpu_mp_announce(void) { u_int cpu; CPU_FOREACH(cpu) { /* Already announced. */ if (cpu == 0) continue; printcpuinfo(cpu); } } void cpu_mp_setmaxid(void) { int cores; #ifdef FDT cores = ofw_cpu_early_foreach(cpu_check_mmu, true); if (cores > 0) { cores = MIN(cores, MAXCPU); if (bootverbose) printf("Found %d CPUs in the device tree\n", cores); mp_ncpus = cores; mp_maxid = cores - 1; cpu_enum_method = CPUS_FDT; } else #endif { if (bootverbose) printf("No CPU data, limiting to 1 core\n"); mp_ncpus = 1; mp_maxid = 0; } if (TUNABLE_INT_FETCH("hw.ncpu", &cores)) { if (cores > 0 && cores < mp_ncpus) { mp_ncpus = cores; mp_maxid = cores - 1; } } } + +void +ipi_all_but_self(u_int ipi) +{ + cpuset_t other_cpus; + + other_cpus = all_cpus; + CPU_CLR(PCPU_GET(cpuid), &other_cpus); + + CTR2(KTR_SMP, "%s: ipi: %x", __func__, ipi); + intr_ipi_send(other_cpus, ipi); +} + +void +ipi_cpu(int cpu, u_int ipi) +{ + cpuset_t cpus; + + CPU_ZERO(&cpus); + CPU_SET(cpu, &cpus); + + CTR3(KTR_SMP, "%s: cpu: %d, ipi: %x", __func__, cpu, ipi); + intr_ipi_send(cpus, ipi); +} + +void +ipi_selected(cpuset_t cpus, u_int ipi) +{ + CTR1(KTR_SMP, "ipi_selected: ipi: %x", ipi); + intr_ipi_send(cpus, ipi); +} diff --git a/sys/riscv/riscv/plic.c b/sys/riscv/riscv/plic.c index 1e8a03127038..406aff5bbff0 100644 --- a/sys/riscv/riscv/plic.c +++ b/sys/riscv/riscv/plic.c @@ -1,468 +1,495 @@ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2018 Ruslan Bukin * All rights reserved. * Copyright (c) 2019 Mitchell Horne * * Portions of this software were developed by SRI International and the * University of Cambridge Computer Laboratory (Department of Computer Science * and Technology) under DARPA contract HR0011-18-C-0016 ("ECATS"), as part of * the DARPA SSITH research programme. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "pic_if.h" #define PLIC_MAX_IRQS 1024 #define PLIC_PRIORITY_BASE 0x000000U #define PLIC_ENABLE_BASE 0x002000U #define PLIC_ENABLE_STRIDE 0x80U #define PLIC_CONTEXT_BASE 0x200000U #define PLIC_CONTEXT_STRIDE 0x1000U #define PLIC_CONTEXT_THRESHOLD 0x0U #define PLIC_CONTEXT_CLAIM 0x4U #define PLIC_PRIORITY(n) (PLIC_PRIORITY_BASE + (n) * sizeof(uint32_t)) #define PLIC_ENABLE(sc, n, h) \ (sc->contexts[h].enable_offset + ((n) / 32) * sizeof(uint32_t)) #define PLIC_THRESHOLD(sc, h) \ (sc->contexts[h].context_offset + PLIC_CONTEXT_THRESHOLD) #define PLIC_CLAIM(sc, h) \ (sc->contexts[h].context_offset + PLIC_CONTEXT_CLAIM) static pic_disable_intr_t plic_disable_intr; static pic_enable_intr_t plic_enable_intr; static pic_map_intr_t plic_map_intr; static pic_setup_intr_t plic_setup_intr; static pic_post_ithread_t plic_post_ithread; static pic_pre_ithread_t plic_pre_ithread; static pic_bind_intr_t plic_bind_intr; struct plic_irqsrc { struct intr_irqsrc isrc; u_int irq; }; struct plic_context { bus_size_t enable_offset; bus_size_t context_offset; }; struct plic_softc { device_t dev; - struct resource * intc_res; + struct resource *mem_res; + struct resource *irq_res; + void *ih; struct plic_irqsrc isrcs[PLIC_MAX_IRQS]; struct plic_context contexts[MAXCPU]; int ndev; }; #define RD4(sc, reg) \ - bus_read_4(sc->intc_res, (reg)) + bus_read_4(sc->mem_res, (reg)) #define WR4(sc, reg, val) \ - bus_write_4(sc->intc_res, (reg), (val)) + bus_write_4(sc->mem_res, (reg), (val)) static u_int plic_irq_cpu; static int riscv_hartid_to_cpu(int hartid) { int i; CPU_FOREACH(i) { if (pcpu_find(i)->pc_hart == hartid) return (i); } return (-1); } static int plic_get_hartid(device_t dev, phandle_t intc) { int hart; /* Check the interrupt controller layout. */ if (OF_searchencprop(intc, "#interrupt-cells", &hart, sizeof(hart)) == -1) { device_printf(dev, "Could not find #interrupt-cells for phandle %u\n", intc); return (-1); } /* * The parent of the interrupt-controller is the CPU we are * interested in, so search for its hart ID. */ if (OF_searchencprop(OF_parent(intc), "reg", (pcell_t *)&hart, sizeof(hart)) == -1) { device_printf(dev, "Could not find hartid\n"); return (-1); } return (hart); } static inline void plic_irq_dispatch(struct plic_softc *sc, u_int irq, struct trapframe *tf) { struct plic_irqsrc *src; src = &sc->isrcs[irq]; if (intr_isrc_dispatch(&src->isrc, tf) != 0) device_printf(sc->dev, "Stray irq %u detected\n", irq); } static int plic_intr(void *arg) { struct plic_softc *sc; struct trapframe *tf; uint32_t pending; uint32_t cpu; sc = arg; cpu = PCPU_GET(cpuid); /* Claim any pending interrupt. */ pending = RD4(sc, PLIC_CLAIM(sc, cpu)); if (pending) { tf = curthread->td_intr_frame; plic_irq_dispatch(sc, pending, tf); } return (FILTER_HANDLED); } static void plic_disable_intr(device_t dev, struct intr_irqsrc *isrc) { struct plic_softc *sc; struct plic_irqsrc *src; sc = device_get_softc(dev); src = (struct plic_irqsrc *)isrc; WR4(sc, PLIC_PRIORITY(src->irq), 0); } static void plic_enable_intr(device_t dev, struct intr_irqsrc *isrc) { struct plic_softc *sc; struct plic_irqsrc *src; sc = device_get_softc(dev); src = (struct plic_irqsrc *)isrc; WR4(sc, PLIC_PRIORITY(src->irq), 1); } static int plic_map_intr(device_t dev, struct intr_map_data *data, struct intr_irqsrc **isrcp) { struct intr_map_data_fdt *daf; struct plic_softc *sc; sc = device_get_softc(dev); if (data->type != INTR_MAP_DATA_FDT) return (ENOTSUP); daf = (struct intr_map_data_fdt *)data; if (daf->ncells != 1 || daf->cells[0] > sc->ndev) return (EINVAL); *isrcp = &sc->isrcs[daf->cells[0]].isrc; return (0); } static int plic_probe(device_t dev) { if (!ofw_bus_status_okay(dev)) return (ENXIO); if (!ofw_bus_is_compatible(dev, "riscv,plic0") && !ofw_bus_is_compatible(dev, "sifive,plic-1.0.0")) return (ENXIO); device_set_desc(dev, "RISC-V PLIC"); return (BUS_PROBE_DEFAULT); } static int plic_attach(device_t dev) { struct plic_irqsrc *isrcs; struct plic_softc *sc; struct intr_pic *pic; pcell_t *cells; uint32_t irq; const char *name; phandle_t node; phandle_t xref; uint32_t cpu; int error; int rid; int nintr; int context; int i; int hart; sc = device_get_softc(dev); sc->dev = dev; node = ofw_bus_get_node(dev); if ((OF_getencprop(node, "riscv,ndev", &sc->ndev, sizeof(sc->ndev))) < 0) { device_printf(dev, "Error: could not get number of devices\n"); return (ENXIO); } if (sc->ndev >= PLIC_MAX_IRQS) { device_printf(dev, "Error: invalid ndev (%d)\n", sc->ndev); return (ENXIO); } /* Request memory resources */ rid = 0; - sc->intc_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, + sc->mem_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, RF_ACTIVE); - if (sc->intc_res == NULL) { + if (sc->mem_res == NULL) { device_printf(dev, "Error: could not allocate memory resources\n"); return (ENXIO); } /* Register the interrupt sources */ isrcs = sc->isrcs; name = device_get_nameunit(sc->dev); for (irq = 1; irq <= sc->ndev; irq++) { isrcs[irq].irq = irq; error = intr_isrc_register(&isrcs[irq].isrc, sc->dev, 0, "%s,%u", name, irq); if (error != 0) return (error); WR4(sc, PLIC_PRIORITY(irq), 0); } /* * Calculate the per-cpu enable and context register offsets. * * This is tricky for a few reasons. The PLIC divides the interrupt * enable, threshold, and claim bits by "context", where each context - * routes to a Core-Local Interrupt Controller (CLIC). + * routes to a core's local interrupt controller. * * The tricky part is that the PLIC spec imposes no restrictions on how * these contexts are laid out. So for example, there is no guarantee * that each CPU will have both a machine mode and supervisor context, * or that different PLIC implementations will organize the context * registers in the same way. On top of this, we must handle the fact * that cpuid != hartid, as they may have been renumbered during boot. * We perform the following steps: * * 1. Examine the PLIC's "interrupts-extended" property and skip any * entries that are not for supervisor external interrupts. * * 2. Walk up the device tree to find the corresponding CPU, and grab - * it's hart ID. + * its hart ID. * * 3. Convert the hart to a cpuid, and calculate the register offsets * based on the context number. + * + * 4. Save the index for the boot hart's S-mode external interrupt in + * order to allocate and setup the corresponding resource, since the + * local interrupt controller newbus device is associated with that + * specific node. */ nintr = OF_getencprop_alloc_multi(node, "interrupts-extended", sizeof(uint32_t), (void **)&cells); if (nintr <= 0) { device_printf(dev, "Could not read interrupts-extended\n"); return (ENXIO); } /* interrupts-extended is a list of phandles and interrupt types. */ + rid = -1; for (i = 0, context = 0; i < nintr; i += 2, context++) { /* Skip M-mode external interrupts */ if (cells[i + 1] != IRQ_EXTERNAL_SUPERVISOR) continue; - /* Get the hart ID from the CLIC's phandle. */ + /* + * Get the hart ID from the core's interrupt controller + * phandle. + */ hart = plic_get_hartid(dev, OF_node_from_xref(cells[i])); if (hart < 0) { OF_prop_free(cells); return (ENXIO); } /* Get the corresponding cpuid. */ cpu = riscv_hartid_to_cpu(hart); if (cpu < 0) { device_printf(dev, "Invalid hart!\n"); OF_prop_free(cells); return (ENXIO); } + if (cpu == 0) + rid = i / 2; + /* Set the enable and context register offsets for the CPU. */ sc->contexts[cpu].enable_offset = PLIC_ENABLE_BASE + context * PLIC_ENABLE_STRIDE; sc->contexts[cpu].context_offset = PLIC_CONTEXT_BASE + context * PLIC_CONTEXT_STRIDE; } OF_prop_free(cells); + if (rid == -1) { + device_printf(dev, + "Could not find local interrupt controller\n"); + return (ENXIO); + } + + sc->irq_res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid, + RF_ACTIVE); + if (sc->irq_res == NULL) { + device_printf(dev, + "Error: could not allocate IRQ resources\n"); + return (ENXIO); + } + /* Set the threshold for each CPU to accept all priorities. */ CPU_FOREACH(cpu) WR4(sc, PLIC_THRESHOLD(sc, cpu), 0); xref = OF_xref_from_node(node); pic = intr_pic_register(sc->dev, xref); if (pic == NULL) return (ENXIO); - csr_set(sie, SIE_SEIE); - - return (intr_pic_claim_root(sc->dev, xref, plic_intr, sc)); + return (bus_setup_intr(dev, sc->irq_res, INTR_TYPE_CLK | INTR_MPSAFE, + plic_intr, NULL, sc, &sc->ih)); } static void plic_pre_ithread(device_t dev, struct intr_irqsrc *isrc) { plic_disable_intr(dev, isrc); } static void plic_post_ithread(device_t dev, struct intr_irqsrc *isrc) { struct plic_softc *sc; struct plic_irqsrc *src; uint32_t cpu; sc = device_get_softc(dev); src = (struct plic_irqsrc *)isrc; cpu = CPU_FFS(&isrc->isrc_cpu) - 1; /* Complete the interrupt. */ WR4(sc, PLIC_CLAIM(sc, cpu), src->irq); plic_enable_intr(dev, isrc); } static int plic_setup_intr(device_t dev, struct intr_irqsrc *isrc, struct resource *res, struct intr_map_data *data) { CPU_ZERO(&isrc->isrc_cpu); plic_bind_intr(dev, isrc); return (0); } static int plic_bind_intr(device_t dev, struct intr_irqsrc *isrc) { struct plic_softc *sc; struct plic_irqsrc *src; uint32_t reg; u_int cpu; sc = device_get_softc(dev); src = (struct plic_irqsrc *)isrc; /* Disable the interrupt source on all CPUs. */ CPU_FOREACH(cpu) { reg = RD4(sc, PLIC_ENABLE(sc, src->irq, cpu)); reg &= ~(1 << (src->irq % 32)); WR4(sc, PLIC_ENABLE(sc, src->irq, cpu), reg); } if (CPU_EMPTY(&isrc->isrc_cpu)) { cpu = plic_irq_cpu = intr_irq_next_cpu(plic_irq_cpu, &all_cpus); CPU_SETOF(cpu, &isrc->isrc_cpu); } else { /* * We will only bind to a single CPU so select the first * CPU found. */ cpu = CPU_FFS(&isrc->isrc_cpu) - 1; } /* Enable the interrupt on the selected CPU only. */ reg = RD4(sc, PLIC_ENABLE(sc, src->irq, cpu)); reg |= (1 << (src->irq % 32)); WR4(sc, PLIC_ENABLE(sc, src->irq, cpu), reg); return (0); } static device_method_t plic_methods[] = { DEVMETHOD(device_probe, plic_probe), DEVMETHOD(device_attach, plic_attach), DEVMETHOD(pic_disable_intr, plic_disable_intr), DEVMETHOD(pic_enable_intr, plic_enable_intr), DEVMETHOD(pic_map_intr, plic_map_intr), DEVMETHOD(pic_pre_ithread, plic_pre_ithread), DEVMETHOD(pic_post_ithread, plic_post_ithread), DEVMETHOD(pic_post_filter, plic_post_ithread), DEVMETHOD(pic_setup_intr, plic_setup_intr), DEVMETHOD(pic_bind_intr, plic_bind_intr), DEVMETHOD_END }; static driver_t plic_driver = { "plic", plic_methods, sizeof(struct plic_softc), }; EARLY_DRIVER_MODULE(plic, simplebus, plic_driver, 0, 0, BUS_PASS_INTERRUPT + BUS_PASS_ORDER_MIDDLE); diff --git a/sys/riscv/riscv/riscv_console.c b/sys/riscv/riscv/riscv_console.c index de61fe24dcee..d6c4ce4540cb 100644 --- a/sys/riscv/riscv/riscv_console.c +++ b/sys/riscv/riscv/riscv_console.c @@ -1,266 +1,264 @@ /*- * Copyright (c) 2015-2017 Ruslan Bukin * All rights reserved. * * Portions of this software were developed by SRI International and the * University of Cambridge Computer Laboratory under DARPA/AFRL contract * FA8750-10-C-0237 ("CTSRD"), as part of the DARPA CRASH research programme. * * Portions of this software were developed by the University of Cambridge * Computer Laboratory as part of the CTSRD Project, with support from the * UK Higher Education Innovation Fund (HEIF). * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include /* bus softc */ struct rcons_softc { struct resource *res[1]; void *ihl[1]; device_t dev; }; /* CN Console interface */ static tsw_outwakeup_t riscvtty_outwakeup; static struct ttydevsw riscv_ttydevsw = { .tsw_flags = TF_NOPREFIX, .tsw_outwakeup = riscvtty_outwakeup, }; static int polltime; static struct callout riscv_callout; static struct tty *tp = NULL; #if defined(KDB) static int alt_break_state; #endif static void riscv_timeout(void *); static cn_probe_t riscv_cnprobe; static cn_init_t riscv_cninit; static cn_term_t riscv_cnterm; static cn_getc_t riscv_cngetc; static cn_putc_t riscv_cnputc; static cn_grab_t riscv_cngrab; static cn_ungrab_t riscv_cnungrab; CONSOLE_DRIVER(riscv); #define MAX_BURST_LEN 1 static void riscv_putc(int c) { sbi_console_putchar(c); } #ifdef EARLY_PRINTF early_putc_t *early_putc = riscv_putc; #endif static void cn_drvinit(void *unused) { if (riscv_consdev.cn_pri != CN_DEAD && riscv_consdev.cn_name[0] != '\0') { tp = tty_alloc(&riscv_ttydevsw, NULL); tty_init_console(tp, 0); tty_makedev(tp, NULL, "%s", "rcons"); polltime = 1; callout_init(&riscv_callout, 1); callout_reset(&riscv_callout, polltime, riscv_timeout, NULL); } } SYSINIT(cndev, SI_SUB_CONFIGURE, SI_ORDER_MIDDLE, cn_drvinit, NULL); static void riscvtty_outwakeup(struct tty *tp) { u_char buf[MAX_BURST_LEN]; int len; int i; for (;;) { len = ttydisc_getc(tp, buf, sizeof(buf)); if (len == 0) break; KASSERT(len == 1, ("tty error")); for (i = 0; i < len; i++) riscv_putc(buf[i]); } } static void riscv_timeout(void *v) { int c; tty_lock(tp); while ((c = riscv_cngetc(NULL)) != -1) ttydisc_rint(tp, c, 0); ttydisc_rint_done(tp); tty_unlock(tp); callout_reset(&riscv_callout, polltime, riscv_timeout, NULL); } static void riscv_cnprobe(struct consdev *cp) { cp->cn_pri = CN_NORMAL; } static void riscv_cninit(struct consdev *cp) { strcpy(cp->cn_name, "rcons"); } static void riscv_cnterm(struct consdev *cp) { } static void riscv_cngrab(struct consdev *cp) { } static void riscv_cnungrab(struct consdev *cp) { } static int riscv_cngetc(struct consdev *cp) { int ch; ch = sbi_console_getchar(); if (ch > 0 && ch < 0xff) { #if defined(KDB) kdb_alt_break(ch, &alt_break_state); #endif return (ch); } return (-1); } static void riscv_cnputc(struct consdev *cp, int c) { riscv_putc(c); } /* Bus interface */ static int rcons_probe(device_t dev) { device_set_desc(dev, "RISC-V console"); return (BUS_PROBE_DEFAULT); } static int rcons_attach(device_t dev) { struct rcons_softc *sc; if (device_get_unit(dev) != 0) return (ENXIO); sc = device_get_softc(dev); sc->dev = dev; - csr_set(sie, SIE_SSIE); - bus_generic_attach(sc->dev); return (0); } static device_method_t rcons_methods[] = { DEVMETHOD(device_probe, rcons_probe), DEVMETHOD(device_attach, rcons_attach), DEVMETHOD_END }; static driver_t rcons_driver = { "rcons", rcons_methods, sizeof(struct rcons_softc) }; DRIVER_MODULE(rcons, nexus, rcons_driver, 0, 0); diff --git a/sys/riscv/riscv/sbi.c b/sys/riscv/riscv/sbi.c index 7c6d19064456..53eadbe54bc6 100644 --- a/sys/riscv/riscv/sbi.c +++ b/sys/riscv/riscv/sbi.c @@ -1,375 +1,418 @@ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2019 Mitchell Horne * Copyright (c) 2021 Jessica Clarke * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include #include #include #include #include +#include #include #include #include #include /* SBI Implementation-Specific Definitions */ #define OPENSBI_VERSION_MAJOR_OFFSET 16 #define OPENSBI_VERSION_MINOR_MASK 0xFFFF struct sbi_softc { device_t dev; }; +struct sbi_devinfo { + struct resource_list rl; +}; + static struct sbi_softc *sbi_softc = NULL; static u_long sbi_spec_version; static u_long sbi_impl_id; static u_long sbi_impl_version; static bool has_time_extension = false; static bool has_ipi_extension = false; static bool has_rfnc_extension = false; static bool has_srst_extension = false; static struct sbi_ret sbi_get_spec_version(void) { return (SBI_CALL0(SBI_EXT_ID_BASE, SBI_BASE_GET_SPEC_VERSION)); } static struct sbi_ret sbi_get_impl_id(void) { return (SBI_CALL0(SBI_EXT_ID_BASE, SBI_BASE_GET_IMPL_ID)); } static struct sbi_ret sbi_get_impl_version(void) { return (SBI_CALL0(SBI_EXT_ID_BASE, SBI_BASE_GET_IMPL_VERSION)); } static struct sbi_ret sbi_get_mvendorid(void) { return (SBI_CALL0(SBI_EXT_ID_BASE, SBI_BASE_GET_MVENDORID)); } static struct sbi_ret sbi_get_marchid(void) { return (SBI_CALL0(SBI_EXT_ID_BASE, SBI_BASE_GET_MARCHID)); } static struct sbi_ret sbi_get_mimpid(void) { return (SBI_CALL0(SBI_EXT_ID_BASE, SBI_BASE_GET_MIMPID)); } static void sbi_shutdown_final(void *dummy __unused, int howto) { if ((howto & RB_POWEROFF) != 0) sbi_system_reset(SBI_SRST_TYPE_SHUTDOWN, SBI_SRST_REASON_NONE); } void sbi_system_reset(u_long reset_type, u_long reset_reason) { /* Use the SRST extension, if available. */ if (has_srst_extension) { (void)SBI_CALL2(SBI_EXT_ID_SRST, SBI_SRST_SYSTEM_RESET, reset_type, reset_reason); } (void)SBI_CALL0(SBI_SHUTDOWN, 0); } void sbi_print_version(void) { u_int major; u_int minor; /* For legacy SBI implementations. */ if (sbi_spec_version == 0) { printf("SBI: Unknown (Legacy) Implementation\n"); printf("SBI Specification Version: 0.1\n"); return; } switch (sbi_impl_id) { case (SBI_IMPL_ID_BBL): printf("SBI: Berkely Boot Loader %lu\n", sbi_impl_version); break; case (SBI_IMPL_ID_XVISOR): printf("SBI: eXtensible Versatile hypervISOR %lu\n", sbi_impl_version); break; case (SBI_IMPL_ID_KVM): printf("SBI: Kernel-based Virtual Machine %lu\n", sbi_impl_version); break; case (SBI_IMPL_ID_RUSTSBI): printf("SBI: RustSBI %lu\n", sbi_impl_version); break; case (SBI_IMPL_ID_DIOSIX): printf("SBI: Diosix %lu\n", sbi_impl_version); break; case (SBI_IMPL_ID_OPENSBI): major = sbi_impl_version >> OPENSBI_VERSION_MAJOR_OFFSET; minor = sbi_impl_version & OPENSBI_VERSION_MINOR_MASK; printf("SBI: OpenSBI v%u.%u\n", major, minor); break; default: printf("SBI: Unrecognized Implementation: %lu\n", sbi_impl_id); break; } major = (sbi_spec_version & SBI_SPEC_VERS_MAJOR_MASK) >> SBI_SPEC_VERS_MAJOR_OFFSET; minor = (sbi_spec_version & SBI_SPEC_VERS_MINOR_MASK); printf("SBI Specification Version: %u.%u\n", major, minor); } void sbi_set_timer(uint64_t val) { struct sbi_ret ret __diagused; /* Use the TIME legacy replacement extension, if available. */ if (has_time_extension) { ret = SBI_CALL1(SBI_EXT_ID_TIME, SBI_TIME_SET_TIMER, val); MPASS(ret.error == SBI_SUCCESS); } else { (void)SBI_CALL1(SBI_SET_TIMER, 0, val); } } void sbi_send_ipi(const u_long *hart_mask) { struct sbi_ret ret __diagused; /* Use the IPI legacy replacement extension, if available. */ if (has_ipi_extension) { ret = SBI_CALL2(SBI_EXT_ID_IPI, SBI_IPI_SEND_IPI, *hart_mask, 0); MPASS(ret.error == SBI_SUCCESS); } else { (void)SBI_CALL1(SBI_SEND_IPI, 0, (uint64_t)hart_mask); } } void sbi_remote_fence_i(const u_long *hart_mask) { struct sbi_ret ret __diagused; /* Use the RFENCE legacy replacement extension, if available. */ if (has_rfnc_extension) { ret = SBI_CALL2(SBI_EXT_ID_RFNC, SBI_RFNC_REMOTE_FENCE_I, *hart_mask, 0); MPASS(ret.error == SBI_SUCCESS); } else { (void)SBI_CALL1(SBI_REMOTE_FENCE_I, 0, (uint64_t)hart_mask); } } void sbi_remote_sfence_vma(const u_long *hart_mask, u_long start, u_long size) { struct sbi_ret ret __diagused; /* Use the RFENCE legacy replacement extension, if available. */ if (has_rfnc_extension) { ret = SBI_CALL4(SBI_EXT_ID_RFNC, SBI_RFNC_REMOTE_SFENCE_VMA, *hart_mask, 0, start, size); MPASS(ret.error == SBI_SUCCESS); } else { (void)SBI_CALL3(SBI_REMOTE_SFENCE_VMA, 0, (uint64_t)hart_mask, start, size); } } void sbi_remote_sfence_vma_asid(const u_long *hart_mask, u_long start, u_long size, u_long asid) { struct sbi_ret ret __diagused; /* Use the RFENCE legacy replacement extension, if available. */ if (has_rfnc_extension) { ret = SBI_CALL5(SBI_EXT_ID_RFNC, SBI_RFNC_REMOTE_SFENCE_VMA_ASID, *hart_mask, 0, start, size, asid); MPASS(ret.error == SBI_SUCCESS); } else { (void)SBI_CALL4(SBI_REMOTE_SFENCE_VMA_ASID, 0, (uint64_t)hart_mask, start, size, asid); } } int sbi_hsm_hart_start(u_long hart, u_long start_addr, u_long priv) { struct sbi_ret ret; ret = SBI_CALL3(SBI_EXT_ID_HSM, SBI_HSM_HART_START, hart, start_addr, priv); return (ret.error != 0 ? (int)ret.error : 0); } void sbi_hsm_hart_stop(void) { (void)SBI_CALL0(SBI_EXT_ID_HSM, SBI_HSM_HART_STOP); } int sbi_hsm_hart_status(u_long hart) { struct sbi_ret ret; ret = SBI_CALL1(SBI_EXT_ID_HSM, SBI_HSM_HART_STATUS, hart); return (ret.error != 0 ? (int)ret.error : (int)ret.value); } void sbi_init(void) { struct sbi_ret sret; /* * Get the spec version. For legacy SBI implementations this will * return an error, otherwise it is guaranteed to succeed. */ sret = sbi_get_spec_version(); if (sret.error != 0) { /* We are running a legacy SBI implementation. */ sbi_spec_version = 0; return; } /* Set the SBI implementation info. */ sbi_spec_version = sret.value; sbi_impl_id = sbi_get_impl_id().value; sbi_impl_version = sbi_get_impl_version().value; /* Set the hardware implementation info. */ mvendorid = sbi_get_mvendorid().value; marchid = sbi_get_marchid().value; mimpid = sbi_get_mimpid().value; /* Probe for legacy replacement extensions. */ if (sbi_probe_extension(SBI_EXT_ID_TIME) != 0) has_time_extension = true; if (sbi_probe_extension(SBI_EXT_ID_IPI) != 0) has_ipi_extension = true; if (sbi_probe_extension(SBI_EXT_ID_RFNC) != 0) has_rfnc_extension = true; if (sbi_probe_extension(SBI_EXT_ID_SRST) != 0) has_srst_extension = true; /* * Probe for legacy extensions. We still rely on many of them to be * implemented, but this is not guaranteed by the spec. */ KASSERT(has_time_extension || sbi_probe_extension(SBI_SET_TIMER) != 0, ("SBI doesn't implement sbi_set_timer()")); KASSERT(sbi_probe_extension(SBI_CONSOLE_PUTCHAR) != 0, ("SBI doesn't implement sbi_console_putchar()")); KASSERT(sbi_probe_extension(SBI_CONSOLE_GETCHAR) != 0, ("SBI doesn't implement sbi_console_getchar()")); KASSERT(has_ipi_extension || sbi_probe_extension(SBI_SEND_IPI) != 0, ("SBI doesn't implement sbi_send_ipi()")); KASSERT(has_rfnc_extension || sbi_probe_extension(SBI_REMOTE_FENCE_I) != 0, ("SBI doesn't implement sbi_remote_fence_i()")); KASSERT(has_rfnc_extension || sbi_probe_extension(SBI_REMOTE_SFENCE_VMA) != 0, ("SBI doesn't implement sbi_remote_sfence_vma()")); KASSERT(has_rfnc_extension || sbi_probe_extension(SBI_REMOTE_SFENCE_VMA_ASID) != 0, ("SBI doesn't implement sbi_remote_sfence_vma_asid()")); KASSERT(has_srst_extension || sbi_probe_extension(SBI_SHUTDOWN) != 0, ("SBI doesn't implement a shutdown or reset extension")); } static void sbi_identify(driver_t *driver, device_t parent) { device_t dev; if (device_find_child(parent, "sbi", -1) != NULL) return; dev = BUS_ADD_CHILD(parent, 0, "sbi", -1); if (dev == NULL) device_printf(parent, "Can't add sbi child\n"); } static int sbi_probe(device_t dev) { device_set_desc(dev, "RISC-V Supervisor Binary Interface"); return (BUS_PROBE_NOWILDCARD); } static int sbi_attach(device_t dev) { struct sbi_softc *sc; +#ifdef SMP + device_t child; + struct sbi_devinfo *di; +#endif if (sbi_softc != NULL) return (ENXIO); sc = device_get_softc(dev); sc->dev = dev; sbi_softc = sc; EVENTHANDLER_REGISTER(shutdown_final, sbi_shutdown_final, NULL, SHUTDOWN_PRI_LAST); +#ifdef SMP + di = malloc(sizeof(*di), M_DEVBUF, M_WAITOK | M_ZERO); + resource_list_init(&di->rl); + child = device_add_child(dev, "sbi_ipi", -1); + if (child == NULL) { + device_printf(dev, "Could not add sbi_ipi child\n"); + return (ENXIO); + } + + device_set_ivars(child, di); +#endif + return (0); } +static struct resource_list * +sbi_get_resource_list(device_t bus, device_t child) +{ + struct sbi_devinfo *di; + + di = device_get_ivars(child); + KASSERT(di != NULL, ("%s: No devinfo", __func__)); + + return (&di->rl); +} + static device_method_t sbi_methods[] = { /* Device interface */ DEVMETHOD(device_identify, sbi_identify), DEVMETHOD(device_probe, sbi_probe), DEVMETHOD(device_attach, sbi_attach), + /* Bus interface */ + DEVMETHOD(bus_alloc_resource, bus_generic_rl_alloc_resource), + DEVMETHOD(bus_activate_resource, bus_generic_activate_resource), + DEVMETHOD(bus_deactivate_resource, bus_generic_deactivate_resource), + DEVMETHOD(bus_release_resource, bus_generic_rl_release_resource), + DEVMETHOD(bus_setup_intr, bus_generic_setup_intr), + DEVMETHOD(bus_teardown_intr, bus_generic_teardown_intr), + DEVMETHOD(bus_get_resource_list, sbi_get_resource_list), + DEVMETHOD(bus_set_resource, bus_generic_rl_set_resource), + DEVMETHOD(bus_get_resource, bus_generic_rl_get_resource), + DEVMETHOD_END }; DEFINE_CLASS_0(sbi, sbi_driver, sbi_methods, sizeof(struct sbi_softc)); EARLY_DRIVER_MODULE(sbi, nexus, sbi_driver, 0, 0, BUS_PASS_CPU + BUS_PASS_ORDER_FIRST); diff --git a/sys/riscv/riscv/sbi_ipi.c b/sys/riscv/riscv/sbi_ipi.c new file mode 100644 index 000000000000..d694b476f5f9 --- /dev/null +++ b/sys/riscv/riscv/sbi_ipi.c @@ -0,0 +1,208 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause + * + * Copyright (c) 2021 Jessica Clarke + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include +#include +#include + +#include "pic_if.h" + +struct sbi_ipi_softc { + device_t dev; + struct resource *irq_res; + void *ih; + struct intr_irqsrc isrc; + uint32_t pending_ipis[MAXCPU]; +}; + +static void +sbi_ipi_pic_ipi_send(device_t dev, struct intr_irqsrc *isrc, cpuset_t cpus, + u_int ipi) +{ + struct sbi_ipi_softc *sc; + struct pcpu *pc; + u_long mask; + u_int cpu; + + sc = device_get_softc(dev); + + KASSERT(isrc == &sc->isrc, ("%s: not the IPI isrc", __func__)); + KASSERT(ipi < INTR_IPI_COUNT, + ("%s: not a valid IPI: %u", __func__, ipi)); + + mask = 0; + STAILQ_FOREACH(pc, &cpuhead, pc_allcpu) { + cpu = pc->pc_cpuid; + if (CPU_ISSET(cpu, &cpus)) { + atomic_set_32(&sc->pending_ipis[cpu], 1u << ipi); + mask |= (1ul << pc->pc_hart); + } + } + sbi_send_ipi(&mask); +} + +static int +sbi_ipi_pic_ipi_setup(device_t dev, u_int ipi, struct intr_irqsrc **isrcp) +{ + struct sbi_ipi_softc *sc; + + sc = device_get_softc(dev); + + KASSERT(ipi < INTR_IPI_COUNT, + ("%s: not a valid IPI: %u", __func__, ipi)); + + *isrcp = &sc->isrc; + + return (0); + +} + +static int +sbi_ipi_intr(void *arg) +{ + struct sbi_ipi_softc *sc; + uint32_t ipi_bitmap; + u_int cpu, ipi; + int bit; + + sc = arg; + + csr_clear(sip, SIP_SSIP); + + cpu = PCPU_GET(cpuid); + + mb(); + + ipi_bitmap = atomic_readandclear_32(&sc->pending_ipis[cpu]); + if (ipi_bitmap == 0) + return (FILTER_HANDLED); + + mb(); + + while ((bit = ffs(ipi_bitmap))) { + ipi = (bit - 1); + ipi_bitmap &= ~(1u << ipi); + + intr_ipi_dispatch(ipi); + } + + return (FILTER_HANDLED); +} + +static int +sbi_ipi_probe(device_t dev) +{ + device_set_desc(dev, "RISC-V SBI Inter-Processor Interrupts"); + + return (BUS_PROBE_NOWILDCARD); +} + +static int +sbi_ipi_attach(device_t dev) +{ + struct sbi_ipi_softc *sc; + const char *name; + int irq, rid, error; + phandle_t iparent; + pcell_t cell; + + sc = device_get_softc(dev); + sc->dev = dev; + + memset(sc->pending_ipis, 0, sizeof(sc->pending_ipis)); + + name = device_get_nameunit(dev); + error = intr_isrc_register(&sc->isrc, sc->dev, INTR_ISRCF_IPI, + "%s,ipi", name); + if (error != 0) { + device_printf(dev, "Can't register interrupt: %d\n", error); + return (ENXIO); + } + + iparent = OF_xref_from_node(ofw_bus_get_node(intr_irq_root_dev)); + cell = IRQ_SOFTWARE_SUPERVISOR; + irq = ofw_bus_map_intr(dev, iparent, 1, &cell); + error = bus_set_resource(dev, SYS_RES_IRQ, 0, irq, 1); + if (error != 0) { + device_printf(dev, "Unable to register IRQ resource\n"); + return (ENXIO); + } + + rid = 0; + sc->irq_res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid, + RF_ACTIVE | RF_SHAREABLE); + if (sc->irq_res == NULL) { + device_printf(dev, "Unable to alloc IRQ resource\n"); + return (ENXIO); + } + + error = bus_setup_intr(dev, sc->irq_res, INTR_TYPE_CLK, + sbi_ipi_intr, NULL, sc, &sc->ih); + if (error != 0) { + device_printf(dev, "Unable to setup IRQ resource\n"); + return (ENXIO); + } + + /* TODO: Define a set of priorities once other IPI sources exist */ + error = intr_ipi_pic_register(dev, 0); + if (error != 0) { + device_printf(dev, "Can't register as IPI source: %d\n", error); + return (ENXIO); + } + + return (0); +} + +static device_method_t sbi_ipi_methods[] = { + /* Device interface */ + DEVMETHOD(device_probe, sbi_ipi_probe), + DEVMETHOD(device_attach, sbi_ipi_attach), + + /* Interrupt controller interface */ + DEVMETHOD(pic_ipi_send, sbi_ipi_pic_ipi_send), + DEVMETHOD(pic_ipi_setup, sbi_ipi_pic_ipi_setup), + + DEVMETHOD_END +}; + +DEFINE_CLASS_0(sbi_ipi, sbi_ipi_driver, sbi_ipi_methods, + sizeof(struct sbi_ipi_softc)); +/* Local interrupt controller attaches during BUS_PASS_ORDER_FIRST */ +EARLY_DRIVER_MODULE(sbi_ipi, sbi, sbi_ipi_driver, 0, 0, + BUS_PASS_INTERRUPT + BUS_PASS_ORDER_EARLY); diff --git a/sys/riscv/riscv/timer.c b/sys/riscv/riscv/timer.c index bb98aee3e2cc..06806872d9fd 100644 --- a/sys/riscv/riscv/timer.c +++ b/sys/riscv/riscv/timer.c @@ -1,295 +1,316 @@ /*- * Copyright (c) 2015-2017 Ruslan Bukin * All rights reserved. * * Portions of this software were developed by SRI International and the * University of Cambridge Computer Laboratory under DARPA/AFRL contract * FA8750-10-C-0237 ("CTSRD"), as part of the DARPA CRASH research programme. * * Portions of this software were developed by the University of Cambridge * Computer Laboratory as part of the CTSRD Project, with support from the * UK Higher Education Innovation Fund (HEIF). * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* * RISC-V Timer */ #include "opt_platform.h" #include #include #include #include #include +#include #include #include #include #include #include #include #include #include +#include #include struct riscv_timer_softc { + struct resource *irq_res; void *ih; uint32_t clkfreq; struct eventtimer et; }; static struct riscv_timer_softc *riscv_timer_sc = NULL; static timecounter_get_t riscv_timer_tc_get_timecount; static timecounter_fill_vdso_timehands_t riscv_timer_tc_fill_vdso_timehands; static struct timecounter riscv_timer_timecount = { .tc_name = "RISC-V Timecounter", .tc_get_timecount = riscv_timer_tc_get_timecount, .tc_poll_pps = NULL, .tc_counter_mask = ~0u, .tc_frequency = 0, .tc_quality = 1000, .tc_fill_vdso_timehands = riscv_timer_tc_fill_vdso_timehands, }; static inline uint64_t get_timecount(void) { return (rdtime()); } static inline void set_timecmp(uint64_t timecmp) { if (has_sstc) csr_write(stimecmp, timecmp); else sbi_set_timer(timecmp); } static u_int riscv_timer_tc_get_timecount(struct timecounter *tc __unused) { return (get_timecount()); } static uint32_t riscv_timer_tc_fill_vdso_timehands(struct vdso_timehands *vdso_th, struct timecounter *tc) { vdso_th->th_algo = VDSO_TH_ALGO_RISCV_RDTIME; bzero(vdso_th->th_res, sizeof(vdso_th->th_res)); return (1); } static int riscv_timer_et_start(struct eventtimer *et, sbintime_t first, sbintime_t period) { uint64_t counts; if (first != 0) { counts = ((uint32_t)et->et_frequency * first) >> 32; set_timecmp(get_timecount() + counts); - csr_set(sie, SIE_STIE); return (0); } return (EINVAL); } static int riscv_timer_et_stop(struct eventtimer *et) { /* Disable timer interrupts. */ csr_clear(sie, SIE_STIE); return (0); } static int riscv_timer_intr(void *arg) { struct riscv_timer_softc *sc; sc = (struct riscv_timer_softc *)arg; csr_clear(sip, SIP_STIP); if (sc->et.et_active) sc->et.et_event_cb(&sc->et, sc->et.et_arg); return (FILTER_HANDLED); } static int riscv_timer_get_timebase(device_t dev, uint32_t *freq) { phandle_t node; int len; node = OF_finddevice("/cpus"); if (node == -1) { if (bootverbose) device_printf(dev, "Can't find cpus node.\n"); return (ENXIO); } len = OF_getproplen(node, "timebase-frequency"); if (len != 4) { if (bootverbose) device_printf(dev, "Can't find timebase-frequency property.\n"); return (ENXIO); } OF_getencprop(node, "timebase-frequency", freq, len); return (0); } static int riscv_timer_probe(device_t dev) { device_set_desc(dev, "RISC-V Timer"); return (BUS_PROBE_DEFAULT); } static int riscv_timer_attach(device_t dev) { struct riscv_timer_softc *sc; - int error; + int irq, rid, error; + phandle_t iparent; + pcell_t cell; sc = device_get_softc(dev); if (riscv_timer_sc != NULL) return (ENXIO); if (device_get_unit(dev) != 0) return (ENXIO); if (riscv_timer_get_timebase(dev, &sc->clkfreq) != 0) { device_printf(dev, "No clock frequency specified\n"); return (ENXIO); } riscv_timer_sc = sc; + iparent = OF_xref_from_node(ofw_bus_get_node(intr_irq_root_dev)); + cell = IRQ_TIMER_SUPERVISOR; + irq = ofw_bus_map_intr(dev, iparent, 1, &cell); + error = bus_set_resource(dev, SYS_RES_IRQ, 0, irq, 1); + if (error != 0) { + device_printf(dev, "Unable to register IRQ resource\n"); + return (ENXIO); + } + + rid = 0; + sc->irq_res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid, + RF_ACTIVE); + if (sc->irq_res == NULL) { + device_printf(dev, "Unable to alloc IRQ resource\n"); + return (ENXIO); + } + /* Setup IRQs handler */ - error = riscv_setup_intr(device_get_nameunit(dev), riscv_timer_intr, - NULL, sc, IRQ_TIMER_SUPERVISOR, INTR_TYPE_CLK, &sc->ih); - if (error) { - device_printf(dev, "Unable to alloc int resource.\n"); + error = bus_setup_intr(dev, sc->irq_res, INTR_TYPE_CLK, + riscv_timer_intr, NULL, sc, &sc->ih); + if (error != 0) { + device_printf(dev, "Unable to setup IRQ resource\n"); return (ENXIO); } riscv_timer_timecount.tc_frequency = sc->clkfreq; riscv_timer_timecount.tc_priv = sc; tc_init(&riscv_timer_timecount); sc->et.et_name = "RISC-V Eventtimer"; sc->et.et_flags = ET_FLAGS_ONESHOT | ET_FLAGS_PERCPU; sc->et.et_quality = 1000; sc->et.et_frequency = sc->clkfreq; sc->et.et_min_period = (0x00000002LLU << 32) / sc->et.et_frequency; sc->et.et_max_period = (0xfffffffeLLU << 32) / sc->et.et_frequency; sc->et.et_start = riscv_timer_et_start; sc->et.et_stop = riscv_timer_et_stop; sc->et.et_priv = sc; et_register(&sc->et); set_cputicker(get_timecount, sc->clkfreq, false); return (0); } static device_method_t riscv_timer_methods[] = { DEVMETHOD(device_probe, riscv_timer_probe), DEVMETHOD(device_attach, riscv_timer_attach), { 0, 0 } }; static driver_t riscv_timer_driver = { "timer", riscv_timer_methods, sizeof(struct riscv_timer_softc), }; EARLY_DRIVER_MODULE(timer, nexus, riscv_timer_driver, 0, 0, BUS_PASS_TIMER + BUS_PASS_ORDER_MIDDLE); void DELAY(int usec) { int64_t counts, counts_per_usec; uint64_t first, last; /* * Check the timers are setup, if not just * use a for loop for the meantime */ if (riscv_timer_sc == NULL) { for (; usec > 0; usec--) for (counts = 200; counts > 0; counts--) /* * Prevent the compiler from optimizing * out the loop */ cpufunc_nullop(); return; } TSENTER(); /* Get the number of times to count */ counts_per_usec = ((riscv_timer_timecount.tc_frequency / 1000000) + 1); /* * Clamp the timeout at a maximum value (about 32 seconds with * a 66MHz clock). *Nobody* should be delay()ing for anywhere * near that length of time and if they are, they should be hung * out to dry. */ if (usec >= (0x80000000U / counts_per_usec)) counts = (0x80000000U / counts_per_usec) - 1; else counts = usec * counts_per_usec; first = get_timecount(); while (counts > 0) { last = get_timecount(); counts -= (int64_t)(last - first); first = last; } TSEXIT(); } diff --git a/sys/riscv/riscv/trap.c b/sys/riscv/riscv/trap.c index f54efc324942..3a7b3fba26b9 100644 --- a/sys/riscv/riscv/trap.c +++ b/sys/riscv/riscv/trap.c @@ -1,459 +1,461 @@ /*- * Copyright (c) 2015-2018 Ruslan Bukin * All rights reserved. * * Portions of this software were developed by SRI International and the * University of Cambridge Computer Laboratory under DARPA/AFRL contract * FA8750-10-C-0237 ("CTSRD"), as part of the DARPA CRASH research programme. * * Portions of this software were developed by the University of Cambridge * Computer Laboratory as part of the CTSRD Project, with support from the * UK Higher Education Innovation Fund (HEIF). * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include #include "opt_ddb.h" #include #include #include #include #include #include #include #include #include #include #include #ifdef KDB #include #endif #include #include #include #include #include #include #include #include #include #include #include #include #ifdef KDTRACE_HOOKS #include #endif #ifdef DDB #include #include #endif +void intr_irq_handler(struct trapframe *tf); + int (*dtrace_invop_jump_addr)(struct trapframe *); /* Called from exception.S */ void do_trap_supervisor(struct trapframe *); void do_trap_user(struct trapframe *); static __inline void call_trapsignal(struct thread *td, int sig, int code, void *addr, int trapno) { ksiginfo_t ksi; ksiginfo_init_trap(&ksi); ksi.ksi_signo = sig; ksi.ksi_code = code; ksi.ksi_addr = addr; ksi.ksi_trapno = trapno; trapsignal(td, &ksi); } int cpu_fetch_syscall_args(struct thread *td) { struct proc *p; syscallarg_t *ap, *dst_ap; struct syscall_args *sa; p = td->td_proc; sa = &td->td_sa; ap = &td->td_frame->tf_a[0]; dst_ap = &sa->args[0]; sa->code = td->td_frame->tf_t[0]; sa->original_code = sa->code; if (__predict_false(sa->code == SYS_syscall || sa->code == SYS___syscall)) { sa->code = *ap++; } else { *dst_ap++ = *ap++; } if (__predict_false(sa->code >= p->p_sysent->sv_size)) sa->callp = &nosys_sysent; else sa->callp = &p->p_sysent->sv_table[sa->code]; KASSERT(sa->callp->sy_narg <= nitems(sa->args), ("Syscall %d takes too many arguments", sa->code)); memcpy(dst_ap, ap, (NARGREG - 1) * sizeof(*dst_ap)); td->td_retval[0] = 0; td->td_retval[1] = 0; return (0); } #include "../../kern/subr_syscall.c" static void print_with_symbol(const char *name, uint64_t value) { #ifdef DDB c_db_sym_t sym; db_expr_t sym_value; db_expr_t offset; const char *sym_name; #endif printf("%7s: 0x%016lx", name, value); #ifdef DDB if (value >= VM_MIN_KERNEL_ADDRESS) { sym = db_search_symbol(value, DB_STGY_ANY, &offset); if (sym != C_DB_SYM_NULL) { db_symbol_values(sym, &sym_name, &sym_value); if (offset != 0) printf(" (%s + 0x%lx)", sym_name, offset); else printf(" (%s)", sym_name); } } #endif printf("\n"); } static void dump_regs(struct trapframe *frame) { char name[6]; int i; for (i = 0; i < nitems(frame->tf_t); i++) { snprintf(name, sizeof(name), "t[%d]", i); print_with_symbol(name, frame->tf_t[i]); } for (i = 0; i < nitems(frame->tf_s); i++) { snprintf(name, sizeof(name), "s[%d]", i); print_with_symbol(name, frame->tf_s[i]); } for (i = 0; i < nitems(frame->tf_a); i++) { snprintf(name, sizeof(name), "a[%d]", i); print_with_symbol(name, frame->tf_a[i]); } print_with_symbol("ra", frame->tf_ra); print_with_symbol("sp", frame->tf_sp); print_with_symbol("gp", frame->tf_gp); print_with_symbol("tp", frame->tf_tp); print_with_symbol("sepc", frame->tf_sepc); printf("sstatus: 0x%016lx\n", frame->tf_sstatus); printf("stval : 0x%016lx\n", frame->tf_stval); } static void ecall_handler(void) { struct thread *td; td = curthread; syscallenter(td); syscallret(td); } static void page_fault_handler(struct trapframe *frame, int usermode) { struct vm_map *map; uint64_t stval; struct thread *td; struct pcb *pcb; vm_prot_t ftype; vm_offset_t va; struct proc *p; int error, sig, ucode; #ifdef KDB bool handled; #endif #ifdef KDB if (kdb_active) { kdb_reenter(); return; } #endif td = curthread; p = td->td_proc; pcb = td->td_pcb; stval = frame->tf_stval; if (td->td_critnest != 0 || td->td_intr_nesting_level != 0 || WITNESS_CHECK(WARN_SLEEPOK | WARN_GIANTOK, NULL, "Kernel page fault") != 0) goto fatal; if (usermode) { if (!VIRT_IS_VALID(stval)) { call_trapsignal(td, SIGSEGV, SEGV_MAPERR, (void *)stval, frame->tf_scause & SCAUSE_CODE); goto done; } map = &p->p_vmspace->vm_map; } else { /* * Enable interrupts for the duration of the page fault. For * user faults this was done already in do_trap_user(). */ intr_enable(); if (stval >= VM_MIN_KERNEL_ADDRESS) { map = kernel_map; } else { if (pcb->pcb_onfault == 0) goto fatal; map = &p->p_vmspace->vm_map; } } va = trunc_page(stval); if (frame->tf_scause == SCAUSE_STORE_PAGE_FAULT) { ftype = VM_PROT_WRITE; } else if (frame->tf_scause == SCAUSE_INST_PAGE_FAULT) { ftype = VM_PROT_EXECUTE; } else { ftype = VM_PROT_READ; } if (VIRT_IS_VALID(va) && pmap_fault(map->pmap, va, ftype)) goto done; error = vm_fault_trap(map, va, ftype, VM_FAULT_NORMAL, &sig, &ucode); if (error != KERN_SUCCESS) { if (usermode) { call_trapsignal(td, sig, ucode, (void *)stval, frame->tf_scause & SCAUSE_CODE); } else { if (pcb->pcb_onfault != 0) { frame->tf_a[0] = error; frame->tf_sepc = pcb->pcb_onfault; return; } goto fatal; } } done: if (usermode) userret(td, frame); return; fatal: dump_regs(frame); #ifdef KDB if (debugger_on_trap) { kdb_why = KDB_WHY_TRAP; handled = kdb_trap(frame->tf_scause & SCAUSE_CODE, 0, frame); kdb_why = KDB_WHY_UNSET; if (handled) return; } #endif panic("Fatal page fault at %#lx: %#lx", frame->tf_sepc, stval); } void do_trap_supervisor(struct trapframe *frame) { uint64_t exception; /* Ensure we came from supervisor mode, interrupts disabled */ KASSERT((csr_read(sstatus) & (SSTATUS_SPP | SSTATUS_SIE)) == SSTATUS_SPP, ("Came from S mode with interrupts enabled")); KASSERT((csr_read(sstatus) & (SSTATUS_SUM)) == 0, ("Came from S mode with SUM enabled")); exception = frame->tf_scause & SCAUSE_CODE; if ((frame->tf_scause & SCAUSE_INTR) != 0) { /* Interrupt */ - riscv_cpu_intr(frame); + intr_irq_handler(frame); return; } #ifdef KDTRACE_HOOKS if (dtrace_trap_func != NULL && (*dtrace_trap_func)(frame, exception)) return; #endif CTR4(KTR_TRAP, "%s: exception=%lu, sepc=%#lx, stval=%#lx", __func__, exception, frame->tf_sepc, frame->tf_stval); switch (exception) { case SCAUSE_LOAD_ACCESS_FAULT: case SCAUSE_STORE_ACCESS_FAULT: case SCAUSE_INST_ACCESS_FAULT: dump_regs(frame); panic("Memory access exception at %#lx: %#lx", frame->tf_sepc, frame->tf_stval); break; case SCAUSE_LOAD_MISALIGNED: case SCAUSE_STORE_MISALIGNED: case SCAUSE_INST_MISALIGNED: dump_regs(frame); panic("Misaligned address exception at %#lx: %#lx", frame->tf_sepc, frame->tf_stval); break; case SCAUSE_STORE_PAGE_FAULT: case SCAUSE_LOAD_PAGE_FAULT: case SCAUSE_INST_PAGE_FAULT: page_fault_handler(frame, 0); break; case SCAUSE_BREAKPOINT: #ifdef KDTRACE_HOOKS if (dtrace_invop_jump_addr != NULL && dtrace_invop_jump_addr(frame) == 0) break; #endif #ifdef KDB kdb_trap(exception, 0, frame); #else dump_regs(frame); panic("No debugger in kernel."); #endif break; case SCAUSE_ILLEGAL_INSTRUCTION: dump_regs(frame); panic("Illegal instruction 0x%0*lx at %#lx", (frame->tf_stval & 0x3) != 0x3 ? 4 : 8, frame->tf_stval, frame->tf_sepc); break; default: dump_regs(frame); panic("Unknown kernel exception %#lx trap value %#lx", exception, frame->tf_stval); } } void do_trap_user(struct trapframe *frame) { uint64_t exception; struct thread *td; struct pcb *pcb; td = curthread; pcb = td->td_pcb; KASSERT(td->td_frame == frame, ("%s: td_frame %p != frame %p", __func__, td->td_frame, frame)); /* Ensure we came from usermode, interrupts disabled */ KASSERT((csr_read(sstatus) & (SSTATUS_SPP | SSTATUS_SIE)) == 0, ("Came from U mode with interrupts enabled")); KASSERT((csr_read(sstatus) & (SSTATUS_SUM)) == 0, ("Came from U mode with SUM enabled")); exception = frame->tf_scause & SCAUSE_CODE; if ((frame->tf_scause & SCAUSE_INTR) != 0) { /* Interrupt */ - riscv_cpu_intr(frame); + intr_irq_handler(frame); return; } intr_enable(); CTR4(KTR_TRAP, "%s: exception=%lu, sepc=%#lx, stval=%#lx", __func__, exception, frame->tf_sepc, frame->tf_stval); switch (exception) { case SCAUSE_LOAD_ACCESS_FAULT: case SCAUSE_STORE_ACCESS_FAULT: case SCAUSE_INST_ACCESS_FAULT: call_trapsignal(td, SIGBUS, BUS_ADRERR, (void *)frame->tf_sepc, exception); userret(td, frame); break; case SCAUSE_LOAD_MISALIGNED: case SCAUSE_STORE_MISALIGNED: case SCAUSE_INST_MISALIGNED: call_trapsignal(td, SIGBUS, BUS_ADRALN, (void *)frame->tf_sepc, exception); userret(td, frame); break; case SCAUSE_STORE_PAGE_FAULT: case SCAUSE_LOAD_PAGE_FAULT: case SCAUSE_INST_PAGE_FAULT: page_fault_handler(frame, 1); break; case SCAUSE_ECALL_USER: frame->tf_sepc += 4; /* Next instruction */ ecall_handler(); break; case SCAUSE_ILLEGAL_INSTRUCTION: if ((pcb->pcb_fpflags & PCB_FP_STARTED) == 0) { /* * May be a FPE trap. Enable FPE usage * for this thread and try again. */ fpe_state_clear(); frame->tf_sstatus &= ~SSTATUS_FS_MASK; frame->tf_sstatus |= SSTATUS_FS_CLEAN; pcb->pcb_fpflags |= PCB_FP_STARTED; break; } call_trapsignal(td, SIGILL, ILL_ILLTRP, (void *)frame->tf_sepc, exception); userret(td, frame); break; case SCAUSE_BREAKPOINT: call_trapsignal(td, SIGTRAP, TRAP_BRKPT, (void *)frame->tf_sepc, exception); userret(td, frame); break; default: dump_regs(frame); panic("Unknown userland exception %#lx, trap value %#lx", exception, frame->tf_stval); } }