diff --git a/sys/arm64/arm64/genassym.c b/sys/arm64/arm64/genassym.c --- a/sys/arm64/arm64/genassym.c +++ b/sys/arm64/arm64/genassym.c @@ -47,6 +47,8 @@ ASSYM(PC_CURPCB, offsetof(struct pcpu, pc_curpcb)); ASSYM(PC_CURTHREAD, offsetof(struct pcpu, pc_curthread)); ASSYM(PC_SSBD, offsetof(struct pcpu, pc_ssbd)); +ASSYM(PC_SDEI_NORMAL_STACK, offsetof(struct pcpu, pc_sdei_normal_stack)); +ASSYM(PC_SDEI_CRITICAL_STACK, offsetof(struct pcpu, pc_sdei_critical_stack)); /* Size of pcb, rounded to keep stack alignment */ ASSYM(PCB_SIZE, roundup2(sizeof(struct pcb), STACKALIGNBYTES + 1)); diff --git a/sys/arm64/conf/std.dev b/sys/arm64/conf/std.dev --- a/sys/arm64/conf/std.dev +++ b/sys/arm64/conf/std.dev @@ -118,6 +118,7 @@ # Firmware device mmio_sram # Generic on-chip SRAM +device sdei # Software defined exception interface # Wireless options options IEEE80211_DEBUG # enable debug msgs diff --git a/sys/arm64/include/intr.h b/sys/arm64/include/intr.h --- a/sys/arm64/include/intr.h +++ b/sys/arm64/include/intr.h @@ -47,11 +47,13 @@ #define ACPI_INTR_XREF 1 #define ACPI_MSI_XREF 2 #define ACPI_GPIO_XREF 3 +#define ACPI_SDEI_XREF 4 #endif #endif /* !LOCORE */ #define INTR_ROOT_FIQ 1 -#define INTR_ROOT_NUM 2 +#define INTR_ROOT_SDEI 2 +#define INTR_ROOT_NUM 3 #endif /* _MACHINE_INTR_H */ diff --git a/sys/arm64/include/pcpu.h b/sys/arm64/include/pcpu.h --- a/sys/arm64/include/pcpu.h +++ b/sys/arm64/include/pcpu.h @@ -50,7 +50,9 @@ struct pmap *pc_curvmpmap; \ uint64_t pc_mpidr; \ u_int pc_bcast_tlbi_workaround; \ - char __pad[197] + void *pc_sdei_normal_stack; \ + void *pc_sdei_critical_stack; \ + char __pad[181] #ifdef _KERNEL diff --git a/sys/arm64/include/stack.h b/sys/arm64/include/stack.h --- a/sys/arm64/include/stack.h +++ b/sys/arm64/include/stack.h @@ -50,9 +50,35 @@ static __inline bool kstack_contains(struct thread *td, vm_offset_t va, size_t len) { - return (va >= td->td_kstack && va + len >= va && - va + len <= td->td_kstack + td->td_kstack_pages * PAGE_SIZE - - sizeof(struct pcb)); + extern u_int mp_maxid; + u_int cpu; + + if (va + len < va) + return (false); + + if (va >= td->td_kstack && va + len <= td->td_kstack + + td->td_kstack_pages * PAGE_SIZE - sizeof(struct pcb)) + return (true); + + for (cpu = 0; cpu <= mp_maxid; cpu++) { + struct pcpu *pcpu; + + pcpu = pcpu_find(cpu); + if (pcpu == NULL) + continue; + if (pcpu->pc_sdei_normal_stack != NULL && + va >= (uintptr_t)pcpu->pc_sdei_normal_stack && + va + len <= (uintptr_t)pcpu->pc_sdei_normal_stack + + PAGE_SIZE) + return (true); + if (pcpu->pc_sdei_critical_stack != NULL && + va >= (uintptr_t)pcpu->pc_sdei_critical_stack && + va + len <= (uintptr_t)pcpu->pc_sdei_critical_stack + + PAGE_SIZE) + return (true); + } + + return (false); } #endif /* _SYS_PROC_H_ */ diff --git a/sys/conf/files.arm64 b/sys/conf/files.arm64 --- a/sys/conf/files.arm64 +++ b/sys/conf/files.arm64 @@ -417,6 +417,9 @@ dev/pci/pci_dw_if.m optional pci fdt dev/psci/psci.c standard +dev/psci/sdei.c optional sdei +dev/psci/sdei_acpi.c optional sdei acpi +dev/psci/sdei_arm64.S optional sdei dev/psci/smccc_arm64.S standard dev/psci/smccc_trng.c standard dev/psci/smccc.c standard diff --git a/sys/dev/psci/sdei.h b/sys/dev/psci/sdei.h new file mode 100644 --- /dev/null +++ b/sys/dev/psci/sdei.h @@ -0,0 +1,65 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause + * + * Copyright (c) 2024 Arm Ltd + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef _PSCI_SDEI_H_ +#define _PSCI_SDEI_H_ + +#define SDEI_FUNC(id) SMCCC_FUNC_ID(SMCCC_FAST_CALL, SMCCC_64BIT_CALL, \ + SMCCC_STD_SECURE_SERVICE_CALLS, id) + +#define SDEI_VERSION SDEI_FUNC(0x20) +#define SDEI_EVENT_REGISTER SDEI_FUNC(0x21) +#define SDEI_EVENT_ENABLE SDEI_FUNC(0x22) +#define SDEI_EVENT_DISABLE SDEI_FUNC(0x23) +#define SDEI_EVENT_CONTEXT SDEI_FUNC(0x24) +#define SDEI_EVENT_COMPLETE SDEI_FUNC(0x25) +#define SDEI_EVENT_COMPLETE_AND_RESUME SDEI_FUNC(0x26) +#define SDEI_EVENT_UNREGISTER SDEI_FUNC(0x27) +#define SDEI_EVENT_STATUS SDEI_FUNC(0x28) +#define SDEI_EVENT_GET_INFO SDEI_FUNC(0x29) +#define SDEI_EVENT_ROUTING_SET SDEI_FUNC(0x2a) +#define SDEI_PE_MASK SDEI_FUNC(0x2b) +#define SDEI_PE_UNMASK SDEI_FUNC(0x2c) +#define SDEI_INTERRUPT_BIND SDEI_FUNC(0x2d) +#define SDEI_INTERRUPT_RELEASE SDEI_FUNC(0x2e) +#define SDEI_EVENT_SIGNAL SDEI_FUNC(0x2f) +#define SDEI_FEATURES SDEI_FUNC(0x30) +#define SDEI_PRIVATE_RESET SDEI_FUNC(0x31) +#define SDEI_SHARED_RESET SDEI_FUNC(0x32) + +/* SDEI_EVENT_GET_INFO */ +#define SDEI_INFO_EV_TYPE 0 +#define SDEI_INFO_EV_TYPE_PRIVATE 0 +#define SDEI_INFO_EV_TYPE_SHARED 1 +#define SDEI_INFO_EV_SIGNALED 1 +#define SDEI_INFO_EV_PRIORITY 2 +#define SDEI_INFO_EV_PRIORITY_NORMAL 0 +#define SDEI_INFO_EV_PRIORITY_CRITICAL 1 +#define SDEI_INFO_EV_ROUTING_MODE 3 +#define SDEI_INFO_EV_ROUTING_AFF 4 + +#endif diff --git a/sys/dev/psci/sdei.c b/sys/dev/psci/sdei.c new file mode 100644 --- /dev/null +++ b/sys/dev/psci/sdei.c @@ -0,0 +1,416 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause + * + * Copyright (c) 2024 Arm Ltd + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include +#include + +#include +#include +#include + +#include "pic_if.h" + +#define SDEI_EVENT_0 0 + +static MALLOC_DEFINE(M_SDEI, "sdei", "SDEI buffers"); + +extern void sdei_handler_hvc(void); +extern void sdei_handler_smc(void); + +#ifdef SMP +static pic_init_secondary_t sdei_init_secondary; +static pic_ipi_send_t sdei_ipi_send; +static pic_ipi_setup_t sdei_ipi_setup; +#endif + +static struct sdei_event **sdei_alloc_private_event(struct sdei_softc *, + uint32_t); +static void sdei_free_private_event(struct sdei_event **); +static bool sdei_attach_event(struct sdei_softc *, struct sdei_event *); +static int sdei_intr(void *); + +static device_method_t sdei_methods[] = { +#ifdef SMP + /* Interrupt controller interface */ + DEVMETHOD(pic_init_secondary, sdei_init_secondary), + DEVMETHOD(pic_ipi_send, sdei_ipi_send), + DEVMETHOD(pic_ipi_setup, sdei_ipi_setup), +#endif + + /* End */ + DEVMETHOD_END +}; + +DEFINE_CLASS_0(sdei, sdei_driver, sdei_methods, sizeof(struct sdei_softc)); + +static int +_sdei_smccc_invoke(struct sdei_softc *sc, register_t a0, register_t a1, + register_t a2, register_t a3, register_t a4, register_t a5, register_t a6, + struct arm_smccc_1_2_regs *res) +{ + struct arm_smccc_1_2_regs args; + + args.a0 = a0; + args.a1 = a1; + args.a2 = a2; + args.a3 = a3; + args.a4 = a4; + args.a5 = a5; + args.a6 = a6; + args.a7 = 0; + + return (sc->sc_smccc_call(&args, res)); +} +#define sdei_smccc_invoke(sc, ...) \ + _arm_smccc_invoke(_sdei_smccc_invoke, sc, __VA_ARGS__) + +static int +sdei_get_info(struct sdei_softc *sc, uint32_t event, uint32_t info, + uint64_t *result) +{ + struct arm_smccc_1_2_regs res; + int ret; + + ret = sdei_smccc_invoke(sc, SDEI_EVENT_GET_INFO, event, info, &res); + if (ret < 0) + return (ret); + + *result = res.a0; + return (0); +} + +static bool +sdei_event_is_private(struct sdei_softc *sc, uint32_t event, bool *priv) +{ + uint64_t res; + + if (sdei_get_info(sc, event, SDEI_INFO_EV_TYPE, &res) != 0) + return (false); + + switch (res) { + case SDEI_INFO_EV_TYPE_PRIVATE: + *priv = true; + break; + case SDEI_INFO_EV_TYPE_SHARED: + *priv = false; + break; + default: + return (false); + } + + return (true); +} + +static bool +sdei_event_is_critical(struct sdei_softc *sc, uint32_t event, bool *crit) +{ + uint64_t res; + + if (sdei_get_info(sc, event, SDEI_INFO_EV_PRIORITY, &res) != 0) + return (false); + + switch (res) { + case SDEI_INFO_EV_PRIORITY_NORMAL: + *crit = false; + break; + case SDEI_INFO_EV_PRIORITY_CRITICAL: + *crit = true; + break; + default: + return (false); + } + + return (true); +} + +static struct sdei_event ** +sdei_alloc_private_event(struct sdei_softc *sc, uint32_t event_id) +{ + struct sdei_event **event; + u_int cpu; + bool critical; + + /* Try reading if the event is critical */ + if (!sdei_event_is_critical(sc, event_id, &critical)) + return (NULL); + + event = mallocarray(mp_maxid + 1, sizeof(*event), M_SDEI, + M_WAITOK | M_ZERO); + CPU_FOREACH(cpu) { + event[cpu] = malloc_domainset(sizeof(**event), + M_SDEI, DOMAINSET_PREF(pcpu_find(cpu)->pc_domain), + M_WAITOK); + event[cpu]->event_critical = critical; + event[cpu]->event_id = event_id; + } + + return (event); +} + +static void +sdei_free_private_event(struct sdei_event **event) +{ + u_int cpu; + + CPU_FOREACH(cpu) { + free(event[cpu], M_SDEI); + } + free(event, M_SDEI); +} + +static bool +sdei_attach_event(struct sdei_softc *sc, struct sdei_event *event) +{ + void (*handler)(void); + uint32_t event_id; + + switch (sc->sc_method) { + case PSCI_METHOD_HVC: + handler = sdei_handler_hvc; + break; + case PSCI_METHOD_SMC: + handler = sdei_handler_smc; + break; + default: + return (false); + } + + MPASS(event != NULL); + event_id = event->event_id; + if (sdei_smccc_invoke(sc, SDEI_EVENT_REGISTER, event_id, + (register_t)handler, (register_t)event, 0, 0, NULL) != 0) + return (false); + + if (sdei_smccc_invoke(sc, SDEI_EVENT_ENABLE, event_id, NULL) != 0) { + /* + * Unregister on failure. Don't check for errors as there's + * nothing we could do other than report it to the user. + */ + sdei_smccc_invoke(sc, SDEI_EVENT_UNREGISTER, event_id, NULL); + return (false); + } + + return (true); +} + +int +sdei_attach(device_t dev) +{ + struct sdei_softc *sc; + int cpu, ret; +#ifdef SMP + bool valid; +#endif + + /* + * The firmware will send exceptions to the highest implemented + * exception level below it, i.e. EL3 will send to EL2 if implemented + * otherwise EL1. + * + * If we are started in EL2, but are currently running in EL1 then + * this will break. Fail to probe in this case so we don't get in to + * a situation where SDEI exceptions are sent to an exception level + * that isn't expecting them. + */ + if (has_hyp() && !in_vhe()) + return (0); + + sc = device_get_softc(dev); + sc->sc_dev = dev; + + switch (sc->sc_method) { + case PSCI_METHOD_HVC: + sc->sc_smccc_call = &arm_smccc_1_2_hvc; + break; + case PSCI_METHOD_SMC: + sc->sc_smccc_call = &arm_smccc_1_2_smc; + break; + default: + return (ENXIO); + } + + sc->sc_pic = intr_pic_register(dev, sc->sc_intr_xref); + if (sc->sc_pic == NULL) { + device_printf(dev, "Unable to register PIC\n"); + return (ENXIO); + } + + if (intr_pic_claim_root(dev, sc->sc_intr_xref, sdei_intr, sc, + INTR_ROOT_SDEI) != 0) { + device_printf(dev, "Unable to claim SDEI interrupt root\n"); + return (ENXIO); + } + + /* Allocate stacks for each CPU */ + CPU_FOREACH(cpu) { + struct pcpu *pcpu; + + pcpu = pcpu_find(cpu); + pcpu->pc_sdei_normal_stack = malloc_domainset_aligned(PAGE_SIZE, + PAGE_SIZE, M_SDEI, DOMAINSET_PREF(pcpu->pc_domain), + M_WAITOK); + pcpu->pc_sdei_critical_stack = malloc_domainset_aligned( + PAGE_SIZE, PAGE_SIZE, M_SDEI, + DOMAINSET_PREF(pcpu->pc_domain), M_WAITOK); + } + +#ifdef SMP + valid = sdei_event_is_private(sc, SDEI_EVENT_0, + &sc->sc_event_0_private); + if (valid && sc->sc_event_0_private) { + sc->sc_event0 = sdei_alloc_private_event(sc, SDEI_EVENT_0); + } + + if (sc->sc_event0 != NULL) { + if (sdei_attach_event(sc, sc->sc_event0[PCPU_GET(cpuid)])) { + /* + * Handle hard stops as they come from panic + * so need to be a NMI. + */ + intr_ipi_pic_register_ipi(IPI_STOP_HARD, dev, 1); + } else { + sdei_free_private_event(sc->sc_event0); + sc->sc_event0 = NULL; + } + } +#endif + + /* + * This can only fail if SDEI is unimplemented. As we've been calling + * into SDEI before now assert it succeeded assert this also succeeds. + */ + ret = sdei_smccc_invoke(sc, SDEI_PE_UNMASK, NULL); + KASSERT(ret == 0, ("%s: Invalid response from SDEI_PE_UNMASK: %d", + device_get_nameunit(dev), ret)); + + return (0); +} + +#ifdef SMP +static void +sdei_init_secondary(device_t dev, uint32_t rootnum) +{ + struct sdei_softc *sc; + int ret __diagused; + bool attached __diagused; + + sc = device_get_softc(dev); + attached = sdei_attach_event(sc, sc->sc_event0[PCPU_GET(cpuid)]); + KASSERT(attached, ("%s: Unable to attach event 0", + device_get_nameunit(dev))); + + /* As above assert SDEI_PE_UNMASK succeeds */ + ret = sdei_smccc_invoke(sc, SDEI_PE_UNMASK, NULL); + KASSERT(ret == 0, ("%s: Invalid response from SDEI_PE_UNMASK: %d", + device_get_nameunit(dev), ret)); +} + +static void +sdei_ipi_send(device_t dev, struct intr_irqsrc *isrc, cpuset_t cpus, + u_int ipi) +{ + struct sdei_softc *sc; + bool send_self; + + KASSERT(ipi == IPI_STOP_HARD, + ("%s: Invalid IPI: %x != IPI_STOP_HARD (%x)", __func__, ipi, + IPI_STOP_HARD)); + + sc = device_get_softc(dev); + send_self = false; + for (int i = 0; i <= mp_maxid; i++) { + if (CPU_ISSET(i, &cpus)) { + if (PCPU_GET(cpuid) == i) { + send_self = true; + } else { + sdei_smccc_invoke(sc, SDEI_EVENT_SIGNAL, + SDEI_EVENT_0, + PCPU_GET_MPIDR(cpuid_to_pcpu[i]), NULL); + } + } + } + if (send_self) { + sdei_smccc_invoke(sc, SDEI_EVENT_SIGNAL, SDEI_EVENT_0, + get_mpidr(), NULL); + } +} + +static int +sdei_ipi_setup(device_t dev, u_int ipi, struct intr_irqsrc **isrcp) +{ + struct sdei_softc *sc; + struct intr_irqsrc *isrc; + + KASSERT(ipi == IPI_STOP_HARD, + ("%s: Invalid IPI: %x != IPI_STOP_HARD (%x)", __func__, ipi, + IPI_STOP_HARD)); + + sc = device_get_softc(dev); + + isrc = &sc->sc_ipi_isrc; + CPU_SET(PCPU_GET(cpuid), &isrc->isrc_cpu); + *isrcp = isrc; + + return (0); +} +#endif + +static int +sdei_intr(void *arg) +{ + struct sdei_softc *sc; + uint32_t event_id; + + sc = arg; + event_id = curthread->td_intr_frame->tf_x[0]; + switch (event_id) { +#ifdef SMP + case SDEI_EVENT_0: + intr_ipi_dispatch(IPI_STOP_HARD); + break; +#endif + default: + device_printf(sc->sc_dev, "Stray SDEI event %u\n", event_id); + } + + return (FILTER_HANDLED); +} diff --git a/sys/dev/psci/sdei_acpi.c b/sys/dev/psci/sdei_acpi.c new file mode 100644 --- /dev/null +++ b/sys/dev/psci/sdei_acpi.c @@ -0,0 +1,97 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause + * + * Copyright (c) 2024 Arm Ltd + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include "opt_acpi.h" + +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include +#include + +static device_identify_t sdei_acpi_identify; +static device_probe_t sdei_acpi_probe; +static device_attach_t sdei_acpi_attach; + +static device_method_t sdei_acpi_methods[] = { + DEVMETHOD(device_identify, sdei_acpi_identify), + DEVMETHOD(device_probe, sdei_acpi_probe), + DEVMETHOD(device_attach, sdei_acpi_attach), + + /* End */ + DEVMETHOD_END +}; + +DEFINE_CLASS_1(sdei, sdei_acpi_driver, sdei_acpi_methods, + sizeof(struct sdei_softc), sdei_driver); + +DRIVER_MODULE(sdei, acpi, sdei_acpi_driver, 0, 0); + +static void +sdei_acpi_identify(driver_t *driver, device_t parent) +{ + ACPI_TABLE_HEADER *sdei; + device_t dev; + if (ACPI_FAILURE(AcpiGetTable(ACPI_SIG_SDEI, 0, &sdei))) + return; + AcpiPutTable(sdei); + + dev = BUS_ADD_CHILD(parent, BUS_PASS_DEFAULT, "sdei", -1); + if (dev == NULL) + return; + acpi_set_private(dev, sdei_acpi_identify); +} + +static int +sdei_acpi_probe(device_t dev) +{ + if (acpi_get_private(dev) != sdei_acpi_identify) + return (ENXIO); + + device_set_desc(dev, "Arm SDEI"); + return (0); +} + +static int +sdei_acpi_attach(device_t dev) +{ + struct sdei_softc *sc; + + sc = device_get_softc(dev); + sc->sc_intr_xref = ACPI_SDEI_XREF; + sc->sc_method = psci_get_method(); + + return (sdei_attach(dev)); +} diff --git a/sys/dev/psci/sdei_arm64.S b/sys/dev/psci/sdei_arm64.S new file mode 100644 --- /dev/null +++ b/sys/dev/psci/sdei_arm64.S @@ -0,0 +1,181 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause + * + * Copyright (c) 2024 Arm Ltd + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include +#include + +#include +#include +#include +#include +#include "assym.inc" + +#include +#include + +/* + * x0: Event number + * x1: struct sdei_event_args address + * x2: Interrupted PC + * x3: Interrupted PSTATE + * x18-x29 must be preserved + */ +LENTRY(sdei_handler) + /* Save the GP registers in the tf_x array */ + stp x0, x1, [x1, #(TF_X + (0 * 8))] + /* x2 and x3 are saved in the caller */ + stp x4, x5, [x1, #(TF_X + (4 * 8))] + stp x6, x7, [x1, #(TF_X + (6 * 8))] + stp x8, x9, [x1, #(TF_X + (8 * 8))] + stp x10, x11, [x1, #(TF_X + (10 * 8))] + stp x12, x13, [x1, #(TF_X + (12 * 8))] + stp x14, x15, [x1, #(TF_X + (14 * 8))] + stp x16, x17, [x1, #(TF_X + (16 * 8))] + stp x18, x19, [x1, #(TF_X + (18 * 8))] + stp x20, x21, [x1, #(TF_X + (20 * 8))] + stp x22, x23, [x1, #(TF_X + (22 * 8))] + stp x24, x25, [x1, #(TF_X + (24 * 8))] + stp x26, x27, [x1, #(TF_X + (26 * 8))] + stp x28, x29, [x1, #(TF_X + (28 * 8))] + /* SP and LR are saved in the caller */ + + /* No ESR or FAR registers */ + stp xzr, xzr, [x1, #(TF_ESR)] + + /* The firmware may have not set the PAN flag correctly */ + EXIT_USER_ACCESS_CHECK(w0, x2) + + /* + * As the CPU may not have been in the kernel when the SDEI + * event was raised we need to restore the pcpu pointer and + * save the userspace stack pointer. Unfortunatly tpidr_el1 + * may have been trashed. Work around this by using tpidr_el2 + * to restore both x18 and tpidr_el1. + */ + mrs x5, CurrentEL + and x5, x5, #(CURRENTEL_EL_MASK) + cmp x5, #(CURRENTEL_EL_EL1) + b.eq .Lel1 + + /* The kernel is in EL2 */ + mrs x18, tpidr_el2 + msr tpidr_el1, x18 + isb + b .Ldone_el + +.Lel1: + /* The kernel is in EL1 */ + mrs x18, tpidr_el1 + +.Ldone_el: + + /* Store sp_el0 in a callee-saved register so we can restore it later */ + mrs x19, sp_el0 +#if defined(PERTHREAD_SSP) + /* Load the SSP canary to sp_el0 */ + ldr x6, [x18, #(PC_CURTHREAD)] + add x6, x6, #(TD_MD_CANARY) + msr sp_el0, x6 +#endif + + /* Load the correct stack pointer */ + ldp x20, x21, [x18, #PC_SDEI_NORMAL_STACK] + ldrb w22, [x1, #TF_SIZE] + cmp w22, #0 + csel x20, x20, x21, eq + add sp, x20, #PAGE_SIZE + + /* + * Create a fake stack frame for unwinding to use to get onto the + * correct stack. If we entered* from the kernel link to its stack, + * otherwise store zeros to the stack to stop unwinding. + */ + and x6, x3, PSR_M_EL_MASK + cmp x5, x6 + csel x29, x29, xzr, eq + csel x2, x2, xzr, eq + stp x29, x2, [sp, #(-16)]! + mov x29, sp + + /* + * Create the stack frame either pointing to the kernel frame or + * stoping a stacktrace. + */ + stp x29, x30, [sp, #(-16)]! + mov x29, sp + mov x20, x1 + + // KMSAN_ENTER + mov x0, x1 + mov x1, #INTR_ROOT_SDEI + bl intr_irq_handler + // KMSAN_LEAVE + + /* Load the return address, skipping fp as it will be loaded below */ + ldp xzr, x30, [sp], #32 + + msr sp_el0, x19 + + /* Restore registers we need to preserve */ + mov x1, x20 + ldp x18, x19, [x1, #(TF_X + (18 * 8))] + ldp x20, x21, [x1, #(TF_X + (20 * 8))] + ldp x22, x23, [x1, #(TF_X + (22 * 8))] + ldp x24, x25, [x1, #(TF_X + (24 * 8))] + ldp x26, x27, [x1, #(TF_X + (26 * 8))] + ldp x28, x29, [x1, #(TF_X + (28 * 8))] + + ret +LEND(sdei_handler) + +.macro sdei_handler_impl insn +EENTRY(sdei_handler_\insn) + /* + * Free up a register to use. x2 and x3 hold elr and spsr. + * x3 is used later to check what exception level was interrupted. + */ + stp x2, x3, [x1, #(TF_ELR)] + stp x2, x3, [x1, #(TF_X + (2 * 8))] + /* Save SP and LR */ + mov x2, sp + stp x2, lr, [x1, #(TF_SP)] + + bl sdei_handler + + ldp x2, lr, [x1, #(TF_SP)] + mov sp, x2 + + ldr x0, =(SDEI_EVENT_COMPLETE) + mov x1, xzr + \insn #0 +EEND(sdei_handler_\insn) +.endm + +sdei_handler_impl hvc +sdei_handler_impl smc + +GNU_PROPERTY_AARCH64_FEATURE_1_NOTE(GNU_PROPERTY_AARCH64_FEATURE_1_VAL) diff --git a/sys/arm64/include/intr.h b/sys/dev/psci/sdeivar.h copy from sys/arm64/include/intr.h copy to sys/dev/psci/sdeivar.h --- a/sys/arm64/include/intr.h +++ b/sys/dev/psci/sdeivar.h @@ -1,6 +1,7 @@ /*- - * Copyright (c) 2014 Andrew Turner - * All rights reserved. + * SPDX-License-Identifier: BSD-2-Clause + * + * Copyright (c) 2024 Arm Ltd * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -24,34 +25,38 @@ * SUCH DAMAGE. */ -#ifndef _MACHINE_INTR_H_ -#define _MACHINE_INTR_H_ - -#ifndef LOCORE -#ifdef FDT -#include +#ifndef _PSCI_SDEIVAR_H_ +#define _PSCI_SDEIVAR_H_ + +#include +#include + +struct sdei_event { + struct trapframe tf; + /* This must be straight after the trapframe as it's read from asm */ + bool event_critical; + + /* These should not be accessed by asm so can be reordered as needed */ + uint32_t event_id; +}; + +struct sdei_softc { + device_t sc_dev; + int (*sc_smccc_call)(const struct arm_smccc_1_2_regs *, + struct arm_smccc_1_2_regs *); + psci_method_t sc_method; + + intptr_t sc_intr_xref; + struct intr_pic *sc_pic; +#ifdef SMP + struct intr_irqsrc sc_ipi_isrc; + bool sc_event_0_private; + struct sdei_event **sc_event0; #endif +}; -#include - -#ifndef NIRQ -#define NIRQ 16384 /* XXX - It should be an option. */ -#endif +DECLARE_CLASS(sdei_driver); -static inline void -arm_irq_memory_barrier(uintptr_t irq) -{ -} +int sdei_attach(device_t); -#ifdef DEV_ACPI -#define ACPI_INTR_XREF 1 -#define ACPI_MSI_XREF 2 -#define ACPI_GPIO_XREF 3 #endif - -#endif /* !LOCORE */ - -#define INTR_ROOT_FIQ 1 -#define INTR_ROOT_NUM 2 - -#endif /* _MACHINE_INTR_H */