Index: sys/arm64/spe/arm_spe.h =================================================================== --- /dev/null +++ sys/arm64/spe/arm_spe.h @@ -0,0 +1,77 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause + * + * Copyright (c) 2024 Arm Ltd + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef _ARM64_ARM_SPE_H_ +#define _ARM64_ARM_SPE_H_ + +/* kqueue events */ +#define ARM_SPE_KQ_BUF 138 +#define ARM_SPE_KQ_SHUTDOWN 139 +#define ARM_SPE_KQ_SIGNAL 140 + +/* spe_backend_read() u64 data encoding */ +#define KQ_BUF_POS_SHIFT 0 +#define KQ_BUF_POS (1 << KQ_BUF_POS_SHIFT) +#define KQ_PARTREC_SHIFT 1 +#define KQ_PARTREC (1 << KQ_PARTREC_SHIFT) +#define KQ_FINAL_BUF_SHIFT 2 +#define KQ_FINAL_BUF (1 << KQ_FINAL_BUF_SHIFT) + +enum arm_spe_ctx_field { + ARM_SPE_CTX_NONE, + ARM_SPE_CTX_PID, + ARM_SPE_CTX_CPU_ID +}; + +enum arm_spe_profiling_level { + ARM_SPE_KERNEL_AND_USER, + ARM_SPE_KERNEL_ONLY, + ARM_SPE_USER_ONLY +}; +struct arm_spe_config { + /* Minimum interval is IMP DEF up to maximum 24 bit value */ + uint32_t interval; + + /* Profile kernel (EL1), userspace (EL0) or both */ + enum arm_spe_profiling_level level; + + /* + * Configure context field in SPE records to store either the + * current PID, the CPU ID or neither + * + * In PID mode, kernel threads without a process context are + * logged as PID 0 + */ + enum arm_spe_ctx_field ctx_field; +}; + +struct arm_spe_svc_buf { + uint32_t ident; + uint8_t buf_idx : 1; +}; + +#endif /* _ARM64_ARM_SPE_H_ */ Index: sys/arm64/spe/arm_spe_acpi.c =================================================================== --- /dev/null +++ sys/arm64/spe/arm_spe_acpi.c @@ -0,0 +1,153 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause + * + * Copyright (c) 2024 Arm Ltd + * Copyright (c) 2022 The FreeBSD Foundation + * + * Portions of this software were developed by Andrew Turner under sponsorship + * from the FreeBSD Foundation. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include + +#include +#include +#include +#include +#include +#include + +#include +#include + +#include + +static device_identify_t arm_spe_acpi_identify; +static device_probe_t arm_spe_acpi_probe; + +static device_method_t arm_spe_acpi_methods[] = { + /* Device interface */ + DEVMETHOD(device_identify, arm_spe_acpi_identify), + DEVMETHOD(device_probe, arm_spe_acpi_probe), + + DEVMETHOD_END, +}; + +DEFINE_CLASS_1(spe, arm_spe_acpi_driver, arm_spe_acpi_methods, + sizeof(struct arm_spe_softc), arm_spe_driver); + +DRIVER_MODULE(spe, acpi, arm_spe_acpi_driver, 0, 0); +MODULE_DEPEND(spe, hwt, 1, 1, 1); +MODULE_VERSION(spe, 1); + +struct madt_data { + u_int irq; + bool found; + bool valid; +}; + +static void +madt_handler(ACPI_SUBTABLE_HEADER *entry, void *arg) +{ + ACPI_MADT_GENERIC_INTERRUPT *intr; + struct madt_data *madt_data; + u_int irq; + + madt_data = (struct madt_data *)arg; + + /* Exit early if we are have decided to not attach */ + if (!madt_data->valid) + return; + + switch(entry->Type) { + case ACPI_MADT_TYPE_GENERIC_INTERRUPT: + intr = (ACPI_MADT_GENERIC_INTERRUPT *)entry; + irq = intr->SpeInterrupt; + + if (irq == 0) { + madt_data->valid = false; + } else if (!madt_data->found) { + madt_data->found = true; + madt_data->irq = irq; + } else if (madt_data->irq != irq) { + madt_data->valid = false; + } + break; + + default: + break; + } +} + +static void +arm_spe_acpi_identify(driver_t *driver, device_t parent) +{ + struct madt_data madt_data; + ACPI_TABLE_MADT *madt; + device_t dev; + vm_paddr_t physaddr; + + physaddr = acpi_find_table(ACPI_SIG_MADT); + if (physaddr == 0) + return; + + madt = acpi_map_table(physaddr, ACPI_SIG_MADT); + if (madt == NULL) { + device_printf(parent, "spe: Unable to map the MADT\n"); + return; + } + + madt_data.irq = 0; + madt_data.found = false; + madt_data.valid = true; + + acpi_walk_subtables(madt + 1, (char *)madt + madt->Header.Length, + madt_handler, &madt_data); + + if (!madt_data.found || !madt_data.valid) + goto out; + + MPASS(madt_data.irq != 0); + + dev = BUS_ADD_CHILD(parent, 0, "spe", -1); + if (dev == NULL) { + device_printf(parent, "add gic child failed\n"); + goto out; + } else { + device_printf(parent, "child added\n"); + } + + BUS_SET_RESOURCE(parent, dev, SYS_RES_IRQ, 0, madt_data.irq, 1); + +out: + acpi_unmap_table(madt); +} + +static int +arm_spe_acpi_probe(device_t dev) +{ + device_set_desc(dev, "ARM Statistical Profiling Extension"); + return (BUS_PROBE_NOWILDCARD); +} + Index: sys/arm64/spe/arm_spe_backend.c =================================================================== --- /dev/null +++ sys/arm64/spe/arm_spe_backend.c @@ -0,0 +1,592 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause + * + * Copyright (c) 2024 Arm Ltd + * Copyright (c) 2022 The FreeBSD Foundation + * + * Portions of this software were developed by Andrew Turner under sponsorship + * from the FreeBSD Foundation. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +/* + * Arm Statistical Profiling Extension (SPE) backend + * + * Basic SPE operation + * + * SPE is enabled and configured on a per-core basis, with each core requiring + * separate code to enable and configure. Each core also requires a separate + * buffer passed as config where the CPU will write profiling data. When the + * profiling buffer is full, an interrupt will be taken on the same CPU. + * + * Driver Design + * + * - HWT allocates a large single buffer per core. This buffer is split in half + * to create a 2 element circular buffer (aka ping-pong buffer) where the + * kernel writes to one half while userspace is copying the other half + * - SMP calls are used to enable and configure each core, with SPE initially + * configured to write to the first half of the buffer + * - When the first half of the buffer is full, a buffer full interrupt will + * immediately switch writing to the second half. The kernel adds the details + * of the half that needs copying to a FIFO STAILQ and notifies userspace via + * kqueue by sending a ARM_SPE_KQ_BUF kevent with how many buffers on the + * queue need servicing + * - The kernel responds to HWT_IOC_BUFPTR_GET ioctl by sending details of the + * first item from the queue + * - The buffers pending copying will not be overwritten until an + * HWT_IOC_SVC_BUF ioctl is received from userspace confirming the data has + * been copied out + * - In the case where both halfs of the buffer are full, profiling will be + * paused until notification via HWT_IOC_SVC_BUF is received + * + * Future improvements and limitations + * + * - Using large buffer sizes should minimise pauses and loss of profiling + * data while kernel is waiting for userspace to copy out data. Since it is + * generally expected that consuming (copying) this data is faster than + * producing it, in practice this has not so far been an issue. If it does + * prove to be an issue even with large buffer sizes then additional buffering + * i.e. n element circular buffers might be required. + * + * - kqueue can only notify and queue one kevent of the same type, with + * subsequent events overwriting data in the first event. The kevent + * ARM_SPE_KQ_BUF can therefore only contain the number of buffers on the + * STAILQ, incrementing each time a new buffer is full. In this case kqueue + * serves just as a notification to userspace to wake up and query the kernel + * with the appropriate ioctl. An alternative might be custom kevents where + * the kevent identifier is encoded with something like n+cpu_id or n+tid. In + * this case data could be sent directly with kqueue via the kevent data and + * fflags elements, avoiding the extra ioctl. + * + */ + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include + +#include +#include +#include +#include +#include +#include + +MALLOC_DECLARE(M_ARM_SPE); + +extern u_int mp_maxid; +extern struct taskqueue *taskqueue_arm_spe; + +void spe_backend_disable_smp(struct hwt_context *ctx); + +static device_t spe_dev; +static struct hwt_backend_ops spe_ops; +static struct hwt_backend backend = { + .ops = &spe_ops, + .name = "spe", + .kva_req = 1, +}; + +static struct arm_spe_info *spe_info; + +static int +spe_backend_init_thread(struct hwt_context *ctx) +{ + return (ENOTSUP); +} + +static void +spe_backend_init_cpu(struct hwt_context *ctx) +{ + struct arm_spe_info *info; + struct arm_spe_softc *sc = device_get_softc(spe_dev); + char lock_name[32]; + char *tmp = "Arm SPE lock/cpu/"; + int cpu_id; + + spe_info = malloc(sizeof(struct arm_spe_info) * mp_ncpus, + M_ARM_SPE, M_WAITOK | M_ZERO); + + sc->spe_info = spe_info; + + CPU_FOREACH_ISSET(cpu_id, &ctx->cpu_map) { + info = &spe_info[cpu_id]; + info->sc = sc; + info->ident = cpu_id; + info->buf_info[0].info = info; + info->buf_info[0].buf_idx = 0; + info->buf_info[1].info = info; + info->buf_info[1].buf_idx = 1; + snprintf(lock_name, sizeof(lock_name), "%s%d", tmp, cpu_id); + mtx_init(&info->lock, lock_name, NULL, MTX_SPIN); + } +} + +static int +spe_backend_init(struct hwt_context *ctx) +{ + struct arm_spe_softc *sc = device_get_softc(spe_dev); + int error = 0; + + /* + * HWT currently specifies buffer size must be a multiple of PAGE_SIZE, + * i.e. minimum 4KB + the maximum PMBIDR.Align is 2KB + * This should never happen but it's good to sense check + */ + if (ctx->bufsize % sc->kva_align != 0) + return (EINVAL); + + /* + * Since we're splitting the buffer in half + PMBLIMITR needs to be page + * aligned, minimum buffer size needs to be 2x PAGE_SIZE + */ + if (ctx->bufsize < (2 * PAGE_SIZE)) + return (EINVAL); + + sc->ctx = ctx; + sc->kqueue_fd = ctx->kqueue_fd; + sc->hwt_td = ctx->hwt_td; + + if (ctx->mode == HWT_MODE_THREAD) + error = spe_backend_init_thread(ctx); + else + spe_backend_init_cpu(ctx); + + return (error); +} + +#ifdef ARM_SPE_DEBUG +static void hex_dump(uint8_t *buf, size_t len) +{ + size_t i; + + printf("--------------------------------------------------------------\n"); + for (i = 0; i < len; ++i) { + if (i % 8 == 0) { + printf(" "); + } + if (i % 16 == 0) { + if (i != 0) { + printf("\r\n"); + } + printf("\t"); + } + printf("%02X ", buf[i]); + } + printf("\r\n"); +} +#endif + +static void +spe_backend_deinit(struct hwt_context *ctx) +{ +#ifdef ARM_SPE_DEBUG + struct arm_spe_info *info; + int cpu_id; + + CPU_FOREACH_ISSET(cpu_id, &ctx->cpu_map) { + info = &spe_info[cpu_id]; + hex_dump((void *)info->kvaddr, 128); + hex_dump((void *)(info->kvaddr + (info->buf_size/2)), 128); + } +#endif + + if (ctx->state == CTX_STATE_RUNNING) { + spe_backend_disable_smp(ctx); + ctx->state = CTX_STATE_STOPPED; + } + + free(spe_info, M_ARM_SPE); +} + +static uint64_t +arm_spe_min_interval(struct arm_spe_softc *sc) +{ + /* IMPLEMENTATION DEFINED */ + switch PMSIDR_Interval_VAL(sc->pmsidr) + { + case PMSIDR_Interval_256: + return (256); + case PMSIDR_Interval_512: + return (512); + case PMSIDR_Interval_768: + return (768); + case PMSIDR_Interval_1024: + return (1024); + case PMSIDR_Interval_1536: + return (1536); + case PMSIDR_Interval_2048: + return (2048); + case PMSIDR_Interval_3072: + return (3072); + case PMSIDR_Interval_4096: + return (4096); + default: + return (4096); + } +} + +static inline void +arm_spe_set_interval(struct arm_spe_info *info, uint64_t interval) +{ + uint64_t min_interval = arm_spe_min_interval(info->sc); + + if (interval < min_interval) + interval = min_interval; + if (interval > (1 << 24)) /* max 24 bits */ + interval = (1 << 24); + + dprintf("%s %lu\n", __func__, interval); + + info->pmsirr &= ~(PMSIRR_INTERVAL_MASK); + info->pmsirr |= (interval << PMSIRR_INTERVAL_SHIFT); +} + +static int +spe_backend_configure(struct hwt_context *ctx, int cpu_id, int session_id) +{ + struct arm_spe_info *info = &spe_info[cpu_id]; + struct arm_spe_config *cfg; + int err = 0; + + mtx_lock_spin(&info->lock); + info->ident = cpu_id; + /* Set defaults */ + info->pmsfcr = 0; + info->pmsevfr = 0xFFFFFFFFFFFFFFFFUL; + info->pmslatfr = 0; + info->pmsirr = + (arm_spe_min_interval(info->sc) << PMSIRR_INTERVAL_SHIFT) + | PMSIRR_RND; + info->pmsicr = 0; + info->pmscr = PMSCR_TS | PMSCR_PA | PMSCR_CX | PMSCR_E1SPE | PMSCR_E0SPE; + + if (ctx->config && + ctx->config_size == sizeof(struct arm_spe_config) && + ctx->config_version == 1) { + cfg = (struct arm_spe_config *)ctx->config; + if (cfg->interval) + arm_spe_set_interval(info, cfg->interval); + if (cfg->level == ARM_SPE_KERNEL_ONLY) + info->pmscr &= ~(PMSCR_E0SPE); /* turn off user */ + if (cfg->level == ARM_SPE_USER_ONLY) + info->pmscr &= ~(PMSCR_E1SPE); /* turn off kern */ + if (cfg->ctx_field) + info->ctx_field = cfg->ctx_field; + } else + err = (EINVAL); + mtx_unlock_spin(&info->lock); + + return (err); +} + + +static void +arm_spe_enable(void *arg __unused) +{ + struct arm_spe_info *info = &spe_info[PCPU_GET(cpuid)];; + uint64_t base, limit; + + dprintf("%s on cpu:%d\n", __func__, PCPU_GET(cpuid)); + + mtx_lock_spin(&info->lock); + + if (info->ctx_field == ARM_SPE_CTX_CPU_ID) + WRITE_SPECIALREG(CONTEXTIDR_EL1_REG, PCPU_GET(cpuid)); + + WRITE_SPECIALREG(PMSFCR_EL1_REG, info->pmsfcr); + WRITE_SPECIALREG(PMSEVFR_EL1_REG, info->pmsevfr); + WRITE_SPECIALREG(PMSLATFR_EL1_REG, info->pmslatfr); + + /* Set the sampling interval */ + WRITE_SPECIALREG(PMSIRR_EL1_REG, info->pmsirr); + isb(); + + /* Write 0 here before enabling sampling */ + WRITE_SPECIALREG(PMSICR_EL1_REG, info->pmsicr); + isb(); + + base = info->kvaddr; + limit = base + (info->buf_size/2); + /* Enable the buffer */ + limit &= PMBLIMITR_LIMIT_MASK; /* Zero lower 12 bits */ + limit |= PMBLIMITR_E; + /* Set the base and limit */ + WRITE_SPECIALREG(PMBPTR_EL1_REG, base); + WRITE_SPECIALREG(PMBLIMITR_EL1_REG, limit); + isb(); + + /* Enable sampling */ + WRITE_SPECIALREG(PMSCR_EL1_REG, info->pmscr); + isb(); + + info->enabled = true; + + mtx_unlock_spin(&info->lock); +} + +static void +spe_backend_enable_smp(struct hwt_context *ctx) +{ + struct arm_spe_info *info; + struct hwt_vm *vm; + int cpu_id; + + HWT_CTX_LOCK(ctx); + CPU_FOREACH_ISSET(cpu_id, &ctx->cpu_map) { + vm = hwt_cpu_get(ctx, cpu_id)->vm; + + info = &spe_info[cpu_id]; + + mtx_lock_spin(&info->lock); + info->kvaddr = vm->kvaddr; + info->buf_size = ctx->bufsize; + mtx_unlock_spin(&info->lock); + } + HWT_CTX_UNLOCK(ctx); + + cpu_id = CPU_FFS(&ctx->cpu_map); + info = &spe_info[cpu_id]; + if (info->ctx_field == ARM_SPE_CTX_PID) + arm64_pid_in_contextidr = true; + else + arm64_pid_in_contextidr = false; + + smp_rendezvous_cpus(ctx->cpu_map, smp_no_rendezvous_barrier, + arm_spe_enable, smp_no_rendezvous_barrier, NULL); +} + +void +arm_spe_disable(void *arg __unused) +{ + struct arm_spe_info *info = &spe_info[PCPU_GET(cpuid)]; + struct arm_spe_buf_info *buf = &info->buf_info[info->buf_idx]; + + if (!info->enabled) + return; + + dprintf("%s on cpu:%d\n", __func__, PCPU_GET(cpuid)); + + /* Disable profiling */ + WRITE_SPECIALREG(PMSCR_EL1_REG, 0x0); + isb(); + + /* Drain any remaining tracing data */ + psb_csync(); + dsb(nsh); + + /* Disable the profiling buffer */ + WRITE_SPECIALREG(PMBLIMITR_EL1_REG, 0); + isb(); + + /* Clear interrupt status reg */ + WRITE_SPECIALREG(PMBSR_EL1_REG, 0x0); + + /* Clear PID/CPU_ID from context ID reg */ + WRITE_SPECIALREG(CONTEXTIDR_EL1_REG, 0); + + mtx_lock_spin(&info->lock); + buf->pmbptr = READ_SPECIALREG(PMBPTR_EL1_REG); + info->enabled = false; + mtx_unlock_spin(&info->lock); +} + +void +spe_backend_disable_smp(struct hwt_context *ctx) +{ + struct kevent kev; + struct arm_spe_info *info; + struct arm_spe_buf_info *buf; + int cpu_id; + int ret; + + /* Disable and send out remaining data in bufs */ + smp_rendezvous_cpus(ctx->cpu_map, smp_no_rendezvous_barrier, + arm_spe_disable, smp_no_rendezvous_barrier, NULL); + + CPU_FOREACH_ISSET(cpu_id, &ctx->cpu_map) { + info = &spe_info[cpu_id]; + buf = &info->buf_info[info->buf_idx]; + arm_spe_send_buffer(buf, 0); + } + + arm64_pid_in_contextidr = false; + + /* + * Tracing on all CPUs has been disabled, and we've sent write ptr + * offsets for all bufs - let userspace know it can shutdown + */ + EV_SET(&kev, ARM_SPE_KQ_SHUTDOWN, EVFILT_USER, 0, NOTE_TRIGGER, 0, NULL); + ret = kqfd_register(ctx->kqueue_fd, &kev, ctx->hwt_td, M_WAITOK); + if (ret) + dprintf("%s kqfd_register ret:%d\n", __func__, ret); +} + +static void +arm_spe_reenable(void *arg __unused) +{ + struct arm_spe_info *info = &spe_info[PCPU_GET(cpuid)];; + + WRITE_SPECIALREG(PMSCR_EL1_REG, info->pmscr); + isb(); +} + +static int +spe_backend_svc_buf(struct hwt_context *ctx, void *data, size_t data_size, + int data_version) +{ + struct arm_spe_info *info; + struct arm_spe_buf_info *buf; + struct arm_spe_svc_buf *s; + int err = 0; + cpuset_t cpu_set; + + if (data_size != sizeof(struct arm_spe_svc_buf)) + return (E2BIG); + + if (data_version != 1) + return (EINVAL); + + s = (struct arm_spe_svc_buf *)data; + if (s->buf_idx > 1) + return (ENODEV); + + info = &spe_info[s->ident]; + mtx_lock_spin(&info->lock); + + buf = &info->buf_info[s->buf_idx]; + + if (!info->enabled) { + err = ENXIO; + goto end; + } + + /* Clear the flag the signals buffer needs servicing */ + buf->buf_svc = false; + + /* Re-enable profiling if we've been waiting for this notification */ + if (buf->buf_wait) { + CPU_SETOF(s->ident, &cpu_set); + + mtx_unlock_spin(&info->lock); + smp_rendezvous_cpus(cpu_set, smp_no_rendezvous_barrier, + arm_spe_reenable, smp_no_rendezvous_barrier, NULL); + mtx_lock_spin(&info->lock); + + buf->buf_wait = false; + } + +end: + mtx_unlock_spin(&info->lock); + return (err); +} + +static int +spe_backend_read(struct hwt_vm *vm, int *ident, vm_offset_t *offset, + uint64_t *data) +{ + struct arm_spe_queue *q; + struct arm_spe_softc *sc = device_get_softc(spe_dev); + int error = 0; + + + mtx_lock_spin(&sc->sc_lock); + + /* Return the first pending buffer that needs servicing */ + q = STAILQ_FIRST(&sc->pending); + if (q == NULL) { + error = ENOENT; + goto error; + } + *ident = q->ident; + *offset = q->offset; + *data = (q->buf_idx << KQ_BUF_POS_SHIFT) | + (q->partial_rec << KQ_PARTREC_SHIFT) | + (q->final_buf << KQ_FINAL_BUF_SHIFT); + + STAILQ_REMOVE_HEAD(&sc->pending, next); + sc->npending--; + +error: + mtx_unlock_spin(&sc->sc_lock); + if (error) + return (error); + + free(q, M_ARM_SPE); + return (0); +} + +static struct hwt_backend_ops spe_ops = { + .hwt_backend_init = spe_backend_init, + .hwt_backend_deinit = spe_backend_deinit, + + .hwt_backend_configure = spe_backend_configure, + .hwt_backend_svc_buf = spe_backend_svc_buf, + .hwt_backend_stop = spe_backend_disable_smp, + + .hwt_backend_enable_smp = spe_backend_enable_smp, + .hwt_backend_disable_smp = spe_backend_disable_smp, + + .hwt_backend_read = spe_backend_read, +}; + +int +spe_register(device_t dev) +{ + int error; + + spe_dev = dev; + + error = hwt_backend_register(&backend); + if (error != 0) { + return (error); + } + + return (0); +} + +static void +arm_spe_init(void) +{ + return; +} +SYSINIT(spe, SI_SUB_DRIVERS, SI_ORDER_FIRST, arm_spe_init, NULL); + Index: sys/arm64/spe/arm_spe_dev.h =================================================================== --- /dev/null +++ sys/arm64/spe/arm_spe_dev.h @@ -0,0 +1,162 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause + * + * Copyright (c) 2024 Arm Ltd + * Copyright (c) 2022 The FreeBSD Foundation + * + * Portions of this software were developed by Andrew Turner under sponsorship + * from the FreeBSD Foundation. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef _ARM64_ARM_SPE_DEV_H_ +#define _ARM64_ARM_SPE_DEV_H_ + +#include +#include + +#include + +#include + +#include + +#define ARM_SPE_DEBUG +#undef ARM_SPE_DEBUG + +#ifdef ARM_SPE_DEBUG +#define dprintf(fmt, ...) printf(fmt, ##__VA_ARGS__) +#else +#define dprintf(fmt, ...) +#endif + +DECLARE_CLASS(arm_spe_driver); + +struct cdev; +struct resource; + +extern bool arm64_pid_in_contextidr; + +int spe_register(device_t dev); +void arm_spe_disable(void *arg __unused); +void spe_backend_disable_smp(struct hwt_context *ctx); +void arm_spe_send_buffer(void *arg, int pending __unused); + +/* + PSB CSYNC is a Profiling Synchronization Barrier encoded in the hint space + * so it is a NOP on earlier architecture. + */ +#define psb_csync() __asm __volatile("hint #17" ::: "memory"); + +struct arm_spe_softc { + device_t dev; + + struct resource *sc_irq_res; + void *sc_irq_cookie; + struct cdev *sc_cdev; + struct mtx sc_lock; + struct task task; + + uint64_t sc_pmsidr; + int kqueue_fd; + struct thread *hwt_td; + struct arm_spe_info *spe_info; + struct hwt_context *ctx; + STAILQ_HEAD(, arm_spe_queue) pending; + uint64_t npending; + + uint64_t pmbidr; + uint64_t pmsidr; + + uint16_t kva_align; +}; + +struct arm_spe_buf_info { + struct arm_spe_info *info; + uint64_t pmbptr; + uint8_t buf_idx : 1; + bool buf_svc : 1; + bool buf_wait : 1; + bool partial_rec : 1; +}; + +struct arm_spe_info { + int ident; /* tid or cpu_id */ + struct mtx lock; + struct arm_spe_softc *sc; + struct task task[2]; + bool enabled : 1; + + /* buffer is split in half as a ping-pong buffer */ + vm_object_t bufobj; + vm_offset_t kvaddr; + size_t buf_size; + uint8_t buf_idx : 1; /* 0 = first half of buf, 1 = 2nd half */ + struct arm_spe_buf_info buf_info[2]; + + /* config */ + enum arm_spe_profiling_level level; + enum arm_spe_ctx_field ctx_field; + /* filters */ + uint64_t pmsfcr; + uint64_t pmsevfr; + uint64_t pmslatfr; + /* interval */ + uint64_t pmsirr; + uint64_t pmsicr; + /* control */ + uint64_t pmscr; +}; + +struct arm_spe_queue { + int ident; + u_int buf_idx : 1; + bool partial_rec : 1; + bool final_buf : 1; + vm_offset_t offset; + STAILQ_ENTRY(arm_spe_queue) next; +}; + +static inline vm_offset_t buf_start_addr(u_int buf_idx, struct arm_spe_info *info) +{ + vm_offset_t addr; + if (buf_idx == 0) + addr = info->kvaddr; + if (buf_idx == 1) + addr = info->kvaddr + (info->buf_size/2); + + return addr; +} + +static inline vm_offset_t buf_end_addr(u_int buf_idx, struct arm_spe_info *info) +{ + vm_offset_t addr; + if (buf_idx == 0) + addr = info->kvaddr + (info->buf_size/2); + if (buf_idx == 1) + addr = info->kvaddr + info->buf_size; + + return addr; +} + +#endif /* _ARM64_ARM_SPE_DEV_H_ */ Index: sys/arm64/spe/arm_spe_dev.c =================================================================== --- /dev/null +++ sys/arm64/spe/arm_spe_dev.c @@ -0,0 +1,321 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause + * + * Copyright (c) 2024 Arm Ltd + * Copyright (c) 2022 The FreeBSD Foundation + * + * Portions of this software were developed by Andrew Turner under sponsorship + * from the FreeBSD Foundation. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include +#include + +MALLOC_DEFINE(M_ARM_SPE, "armspe", "Arm SPE tracing"); + +/* + * taskqueue(9) used for sleepable routines called from interrupt handlers + */ +TASKQUEUE_FAST_DEFINE_THREAD(arm_spe); + +void arm_spe_send_buffer(void *, int); +static void arm_spe_error(void *, int); +static int arm_spe_intr(void *); +device_attach_t arm_spe_attach; + +static device_method_t arm_spe_methods[] = { + /* Device interface */ + DEVMETHOD(device_attach, arm_spe_attach), + + DEVMETHOD_END, +}; + +DEFINE_CLASS_0(spe, arm_spe_driver, arm_spe_methods, + sizeof(struct arm_spe_softc)); + +int +arm_spe_attach(device_t dev) +{ + struct arm_spe_softc *sc; + int error, rid; + + sc = device_get_softc(dev); + sc->dev = dev; + + sc->pmbidr = READ_SPECIALREG(PMBIDR_EL1_REG); + sc->pmsidr = READ_SPECIALREG(PMSIDR_EL1_REG); + device_printf(dev, "PMBIDR_EL1: %#lx\n", sc->pmbidr); + device_printf(dev, "PMSIDR_EL1: %#lx\n", sc->pmsidr); + if (sc->pmbidr & PMBIDR_P) { + device_printf(dev, "Profiling Buffer is owned by a higher Exception level\n"); + return (EPERM); + } + + sc->kva_align = 1 << ((sc->pmbidr & PMBIDR_Align_MASK) >> PMBIDR_Align_SHIFT); + if (sc->kva_align > UL(2048)) { + device_printf(dev, "Invalid PMBIDR.Align value of %d\n", sc->kva_align); + return (EINVAL); + } + + rid = 0; + sc->sc_irq_res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid, + RF_ACTIVE); + if (sc->sc_irq_res == NULL) { + device_printf(dev, "Unable to allocate interrupt\n"); + return (ENXIO); + } + error = bus_setup_intr(dev, sc->sc_irq_res, + INTR_TYPE_MISC | INTR_MPSAFE, arm_spe_intr, NULL, sc, + &sc->sc_irq_cookie); + if (error != 0) { + device_printf(dev, "Unable to set up interrupt\n"); + return (error); + } + + mtx_init(&sc->sc_lock, "Arm SPE lock", NULL, MTX_SPIN); + + STAILQ_INIT(&sc->pending); + sc->npending = 0; + + spe_register(dev); + + return (0); +} + +/* Interrupt handler runs on the same core that triggered the exception */ +static int +arm_spe_intr(void *arg) +{ + int cpu_id = PCPU_GET(cpuid); + struct arm_spe_softc *sc = arg; + uint64_t pmbsr; + uint64_t base, limit; + uint8_t ec; + struct arm_spe_info *info = &sc->spe_info[cpu_id]; + uint8_t i = info->buf_idx; + struct arm_spe_buf_info *buf = &info->buf_info[i]; + struct arm_spe_buf_info *prev_buf = &info->buf_info[!i]; + device_t dev = sc->dev; + + /* Make sure the profiling data is visible to the CPU */ + psb_csync(); + dsb(nsh); + + /* Make sure any HW update of PMBPTR_EL1 is visible to the CPU */ + isb(); + + pmbsr = READ_SPECIALREG(PMBSR_EL1_REG); + + if (!(pmbsr & PMBSR_S)) + return (FILTER_STRAY); + + /* Event Class */ + ec = PMBSR_EC_VAL(pmbsr); + switch (ec) + { + case PMBSR_EC_OTHER_BUF_MGMT: /* Other buffer management event */ + break; + case PMBSR_EC_GRAN_PROT_CHK: /* Granule Protection Check fault */ + device_printf(dev, "PMBSR_EC_GRAN_PROT_CHK\n"); + break; + case PMBSR_EC_STAGE1_DA: /* Stage 1 Data Abort */ + device_printf(dev, "PMBSR_EC_STAGE1_DA\n"); + break; + case PMBSR_EC_STAGE2_DA: /* Stage 2 Data Abort */ + device_printf(dev, "PMBSR_EC_STAGE2_DA\n"); + break; + default: + /* Unknown EC */ + device_printf(dev, "unknown PMBSR_EC: %#x\n", ec); + arm_spe_disable(NULL); + TASK_INIT(&sc->task, 0, (task_fn_t *)arm_spe_error, sc->ctx); + taskqueue_enqueue(taskqueue_arm_spe, &sc->task); + return (FILTER_HANDLED); + } + + switch (ec) + { + case PMBSR_EC_OTHER_BUF_MGMT: + /* Buffer Status Code = buffer filled */ + if ((pmbsr & PMBSR_MSS_BSC_MASK) == 0b000001) + dprintf("%s SPE buffer full event (cpu:%d)\n", + __func__, cpu_id); + break; + case PMBSR_EC_GRAN_PROT_CHK: + case PMBSR_EC_STAGE1_DA: + case PMBSR_EC_STAGE2_DA: + /* + * If we have one of these, we've messed up the + * programming somehow (e.g. passed invalid memory to + * SPE) and can't recover + */ + arm_spe_disable(NULL); + TASK_INIT(&sc->task, 0, (task_fn_t *)arm_spe_error, sc->ctx); + taskqueue_enqueue(taskqueue_arm_spe, &sc->task); + /* PMBPTR_EL1 is fault address if PMBSR_DL is 1 */ + device_printf(dev, "CPU:%d PMBSR_EL1:%#lx\n", cpu_id, pmbsr); + device_printf(dev, "PMBPTR_EL1:%#lx PMBLIMITR_EL1:%#lx\n", + READ_SPECIALREG(PMBPTR_EL1_REG), + READ_SPECIALREG(PMBLIMITR_EL1_REG)); + return (FILTER_HANDLED); + } + + mtx_lock_spin(&info->lock); + + /* + * Data Loss bit - pmbptr might not be pointing to the end of the last + * complete record + */ + if ((pmbsr & PMBSR_DL) == 1) + buf->partial_rec = 1; + buf->pmbptr = READ_SPECIALREG(PMBPTR_EL1_REG); + buf->buf_svc = true; + + /* Setup regs ready to start writing to the other half of the buffer */ + info->buf_idx = !info->buf_idx; + base = buf_start_addr(info->buf_idx, info); + limit = base + (info->buf_size/2); + limit &= PMBLIMITR_LIMIT_MASK; + limit |= PMBLIMITR_E; + WRITE_SPECIALREG(PMBPTR_EL1_REG, base); + WRITE_SPECIALREG(PMBLIMITR_EL1_REG, limit); + isb(); + + /* + * Notify userspace via kqueue that buffer is full and needs copying + * out - since kqueue can sleep, don't do this in the interrupt handler, + * add to a taskqueue to be scheduled later instead + */ + TASK_INIT(&info->task[i], 0, (task_fn_t *)arm_spe_send_buffer, buf); + taskqueue_enqueue(taskqueue_arm_spe, &info->task[i]); + + /* + * It's possible userspace hasn't yet notified us they've copied out the + * other half of the buffer + * + * This might be because: + * a) Kernel hasn't scheduled the task via taskqueue to notify + * userspace to copy out the data + * b) Userspace is still copying the buffer or hasn't notified us + * back via the HWT_IOC_SVC_BUF ioctl + * + * Either way we need to avoid overwriting uncopied data in the + * buffer, so disable profiling until we receive that SVC_BUF + * ioctl + * + * Using a larger buffer size should help to minimise these events and + * loss of profiling data while profiling is disabled + */ + if (prev_buf->buf_svc) { + device_printf(sc->dev, "cpu%d: buffer full interrupt, but other" + " half of buffer has not been copied out - consider" + " increasing buffer size to minimise loss of profiling data\n", + cpu_id); + WRITE_SPECIALREG(PMSCR_EL1_REG, 0x0); + prev_buf->buf_wait = true; + } + + mtx_unlock_spin(&info->lock); + + /* Clear Profiling Buffer Status Register */ + WRITE_SPECIALREG(PMBSR_EL1_REG, 0); + + isb(); + + return (FILTER_HANDLED); +} + +/* note: Scheduled and run via taskqueue, so can run on any CPU at any time */ +void +arm_spe_send_buffer(void *arg, int pending __unused) +{ + struct arm_spe_buf_info *buf = (struct arm_spe_buf_info *)arg; + struct arm_spe_info *info = buf->info; + struct arm_spe_queue *queue; + struct kevent kev; + int ret; + + queue = malloc(sizeof(struct arm_spe_queue), M_ARM_SPE, + M_WAITOK | M_ZERO); + + mtx_lock_spin(&info->lock); + + /* Add to queue for userspace to pickup */ + queue->ident = info->ident; + queue->offset = buf->pmbptr - buf_start_addr(buf->buf_idx, info); + queue->buf_idx = buf->buf_idx; + queue->final_buf = !info->enabled; + queue->partial_rec = buf->partial_rec; + mtx_unlock_spin(&info->lock); + + mtx_lock_spin(&info->sc->sc_lock); + STAILQ_INSERT_TAIL(&info->sc->pending, queue, next); + info->sc->npending++; + EV_SET(&kev, ARM_SPE_KQ_BUF, EVFILT_USER, 0, NOTE_TRIGGER, + info->sc->npending, NULL); + mtx_unlock_spin(&info->sc->sc_lock); + + /* Notify userspace */ + ret = kqfd_register(info->sc->kqueue_fd, &kev, info->sc->hwt_td, + M_WAITOK); + if (ret) { + dprintf("%s kqfd_register ret:%d\n", __func__, ret); + arm_spe_error(info->sc->ctx, 0); + } +} + + +static void +arm_spe_error(void *arg, int pending __unused) +{ + struct hwt_context *ctx = arg; + struct kevent kev; + int ret; + + smp_rendezvous_cpus(ctx->cpu_map, smp_no_rendezvous_barrier, + arm_spe_disable, smp_no_rendezvous_barrier, NULL); + + EV_SET(&kev, ARM_SPE_KQ_SHUTDOWN, EVFILT_USER, 0, NOTE_TRIGGER, 0, NULL); + ret = kqfd_register(ctx->kqueue_fd, &kev, ctx->hwt_td, M_WAITOK); + if (ret) + dprintf("%s kqfd_register ret:%d\n", __func__, ret); +} Index: sys/arm64/spe/arm_spe_fdt.c =================================================================== --- /dev/null +++ sys/arm64/spe/arm_spe_fdt.c @@ -0,0 +1,80 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause + * + * Copyright (c) 2024 Arm Ltd + * Copyright (c) 2022 The FreeBSD Foundation + * + * Portions of this software were developed by Andrew Turner under sponsorship + * from the FreeBSD Foundation. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include + +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include + +static device_probe_t arm_spe_fdt_probe; + +static struct ofw_compat_data compat_data[] = { + {"arm,statistical-profiling-extension-v1", true}, + {NULL, false}, +}; + +static device_method_t arm_spe_fdt_methods[] = { + /* Device interface */ + DEVMETHOD(device_probe, arm_spe_fdt_probe), + + DEVMETHOD_END, +}; + +DEFINE_CLASS_1(spe, arm_spe_fdt_driver, arm_spe_fdt_methods, + sizeof(struct arm_spe_softc), arm_spe_driver); + +DRIVER_MODULE(spe, simplebus, arm_spe_fdt_driver, 0, 0); +MODULE_DEPEND(spe, hwt, 1, 1, 1); +MODULE_VERSION(spe, 1); + +static int +arm_spe_fdt_probe(device_t dev) +{ + if (!ofw_bus_status_okay(dev)) + return (ENXIO); + + if (!ofw_bus_search_compatible(dev, compat_data)->ocd_data) + return (ENXIO); + + device_set_desc(dev, "ARM Statistical Profiling Extension"); + return (BUS_PROBE_DEFAULT); +} + Index: sys/conf/files.arm64 =================================================================== --- sys/conf/files.arm64 +++ sys/conf/files.arm64 @@ -25,6 +25,10 @@ arm64/acpica/acpi_wakeup.c optional acpi arm64/acpica/pci_cfgreg.c optional acpi pci arm64/arm64/autoconf.c standard +arm64/spe/arm_spe_backend.c optional hwt spe +arm64/spe/arm_spe_dev.c optional hwt spe +arm64/spe/arm_spe_acpi.c optional hwt spe acpi +arm64/spe/arm_spe_fdt.c optional hwt spe fdt arm64/arm64/bus_machdep.c standard arm64/arm64/bus_space_asm.S standard arm64/arm64/busdma_bounce.c standard Index: sys/modules/Makefile =================================================================== --- sys/modules/Makefile +++ sys/modules/Makefile @@ -376,6 +376,7 @@ smbfs \ snp \ sound \ + ${_spe} \ ${_speaker} \ spi \ ${_splash} \ @@ -665,6 +666,10 @@ .endif .endif +.if ${MACHINE_CPUARCH} == "aarch64" +_spe= spe +.endif + .if ${MACHINE_CPUARCH} == "aarch64" || ${MACHINE_CPUARCH} == "arm" || \ ${MACHINE_CPUARCH} == "riscv" .if !empty(OPT_FDT) Index: sys/modules/spe/Makefile =================================================================== --- /dev/null +++ sys/modules/spe/Makefile @@ -0,0 +1,24 @@ +# $FreeBSD$ + +.PATH: ${SRCTOP}/sys/arm64/spe + +KMOD = spe +SRCS = \ + arm_spe_dev.c \ + arm_spe_backend.c \ + arm_spe.h \ + arm_spe_dev.h + +SRCS+= opt_platform.h + +SRCS.DEV_ACPI= \ + arm_spe_acpi.c + +.if !empty(OPT_FDT) +SRCS+= \ + arm_spe_fdt.c +.endif + +EXPORT_SYMS= YES + +.include