diff --git a/sys/arm64/arm64/cpu_feat.c b/sys/arm64/arm64/cpu_feat.c new file mode 100644 --- /dev/null +++ b/sys/arm64/arm64/cpu_feat.c @@ -0,0 +1,86 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause + * + * Copyright (c) 2024 Arm Ltd + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include +#include +#include + +#include +#include + +void +enable_cpu_feat(uint32_t stage) +{ + struct cpu_feat **featp, *feat; + uint32_t midr; + u_int errata_id; + cpu_feat_errata errata_status; + + MPASS((stage & ~CPU_FEAT_STAGE_MASK) == 0); + + midr = get_midr(); + SET_FOREACH(featp, cpu_feat_set) { + feat = *featp; + + /* Run the enablement code at the correct stage of boot */ + if ((feat->feat_flags & CPU_FEAT_STAGE_MASK) != stage) + continue; + + /* If the feature is system wide run on a single CPU */ + if ((feat->feat_flags & CPU_FEAT_SCOPE_MASK)==CPU_FEAT_SYSTEM && + PCPU_GET(cpuid) != 0) + continue; + + if (feat->feat_check != NULL && !feat->feat_check(feat, midr)) + continue; + + /* + * Check if the feature has any errata that may need a + * workaround applied (or it is to install the workaround for + * known errata. + */ + /* TODO: Check the Errata Management ABI if possible */ + errata_status = ERRATA_NONE; + errata_id = 0; + if (feat->feat_has_errata != NULL && + feat->feat_has_errata(feat, midr, &errata_id)) { + /* Assume we are affected until told otherwise */ + errata_status = ERRATA_AFFECTED; + } + + feat->feat_enable(feat, errata_status, errata_id); + } +} + +static void +enable_cpu_feat_after_dev(void *dummy __unused) +{ + MPASS(PCPU_GET(cpuid) == 0); + enable_cpu_feat(CPU_FEAT_AFTER_DEV); +} +SYSINIT(enable_cpu_feat_after_dev, SI_SUB_CONFIGURE, SI_ORDER_MIDDLE, + enable_cpu_feat_after_dev, NULL); diff --git a/sys/arm64/arm64/machdep.c b/sys/arm64/arm64/machdep.c --- a/sys/arm64/arm64/machdep.c +++ b/sys/arm64/arm64/machdep.c @@ -77,6 +77,7 @@ #include #include +#include #include #include #include @@ -999,6 +1000,9 @@ panic("Invalid bus configuration: %s", kern_getenv("kern.cfg.order")); + /* Detect early CPU feature support */ + enable_cpu_feat(CPU_FEAT_EARLY_BOOT); + /* * Check if pointer authentication is available on this system, and * if so enable its use. This needs to be called before init_proc0 diff --git a/sys/arm64/arm64/mp_machdep.c b/sys/arm64/arm64/mp_machdep.c --- a/sys/arm64/arm64/mp_machdep.c +++ b/sys/arm64/arm64/mp_machdep.c @@ -56,6 +56,7 @@ #include #include +#include #include #include #include @@ -224,6 +225,9 @@ /* Ensure the stores in identify_cpu have completed */ atomic_thread_fence_acq_rel(); + /* Detect early CPU feature support */ + enable_cpu_feat(CPU_FEAT_EARLY_BOOT); + /* Signal the BSP and spin until it has released all APs. */ atomic_add_int(&aps_started, 1); while (!atomic_load_int(&aps_ready)) @@ -241,6 +245,7 @@ pcpup->pc_curpmap = pmap0; install_cpu_errata(); + enable_cpu_feat(CPU_FEAT_AFTER_DEV); intr_pic_init_secondary(); diff --git a/sys/arm64/include/cpu_feat.h b/sys/arm64/include/cpu_feat.h new file mode 100644 --- /dev/null +++ b/sys/arm64/include/cpu_feat.h @@ -0,0 +1,67 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause + * + * Copyright (c) 2024 Arm Ltd + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef _MACHINE_CPU_FEAT_H_ +#define _MACHINE_CPU_FEAT_H_ + +#include + +typedef enum { + ERRATA_NONE, /* No errata for this feature on this system. */ + ERRATA_AFFECTED, /* There is errata on this system. */ + ERRATA_FW_MITIGAION, /* There is errata, and a firmware */ + /* mitigation. The mitigation may need a */ + /* kernel component. */ +} cpu_feat_errata; + +#define CPU_FEAT_STAGE_MASK 0x00000001 +#define CPU_FEAT_EARLY_BOOT 0x00000000 +#define CPU_FEAT_AFTER_DEV 0x00000001 + +#define CPU_FEAT_SCOPE_MASK 0x00000010 +#define CPU_FEAT_PER_CPU 0x00000000 +#define CPU_FEAT_SYSTEM 0x00000010 + +struct cpu_feat; + +typedef bool (cpu_feat_check)(const struct cpu_feat *, u_int); +typedef bool (cpu_feat_has_errata)(const struct cpu_feat *, u_int, + u_int *); +typedef void (cpu_feat_enable)(const struct cpu_feat *, cpu_feat_errata, u_int); + +struct cpu_feat { + const char *feat_name; + cpu_feat_check *feat_check; + cpu_feat_has_errata *feat_has_errata; + cpu_feat_enable *feat_enable; + uint32_t feat_flags; +}; +SET_DECLARE(cpu_feat_set, struct cpu_feat); + +void enable_cpu_feat(uint32_t); + +#endif /* _MACHINE_CPU_FEAT_H_ */ diff --git a/sys/conf/files.arm64 b/sys/conf/files.arm64 --- a/sys/conf/files.arm64 +++ b/sys/conf/files.arm64 @@ -32,6 +32,7 @@ arm64/arm64/clock.c standard arm64/arm64/copyinout.S standard arm64/arm64/cpu_errata.c standard +arm64/arm64/cpu_feat.c standard arm64/arm64/cpufunc_asm.S standard arm64/arm64/db_disasm.c optional ddb arm64/arm64/db_interface.c optional ddb