Page MenuHomeFreeBSD
Differential D47817

arm64: Use the PAN msr mnemonic rather than .inst
ClosedPublic
Actions

Authored by andrew on Nov 27 2024, 6:12 PM.
Tags
None
Referenced Files
Unknown Object (File)
Mon, Aug 25, 9:55 AM
Unknown Object (File)
Sat, Aug 23, 7:05 PM
Unknown Object (File)
Fri, Aug 15, 3:59 PM
Unknown Object (File)
Thu, Aug 14, 11:08 PM
Unknown Object (File)
Thu, Aug 14, 11:21 AM
Unknown Object (File)
Sat, Aug 2, 5:04 AM
Unknown Object (File)
Jul 30 2025, 10:46 AM
Unknown Object (File)
Jul 28 2025, 10:26 PM
View All 49 Files
Subscribers
emaste
imp

Details

Reviewers
manu
Group Reviewers
ARM
Commits
rG938e4b131c48: arm64: Use the PAN msr mnemonic rather than .inst
Summary

Switch from creating the msr instructions to manage PAN to use the
"msr pan, #1" instruction directly. When this was added clang didn't
have support to assemble the instructions. This appears to have been
added to clang 13 which is sufficiently old enough.

Binutils releases from around the same time appear to have added this
instruction so any modern gcc should also support this instruction.

Sponsored by: Arm Ltd

Diff Detail

Repository
rG FreeBSD src repository
Lint
Lint Skipped
Unit
Tests Skipped
Build Status
Buildable 60852
Build 57736: arc lint + arc unit

Revision Contents
Changeset List

PathSize
sys/
arm64/
arm64/
10 lines
include/
12 lines

Diff 147079

View Options

sys/arm64/arm64/machdep.c

Show All 36 Lines
#include <sys/sbuf.h> #include <sys/sbuf.h>
#include <sys/smp.h> #include <sys/smp.h>
#include <sys/sysctl.h> #include <sys/sysctl.h>
#include <sys/sysent.h> #include <sys/sysent.h>
#include <sys/systm.h> #include <sys/systm.h>
#include <machine/atomic.h> #include <machine/atomic.h>
#include <machine/cpu.h> #include <machine/cpu.h>
#include <machine/cpu_feat.h>
#include <machine/cpufunc.h> #include <machine/cpufunc.h>
#include <machine/elf.h> #include <machine/elf.h>
#include <machine/md_var.h> #include <machine/md_var.h>
#include <machine/undefined.h> #include <machine/undefined.h>
static MALLOC_DEFINE(M_IDENTCPU, "CPU ID", "arm64 CPU identification memory"); static MALLOC_DEFINE(M_IDENTCPU, "CPU ID", "arm64 CPU identification memory");
struct cpu_desc; struct cpu_desc;
Show All 18 Lines
static int allow_dic = 1; static int allow_dic = 1;
SYSCTL_INT(_machdep_cache, OID_AUTO, allow_dic, CTLFLAG_RDTUN, &allow_dic, 0, SYSCTL_INT(_machdep_cache, OID_AUTO, allow_dic, CTLFLAG_RDTUN, &allow_dic, 0,
"Allow optimizations based on the DIC cache bit"); "Allow optimizations based on the DIC cache bit");
static int allow_idc = 1; static int allow_idc = 1;
SYSCTL_INT(_machdep_cache, OID_AUTO, allow_idc, CTLFLAG_RDTUN, &allow_idc, 0, SYSCTL_INT(_machdep_cache, OID_AUTO, allow_idc, CTLFLAG_RDTUN, &allow_idc, 0,
"Allow optimizations based on the IDC cache bit"); "Allow optimizations based on the IDC cache bit");
static bool emulate_ctr = false;
static void check_cpu_regs(u_int cpu, struct cpu_desc *desc, static void check_cpu_regs(u_int cpu, struct cpu_desc *desc,
struct cpu_desc *prev_desc); struct cpu_desc *prev_desc);
static uint64_t update_special_reg_field(uint64_t user_reg, u_int type, static uint64_t update_special_reg_field(uint64_t user_reg, u_int type,
uint64_t value, u_int width, u_int shift, bool sign); uint64_t value, u_int width, u_int shift, bool sign);
/* /*
* The default implementation of I-cache sync assumes we have an * The default implementation of I-cache sync assumes we have an
* aliasing cache until we know otherwise. * aliasing cache until we know otherwise.
▲ Show 20 Lines • Show All 1,978 Lines • ▼ Show 20 Linesstatic const struct mrs_user_reg user_regs[] = {
USER_REG(ID_AA64MMFR4_EL1, id_aa64mmfr4, true), USER_REG(ID_AA64MMFR4_EL1, id_aa64mmfr4, true),
USER_REG(ID_AA64PFR0_EL1, id_aa64pfr0, true), USER_REG(ID_AA64PFR0_EL1, id_aa64pfr0, true),
USER_REG(ID_AA64PFR1_EL1, id_aa64pfr1, true), USER_REG(ID_AA64PFR1_EL1, id_aa64pfr1, true),
USER_REG(ID_AA64PFR2_EL1, id_aa64pfr2, true), USER_REG(ID_AA64PFR2_EL1, id_aa64pfr2, true),
USER_REG(ID_AA64ZFR0_EL1, id_aa64zfr0, true), USER_REG(ID_AA64ZFR0_EL1, id_aa64zfr0, true),
USER_REG(CTR_EL0, ctr, true),
#ifdef COMPAT_FREEBSD32 #ifdef COMPAT_FREEBSD32
USER_REG(ID_ISAR5_EL1, id_isar5, false), USER_REG(ID_ISAR5_EL1, id_isar5, false),
USER_REG(MVFR0_EL1, mvfr0, false), USER_REG(MVFR0_EL1, mvfr0, false),
USER_REG(MVFR1_EL1, mvfr1, false), USER_REG(MVFR1_EL1, mvfr1, false),
#endif /* COMPAT_FREEBSD32 */ #endif /* COMPAT_FREEBSD32 */
}; };
#define CPU_DESC_FIELD(desc, idx) \ #define CPU_DESC_FIELD(desc, idx) \
*(uint64_t *)((char *)&(desc) + user_regs[(idx)].offset) *(uint64_t *)((char *)&(desc) + user_regs[(idx)].offset)
static bool
user_ctr_has_neoverse_n1_1542419(uint32_t midr, uint64_t ctr)
{
/* Skip non-Neoverse-N1 */
if (!CPU_MATCH(CPU_IMPL_MASK | CPU_PART_MASK, CPU_IMPL_ARM,
CPU_PART_NEOVERSE_N1, 0, 0))
return (false);
switch (CPU_VAR(midr)) {
default:
break;
case 4:
/* Fixed in r4p1 */
if (CPU_REV(midr) > 0)
break;
/* FALLTHROUGH */
case 3:
/* If DIC is enabled (coherent icache) then we are affected */
return (CTR_DIC_VAL(ctr) != 0);
}
return (false);
}
static bool
user_ctr_check(const struct cpu_feat *feat __unused, u_int midr __unused)
{
if (emulate_ctr)
return (true);
if (user_ctr_has_neoverse_n1_1542419(midr, READ_SPECIALREG(ctr_el0)))
return (true);
return (false);
}
static bool
user_ctr_has_errata(const struct cpu_feat *feat __unused, u_int midr,
u_int *errata_id)
{
if (user_ctr_has_neoverse_n1_1542419(midr, READ_SPECIALREG(ctr_el0))) {
*errata_id = 1542419;
return (true);
}
return (false);
}
static void
user_ctr_enable(const struct cpu_feat *feat __unused,
cpu_feat_errata errata_status, u_int errata_id __unused)
{
MPASS(emulate_ctr || errata_status != ERRATA_NONE);
/*
* The Errata Management Firmware Interface may incorrectly mark
* this as firmware mitigated. We should ignore that as there is
* a kernel component to the mitigation.
*/
if (errata_status != ERRATA_NONE && errata_id == 1542419 &&
PCPU_GET(cpuid) == 0) {
/* Clear fields we will change */
user_cpu_desc.ctr &= ~(CTR_DIC_MASK | CTR_ILINE_WIDTH);
/*
* Set DIC to none so userspace will execute an 'ic ivau'
* instruction that can be trapped by EL3.
*/
user_cpu_desc.ctr |= CTR_DIC_NONE;
/*
* Set the i-cache line size to be page size to reduce the
* number of times userspace needs to execute the 'ic ivau'
* instruction. The ctr_el0.IminLine is log2 the number of
* 4-byte words the instruction covers. As PAGE_SHIFT is log2
* of the number of bytes in a page we need to subtract 2.
*/
user_cpu_desc.ctr |= (PAGE_SHIFT - 2) << CTR_ILINE_SHIFT;
l_user_cpu_desc.ctr = user_cpu_desc.ctr;
}
WRITE_SPECIALREG(sctlr_el1,
READ_SPECIALREG(sctlr_el1) & ~SCTLR_UCT);
isb();
}
static struct cpu_feat user_ctr = {
.feat_name = "Trap CTR_EL0",
.feat_check = user_ctr_check,
.feat_has_errata = user_ctr_has_errata,
.feat_enable = user_ctr_enable,
.feat_flags = CPU_FEAT_AFTER_DEV | CPU_FEAT_PER_CPU,
};
DATA_SET(cpu_feat_set, user_ctr);
static int static int
user_ctr_handler(vm_offset_t va, uint32_t insn, struct trapframe *frame,
uint32_t esr)
{
uint64_t value;
int reg;
if ((insn & MRS_MASK) != MRS_VALUE)
return (0);
/* Check if this is the ctr_el0 register */
/* TODO: Add macros to armreg.h */
if (mrs_Op0(insn) != 3 || mrs_Op1(insn) != 3 || mrs_CRn(insn) != 0 ||
mrs_CRm(insn) != 0 || mrs_Op2(insn) != 1)
return (0);
if (SV_CURPROC_ABI() == SV_ABI_FREEBSD)
value = user_cpu_desc.ctr;
else
value = l_user_cpu_desc.ctr;
/*
* We will handle this instruction, move to the next so we
* don't trap here again.
*/
frame->tf_elr += INSN_SIZE;
reg = MRS_REGISTER(insn);
/* If reg is 31 then write to xzr, i.e. do nothing */
if (reg == 31)
return (1);
if (reg < nitems(frame->tf_x))
frame->tf_x[reg] = value;
else if (reg == 30)
frame->tf_lr = value;
return (1);
}
static int
user_mrs_handler(vm_offset_t va, uint32_t insn, struct trapframe *frame, user_mrs_handler(vm_offset_t va, uint32_t insn, struct trapframe *frame,
uint32_t esr) uint32_t esr)
{ {
uint64_t value; uint64_t value;
int CRm, Op2, i, reg; int CRm, Op2, i, reg;
if ((insn & MRS_MASK) != MRS_VALUE) if ((insn & MRS_MASK) != MRS_VALUE)
return (0); return (0);
▲ Show 20 Lines • Show All 393 Lines • ▼ Show 20 Lines if ((elf_hwcap & HWCAP_ATOMICS) != 0) {
if (bootverbose) if (bootverbose)
printf("Enabling LSE atomics in the kernel\n"); printf("Enabling LSE atomics in the kernel\n");
} }
#ifdef LSE_ATOMICS #ifdef LSE_ATOMICS
if (!lse_supported) if (!lse_supported)
panic("CPU does not support LSE atomic instructions"); panic("CPU does not support LSE atomic instructions");
#endif #endif
install_undef_handler(true, user_ctr_handler);
install_undef_handler(true, user_mrs_handler); install_undef_handler(true, user_mrs_handler);
} }
SYSINIT(identify_cpu, SI_SUB_CPU, SI_ORDER_MIDDLE, identify_cpu_sysinit, NULL); SYSINIT(identify_cpu, SI_SUB_CPU, SI_ORDER_MIDDLE, identify_cpu_sysinit, NULL);
static void static void
cpu_features_sysinit(void *dummy __unused) cpu_features_sysinit(void *dummy __unused)
{ {
struct sbuf sb; struct sbuf sb;
▲ Show 20 Lines • Show All 516 Lines • ▼ Show 20 Lines case 2:
/* FALLTHROUGH */ /* FALLTHROUGH */
case 3: case 3:
if (CPU_AFF3(desc->mpidr) != CPU_AFF3(prev_desc->mpidr)) if (CPU_AFF3(desc->mpidr) != CPU_AFF3(prev_desc->mpidr))
cpu_aff_levels = 4; cpu_aff_levels = 4;
break; break;
} }
if (desc->ctr != prev_desc->ctr) { if (desc->ctr != prev_desc->ctr) {
/*
* If the cache is different on different cores we should
* emulate for userspace to provide a uniform value
*/
emulate_ctr = true;
/* /*
* If the cache type register is different we may * If the cache type register is different we may
* have a different l1 cache type. * have a different l1 cache type.
*/ */
identify_cache(desc->ctr); identify_cache(desc->ctr);
} }
} }
View Options

sys/arm64/include/asm.h

Show All 36 Lines
#include <sys/sbuf.h> #include <sys/sbuf.h>
#include <sys/smp.h> #include <sys/smp.h>
#include <sys/sysctl.h> #include <sys/sysctl.h>
#include <sys/sysent.h> #include <sys/sysent.h>
#include <sys/systm.h> #include <sys/systm.h>
#include <machine/atomic.h> #include <machine/atomic.h>
#include <machine/cpu.h> #include <machine/cpu.h>
#include <machine/cpu_feat.h>
#include <machine/cpufunc.h> #include <machine/cpufunc.h>
#include <machine/elf.h> #include <machine/elf.h>
#include <machine/md_var.h> #include <machine/md_var.h>
#include <machine/undefined.h> #include <machine/undefined.h>
static MALLOC_DEFINE(M_IDENTCPU, "CPU ID", "arm64 CPU identification memory"); static MALLOC_DEFINE(M_IDENTCPU, "CPU ID", "arm64 CPU identification memory");
struct cpu_desc; struct cpu_desc;
Show All 18 Lines
static int allow_dic = 1; static int allow_dic = 1;
SYSCTL_INT(_machdep_cache, OID_AUTO, allow_dic, CTLFLAG_RDTUN, &allow_dic, 0, SYSCTL_INT(_machdep_cache, OID_AUTO, allow_dic, CTLFLAG_RDTUN, &allow_dic, 0,
"Allow optimizations based on the DIC cache bit"); "Allow optimizations based on the DIC cache bit");
static int allow_idc = 1; static int allow_idc = 1;
SYSCTL_INT(_machdep_cache, OID_AUTO, allow_idc, CTLFLAG_RDTUN, &allow_idc, 0, SYSCTL_INT(_machdep_cache, OID_AUTO, allow_idc, CTLFLAG_RDTUN, &allow_idc, 0,
"Allow optimizations based on the IDC cache bit"); "Allow optimizations based on the IDC cache bit");
static bool emulate_ctr = false;
static void check_cpu_regs(u_int cpu, struct cpu_desc *desc, static void check_cpu_regs(u_int cpu, struct cpu_desc *desc,
struct cpu_desc *prev_desc); struct cpu_desc *prev_desc);
static uint64_t update_special_reg_field(uint64_t user_reg, u_int type, static uint64_t update_special_reg_field(uint64_t user_reg, u_int type,
uint64_t value, u_int width, u_int shift, bool sign); uint64_t value, u_int width, u_int shift, bool sign);
/* /*
* The default implementation of I-cache sync assumes we have an * The default implementation of I-cache sync assumes we have an
* aliasing cache until we know otherwise. * aliasing cache until we know otherwise.
▲ Show 20 Lines • Show All 1,978 Lines • ▼ Show 20 Linesstatic const struct mrs_user_reg user_regs[] = {
USER_REG(ID_AA64MMFR4_EL1, id_aa64mmfr4, true), USER_REG(ID_AA64MMFR4_EL1, id_aa64mmfr4, true),
USER_REG(ID_AA64PFR0_EL1, id_aa64pfr0, true), USER_REG(ID_AA64PFR0_EL1, id_aa64pfr0, true),
USER_REG(ID_AA64PFR1_EL1, id_aa64pfr1, true), USER_REG(ID_AA64PFR1_EL1, id_aa64pfr1, true),
USER_REG(ID_AA64PFR2_EL1, id_aa64pfr2, true), USER_REG(ID_AA64PFR2_EL1, id_aa64pfr2, true),
USER_REG(ID_AA64ZFR0_EL1, id_aa64zfr0, true), USER_REG(ID_AA64ZFR0_EL1, id_aa64zfr0, true),
USER_REG(CTR_EL0, ctr, true),
#ifdef COMPAT_FREEBSD32 #ifdef COMPAT_FREEBSD32
USER_REG(ID_ISAR5_EL1, id_isar5, false), USER_REG(ID_ISAR5_EL1, id_isar5, false),
USER_REG(MVFR0_EL1, mvfr0, false), USER_REG(MVFR0_EL1, mvfr0, false),
USER_REG(MVFR1_EL1, mvfr1, false), USER_REG(MVFR1_EL1, mvfr1, false),
#endif /* COMPAT_FREEBSD32 */ #endif /* COMPAT_FREEBSD32 */
}; };
#define CPU_DESC_FIELD(desc, idx) \ #define CPU_DESC_FIELD(desc, idx) \
*(uint64_t *)((char *)&(desc) + user_regs[(idx)].offset) *(uint64_t *)((char *)&(desc) + user_regs[(idx)].offset)
static bool
user_ctr_has_neoverse_n1_1542419(uint32_t midr, uint64_t ctr)
{
/* Skip non-Neoverse-N1 */
if (!CPU_MATCH(CPU_IMPL_MASK | CPU_PART_MASK, CPU_IMPL_ARM,
CPU_PART_NEOVERSE_N1, 0, 0))
return (false);
switch (CPU_VAR(midr)) {
default:
break;
case 4:
/* Fixed in r4p1 */
if (CPU_REV(midr) > 0)
break;
/* FALLTHROUGH */
case 3:
/* If DIC is enabled (coherent icache) then we are affected */
return (CTR_DIC_VAL(ctr) != 0);
}
return (false);
}
static bool
user_ctr_check(const struct cpu_feat *feat __unused, u_int midr __unused)
{
if (emulate_ctr)
return (true);
if (user_ctr_has_neoverse_n1_1542419(midr, READ_SPECIALREG(ctr_el0)))
return (true);
return (false);
}
static bool
user_ctr_has_errata(const struct cpu_feat *feat __unused, u_int midr,
u_int *errata_id)
{
if (user_ctr_has_neoverse_n1_1542419(midr, READ_SPECIALREG(ctr_el0))) {
*errata_id = 1542419;
return (true);
}
return (false);
}
static void
user_ctr_enable(const struct cpu_feat *feat __unused,
cpu_feat_errata errata_status, u_int errata_id __unused)
{
MPASS(emulate_ctr || errata_status != ERRATA_NONE);
/*
* The Errata Management Firmware Interface may incorrectly mark
* this as firmware mitigated. We should ignore that as there is
* a kernel component to the mitigation.
*/
if (errata_status != ERRATA_NONE && errata_id == 1542419 &&
PCPU_GET(cpuid) == 0) {
/* Clear fields we will change */
user_cpu_desc.ctr &= ~(CTR_DIC_MASK | CTR_ILINE_WIDTH);
/*
* Set DIC to none so userspace will execute an 'ic ivau'
* instruction that can be trapped by EL3.
*/
user_cpu_desc.ctr |= CTR_DIC_NONE;
/*
* Set the i-cache line size to be page size to reduce the
* number of times userspace needs to execute the 'ic ivau'
* instruction. The ctr_el0.IminLine is log2 the number of
* 4-byte words the instruction covers. As PAGE_SHIFT is log2
* of the number of bytes in a page we need to subtract 2.
*/
user_cpu_desc.ctr |= (PAGE_SHIFT - 2) << CTR_ILINE_SHIFT;
l_user_cpu_desc.ctr = user_cpu_desc.ctr;
}
WRITE_SPECIALREG(sctlr_el1,
READ_SPECIALREG(sctlr_el1) & ~SCTLR_UCT);
isb();
}
static struct cpu_feat user_ctr = {
.feat_name = "Trap CTR_EL0",
.feat_check = user_ctr_check,
.feat_has_errata = user_ctr_has_errata,
.feat_enable = user_ctr_enable,
.feat_flags = CPU_FEAT_AFTER_DEV | CPU_FEAT_PER_CPU,
};
DATA_SET(cpu_feat_set, user_ctr);
static int static int
user_ctr_handler(vm_offset_t va, uint32_t insn, struct trapframe *frame,
uint32_t esr)
{
uint64_t value;
int reg;
if ((insn & MRS_MASK) != MRS_VALUE)
return (0);
/* Check if this is the ctr_el0 register */
/* TODO: Add macros to armreg.h */
if (mrs_Op0(insn) != 3 || mrs_Op1(insn) != 3 || mrs_CRn(insn) != 0 ||
mrs_CRm(insn) != 0 || mrs_Op2(insn) != 1)
return (0);
if (SV_CURPROC_ABI() == SV_ABI_FREEBSD)
value = user_cpu_desc.ctr;
else
value = l_user_cpu_desc.ctr;
/*
* We will handle this instruction, move to the next so we
* don't trap here again.
*/
frame->tf_elr += INSN_SIZE;
reg = MRS_REGISTER(insn);
/* If reg is 31 then write to xzr, i.e. do nothing */
if (reg == 31)
return (1);
if (reg < nitems(frame->tf_x))
frame->tf_x[reg] = value;
else if (reg == 30)
frame->tf_lr = value;
return (1);
}
static int
user_mrs_handler(vm_offset_t va, uint32_t insn, struct trapframe *frame, user_mrs_handler(vm_offset_t va, uint32_t insn, struct trapframe *frame,
uint32_t esr) uint32_t esr)
{ {
uint64_t value; uint64_t value;
int CRm, Op2, i, reg; int CRm, Op2, i, reg;
if ((insn & MRS_MASK) != MRS_VALUE) if ((insn & MRS_MASK) != MRS_VALUE)
return (0); return (0);
▲ Show 20 Lines • Show All 393 Lines • ▼ Show 20 Lines if ((elf_hwcap & HWCAP_ATOMICS) != 0) {
if (bootverbose) if (bootverbose)
printf("Enabling LSE atomics in the kernel\n"); printf("Enabling LSE atomics in the kernel\n");
} }
#ifdef LSE_ATOMICS #ifdef LSE_ATOMICS
if (!lse_supported) if (!lse_supported)
panic("CPU does not support LSE atomic instructions"); panic("CPU does not support LSE atomic instructions");
#endif #endif
install_undef_handler(true, user_ctr_handler);
install_undef_handler(true, user_mrs_handler); install_undef_handler(true, user_mrs_handler);
} }
SYSINIT(identify_cpu, SI_SUB_CPU, SI_ORDER_MIDDLE, identify_cpu_sysinit, NULL); SYSINIT(identify_cpu, SI_SUB_CPU, SI_ORDER_MIDDLE, identify_cpu_sysinit, NULL);
static void static void
cpu_features_sysinit(void *dummy __unused) cpu_features_sysinit(void *dummy __unused)
{ {
struct sbuf sb; struct sbuf sb;
▲ Show 20 Lines • Show All 516 Lines • ▼ Show 20 Lines case 2:
/* FALLTHROUGH */ /* FALLTHROUGH */
case 3: case 3:
if (CPU_AFF3(desc->mpidr) != CPU_AFF3(prev_desc->mpidr)) if (CPU_AFF3(desc->mpidr) != CPU_AFF3(prev_desc->mpidr))
cpu_aff_levels = 4; cpu_aff_levels = 4;
break; break;
} }
if (desc->ctr != prev_desc->ctr) { if (desc->ctr != prev_desc->ctr) {
/*
* If the cache is different on different cores we should
* emulate for userspace to provide a uniform value
*/
emulate_ctr = true;
/* /*
* If the cache type register is different we may * If the cache type register is different we may
* have a different l1 cache type. * have a different l1 cache type.
*/ */
identify_cache(desc->ctr); identify_cache(desc->ctr);
} }
} }