diff --git a/sys/arm64/arm64/efirt_machdep.c b/sys/arm64/arm64/efirt_machdep.c --- a/sys/arm64/arm64/efirt_machdep.c +++ b/sys/arm64/arm64/efirt_machdep.c @@ -241,6 +241,7 @@ int efi_arch_enter(void) { + uint64_t tcr; CRITICAL_ASSERT(curthread); curthread->td_md.md_efirt_dis_pf = vm_fault_disable_pagefaults(); @@ -249,7 +250,17 @@ * Temporarily switch to EFI's page table. However, we leave curpmap * unchanged in order to prevent its ASID from being reclaimed before * we switch back to its page table in efi_arch_leave(). + * + * UEFI sdoesn't care about TBI, so enable it. It's more likely + * userspace will have TBI on as it's only disabled for backwards + * compatibility. */ + tcr = READ_SPECIALREG(tcr_el1); + if ((tcr & MD_TCR_FIELDS) != TCR_TBI0) { + tcr &= ~MD_TCR_FIELDS; + tcr |= TCR_TBI0; + WRITE_SPECIALREG(tcr_el1, tcr); + } set_ttbr0(efi_ttbr0); if (PCPU_GET(bcast_tlbi_workaround) != 0) invalidate_local_icache(); @@ -260,6 +271,7 @@ void efi_arch_leave(void) { + uint64_t proc_tcr, tcr; /* * Restore the pcpu pointer. Some UEFI implementations trash it and @@ -271,6 +283,13 @@ __asm __volatile( "mrs x18, tpidr_el1 \n" ); + proc_tcr = curthread->td_proc->p_md.md_tcr; + tcr = READ_SPECIALREG(tcr_el1); + if ((tcr & MD_TCR_FIELDS) != proc_tcr) { + tcr &= ~MD_TCR_FIELDS; + tcr |= proc_tcr; + WRITE_SPECIALREG(tcr_el1, tcr); + } set_ttbr0(pmap_to_ttbr0(PCPU_GET(curpmap))); if (PCPU_GET(bcast_tlbi_workaround) != 0) invalidate_local_icache(); diff --git a/sys/arm64/arm64/elf_machdep.c b/sys/arm64/arm64/elf_machdep.c --- a/sys/arm64/arm64/elf_machdep.c +++ b/sys/arm64/arm64/elf_machdep.c @@ -65,7 +65,13 @@ u_long __read_frequently linux_elf_hwcap3; u_long __read_frequently linux_elf_hwcap4; -struct arm64_addr_mask elf64_addr_mask; +struct arm64_addr_mask elf64_addr_mask = { + .code = TBI_ADDR_MASK, + .data = TBI_ADDR_MASK, +}; +#ifdef COMPAT_FREEBSD14 +struct arm64_addr_mask elf64_addr_mask_14; +#endif static void arm64_exec_protect(struct image_params *, int); @@ -136,7 +142,14 @@ if (buf != NULL) { KASSERT(*sizep == sizeof(elf64_addr_mask), ("%s: invalid size", __func__)); - memcpy(buf, &elf64_addr_mask, sizeof(elf64_addr_mask)); +#ifdef COMPAT_FREEBSD14 + /* running an old binary use the old address mask */ + if (td->td_proc->p_osrel < TBI_VERSION) + memcpy(buf, &elf64_addr_mask_14, + sizeof(elf64_addr_mask_14)); + else +#endif + memcpy(buf, &elf64_addr_mask, sizeof(elf64_addr_mask)); } *sizep = sizeof(elf64_addr_mask); diff --git a/sys/arm64/arm64/exec_machdep.c b/sys/arm64/arm64/exec_machdep.c --- a/sys/arm64/arm64/exec_machdep.c +++ b/sys/arm64/arm64/exec_machdep.c @@ -51,6 +51,7 @@ #include #include +#include #include #include #include @@ -411,6 +412,7 @@ { struct trapframe *tf = td->td_frame; struct pcb *pcb = td->td_pcb; + uint64_t new_tcr, tcr; memset(tf, 0, sizeof(struct trapframe)); @@ -433,6 +435,35 @@ */ bzero(&pcb->pcb_dbg_regs, sizeof(pcb->pcb_dbg_regs)); + /* If the process is new enough enable TBI */ + if (td->td_proc->p_osrel >= TBI_VERSION) + new_tcr = TCR_TBI0; + else + new_tcr = 0; + td->td_proc->p_md.md_tcr = new_tcr; + + /* TODO: should create a pmap function for this... */ + tcr = READ_SPECIALREG(tcr_el1); + if ((tcr & MD_TCR_FIELDS) != new_tcr) { + uint64_t asid; + + tcr &= ~MD_TCR_FIELDS; + tcr |= new_tcr; + WRITE_SPECIALREG(tcr_el1, tcr); + isb(); + + /* + * TCR_EL1.TBI0 is permitted to be cached in the TLB, so + * we need to perform a TLB invalidation. + */ + asid = READ_SPECIALREG(ttbr0_el1) & TTBR_ASID_MASK; + __asm __volatile( + "tlbi aside1is, %0 \n" + "dsb ish \n" + "isb \n" + : : "r" (asid)); + } + /* Generate new pointer authentication keys */ ptrauth_exec(td); } diff --git a/sys/arm64/arm64/genassym.c b/sys/arm64/arm64/genassym.c --- a/sys/arm64/arm64/genassym.c +++ b/sys/arm64/arm64/genassym.c @@ -64,6 +64,7 @@ ASSYM(PCB_FLAGS, offsetof(struct pcb, pcb_flags)); ASSYM(P_PID, offsetof(struct proc, p_pid)); +ASSYM(P_MD_TCR, offsetof(struct proc, p_md.md_tcr)); ASSYM(SF_UC, offsetof(struct sigframe, sf_uc)); diff --git a/sys/arm64/arm64/pmap.c b/sys/arm64/arm64/pmap.c --- a/sys/arm64/arm64/pmap.c +++ b/sys/arm64/arm64/pmap.c @@ -469,7 +469,7 @@ vm_offset_t va); static void pmap_abort_ptp(pmap_t pmap, vm_offset_t va, vm_page_t mpte); -static bool pmap_activate_int(pmap_t pmap); +static bool pmap_activate_int(struct thread *td, pmap_t pmap); static void pmap_alloc_asid(pmap_t pmap); static int pmap_change_props_locked(vm_offset_t va, vm_size_t size, vm_prot_t prot, int mode, bool skip_unmapped); @@ -9113,7 +9113,7 @@ SYSINIT(pmap_init_cnp, SI_SUB_SMP, SI_ORDER_ANY, pmap_init_cnp, NULL); static bool -pmap_activate_int(pmap_t pmap) +pmap_activate_int(struct thread *td, pmap_t pmap) { struct asid_set *set; int epoch; @@ -9152,6 +9152,15 @@ pmap_alloc_asid(pmap); if (pmap->pm_stage == PM_STAGE1) { + uint64_t new_tcr, tcr; + + new_tcr = td->td_proc->p_md.md_tcr; + tcr = READ_SPECIALREG(tcr_el1); + if ((tcr & MD_TCR_FIELDS) != new_tcr) { + tcr &= ~MD_TCR_FIELDS; + tcr |= new_tcr; + WRITE_SPECIALREG(tcr_el1, tcr); + } set_ttbr0(pmap_to_ttbr0(pmap)); if (PCPU_GET(bcast_tlbi_workaround) != 0) invalidate_local_icache(); @@ -9165,7 +9174,7 @@ PMAP_ASSERT_STAGE2(pmap); - (void)pmap_activate_int(pmap); + (void)pmap_activate_int(NULL, pmap); } void @@ -9176,7 +9185,7 @@ pmap = vmspace_pmap(td->td_proc->p_vmspace); PMAP_ASSERT_STAGE1(pmap); critical_enter(); - (void)pmap_activate_int(pmap); + (void)pmap_activate_int(td, pmap); critical_exit(); } @@ -9202,7 +9211,7 @@ * to a user process. */ - if (pmap_activate_int(vmspace_pmap(new->td_proc->p_vmspace))) { + if (pmap_activate_int(new, vmspace_pmap(new->td_proc->p_vmspace))) { /* * Stop userspace from training the branch predictor against * other processes. This will call into a CPU specific diff --git a/sys/arm64/arm64/ptrauth.c b/sys/arm64/arm64/ptrauth.c --- a/sys/arm64/arm64/ptrauth.c +++ b/sys/arm64/arm64/ptrauth.c @@ -149,6 +149,10 @@ enable_ptrauth = true; elf64_addr_mask.code |= PAC_ADDR_MASK; elf64_addr_mask.data |= PAC_ADDR_MASK; +#ifdef COMPAT_FREEBSD14 + elf64_addr_mask_14.code |= PAC_ADDR_MASK_14; + elf64_addr_mask_14.data |= PAC_ADDR_MASK_14; +#endif } diff --git a/sys/arm64/arm64/swtch.S b/sys/arm64/arm64/swtch.S --- a/sys/arm64/arm64/swtch.S +++ b/sys/arm64/arm64/swtch.S @@ -37,6 +37,8 @@ #include #include +#include + .macro clear_step_flag pcbflags, tmp tbz \pcbflags, #PCB_SINGLE_STEP_SHIFT, 999f mrs \tmp, mdscr_el1 @@ -239,6 +241,16 @@ msr daifset, #(DAIF_D | DAIF_INTR) ldr x0, [x18, #PC_CURTHREAD] + + /* Set the per-process tcr_el1 fields */ + ldr x10, [x0, #TD_PROC] + ldr x10, [x10, #P_MD_TCR] + mrs x11, tcr_el1 + and x11, x11, #(~MD_TCR_FIELDS) + orr x11, x11, x10 + msr tcr_el1, x11 + /* No isb as the eret below is the context-synchronising event */ + bl ptrauth_enter_el0 /* Restore sp, lr, elr, and spsr */ diff --git a/sys/arm64/arm64/vm_machdep.c b/sys/arm64/arm64/vm_machdep.c --- a/sys/arm64/arm64/vm_machdep.c +++ b/sys/arm64/arm64/vm_machdep.c @@ -120,6 +120,9 @@ td2->td_md.md_spinlock_count = 1; td2->td_md.md_saved_daif = PSR_DAIF_DEFAULT; + /* Copy the TCR_EL1 value */ + td2->td_proc->p_md.md_tcr = td1->td_proc->p_md.md_tcr; + #if defined(PERTHREAD_SSP) /* Set the new canary */ arc4random_buf(&td2->td_md.md_canary, sizeof(td2->td_md.md_canary)); diff --git a/sys/arm64/include/cpu.h b/sys/arm64/include/cpu.h --- a/sys/arm64/include/cpu.h +++ b/sys/arm64/include/cpu.h @@ -226,6 +226,9 @@ struct arm64_addr_mask; extern struct arm64_addr_mask elf64_addr_mask; +#ifdef COMPAT_FREEBSD14 +extern struct arm64_addr_mask elf64_addr_mask_14; +#endif typedef void (*cpu_reset_hook_t)(void); extern cpu_reset_hook_t cpu_reset_hook; diff --git a/sys/arm64/include/elf.h b/sys/arm64/include/elf.h --- a/sys/arm64/include/elf.h +++ b/sys/arm64/include/elf.h @@ -93,6 +93,9 @@ #define ET_DYN_LOAD_ADDR 0x100000 #endif +/* First __FreeBSD_version that supports Top Byte Ignore (TBI) */ +#define TBI_VERSION 1500058 + /* HWCAP */ #define HWCAP_FP (1 << 0) #define HWCAP_ASIMD (1 << 1) diff --git a/sys/arm64/include/proc.h b/sys/arm64/include/proc.h --- a/sys/arm64/include/proc.h +++ b/sys/arm64/include/proc.h @@ -35,6 +35,7 @@ #ifndef _MACHINE_PROC_H_ #define _MACHINE_PROC_H_ +#ifndef LOCORE struct ptrauth_key { uint64_t pa_key_lo; uint64_t pa_key_hi; @@ -73,8 +74,12 @@ }; struct mdproc { - long md_dummy; + uint64_t md_tcr; /* TCR_EL1 fields to update */ }; +#endif /* !LOCORE */ + +/* Fields that can be set in md_tcr */ +#define MD_TCR_FIELDS TCR_TBI0 #define KINFO_PROC_SIZE 1088 #define KINFO_PROC32_SIZE 816 diff --git a/sys/arm64/include/vmparam.h b/sys/arm64/include/vmparam.h --- a/sys/arm64/include/vmparam.h +++ b/sys/arm64/include/vmparam.h @@ -209,7 +209,8 @@ #define KMSAN_ORIG_MAX_ADDRESS (0xffff028000000000UL) /* The address bits that hold a pointer authentication code */ -#define PAC_ADDR_MASK (0xff7f000000000000UL) +#define PAC_ADDR_MASK (0x007f000000000000UL) +#define PAC_ADDR_MASK_14 (0xff7f000000000000UL) /* The top-byte ignore address bits */ #define TBI_ADDR_MASK 0xff00000000000000UL diff --git a/sys/sys/param.h b/sys/sys/param.h --- a/sys/sys/param.h +++ b/sys/sys/param.h @@ -74,7 +74,7 @@ * cannot include sys/param.h and should only be updated here. */ #undef __FreeBSD_version -#define __FreeBSD_version 1500057 +#define __FreeBSD_version 1500058 /* * __FreeBSD_kernel__ indicates that this system uses the kernel of FreeBSD, diff --git a/tests/sys/kern/ptrace_test.c b/tests/sys/kern/ptrace_test.c --- a/tests/sys/kern/ptrace_test.c +++ b/tests/sys/kern/ptrace_test.c @@ -3239,7 +3239,7 @@ ATF_REQUIRE(ptrace(PT_GETREGSET, wpid, (caddr_t)&vec, NT_ARM_ADDR_MASK) != -1); REQUIRE_EQ(addr_mask.code, addr_mask.data); - ATF_REQUIRE(addr_mask.code == 0 || + ATF_REQUIRE(addr_mask.code == 0xff00000000000000ul || addr_mask.code == 0xff7f000000000000UL); #endif