diff --git a/sys/arm64/arm64/exception.S b/sys/arm64/arm64/exception.S --- a/sys/arm64/arm64/exception.S +++ b/sys/arm64/arm64/exception.S @@ -66,6 +66,13 @@ mrs x18, tpidr_el1 add x29, sp, #(TF_SIZE) .if \el == 0 +#if defined(PERTHREAD_SSP) + /* Load the SSP canary to sp_el0 */ + ldr x1, [x18, #(PC_CURTHREAD)] + add x1, x1, #(TD_MD_CANARY) + msr sp_el0, x1 +#endif + /* Apply the SSBD (CVE-2018-3639) workaround if needed */ ldr x1, [x18, #PC_SSBD] cbz x1, 1f diff --git a/sys/arm64/arm64/genassym.c b/sys/arm64/arm64/genassym.c --- a/sys/arm64/arm64/genassym.c +++ b/sys/arm64/arm64/genassym.c @@ -73,6 +73,7 @@ ASSYM(TD_FLAGS, offsetof(struct thread, td_flags)); ASSYM(TD_FRAME, offsetof(struct thread, td_frame)); ASSYM(TD_LOCK, offsetof(struct thread, td_lock)); +ASSYM(TD_MD_CANARY, offsetof(struct thread, td_md.md_canary)); ASSYM(TF_SIZE, sizeof(struct trapframe)); ASSYM(TF_SP, offsetof(struct trapframe, tf_sp)); diff --git a/sys/arm64/arm64/locore.S b/sys/arm64/arm64/locore.S --- a/sys/arm64/arm64/locore.S +++ b/sys/arm64/arm64/locore.S @@ -116,6 +116,13 @@ cmp x15, x14 b.lo 1b +#if defined(PERTHREAD_SSP) + /* Set sp_el0 to the boot canary for early per-thread SSP to work */ + adrp x15, boot_canary + add x15, x15, :lo12:boot_canary + msr sp_el0, x15 +#endif + /* Backup the module pointer */ mov x1, x0 @@ -200,6 +207,13 @@ ldr x4, [x4] mov sp, x4 +#if defined(PERTHREAD_SSP) + /* Set sp_el0 to the boot canary for early per-thread SSP to work */ + adrp x15, boot_canary + add x15, x15, :lo12:boot_canary + msr sp_el0, x15 +#endif + /* Load the kernel ttbr0 pagetable */ msr ttbr0_el1, x27 isb diff --git a/sys/arm64/arm64/machdep.c b/sys/arm64/arm64/machdep.c --- a/sys/arm64/arm64/machdep.c +++ b/sys/arm64/arm64/machdep.c @@ -109,6 +109,14 @@ */ struct pcpu pcpu0; +#if defined(PERTHREAD_SSP) +/* + * The boot SSP canary. Will be replaced with a per-thread canary when + * scheduling has started. + */ +uintptr_t boot_canary = 0x49a2d892bc05a0b1ul; +#endif + static struct trapframe proc0_tf; int early_boot = 1; @@ -136,6 +144,16 @@ int (*apei_nmi)(void); +#if defined(PERTHREAD_SSP_WARNING) +static void +print_ssp_warning(void *data __unused) +{ + printf("WARNING: Per-thread SSP is enabled but the compiler is too old to support it\n"); +} +SYSINIT(ssp_warn, SI_SUB_COPYRIGHT, SI_ORDER_ANY, print_ssp_warning, NULL); +SYSINIT(ssp_warn2, SI_SUB_LAST, SI_ORDER_ANY, print_ssp_warning, NULL); +#endif + static void pan_setup(void) { @@ -347,6 +365,9 @@ proc_linkup0(&proc0, &thread0); thread0.td_kstack = kstack; thread0.td_kstack_pages = KSTACK_PAGES; +#if defined(PERTHREAD_SSP) + thread0.td_md.md_canary = boot_canary; +#endif thread0.td_pcb = (struct pcb *)(thread0.td_kstack + thread0.td_kstack_pages * PAGE_SIZE) - 1; thread0.td_pcb->pcb_fpflags = 0; @@ -781,6 +802,7 @@ "mov x18, %0 \n" "msr tpidr_el1, %0" :: "r"(pcpup)); + /* locore.S sets sp_el0 to &thread0 so no need to set it here. */ PCPU_SET(curthread, &thread0); PCPU_SET(midr, get_midr()); diff --git a/sys/arm64/arm64/pmap.c b/sys/arm64/arm64/pmap.c --- a/sys/arm64/arm64/pmap.c +++ b/sys/arm64/arm64/pmap.c @@ -6619,6 +6619,10 @@ /* Store the new curthread */ PCPU_SET(curthread, new); +#if defined(PERTHREAD_SSP) + /* Set the new threads SSP canary */ + __asm("msr sp_el0, %0" :: "r"(&new->td_md.md_canary)); +#endif /* And the new pcb */ pcb = new->td_pcb; diff --git a/sys/arm64/arm64/vm_machdep.c b/sys/arm64/arm64/vm_machdep.c --- a/sys/arm64/arm64/vm_machdep.c +++ b/sys/arm64/arm64/vm_machdep.c @@ -113,6 +113,11 @@ /* Setup to release spin count in fork_exit(). */ td2->td_md.md_spinlock_count = 1; td2->td_md.md_saved_daif = PSR_DAIF_DEFAULT; + +#if defined(PERTHREAD_SSP) + /* Set the new canary */ + arc4random_buf(&td2->td_md.md_canary, sizeof(td2->td_md.md_canary)); +#endif } void @@ -187,6 +192,11 @@ /* Setup to release spin count in fork_exit(). */ td->td_md.md_spinlock_count = 1; td->td_md.md_saved_daif = PSR_DAIF_DEFAULT; + +#if defined(PERTHREAD_SSP) + /* Set the new canary */ + arc4random_buf(&td->td_md.md_canary, sizeof(td->td_md.md_canary)); +#endif } /* diff --git a/sys/arm64/conf/std.arm64 b/sys/arm64/conf/std.arm64 --- a/sys/arm64/conf/std.arm64 +++ b/sys/arm64/conf/std.arm64 @@ -67,6 +67,7 @@ options RCTL # Resource limits options INTRNG options LINUX_BOOT_ABI # Boot using booti command from U-Boot +options PERTHREAD_SSP # Per-thread SSP canary # Debugging support. Always need this: options KDB # Enable kernel debugger support. diff --git a/sys/arm64/include/proc.h b/sys/arm64/include/proc.h --- a/sys/arm64/include/proc.h +++ b/sys/arm64/include/proc.h @@ -37,6 +37,7 @@ struct mdthread { int md_spinlock_count; /* (k) */ register_t md_saved_daif; /* (k) */ + uintptr_t md_canary; }; struct mdproc { diff --git a/sys/conf/Makefile.arm64 b/sys/conf/Makefile.arm64 --- a/sys/conf/Makefile.arm64 +++ b/sys/conf/Makefile.arm64 @@ -30,6 +30,20 @@ LINUX_DTS_VERSION!= awk '/freebsd,dts-version/ { sub(/;$$/,"", $$NF); print $$NF }' $S/dts/freebsd-compatible.dts CFLAGS += -DLINUX_DTS_VERSION=\"${LINUX_DTS_VERSION}\" +PERTHREAD_SSP_ENABLED!= grep PERTHREAD_SSP opt_global.h || true ; echo +.if !empty(PERTHREAD_SSP_ENABLED) +. if ${COMPILER_TYPE} == "clang" && ${COMPILER_VERSION} >= 130000 +ARM64_SSP_CFLAGS = -mstack-protector-guard=sysreg +ARM64_SSP_CFLAGS += -mstack-protector-guard-reg=sp_el0 +ARM64_SSP_CFLAGS += -mstack-protector-guard-offset=0 +. else +ARM64_SSP_CFLAGS += -DPERTHREAD_SSP_WARNING +. warning "Compiler is too old to support PERTHREAD_SSP" +. endif +CFLAGS += ${ARM64_SSP_CFLAGS} +ARCH_FLAGS += ${ARM64_SSP_CFLAGS} +.endif + # Use a custom SYSTEM_LD command to generate the elf kernel, so we can # set the text segment start address, and also strip the "arm mapping # symbols" which have names like $a.0 and $d.2; see the document diff --git a/sys/conf/options.arm64 b/sys/conf/options.arm64 --- a/sys/conf/options.arm64 +++ b/sys/conf/options.arm64 @@ -8,6 +8,10 @@ LINUX_BOOT_ABI opt_global.h LSE_ATOMICS opt_global.h +# Per-thread stack smashing protection support +# Needs clang >= 13 +PERTHREAD_SSP opt_global.h + # Binary compatibility COMPAT_FREEBSD32 opt_global.h