diff --git a/sys/arm64/arm64/swtch.S b/sys/arm64/arm64/swtch.S index 081bdfe88126..d7f915a91946 100644 --- a/sys/arm64/arm64/swtch.S +++ b/sys/arm64/arm64/swtch.S @@ -1,296 +1,272 @@ /*- * Copyright (c) 2014 Andrew Turner * Copyright (c) 2014 The FreeBSD Foundation * All rights reserved. * * This software was developed by Andrew Turner under sponsorship from * the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * */ #include "assym.inc" #include "opt_kstack_pages.h" #include "opt_sched.h" #include #include __FBSDID("$FreeBSD$"); .macro clear_step_flag pcbflags, tmp tbz \pcbflags, #PCB_SINGLE_STEP_SHIFT, 999f mrs \tmp, mdscr_el1 bic \tmp, \tmp, #MDSCR_SS msr mdscr_el1, \tmp isb 999: .endm .macro set_step_flag pcbflags, tmp tbz \pcbflags, #PCB_SINGLE_STEP_SHIFT, 999f mrs \tmp, mdscr_el1 orr \tmp, \tmp, #MDSCR_SS msr mdscr_el1, \tmp isb 999: .endm /* * void cpu_throw(struct thread *old, struct thread *new) */ ENTRY(cpu_throw) /* Of old == NULL skip disabling stepping */ cbz x0, 1f /* If we were single stepping, disable it */ ldr x4, [x0, #TD_PCB] ldr w5, [x4, #PCB_FLAGS] clear_step_flag w5, x6 1: #ifdef VFP /* Backup the new thread pointer around a call to C code */ mov x19, x1 bl vfp_discard mov x0, x19 #else mov x0, x1 #endif /* This returns the thread pointer so no need to save it */ bl ptrauth_switch /* This returns the thread pcb */ bl pmap_switch mov x4, x0 /* If we are single stepping, enable it */ ldr w5, [x4, #PCB_FLAGS] set_step_flag w5, x6 /* Restore the registers */ ldp x5, x6, [x4, #PCB_SP] mov sp, x5 msr tpidr_el0, x6 ldr x6, [x4, #PCB_TPIDRRO] msr tpidrro_el0, x6 - ldp x8, x9, [x4, #PCB_REGS + 8 * 8] - ldp x10, x11, [x4, #PCB_REGS + 10 * 8] - ldp x12, x13, [x4, #PCB_REGS + 12 * 8] - ldp x14, x15, [x4, #PCB_REGS + 14 * 8] - ldp x16, x17, [x4, #PCB_REGS + 16 * 8] - ldr x19, [x4, #PCB_REGS + 19 * 8] - ldp x20, x21, [x4, #PCB_REGS + 20 * 8] - ldp x22, x23, [x4, #PCB_REGS + 22 * 8] - ldp x24, x25, [x4, #PCB_REGS + 24 * 8] - ldp x26, x27, [x4, #PCB_REGS + 26 * 8] - ldp x28, x29, [x4, #PCB_REGS + 28 * 8] - ldr lr, [x4, #PCB_LR] + ldp x19, x20, [x4, #PCB_REGS + 19 * 8] + ldp x21, x22, [x4, #PCB_REGS + 21 * 8] + ldp x23, x24, [x4, #PCB_REGS + 23 * 8] + ldp x25, x26, [x4, #PCB_REGS + 25 * 8] + ldp x27, x28, [x4, #PCB_REGS + 27 * 8] + ldp x29, lr, [x4, #PCB_REGS + 29 * 8] ret END(cpu_throw) /* * void cpu_switch(struct thread *old, struct thread *new, struct mtx *mtx) * * x0 = old * x1 = new * x2 = mtx * x3 to x7, x16 and x17 are caller saved */ ENTRY(cpu_switch) /* * Save the old context. */ ldr x4, [x0, #TD_PCB] /* Store the callee-saved registers */ - stp x8, x9, [x4, #PCB_REGS + 8 * 8] - stp x10, x11, [x4, #PCB_REGS + 10 * 8] - stp x12, x13, [x4, #PCB_REGS + 12 * 8] - stp x14, x15, [x4, #PCB_REGS + 14 * 8] - stp x16, x17, [x4, #PCB_REGS + 16 * 8] - stp x18, x19, [x4, #PCB_REGS + 18 * 8] - stp x20, x21, [x4, #PCB_REGS + 20 * 8] - stp x22, x23, [x4, #PCB_REGS + 22 * 8] - stp x24, x25, [x4, #PCB_REGS + 24 * 8] - stp x26, x27, [x4, #PCB_REGS + 26 * 8] - stp x28, x29, [x4, #PCB_REGS + 28 * 8] - str lr, [x4, #PCB_LR] + stp x19, x20, [x4, #PCB_REGS + 19 * 8] + stp x21, x22, [x4, #PCB_REGS + 21 * 8] + stp x23, x24, [x4, #PCB_REGS + 23 * 8] + stp x25, x26, [x4, #PCB_REGS + 25 * 8] + stp x27, x28, [x4, #PCB_REGS + 27 * 8] + stp x29, lr, [x4, #PCB_REGS + 29 * 8] /* And the old stack pointer */ mov x5, sp mrs x6, tpidrro_el0 str x6, [x4, #PCB_TPIDRRO] mrs x6, tpidr_el0 stp x5, x6, [x4, #PCB_SP] /* If we were single stepping, disable it */ ldr w5, [x4, #PCB_FLAGS] clear_step_flag w5, x6 mov x19, x0 mov x20, x1 mov x21, x2 #ifdef VFP /* Load the pcb address */ mov x1, x4 bl vfp_save_state mov x0, x20 #else mov x0, x1 #endif /* This returns the thread pointer so no need to save it */ bl ptrauth_switch /* This returns the thread pcb */ bl pmap_switch /* Move the new pcb out of the way */ mov x4, x0 mov x2, x21 mov x1, x20 mov x0, x19 /* * Release the old thread. */ stlr x2, [x0, #TD_LOCK] #if defined(SCHED_ULE) && defined(SMP) /* Spin if TD_LOCK points to a blocked_lock */ ldr x2, =_C_LABEL(blocked_lock) 1: ldar x3, [x1, #TD_LOCK] cmp x3, x2 b.eq 1b #endif /* If we are single stepping, enable it */ ldr w5, [x4, #PCB_FLAGS] set_step_flag w5, x6 /* Restore the registers */ ldp x5, x6, [x4, #PCB_SP] mov sp, x5 msr tpidr_el0, x6 ldr x6, [x4, #PCB_TPIDRRO] msr tpidrro_el0, x6 - ldp x8, x9, [x4, #PCB_REGS + 8 * 8] - ldp x10, x11, [x4, #PCB_REGS + 10 * 8] - ldp x12, x13, [x4, #PCB_REGS + 12 * 8] - ldp x14, x15, [x4, #PCB_REGS + 14 * 8] - ldp x16, x17, [x4, #PCB_REGS + 16 * 8] - ldr x19, [x4, #PCB_REGS + 19 * 8] - ldp x20, x21, [x4, #PCB_REGS + 20 * 8] - ldp x22, x23, [x4, #PCB_REGS + 22 * 8] - ldp x24, x25, [x4, #PCB_REGS + 24 * 8] - ldp x26, x27, [x4, #PCB_REGS + 26 * 8] - ldp x28, x29, [x4, #PCB_REGS + 28 * 8] - ldr lr, [x4, #PCB_LR] + ldp x19, x20, [x4, #PCB_REGS + 19 * 8] + ldp x21, x22, [x4, #PCB_REGS + 21 * 8] + ldp x23, x24, [x4, #PCB_REGS + 23 * 8] + ldp x25, x26, [x4, #PCB_REGS + 25 * 8] + ldp x27, x28, [x4, #PCB_REGS + 27 * 8] + ldp x29, lr, [x4, #PCB_REGS + 29 * 8] str xzr, [x4, #PCB_REGS + 18 * 8] ret END(cpu_switch) ENTRY(fork_trampoline) - mov x0, x8 - mov x1, x9 + mov x0, x19 + mov x1, x20 mov x2, sp mov fp, #0 /* Stack traceback stops here. */ bl _C_LABEL(fork_exit) /* * Disable interrupts as we are setting userspace specific * state that we won't handle correctly in an interrupt while * in the kernel. */ msr daifset, #(DAIF_D | DAIF_INTR) ldr x0, [x18, #PC_CURTHREAD] bl ptrauth_enter_el0 /* Restore sp, lr, elr, and spsr */ ldp x18, lr, [sp, #TF_SP] ldp x10, x11, [sp, #TF_ELR] msr sp_el0, x18 msr spsr_el1, x11 msr elr_el1, x10 /* Restore the CPU registers */ ldp x0, x1, [sp, #TF_X + 0 * 8] ldp x2, x3, [sp, #TF_X + 2 * 8] ldp x4, x5, [sp, #TF_X + 4 * 8] ldp x6, x7, [sp, #TF_X + 6 * 8] ldp x8, x9, [sp, #TF_X + 8 * 8] ldp x10, x11, [sp, #TF_X + 10 * 8] ldp x12, x13, [sp, #TF_X + 12 * 8] ldp x14, x15, [sp, #TF_X + 14 * 8] ldp x16, x17, [sp, #TF_X + 16 * 8] ldp x18, x19, [sp, #TF_X + 18 * 8] ldp x20, x21, [sp, #TF_X + 20 * 8] ldp x22, x23, [sp, #TF_X + 22 * 8] ldp x24, x25, [sp, #TF_X + 24 * 8] ldp x26, x27, [sp, #TF_X + 26 * 8] ldp x28, x29, [sp, #TF_X + 28 * 8] /* * No need for interrupts reenabling since PSR * will be set to the desired value anyway. */ ERET END(fork_trampoline) ENTRY(savectx) /* Store the callee-saved registers */ - stp x8, x9, [x0, #PCB_REGS + 8 * 8] - stp x10, x11, [x0, #PCB_REGS + 10 * 8] - stp x12, x13, [x0, #PCB_REGS + 12 * 8] - stp x14, x15, [x0, #PCB_REGS + 14 * 8] - stp x16, x17, [x0, #PCB_REGS + 16 * 8] - stp x18, x19, [x0, #PCB_REGS + 18 * 8] - stp x20, x21, [x0, #PCB_REGS + 20 * 8] - stp x22, x23, [x0, #PCB_REGS + 22 * 8] - stp x24, x25, [x0, #PCB_REGS + 24 * 8] - stp x26, x27, [x0, #PCB_REGS + 26 * 8] - stp x28, x29, [x0, #PCB_REGS + 28 * 8] - str lr, [x0, #PCB_LR] + stp x19, x20, [x0, #PCB_REGS + 19 * 8] + stp x21, x22, [x0, #PCB_REGS + 21 * 8] + stp x23, x24, [x0, #PCB_REGS + 23 * 8] + stp x25, x26, [x0, #PCB_REGS + 25 * 8] + stp x27, x28, [x0, #PCB_REGS + 27 * 8] + stp x29, lr, [x0, #PCB_REGS + 29 * 8] /* And the old stack pointer */ mov x5, sp mrs x6, tpidrro_el0 str x6, [x0, #PCB_TPIDRRO] mrs x6, tpidr_el0 stp x5, x6, [x0, #PCB_SP] /* Store the VFP registers */ #ifdef VFP mov x28, lr mov x1, x0 /* move pcb to the correct register */ mov x0, xzr /* td = NULL */ bl vfp_save_state mov lr, x28 #endif ret END(savectx) diff --git a/sys/arm64/arm64/vm_machdep.c b/sys/arm64/arm64/vm_machdep.c index c52a7e2fc5c1..feb439314f50 100644 --- a/sys/arm64/arm64/vm_machdep.c +++ b/sys/arm64/arm64/vm_machdep.c @@ -1,312 +1,312 @@ /*- * Copyright (c) 2014 Andrew Turner * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * */ #include "opt_platform.h" #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #ifdef VFP #include #endif #include /* * Finish a fork operation, with process p2 nearly set up. * Copy and update the pcb, set up the stack so that the child * ready to run and return to user mode. */ void cpu_fork(struct thread *td1, struct proc *p2, struct thread *td2, int flags) { struct pcb *pcb2; struct trapframe *tf; if ((flags & RFPROC) == 0) return; if (td1 == curthread) { /* * Save the tpidr_el0 and the vfp state, these normally happen * in cpu_switch, but if userland changes these then forks * this may not have happened. */ td1->td_pcb->pcb_tpidr_el0 = READ_SPECIALREG(tpidr_el0); td1->td_pcb->pcb_tpidrro_el0 = READ_SPECIALREG(tpidrro_el0); #ifdef VFP if ((td1->td_pcb->pcb_fpflags & PCB_FP_STARTED) != 0) vfp_save_state(td1, td1->td_pcb); #endif } pcb2 = (struct pcb *)(td2->td_kstack + td2->td_kstack_pages * PAGE_SIZE) - 1; td2->td_pcb = pcb2; bcopy(td1->td_pcb, pcb2, sizeof(*pcb2)); /* Clear the debug register state. */ bzero(&pcb2->pcb_dbg_regs, sizeof(pcb2->pcb_dbg_regs)); ptrauth_fork(td2, td1); tf = (struct trapframe *)STACKALIGN((struct trapframe *)pcb2 - 1); bcopy(td1->td_frame, tf, sizeof(*tf)); tf->tf_x[0] = 0; tf->tf_x[1] = 0; tf->tf_spsr = td1->td_frame->tf_spsr & (PSR_M_32 | PSR_DAIF); td2->td_frame = tf; /* Set the return value registers for fork() */ - td2->td_pcb->pcb_x[8] = (uintptr_t)fork_return; - td2->td_pcb->pcb_x[9] = (uintptr_t)td2; + td2->td_pcb->pcb_x[19] = (uintptr_t)fork_return; + td2->td_pcb->pcb_x[20] = (uintptr_t)td2; td2->td_pcb->pcb_lr = (uintptr_t)fork_trampoline; td2->td_pcb->pcb_sp = (uintptr_t)td2->td_frame; td2->td_pcb->pcb_fpusaved = &td2->td_pcb->pcb_fpustate; td2->td_pcb->pcb_vfpcpu = UINT_MAX; /* Setup to release spin count in fork_exit(). */ td2->td_md.md_spinlock_count = 1; td2->td_md.md_saved_daif = PSR_DAIF_DEFAULT; #if defined(PERTHREAD_SSP) /* Set the new canary */ arc4random_buf(&td2->td_md.md_canary, sizeof(td2->td_md.md_canary)); #endif } void cpu_reset(void) { psci_reset(); printf("cpu_reset failed"); while(1) __asm volatile("wfi" ::: "memory"); } void cpu_thread_swapin(struct thread *td) { } void cpu_thread_swapout(struct thread *td) { } void cpu_set_syscall_retval(struct thread *td, int error) { struct trapframe *frame; frame = td->td_frame; if (__predict_true(error == 0)) { frame->tf_x[0] = td->td_retval[0]; frame->tf_x[1] = td->td_retval[1]; frame->tf_spsr &= ~PSR_C; /* carry bit */ return; } switch (error) { case ERESTART: frame->tf_elr -= 4; break; case EJUSTRETURN: break; default: frame->tf_spsr |= PSR_C; /* carry bit */ frame->tf_x[0] = error; break; } } /* * Initialize machine state, mostly pcb and trap frame for a new * thread, about to return to userspace. Put enough state in the new * thread's PCB to get it to go back to the fork_return(), which * finalizes the thread state and handles peculiarities of the first * return to userspace for the new thread. */ void cpu_copy_thread(struct thread *td, struct thread *td0) { bcopy(td0->td_frame, td->td_frame, sizeof(struct trapframe)); bcopy(td0->td_pcb, td->td_pcb, sizeof(struct pcb)); - td->td_pcb->pcb_x[8] = (uintptr_t)fork_return; - td->td_pcb->pcb_x[9] = (uintptr_t)td; + td->td_pcb->pcb_x[19] = (uintptr_t)fork_return; + td->td_pcb->pcb_x[20] = (uintptr_t)td; td->td_pcb->pcb_lr = (uintptr_t)fork_trampoline; td->td_pcb->pcb_sp = (uintptr_t)td->td_frame; td->td_pcb->pcb_fpflags &= ~(PCB_FP_STARTED | PCB_FP_KERN | PCB_FP_NOSAVE); td->td_pcb->pcb_fpusaved = &td->td_pcb->pcb_fpustate; td->td_pcb->pcb_vfpcpu = UINT_MAX; /* Setup to release spin count in fork_exit(). */ td->td_md.md_spinlock_count = 1; td->td_md.md_saved_daif = PSR_DAIF_DEFAULT; #if defined(PERTHREAD_SSP) /* Set the new canary */ arc4random_buf(&td->td_md.md_canary, sizeof(td->td_md.md_canary)); #endif /* Generate new pointer authentication keys. */ ptrauth_copy_thread(td, td0); } /* * Set that machine state for performing an upcall that starts * the entry function with the given argument. */ void cpu_set_upcall(struct thread *td, void (*entry)(void *), void *arg, stack_t *stack) { struct trapframe *tf = td->td_frame; /* 32bits processes use r13 for sp */ if (td->td_frame->tf_spsr & PSR_M_32) { tf->tf_x[13] = STACKALIGN((uintptr_t)stack->ss_sp + stack->ss_size); if ((register_t)entry & 1) tf->tf_spsr |= PSR_T; } else tf->tf_sp = STACKALIGN((uintptr_t)stack->ss_sp + stack->ss_size); tf->tf_elr = (register_t)entry; tf->tf_x[0] = (register_t)arg; } int cpu_set_user_tls(struct thread *td, void *tls_base) { struct pcb *pcb; if ((uintptr_t)tls_base >= VM_MAXUSER_ADDRESS) return (EINVAL); pcb = td->td_pcb; if (td->td_frame->tf_spsr & PSR_M_32) { /* 32bits arm stores the user TLS into tpidrro */ pcb->pcb_tpidrro_el0 = (register_t)tls_base; pcb->pcb_tpidr_el0 = (register_t)tls_base; if (td == curthread) { WRITE_SPECIALREG(tpidrro_el0, tls_base); WRITE_SPECIALREG(tpidr_el0, tls_base); } } else { pcb->pcb_tpidr_el0 = (register_t)tls_base; if (td == curthread) WRITE_SPECIALREG(tpidr_el0, tls_base); } return (0); } void cpu_thread_exit(struct thread *td) { } void cpu_thread_alloc(struct thread *td) { td->td_pcb = (struct pcb *)(td->td_kstack + td->td_kstack_pages * PAGE_SIZE) - 1; td->td_frame = (struct trapframe *)STACKALIGN( (struct trapframe *)td->td_pcb - 1); ptrauth_thread_alloc(td); } void cpu_thread_free(struct thread *td) { } void cpu_thread_clean(struct thread *td) { } /* * Intercept the return address from a freshly forked process that has NOT * been scheduled yet. * * This is needed to make kernel threads stay in kernel mode. */ void cpu_fork_kthread_handler(struct thread *td, void (*func)(void *), void *arg) { - td->td_pcb->pcb_x[8] = (uintptr_t)func; - td->td_pcb->pcb_x[9] = (uintptr_t)arg; + td->td_pcb->pcb_x[19] = (uintptr_t)func; + td->td_pcb->pcb_x[20] = (uintptr_t)arg; } void cpu_exit(struct thread *td) { } bool cpu_exec_vmspace_reuse(struct proc *p __unused, vm_map_t map __unused) { return (true); } int cpu_procctl(struct thread *td __unused, int idtype __unused, id_t id __unused, int com __unused, void *data __unused) { return (EINVAL); }