diff --git a/sys/arm64/arm64/swtch.S b/sys/arm64/arm64/swtch.S
index 081bdfe88126..d7f915a91946 100644
--- a/sys/arm64/arm64/swtch.S
+++ b/sys/arm64/arm64/swtch.S
@@ -1,296 +1,272 @@
/*-
* Copyright (c) 2014 Andrew Turner
* Copyright (c) 2014 The FreeBSD Foundation
* All rights reserved.
*
* This software was developed by Andrew Turner under sponsorship from
* the FreeBSD Foundation.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*
*/
#include "assym.inc"
#include "opt_kstack_pages.h"
#include "opt_sched.h"
#include <machine/asm.h>
#include <machine/armreg.h>
__FBSDID("$FreeBSD$");
.macro clear_step_flag pcbflags, tmp
tbz \pcbflags, #PCB_SINGLE_STEP_SHIFT, 999f
mrs \tmp, mdscr_el1
bic \tmp, \tmp, #MDSCR_SS
msr mdscr_el1, \tmp
isb
999:
.endm
.macro set_step_flag pcbflags, tmp
tbz \pcbflags, #PCB_SINGLE_STEP_SHIFT, 999f
mrs \tmp, mdscr_el1
orr \tmp, \tmp, #MDSCR_SS
msr mdscr_el1, \tmp
isb
999:
.endm
/*
* void cpu_throw(struct thread *old, struct thread *new)
*/
ENTRY(cpu_throw)
/* Of old == NULL skip disabling stepping */
cbz x0, 1f
/* If we were single stepping, disable it */
ldr x4, [x0, #TD_PCB]
ldr w5, [x4, #PCB_FLAGS]
clear_step_flag w5, x6
1:
#ifdef VFP
/* Backup the new thread pointer around a call to C code */
mov x19, x1
bl vfp_discard
mov x0, x19
#else
mov x0, x1
#endif
/* This returns the thread pointer so no need to save it */
bl ptrauth_switch
/* This returns the thread pcb */
bl pmap_switch
mov x4, x0
/* If we are single stepping, enable it */
ldr w5, [x4, #PCB_FLAGS]
set_step_flag w5, x6
/* Restore the registers */
ldp x5, x6, [x4, #PCB_SP]
mov sp, x5
msr tpidr_el0, x6
ldr x6, [x4, #PCB_TPIDRRO]
msr tpidrro_el0, x6
- ldp x8, x9, [x4, #PCB_REGS + 8 * 8]
- ldp x10, x11, [x4, #PCB_REGS + 10 * 8]
- ldp x12, x13, [x4, #PCB_REGS + 12 * 8]
- ldp x14, x15, [x4, #PCB_REGS + 14 * 8]
- ldp x16, x17, [x4, #PCB_REGS + 16 * 8]
- ldr x19, [x4, #PCB_REGS + 19 * 8]
- ldp x20, x21, [x4, #PCB_REGS + 20 * 8]
- ldp x22, x23, [x4, #PCB_REGS + 22 * 8]
- ldp x24, x25, [x4, #PCB_REGS + 24 * 8]
- ldp x26, x27, [x4, #PCB_REGS + 26 * 8]
- ldp x28, x29, [x4, #PCB_REGS + 28 * 8]
- ldr lr, [x4, #PCB_LR]
+ ldp x19, x20, [x4, #PCB_REGS + 19 * 8]
+ ldp x21, x22, [x4, #PCB_REGS + 21 * 8]
+ ldp x23, x24, [x4, #PCB_REGS + 23 * 8]
+ ldp x25, x26, [x4, #PCB_REGS + 25 * 8]
+ ldp x27, x28, [x4, #PCB_REGS + 27 * 8]
+ ldp x29, lr, [x4, #PCB_REGS + 29 * 8]
ret
END(cpu_throw)
/*
* void cpu_switch(struct thread *old, struct thread *new, struct mtx *mtx)
*
* x0 = old
* x1 = new
* x2 = mtx
* x3 to x7, x16 and x17 are caller saved
*/
ENTRY(cpu_switch)
/*
* Save the old context.
*/
ldr x4, [x0, #TD_PCB]
/* Store the callee-saved registers */
- stp x8, x9, [x4, #PCB_REGS + 8 * 8]
- stp x10, x11, [x4, #PCB_REGS + 10 * 8]
- stp x12, x13, [x4, #PCB_REGS + 12 * 8]
- stp x14, x15, [x4, #PCB_REGS + 14 * 8]
- stp x16, x17, [x4, #PCB_REGS + 16 * 8]
- stp x18, x19, [x4, #PCB_REGS + 18 * 8]
- stp x20, x21, [x4, #PCB_REGS + 20 * 8]
- stp x22, x23, [x4, #PCB_REGS + 22 * 8]
- stp x24, x25, [x4, #PCB_REGS + 24 * 8]
- stp x26, x27, [x4, #PCB_REGS + 26 * 8]
- stp x28, x29, [x4, #PCB_REGS + 28 * 8]
- str lr, [x4, #PCB_LR]
+ stp x19, x20, [x4, #PCB_REGS + 19 * 8]
+ stp x21, x22, [x4, #PCB_REGS + 21 * 8]
+ stp x23, x24, [x4, #PCB_REGS + 23 * 8]
+ stp x25, x26, [x4, #PCB_REGS + 25 * 8]
+ stp x27, x28, [x4, #PCB_REGS + 27 * 8]
+ stp x29, lr, [x4, #PCB_REGS + 29 * 8]
/* And the old stack pointer */
mov x5, sp
mrs x6, tpidrro_el0
str x6, [x4, #PCB_TPIDRRO]
mrs x6, tpidr_el0
stp x5, x6, [x4, #PCB_SP]
/* If we were single stepping, disable it */
ldr w5, [x4, #PCB_FLAGS]
clear_step_flag w5, x6
mov x19, x0
mov x20, x1
mov x21, x2
#ifdef VFP
/* Load the pcb address */
mov x1, x4
bl vfp_save_state
mov x0, x20
#else
mov x0, x1
#endif
/* This returns the thread pointer so no need to save it */
bl ptrauth_switch
/* This returns the thread pcb */
bl pmap_switch
/* Move the new pcb out of the way */
mov x4, x0
mov x2, x21
mov x1, x20
mov x0, x19
/*
* Release the old thread.
*/
stlr x2, [x0, #TD_LOCK]
#if defined(SCHED_ULE) && defined(SMP)
/* Spin if TD_LOCK points to a blocked_lock */
ldr x2, =_C_LABEL(blocked_lock)
1:
ldar x3, [x1, #TD_LOCK]
cmp x3, x2
b.eq 1b
#endif
/* If we are single stepping, enable it */
ldr w5, [x4, #PCB_FLAGS]
set_step_flag w5, x6
/* Restore the registers */
ldp x5, x6, [x4, #PCB_SP]
mov sp, x5
msr tpidr_el0, x6
ldr x6, [x4, #PCB_TPIDRRO]
msr tpidrro_el0, x6
- ldp x8, x9, [x4, #PCB_REGS + 8 * 8]
- ldp x10, x11, [x4, #PCB_REGS + 10 * 8]
- ldp x12, x13, [x4, #PCB_REGS + 12 * 8]
- ldp x14, x15, [x4, #PCB_REGS + 14 * 8]
- ldp x16, x17, [x4, #PCB_REGS + 16 * 8]
- ldr x19, [x4, #PCB_REGS + 19 * 8]
- ldp x20, x21, [x4, #PCB_REGS + 20 * 8]
- ldp x22, x23, [x4, #PCB_REGS + 22 * 8]
- ldp x24, x25, [x4, #PCB_REGS + 24 * 8]
- ldp x26, x27, [x4, #PCB_REGS + 26 * 8]
- ldp x28, x29, [x4, #PCB_REGS + 28 * 8]
- ldr lr, [x4, #PCB_LR]
+ ldp x19, x20, [x4, #PCB_REGS + 19 * 8]
+ ldp x21, x22, [x4, #PCB_REGS + 21 * 8]
+ ldp x23, x24, [x4, #PCB_REGS + 23 * 8]
+ ldp x25, x26, [x4, #PCB_REGS + 25 * 8]
+ ldp x27, x28, [x4, #PCB_REGS + 27 * 8]
+ ldp x29, lr, [x4, #PCB_REGS + 29 * 8]
str xzr, [x4, #PCB_REGS + 18 * 8]
ret
END(cpu_switch)
ENTRY(fork_trampoline)
- mov x0, x8
- mov x1, x9
+ mov x0, x19
+ mov x1, x20
mov x2, sp
mov fp, #0 /* Stack traceback stops here. */
bl _C_LABEL(fork_exit)
/*
* Disable interrupts as we are setting userspace specific
* state that we won't handle correctly in an interrupt while
* in the kernel.
*/
msr daifset, #(DAIF_D | DAIF_INTR)
ldr x0, [x18, #PC_CURTHREAD]
bl ptrauth_enter_el0
/* Restore sp, lr, elr, and spsr */
ldp x18, lr, [sp, #TF_SP]
ldp x10, x11, [sp, #TF_ELR]
msr sp_el0, x18
msr spsr_el1, x11
msr elr_el1, x10
/* Restore the CPU registers */
ldp x0, x1, [sp, #TF_X + 0 * 8]
ldp x2, x3, [sp, #TF_X + 2 * 8]
ldp x4, x5, [sp, #TF_X + 4 * 8]
ldp x6, x7, [sp, #TF_X + 6 * 8]
ldp x8, x9, [sp, #TF_X + 8 * 8]
ldp x10, x11, [sp, #TF_X + 10 * 8]
ldp x12, x13, [sp, #TF_X + 12 * 8]
ldp x14, x15, [sp, #TF_X + 14 * 8]
ldp x16, x17, [sp, #TF_X + 16 * 8]
ldp x18, x19, [sp, #TF_X + 18 * 8]
ldp x20, x21, [sp, #TF_X + 20 * 8]
ldp x22, x23, [sp, #TF_X + 22 * 8]
ldp x24, x25, [sp, #TF_X + 24 * 8]
ldp x26, x27, [sp, #TF_X + 26 * 8]
ldp x28, x29, [sp, #TF_X + 28 * 8]
/*
* No need for interrupts reenabling since PSR
* will be set to the desired value anyway.
*/
ERET
END(fork_trampoline)
ENTRY(savectx)
/* Store the callee-saved registers */
- stp x8, x9, [x0, #PCB_REGS + 8 * 8]
- stp x10, x11, [x0, #PCB_REGS + 10 * 8]
- stp x12, x13, [x0, #PCB_REGS + 12 * 8]
- stp x14, x15, [x0, #PCB_REGS + 14 * 8]
- stp x16, x17, [x0, #PCB_REGS + 16 * 8]
- stp x18, x19, [x0, #PCB_REGS + 18 * 8]
- stp x20, x21, [x0, #PCB_REGS + 20 * 8]
- stp x22, x23, [x0, #PCB_REGS + 22 * 8]
- stp x24, x25, [x0, #PCB_REGS + 24 * 8]
- stp x26, x27, [x0, #PCB_REGS + 26 * 8]
- stp x28, x29, [x0, #PCB_REGS + 28 * 8]
- str lr, [x0, #PCB_LR]
+ stp x19, x20, [x0, #PCB_REGS + 19 * 8]
+ stp x21, x22, [x0, #PCB_REGS + 21 * 8]
+ stp x23, x24, [x0, #PCB_REGS + 23 * 8]
+ stp x25, x26, [x0, #PCB_REGS + 25 * 8]
+ stp x27, x28, [x0, #PCB_REGS + 27 * 8]
+ stp x29, lr, [x0, #PCB_REGS + 29 * 8]
/* And the old stack pointer */
mov x5, sp
mrs x6, tpidrro_el0
str x6, [x0, #PCB_TPIDRRO]
mrs x6, tpidr_el0
stp x5, x6, [x0, #PCB_SP]
/* Store the VFP registers */
#ifdef VFP
mov x28, lr
mov x1, x0 /* move pcb to the correct register */
mov x0, xzr /* td = NULL */
bl vfp_save_state
mov lr, x28
#endif
ret
END(savectx)
diff --git a/sys/arm64/arm64/vm_machdep.c b/sys/arm64/arm64/vm_machdep.c
index c52a7e2fc5c1..feb439314f50 100644
--- a/sys/arm64/arm64/vm_machdep.c
+++ b/sys/arm64/arm64/vm_machdep.c
@@ -1,312 +1,312 @@
/*-
* Copyright (c) 2014 Andrew Turner
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*
*/
#include "opt_platform.h"
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/limits.h>
#include <sys/proc.h>
#include <sys/sf_buf.h>
#include <sys/signal.h>
#include <sys/sysent.h>
#include <sys/unistd.h>
#include <vm/vm.h>
#include <vm/vm_page.h>
#include <vm/vm_map.h>
#include <vm/uma.h>
#include <vm/uma_int.h>
#include <machine/armreg.h>
#include <machine/cpu.h>
#include <machine/md_var.h>
#include <machine/pcb.h>
#include <machine/frame.h>
#ifdef VFP
#include <machine/vfp.h>
#endif
#include <dev/psci/psci.h>
/*
* Finish a fork operation, with process p2 nearly set up.
* Copy and update the pcb, set up the stack so that the child
* ready to run and return to user mode.
*/
void
cpu_fork(struct thread *td1, struct proc *p2, struct thread *td2, int flags)
{
struct pcb *pcb2;
struct trapframe *tf;
if ((flags & RFPROC) == 0)
return;
if (td1 == curthread) {
/*
* Save the tpidr_el0 and the vfp state, these normally happen
* in cpu_switch, but if userland changes these then forks
* this may not have happened.
*/
td1->td_pcb->pcb_tpidr_el0 = READ_SPECIALREG(tpidr_el0);
td1->td_pcb->pcb_tpidrro_el0 = READ_SPECIALREG(tpidrro_el0);
#ifdef VFP
if ((td1->td_pcb->pcb_fpflags & PCB_FP_STARTED) != 0)
vfp_save_state(td1, td1->td_pcb);
#endif
}
pcb2 = (struct pcb *)(td2->td_kstack +
td2->td_kstack_pages * PAGE_SIZE) - 1;
td2->td_pcb = pcb2;
bcopy(td1->td_pcb, pcb2, sizeof(*pcb2));
/* Clear the debug register state. */
bzero(&pcb2->pcb_dbg_regs, sizeof(pcb2->pcb_dbg_regs));
ptrauth_fork(td2, td1);
tf = (struct trapframe *)STACKALIGN((struct trapframe *)pcb2 - 1);
bcopy(td1->td_frame, tf, sizeof(*tf));
tf->tf_x[0] = 0;
tf->tf_x[1] = 0;
tf->tf_spsr = td1->td_frame->tf_spsr & (PSR_M_32 | PSR_DAIF);
td2->td_frame = tf;
/* Set the return value registers for fork() */
- td2->td_pcb->pcb_x[8] = (uintptr_t)fork_return;
- td2->td_pcb->pcb_x[9] = (uintptr_t)td2;
+ td2->td_pcb->pcb_x[19] = (uintptr_t)fork_return;
+ td2->td_pcb->pcb_x[20] = (uintptr_t)td2;
td2->td_pcb->pcb_lr = (uintptr_t)fork_trampoline;
td2->td_pcb->pcb_sp = (uintptr_t)td2->td_frame;
td2->td_pcb->pcb_fpusaved = &td2->td_pcb->pcb_fpustate;
td2->td_pcb->pcb_vfpcpu = UINT_MAX;
/* Setup to release spin count in fork_exit(). */
td2->td_md.md_spinlock_count = 1;
td2->td_md.md_saved_daif = PSR_DAIF_DEFAULT;
#if defined(PERTHREAD_SSP)
/* Set the new canary */
arc4random_buf(&td2->td_md.md_canary, sizeof(td2->td_md.md_canary));
#endif
}
void
cpu_reset(void)
{
psci_reset();
printf("cpu_reset failed");
while(1)
__asm volatile("wfi" ::: "memory");
}
void
cpu_thread_swapin(struct thread *td)
{
}
void
cpu_thread_swapout(struct thread *td)
{
}
void
cpu_set_syscall_retval(struct thread *td, int error)
{
struct trapframe *frame;
frame = td->td_frame;
if (__predict_true(error == 0)) {
frame->tf_x[0] = td->td_retval[0];
frame->tf_x[1] = td->td_retval[1];
frame->tf_spsr &= ~PSR_C; /* carry bit */
return;
}
switch (error) {
case ERESTART:
frame->tf_elr -= 4;
break;
case EJUSTRETURN:
break;
default:
frame->tf_spsr |= PSR_C; /* carry bit */
frame->tf_x[0] = error;
break;
}
}
/*
* Initialize machine state, mostly pcb and trap frame for a new
* thread, about to return to userspace. Put enough state in the new
* thread's PCB to get it to go back to the fork_return(), which
* finalizes the thread state and handles peculiarities of the first
* return to userspace for the new thread.
*/
void
cpu_copy_thread(struct thread *td, struct thread *td0)
{
bcopy(td0->td_frame, td->td_frame, sizeof(struct trapframe));
bcopy(td0->td_pcb, td->td_pcb, sizeof(struct pcb));
- td->td_pcb->pcb_x[8] = (uintptr_t)fork_return;
- td->td_pcb->pcb_x[9] = (uintptr_t)td;
+ td->td_pcb->pcb_x[19] = (uintptr_t)fork_return;
+ td->td_pcb->pcb_x[20] = (uintptr_t)td;
td->td_pcb->pcb_lr = (uintptr_t)fork_trampoline;
td->td_pcb->pcb_sp = (uintptr_t)td->td_frame;
td->td_pcb->pcb_fpflags &= ~(PCB_FP_STARTED | PCB_FP_KERN | PCB_FP_NOSAVE);
td->td_pcb->pcb_fpusaved = &td->td_pcb->pcb_fpustate;
td->td_pcb->pcb_vfpcpu = UINT_MAX;
/* Setup to release spin count in fork_exit(). */
td->td_md.md_spinlock_count = 1;
td->td_md.md_saved_daif = PSR_DAIF_DEFAULT;
#if defined(PERTHREAD_SSP)
/* Set the new canary */
arc4random_buf(&td->td_md.md_canary, sizeof(td->td_md.md_canary));
#endif
/* Generate new pointer authentication keys. */
ptrauth_copy_thread(td, td0);
}
/*
* Set that machine state for performing an upcall that starts
* the entry function with the given argument.
*/
void
cpu_set_upcall(struct thread *td, void (*entry)(void *), void *arg,
stack_t *stack)
{
struct trapframe *tf = td->td_frame;
/* 32bits processes use r13 for sp */
if (td->td_frame->tf_spsr & PSR_M_32) {
tf->tf_x[13] = STACKALIGN((uintptr_t)stack->ss_sp + stack->ss_size);
if ((register_t)entry & 1)
tf->tf_spsr |= PSR_T;
} else
tf->tf_sp = STACKALIGN((uintptr_t)stack->ss_sp + stack->ss_size);
tf->tf_elr = (register_t)entry;
tf->tf_x[0] = (register_t)arg;
}
int
cpu_set_user_tls(struct thread *td, void *tls_base)
{
struct pcb *pcb;
if ((uintptr_t)tls_base >= VM_MAXUSER_ADDRESS)
return (EINVAL);
pcb = td->td_pcb;
if (td->td_frame->tf_spsr & PSR_M_32) {
/* 32bits arm stores the user TLS into tpidrro */
pcb->pcb_tpidrro_el0 = (register_t)tls_base;
pcb->pcb_tpidr_el0 = (register_t)tls_base;
if (td == curthread) {
WRITE_SPECIALREG(tpidrro_el0, tls_base);
WRITE_SPECIALREG(tpidr_el0, tls_base);
}
} else {
pcb->pcb_tpidr_el0 = (register_t)tls_base;
if (td == curthread)
WRITE_SPECIALREG(tpidr_el0, tls_base);
}
return (0);
}
void
cpu_thread_exit(struct thread *td)
{
}
void
cpu_thread_alloc(struct thread *td)
{
td->td_pcb = (struct pcb *)(td->td_kstack +
td->td_kstack_pages * PAGE_SIZE) - 1;
td->td_frame = (struct trapframe *)STACKALIGN(
(struct trapframe *)td->td_pcb - 1);
ptrauth_thread_alloc(td);
}
void
cpu_thread_free(struct thread *td)
{
}
void
cpu_thread_clean(struct thread *td)
{
}
/*
* Intercept the return address from a freshly forked process that has NOT
* been scheduled yet.
*
* This is needed to make kernel threads stay in kernel mode.
*/
void
cpu_fork_kthread_handler(struct thread *td, void (*func)(void *), void *arg)
{
- td->td_pcb->pcb_x[8] = (uintptr_t)func;
- td->td_pcb->pcb_x[9] = (uintptr_t)arg;
+ td->td_pcb->pcb_x[19] = (uintptr_t)func;
+ td->td_pcb->pcb_x[20] = (uintptr_t)arg;
}
void
cpu_exit(struct thread *td)
{
}
bool
cpu_exec_vmspace_reuse(struct proc *p __unused, vm_map_t map __unused)
{
return (true);
}
int
cpu_procctl(struct thread *td __unused, int idtype __unused, id_t id __unused,
int com __unused, void *data __unused)
{
return (EINVAL);
}