diff --git a/sys/arm/arm/exec_machdep.c b/sys/arm/arm/exec_machdep.c index c14bd51146ef..96b382c9083f 100644 --- a/sys/arm/arm/exec_machdep.c +++ b/sys/arm/arm/exec_machdep.c @@ -1,397 +1,392 @@ /* $NetBSD: arm32_machdep.c,v 1.44 2004/03/24 15:34:47 atatat Exp $ */ /*- * SPDX-License-Identifier: BSD-4-Clause * * Copyright (c) 2004 Olivier Houchard * Copyright (c) 1994-1998 Mark Brinicombe. * Copyright (c) 1994 Brini. * All rights reserved. * * This code is derived from software written for Brini by Mark Brinicombe * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. All advertising materials mentioning features or use of this software * must display the following acknowledgement: * This product includes software developed by Mark Brinicombe * for the NetBSD Project. * 4. The name of the company nor the name of the author may be used to * endorse or promote products derived from this software without specific * prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include _Static_assert(sizeof(mcontext_t) == 208, "mcontext_t size incorrect"); _Static_assert(sizeof(ucontext_t) == 260, "ucontext_t size incorrect"); _Static_assert(sizeof(siginfo_t) == 64, "siginfo_t size incorrect"); /* * Clear registers on exec */ void exec_setregs(struct thread *td, struct image_params *imgp, uintptr_t stack) { struct trapframe *tf = td->td_frame; memset(tf, 0, sizeof(*tf)); tf->tf_usr_sp = stack; tf->tf_usr_lr = imgp->entry_addr; tf->tf_svc_lr = 0x77777777; tf->tf_pc = imgp->entry_addr; tf->tf_spsr = PSR_USR32_MODE; if ((register_t)imgp->entry_addr & 1) tf->tf_spsr |= PSR_T; } #ifdef VFP /* * Get machine VFP context. */ void get_vfpcontext(struct thread *td, mcontext_vfp_t *vfp) { struct pcb *pcb; - MPASS(td == curthread); + MPASS(td == curthread || TD_IS_SUSPENDED(td) || + P_SHOULDSTOP(td->td_proc)); pcb = td->td_pcb; - if ((pcb->pcb_fpflags & PCB_FP_STARTED) != 0) { + if ((pcb->pcb_fpflags & PCB_FP_STARTED) != 0 && td == curthread) { critical_enter(); vfp_store(&pcb->pcb_vfpstate, false); critical_exit(); } KASSERT(pcb->pcb_vfpsaved == &pcb->pcb_vfpstate, ("Called get_vfpcontext while the kernel is using the VFP")); - memcpy(vfp->mcv_reg, pcb->pcb_vfpstate.reg, - sizeof(vfp->mcv_reg)); - vfp->mcv_fpscr = pcb->pcb_vfpstate.fpscr; + memcpy(vfp, &pcb->pcb_vfpstate, sizeof(*vfp)); } /* * Set machine VFP context. */ void set_vfpcontext(struct thread *td, mcontext_vfp_t *vfp) { struct pcb *pcb; - MPASS(td == curthread); - pcb = td->td_pcb; - if ((pcb->pcb_fpflags & PCB_FP_STARTED) != 0) { + if (td == curthread) { critical_enter(); vfp_discard(td); critical_exit(); } KASSERT(pcb->pcb_vfpsaved == &pcb->pcb_vfpstate, ("Called set_vfpcontext while the kernel is using the VFP")); - memcpy(pcb->pcb_vfpstate.reg, vfp->mcv_reg, - sizeof(pcb->pcb_vfpstate.reg)); - pcb->pcb_vfpstate.fpscr = vfp->mcv_fpscr; + memcpy(&pcb->pcb_vfpstate, vfp, sizeof(*vfp)); } #endif int arm_get_vfpstate(struct thread *td, void *args) { int rv; struct arm_get_vfpstate_args ua; mcontext_vfp_t mcontext_vfp; rv = copyin(args, &ua, sizeof(ua)); if (rv != 0) return (rv); if (ua.mc_vfp_size != sizeof(mcontext_vfp_t)) return (EINVAL); #ifdef VFP get_vfpcontext(td, &mcontext_vfp); #else bzero(&mcontext_vfp, sizeof(mcontext_vfp)); #endif rv = copyout(&mcontext_vfp, ua.mc_vfp, sizeof(mcontext_vfp)); if (rv != 0) return (rv); return (0); } /* * Get machine context. */ int get_mcontext(struct thread *td, mcontext_t *mcp, int clear_ret) { struct trapframe *tf = td->td_frame; __greg_t *gr = mcp->__gregs; mcontext_vfp_t mcontext_vfp; int rv; if (clear_ret & GET_MC_CLEAR_RET) { gr[_REG_R0] = 0; gr[_REG_CPSR] = tf->tf_spsr & ~PSR_C; } else { gr[_REG_R0] = tf->tf_r0; gr[_REG_CPSR] = tf->tf_spsr; } gr[_REG_R1] = tf->tf_r1; gr[_REG_R2] = tf->tf_r2; gr[_REG_R3] = tf->tf_r3; gr[_REG_R4] = tf->tf_r4; gr[_REG_R5] = tf->tf_r5; gr[_REG_R6] = tf->tf_r6; gr[_REG_R7] = tf->tf_r7; gr[_REG_R8] = tf->tf_r8; gr[_REG_R9] = tf->tf_r9; gr[_REG_R10] = tf->tf_r10; gr[_REG_R11] = tf->tf_r11; gr[_REG_R12] = tf->tf_r12; gr[_REG_SP] = tf->tf_usr_sp; gr[_REG_LR] = tf->tf_usr_lr; gr[_REG_PC] = tf->tf_pc; #ifdef VFP if (mcp->mc_vfp_size != sizeof(mcontext_vfp_t)) return (EINVAL); get_vfpcontext(td, &mcontext_vfp); #else bzero(&mcontext_vfp, sizeof(mcontext_vfp)); #endif if (mcp->mc_vfp_ptr != NULL) { rv = copyout(&mcontext_vfp, mcp->mc_vfp_ptr, sizeof(mcontext_vfp)); if (rv != 0) return (rv); } return (0); } /* * Set machine context. * * However, we don't set any but the user modifiable flags, and we won't * touch the cs selector. */ int set_mcontext(struct thread *td, mcontext_t *mcp) { mcontext_vfp_t mc_vfp, *vfp; struct trapframe *tf = td->td_frame; const __greg_t *gr = mcp->__gregs; int spsr; /* * Make sure the processor mode has not been tampered with and * interrupts have not been disabled. */ spsr = gr[_REG_CPSR]; if ((spsr & PSR_MODE) != PSR_USR32_MODE || (spsr & (PSR_I | PSR_F)) != 0) return (EINVAL); #ifdef WITNESS if (mcp->mc_vfp_size != 0 && mcp->mc_vfp_size != sizeof(mc_vfp)) { printf("%s: %s: Malformed mc_vfp_size: %d (0x%08X)\n", td->td_proc->p_comm, __func__, mcp->mc_vfp_size, mcp->mc_vfp_size); } else if (mcp->mc_vfp_size != 0 && mcp->mc_vfp_ptr == NULL) { printf("%s: %s: c_vfp_size != 0 but mc_vfp_ptr == NULL\n", td->td_proc->p_comm, __func__); } #endif if (mcp->mc_vfp_size == sizeof(mc_vfp) && mcp->mc_vfp_ptr != NULL) { if (copyin(mcp->mc_vfp_ptr, &mc_vfp, sizeof(mc_vfp)) != 0) return (EFAULT); vfp = &mc_vfp; } else { vfp = NULL; } tf->tf_r0 = gr[_REG_R0]; tf->tf_r1 = gr[_REG_R1]; tf->tf_r2 = gr[_REG_R2]; tf->tf_r3 = gr[_REG_R3]; tf->tf_r4 = gr[_REG_R4]; tf->tf_r5 = gr[_REG_R5]; tf->tf_r6 = gr[_REG_R6]; tf->tf_r7 = gr[_REG_R7]; tf->tf_r8 = gr[_REG_R8]; tf->tf_r9 = gr[_REG_R9]; tf->tf_r10 = gr[_REG_R10]; tf->tf_r11 = gr[_REG_R11]; tf->tf_r12 = gr[_REG_R12]; tf->tf_usr_sp = gr[_REG_SP]; tf->tf_usr_lr = gr[_REG_LR]; tf->tf_pc = gr[_REG_PC]; tf->tf_spsr = gr[_REG_CPSR]; #ifdef VFP if (vfp != NULL) set_vfpcontext(td, vfp); #endif return (0); } void sendsig(sig_t catcher, ksiginfo_t *ksi, sigset_t *mask) { struct thread *td; struct proc *p; struct trapframe *tf; struct sigframe *fp, frame; struct sigacts *psp; struct sysentvec *sysent; int onstack; int sig; td = curthread; p = td->td_proc; PROC_LOCK_ASSERT(p, MA_OWNED); sig = ksi->ksi_signo; psp = p->p_sigacts; mtx_assert(&psp->ps_mtx, MA_OWNED); tf = td->td_frame; onstack = sigonstack(tf->tf_usr_sp); CTR4(KTR_SIG, "sendsig: td=%p (%s) catcher=%p sig=%d", td, p->p_comm, catcher, sig); /* Allocate and validate space for the signal handler context. */ if ((td->td_pflags & TDP_ALTSTACK) != 0 && !(onstack) && SIGISMEMBER(psp->ps_sigonstack, sig)) { fp = (struct sigframe *)((uintptr_t)td->td_sigstk.ss_sp + td->td_sigstk.ss_size); #if defined(COMPAT_43) td->td_sigstk.ss_flags |= SS_ONSTACK; #endif } else fp = (struct sigframe *)td->td_frame->tf_usr_sp; /* make room on the stack */ fp--; /* make the stack aligned */ fp = (struct sigframe *)STACKALIGN(fp); /* Populate the siginfo frame. */ bzero(&frame, sizeof(frame)); get_mcontext(td, &frame.sf_uc.uc_mcontext, 0); frame.sf_si = ksi->ksi_info; frame.sf_uc.uc_sigmask = *mask; frame.sf_uc.uc_stack = td->td_sigstk; frame.sf_uc.uc_stack.ss_flags = (td->td_pflags & TDP_ALTSTACK) != 0 ? (onstack ? SS_ONSTACK : 0) : SS_DISABLE; mtx_unlock(&psp->ps_mtx); PROC_UNLOCK(td->td_proc); /* Copy the sigframe out to the user's stack. */ if (copyout(&frame, fp, sizeof(*fp)) != 0) { /* Process has trashed its stack. Kill it. */ CTR2(KTR_SIG, "sendsig: sigexit td=%p fp=%p", td, fp); PROC_LOCK(p); sigexit(td, SIGILL); } /* * Build context to run handler in. We invoke the handler * directly, only returning via the trampoline. Note the * trampoline version numbers are coordinated with machine- * dependent code in libc. */ tf->tf_r0 = sig; tf->tf_r1 = (register_t)&fp->sf_si; tf->tf_r2 = (register_t)&fp->sf_uc; /* the trampoline uses r5 as the uc address */ tf->tf_r5 = (register_t)&fp->sf_uc; tf->tf_pc = (register_t)catcher; tf->tf_usr_sp = (register_t)fp; sysent = p->p_sysent; if (PROC_HAS_SHP(p)) tf->tf_usr_lr = (register_t)PROC_SIGCODE(p); else tf->tf_usr_lr = (register_t)(PROC_PS_STRINGS(p) - *(sysent->sv_szsigcode)); /* Set the mode to enter in the signal handler */ #if __ARM_ARCH >= 7 if ((register_t)catcher & 1) tf->tf_spsr |= PSR_T; else tf->tf_spsr &= ~PSR_T; #endif CTR3(KTR_SIG, "sendsig: return td=%p pc=%#x sp=%#x", td, tf->tf_usr_lr, tf->tf_usr_sp); PROC_LOCK(p); mtx_lock(&psp->ps_mtx); } int sys_sigreturn(struct thread *td, struct sigreturn_args *uap) { ucontext_t uc; int error; if (uap == NULL) return (EFAULT); if (copyin(uap->sigcntxp, &uc, sizeof(uc))) return (EFAULT); /* Restore register context. */ error = set_mcontext(td, &uc.uc_mcontext); if (error != 0) return (error); /* Restore signal mask. */ kern_sigprocmask(td, SIG_SETMASK, &uc.uc_sigmask, NULL, 0); return (EJUSTRETURN); } diff --git a/sys/arm/arm/vfp.c b/sys/arm/arm/vfp.c index 915d65c1b790..d51c4b6e0618 100644 --- a/sys/arm/arm/vfp.c +++ b/sys/arm/arm/vfp.c @@ -1,503 +1,501 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2014 Ian Lepore * Copyright (c) 2012 Mark Tinguely * * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #ifdef VFP #include #include #include #include #include #include #include #include #include #include #include #include /* function prototypes */ static int vfp_bounce(u_int, u_int, struct trapframe *, int); static void vfp_restore(struct vfp_state *); extern int vfp_exists; static struct undefined_handler vfp10_uh, vfp11_uh; /* If true the VFP unit has 32 double registers, otherwise it has 16 */ static int is_d32; struct fpu_kern_ctx { struct vfp_state *prev; #define FPU_KERN_CTX_DUMMY 0x01 /* avoided save for the kern thread */ #define FPU_KERN_CTX_INUSE 0x02 uint32_t flags; struct vfp_state state; }; /* * About .fpu directives in this file... * * We should need simply .fpu vfpv3, but clang 3.5 has a quirk where setting * vfpv3 doesn't imply that vfp2 features are also available -- both have to be * explicitly set to get all the features of both. This is probably a bug in * clang, so it may get fixed and require changes here some day. Other changes * are probably coming in clang too, because there is email and open PRs * indicating they want to completely disable the ability to use .fpu and * similar directives in inline asm. That would be catastrophic for us, * hopefully they come to their senses. There was also some discusion of a new * syntax such as .push fpu=vfpv3; ...; .pop fpu; and that would be ideal for * us, better than what we have now really. * * For gcc, each .fpu directive completely overrides the prior directive, unlike * with clang, but luckily on gcc saying v3 implies all the v2 features as well. */ #define fmxr(reg, val) \ __asm __volatile(" .fpu vfpv2\n .fpu vfpv3\n" \ " vmsr " __STRING(reg) ", %0" :: "r"(val)); #define fmrx(reg) \ ({ u_int val = 0;\ __asm __volatile(" .fpu vfpv2\n .fpu vfpv3\n" \ " vmrs %0, " __STRING(reg) : "=r"(val)); \ val; \ }) static u_int get_coprocessorACR(void) { u_int val; __asm __volatile("mrc p15, 0, %0, c1, c0, 2" : "=r" (val) : : "cc"); return val; } static void set_coprocessorACR(u_int val) { __asm __volatile("mcr p15, 0, %0, c1, c0, 2\n\t" : : "r" (val) : "cc"); isb(); } static void vfp_enable(void) { uint32_t fpexc; fpexc = fmrx(fpexc); fmxr(fpexc, fpexc | VFPEXC_EN); isb(); } static void vfp_disable(void) { uint32_t fpexc; fpexc = fmrx(fpexc); fmxr(fpexc, fpexc & ~VFPEXC_EN); isb(); } /* called for each cpu */ void vfp_init(void) { u_int fpsid, tmp; u_int coproc, vfp_arch; coproc = get_coprocessorACR(); coproc |= COPROC10 | COPROC11; set_coprocessorACR(coproc); fpsid = fmrx(fpsid); /* read the vfp system id */ if (!(fpsid & VFPSID_HARDSOFT_IMP)) { vfp_exists = 1; is_d32 = 0; PCPU_SET(vfpsid, fpsid); /* save the fpsid */ elf_hwcap |= HWCAP_VFP; vfp_arch = (fpsid & VFPSID_SUBVERSION2_MASK) >> VFPSID_SUBVERSION_OFF; if (vfp_arch >= VFP_ARCH3) { tmp = fmrx(mvfr0); PCPU_SET(vfpmvfr0, tmp); elf_hwcap |= HWCAP_VFPv3; if ((tmp & VMVFR0_RB_MASK) == 2) { elf_hwcap |= HWCAP_VFPD32; is_d32 = 1; } else elf_hwcap |= HWCAP_VFPv3D16; tmp = fmrx(mvfr1); PCPU_SET(vfpmvfr1, tmp); if (PCPU_GET(cpuid) == 0) { if ((tmp & VMVFR1_FZ_MASK) == 0x1) { /* Denormals arithmetic support */ initial_fpscr &= ~VFPSCR_FZ; thread0.td_pcb->pcb_vfpstate.fpscr = initial_fpscr; } } if ((tmp & VMVFR1_LS_MASK) >> VMVFR1_LS_OFF == 1 && (tmp & VMVFR1_I_MASK) >> VMVFR1_I_OFF == 1 && (tmp & VMVFR1_SP_MASK) >> VMVFR1_SP_OFF == 1) elf_hwcap |= HWCAP_NEON; if ((tmp & VMVFR1_FMAC_MASK) >> VMVFR1_FMAC_OFF == 1) elf_hwcap |= HWCAP_VFPv4; } /* initialize the coprocess 10 and 11 calls * These are called to restore the registers and enable * the VFP hardware. */ if (vfp10_uh.uh_handler == NULL) { vfp10_uh.uh_handler = vfp_bounce; vfp11_uh.uh_handler = vfp_bounce; install_coproc_handler_static(10, &vfp10_uh); install_coproc_handler_static(11, &vfp11_uh); } } } SYSINIT(vfp, SI_SUB_CPU, SI_ORDER_ANY, vfp_init, NULL); /* start VFP unit, restore the vfp registers from the PCB and retry * the instruction */ static int vfp_bounce(u_int addr, u_int insn, struct trapframe *frame, int code) { u_int cpu, fpexc; struct pcb *curpcb; ksiginfo_t ksi; if ((code & FAULT_USER) == 0) panic("undefined floating point instruction in supervisor mode"); critical_enter(); /* * If the VFP is already on and we got an undefined instruction, then * something tried to executate a truly invalid instruction that maps to * the VFP. */ fpexc = fmrx(fpexc); if (fpexc & VFPEXC_EN) { /* Clear any exceptions */ fmxr(fpexc, fpexc & ~(VFPEXC_EX | VFPEXC_FP2V)); /* kill the process - we do not handle emulation */ critical_exit(); if (fpexc & VFPEXC_EX) { /* We have an exception, signal a SIGFPE */ ksiginfo_init_trap(&ksi); ksi.ksi_signo = SIGFPE; if (fpexc & VFPEXC_UFC) ksi.ksi_code = FPE_FLTUND; else if (fpexc & VFPEXC_OFC) ksi.ksi_code = FPE_FLTOVF; else if (fpexc & VFPEXC_IOC) ksi.ksi_code = FPE_FLTINV; ksi.ksi_addr = (void *)addr; trapsignal(curthread, &ksi); return 0; } return 1; } /* * If the last time this thread used the VFP it was on this core, and * the last thread to use the VFP on this core was this thread, then the * VFP state is valid, otherwise restore this thread's state to the VFP. */ fmxr(fpexc, fpexc | VFPEXC_EN); curpcb = curthread->td_pcb; cpu = PCPU_GET(cpuid); if (curpcb->pcb_vfpcpu != cpu || curthread != PCPU_GET(fpcurthread)) { if (curpcb->pcb_vfpsaved == NULL) curpcb->pcb_vfpsaved = &curpcb->pcb_vfpstate; vfp_restore(curpcb->pcb_vfpsaved); curpcb->pcb_vfpcpu = cpu; PCPU_SET(fpcurthread, curthread); } critical_exit(); return (0); } /* * Restore the given state to the VFP hardware. */ static void vfp_restore(struct vfp_state *vfpsave) { uint32_t fpexc; /* On vfpv3 we may need to restore FPINST and FPINST2 */ fpexc = vfpsave->fpexec; if (fpexc & VFPEXC_EX) { fmxr(fpinst, vfpsave->fpinst); if (fpexc & VFPEXC_FP2V) fmxr(fpinst2, vfpsave->fpinst2); } fmxr(fpscr, vfpsave->fpscr); __asm __volatile( " .fpu vfpv2\n" " .fpu vfpv3\n" " vldmia %0!, {d0-d15}\n" /* d0-d15 */ " cmp %1, #0\n" /* -D16 or -D32? */ " vldmiane %0!, {d16-d31}\n" /* d16-d31 */ " addeq %0, %0, #128\n" /* skip missing regs */ : "+&r" (vfpsave) : "r" (is_d32) : "cc" ); fmxr(fpexc, fpexc); } /* * If the VFP is on, save its current state and turn it off if requested to do * so. If the VFP is not on, does not change the values at *vfpsave. Caller is * responsible for preventing a context switch while this is running. */ void vfp_store(struct vfp_state *vfpsave, boolean_t disable_vfp) { uint32_t fpexc; fpexc = fmrx(fpexc); /* Is the vfp enabled? */ if (fpexc & VFPEXC_EN) { vfpsave->fpexec = fpexc; vfpsave->fpscr = fmrx(fpscr); /* On vfpv3 we may need to save FPINST and FPINST2 */ if (fpexc & VFPEXC_EX) { vfpsave->fpinst = fmrx(fpinst); if (fpexc & VFPEXC_FP2V) vfpsave->fpinst2 = fmrx(fpinst2); fpexc &= ~VFPEXC_EX; } __asm __volatile( " .fpu vfpv2\n" " .fpu vfpv3\n" " vstmia %0!, {d0-d15}\n" /* d0-d15 */ " cmp %1, #0\n" /* -D16 or -D32? */ " vstmiane %0!, {d16-d31}\n" /* d16-d31 */ " addeq %0, %0, #128\n" /* skip missing regs */ : "+&r" (vfpsave) : "r" (is_d32) : "cc" ); if (disable_vfp) fmxr(fpexc , fpexc & ~VFPEXC_EN); } } /* * The current thread is dying. If the state currently in the hardware belongs * to the current thread, set fpcurthread to NULL to indicate that the VFP * hardware state does not belong to any thread. If the VFP is on, turn it off. - * Called only from cpu_throw(), so we don't have to worry about a context - * switch here. */ void vfp_discard(struct thread *td) { u_int tmp; if (PCPU_GET(fpcurthread) == td) PCPU_SET(fpcurthread, NULL); tmp = fmrx(fpexc); if (tmp & VFPEXC_EN) fmxr(fpexc, tmp & ~VFPEXC_EN); } void vfp_save_state(struct thread *td, struct pcb *pcb) { int32_t fpexc; KASSERT(pcb != NULL, ("NULL vfp pcb")); KASSERT(td == NULL || td->td_pcb == pcb, ("Invalid vfp pcb")); /* * savectx() will be called on panic with dumppcb as an argument, * dumppcb doesn't have pcb_vfpsaved set, so set it to save * the VFP registers. */ if (pcb->pcb_vfpsaved == NULL) pcb->pcb_vfpsaved = &pcb->pcb_vfpstate; if (td == NULL) td = curthread; critical_enter(); /* * Only store the registers if the VFP is enabled, * i.e. return if we are trapping on FP access. */ fpexc = fmrx(fpexc); if (fpexc & VFPEXC_EN) { KASSERT(PCPU_GET(fpcurthread) == td, ("Storing an invalid VFP state")); vfp_store(pcb->pcb_vfpsaved, true); } critical_exit(); } void fpu_kern_enter(struct thread *td, struct fpu_kern_ctx *ctx, u_int flags) { struct pcb *pcb; pcb = td->td_pcb; KASSERT((flags & FPU_KERN_NOCTX) != 0 || ctx != NULL, ("ctx is required when !FPU_KERN_NOCTX")); KASSERT(ctx == NULL || (ctx->flags & FPU_KERN_CTX_INUSE) == 0, ("using inuse ctx")); KASSERT((pcb->pcb_fpflags & PCB_FP_NOSAVE) == 0, ("recursive fpu_kern_enter while in PCB_FP_NOSAVE state")); if ((flags & FPU_KERN_NOCTX) != 0) { critical_enter(); if (curthread == PCPU_GET(fpcurthread)) { vfp_save_state(curthread, pcb); } PCPU_SET(fpcurthread, NULL); vfp_enable(); pcb->pcb_fpflags |= PCB_FP_KERN | PCB_FP_NOSAVE | PCB_FP_STARTED; return; } if ((flags & FPU_KERN_KTHR) != 0 && is_fpu_kern_thread(0)) { ctx->flags = FPU_KERN_CTX_DUMMY | FPU_KERN_CTX_INUSE; return; } /* * Check either we are already using the VFP in the kernel, or * the the saved state points to the default user space. */ KASSERT((pcb->pcb_fpflags & PCB_FP_KERN) != 0 || pcb->pcb_vfpsaved == &pcb->pcb_vfpstate, ("Mangled pcb_vfpsaved %x %p %p", pcb->pcb_fpflags, pcb->pcb_vfpsaved, &pcb->pcb_vfpstate)); ctx->flags = FPU_KERN_CTX_INUSE; vfp_save_state(curthread, pcb); ctx->prev = pcb->pcb_vfpsaved; pcb->pcb_vfpsaved = &ctx->state; pcb->pcb_fpflags |= PCB_FP_KERN; pcb->pcb_fpflags &= ~PCB_FP_STARTED; return; } int fpu_kern_leave(struct thread *td, struct fpu_kern_ctx *ctx) { struct pcb *pcb; pcb = td->td_pcb; if ((pcb->pcb_fpflags & PCB_FP_NOSAVE) != 0) { KASSERT(ctx == NULL, ("non-null ctx after FPU_KERN_NOCTX")); KASSERT(PCPU_GET(fpcurthread) == NULL, ("non-NULL fpcurthread for PCB_FP_NOSAVE")); CRITICAL_ASSERT(td); vfp_disable(); pcb->pcb_fpflags &= ~(PCB_FP_NOSAVE | PCB_FP_STARTED); critical_exit(); } else { KASSERT((ctx->flags & FPU_KERN_CTX_INUSE) != 0, ("FPU context not inuse")); ctx->flags &= ~FPU_KERN_CTX_INUSE; if (is_fpu_kern_thread(0) && (ctx->flags & FPU_KERN_CTX_DUMMY) != 0) return (0); KASSERT((ctx->flags & FPU_KERN_CTX_DUMMY) == 0, ("dummy ctx")); critical_enter(); vfp_discard(td); critical_exit(); pcb->pcb_fpflags &= ~PCB_FP_STARTED; pcb->pcb_vfpsaved = ctx->prev; } if (pcb->pcb_vfpsaved == &pcb->pcb_vfpstate) { pcb->pcb_fpflags &= ~PCB_FP_KERN; } else { KASSERT((pcb->pcb_fpflags & PCB_FP_KERN) != 0, ("unpaired fpu_kern_leave")); } return (0); } int fpu_kern_thread(u_int flags __unused) { struct pcb *pcb = curthread->td_pcb; KASSERT((curthread->td_pflags & TDP_KTHREAD) != 0, ("Only kthread may use fpu_kern_thread")); KASSERT(pcb->pcb_vfpsaved == &pcb->pcb_vfpstate, ("Mangled pcb_vfpsaved")); KASSERT((pcb->pcb_fpflags & PCB_FP_KERN) == 0, ("Thread already setup for the VFP")); pcb->pcb_fpflags |= PCB_FP_KERN; return (0); } int is_fpu_kern_thread(u_int flags __unused) { struct pcb *curpcb; if ((curthread->td_pflags & TDP_KTHREAD) == 0) return (0); curpcb = curthread->td_pcb; return ((curpcb->pcb_fpflags & PCB_FP_KERN) != 0); } #endif