diff --git a/share/man/man9/fpu_kern.9 b/share/man/man9/fpu_kern.9 --- a/share/man/man9/fpu_kern.9 +++ b/share/man/man9/fpu_kern.9 @@ -185,7 +185,8 @@ .Sh NOTES The .Nm -is currently implemented only for the i386, amd64, and arm64 architectures. +is currently implemented only for the i386, amd64, arm64, and powerpc +architectures. .Pp There is no way to handle floating point exceptions raised from kernel mode. @@ -205,6 +206,8 @@ .An Konstantin Belousov Aq Mt kib@FreeBSD.org . The arm64 support was added by .An Andrew Turner Aq Mt andrew@FreeBSD.org . +The powerpc support was added by +.An Shawn Anastasio Aq Mt sanastasio@raptorengineering.com . .Sh BUGS .Fn fpu_kern_leave should probably have type diff --git a/sys/powerpc/include/altivec.h b/sys/powerpc/include/altivec.h --- a/sys/powerpc/include/altivec.h +++ b/sys/powerpc/include/altivec.h @@ -35,5 +35,7 @@ void enable_vec(struct thread *); void save_vec(struct thread *); void save_vec_nodrop(struct thread *); +void enable_vec_kern(void); +void disable_vec(struct thread *td); #endif /* _MACHINE_ALTIVEC_H_ */ diff --git a/sys/powerpc/include/fpu.h b/sys/powerpc/include/fpu.h --- a/sys/powerpc/include/fpu.h +++ b/sys/powerpc/include/fpu.h @@ -76,6 +76,26 @@ void save_fpu_nodrop(struct thread *); void cleanup_fpscr(void); u_int get_fpu_exception(struct thread *); +void enable_fpu_kern(void); +void disable_fpu(struct thread *td); + +/* + * Flags for fpu_kern_alloc_ctx(), fpu_kern_enter() and fpu_kern_thread(). + */ +#define FPU_KERN_NORMAL 0x0000 +#define FPU_KERN_NOWAIT 0x0001 +#define FPU_KERN_KTHR 0x0002 +#define FPU_KERN_NOCTX 0x0004 + +struct fpu_kern_ctx; + +struct fpu_kern_ctx *fpu_kern_alloc_ctx(u_int flags); +void fpu_kern_free_ctx(struct fpu_kern_ctx *ctx); +void fpu_kern_enter(struct thread *td, struct fpu_kern_ctx *ctx, + u_int flags); +int fpu_kern_leave(struct thread *td, struct fpu_kern_ctx *ctx); +int fpu_kern_thread(u_int flags); +int is_fpu_kern_thread(u_int flags); #endif /* _KERNEL */ diff --git a/sys/powerpc/include/pcb.h b/sys/powerpc/include/pcb.h --- a/sys/powerpc/include/pcb.h +++ b/sys/powerpc/include/pcb.h @@ -48,7 +48,7 @@ register_t pcb_toc; /* toc pointer */ register_t pcb_lr; /* link register */ register_t pcb_dscr; /* dscr value */ - register_t pcb_fscr; + register_t pcb_fscr; register_t pcb_tar; struct pmap *pcb_pm; /* pmap of our vmspace */ jmp_buf *pcb_onfault; /* For use during @@ -56,11 +56,14 @@ int pcb_flags; #define PCB_FPU 0x1 /* Process uses FPU */ #define PCB_FPREGS 0x2 /* Process had FPU registers initialized */ -#define PCB_VEC 0x4 /* Process had Altivec initialized */ +#define PCB_VEC 0x4 /* Process uses Altivec */ #define PCB_VSX 0x8 /* Process had VSX initialized */ #define PCB_CDSCR 0x10 /* Process had Custom DSCR initialized */ #define PCB_HTM 0x20 /* Process had HTM initialized */ #define PCB_CFSCR 0x40 /* Process had FSCR updated */ +#define PCB_KERN_FPU 0x80 /* Kernel is using FPU/Vector unit */ +#define PCB_KERN_FPU_NOSAVE 0x100 /* FPU/Vec state not saved for kernel use */ +#define PCB_VECREGS 0x200 /* Process had Altivec registers initialized */ struct fpu { union { #if _BYTE_ORDER == _BIG_ENDIAN diff --git a/sys/powerpc/powerpc/altivec.c b/sys/powerpc/powerpc/altivec.c --- a/sys/powerpc/powerpc/altivec.c +++ b/sys/powerpc/powerpc/altivec.c @@ -105,10 +105,11 @@ * the thread, initialise the vector registers and VSCR to 0, and * set the flag to indicate that the vector unit is in use. */ + pcb->pcb_flags |= PCB_VEC; tf->srr1 |= PSL_VEC; - if (!(pcb->pcb_flags & PCB_VEC)) { + if (!(pcb->pcb_flags & PCB_VECREGS)) { memset(&pcb->pcb_vec, 0, sizeof pcb->pcb_vec); - pcb->pcb_flags |= PCB_VEC; + pcb->pcb_flags |= PCB_VECREGS; } /* @@ -170,3 +171,32 @@ if (td == PCPU_GET(vecthread)) save_vec_int(td); } + +void +enable_vec_kern(void) +{ + mtmsr(mfmsr() | PSL_VEC); +} + +void +disable_vec(struct thread *td) +{ + register_t msr; + struct pcb *pcb; + struct trapframe *tf; + + pcb = td->td_pcb; + tf = trapframe(td); + + /* Disable PSL_VEC in kernel (if enabled) */ + msr = mfmsr() & ~PSL_VEC; + isync(); + mtmsr(msr); + + /* + * Disable PSL_VEC in userspace. It will be re-enabled when + * an Altivec instruction is executed. + */ + tf->srr1 &= ~PSL_VEC; + pcb->pcb_flags &= ~PCB_VEC; +} diff --git a/sys/powerpc/powerpc/exec_machdep.c b/sys/powerpc/powerpc/exec_machdep.c --- a/sys/powerpc/powerpc/exec_machdep.c +++ b/sys/powerpc/powerpc/exec_machdep.c @@ -441,12 +441,14 @@ * Repeat for Altivec context */ - if (pcb->pcb_flags & PCB_VEC) { - KASSERT(td == curthread, - ("get_mcontext: fp save not curthread")); - critical_enter(); - save_vec(td); - critical_exit(); + if (pcb->pcb_flags & PCB_VECREGS) { + if (pcb->pcb_flags & PCB_VEC) { + KASSERT(td == curthread, + ("get_mcontext: altivec save not curthread")); + critical_enter(); + save_vec(td); + critical_exit(); + } mcp->mc_flags |= _MC_AV_VALID; mcp->mc_vscr = pcb->pcb_vec.vscr; mcp->mc_vrsave = pcb->pcb_vec.vrsave; @@ -543,11 +545,8 @@ } if (mcp->mc_flags & _MC_AV_VALID) { - if ((pcb->pcb_flags & PCB_VEC) != PCB_VEC) { - critical_enter(); - enable_vec(td); - critical_exit(); - } + /* enable_vec() will happen lazily on a fault */ + pcb->pcb_flags |= PCB_VECREGS; pcb->pcb_vec.vscr = mcp->mc_vscr; pcb->pcb_vec.vrsave = mcp->mc_vrsave; memcpy(pcb->pcb_vec.vr, mcp->mc_avec, sizeof(mcp->mc_avec)); diff --git a/sys/powerpc/powerpc/fpu.c b/sys/powerpc/powerpc/fpu.c --- a/sys/powerpc/powerpc/fpu.c +++ b/sys/powerpc/powerpc/fpu.c @@ -42,6 +42,7 @@ #include #include #include +#include static void save_fpu_int(struct thread *td) @@ -259,3 +260,127 @@ return ucode; } +void +enable_fpu_kern(void) +{ + register_t msr; + + msr = mfmsr() | PSL_FP; + + if (cpu_features & PPC_FEATURE_HAS_VSX) + msr |= PSL_VSX; + + mtmsr(msr); +} + +void +disable_fpu(struct thread *td) +{ + register_t msr; + struct pcb *pcb; + struct trapframe *tf; + + pcb = td->td_pcb; + tf = trapframe(td); + + /* Disable FPU in kernel (if enabled) */ + msr = mfmsr() & ~(PSL_FP | PSL_VSX); + isync(); + mtmsr(msr); + + /* + * Disable FPU in userspace. It will be re-enabled when + * an FP or VSX instruction is executed. + */ + tf->srr1 &= ~(PSL_FP | PSL_VSX); + pcb->pcb_flags &= ~(PCB_FPU | PCB_VSX); +} + +#ifndef __SPE__ +/* + * XXX: Implement fpu_kern_alloc_ctx/fpu_kern_free_ctx once fpu_kern_enter and + * fpu_kern_leave can handle !FPU_KERN_NOCTX. + */ +struct fpu_kern_ctx { +#define FPU_KERN_CTX_DUMMY 0x01 /* avoided save for the kern thread */ +#define FPU_KERN_CTX_INUSE 0x02 + uint32_t flags; +}; + +void +fpu_kern_enter(struct thread *td, struct fpu_kern_ctx *ctx, u_int flags) +{ + struct pcb *pcb; + + pcb = td->td_pcb; + + KASSERT((flags & FPU_KERN_NOCTX) != 0 || ctx != NULL, + ("ctx is required when !FPU_KERN_NOCTX")); + KASSERT(ctx == NULL || (ctx->flags & FPU_KERN_CTX_INUSE) == 0, + ("using inuse ctx")); + KASSERT((pcb->pcb_flags & PCB_KERN_FPU_NOSAVE) == 0, + ("recursive fpu_kern_enter while in PCB_KERN_FPU_NOSAVE state")); + + if ((flags & FPU_KERN_NOCTX) != 0) { + critical_enter(); + + if (pcb->pcb_flags & PCB_FPU) { + save_fpu(td); + pcb->pcb_flags |= PCB_FPREGS; + } + enable_fpu_kern(); + + if (pcb->pcb_flags & PCB_VEC) { + save_vec(td); + pcb->pcb_flags |= PCB_VECREGS; + } + enable_vec_kern(); + + pcb->pcb_flags |= PCB_KERN_FPU | PCB_KERN_FPU_NOSAVE; + return; + } + + KASSERT(0, ("fpu_kern_enter with !FPU_KERN_NOCTX not implemented!")); +} + +int +fpu_kern_leave(struct thread *td, struct fpu_kern_ctx *ctx) +{ + struct pcb *pcb; + + pcb = td->td_pcb; + + if ((pcb->pcb_flags & PCB_KERN_FPU_NOSAVE) != 0) { + KASSERT(ctx == NULL, ("non-null ctx after FPU_KERN_NOCTX")); + KASSERT(PCPU_GET(fpcurthread) == NULL, + ("non-NULL fpcurthread for PCB_FP_NOSAVE")); + CRITICAL_ASSERT(td); + + /* Disable FPU, VMX, and VSX */ + disable_fpu(td); + disable_vec(td); + + pcb->pcb_flags &= ~PCB_KERN_FPU_NOSAVE; + + critical_exit(); + } else { + KASSERT(0, ("fpu_kern_leave with !FPU_KERN_NOCTX not implemented!")); + } + + pcb->pcb_flags &= ~PCB_KERN_FPU; + + return 0; +} + +int +is_fpu_kern_thread(u_int flags __unused) +{ + struct pcb *curpcb; + + if ((curthread->td_pflags & TDP_KTHREAD) == 0) + return (0); + curpcb = curthread->td_pcb; + return ((curpcb->pcb_flags & PCB_KERN_FPU) != 0); +} + +#endif /* !__SPE__ */ diff --git a/sys/sys/param.h b/sys/sys/param.h --- a/sys/sys/param.h +++ b/sys/sys/param.h @@ -75,7 +75,7 @@ * cannot include sys/param.h and should only be updated here. */ #undef __FreeBSD_version -#define __FreeBSD_version 1500000 +#define __FreeBSD_version 1500001 /* * __FreeBSD_kernel__ indicates that this system uses the kernel of FreeBSD,