Changeset View
Changeset View
Standalone View
Standalone View
sys/amd64/amd64/fpu.c
Show First 20 Lines • Show All 41 Lines • ▼ Show 20 Lines | |||||
#include <sys/kernel.h> | #include <sys/kernel.h> | ||||
#include <sys/lock.h> | #include <sys/lock.h> | ||||
#include <sys/malloc.h> | #include <sys/malloc.h> | ||||
#include <sys/module.h> | #include <sys/module.h> | ||||
#include <sys/mutex.h> | #include <sys/mutex.h> | ||||
#include <sys/mutex.h> | #include <sys/mutex.h> | ||||
#include <sys/proc.h> | #include <sys/proc.h> | ||||
#include <sys/sysctl.h> | #include <sys/sysctl.h> | ||||
#include <sys/sysent.h> | |||||
#include <machine/bus.h> | #include <machine/bus.h> | ||||
#include <sys/rman.h> | #include <sys/rman.h> | ||||
#include <sys/signalvar.h> | #include <sys/signalvar.h> | ||||
#include <vm/uma.h> | #include <vm/uma.h> | ||||
#include <machine/cputypes.h> | #include <machine/cputypes.h> | ||||
#include <machine/frame.h> | #include <machine/frame.h> | ||||
#include <machine/intr_machdep.h> | #include <machine/intr_machdep.h> | ||||
Show All 18 Lines | |||||
#define fnstcw(addr) __asm __volatile("fnstcw %0" : "=m" (*(addr))) | #define fnstcw(addr) __asm __volatile("fnstcw %0" : "=m" (*(addr))) | ||||
#define fnstsw(addr) __asm __volatile("fnstsw %0" : "=am" (*(addr))) | #define fnstsw(addr) __asm __volatile("fnstsw %0" : "=am" (*(addr))) | ||||
#define fxrstor(addr) __asm __volatile("fxrstor %0" : : "m" (*(addr))) | #define fxrstor(addr) __asm __volatile("fxrstor %0" : : "m" (*(addr))) | ||||
#define fxsave(addr) __asm __volatile("fxsave %0" : "=m" (*(addr))) | #define fxsave(addr) __asm __volatile("fxsave %0" : "=m" (*(addr))) | ||||
#define ldmxcsr(csr) __asm __volatile("ldmxcsr %0" : : "m" (csr)) | #define ldmxcsr(csr) __asm __volatile("ldmxcsr %0" : : "m" (csr)) | ||||
#define stmxcsr(addr) __asm __volatile("stmxcsr %0" : : "m" (*(addr))) | #define stmxcsr(addr) __asm __volatile("stmxcsr %0" : : "m" (*(addr))) | ||||
static __inline void | static __inline void | ||||
xrstor(char *addr, uint64_t mask) | xrstor32(char *addr, uint64_t mask) | ||||
{ | { | ||||
uint32_t low, hi; | uint32_t low, hi; | ||||
low = mask; | low = mask; | ||||
hi = mask >> 32; | hi = mask >> 32; | ||||
__asm __volatile("xrstor %0" : : "m" (*addr), "a" (low), "d" (hi)); | __asm __volatile("xrstor %0" : : "m" (*addr), "a" (low), "d" (hi)); | ||||
} | } | ||||
static __inline void | static __inline void | ||||
xsave(char *addr, uint64_t mask) | xrstor64(char *addr, uint64_t mask) | ||||
{ | { | ||||
uint32_t low, hi; | uint32_t low, hi; | ||||
low = mask; | low = mask; | ||||
hi = mask >> 32; | hi = mask >> 32; | ||||
__asm __volatile("xrstor64 %0" : : "m" (*addr), "a" (low), "d" (hi)); | |||||
} | |||||
static __inline void | |||||
xsave32(char *addr, uint64_t mask) | |||||
{ | |||||
uint32_t low, hi; | |||||
low = mask; | |||||
hi = mask >> 32; | |||||
__asm __volatile("xsave %0" : "=m" (*addr) : "a" (low), "d" (hi) : | __asm __volatile("xsave %0" : "=m" (*addr) : "a" (low), "d" (hi) : | ||||
"memory"); | "memory"); | ||||
} | } | ||||
static __inline void | static __inline void | ||||
xsaveopt(char *addr, uint64_t mask) | xsave64(char *addr, uint64_t mask) | ||||
{ | { | ||||
uint32_t low, hi; | uint32_t low, hi; | ||||
low = mask; | low = mask; | ||||
hi = mask >> 32; | hi = mask >> 32; | ||||
__asm __volatile("xsave64 %0" : "=m" (*addr) : "a" (low), "d" (hi) : | |||||
"memory"); | |||||
} | |||||
static __inline void | |||||
xsaveopt32(char *addr, uint64_t mask) | |||||
{ | |||||
uint32_t low, hi; | |||||
low = mask; | |||||
hi = mask >> 32; | |||||
__asm __volatile("xsaveopt %0" : "=m" (*addr) : "a" (low), "d" (hi) : | __asm __volatile("xsaveopt %0" : "=m" (*addr) : "a" (low), "d" (hi) : | ||||
"memory"); | "memory"); | ||||
} | } | ||||
static __inline void | |||||
xsaveopt64(char *addr, uint64_t mask) | |||||
{ | |||||
uint32_t low, hi; | |||||
low = mask; | |||||
hi = mask >> 32; | |||||
__asm __volatile("xsaveopt64 %0" : "=m" (*addr) : "a" (low), "d" (hi) : | |||||
"memory"); | |||||
} | |||||
#else /* !(__GNUCLIKE_ASM && !lint) */ | #else /* !(__GNUCLIKE_ASM && !lint) */ | ||||
void fldcw(u_short cw); | void fldcw(u_short cw); | ||||
void fnclex(void); | void fnclex(void); | ||||
void fninit(void); | void fninit(void); | ||||
void fnstcw(caddr_t addr); | void fnstcw(caddr_t addr); | ||||
void fnstsw(caddr_t addr); | void fnstsw(caddr_t addr); | ||||
void fxsave(caddr_t addr); | void fxsave(caddr_t addr); | ||||
void fxrstor(caddr_t addr); | void fxrstor(caddr_t addr); | ||||
void ldmxcsr(u_int csr); | void ldmxcsr(u_int csr); | ||||
void stmxcsr(u_int *csr); | void stmxcsr(u_int *csr); | ||||
void xrstor(char *addr, uint64_t mask); | void xrstor32(char *addr, uint64_t mask); | ||||
void xsave(char *addr, uint64_t mask); | void xrstor64(char *addr, uint64_t mask); | ||||
void xsaveopt(char *addr, uint64_t mask); | void xsave32(char *addr, uint64_t mask); | ||||
void xsave64(char *addr, uint64_t mask); | |||||
void xsaveopt32(char *addr, uint64_t mask); | |||||
void xsaveopt64(char *addr, uint64_t mask); | |||||
#endif /* __GNUCLIKE_ASM && !lint */ | #endif /* __GNUCLIKE_ASM && !lint */ | ||||
#define start_emulating() load_cr0(rcr0() | CR0_TS) | #define start_emulating() load_cr0(rcr0() | CR0_TS) | ||||
#define stop_emulating() clts() | #define stop_emulating() clts() | ||||
CTASSERT(sizeof(struct savefpu) == 512); | CTASSERT(sizeof(struct savefpu) == 512); | ||||
CTASSERT(sizeof(struct xstate_hdr) == 64); | CTASSERT(sizeof(struct xstate_hdr) == 64); | ||||
Show All 24 Lines | |||||
static struct savefpu *fpu_initialstate; | static struct savefpu *fpu_initialstate; | ||||
static struct xsave_area_elm_descr { | static struct xsave_area_elm_descr { | ||||
u_int offset; | u_int offset; | ||||
u_int size; | u_int size; | ||||
} *xsave_area_desc; | } *xsave_area_desc; | ||||
static void | static void | ||||
fpusave_xsaveopt(void *addr) | fpusave_xsaveopt64(void *addr) | ||||
{ | { | ||||
xsaveopt64((char *)addr, xsave_mask); | |||||
} | |||||
xsaveopt((char *)addr, xsave_mask); | static void | ||||
fpusave_xsaveopt3264(void *addr) | |||||
{ | |||||
if (SV_CURPROC_FLAG(SV_ILP32)) | |||||
xsaveopt32((char *)addr, xsave_mask); | |||||
else | |||||
xsaveopt64((char *)addr, xsave_mask); | |||||
} | } | ||||
static void | static void | ||||
fpusave_xsave(void *addr) | fpusave_xsave64(void *addr) | ||||
{ | { | ||||
xsave64((char *)addr, xsave_mask); | |||||
} | |||||
xsave((char *)addr, xsave_mask); | static void | ||||
fpusave_xsave3264(void *addr) | |||||
{ | |||||
if (SV_CURPROC_FLAG(SV_ILP32)) | |||||
xsave32((char *)addr, xsave_mask); | |||||
else | |||||
xsave64((char *)addr, xsave_mask); | |||||
} | } | ||||
static void | static void | ||||
fpurestore_xrstor(void *addr) | fpurestore_xrstor64(void *addr) | ||||
{ | { | ||||
xrstor64((char *)addr, xsave_mask); | |||||
} | |||||
xrstor((char *)addr, xsave_mask); | static void | ||||
fpurestore_xrstor3264(void *addr) | |||||
{ | |||||
if (SV_CURPROC_FLAG(SV_ILP32)) | |||||
xrstor32((char *)addr, xsave_mask); | |||||
else | |||||
xrstor64((char *)addr, xsave_mask); | |||||
} | } | ||||
static void | static void | ||||
fpusave_fxsave(void *addr) | fpusave_fxsave(void *addr) | ||||
{ | { | ||||
fxsave((char *)addr); | fxsave((char *)addr); | ||||
} | } | ||||
Show All 16 Lines | init_xsave(void) | ||||
use_xsave = 1; | use_xsave = 1; | ||||
TUNABLE_INT_FETCH("hw.use_xsave", &use_xsave); | TUNABLE_INT_FETCH("hw.use_xsave", &use_xsave); | ||||
} | } | ||||
DEFINE_IFUNC(, void, fpusave, (void *)) | DEFINE_IFUNC(, void, fpusave, (void *)) | ||||
{ | { | ||||
init_xsave(); | init_xsave(); | ||||
if (use_xsave) | if (!use_xsave) | ||||
return ((cpu_stdext_feature & CPUID_EXTSTATE_XSAVEOPT) != 0 ? | |||||
fpusave_xsaveopt : fpusave_xsave); | |||||
return (fpusave_fxsave); | return (fpusave_fxsave); | ||||
if ((cpu_stdext_feature & CPUID_EXTSTATE_XSAVEOPT) != 0) { | |||||
return ((cpu_stdext_feature & CPUID_STDEXT_NFPUSG) != 0 ? | |||||
fpusave_xsaveopt64 : fpusave_xsaveopt3264); | |||||
} | } | ||||
return ((cpu_stdext_feature & CPUID_STDEXT_NFPUSG) != 0 ? | |||||
fpusave_xsave64 : fpusave_xsave3264); | |||||
} | |||||
DEFINE_IFUNC(, void, fpurestore, (void *)) | DEFINE_IFUNC(, void, fpurestore, (void *)) | ||||
{ | { | ||||
init_xsave(); | init_xsave(); | ||||
return (use_xsave ? fpurestore_xrstor : fpurestore_fxrstor); | if (!use_xsave) | ||||
return (fpurestore_fxrstor); | |||||
return ((cpu_stdext_feature & CPUID_STDEXT_NFPUSG) != 0 ? | |||||
fpurestore_xrstor64 : fpurestore_xrstor3264); | |||||
} | } | ||||
void | void | ||||
fpususpend(void *addr) | fpususpend(void *addr) | ||||
{ | { | ||||
u_long cr0; | u_long cr0; | ||||
cr0 = rcr0(); | cr0 = rcr0(); | ||||
▲ Show 20 Lines • Show All 50 Lines • ▼ Show 20 Lines | if ((cp[0] & CPUID_EXTSTATE_XSAVEOPT) != 0) { | ||||
* to XSAVEOPT. We assume that XSAVE encoding used | * to XSAVEOPT. We assume that XSAVE encoding used | ||||
* REX byte, and set the bit 4 of the r/m byte. | * REX byte, and set the bit 4 of the r/m byte. | ||||
* | * | ||||
* It seems that some BIOSes give control to the OS | * It seems that some BIOSes give control to the OS | ||||
* with CR0.WP already set, making the kernel text | * with CR0.WP already set, making the kernel text | ||||
* read-only before cpu_startup(). | * read-only before cpu_startup(). | ||||
*/ | */ | ||||
old_wp = disable_wp(); | old_wp = disable_wp(); | ||||
ctx_switch_xsave32[3] |= 0x10; | |||||
ctx_switch_xsave[3] |= 0x10; | ctx_switch_xsave[3] |= 0x10; | ||||
restore_wp(old_wp); | restore_wp(old_wp); | ||||
} | } | ||||
} | } | ||||
/* | /* | ||||
* Calculate the fpu save area size. | * Calculate the fpu save area size. | ||||
*/ | */ | ||||
▲ Show 20 Lines • Show All 921 Lines • Show Last 20 Lines |