Changeset View
Changeset View
Standalone View
Standalone View
sys/i386/i386/npx.c
Show First 20 Lines • Show All 61 Lines • ▼ Show 20 Lines | |||||
#include <machine/frame.h> | #include <machine/frame.h> | ||||
#include <machine/md_var.h> | #include <machine/md_var.h> | ||||
#include <machine/pcb.h> | #include <machine/pcb.h> | ||||
#include <machine/psl.h> | #include <machine/psl.h> | ||||
#include <machine/resource.h> | #include <machine/resource.h> | ||||
#include <machine/specialreg.h> | #include <machine/specialreg.h> | ||||
#include <machine/segments.h> | #include <machine/segments.h> | ||||
#include <machine/ucontext.h> | #include <machine/ucontext.h> | ||||
#include <x86/ifunc.h> | |||||
#include <machine/intr_machdep.h> | #include <machine/intr_machdep.h> | ||||
#ifdef DEV_ISA | #ifdef DEV_ISA | ||||
#include <isa/isavar.h> | #include <isa/isavar.h> | ||||
#endif | #endif | ||||
/* | /* | ||||
▲ Show 20 Lines • Show All 100 Lines • ▼ Show 20 Lines | |||||
* Ensure the copy of XCR0 saved in a core is contained in the padding | * Ensure the copy of XCR0 saved in a core is contained in the padding | ||||
* area. | * area. | ||||
*/ | */ | ||||
CTASSERT(X86_XSTATE_XCR0_OFFSET >= offsetof(struct savexmm, sv_pad) && | CTASSERT(X86_XSTATE_XCR0_OFFSET >= offsetof(struct savexmm, sv_pad) && | ||||
X86_XSTATE_XCR0_OFFSET + sizeof(uint64_t) <= sizeof(struct savexmm)); | X86_XSTATE_XCR0_OFFSET + sizeof(uint64_t) <= sizeof(struct savexmm)); | ||||
static void fpu_clean_state(void); | static void fpu_clean_state(void); | ||||
static void fpusave(union savefpu *); | |||||
static void fpurstor(union savefpu *); | static void fpurstor(union savefpu *); | ||||
int hw_float; | int hw_float; | ||||
SYSCTL_INT(_hw, HW_FLOATINGPT, floatingpoint, CTLFLAG_RD, | SYSCTL_INT(_hw, HW_FLOATINGPT, floatingpoint, CTLFLAG_RD, | ||||
&hw_float, 0, "Floating point instructions executed in hardware"); | &hw_float, 0, "Floating point instructions executed in hardware"); | ||||
int use_xsave; | int use_xsave; | ||||
uint64_t xsave_mask; | uint64_t xsave_mask; | ||||
static uma_zone_t fpu_save_area_zone; | static uma_zone_t fpu_save_area_zone; | ||||
static union savefpu *npx_initialstate; | static union savefpu *npx_initialstate; | ||||
struct xsave_area_elm_descr { | struct xsave_area_elm_descr { | ||||
u_int offset; | u_int offset; | ||||
u_int size; | u_int size; | ||||
} *xsave_area_desc; | } *xsave_area_desc; | ||||
static int use_xsaveopt; | |||||
static volatile u_int npx_traps_while_probing; | static volatile u_int npx_traps_while_probing; | ||||
alias_for_inthand_t probetrap; | alias_for_inthand_t probetrap; | ||||
__asm(" \n\ | __asm(" \n\ | ||||
.text \n\ | .text \n\ | ||||
.p2align 2,0x90 \n\ | .p2align 2,0x90 \n\ | ||||
.type " __XSTRING(CNAME(probetrap)) ",@function \n\ | .type " __XSTRING(CNAME(probetrap)) ",@function \n\ | ||||
" __XSTRING(CNAME(probetrap)) ": \n\ | " __XSTRING(CNAME(probetrap)) ": \n\ | ||||
▲ Show 20 Lines • Show All 90 Lines • ▼ Show 20 Lines | #endif | ||||
__asm __volatile("smsw %%ax; orb %0,%%al; lmsw %%ax" : : | __asm __volatile("smsw %%ax; orb %0,%%al; lmsw %%ax" : : | ||||
"n" (CR0_EM | CR0_MP) : "ax"); | "n" (CR0_EM | CR0_MP) : "ax"); | ||||
cleanup: | cleanup: | ||||
idt[IDT_MF] = save_idt_npxtrap; | idt[IDT_MF] = save_idt_npxtrap; | ||||
return (hw_float); | return (hw_float); | ||||
} | } | ||||
static void | |||||
npxsave_xsaveopt(union savefpu *addr) | |||||
{ | |||||
xsaveopt((char *)addr, xsave_mask); | |||||
} | |||||
static void | |||||
fpusave_xsave(union savefpu *addr) | |||||
{ | |||||
xsave((char *)addr, xsave_mask); | |||||
} | |||||
static void | |||||
fpusave_fxsave(union savefpu *addr) | |||||
{ | |||||
fxsave((char *)addr); | |||||
} | |||||
static void | |||||
fpusave_fnsave(union savefpu *addr) | |||||
{ | |||||
fnsave((char *)addr); | |||||
} | |||||
static void | |||||
init_xsave(void) | |||||
{ | |||||
if (use_xsave) | |||||
return; | |||||
if (!cpu_fxsr || (cpu_feature2 & CPUID2_XSAVE) == 0) | |||||
return; | |||||
use_xsave = 1; | |||||
TUNABLE_INT_FETCH("hw.use_xsave", &use_xsave); | |||||
} | |||||
DEFINE_IFUNC(, void, npxsave_core, (union savefpu *), static) | |||||
{ | |||||
init_xsave(); | |||||
if (use_xsave) | |||||
return ((cpu_stdext_feature & CPUID_EXTSTATE_XSAVEOPT) != 0 ? | |||||
npxsave_xsaveopt : fpusave_xsave); | |||||
if (cpu_fxsr) | |||||
return (fpusave_fxsave); | |||||
return (fpusave_fnsave); | |||||
} | |||||
DEFINE_IFUNC(, void, fpusave, (union savefpu *), static) | |||||
{ | |||||
init_xsave(); | |||||
if (use_xsave) | |||||
return (fpusave_xsave); | |||||
if (cpu_fxsr) | |||||
return (fpusave_fxsave); | |||||
return (fpusave_fnsave); | |||||
} | |||||
/* | /* | ||||
* Enable XSAVE if supported and allowed by user. | * Enable XSAVE if supported and allowed by user. | ||||
* Calculate the xsave_mask. | * Calculate the xsave_mask. | ||||
*/ | */ | ||||
static void | static void | ||||
npxinit_bsp1(void) | npxinit_bsp1(void) | ||||
{ | { | ||||
u_int cp[4]; | u_int cp[4]; | ||||
uint64_t xsave_mask_user; | uint64_t xsave_mask_user; | ||||
if (cpu_fxsr && (cpu_feature2 & CPUID2_XSAVE) != 0) { | |||||
use_xsave = 1; | |||||
TUNABLE_INT_FETCH("hw.use_xsave", &use_xsave); | |||||
} | |||||
if (!use_xsave) | if (!use_xsave) | ||||
return; | return; | ||||
cpuid_count(0xd, 0x0, cp); | cpuid_count(0xd, 0x0, cp); | ||||
xsave_mask = XFEATURE_ENABLED_X87 | XFEATURE_ENABLED_SSE; | xsave_mask = XFEATURE_ENABLED_X87 | XFEATURE_ENABLED_SSE; | ||||
if ((cp[0] & xsave_mask) != xsave_mask) | if ((cp[0] & xsave_mask) != xsave_mask) | ||||
panic("CPU0 does not support X87 or SSE: %x", cp[0]); | panic("CPU0 does not support X87 or SSE: %x", cp[0]); | ||||
xsave_mask = ((uint64_t)cp[3] << 32) | cp[0]; | xsave_mask = ((uint64_t)cp[3] << 32) | cp[0]; | ||||
xsave_mask_user = xsave_mask; | xsave_mask_user = xsave_mask; | ||||
TUNABLE_QUAD_FETCH("hw.xsave_mask", &xsave_mask_user); | TUNABLE_QUAD_FETCH("hw.xsave_mask", &xsave_mask_user); | ||||
xsave_mask_user |= XFEATURE_ENABLED_X87 | XFEATURE_ENABLED_SSE; | xsave_mask_user |= XFEATURE_ENABLED_X87 | XFEATURE_ENABLED_SSE; | ||||
xsave_mask &= xsave_mask_user; | xsave_mask &= xsave_mask_user; | ||||
if ((xsave_mask & XFEATURE_AVX512) != XFEATURE_AVX512) | if ((xsave_mask & XFEATURE_AVX512) != XFEATURE_AVX512) | ||||
xsave_mask &= ~XFEATURE_AVX512; | xsave_mask &= ~XFEATURE_AVX512; | ||||
if ((xsave_mask & XFEATURE_MPX) != XFEATURE_MPX) | if ((xsave_mask & XFEATURE_MPX) != XFEATURE_MPX) | ||||
xsave_mask &= ~XFEATURE_MPX; | xsave_mask &= ~XFEATURE_MPX; | ||||
cpuid_count(0xd, 0x1, cp); | |||||
if ((cp[0] & CPUID_EXTSTATE_XSAVEOPT) != 0) | |||||
use_xsaveopt = 1; | |||||
} | } | ||||
/* | /* | ||||
* Calculate the fpu save area size. | * Calculate the fpu save area size. | ||||
*/ | */ | ||||
static void | static void | ||||
npxinit_bsp2(void) | npxinit_bsp2(void) | ||||
{ | { | ||||
u_int cp[4]; | u_int cp[4]; | ||||
if (use_xsave) { | if (use_xsave) { | ||||
▲ Show 20 Lines • Show All 489 Lines • ▼ Show 20 Lines | |||||
* Wrapper for fpusave() called from context switch routines. | * Wrapper for fpusave() called from context switch routines. | ||||
* | * | ||||
* npxsave() must be called with interrupts disabled, so that it clears | * npxsave() must be called with interrupts disabled, so that it clears | ||||
* fpcurthread atomically with saving the state. We require callers to do the | * fpcurthread atomically with saving the state. We require callers to do the | ||||
* disabling, since most callers need to disable interrupts anyway to call | * disabling, since most callers need to disable interrupts anyway to call | ||||
* npxsave() atomically with checking fpcurthread. | * npxsave() atomically with checking fpcurthread. | ||||
*/ | */ | ||||
void | void | ||||
npxsave(addr) | npxsave(union savefpu *addr) | ||||
union savefpu *addr; | |||||
{ | { | ||||
stop_emulating(); | stop_emulating(); | ||||
if (use_xsaveopt) | npxsave_core(addr); | ||||
xsaveopt((char *)addr, xsave_mask); | |||||
else | |||||
fpusave(addr); | |||||
start_emulating(); | start_emulating(); | ||||
PCPU_SET(fpcurthread, NULL); | PCPU_SET(fpcurthread, NULL); | ||||
} | } | ||||
/* | /* | ||||
* Unconditionally save the current co-processor state across suspend and | * Unconditionally save the current co-processor state across suspend and | ||||
* resume. | * resume. | ||||
*/ | */ | ||||
▲ Show 20 Lines • Show All 193 Lines • ▼ Show 20 Lines | if (td == PCPU_GET(fpcurthread) && PCB_USER_FPU(pcb)) { | ||||
critical_exit(); | critical_exit(); | ||||
error = npxsetxstate(td, xfpustate, xfpustate_size); | error = npxsetxstate(td, xfpustate, xfpustate_size); | ||||
if (error != 0) | if (error != 0) | ||||
return (error); | return (error); | ||||
bcopy(addr, get_pcb_user_save_td(td), sizeof(*addr)); | bcopy(addr, get_pcb_user_save_td(td), sizeof(*addr)); | ||||
npxuserinited(td); | npxuserinited(td); | ||||
} | } | ||||
return (0); | return (0); | ||||
} | |||||
static void | |||||
fpusave(addr) | |||||
union savefpu *addr; | |||||
{ | |||||
if (use_xsave) | |||||
xsave((char *)addr, xsave_mask); | |||||
else if (cpu_fxsr) | |||||
fxsave(addr); | |||||
else | |||||
fnsave(addr); | |||||
} | } | ||||
static void | static void | ||||
npx_fill_fpregs_xmm1(struct savexmm *sv_xmm, struct save87 *sv_87) | npx_fill_fpregs_xmm1(struct savexmm *sv_xmm, struct save87 *sv_87) | ||||
{ | { | ||||
struct env87 *penv_87; | struct env87 *penv_87; | ||||
struct envxmm *penv_xmm; | struct envxmm *penv_xmm; | ||||
int i; | int i; | ||||
▲ Show 20 Lines • Show All 337 Lines • Show Last 20 Lines |