diff --git a/sys/x86/include/fpu.h b/sys/x86/include/fpu.h --- a/sys/x86/include/fpu.h +++ b/sys/x86/include/fpu.h @@ -233,6 +233,12 @@ */ #define fpu_enable() clts() #define fpu_disable() load_cr0(rcr0() | CR0_TS) + +bool xsave_extfeature_supported(uint64_t feature, bool supervisor); +bool xsave_extension_supported(uint64_t extension); +size_t xsave_area_hdr_offset(void); +size_t xsave_area_offset(uint64_t xstate_bv, uint64_t feature, bool compact); +size_t xsave_area_size(uint64_t xstate_bv, bool compact); #endif #endif /* !_X86_FPU_H_ */ diff --git a/sys/x86/include/specialreg.h b/sys/x86/include/specialreg.h --- a/sys/x86/include/specialreg.h +++ b/sys/x86/include/specialreg.h @@ -370,6 +370,14 @@ #define CPUID_EXTSTATE_XINUSE 0x00000004 #define CPUID_EXTSTATE_XSAVES 0x00000008 +/* + * CPUID instruction 0xd Processor Extended State Enumeration + * Sub-leaf > 1 ecx info + */ +#define CPUID_EXTSTATE_SUPERVISOR 0x00000001 +#define CPUID_EXTSTATE_ALIGNED 0x00000002 +#define CPUID_EXTSTATE_XFD_SUPPORTED 0x00000004 + /* * AMD extended function 8000_0007h ebx info */ diff --git a/sys/x86/x86/cpu_machdep.c b/sys/x86/x86/cpu_machdep.c --- a/sys/x86/x86/cpu_machdep.c +++ b/sys/x86/x86/cpu_machdep.c @@ -1733,6 +1733,167 @@ #endif } +#define EXTSTATE_INFO_LEN (ilog2_const(XFEATURE_ENABLED_TILEDATA) + 1) +struct extstate_info { + size_t offset; + size_t size; + uint64_t flags; +}; +static struct extstate_info extstate_info[EXTSTATE_INFO_LEN] __read_mostly; +static uint64_t extstate_extensions; +static uint64_t extfeatures_user; +static uint64_t extfeatures_supervisor; + +static __inline bool +cpu_xsave_supported(void) +{ + return ((cpu_feature2 & (CPUID2_XSAVE | CPUID2_OSXSAVE)) == + (CPUID2_XSAVE | CPUID2_OSXSAVE)); +} + +static void +cpu_extstate_info_init(void *arg __unused) +{ + int i; + uint32_t regs[4]; + struct extstate_info *fip; + + if (!cpu_xsave_supported()) + return; + + cpuid_count(0xd, 0, regs); + extfeatures_user = regs[0] | ((uint64_t)regs[3] << 32); + + cpuid_count(0xd, 1, regs); + extstate_extensions = regs[0]; + extfeatures_supervisor = regs[2] | ((uint64_t)regs[3] << 32); + + for (i = 0; i < EXTSTATE_INFO_LEN; i++) { + fip = &extstate_info[i]; + cpuid_count(0xd, i, regs); + fip->size = regs[0]; + fip->flags = regs[2]; + fip->offset = (fip->flags & CPUID_EXTSTATE_SUPERVISOR) ? + -1 : regs[1]; + } +} +SYSINIT(extstate_info_init, SI_SUB_CPU, SI_ORDER_MIDDLE, + cpu_extstate_info_init, NULL); + +static __inline void +cpu_extfeature_check(uint64_t feature) +{ + + KASSERT((feature & (feature - 1)) == 0, + ("%s: invalid XFEATURE 0x%lx", __func__, feature)); + KASSERT(feature <= XFEATURE_ENABLED_TILEDATA, + ("%s: unknown XFEATURE 0x%lx", __func__, feature)); +} + +static __inline void +cpu_extstate_bv_check(uint64_t xstate_bv) +{ + + KASSERT(xstate_bv != 0 && + fls(xstate_bv) - 1 <= XFEATURE_ENABLED_TILEDATA, + ("%s: invalid XSTATE_BV 0x%lx", __func__, xstate_bv)); +} + +/* + * Returns whether the XFEATURE 'feature' is supported as a user state + * or supervisor state component. + * + * Always returns false if XSAVE is not supported. + */ +bool +xsave_extfeature_supported(uint64_t feature, bool supervisor) +{ + if (!cpu_xsave_supported()) + return (false); + + cpu_extfeature_check(feature); + if (supervisor) + return ((extfeatures_supervisor & feature) != 0); + else + return ((extfeatures_user & feature) != 0); +} + +/* + * Returns whether the XFEATURE 'feature' is supported as a user state + * or supervisor state component. + * + * Always returns false if XSAVE is not supported. + */ +bool +xsave_extension_supported(uint64_t extension) +{ + if (!cpu_xsave_supported()) + return (false); + + return ((extstate_extensions & extension) != 0); +} + +/* + * Returns offset for XFEATURE 'feature' given the requested feature bitmap + * 'xstate_bv', and extended region format ('compact'). + * + * Returns -1 if XSAVE is not supported. + */ +size_t +xsave_area_offset(uint64_t xstate_bv, uint64_t feature, + bool compact) +{ + int i, idx; + size_t offs; + struct extstate_info *xip; + + if (!cpu_xsave_supported()) + return (-1); + cpu_extstate_bv_check(xstate_bv); + cpu_extfeature_check(feature); + idx = ilog2_const(feature); + + if (!compact) + return (extstate_info[idx].offset); + offs = sizeof(struct savefpu) + sizeof(struct xstate_hdr); + xstate_bv &= ~(XFEATURE_ENABLED_X87 | XFEATURE_ENABLED_SSE); + while ((i = ffs(xstate_bv) - 1) > 0 && i < idx) { + xip = &extstate_info[i]; + if (xip->flags & CPUID_EXTSTATE_ALIGNED) + offs = roundup2(offs, 64); + offs += xip->size; + xstate_bv &= ~(1 << i); + } + + return (offs); +} + +/* + * Returns the XSAVE area size for the requested feature bitmap + * 'xstate_bv' and extended region format ('compact'). + * + * Returns 0 if XSAVE is not supported. + */ +size_t +xsave_area_size(uint64_t xstate_bv, bool compact) +{ + int last_idx; + + if (!cpu_xsave_supported()) + return (0); + cpu_extstate_bv_check(xstate_bv); + last_idx = fls(xstate_bv) - 1; + + return (xsave_area_offset(xstate_bv, 1 << last_idx, compact) + + extstate_info[last_idx].size); +} + +size_t +xsave_area_hdr_offset(void) +{ + return (sizeof(struct savefpu)); +} + DEFINE_IFUNC(, uint64_t, rdtsc_ordered, (void)) { bool cpu_is_amd = cpu_vendor_id == CPU_VENDOR_AMD ||