diff --git a/sys/amd64/include/cpufunc.h b/sys/amd64/include/cpufunc.h --- a/sys/amd64/include/cpufunc.h +++ b/sys/amd64/include/cpufunc.h @@ -355,6 +355,20 @@ return (low | ((uint64_t)high << 32)); } +static __inline uint64_t +rdtsc_ordered_lfence(void) +{ + lfence(); + return (rdtsc()); +} + +static __inline uint64_t +rdtsc_ordered_mfence(void) +{ + mfence(); + return (rdtsc()); +} + static __inline uint64_t rdtscp(void) { diff --git a/sys/i386/include/cpufunc.h b/sys/i386/include/cpufunc.h --- a/sys/i386/include/cpufunc.h +++ b/sys/i386/include/cpufunc.h @@ -394,6 +394,20 @@ return (rv); } +static __inline uint64_t +rdtsc_ordered_lfence(void) +{ + lfence(); + return (rdtsc()); +} + +static __inline uint64_t +rdtsc_ordered_mfence(void) +{ + mfence(); + return (rdtsc()); +} + static __inline uint64_t rdtscp(void) { diff --git a/sys/x86/include/x86_var.h b/sys/x86/include/x86_var.h --- a/sys/x86/include/x86_var.h +++ b/sys/x86/include/x86_var.h @@ -154,6 +154,7 @@ int minidumpsys(struct dumperinfo *); struct pcb *get_pcb_td(struct thread *td); void x86_set_fork_retval(struct thread *td); +uint64_t rdtsc_ordered(void); /* * MSR ops for x86_msr_op() diff --git a/sys/x86/x86/cpu_machdep.c b/sys/x86/x86/cpu_machdep.c --- a/sys/x86/x86/cpu_machdep.c +++ b/sys/x86/x86/cpu_machdep.c @@ -76,6 +76,7 @@ #include #include +#include #include #include #include @@ -88,6 +89,7 @@ #include #endif #include +#include #include #include @@ -1508,3 +1510,17 @@ return (false); #endif } + +DEFINE_IFUNC(, uint64_t, rdtsc_ordered, (void)) +{ + bool cpu_is_amd = cpu_vendor_id == CPU_VENDOR_AMD || + cpu_vendor_id == CPU_VENDOR_HYGON; + + if ((amd_feature & AMDID_RDTSCP) != 0) + return (rdtscp); + else if ((cpu_feature & CPUID_SSE2) != 0) + return (cpu_is_amd ? rdtsc_ordered_mfence : + rdtsc_ordered_lfence); + else + return (rdtsc); +}