Index: sys/amd64/include/cpufunc.h =================================================================== --- sys/amd64/include/cpufunc.h +++ sys/amd64/include/cpufunc.h @@ -355,6 +355,20 @@ return (low | ((uint64_t)high << 32)); } +static __inline uint64_t +rdtsc_ordered_lfence(void) +{ + lfence(); + return (rdtsc()); +} + +static __inline uint64_t +rdtsc_ordered_mfence(void) +{ + mfence(); + return (rdtsc()); +} + static __inline uint64_t rdtscp(void) { @@ -1017,6 +1031,10 @@ #endif /* __GNUCLIKE_ASM && __CC_SUPPORTS___INLINE */ +#ifdef _KERNEL +uint64_t rdtsc_ordered(void); +#endif /* _KERNEL */ + void reset_dbregs(void); #ifdef _KERNEL Index: sys/conf/files.x86 =================================================================== --- sys/conf/files.x86 +++ sys/conf/files.x86 @@ -319,6 +319,7 @@ x86/x86/mp_watchdog.c optional mp_watchdog smp x86/x86/nexus.c standard x86/x86/pvclock.c standard +x86/x86/rdtsc_ordered.c standard x86/x86/stack_machdep.c optional ddb | stack x86/x86/tsc.c standard x86/x86/ucode.c standard Index: sys/i386/include/cpufunc.h =================================================================== --- sys/i386/include/cpufunc.h +++ sys/i386/include/cpufunc.h @@ -394,6 +394,20 @@ return (rv); } +static __inline uint64_t +rdtsc_ordered_lfence(void) +{ + lfence(); + return (rdtsc()); +} + +static __inline uint64_t +rdtsc_ordered_mfence(void) +{ + mfence(); + return (rdtsc()); +} + static __inline uint64_t rdtscp(void) { @@ -832,6 +846,10 @@ #endif /* __GNUCLIKE_ASM && __CC_SUPPORTS___INLINE */ +#ifdef _KERNEL +uint64_t rdtsc_ordered(void); +#endif /* _KERNEL */ + void reset_dbregs(void); #ifdef _KERNEL Index: sys/x86/x86/rdtsc_ordered.c =================================================================== --- /dev/null +++ sys/x86/x86/rdtsc_ordered.c @@ -0,0 +1,54 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause-FreeBSD + * + * Copyright (c) 2021 Juniper Networks, Inc. + * Copyright (c) 2021 Klara, Inc. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include +__FBSDID("$FreeBSD$"); + +#include +#include + +#include +#include +#include +#include + +#include + +DEFINE_IFUNC(, uint64_t, rdtsc_ordered, (void)) +{ + bool cpu_is_amd = cpu_vendor_id == CPU_VENDOR_AMD || + cpu_vendor_id == CPU_VENDOR_HYGON; + + if ((amd_feature & AMDID_RDTSCP) != 0) + return (rdtscp); + else if ((cpu_feature & CPUID_SSE2) != 0) + return (cpu_is_amd ? rdtsc_ordered_mfence : + rdtsc_ordered_lfence); + else + return (rdtsc); +}