Page MenuHomeFreeBSD
Files F142172518

D27986.id81971.diff
No OneTemporary
Actions

D27986.id81971.diff
View Options

diff --git a/lib/libc/x86/sys/__vdso_gettc.c b/lib/libc/x86/sys/__vdso_gettc.c
--- a/lib/libc/x86/sys/__vdso_gettc.c
+++ b/lib/libc/x86/sys/__vdso_gettc.c
@@ -53,57 +53,151 @@
#include <x86/ifunc.h>
#include "libc_private.h"
-static void
-rdtsc_mb_lfence(void)
+static inline u_int
+rdtsc_low(const struct vdso_timehands *th)
{
+ u_int rv;
- lfence();
+ __asm __volatile("rdtsc; shrd %%cl, %%edx, %0"
+ : "=a" (rv) : "c" (th->th_x86_shift) : "edx");
+ return (rv);
}
-static void
-rdtsc_mb_mfence(void)
+static inline u_int
+rdtscp_low(const struct vdso_timehands *th)
{
+ u_int rv;
- mfence();
+ __asm __volatile("rdtscp; movl %%edi,%%ecx; shrd %%cl, %%edx, %0"
+ : "=a" (rv) : "D" (th->th_x86_shift) : "ecx", "edx");
+ return (rv);
}
-static void
-rdtsc_mb_none(void)
+static u_int
+rdtsc_low_mb_lfence(const struct vdso_timehands *th)
{
+ lfence();
+ return (rdtsc_low(th));
}
-DEFINE_UIFUNC(static, void, rdtsc_mb, (void))
+static u_int
+rdtsc_low_mb_mfence(const struct vdso_timehands *th)
{
- u_int p[4];
- /* Not a typo, string matches our do_cpuid() registers use. */
- static const char intel_id[] = "GenuntelineI";
-
- if ((cpu_feature & CPUID_SSE2) == 0)
- return (rdtsc_mb_none);
- do_cpuid(0, p);
- return (memcmp(p + 1, intel_id, sizeof(intel_id) - 1) == 0 ?
- rdtsc_mb_lfence : rdtsc_mb_mfence);
+ mfence();
+ return (rdtsc_low(th));
}
static u_int
-__vdso_gettc_rdtsc_low(const struct vdso_timehands *th)
+rdtsc_low_mb_none(const struct vdso_timehands *th)
{
- u_int rv;
+ return (rdtsc_low(th));
+}
- rdtsc_mb();
- __asm __volatile("rdtsc; shrd %%cl, %%edx, %0"
- : "=a" (rv) : "c" (th->th_x86_shift) : "edx");
- return (rv);
+static u_int
+rdtsc32_mb_lfence(void)
+{
+ lfence();
+ return (rdtsc32());
}
static u_int
-__vdso_rdtsc32(void)
+rdtsc32_mb_mfence(void)
{
+ mfence();
+ return (rdtsc32());
+}
- rdtsc_mb();
+static u_int
+rdtsc32_mb_none(void)
+{
return (rdtsc32());
}
+static u_int
+rdtscp32_(void)
+{
+ return (rdtscp32());
+}
+
+struct tsc_selector_tag {
+ u_int (*ts_rdtsc32)(void);
+ u_int (*ts_rdtsc_low)(const struct vdso_timehands *);
+};
+
+static const struct tsc_selector_tag tsc_selector[] = {
+ [0] = { /* Intel or AMD Zen+, LFENCE */
+ .ts_rdtsc32 = rdtsc32_mb_lfence,
+ .ts_rdtsc_low = rdtsc_low_mb_lfence,
+ },
+ [1] = { /* AMD, MFENCE */
+ .ts_rdtsc32 = rdtsc32_mb_mfence,
+ .ts_rdtsc_low = rdtsc_low_mb_mfence,
+ },
+ [2] = { /* No SSE2 */
+ .ts_rdtsc32 = rdtsc32_mb_none,
+ .ts_rdtsc_low = rdtsc_low_mb_none,
+ },
+ [3] = { /* RDTSCP */
+ .ts_rdtsc32 = rdtscp32_,
+ .ts_rdtsc_low = rdtscp_low,
+ },
+};
+
+static int
+tsc_selector_idx(u_int cpu_feature)
+{
+ u_int amd_feature, cpu_exthigh, cpu_id, p[4], v[3];
+ static const char amd_id[] = "AuthenticAMD";
+ static const char hygon_id[] = "HygonGenuine";
+ bool amd_cpu;
+
+ if (cpu_feature == 0)
+ return (2); /* should not happen due to RDTSC */
+
+ do_cpuid(0, p);
+ v[0] = p[1];
+ v[1] = p[3];
+ v[2] = p[2];
+ amd_cpu = memcmp(v, amd_id, sizeof(amd_id) - 1) == 0 ||
+ memcmp(v, hygon_id, sizeof(hygon_id) - 1) == 0;
+
+ do_cpuid(1, p);
+ cpu_id = p[0];
+
+ if (amd_cpu && CPUID_TO_FAMILY(cpu_id) >= 0x17)
+ return (0);
+
+ if (cpu_feature != 0) {
+ do_cpuid(0x80000000, p);
+ cpu_exthigh = p[0];
+ } else {
+ cpu_exthigh = 0;
+ }
+ if (cpu_exthigh >= 0x80000001) {
+ do_cpuid(0x80000001, p);
+ amd_feature = p[3];
+ } else {
+ amd_feature = 0;
+ }
+
+ if ((amd_feature & AMDID_RDTSCP) != 0)
+ return (3);
+ if ((cpu_feature & CPUID_SSE2) == 0)
+ return (2);
+ return (amd_cpu ? 1 : 0);
+}
+
+DEFINE_UIFUNC(static, u_int, __vdso_gettc_rdtsc_low,
+ (const struct vdso_timehands *th))
+{
+ return (tsc_selector[tsc_selector_idx(cpu_feature)].ts_rdtsc_low);
+}
+
+DEFINE_UIFUNC(static, u_int, __vdso_gettc_rdtsc32, (void))
+{
+ return (tsc_selector[tsc_selector_idx(cpu_feature)].ts_rdtsc32);
+}
+
#define HPET_DEV_MAP_MAX 10
static volatile char *hpet_dev_map[HPET_DEV_MAP_MAX];
@@ -199,7 +293,7 @@
scale = tsc_ref->tsc_scale;
ofs = tsc_ref->tsc_ofs;
- rdtsc_mb();
+ mfence(); /* XXXKIB */
tsc = rdtsc();
/* ret = ((tsc * scale) >> 64) + ofs */
@@ -231,7 +325,7 @@
switch (th->th_algo) {
case VDSO_TH_ALGO_X86_TSC:
*tc = th->th_x86_shift > 0 ? __vdso_gettc_rdtsc_low(th) :
- __vdso_rdtsc32();
+ __vdso_gettc_rdtsc32();
return (0);
case VDSO_TH_ALGO_X86_HPET:
idx = th->th_x86_hpet_idx;
diff --git a/sys/amd64/include/cpufunc.h b/sys/amd64/include/cpufunc.h
--- a/sys/amd64/include/cpufunc.h
+++ b/sys/amd64/include/cpufunc.h
@@ -412,6 +412,15 @@
return (rv);
}
+static __inline uint32_t
+rdtscp32(void)
+{
+ uint32_t rv;
+
+ __asm __volatile("rdtscp" : "=a" (rv) : : "ecx", "edx");
+ return (rv);
+}
+
static __inline void
wbinvd(void)
{
diff --git a/sys/i386/include/cpufunc.h b/sys/i386/include/cpufunc.h
--- a/sys/i386/include/cpufunc.h
+++ b/sys/i386/include/cpufunc.h
@@ -412,6 +412,15 @@
return (rv);
}
+static __inline uint32_t
+rdtscp32(void)
+{
+ uint32_t rv;
+
+ __asm __volatile("rdtscp" : "=a" (rv) : : "ecx", "edx");
+ return (rv);
+}
+
static __inline void
wbinvd(void)
{
diff --git a/sys/x86/x86/identcpu.c b/sys/x86/x86/identcpu.c
--- a/sys/x86/x86/identcpu.c
+++ b/sys/x86/x86/identcpu.c
@@ -223,7 +223,7 @@
} cpu_vendors[] = {
{ INTEL_VENDOR_ID, CPU_VENDOR_INTEL }, /* GenuineIntel */
{ AMD_VENDOR_ID, CPU_VENDOR_AMD }, /* AuthenticAMD */
- { HYGON_VENDOR_ID, CPU_VENDOR_HYGON }, /* HygonGenuine*/
+ { HYGON_VENDOR_ID, CPU_VENDOR_HYGON }, /* HygonGenuine */
{ CENTAUR_VENDOR_ID, CPU_VENDOR_CENTAUR }, /* CentaurHauls */
#ifdef __i386__
{ NSC_VENDOR_ID, CPU_VENDOR_NSC }, /* Geode by NSC */
diff --git a/sys/x86/x86/tsc.c b/sys/x86/x86/tsc.c
--- a/sys/x86/x86/tsc.c
+++ b/sys/x86/x86/tsc.c
@@ -91,12 +91,14 @@
int status);
static void tsc_freq_changing(void *arg, const struct cf_level *level,
int *status);
-static unsigned tsc_get_timecount(struct timecounter *tc);
-static inline unsigned tsc_get_timecount_low(struct timecounter *tc);
-static unsigned tsc_get_timecount_lfence(struct timecounter *tc);
-static unsigned tsc_get_timecount_low_lfence(struct timecounter *tc);
-static unsigned tsc_get_timecount_mfence(struct timecounter *tc);
-static unsigned tsc_get_timecount_low_mfence(struct timecounter *tc);
+static u_int tsc_get_timecount(struct timecounter *tc);
+static inline u_int tsc_get_timecount_low(struct timecounter *tc);
+static u_int tsc_get_timecount_lfence(struct timecounter *tc);
+static u_int tsc_get_timecount_low_lfence(struct timecounter *tc);
+static u_int tsc_get_timecount_mfence(struct timecounter *tc);
+static u_int tsc_get_timecount_low_mfence(struct timecounter *tc);
+static u_int tscp_get_timecount(struct timecounter *tc);
+static u_int tscp_get_timecount_low(struct timecounter *tc);
static void tsc_levels_changed(void *arg, int unit);
static uint32_t x86_tsc_vdso_timehands(struct vdso_timehands *vdso_th,
struct timecounter *tc);
@@ -628,7 +630,25 @@
init:
for (shift = 0; shift <= 31 && (tsc_freq >> shift) > max_freq; shift++)
;
- if ((cpu_feature & CPUID_SSE2) != 0 && mp_ncpus > 1) {
+
+ /*
+ * Timecounter implementation selection, top to bottom:
+ * - For AMD Zens and newer, use LFENCE;RDTSC.
+ * - If RDTSCP is available, use RDTSCP.
+ * - If fence instructions are provided (SSE2), use LFENCE;RDTSC
+ * on Intel, and MFENCE;RDTSC on AMD.
+ * - For really old CPUs, just use RDTSC.
+ */
+ if ((cpu_vendor_id == CPU_VENDOR_AMD ||
+ cpu_vendor_id == CPU_VENDOR_HYGON) &&
+ CPUID_TO_FAMILY(cpu_id) >= 0x17) {
+ tsc_timecounter.tc_get_timecount = shift > 0 ?
+ tsc_get_timecount_low_lfence :
+ tsc_get_timecount_lfence;
+ } else if ((amd_feature & AMDID_RDTSCP) != 0) {
+ tsc_timecounter.tc_get_timecount = shift > 0 ?
+ tscp_get_timecount_low : tscp_get_timecount;
+ } else if ((cpu_feature & CPUID_SSE2) != 0 && mp_ncpus > 1) {
if (cpu_vendor_id == CPU_VENDOR_AMD ||
cpu_vendor_id == CPU_VENDOR_HYGON) {
tsc_timecounter.tc_get_timecount = shift > 0 ?
@@ -783,6 +803,13 @@
return (rdtsc32());
}
+static u_int
+tscp_get_timecount(struct timecounter *tc __unused)
+{
+
+ return (rdtscp32());
+}
+
static inline u_int
tsc_get_timecount_low(struct timecounter *tc)
{
@@ -793,6 +820,16 @@
return (rv);
}
+static u_int
+tscp_get_timecount_low(struct timecounter *tc)
+{
+ uint32_t rv;
+
+ __asm __volatile("rdtscp; movl %1, %%ecx; shrd %%cl, %%edx, %0"
+ : "=a" (rv) : "m" (tc->tc_priv) : "ecx", "edx");
+ return (rv);
+}
+
static u_int
tsc_get_timecount_lfence(struct timecounter *tc __unused)
{

File Metadata

Mime Type
text/plain
Expires
Sat, Jan 17, 7:02 PM (12 h, 47 m)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
27697850
Default Alt Text
D27986.id81971.diff (8 KB)

Event Timeline