Changeset View
Standalone View
sys/x86/x86/tsc.c
Show First 20 Lines • Show All 85 Lines • ▼ Show 20 Lines | |||||
SYSCTL_INT(_machdep, OID_AUTO, disable_tsc_calibration, CTLFLAG_RDTUN, | SYSCTL_INT(_machdep, OID_AUTO, disable_tsc_calibration, CTLFLAG_RDTUN, | ||||
&tsc_skip_calibration, 0, | &tsc_skip_calibration, 0, | ||||
"Disable TSC frequency calibration"); | "Disable TSC frequency calibration"); | ||||
static void tsc_freq_changed(void *arg, const struct cf_level *level, | static void tsc_freq_changed(void *arg, const struct cf_level *level, | ||||
int status); | int status); | ||||
static void tsc_freq_changing(void *arg, const struct cf_level *level, | static void tsc_freq_changing(void *arg, const struct cf_level *level, | ||||
int *status); | int *status); | ||||
static unsigned tsc_get_timecount(struct timecounter *tc); | static u_int tsc_get_timecount(struct timecounter *tc); | ||||
static inline unsigned tsc_get_timecount_low(struct timecounter *tc); | static inline u_int tsc_get_timecount_low(struct timecounter *tc); | ||||
static unsigned tsc_get_timecount_lfence(struct timecounter *tc); | static u_int tsc_get_timecount_lfence(struct timecounter *tc); | ||||
static unsigned tsc_get_timecount_low_lfence(struct timecounter *tc); | static u_int tsc_get_timecount_low_lfence(struct timecounter *tc); | ||||
static unsigned tsc_get_timecount_mfence(struct timecounter *tc); | static u_int tsc_get_timecount_mfence(struct timecounter *tc); | ||||
static unsigned tsc_get_timecount_low_mfence(struct timecounter *tc); | static u_int tsc_get_timecount_low_mfence(struct timecounter *tc); | ||||
static u_int tscp_get_timecount(struct timecounter *tc); | |||||
static u_int tscp_get_timecount_low(struct timecounter *tc); | |||||
static void tsc_levels_changed(void *arg, int unit); | static void tsc_levels_changed(void *arg, int unit); | ||||
static uint32_t x86_tsc_vdso_timehands(struct vdso_timehands *vdso_th, | static uint32_t x86_tsc_vdso_timehands(struct vdso_timehands *vdso_th, | ||||
struct timecounter *tc); | struct timecounter *tc); | ||||
#ifdef COMPAT_FREEBSD32 | #ifdef COMPAT_FREEBSD32 | ||||
static uint32_t x86_tsc_vdso_timehands32(struct vdso_timehands32 *vdso_th32, | static uint32_t x86_tsc_vdso_timehands32(struct vdso_timehands32 *vdso_th32, | ||||
struct timecounter *tc); | struct timecounter *tc); | ||||
#endif | #endif | ||||
▲ Show 20 Lines • Show All 515 Lines • ▼ Show 20 Lines | |||||
#endif /* SMP */ | #endif /* SMP */ | ||||
if (tsc_is_invariant) | if (tsc_is_invariant) | ||||
tsc_timecounter.tc_quality = 1000; | tsc_timecounter.tc_quality = 1000; | ||||
max_freq >>= tsc_shift; | max_freq >>= tsc_shift; | ||||
init: | init: | ||||
for (shift = 0; shift <= 31 && (tsc_freq >> shift) > max_freq; shift++) | for (shift = 0; shift <= 31 && (tsc_freq >> shift) > max_freq; shift++) | ||||
; | ; | ||||
if ((cpu_feature & CPUID_SSE2) != 0 && mp_ncpus > 1) { | if ((amd_feature & AMDID_RDTSCP) != 0) { | ||||
tsc_timecounter.tc_get_timecount = shift > 0 ? | |||||
tscp_get_timecount_low : tscp_get_timecount; | |||||
} else if ((cpu_feature & CPUID_SSE2) != 0 && mp_ncpus > 1) { | |||||
if (cpu_vendor_id == CPU_VENDOR_AMD || | if (cpu_vendor_id == CPU_VENDOR_AMD || | ||||
cpu_vendor_id == CPU_VENDOR_HYGON) { | cpu_vendor_id == CPU_VENDOR_HYGON) { | ||||
tsc_timecounter.tc_get_timecount = shift > 0 ? | tsc_timecounter.tc_get_timecount = shift > 0 ? | ||||
tsc_get_timecount_low_mfence : | tsc_get_timecount_low_mfence : | ||||
tsc_get_timecount_mfence; | tsc_get_timecount_mfence; | ||||
} else { | } else { | ||||
tsc_timecounter.tc_get_timecount = shift > 0 ? | tsc_timecounter.tc_get_timecount = shift > 0 ? | ||||
tsc_get_timecount_low_lfence : | tsc_get_timecount_low_lfence : | ||||
▲ Show 20 Lines • Show All 138 Lines • ▼ Show 20 Lines | |||||
static u_int | static u_int | ||||
tsc_get_timecount(struct timecounter *tc __unused) | tsc_get_timecount(struct timecounter *tc __unused) | ||||
{ | { | ||||
return (rdtsc32()); | return (rdtsc32()); | ||||
} | } | ||||
static u_int | |||||
tscp_get_timecount(struct timecounter *tc __unused) | |||||
{ | |||||
uint32_t rv; | |||||
__asm __volatile("rdtscp" | |||||
: "=a" (rv) :: "ecx", "edx"); | |||||
return (rv); | |||||
markj: This is rdtscp32(). | |||||
Done Inline ActionsWell, except that we do not have such helper in cpufunc.h. I considered adding it, but decided that single-use does not make sense, additional argument for me was that this file contains enough asm already. If you prefer rdtscp32(), I will add it. kib: Well, except that we do not have such helper in cpufunc.h. I considered adding it, but decided… | |||||
Done Inline ActionsThere would be 2 uses: here and in __vdso_gettc.c, where rdtscp32() is already defined locally. Sorry for not being more clear, the duplication is why I made the suggestion. markj: There would be 2 uses: here and in __vdso_gettc.c, where rdtscp32() is already defined locally. | |||||
} | |||||
static inline u_int | static inline u_int | ||||
tsc_get_timecount_low(struct timecounter *tc) | tsc_get_timecount_low(struct timecounter *tc) | ||||
{ | { | ||||
uint32_t rv; | uint32_t rv; | ||||
__asm __volatile("rdtsc; shrd %%cl, %%edx, %0" | __asm __volatile("rdtsc; shrd %%cl, %%edx, %0" | ||||
: "=a" (rv) : "c" ((int)(intptr_t)tc->tc_priv) : "edx"); | : "=a" (rv) : "c" ((int)(intptr_t)tc->tc_priv) : "edx"); | ||||
return (rv); | |||||
} | |||||
static u_int | |||||
tscp_get_timecount_low(struct timecounter *tc) | |||||
{ | |||||
uint32_t rv; | |||||
__asm __volatile("rdtscp; movl %1, %%ecx; shrd %%cl, %%edx, %0" | |||||
: "=a" (rv) : "m" (tc->tc_priv) : "ecx", "edx"); | |||||
Not Done Inline ActionsDoesn't tc_priv need to be cast to 32 bits like in tsc_get_timecount_low()? markj: Doesn't tc_priv need to be cast to 32 bits like in tsc_get_timecount_low()? | |||||
Done Inline ActionsCasting would make it into r-value from l-value, which does not allow to use m specifier. Inline asm is happy without casting as well, because movl %1,%%ecx explicitly informs assembler about operand size (both 'l' and target size). I believe the cast in tsc_get_timecount_low() is not strictly needed, since there is no harm from loading whole 64bit of tc_priv into %rcx on amd64. But I do not want to touch it ATM. kib: Casting would make it into r-value from l-value, which does not allow to use m specifier. | |||||
return (rv); | return (rv); | ||||
} | } | ||||
static u_int | static u_int | ||||
tsc_get_timecount_lfence(struct timecounter *tc __unused) | tsc_get_timecount_lfence(struct timecounter *tc __unused) | ||||
{ | { | ||||
lfence(); | lfence(); | ||||
▲ Show 20 Lines • Show All 51 Lines • Show Last 20 Lines |
This is rdtscp32().