Page MenuHomeFreeBSD
Differential D27986 Diff 81719 lib/libc/x86/sys/__vdso_gettc.c

Changeset View

Standalone View

View Options

lib/libc/x86/sys/__vdso_gettc.c

Show First 20 Lines • Show All 47 Lines • ▼ Show 20 Lines
#include <machine/specialreg.h> #include <machine/specialreg.h>
#include <dev/acpica/acpi_hpet.h> #include <dev/acpica/acpi_hpet.h>
#ifdef WANT_HYPERV #ifdef WANT_HYPERV
#include <dev/hyperv/hyperv.h> #include <dev/hyperv/hyperv.h>
#endif #endif
#include <x86/ifunc.h> #include <x86/ifunc.h>
#include "libc_private.h" #include "libc_private.h"
static void static inline u_int
rdtsc_mb_lfence(void) rdtsc_low(u_int shift)
{ {
u_int rv;
lfence(); __asm __volatile("rdtsc; shrd %%cl, %%edx, %0"
: "=a" (rv) : "c" (shift) : "edx");
return (rv);
} }
static void static inline u_int
rdtsc_mb_mfence(void) rdtscp_low(const struct vdso_timehands *th)
{ {
u_int rv;
__asm __volatile("rdtscp; movl %%edi,%%ecx; shrd %%cl, %%edx, %0"
: "=a" (rv) : "D" (th->th_x86_shift) : "ecx", "edx");
return (rv);
}
static u_int
rdtsc_low_mb_lfence(const struct vdso_timehands *th)
{
lfence();
return (rdtsc_low(th->th_x86_shift));
}
static u_int
rdtsc_low_mb_mfence(const struct vdso_timehands *th)
{
mfence(); mfence();
return (rdtsc_low(th->th_x86_shift));
} }
static void static u_int
rdtsc_mb_none(void) rdtsc_low_mb_none(const struct vdso_timehands *th)
{ {
return (rdtsc_low(th->th_x86_shift));
markjUnsubmitted
Done
Inline Actions

IMO it is a bit neater to dereference th in rdtsc_low() rather than its callers, like rdtscp_low().

markj: IMO it is a bit neater to dereference `th` in rdtsc_low() rather than its callers, like…
} }
DEFINE_UIFUNC(static, void, rdtsc_mb, (void)) DEFINE_UIFUNC(static, u_int, __vdso_gettc_rdtsc_low,
(const struct vdso_timehands *th))
{ {
u_int p[4]; u_int amd_feature, cpu_exthigh, p[4];
/* Not a typo, string matches our do_cpuid() registers use. */ /* Not a typo, string matches our do_cpuid() registers use. */
static const char intel_id[] = "GenuntelineI"; static const char intel_id[] = "GenuntelineI";
if (cpu_feature != 0) {
do_cpuid(0x80000000, p);
cpu_exthigh = p[0];
} else {
cpu_exthigh = 0;
}
if (cpu_exthigh >= 0x80000001) {
do_cpuid(0x80000001, p);
amd_feature = p[3];
} else {
amd_feature = 0;
}
if ((amd_feature & AMDID_RDTSCP) != 0)
return (rdtscp_low);
if ((cpu_feature & CPUID_SSE2) == 0) if ((cpu_feature & CPUID_SSE2) == 0)
return (rdtsc_mb_none); return (rdtsc_low_mb_none);
do_cpuid(0, p); do_cpuid(0, p);
return (memcmp(p + 1, intel_id, sizeof(intel_id) - 1) == 0 ? return (memcmp(p + 1, intel_id, sizeof(intel_id) - 1) == 0 ?
rdtsc_mb_lfence : rdtsc_mb_mfence); rdtsc_low_mb_lfence : rdtsc_low_mb_mfence);
} }
static u_int static u_int
__vdso_gettc_rdtsc_low(const struct vdso_timehands *th) rdtsc32_mb_lfence(void)
{ {
u_int rv; lfence();
return (rdtsc32());
rdtsc_mb();
__asm __volatile("rdtsc; shrd %%cl, %%edx, %0"
: "=a" (rv) : "c" (th->th_x86_shift) : "edx");
return (rv);
} }
static u_int static u_int
__vdso_rdtsc32(void) rdtsc32_mb_mfence(void)
{ {
mfence();
return (rdtsc32());
}
rdtsc_mb(); static u_int
rdtsc32_mb_none(void)
{
return (rdtsc32()); return (rdtsc32());
} }
static u_int
rdtscp32(void)
{
uint32_t rv;
__asm __volatile("rdtscp"
: "=a" (rv) :: "ecx", "edx");
return (rv);
}
DEFINE_UIFUNC(static, u_int, __vdso_gettc_rdtsc32, (void))
{
u_int amd_feature, cpu_exthigh, p[4];
/* Not a typo, string matches our do_cpuid() registers use. */
static const char intel_id[] = "GenuntelineI";
if (cpu_feature != 0) {
do_cpuid(0x80000000, p);
cpu_exthigh = p[0];
} else {
cpu_exthigh = 0;
}
if (cpu_exthigh >= 0x80000001) {
do_cpuid(0x80000001, p);
amd_feature = p[3];
} else {
amd_feature = 0;
}
if ((amd_feature & AMDID_RDTSCP) != 0)
return (rdtscp32);
if ((cpu_feature & CPUID_SSE2) == 0)
return (rdtsc32_mb_none);
do_cpuid(0, p);
return (memcmp(p + 1, intel_id, sizeof(intel_id) - 1) == 0 ?
rdtsc32_mb_lfence : rdtsc32_mb_mfence);
}
#define HPET_DEV_MAP_MAX 10 #define HPET_DEV_MAP_MAX 10
static volatile char *hpet_dev_map[HPET_DEV_MAP_MAX]; static volatile char *hpet_dev_map[HPET_DEV_MAP_MAX];
static void static void
__vdso_init_hpet(uint32_t u) __vdso_init_hpet(uint32_t u)
{ {
static const char devprefix[] = "/dev/hpet"; static const char devprefix[] = "/dev/hpet";
char devname[64], *c, *c1, t; char devname[64], *c, *c1, t;
▲ Show 20 Lines • Show All 79 Lines • ▼ Show 20 Lines__vdso_hyperv_tsc(struct hyperv_reftsc *tsc_ref, u_int *tc)
uint64_t disc, ret, tsc, scale; uint64_t disc, ret, tsc, scale;
uint32_t seq; uint32_t seq;
int64_t ofs; int64_t ofs;
while ((seq = atomic_load_acq_int(&tsc_ref->tsc_seq)) != 0) { while ((seq = atomic_load_acq_int(&tsc_ref->tsc_seq)) != 0) {
scale = tsc_ref->tsc_scale; scale = tsc_ref->tsc_scale;
ofs = tsc_ref->tsc_ofs; ofs = tsc_ref->tsc_ofs;
rdtsc_mb(); mfence(); /* XXXKIB */
tsc = rdtsc(); tsc = rdtsc();
/* ret = ((tsc * scale) >> 64) + ofs */ /* ret = ((tsc * scale) >> 64) + ofs */
__asm__ __volatile__ ("mulq %3" : __asm__ __volatile__ ("mulq %3" :
"=d" (ret), "=a" (disc) : "=d" (ret), "=a" (disc) :
"a" (tsc), "r" (scale)); "a" (tsc), "r" (scale));
ret += ofs; ret += ofs;
Show All 15 Lines
__vdso_gettc(const struct vdso_timehands *th, u_int *tc) __vdso_gettc(const struct vdso_timehands *th, u_int *tc)
{ {
volatile char *map; volatile char *map;
uint32_t idx; uint32_t idx;
switch (th->th_algo) { switch (th->th_algo) {
case VDSO_TH_ALGO_X86_TSC: case VDSO_TH_ALGO_X86_TSC:
*tc = th->th_x86_shift > 0 ? __vdso_gettc_rdtsc_low(th) : *tc = th->th_x86_shift > 0 ? __vdso_gettc_rdtsc_low(th) :
__vdso_rdtsc32(); __vdso_gettc_rdtsc32();
return (0); return (0);
case VDSO_TH_ALGO_X86_HPET: case VDSO_TH_ALGO_X86_HPET:
idx = th->th_x86_hpet_idx; idx = th->th_x86_hpet_idx;
if (idx >= HPET_DEV_MAP_MAX) if (idx >= HPET_DEV_MAP_MAX)
return (ENOSYS); return (ENOSYS);
map = (volatile char *)atomic_load_acq_ptr( map = (volatile char *)atomic_load_acq_ptr(
(volatile uintptr_t *)&hpet_dev_map[idx]); (volatile uintptr_t *)&hpet_dev_map[idx]);
if (map == NULL) { if (map == NULL) {
Show All 28 Lines