Index: head/lib/libc/Makefile =================================================================== --- head/lib/libc/Makefile +++ head/lib/libc/Makefile @@ -110,6 +110,9 @@ ${LIBC_ARCH} == "mips" .include "${LIBC_SRCTOP}/softfloat/Makefile.inc" .endif +.if ${LIBC_ARCH} == "i386" || ${LIBC_ARCH} == "amd64" +.include "${LIBC_SRCTOP}/x86/sys/Makefile.inc" +.endif .if ${MK_NIS} != "no" CFLAGS+= -DYP .include "${LIBC_SRCTOP}/yp/Makefile.inc" Index: head/lib/libc/aarch64/sys/__vdso_gettc.c =================================================================== --- head/lib/libc/aarch64/sys/__vdso_gettc.c +++ head/lib/libc/aarch64/sys/__vdso_gettc.c @@ -34,6 +34,7 @@ #include #include #include +#include #include "libc_private.h" static inline uint64_t @@ -55,14 +56,15 @@ } #pragma weak __vdso_gettc -u_int -__vdso_gettc(const struct vdso_timehands *th) +int +__vdso_gettc(const struct vdso_timehands *th, u_int *tc) { - uint64_t val; + if (th->th_algo != VDSO_TH_ALGO_ARM_GENTIM) + return (ENOSYS); __asm __volatile("isb" : : : "memory"); - val = th->th_physical == 0 ? cp15_cntvct_get() : cp15_cntpct_get(); - return (val); + *tc = th->th_physical == 0 ? cp15_cntvct_get() : cp15_cntpct_get(); + return (0); } #pragma weak __vdso_gettimekeep Index: head/lib/libc/amd64/sys/Makefile.inc =================================================================== --- head/lib/libc/amd64/sys/Makefile.inc +++ head/lib/libc/amd64/sys/Makefile.inc @@ -2,7 +2,7 @@ # $FreeBSD$ SRCS+= amd64_get_fsbase.c amd64_get_gsbase.c amd64_set_fsbase.c \ - amd64_set_gsbase.c __vdso_gettc.c + amd64_set_gsbase.c MDASM= vfork.S brk.S cerror.S exect.S getcontext.S ptrace.S \ sbrk.S setlogin.S sigreturn.S Index: head/lib/libc/amd64/sys/__vdso_gettc.c =================================================================== --- head/lib/libc/amd64/sys/__vdso_gettc.c +++ head/lib/libc/amd64/sys/__vdso_gettc.c @@ -1,70 +0,0 @@ -/*- - * Copyright (c) 2012 Konstantin Belousov - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ - -#include -__FBSDID("$FreeBSD$"); - -#include -#include -#include -#include -#include -#include "libc_private.h" - -static u_int -__vdso_gettc_low(const struct vdso_timehands *th) -{ - u_int rv; - - __asm __volatile("lfence; rdtsc; shrd %%cl, %%edx, %0" - : "=a" (rv) : "c" (th->th_x86_shift) : "edx"); - return (rv); -} - -static u_int -__vdso_rdtsc32(void) -{ - u_int rv; - - __asm __volatile("lfence;rdtsc" : "=a" (rv) : : "edx"); - return (rv); -} - -#pragma weak __vdso_gettc -u_int -__vdso_gettc(const struct vdso_timehands *th) -{ - - return (th->th_x86_shift > 0 ? __vdso_gettc_low(th) : - __vdso_rdtsc32()); -} - -#pragma weak __vdso_gettimekeep -int -__vdso_gettimekeep(struct vdso_timekeep **tk) -{ - - return (_elf_aux_info(AT_TIMEKEEP, tk, sizeof(*tk))); -} Index: head/lib/libc/arm/sys/__vdso_gettc.c =================================================================== --- head/lib/libc/arm/sys/__vdso_gettc.c +++ head/lib/libc/arm/sys/__vdso_gettc.c @@ -35,6 +35,7 @@ #include #include #include +#include #include "libc_private.h" #if __ARM_ARCH >= 6 @@ -58,11 +59,12 @@ #endif #pragma weak __vdso_gettc -u_int -__vdso_gettc(const struct vdso_timehands *th) +int +__vdso_gettc(const struct vdso_timehands *th, u_int *tc) { - uint64_t val; + if (th->th_algo != VDSO_TH_ALGO_ARM_GENTIM) + return (ENOSYS); #if __ARM_ARCH >= 6 /* * Userspace gettimeofday() is only enabled on ARMv7 CPUs, but @@ -70,11 +72,12 @@ * armv7-a directive does not work. */ __asm __volatile(".word\t0xf57ff06f" : : : "memory"); /* isb */ - val = th->th_physical == 0 ? cp15_cntvct_get() : cp15_cntpct_get(); + *tc = th->th_physical == 0 ? cp15_cntvct_get() : cp15_cntpct_get(); + return (0); #else - val = 0; + *tc = 0; + return (ENOSYS); #endif - return (val); } #pragma weak __vdso_gettimekeep Index: head/lib/libc/i386/sys/Makefile.inc =================================================================== --- head/lib/libc/i386/sys/Makefile.inc +++ head/lib/libc/i386/sys/Makefile.inc @@ -5,8 +5,7 @@ SRCS+= i386_clr_watch.c i386_set_watch.c i386_vm86.c .endif SRCS+= i386_get_fsbase.c i386_get_gsbase.c i386_get_ioperm.c i386_get_ldt.c \ - i386_set_fsbase.c i386_set_gsbase.c i386_set_ioperm.c i386_set_ldt.c \ - __vdso_gettc.c + i386_set_fsbase.c i386_set_gsbase.c i386_set_ioperm.c i386_set_ldt.c MDASM= Ovfork.S brk.S cerror.S exect.S getcontext.S ptrace.S \ sbrk.S setlogin.S sigreturn.S syscall.S Index: head/lib/libc/i386/sys/__vdso_gettc.c =================================================================== --- head/lib/libc/i386/sys/__vdso_gettc.c +++ head/lib/libc/i386/sys/__vdso_gettc.c @@ -1,114 +0,0 @@ -/*- - * Copyright (c) 2012 Konstantin Belousov - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ - -#include -__FBSDID("$FreeBSD$"); - -#include -#include -#include -#include -#include -#include -#include "libc_private.h" - -static int lfence_works = -1; - -static int -get_lfence_usage(void) -{ - u_int cpuid_supported, p[4]; - - if (lfence_works == -1) { - __asm __volatile( - " pushfl\n" - " popl %%eax\n" - " movl %%eax,%%ecx\n" - " xorl $0x200000,%%eax\n" - " pushl %%eax\n" - " popfl\n" - " pushfl\n" - " popl %%eax\n" - " xorl %%eax,%%ecx\n" - " je 1f\n" - " movl $1,%0\n" - " jmp 2f\n" - "1: movl $0,%0\n" - "2:\n" - : "=r" (cpuid_supported) : : "eax", "ecx"); - if (cpuid_supported) { - __asm __volatile( - " pushl %%ebx\n" - " cpuid\n" - " movl %%ebx,%1\n" - " popl %%ebx\n" - : "=a" (p[0]), "=r" (p[1]), "=c" (p[2]), "=d" (p[3]) - : "0" (0x1)); - lfence_works = (p[3] & CPUID_SSE2) != 0; - } else - lfence_works = 0; - } - return (lfence_works); -} - -static u_int -__vdso_gettc_low(const struct vdso_timehands *th) -{ - u_int rv; - - if (get_lfence_usage() == 1) - lfence(); - __asm __volatile("rdtsc; shrd %%cl, %%edx, %0" - : "=a" (rv) : "c" (th->th_x86_shift) : "edx"); - return (rv); -} - -static u_int -__vdso_rdtsc32(void) -{ - u_int rv; - - if (get_lfence_usage() == 1) - lfence(); - rv = rdtsc32(); - return (rv); -} - -#pragma weak __vdso_gettc -u_int -__vdso_gettc(const struct vdso_timehands *th) -{ - - return (th->th_x86_shift > 0 ? __vdso_gettc_low(th) : - __vdso_rdtsc32()); -} - -#pragma weak __vdso_gettimekeep -int -__vdso_gettimekeep(struct vdso_timekeep **tk) -{ - - return (_elf_aux_info(AT_TIMEKEEP, tk, sizeof(*tk))); -} Index: head/lib/libc/sys/__vdso_gettimeofday.c =================================================================== --- head/lib/libc/sys/__vdso_gettimeofday.c +++ head/lib/libc/sys/__vdso_gettimeofday.c @@ -34,12 +34,16 @@ #include #include "libc_private.h" -static u_int -tc_delta(const struct vdso_timehands *th) +static int +tc_delta(const struct vdso_timehands *th, u_int *delta) { + int error; + u_int tc; - return ((__vdso_gettc(th) - th->th_offset_count) & - th->th_counter_mask); + error = __vdso_gettc(th, &tc); + if (error == 0) + *delta = (tc - th->th_offset_count) & th->th_counter_mask; + return (error); } /* @@ -56,6 +60,8 @@ { struct vdso_timehands *th; uint32_t curr, gen; + u_int delta; + int error; do { if (!tk->tk_enabled) @@ -63,11 +69,14 @@ curr = atomic_load_acq_32(&tk->tk_current); th = &tk->tk_th[curr]; - if (th->th_algo != VDSO_TH_ALGO_1) - return (ENOSYS); gen = atomic_load_acq_32(&th->th_gen); *bt = th->th_offset; - bintime_addx(bt, th->th_scale * tc_delta(th)); + error = tc_delta(th, &delta); + if (error == EAGAIN) + continue; + if (error != 0) + return (error); + bintime_addx(bt, th->th_scale * delta); if (abs) bintime_add(bt, &th->th_boottime); Index: head/lib/libc/sys/trivial-vdso_tc.c =================================================================== --- head/lib/libc/sys/trivial-vdso_tc.c +++ head/lib/libc/sys/trivial-vdso_tc.c @@ -32,11 +32,11 @@ #include #pragma weak __vdso_gettc -u_int -__vdso_gettc(const struct vdso_timehands *th) +int +__vdso_gettc(const struct vdso_timehands *th, u_int *tc) { - return (0); + return (ENOSYS); } #pragma weak __vdso_gettimekeep Index: head/lib/libc/x86/sys/Makefile.inc =================================================================== --- head/lib/libc/x86/sys/Makefile.inc +++ head/lib/libc/x86/sys/Makefile.inc @@ -0,0 +1,6 @@ +# $FreeBSD$ + +.PATH: ${LIBC_SRCTOP}/x86/sys + +SRCS+= \ + __vdso_gettc.c Index: head/lib/libc/x86/sys/__vdso_gettc.c =================================================================== --- head/lib/libc/x86/sys/__vdso_gettc.c +++ head/lib/libc/x86/sys/__vdso_gettc.c @@ -0,0 +1,179 @@ +/*- + * Copyright (c) 2012 Konstantin Belousov + * Copyright (c) 2016 The FreeBSD Foundation + * All rights reserved. + * + * Portions of this software were developed by Konstantin Belousov + * under sponsorship from the FreeBSD Foundation. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include +__FBSDID("$FreeBSD$"); + +#include +#include "namespace.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include "un-namespace.h" +#include +#include +#include +#include "libc_private.h" + +static void +lfence_mb(void) +{ +#if defined(__i386__) + static int lfence_works = -1; + u_int cpuid_supported, p[4]; + + if (lfence_works == -1) { + __asm __volatile( + " pushfl\n" + " popl %%eax\n" + " movl %%eax,%%ecx\n" + " xorl $0x200000,%%eax\n" + " pushl %%eax\n" + " popfl\n" + " pushfl\n" + " popl %%eax\n" + " xorl %%eax,%%ecx\n" + " je 1f\n" + " movl $1,%0\n" + " jmp 2f\n" + "1: movl $0,%0\n" + "2:\n" + : "=r" (cpuid_supported) : : "eax", "ecx", "cc"); + if (cpuid_supported) { + __asm __volatile( + " pushl %%ebx\n" + " cpuid\n" + " movl %%ebx,%1\n" + " popl %%ebx\n" + : "=a" (p[0]), "=r" (p[1]), "=c" (p[2]), "=d" (p[3]) + : "0" (0x1)); + lfence_works = (p[3] & CPUID_SSE2) != 0; + } else + lfence_works = 0; + } + if (lfence_works == 1) + lfence(); +#elif defined(__amd64__) + lfence(); +#else +#error "arch" +#endif +} + +static u_int +__vdso_gettc_rdtsc_low(const struct vdso_timehands *th) +{ + u_int rv; + + lfence_mb(); + __asm __volatile("rdtsc; shrd %%cl, %%edx, %0" + : "=a" (rv) : "c" (th->th_x86_shift) : "edx"); + return (rv); +} + +static u_int +__vdso_rdtsc32(void) +{ + + lfence_mb(); + return (rdtsc32()); +} + +static char *hpet_dev_map = NULL; +static uint32_t hpet_idx = 0xffffffff; + +static void +__vdso_init_hpet(uint32_t u) +{ + static const char devprefix[] = "/dev/hpet"; + char devname[64], *c, *c1, t; + int fd; + + c1 = c = stpcpy(devname, devprefix); + u = hpet_idx; + do { + *c++ = u % 10 + '0'; + u /= 10; + } while (u != 0); + *c = '\0'; + for (c--; c1 != c; c1++, c--) { + t = *c1; + *c1 = *c; + *c = t; + } + fd = _open(devname, O_RDONLY); + if (fd == -1) { + hpet_dev_map = MAP_FAILED; + return; + } + if (hpet_dev_map != NULL && hpet_dev_map != MAP_FAILED) + munmap(hpet_dev_map, PAGE_SIZE); + hpet_dev_map = mmap(NULL, PAGE_SIZE, PROT_READ, MAP_SHARED, fd, 0); + _close(fd); +} + +#pragma weak __vdso_gettc +int +__vdso_gettc(const struct vdso_timehands *th, u_int *tc) +{ + uint32_t tmp; + + switch (th->th_algo) { + case VDSO_TH_ALGO_X86_TSC: + *tc = th->th_x86_shift > 0 ? __vdso_gettc_rdtsc_low(th) : + __vdso_rdtsc32(); + return (0); + case VDSO_TH_ALGO_X86_HPET: + tmp = th->th_x86_hpet_idx; + if (hpet_dev_map == NULL || tmp != hpet_idx) { + hpet_idx = tmp; + __vdso_init_hpet(hpet_idx); + } + if (hpet_dev_map == MAP_FAILED) + return (ENOSYS); + *tc = *(volatile uint32_t *)(hpet_dev_map + HPET_MAIN_COUNTER); + return (0); + default: + return (ENOSYS); + } +} + +#pragma weak __vdso_gettimekeep +int +__vdso_gettimekeep(struct vdso_timekeep **tk) +{ + + return (_elf_aux_info(AT_TIMEKEEP, tk, sizeof(*tk))); +} Index: head/sys/arm/arm/generic_timer.c =================================================================== --- head/sys/arm/arm/generic_timer.c +++ head/sys/arm/arm/generic_timer.c @@ -105,6 +105,10 @@ { -1, 0 } }; +static uint32_t arm_tmr_fill_vdso_timehands(struct vdso_timehands *vdso_th, + struct timecounter *tc); +static void arm_tmr_do_delay(int usec, void *); + static timecounter_get_t arm_tmr_get_timecount; static struct timecounter arm_tmr_timecount = { @@ -114,6 +118,7 @@ .tc_counter_mask = ~0u, .tc_frequency = 0, .tc_quality = 1000, + .tc_fill_vdso_timehands = arm_tmr_fill_vdso_timehands, }; #ifdef __arm__ @@ -128,10 +133,6 @@ #define set_el1(x, val) WRITE_SPECIALREG(x ##_el1, val) #endif -static uint32_t arm_tmr_fill_vdso_timehands(struct vdso_timehands *vdso_th, - struct timecounter *tc); -static void arm_tmr_do_delay(int usec, void *); - static int get_freq(void) { @@ -412,8 +413,6 @@ } } - arm_cpu_fill_vdso_timehands = arm_tmr_fill_vdso_timehands; - arm_tmr_timecount.tc_frequency = sc->clkfreq; tc_init(&arm_tmr_timecount); @@ -535,7 +534,8 @@ struct timecounter *tc) { + vdso_th->th_algo = VDSO_TH_ALGO_ARM_GENTIM; vdso_th->th_physical = arm_tmr_sc->physical; bzero(vdso_th->th_res, sizeof(vdso_th->th_res)); - return (tc == &arm_tmr_timecount); + return (1); } Index: head/sys/arm/arm/machdep.c =================================================================== --- head/sys/arm/arm/machdep.c +++ head/sys/arm/arm/machdep.c @@ -2003,14 +2003,3 @@ #endif /* __ARM_ARCH < 6 */ #endif /* FDT */ - -uint32_t (*arm_cpu_fill_vdso_timehands)(struct vdso_timehands *, - struct timecounter *); - -uint32_t -cpu_fill_vdso_timehands(struct vdso_timehands *vdso_th, struct timecounter *tc) -{ - - return (arm_cpu_fill_vdso_timehands != NULL ? - arm_cpu_fill_vdso_timehands(vdso_th, tc) : 0); -} Index: head/sys/arm/include/md_var.h =================================================================== --- head/sys/arm/include/md_var.h +++ head/sys/arm/include/md_var.h @@ -45,11 +45,6 @@ extern int _min_memcpy_size; extern int _min_bzero_size; -struct vdso_timehands; -struct timecounter; -extern uint32_t (*arm_cpu_fill_vdso_timehands)(struct vdso_timehands *, - struct timecounter *); - #define DST_IS_USER 0x1 #define SRC_IS_USER 0x2 #define IS_PHYSICAL 0x4 Index: head/sys/arm/include/vdso.h =================================================================== --- head/sys/arm/include/vdso.h +++ head/sys/arm/include/vdso.h @@ -32,4 +32,6 @@ uint32_t th_physical; \ uint32_t th_res[7]; +#define VDSO_TH_ALGO_ARM_GENTIM VDSO_TH_ALGO_1 + #endif Index: head/sys/arm64/arm64/machdep.c =================================================================== --- head/sys/arm64/arm64/machdep.c +++ head/sys/arm64/arm64/machdep.c @@ -1005,17 +1005,6 @@ early_boot = 0; } -uint32_t (*arm_cpu_fill_vdso_timehands)(struct vdso_timehands *, - struct timecounter *); - -uint32_t -cpu_fill_vdso_timehands(struct vdso_timehands *vdso_th, struct timecounter *tc) -{ - - return (arm_cpu_fill_vdso_timehands != NULL ? - arm_cpu_fill_vdso_timehands(vdso_th, tc) : 0); -} - #ifdef DDB #include Index: head/sys/arm64/include/md_var.h =================================================================== --- head/sys/arm64/include/md_var.h +++ head/sys/arm64/include/md_var.h @@ -47,9 +47,4 @@ void dump_drop_page(vm_paddr_t); int minidumpsys(struct dumperinfo *); -struct vdso_timehands; -struct timecounter; -extern uint32_t (*arm_cpu_fill_vdso_timehands)(struct vdso_timehands *, - struct timecounter *); - #endif /* !_MACHINE_MD_VAR_H_ */ Index: head/sys/arm64/include/vdso.h =================================================================== --- head/sys/arm64/include/vdso.h +++ head/sys/arm64/include/vdso.h @@ -32,4 +32,6 @@ uint32_t th_physical; \ uint32_t th_res[7]; +#define VDSO_TH_ALGO_ARM_GENTIM VDSO_TH_ALGO_1 + #endif /* !_MACHINE_VDSO_H_ */ Index: head/sys/dev/acpica/acpi_hpet.h =================================================================== --- head/sys/dev/acpica/acpi_hpet.h +++ head/sys/dev/acpica/acpi_hpet.h @@ -64,4 +64,15 @@ #define HPET_MIN_CYCLES 128 /* Period considered reliable. */ +#ifdef _KERNEL +struct timecounter; +struct vdso_timehands; +struct vdso_timehands32; + +uint32_t hpet_vdso_timehands(struct vdso_timehands *vdso_th, + struct timecounter *tc); +uint32_t hpet_vdso_timehands32(struct vdso_timehands32 *vdso_th32, + struct timecounter *tc); +#endif + #endif /* !__ACPI_HPET_H__ */ Index: head/sys/dev/acpica/acpi_hpet.c =================================================================== --- head/sys/dev/acpica/acpi_hpet.c +++ head/sys/dev/acpica/acpi_hpet.c @@ -29,6 +29,8 @@ __FBSDID("$FreeBSD$"); #include "opt_acpi.h" +#include "opt_compat.h" + #if defined(__amd64__) #define DEV_APIC #else @@ -47,6 +49,7 @@ #include #include #include +#include #include #include @@ -141,6 +144,35 @@ return (bus_read_4(sc->mem_res, HPET_MAIN_COUNTER)); } +uint32_t +hpet_vdso_timehands(struct vdso_timehands *vdso_th, struct timecounter *tc) +{ + struct hpet_softc *sc; + + sc = tc->tc_priv; + vdso_th->th_algo = VDSO_TH_ALGO_X86_HPET; + vdso_th->th_x86_shift = 0; + vdso_th->th_x86_hpet_idx = device_get_unit(sc->dev); + bzero(vdso_th->th_res, sizeof(vdso_th->th_res)); + return (sc->mmap_allow != 0); +} + +#ifdef COMPAT_FREEBSD32 +uint32_t +hpet_vdso_timehands32(struct vdso_timehands32 *vdso_th32, + struct timecounter *tc) +{ + struct hpet_softc *sc; + + sc = tc->tc_priv; + vdso_th32->th_algo = VDSO_TH_ALGO_X86_HPET; + vdso_th32->th_x86_shift = 0; + vdso_th32->th_x86_hpet_idx = device_get_unit(sc->dev); + bzero(vdso_th32->th_res, sizeof(vdso_th32->th_res)); + return (sc->mmap_allow != 0); +} +#endif + static void hpet_enable(struct hpet_softc *sc) { @@ -537,6 +569,10 @@ sc->tc.tc_quality = 950, sc->tc.tc_frequency = sc->freq; sc->tc.tc_priv = sc; + sc->tc.tc_fill_vdso_timehands = hpet_vdso_timehands; +#ifdef COMPAT_FREEBSD32 + sc->tc.tc_fill_vdso_timehands32 = hpet_vdso_timehands32; +#endif tc_init(&sc->tc); } /* If not disabled - setup and announce event timers. */ Index: head/sys/kern/kern_tc.c =================================================================== --- head/sys/kern/kern_tc.c +++ head/sys/kern/kern_tc.c @@ -6,11 +6,14 @@ * this stuff is worth it, you can buy me a beer in return. Poul-Henning Kamp * ---------------------------------------------------------------------------- * - * Copyright (c) 2011 The FreeBSD Foundation + * Copyright (c) 2011, 2015, 2016 The FreeBSD Foundation * All rights reserved. * * Portions of this software were developed by Julien Ridoux at the University * of Melbourne under sponsorship from the FreeBSD Foundation. + * + * Portions of this software were developed by Konstantin Belousov + * under sponsorship from the FreeBSD Foundation. */ #include @@ -2130,13 +2133,16 @@ uint32_t enabled; th = timehands; - vdso_th->th_algo = VDSO_TH_ALGO_1; vdso_th->th_scale = th->th_scale; vdso_th->th_offset_count = th->th_offset_count; vdso_th->th_counter_mask = th->th_counter->tc_counter_mask; vdso_th->th_offset = th->th_offset; vdso_th->th_boottime = th->th_boottime; - enabled = cpu_fill_vdso_timehands(vdso_th, th->th_counter); + if (th->th_counter->tc_fill_vdso_timehands != NULL) { + enabled = th->th_counter->tc_fill_vdso_timehands(vdso_th, + th->th_counter); + } else + enabled = 0; if (!vdso_th_enable) enabled = 0; return (enabled); @@ -2150,7 +2156,6 @@ uint32_t enabled; th = timehands; - vdso_th32->th_algo = VDSO_TH_ALGO_1; *(uint64_t *)&vdso_th32->th_scale[0] = th->th_scale; vdso_th32->th_offset_count = th->th_offset_count; vdso_th32->th_counter_mask = th->th_counter->tc_counter_mask; @@ -2158,7 +2163,11 @@ *(uint64_t *)&vdso_th32->th_offset.frac[0] = th->th_offset.frac; vdso_th32->th_boottime.sec = th->th_boottime.sec; *(uint64_t *)&vdso_th32->th_boottime.frac[0] = th->th_boottime.frac; - enabled = cpu_fill_vdso_timehands32(vdso_th32, th->th_counter); + if (th->th_counter->tc_fill_vdso_timehands32 != NULL) { + enabled = th->th_counter->tc_fill_vdso_timehands32(vdso_th32, + th->th_counter); + } else + enabled = 0; if (!vdso_th_enable) enabled = 0; return (enabled); Index: head/sys/sys/timetc.h =================================================================== --- head/sys/sys/timetc.h +++ head/sys/sys/timetc.h @@ -28,8 +28,14 @@ */ struct timecounter; +struct vdso_timehands; +struct vdso_timehands32; typedef u_int timecounter_get_t(struct timecounter *); typedef void timecounter_pps_t(struct timecounter *); +typedef uint32_t timecounter_fill_vdso_timehands_t(struct vdso_timehands *, + struct timecounter *); +typedef uint32_t timecounter_fill_vdso_timehands32_t(struct vdso_timehands32 *, + struct timecounter *); struct timecounter { timecounter_get_t *tc_get_timecount; @@ -68,6 +74,8 @@ /* Pointer to the timecounter's private parts. */ struct timecounter *tc_next; /* Pointer to the next timecounter. */ + timecounter_fill_vdso_timehands_t *tc_fill_vdso_timehands; + timecounter_fill_vdso_timehands32_t *tc_fill_vdso_timehands32; }; extern struct timecounter *timecounter; Index: head/sys/sys/vdso.h =================================================================== --- head/sys/sys/vdso.h +++ head/sys/sys/vdso.h @@ -53,6 +53,7 @@ #define VDSO_TK_VER_1 0x1 #define VDSO_TK_VER_CURR VDSO_TK_VER_1 #define VDSO_TH_ALGO_1 0x1 +#define VDSO_TH_ALGO_2 0x2 #ifndef _KERNEL @@ -62,7 +63,7 @@ int __vdso_clock_gettime(clockid_t clock_id, struct timespec *ts); int __vdso_gettimeofday(struct timeval *tv, struct timezone *tz); -u_int __vdso_gettc(const struct vdso_timehands *vdso_th); +int __vdso_gettc(const struct vdso_timehands *vdso_th, u_int *tc); int __vdso_gettimekeep(struct vdso_timekeep **tk); #endif Index: head/sys/x86/include/vdso.h =================================================================== --- head/sys/x86/include/vdso.h +++ head/sys/x86/include/vdso.h @@ -1,7 +1,11 @@ /*- * Copyright 2012 Konstantin Belousov . + * Copyright 2016 The FreeBSD Foundation. * All rights reserved. * + * Portions of this software were developed by Konstantin Belousov + * under sponsorship from the FreeBSD Foundation. + * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: @@ -30,7 +34,11 @@ #define VDSO_TIMEHANDS_MD \ uint32_t th_x86_shift; \ - uint32_t th_res[7]; + uint32_t th_x86_hpet_idx; \ + uint32_t th_res[6]; + +#define VDSO_TH_ALGO_X86_TSC VDSO_TH_ALGO_1 +#define VDSO_TH_ALGO_X86_HPET VDSO_TH_ALGO_2 #ifdef _KERNEL #ifdef COMPAT_FREEBSD32 Index: head/sys/x86/x86/tsc.c =================================================================== --- head/sys/x86/x86/tsc.c +++ head/sys/x86/x86/tsc.c @@ -48,6 +48,7 @@ #include #include #include +#include #include "cpufreq_if.h" @@ -93,14 +94,22 @@ static unsigned tsc_get_timecount_mfence(struct timecounter *tc); static unsigned tsc_get_timecount_low_mfence(struct timecounter *tc); static void tsc_levels_changed(void *arg, int unit); +static uint32_t x86_tsc_vdso_timehands(struct vdso_timehands *vdso_th, + struct timecounter *tc); +#ifdef COMPAT_FREEBSD32 +static uint32_t x86_tsc_vdso_timehands32(struct vdso_timehands32 *vdso_th32, + struct timecounter *tc); +#endif static struct timecounter tsc_timecounter = { - tsc_get_timecount, /* get_timecount */ - 0, /* no poll_pps */ - ~0u, /* counter_mask */ - 0, /* frequency */ - "TSC", /* name */ - 800, /* quality (adjusted in code) */ + .tc_get_timecount = tsc_get_timecount, + .tc_counter_mask = ~0u, + .tc_name = "TSC", + .tc_quality = 800, /* adjusted in code */ + .tc_fill_vdso_timehands = x86_tsc_vdso_timehands, +#ifdef COMPAT_FREEBSD32 + .tc_fill_vdso_timehands32 = x86_tsc_vdso_timehands32, +#endif }; static void @@ -724,23 +733,27 @@ return (tsc_get_timecount_low(tc)); } -uint32_t -cpu_fill_vdso_timehands(struct vdso_timehands *vdso_th, struct timecounter *tc) +static uint32_t +x86_tsc_vdso_timehands(struct vdso_timehands *vdso_th, struct timecounter *tc) { + vdso_th->th_algo = VDSO_TH_ALGO_X86_TSC; vdso_th->th_x86_shift = (int)(intptr_t)tc->tc_priv; + vdso_th->th_x86_hpet_idx = 0xffffffff; bzero(vdso_th->th_res, sizeof(vdso_th->th_res)); - return (tc == &tsc_timecounter); + return (1); } #ifdef COMPAT_FREEBSD32 -uint32_t -cpu_fill_vdso_timehands32(struct vdso_timehands32 *vdso_th32, +static uint32_t +x86_tsc_vdso_timehands32(struct vdso_timehands32 *vdso_th32, struct timecounter *tc) { + vdso_th32->th_algo = VDSO_TH_ALGO_X86_TSC; vdso_th32->th_x86_shift = (int)(intptr_t)tc->tc_priv; + vdso_th32->th_x86_hpet_idx = 0xffffffff; bzero(vdso_th32->th_res, sizeof(vdso_th32->th_res)); - return (tc == &tsc_timecounter); + return (1); } #endif