diff --git a/sys/amd64/linux/linux_vdso_gtod.c b/sys/amd64/linux/linux_vdso_gtod.c index 57d3f4b45dd4..65e45ce29505 100644 --- a/sys/amd64/linux/linux_vdso_gtod.c +++ b/sys/amd64/linux/linux_vdso_gtod.c @@ -1,145 +1,146 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2021 Dmitry Chagin * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #define _KERNEL #include #undef _KERNEL #include #include +#include #include #include #include #include #include /* The kernel fixup this at vDSO install */ uintptr_t *kern_timekeep_base = NULL; uint32_t kern_tsc_selector = 0; #include /* for debug purpose */ static int write(int fd, const void *buf, size_t size) { int res; __asm__ __volatile__ ( "syscall" : "=a"(res) : "a"(LINUX_SYS_write), "D"(fd), "S"(buf), "d"(size) : "cc", "rcx", "r11", "memory" ); return (res); } static int __vdso_clock_gettime_fallback(clockid_t clock_id, struct l_timespec *ts) { int res; __asm__ __volatile__ ( "syscall" : "=a"(res) : "a"(LINUX_SYS_linux_clock_gettime), "D"(clock_id), "S"(ts) : "cc", "rcx", "r11", "memory" ); return (res); } static int __vdso_gettimeofday_fallback(l_timeval *tv, struct timezone *tz) { int res; __asm__ __volatile__ ( "syscall" : "=a"(res) : "a"(LINUX_SYS_gettimeofday), "D"(tv), "S"(tz) : "cc", "rcx", "r11", "memory" ); return (res); } static int __vdso_clock_getres_fallback(clockid_t clock_id, struct l_timespec *ts) { int res; __asm__ __volatile__ ( "syscall" : "=a"(res) : "a"(LINUX_SYS_linux_clock_getres), "D"(clock_id), "S"(ts) : "cc", "rcx", "r11", "memory" ); return (res); } static int __vdso_getcpu_fallback(uint32_t *cpu, uint32_t *node, void *cache) { int res; __asm__ __volatile__ ( "syscall" : "=a"(res) : "a"(LINUX_SYS_linux_getcpu), "D"(cpu), "S"(node), "d"(cache) : "cc", "rcx", "r11", "memory" ); return (res); } static int __vdso_time_fallback(long *tm) { int res; __asm__ __volatile__ ( "syscall" : "=a"(res) : "a"(LINUX_SYS_linux_time), "D"(tm) : "cc", "rcx", "r11", "memory" ); return (res); } #include diff --git a/sys/amd64/linux32/linux32_vdso_gtod.c b/sys/amd64/linux32/linux32_vdso_gtod.c index 0caf787c9503..bbe15e462c09 100644 --- a/sys/amd64/linux32/linux32_vdso_gtod.c +++ b/sys/amd64/linux32/linux32_vdso_gtod.c @@ -1,145 +1,146 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2021 Dmitry Chagin * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #define _KERNEL #include #undef _KERNEL #include -#include -#include +#include +#include +#include #include #include #include #include /* The kernel fixup this at vDSO install */ uintptr_t *kern_timekeep_base = NULL; uint32_t kern_tsc_selector = 0; #include static int write(int fd, const void *buf, size_t size) { int res; __asm__ __volatile__ ( "int $0x80" : "=a"(res) : "a"(LINUX32_SYS_write), "b"(fd), "c"(buf), "d"(size) : "cc", "memory" ); return (res); } static int __vdso_clock_gettime_fallback(clockid_t clock_id, struct l_timespec *ts) { int res; __asm__ __volatile__ ( "int $0x80" : "=a"(res) : "a"(LINUX32_SYS_linux_clock_gettime), "b"(clock_id), "c"(ts) : "cc", "memory" ); return (res); } static int __vdso_clock_gettime64_fallback(clockid_t clock_id, struct l_timespec64 *ts) { int res; __asm__ __volatile__ ( "int $0x80" : "=a"(res) : "a"(LINUX32_SYS_linux_clock_gettime64), "b"(clock_id), "c"(ts) : "cc", "memory" ); return (res); } static int __vdso_gettimeofday_fallback(l_timeval *tv, struct timezone *tz) { int res; __asm__ __volatile__ ( "int $0x80" : "=a"(res) : "a"(LINUX32_SYS_linux_gettimeofday), "b"(tv), "c"(tz) : "cc", "memory" ); return (res); } static int __vdso_clock_getres_fallback(clockid_t clock_id, struct l_timespec *ts) { int res; __asm__ __volatile__ ( "int $0x80" : "=a"(res) : "a"(LINUX32_SYS_linux_clock_getres), "b"(clock_id), "c"(ts) : "cc", "memory" ); return (res); } static int __vdso_time_fallback(long *tm) { int res; __asm__ __volatile__ ( "int $0x80" : "=a"(res) : "a"(LINUX32_SYS_linux_time), "b"(tm) : "cc", "memory" ); return (res); } #include diff --git a/sys/i386/linux/linux_vdso_gtod.c b/sys/i386/linux/linux_vdso_gtod.c index 046b864c54f5..8619a21cd3f4 100644 --- a/sys/i386/linux/linux_vdso_gtod.c +++ b/sys/i386/linux/linux_vdso_gtod.c @@ -1,144 +1,145 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2021 Dmitry Chagin * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #define _KERNEL #include #undef _KERNEL #include #include +#include #include #include #include #include #include /* The kernel fixup this at vDSO install */ uintptr_t *kern_timekeep_base = NULL; uint32_t kern_tsc_selector = 0; #include static int write(int fd, const void *buf, size_t size) { int res; __asm__ __volatile__ ( "int $0x80" : "=a"(res) : "a"(LINUX_SYS_write), "b"(fd), "c"(buf), "d"(size) : "cc", "memory" ); return (res); } static int __vdso_clock_gettime_fallback(clockid_t clock_id, struct l_timespec *ts) { int res; __asm__ __volatile__ ( "int $0x80" : "=a"(res) : "a"(LINUX_SYS_linux_clock_gettime), "b"(clock_id), "c"(ts) : "cc", "memory" ); return (res); } static int __vdso_clock_gettime64_fallback(clockid_t clock_id, struct l_timespec64 *ts) { int res; __asm__ __volatile__ ( "int $0x80" : "=a"(res) : "a"(LINUX_SYS_linux_clock_gettime64), "b"(clock_id), "c"(ts) : "cc", "memory" ); return (res); } static int __vdso_gettimeofday_fallback(l_timeval *tv, struct timezone *tz) { int res; __asm__ __volatile__ ( "int $0x80" : "=a"(res) : "a"(LINUX_SYS_gettimeofday), "b"(tv), "c"(tz) : "cc", "memory" ); return (res); } static int __vdso_clock_getres_fallback(clockid_t clock_id, struct l_timespec *ts) { int res; __asm__ __volatile__ ( "int $0x80" : "=a"(res) : "a"(LINUX_SYS_linux_clock_getres), "b"(clock_id), "c"(ts) : "cc", "memory" ); return (res); } static int __vdso_time_fallback(long *tm) { int res; __asm__ __volatile__ ( "int $0x80" : "=a"(res) : "a"(LINUX_SYS_linux_time), "b"(tm) : "cc", "memory" ); return (res); } #include diff --git a/sys/x86/linux/linux_vdso_gettc_x86.inc b/sys/x86/linux/linux_vdso_gettc_x86.inc index ade78a03486b..11c2a531e97e 100644 --- a/sys/x86/linux/linux_vdso_gettc_x86.inc +++ b/sys/x86/linux/linux_vdso_gettc_x86.inc @@ -1,164 +1,151 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 2012 Konstantin Belousov * Copyright (c) 2016, 2017, 2019 The FreeBSD Foundation * Copyright (c) 2021 Dmitry Chagin * * Portions of this software were developed by Konstantin Belousov * under sponsorship from the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ -#include -__FBSDID("$FreeBSD$"); - -#include - -#if defined(__i386__) || defined(COMPAT_LINUX32) -#include -#include -#else -#include -#include -#endif - static inline u_int rdtsc_low(const struct vdso_timehands *th) { u_int rv; __asm __volatile("rdtsc; shrd %%cl, %%edx, %0" : "=a" (rv) : "c" (th->th_x86_shift) : "edx"); return (rv); } static inline u_int rdtscp_low(const struct vdso_timehands *th) { u_int rv; __asm __volatile("rdtscp; movl %%edi,%%ecx; shrd %%cl, %%edx, %0" : "=a" (rv) : "D" (th->th_x86_shift) : "ecx", "edx"); return (rv); } static u_int rdtsc_low_mb_lfence(const struct vdso_timehands *th) { lfence(); return (rdtsc_low(th)); } static u_int rdtsc_low_mb_mfence(const struct vdso_timehands *th) { mfence(); return (rdtsc_low(th)); } static u_int rdtsc_low_mb_none(const struct vdso_timehands *th) { return (rdtsc_low(th)); } static u_int rdtsc32_mb_lfence(void) { lfence(); return (rdtsc32()); } static u_int rdtsc32_mb_mfence(void) { mfence(); return (rdtsc32()); } static u_int rdtsc32_mb_none(void) { return (rdtsc32()); } static u_int rdtscp32_(void) { return (rdtscp32()); } struct tsc_selector_tag { u_int (*ts_rdtsc32)(void); u_int (*ts_rdtsc_low)(const struct vdso_timehands *); }; static const struct tsc_selector_tag tsc_selector[] = { [0] = { /* Intel, LFENCE */ .ts_rdtsc32 = rdtsc32_mb_lfence, .ts_rdtsc_low = rdtsc_low_mb_lfence, }, [1] = { /* AMD, MFENCE */ .ts_rdtsc32 = rdtsc32_mb_mfence, .ts_rdtsc_low = rdtsc_low_mb_mfence, }, [2] = { /* No SSE2 */ .ts_rdtsc32 = rdtsc32_mb_none, .ts_rdtsc_low = rdtsc_low_mb_none, }, [3] = { /* RDTSCP */ .ts_rdtsc32 = rdtscp32_, .ts_rdtsc_low = rdtscp_low, }, }; static u_int __vdso_gettc_rdtsc_low(const struct vdso_timehands *th) { return (tsc_selector[kern_tsc_selector].ts_rdtsc_low(th)); } static u_int __vdso_gettc_rdtsc32(void) { return (tsc_selector[kern_tsc_selector].ts_rdtsc32()); } int __vdso_gettc(const struct vdso_timehands *th, u_int *tc) { switch (th->th_algo) { case VDSO_TH_ALGO_X86_TSC: *tc = th->th_x86_shift > 0 ? __vdso_gettc_rdtsc_low(th) : __vdso_gettc_rdtsc32(); return (0); case VDSO_TH_ALGO_X86_HPET: /* TODO */ default: return (ENOSYS); } }