diff --git a/sys/amd64/include/atomic.h b/sys/amd64/include/atomic.h index 61cb79645c10..c821e77e5b8b 100644 --- a/sys/amd64/include/atomic.h +++ b/sys/amd64/include/atomic.h @@ -1,684 +1,692 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (c) 1998 Doug Rabson * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #ifndef _MACHINE_ATOMIC_H_ #define _MACHINE_ATOMIC_H_ #ifndef _SYS_CDEFS_H_ #error this file needs sys/cdefs.h as a prerequisite #endif /* * To express interprocessor (as opposed to processor and device) memory * ordering constraints, use the atomic_*() functions with acquire and release * semantics rather than the *mb() functions. An architecture's memory * ordering (or memory consistency) model governs the order in which a * program's accesses to different locations may be performed by an * implementation of that architecture. In general, for memory regions * defined as writeback cacheable, the memory ordering implemented by amd64 * processors preserves the program ordering of a load followed by a load, a * load followed by a store, and a store followed by a store. Only a store * followed by a load to a different memory location may be reordered. * Therefore, except for special cases, like non-temporal memory accesses or * memory regions defined as write combining, the memory ordering effects * provided by the sfence instruction in the wmb() function and the lfence * instruction in the rmb() function are redundant. In contrast, the * atomic_*() functions with acquire and release semantics do not perform * redundant instructions for ordinary cases of interprocessor memory * ordering on any architecture. */ #define mb() __asm __volatile("mfence;" : : : "memory") #define wmb() __asm __volatile("sfence;" : : : "memory") #define rmb() __asm __volatile("lfence;" : : : "memory") #ifdef _KERNEL /* * OFFSETOF_MONITORBUF == __pcpu_offset(pc_monitorbuf). * * The open-coded number is used instead of the symbolic expression to * avoid a dependency on sys/pcpu.h in machine/atomic.h consumers. * An assertion in amd64/vm_machdep.c ensures that the value is correct. */ #define OFFSETOF_MONITORBUF 0x100 #endif -#if defined(KCSAN) && !defined(KCSAN_RUNTIME) +#ifndef SAN_RUNTIME +#if defined(KASAN) +#define ATOMIC_SAN_PREFIX kasan +#elif defined(KCSAN) +#define ATOMIC_SAN_PREFIX kcsan +#endif +#endif + +#ifdef ATOMIC_SAN_PREFIX #include #else #include /* * Various simple operations on memory, each of which is atomic in the * presence of interrupts and multiple processors. * * atomic_set_char(P, V) (*(u_char *)(P) |= (V)) * atomic_clear_char(P, V) (*(u_char *)(P) &= ~(V)) * atomic_add_char(P, V) (*(u_char *)(P) += (V)) * atomic_subtract_char(P, V) (*(u_char *)(P) -= (V)) * * atomic_set_short(P, V) (*(u_short *)(P) |= (V)) * atomic_clear_short(P, V) (*(u_short *)(P) &= ~(V)) * atomic_add_short(P, V) (*(u_short *)(P) += (V)) * atomic_subtract_short(P, V) (*(u_short *)(P) -= (V)) * * atomic_set_int(P, V) (*(u_int *)(P) |= (V)) * atomic_clear_int(P, V) (*(u_int *)(P) &= ~(V)) * atomic_add_int(P, V) (*(u_int *)(P) += (V)) * atomic_subtract_int(P, V) (*(u_int *)(P) -= (V)) * atomic_swap_int(P, V) (return (*(u_int *)(P)); *(u_int *)(P) = (V);) * atomic_readandclear_int(P) (return (*(u_int *)(P)); *(u_int *)(P) = 0;) * * atomic_set_long(P, V) (*(u_long *)(P) |= (V)) * atomic_clear_long(P, V) (*(u_long *)(P) &= ~(V)) * atomic_add_long(P, V) (*(u_long *)(P) += (V)) * atomic_subtract_long(P, V) (*(u_long *)(P) -= (V)) * atomic_swap_long(P, V) (return (*(u_long *)(P)); *(u_long *)(P) = (V);) * atomic_readandclear_long(P) (return (*(u_long *)(P)); *(u_long *)(P) = 0;) */ /* * The above functions are expanded inline in the statically-linked * kernel. Lock prefixes are generated if an SMP kernel is being * built. * * Kernel modules call real functions which are built into the kernel. * This allows kernel modules to be portable between UP and SMP systems. */ #if !defined(__GNUCLIKE_ASM) #define ATOMIC_ASM(NAME, TYPE, OP, CONS, V) \ void atomic_##NAME##_##TYPE(volatile u_##TYPE *p, u_##TYPE v); \ void atomic_##NAME##_barr_##TYPE(volatile u_##TYPE *p, u_##TYPE v) int atomic_cmpset_char(volatile u_char *dst, u_char expect, u_char src); int atomic_cmpset_short(volatile u_short *dst, u_short expect, u_short src); int atomic_cmpset_int(volatile u_int *dst, u_int expect, u_int src); int atomic_cmpset_long(volatile u_long *dst, u_long expect, u_long src); int atomic_fcmpset_char(volatile u_char *dst, u_char *expect, u_char src); int atomic_fcmpset_short(volatile u_short *dst, u_short *expect, u_short src); int atomic_fcmpset_int(volatile u_int *dst, u_int *expect, u_int src); int atomic_fcmpset_long(volatile u_long *dst, u_long *expect, u_long src); u_int atomic_fetchadd_int(volatile u_int *p, u_int v); u_long atomic_fetchadd_long(volatile u_long *p, u_long v); int atomic_testandset_int(volatile u_int *p, u_int v); int atomic_testandset_long(volatile u_long *p, u_int v); int atomic_testandclear_int(volatile u_int *p, u_int v); int atomic_testandclear_long(volatile u_long *p, u_int v); void atomic_thread_fence_acq(void); void atomic_thread_fence_acq_rel(void); void atomic_thread_fence_rel(void); void atomic_thread_fence_seq_cst(void); #define ATOMIC_LOAD(TYPE) \ u_##TYPE atomic_load_acq_##TYPE(volatile u_##TYPE *p) #define ATOMIC_STORE(TYPE) \ void atomic_store_rel_##TYPE(volatile u_##TYPE *p, u_##TYPE v) #else /* !KLD_MODULE && __GNUCLIKE_ASM */ /* * For userland, always use lock prefixes so that the binaries will run * on both SMP and !SMP systems. */ #if defined(SMP) || !defined(_KERNEL) || defined(KLD_MODULE) #define MPLOCKED "lock ; " #else #define MPLOCKED #endif /* * The assembly is volatilized to avoid code chunk removal by the compiler. * GCC aggressively reorders operations and memory clobbering is necessary * in order to avoid that for memory barriers. */ #define ATOMIC_ASM(NAME, TYPE, OP, CONS, V) \ static __inline void \ atomic_##NAME##_##TYPE(volatile u_##TYPE *p, u_##TYPE v)\ { \ __asm __volatile(MPLOCKED OP \ : "+m" (*p) \ : CONS (V) \ : "cc"); \ } \ \ static __inline void \ atomic_##NAME##_barr_##TYPE(volatile u_##TYPE *p, u_##TYPE v)\ { \ __asm __volatile(MPLOCKED OP \ : "+m" (*p) \ : CONS (V) \ : "memory", "cc"); \ } \ struct __hack /* * Atomic compare and set, used by the mutex functions. * * cmpset: * if (*dst == expect) * *dst = src * * fcmpset: * if (*dst == *expect) * *dst = src * else * *expect = *dst * * Returns 0 on failure, non-zero on success. */ #define ATOMIC_CMPSET(TYPE) \ static __inline int \ atomic_cmpset_##TYPE(volatile u_##TYPE *dst, u_##TYPE expect, u_##TYPE src) \ { \ u_char res; \ \ __asm __volatile( \ " " MPLOCKED " " \ " cmpxchg %3,%1 ; " \ "# atomic_cmpset_" #TYPE " " \ : "=@cce" (res), /* 0 */ \ "+m" (*dst), /* 1 */ \ "+a" (expect) /* 2 */ \ : "r" (src) /* 3 */ \ : "memory", "cc"); \ return (res); \ } \ \ static __inline int \ atomic_fcmpset_##TYPE(volatile u_##TYPE *dst, u_##TYPE *expect, u_##TYPE src) \ { \ u_char res; \ \ __asm __volatile( \ " " MPLOCKED " " \ " cmpxchg %3,%1 ; " \ "# atomic_fcmpset_" #TYPE " " \ : "=@cce" (res), /* 0 */ \ "+m" (*dst), /* 1 */ \ "+a" (*expect) /* 2 */ \ : "r" (src) /* 3 */ \ : "memory", "cc"); \ return (res); \ } ATOMIC_CMPSET(char); ATOMIC_CMPSET(short); ATOMIC_CMPSET(int); ATOMIC_CMPSET(long); /* * Atomically add the value of v to the integer pointed to by p and return * the previous value of *p. */ static __inline u_int atomic_fetchadd_int(volatile u_int *p, u_int v) { __asm __volatile( " " MPLOCKED " " " xaddl %0,%1 ; " "# atomic_fetchadd_int" : "+r" (v), /* 0 */ "+m" (*p) /* 1 */ : : "cc"); return (v); } /* * Atomically add the value of v to the long integer pointed to by p and return * the previous value of *p. */ static __inline u_long atomic_fetchadd_long(volatile u_long *p, u_long v) { __asm __volatile( " " MPLOCKED " " " xaddq %0,%1 ; " "# atomic_fetchadd_long" : "+r" (v), /* 0 */ "+m" (*p) /* 1 */ : : "cc"); return (v); } static __inline int atomic_testandset_int(volatile u_int *p, u_int v) { u_char res; __asm __volatile( " " MPLOCKED " " " btsl %2,%1 ; " "# atomic_testandset_int" : "=@ccc" (res), /* 0 */ "+m" (*p) /* 1 */ : "Ir" (v & 0x1f) /* 2 */ : "cc"); return (res); } static __inline int atomic_testandset_long(volatile u_long *p, u_int v) { u_char res; __asm __volatile( " " MPLOCKED " " " btsq %2,%1 ; " "# atomic_testandset_long" : "=@ccc" (res), /* 0 */ "+m" (*p) /* 1 */ : "Jr" ((u_long)(v & 0x3f)) /* 2 */ : "cc"); return (res); } static __inline int atomic_testandclear_int(volatile u_int *p, u_int v) { u_char res; __asm __volatile( " " MPLOCKED " " " btrl %2,%1 ; " "# atomic_testandclear_int" : "=@ccc" (res), /* 0 */ "+m" (*p) /* 1 */ : "Ir" (v & 0x1f) /* 2 */ : "cc"); return (res); } static __inline int atomic_testandclear_long(volatile u_long *p, u_int v) { u_char res; __asm __volatile( " " MPLOCKED " " " btrq %2,%1 ; " "# atomic_testandclear_long" : "=@ccc" (res), /* 0 */ "+m" (*p) /* 1 */ : "Jr" ((u_long)(v & 0x3f)) /* 2 */ : "cc"); return (res); } /* * We assume that a = b will do atomic loads and stores. Due to the * IA32 memory model, a simple store guarantees release semantics. * * However, a load may pass a store if they are performed on distinct * addresses, so we need a Store/Load barrier for sequentially * consistent fences in SMP kernels. We use "lock addl $0,mem" for a * Store/Load barrier, as recommended by the AMD Software Optimization * Guide, and not mfence. To avoid false data dependencies, we use a * special address for "mem". In the kernel, we use a private per-cpu * cache line. In user space, we use a word in the stack's red zone * (-8(%rsp)). * * For UP kernels, however, the memory of the single processor is * always consistent, so we only need to stop the compiler from * reordering accesses in a way that violates the semantics of acquire * and release. */ #if defined(_KERNEL) #if defined(SMP) || defined(KLD_MODULE) static __inline void __storeload_barrier(void) { __asm __volatile("lock; addl $0,%%gs:%0" : "+m" (*(u_int *)OFFSETOF_MONITORBUF) : : "memory", "cc"); } #else /* _KERNEL && UP */ static __inline void __storeload_barrier(void) { __compiler_membar(); } #endif /* SMP */ #else /* !_KERNEL */ static __inline void __storeload_barrier(void) { __asm __volatile("lock; addl $0,-8(%%rsp)" : : : "memory", "cc"); } #endif /* _KERNEL*/ #define ATOMIC_LOAD(TYPE) \ static __inline u_##TYPE \ atomic_load_acq_##TYPE(volatile u_##TYPE *p) \ { \ u_##TYPE res; \ \ res = *p; \ __compiler_membar(); \ return (res); \ } \ struct __hack #define ATOMIC_STORE(TYPE) \ static __inline void \ atomic_store_rel_##TYPE(volatile u_##TYPE *p, u_##TYPE v) \ { \ \ __compiler_membar(); \ *p = v; \ } \ struct __hack static __inline void atomic_thread_fence_acq(void) { __compiler_membar(); } static __inline void atomic_thread_fence_rel(void) { __compiler_membar(); } static __inline void atomic_thread_fence_acq_rel(void) { __compiler_membar(); } static __inline void atomic_thread_fence_seq_cst(void) { __storeload_barrier(); } #endif /* KLD_MODULE || !__GNUCLIKE_ASM */ ATOMIC_ASM(set, char, "orb %b1,%0", "iq", v); ATOMIC_ASM(clear, char, "andb %b1,%0", "iq", ~v); ATOMIC_ASM(add, char, "addb %b1,%0", "iq", v); ATOMIC_ASM(subtract, char, "subb %b1,%0", "iq", v); ATOMIC_ASM(set, short, "orw %w1,%0", "ir", v); ATOMIC_ASM(clear, short, "andw %w1,%0", "ir", ~v); ATOMIC_ASM(add, short, "addw %w1,%0", "ir", v); ATOMIC_ASM(subtract, short, "subw %w1,%0", "ir", v); ATOMIC_ASM(set, int, "orl %1,%0", "ir", v); ATOMIC_ASM(clear, int, "andl %1,%0", "ir", ~v); ATOMIC_ASM(add, int, "addl %1,%0", "ir", v); ATOMIC_ASM(subtract, int, "subl %1,%0", "ir", v); ATOMIC_ASM(set, long, "orq %1,%0", "er", v); ATOMIC_ASM(clear, long, "andq %1,%0", "er", ~v); ATOMIC_ASM(add, long, "addq %1,%0", "er", v); ATOMIC_ASM(subtract, long, "subq %1,%0", "er", v); #define ATOMIC_LOADSTORE(TYPE) \ ATOMIC_LOAD(TYPE); \ ATOMIC_STORE(TYPE) ATOMIC_LOADSTORE(char); ATOMIC_LOADSTORE(short); ATOMIC_LOADSTORE(int); ATOMIC_LOADSTORE(long); #undef ATOMIC_ASM #undef ATOMIC_LOAD #undef ATOMIC_STORE #undef ATOMIC_LOADSTORE #ifndef WANT_FUNCTIONS /* Read the current value and store a new value in the destination. */ #ifdef __GNUCLIKE_ASM static __inline u_int atomic_swap_int(volatile u_int *p, u_int v) { __asm __volatile( " xchgl %1,%0 ; " "# atomic_swap_int" : "+r" (v), /* 0 */ "+m" (*p)); /* 1 */ return (v); } static __inline u_long atomic_swap_long(volatile u_long *p, u_long v) { __asm __volatile( " xchgq %1,%0 ; " "# atomic_swap_long" : "+r" (v), /* 0 */ "+m" (*p)); /* 1 */ return (v); } #else /* !__GNUCLIKE_ASM */ u_int atomic_swap_int(volatile u_int *p, u_int v); u_long atomic_swap_long(volatile u_long *p, u_long v); #endif /* __GNUCLIKE_ASM */ #define atomic_set_acq_char atomic_set_barr_char #define atomic_set_rel_char atomic_set_barr_char #define atomic_clear_acq_char atomic_clear_barr_char #define atomic_clear_rel_char atomic_clear_barr_char #define atomic_add_acq_char atomic_add_barr_char #define atomic_add_rel_char atomic_add_barr_char #define atomic_subtract_acq_char atomic_subtract_barr_char #define atomic_subtract_rel_char atomic_subtract_barr_char #define atomic_cmpset_acq_char atomic_cmpset_char #define atomic_cmpset_rel_char atomic_cmpset_char #define atomic_fcmpset_acq_char atomic_fcmpset_char #define atomic_fcmpset_rel_char atomic_fcmpset_char #define atomic_set_acq_short atomic_set_barr_short #define atomic_set_rel_short atomic_set_barr_short #define atomic_clear_acq_short atomic_clear_barr_short #define atomic_clear_rel_short atomic_clear_barr_short #define atomic_add_acq_short atomic_add_barr_short #define atomic_add_rel_short atomic_add_barr_short #define atomic_subtract_acq_short atomic_subtract_barr_short #define atomic_subtract_rel_short atomic_subtract_barr_short #define atomic_cmpset_acq_short atomic_cmpset_short #define atomic_cmpset_rel_short atomic_cmpset_short #define atomic_fcmpset_acq_short atomic_fcmpset_short #define atomic_fcmpset_rel_short atomic_fcmpset_short #define atomic_set_acq_int atomic_set_barr_int #define atomic_set_rel_int atomic_set_barr_int #define atomic_clear_acq_int atomic_clear_barr_int #define atomic_clear_rel_int atomic_clear_barr_int #define atomic_add_acq_int atomic_add_barr_int #define atomic_add_rel_int atomic_add_barr_int #define atomic_subtract_acq_int atomic_subtract_barr_int #define atomic_subtract_rel_int atomic_subtract_barr_int #define atomic_cmpset_acq_int atomic_cmpset_int #define atomic_cmpset_rel_int atomic_cmpset_int #define atomic_fcmpset_acq_int atomic_fcmpset_int #define atomic_fcmpset_rel_int atomic_fcmpset_int #define atomic_set_acq_long atomic_set_barr_long #define atomic_set_rel_long atomic_set_barr_long #define atomic_clear_acq_long atomic_clear_barr_long #define atomic_clear_rel_long atomic_clear_barr_long #define atomic_add_acq_long atomic_add_barr_long #define atomic_add_rel_long atomic_add_barr_long #define atomic_subtract_acq_long atomic_subtract_barr_long #define atomic_subtract_rel_long atomic_subtract_barr_long #define atomic_cmpset_acq_long atomic_cmpset_long #define atomic_cmpset_rel_long atomic_cmpset_long #define atomic_fcmpset_acq_long atomic_fcmpset_long #define atomic_fcmpset_rel_long atomic_fcmpset_long #define atomic_readandclear_int(p) atomic_swap_int(p, 0) #define atomic_readandclear_long(p) atomic_swap_long(p, 0) #define atomic_testandset_acq_long atomic_testandset_long /* Operations on 8-bit bytes. */ #define atomic_set_8 atomic_set_char #define atomic_set_acq_8 atomic_set_acq_char #define atomic_set_rel_8 atomic_set_rel_char #define atomic_clear_8 atomic_clear_char #define atomic_clear_acq_8 atomic_clear_acq_char #define atomic_clear_rel_8 atomic_clear_rel_char #define atomic_add_8 atomic_add_char #define atomic_add_acq_8 atomic_add_acq_char #define atomic_add_rel_8 atomic_add_rel_char #define atomic_subtract_8 atomic_subtract_char #define atomic_subtract_acq_8 atomic_subtract_acq_char #define atomic_subtract_rel_8 atomic_subtract_rel_char #define atomic_load_acq_8 atomic_load_acq_char #define atomic_store_rel_8 atomic_store_rel_char #define atomic_cmpset_8 atomic_cmpset_char #define atomic_cmpset_acq_8 atomic_cmpset_acq_char #define atomic_cmpset_rel_8 atomic_cmpset_rel_char #define atomic_fcmpset_8 atomic_fcmpset_char #define atomic_fcmpset_acq_8 atomic_fcmpset_acq_char #define atomic_fcmpset_rel_8 atomic_fcmpset_rel_char /* Operations on 16-bit words. */ #define atomic_set_16 atomic_set_short #define atomic_set_acq_16 atomic_set_acq_short #define atomic_set_rel_16 atomic_set_rel_short #define atomic_clear_16 atomic_clear_short #define atomic_clear_acq_16 atomic_clear_acq_short #define atomic_clear_rel_16 atomic_clear_rel_short #define atomic_add_16 atomic_add_short #define atomic_add_acq_16 atomic_add_acq_short #define atomic_add_rel_16 atomic_add_rel_short #define atomic_subtract_16 atomic_subtract_short #define atomic_subtract_acq_16 atomic_subtract_acq_short #define atomic_subtract_rel_16 atomic_subtract_rel_short #define atomic_load_acq_16 atomic_load_acq_short #define atomic_store_rel_16 atomic_store_rel_short #define atomic_cmpset_16 atomic_cmpset_short #define atomic_cmpset_acq_16 atomic_cmpset_acq_short #define atomic_cmpset_rel_16 atomic_cmpset_rel_short #define atomic_fcmpset_16 atomic_fcmpset_short #define atomic_fcmpset_acq_16 atomic_fcmpset_acq_short #define atomic_fcmpset_rel_16 atomic_fcmpset_rel_short /* Operations on 32-bit double words. */ #define atomic_set_32 atomic_set_int #define atomic_set_acq_32 atomic_set_acq_int #define atomic_set_rel_32 atomic_set_rel_int #define atomic_clear_32 atomic_clear_int #define atomic_clear_acq_32 atomic_clear_acq_int #define atomic_clear_rel_32 atomic_clear_rel_int #define atomic_add_32 atomic_add_int #define atomic_add_acq_32 atomic_add_acq_int #define atomic_add_rel_32 atomic_add_rel_int #define atomic_subtract_32 atomic_subtract_int #define atomic_subtract_acq_32 atomic_subtract_acq_int #define atomic_subtract_rel_32 atomic_subtract_rel_int #define atomic_load_acq_32 atomic_load_acq_int #define atomic_store_rel_32 atomic_store_rel_int #define atomic_cmpset_32 atomic_cmpset_int #define atomic_cmpset_acq_32 atomic_cmpset_acq_int #define atomic_cmpset_rel_32 atomic_cmpset_rel_int #define atomic_fcmpset_32 atomic_fcmpset_int #define atomic_fcmpset_acq_32 atomic_fcmpset_acq_int #define atomic_fcmpset_rel_32 atomic_fcmpset_rel_int #define atomic_swap_32 atomic_swap_int #define atomic_readandclear_32 atomic_readandclear_int #define atomic_fetchadd_32 atomic_fetchadd_int #define atomic_testandset_32 atomic_testandset_int #define atomic_testandclear_32 atomic_testandclear_int /* Operations on 64-bit quad words. */ #define atomic_set_64 atomic_set_long #define atomic_set_acq_64 atomic_set_acq_long #define atomic_set_rel_64 atomic_set_rel_long #define atomic_clear_64 atomic_clear_long #define atomic_clear_acq_64 atomic_clear_acq_long #define atomic_clear_rel_64 atomic_clear_rel_long #define atomic_add_64 atomic_add_long #define atomic_add_acq_64 atomic_add_acq_long #define atomic_add_rel_64 atomic_add_rel_long #define atomic_subtract_64 atomic_subtract_long #define atomic_subtract_acq_64 atomic_subtract_acq_long #define atomic_subtract_rel_64 atomic_subtract_rel_long #define atomic_load_acq_64 atomic_load_acq_long #define atomic_store_rel_64 atomic_store_rel_long #define atomic_cmpset_64 atomic_cmpset_long #define atomic_cmpset_acq_64 atomic_cmpset_acq_long #define atomic_cmpset_rel_64 atomic_cmpset_rel_long #define atomic_fcmpset_64 atomic_fcmpset_long #define atomic_fcmpset_acq_64 atomic_fcmpset_acq_long #define atomic_fcmpset_rel_64 atomic_fcmpset_rel_long #define atomic_swap_64 atomic_swap_long #define atomic_readandclear_64 atomic_readandclear_long #define atomic_fetchadd_64 atomic_fetchadd_long #define atomic_testandset_64 atomic_testandset_long #define atomic_testandclear_64 atomic_testandclear_long /* Operations on pointers. */ #define atomic_set_ptr atomic_set_long #define atomic_set_acq_ptr atomic_set_acq_long #define atomic_set_rel_ptr atomic_set_rel_long #define atomic_clear_ptr atomic_clear_long #define atomic_clear_acq_ptr atomic_clear_acq_long #define atomic_clear_rel_ptr atomic_clear_rel_long #define atomic_add_ptr atomic_add_long #define atomic_add_acq_ptr atomic_add_acq_long #define atomic_add_rel_ptr atomic_add_rel_long #define atomic_subtract_ptr atomic_subtract_long #define atomic_subtract_acq_ptr atomic_subtract_acq_long #define atomic_subtract_rel_ptr atomic_subtract_rel_long #define atomic_load_acq_ptr atomic_load_acq_long #define atomic_store_rel_ptr atomic_store_rel_long #define atomic_cmpset_ptr atomic_cmpset_long #define atomic_cmpset_acq_ptr atomic_cmpset_acq_long #define atomic_cmpset_rel_ptr atomic_cmpset_rel_long #define atomic_fcmpset_ptr atomic_fcmpset_long #define atomic_fcmpset_acq_ptr atomic_fcmpset_acq_long #define atomic_fcmpset_rel_ptr atomic_fcmpset_rel_long #define atomic_swap_ptr atomic_swap_long #define atomic_readandclear_ptr atomic_readandclear_long #endif /* !WANT_FUNCTIONS */ -#endif /* KCSAN && !KCSAN_RUNTIME */ +#endif /* !ATOMIC_SAN_PREFIX */ #endif /* !_MACHINE_ATOMIC_H_ */ diff --git a/sys/arm64/arm64/bus_machdep.c b/sys/arm64/arm64/bus_machdep.c index 7fc83e6924f2..69d7c5b591b2 100644 --- a/sys/arm64/arm64/bus_machdep.c +++ b/sys/arm64/arm64/bus_machdep.c @@ -1,292 +1,294 @@ /*- * Copyright (c) 2014 Andrew Turner * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * */ -#define KCSAN_RUNTIME +#if defined(KASAN) || defined(KCSAN) +#define SAN_RUNTIME +#endif #include "opt_platform.h" #include __FBSDID("$FreeBSD$"); #include #include #include uint8_t generic_bs_r_1(void *, bus_space_handle_t, bus_size_t); uint16_t generic_bs_r_2(void *, bus_space_handle_t, bus_size_t); uint32_t generic_bs_r_4(void *, bus_space_handle_t, bus_size_t); uint64_t generic_bs_r_8(void *, bus_space_handle_t, bus_size_t); void generic_bs_rm_1(void *, bus_space_handle_t, bus_size_t, uint8_t *, bus_size_t); void generic_bs_rm_2(void *, bus_space_handle_t, bus_size_t, uint16_t *, bus_size_t); void generic_bs_rm_4(void *, bus_space_handle_t, bus_size_t, uint32_t *, bus_size_t); void generic_bs_rm_8(void *, bus_space_handle_t, bus_size_t, uint64_t *, bus_size_t); void generic_bs_rr_1(void *, bus_space_handle_t, bus_size_t, uint8_t *, bus_size_t); void generic_bs_rr_2(void *, bus_space_handle_t, bus_size_t, uint16_t *, bus_size_t); void generic_bs_rr_4(void *, bus_space_handle_t, bus_size_t, uint32_t *, bus_size_t); void generic_bs_rr_8(void *, bus_space_handle_t, bus_size_t, uint64_t *, bus_size_t); void generic_bs_w_1(void *, bus_space_handle_t, bus_size_t, uint8_t); void generic_bs_w_2(void *, bus_space_handle_t, bus_size_t, uint16_t); void generic_bs_w_4(void *, bus_space_handle_t, bus_size_t, uint32_t); void generic_bs_w_8(void *, bus_space_handle_t, bus_size_t, uint64_t); void generic_bs_wm_1(void *, bus_space_handle_t, bus_size_t, const uint8_t *, bus_size_t); void generic_bs_wm_2(void *, bus_space_handle_t, bus_size_t, const uint16_t *, bus_size_t); void generic_bs_wm_4(void *, bus_space_handle_t, bus_size_t, const uint32_t *, bus_size_t); void generic_bs_wm_8(void *, bus_space_handle_t, bus_size_t, const uint64_t *, bus_size_t); void generic_bs_wr_1(void *, bus_space_handle_t, bus_size_t, const uint8_t *, bus_size_t); void generic_bs_wr_2(void *, bus_space_handle_t, bus_size_t, const uint16_t *, bus_size_t); void generic_bs_wr_4(void *, bus_space_handle_t, bus_size_t, const uint32_t *, bus_size_t); void generic_bs_wr_8(void *, bus_space_handle_t, bus_size_t, const uint64_t *, bus_size_t); int generic_bs_peek_1(void *, bus_space_handle_t, bus_size_t , uint8_t *); int generic_bs_peek_2(void *, bus_space_handle_t, bus_size_t , uint16_t *); int generic_bs_peek_4(void *, bus_space_handle_t, bus_size_t , uint32_t *); int generic_bs_peek_8(void *, bus_space_handle_t, bus_size_t , uint64_t *); int generic_bs_poke_1(void *, bus_space_handle_t, bus_size_t, uint8_t); int generic_bs_poke_2(void *, bus_space_handle_t, bus_size_t, uint16_t); int generic_bs_poke_4(void *, bus_space_handle_t, bus_size_t, uint32_t); int generic_bs_poke_8(void *, bus_space_handle_t, bus_size_t, uint64_t); static int generic_bs_map(void *t, bus_addr_t bpa, bus_size_t size, int flags, bus_space_handle_t *bshp) { void *va; va = pmap_mapdev(bpa, size); if (va == NULL) return (ENOMEM); *bshp = (bus_space_handle_t)va; return (0); } static void generic_bs_unmap(void *t, bus_space_handle_t bsh, bus_size_t size) { pmap_unmapdev(bsh, size); } static void generic_bs_barrier(void *t, bus_space_handle_t bsh, bus_size_t offset, bus_size_t size, int flags) { } static int generic_bs_subregion(void *t, bus_space_handle_t bsh, bus_size_t offset, bus_size_t size, bus_space_handle_t *nbshp) { *nbshp = bsh + offset; return (0); } /* * Write `count' 1, 2, 4, or 8 byte value `val' to bus space described * by tag/handle starting at `offset'. */ static void generic_bs_sr_1(void *t, bus_space_handle_t bsh, bus_size_t offset, uint8_t value, size_t count) { for (; count != 0; count--, offset++) generic_bs_w_1(t, bsh, offset, value); } static void generic_bs_sr_2(void *t, bus_space_handle_t bsh, bus_size_t offset, uint16_t value, size_t count) { for (; count != 0; count--, offset += 2) generic_bs_w_2(t, bsh, offset, value); } static void generic_bs_sr_4(void *t, bus_space_handle_t bsh, bus_size_t offset, uint32_t value, size_t count) { for (; count != 0; count--, offset += 4) generic_bs_w_4(t, bsh, offset, value); } static void generic_bs_sr_8(void *t, bus_space_handle_t bsh, bus_size_t offset, uint64_t value, size_t count) { for (; count != 0; count--, offset += 8) generic_bs_w_8(t, bsh, offset, value); } struct bus_space memmap_bus = { /* cookie */ .bs_cookie = NULL, /* mapping/unmapping */ .bs_map = generic_bs_map, .bs_unmap = generic_bs_unmap, .bs_subregion = generic_bs_subregion, /* allocation/deallocation */ .bs_alloc = NULL, .bs_free = NULL, /* barrier */ .bs_barrier = generic_bs_barrier, /* read single */ .bs_r_1 = generic_bs_r_1, .bs_r_2 = generic_bs_r_2, .bs_r_4 = generic_bs_r_4, .bs_r_8 = generic_bs_r_8, /* read multiple */ .bs_rm_1 = generic_bs_rm_1, .bs_rm_2 = generic_bs_rm_2, .bs_rm_4 = generic_bs_rm_4, .bs_rm_8 = generic_bs_rm_8, /* read region */ .bs_rr_1 = generic_bs_rr_1, .bs_rr_2 = generic_bs_rr_2, .bs_rr_4 = generic_bs_rr_4, .bs_rr_8 = generic_bs_rr_8, /* write single */ .bs_w_1 = generic_bs_w_1, .bs_w_2 = generic_bs_w_2, .bs_w_4 = generic_bs_w_4, .bs_w_8 = generic_bs_w_8, /* write multiple */ .bs_wm_1 = generic_bs_wm_1, .bs_wm_2 = generic_bs_wm_2, .bs_wm_4 = generic_bs_wm_4, .bs_wm_8 = generic_bs_wm_8, /* write region */ .bs_wr_1 = generic_bs_wr_1, .bs_wr_2 = generic_bs_wr_2, .bs_wr_4 = generic_bs_wr_4, .bs_wr_8 = generic_bs_wr_8, /* set multiple */ .bs_sm_1 = NULL, .bs_sm_2 = NULL, .bs_sm_4 = NULL, .bs_sm_8 = NULL, /* set region */ .bs_sr_1 = generic_bs_sr_1, .bs_sr_2 = generic_bs_sr_2, .bs_sr_4 = generic_bs_sr_4, .bs_sr_8 = generic_bs_sr_8, /* copy */ .bs_c_1 = NULL, .bs_c_2 = NULL, .bs_c_4 = NULL, .bs_c_8 = NULL, /* read single stream */ .bs_r_1_s = NULL, .bs_r_2_s = NULL, .bs_r_4_s = NULL, .bs_r_8_s = NULL, /* read multiple stream */ .bs_rm_1_s = generic_bs_rm_1, .bs_rm_2_s = generic_bs_rm_2, .bs_rm_4_s = generic_bs_rm_4, .bs_rm_8_s = generic_bs_rm_8, /* read region stream */ .bs_rr_1_s = NULL, .bs_rr_2_s = NULL, .bs_rr_4_s = NULL, .bs_rr_8_s = NULL, /* write single stream */ .bs_w_1_s = NULL, .bs_w_2_s = NULL, .bs_w_4_s = NULL, .bs_w_8_s = NULL, /* write multiple stream */ .bs_wm_1_s = generic_bs_wm_1, .bs_wm_2_s = generic_bs_wm_2, .bs_wm_4_s = generic_bs_wm_4, .bs_wm_8_s = generic_bs_wm_8, /* write region stream */ .bs_wr_1_s = NULL, .bs_wr_2_s = NULL, .bs_wr_4_s = NULL, .bs_wr_8_s = NULL, /* peek */ .bs_peek_1 = generic_bs_peek_1, .bs_peek_2 = generic_bs_peek_2, .bs_peek_4 = generic_bs_peek_4, .bs_peek_8 = generic_bs_peek_8, /* poke */ .bs_poke_1 = generic_bs_poke_1, .bs_poke_2 = generic_bs_poke_2, .bs_poke_4 = generic_bs_poke_4, .bs_poke_8 = generic_bs_poke_8, }; #ifdef FDT bus_space_tag_t fdtbus_bs_tag = &memmap_bus; #endif diff --git a/sys/arm64/include/bus.h b/sys/arm64/include/bus.h index 417b918f595f..e2862aa65fb0 100644 --- a/sys/arm64/include/bus.h +++ b/sys/arm64/include/bus.h @@ -1,502 +1,510 @@ /* $NetBSD: bus.h,v 1.11 2003/07/28 17:35:54 thorpej Exp $ */ /*- * Copyright (c) 1996, 1997, 1998, 2001 The NetBSD Foundation, Inc. * All rights reserved. * * This code is derived from software contributed to The NetBSD Foundation * by Jason R. Thorpe of the Numerical Aerospace Simulation Facility, * NASA Ames Research Center. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ /*- * Copyright (c) 1996 Charles M. Hannum. All rights reserved. * Copyright (c) 1996 Christopher G. Demetriou. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. All advertising materials mentioning features or use of this software * must display the following acknowledgement: * This product includes software developed by Christopher G. Demetriou * for the NetBSD Project. * 4. The name of the author may not be used to endorse or promote products * derived from this software without specific prior written permission * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * * From: sys/arm/include/bus.h * * $FreeBSD$ */ #ifndef _MACHINE_BUS_H_ #define _MACHINE_BUS_H_ #include #define BUS_SPACE_ALIGNED_POINTER(p, t) ALIGNED_POINTER(p, t) #define BUS_SPACE_MAXADDR_24BIT 0xFFFFFFUL #define BUS_SPACE_MAXADDR_32BIT 0xFFFFFFFFUL #define BUS_SPACE_MAXADDR_40BIT 0xFFFFFFFFFFUL #define BUS_SPACE_MAXSIZE_24BIT 0xFFFFFFUL #define BUS_SPACE_MAXSIZE_32BIT 0xFFFFFFFFUL #define BUS_SPACE_MAXSIZE_40BIT 0xFFFFFFFFFFUL #define BUS_SPACE_MAXADDR 0xFFFFFFFFFFFFFFFFUL #define BUS_SPACE_MAXSIZE 0xFFFFFFFFFFFFFFFFUL #define BUS_SPACE_MAP_CACHEABLE 0x01 #define BUS_SPACE_MAP_LINEAR 0x02 #define BUS_SPACE_MAP_PREFETCHABLE 0x04 #define BUS_SPACE_UNRESTRICTED (~0) #define BUS_SPACE_BARRIER_READ 0x01 #define BUS_SPACE_BARRIER_WRITE 0x02 -#if defined(KCSAN) && !defined(KCSAN_RUNTIME) +#ifndef SAN_RUNTIME +#if defined(KASAN) +#define BUS_SAN_PREFIX kasan +#elif defined(KCSAN) +#define BUS_SAN_PREFIX kcsan +#endif +#endif + +#ifdef BUS_SAN_PREFIX #include #else struct bus_space { /* cookie */ void *bs_cookie; /* mapping/unmapping */ int (*bs_map) (void *, bus_addr_t, bus_size_t, int, bus_space_handle_t *); void (*bs_unmap) (void *, bus_space_handle_t, bus_size_t); int (*bs_subregion) (void *, bus_space_handle_t, bus_size_t, bus_size_t, bus_space_handle_t *); /* allocation/deallocation */ int (*bs_alloc) (void *, bus_addr_t, bus_addr_t, bus_size_t, bus_size_t, bus_size_t, int, bus_addr_t *, bus_space_handle_t *); void (*bs_free) (void *, bus_space_handle_t, bus_size_t); /* get kernel virtual address */ /* barrier */ void (*bs_barrier) (void *, bus_space_handle_t, bus_size_t, bus_size_t, int); /* read single */ u_int8_t (*bs_r_1) (void *, bus_space_handle_t, bus_size_t); u_int16_t (*bs_r_2) (void *, bus_space_handle_t, bus_size_t); u_int32_t (*bs_r_4) (void *, bus_space_handle_t, bus_size_t); u_int64_t (*bs_r_8) (void *, bus_space_handle_t, bus_size_t); /* read multiple */ void (*bs_rm_1) (void *, bus_space_handle_t, bus_size_t, u_int8_t *, bus_size_t); void (*bs_rm_2) (void *, bus_space_handle_t, bus_size_t, u_int16_t *, bus_size_t); void (*bs_rm_4) (void *, bus_space_handle_t, bus_size_t, u_int32_t *, bus_size_t); void (*bs_rm_8) (void *, bus_space_handle_t, bus_size_t, u_int64_t *, bus_size_t); /* read region */ void (*bs_rr_1) (void *, bus_space_handle_t, bus_size_t, u_int8_t *, bus_size_t); void (*bs_rr_2) (void *, bus_space_handle_t, bus_size_t, u_int16_t *, bus_size_t); void (*bs_rr_4) (void *, bus_space_handle_t, bus_size_t, u_int32_t *, bus_size_t); void (*bs_rr_8) (void *, bus_space_handle_t, bus_size_t, u_int64_t *, bus_size_t); /* write single */ void (*bs_w_1) (void *, bus_space_handle_t, bus_size_t, u_int8_t); void (*bs_w_2) (void *, bus_space_handle_t, bus_size_t, u_int16_t); void (*bs_w_4) (void *, bus_space_handle_t, bus_size_t, u_int32_t); void (*bs_w_8) (void *, bus_space_handle_t, bus_size_t, u_int64_t); /* write multiple */ void (*bs_wm_1) (void *, bus_space_handle_t, bus_size_t, const u_int8_t *, bus_size_t); void (*bs_wm_2) (void *, bus_space_handle_t, bus_size_t, const u_int16_t *, bus_size_t); void (*bs_wm_4) (void *, bus_space_handle_t, bus_size_t, const u_int32_t *, bus_size_t); void (*bs_wm_8) (void *, bus_space_handle_t, bus_size_t, const u_int64_t *, bus_size_t); /* write region */ void (*bs_wr_1) (void *, bus_space_handle_t, bus_size_t, const u_int8_t *, bus_size_t); void (*bs_wr_2) (void *, bus_space_handle_t, bus_size_t, const u_int16_t *, bus_size_t); void (*bs_wr_4) (void *, bus_space_handle_t, bus_size_t, const u_int32_t *, bus_size_t); void (*bs_wr_8) (void *, bus_space_handle_t, bus_size_t, const u_int64_t *, bus_size_t); /* set multiple */ void (*bs_sm_1) (void *, bus_space_handle_t, bus_size_t, u_int8_t, bus_size_t); void (*bs_sm_2) (void *, bus_space_handle_t, bus_size_t, u_int16_t, bus_size_t); void (*bs_sm_4) (void *, bus_space_handle_t, bus_size_t, u_int32_t, bus_size_t); void (*bs_sm_8) (void *, bus_space_handle_t, bus_size_t, u_int64_t, bus_size_t); /* set region */ void (*bs_sr_1) (void *, bus_space_handle_t, bus_size_t, u_int8_t, bus_size_t); void (*bs_sr_2) (void *, bus_space_handle_t, bus_size_t, u_int16_t, bus_size_t); void (*bs_sr_4) (void *, bus_space_handle_t, bus_size_t, u_int32_t, bus_size_t); void (*bs_sr_8) (void *, bus_space_handle_t, bus_size_t, u_int64_t, bus_size_t); /* copy */ void (*bs_c_1) (void *, bus_space_handle_t, bus_size_t, bus_space_handle_t, bus_size_t, bus_size_t); void (*bs_c_2) (void *, bus_space_handle_t, bus_size_t, bus_space_handle_t, bus_size_t, bus_size_t); void (*bs_c_4) (void *, bus_space_handle_t, bus_size_t, bus_space_handle_t, bus_size_t, bus_size_t); void (*bs_c_8) (void *, bus_space_handle_t, bus_size_t, bus_space_handle_t, bus_size_t, bus_size_t); /* read single stream */ u_int8_t (*bs_r_1_s) (void *, bus_space_handle_t, bus_size_t); u_int16_t (*bs_r_2_s) (void *, bus_space_handle_t, bus_size_t); u_int32_t (*bs_r_4_s) (void *, bus_space_handle_t, bus_size_t); u_int64_t (*bs_r_8_s) (void *, bus_space_handle_t, bus_size_t); /* read multiple stream */ void (*bs_rm_1_s) (void *, bus_space_handle_t, bus_size_t, u_int8_t *, bus_size_t); void (*bs_rm_2_s) (void *, bus_space_handle_t, bus_size_t, u_int16_t *, bus_size_t); void (*bs_rm_4_s) (void *, bus_space_handle_t, bus_size_t, u_int32_t *, bus_size_t); void (*bs_rm_8_s) (void *, bus_space_handle_t, bus_size_t, u_int64_t *, bus_size_t); /* read region stream */ void (*bs_rr_1_s) (void *, bus_space_handle_t, bus_size_t, u_int8_t *, bus_size_t); void (*bs_rr_2_s) (void *, bus_space_handle_t, bus_size_t, u_int16_t *, bus_size_t); void (*bs_rr_4_s) (void *, bus_space_handle_t, bus_size_t, u_int32_t *, bus_size_t); void (*bs_rr_8_s) (void *, bus_space_handle_t, bus_size_t, u_int64_t *, bus_size_t); /* write single stream */ void (*bs_w_1_s) (void *, bus_space_handle_t, bus_size_t, u_int8_t); void (*bs_w_2_s) (void *, bus_space_handle_t, bus_size_t, u_int16_t); void (*bs_w_4_s) (void *, bus_space_handle_t, bus_size_t, u_int32_t); void (*bs_w_8_s) (void *, bus_space_handle_t, bus_size_t, u_int64_t); /* write multiple stream */ void (*bs_wm_1_s) (void *, bus_space_handle_t, bus_size_t, const u_int8_t *, bus_size_t); void (*bs_wm_2_s) (void *, bus_space_handle_t, bus_size_t, const u_int16_t *, bus_size_t); void (*bs_wm_4_s) (void *, bus_space_handle_t, bus_size_t, const u_int32_t *, bus_size_t); void (*bs_wm_8_s) (void *, bus_space_handle_t, bus_size_t, const u_int64_t *, bus_size_t); /* write region stream */ void (*bs_wr_1_s) (void *, bus_space_handle_t, bus_size_t, const u_int8_t *, bus_size_t); void (*bs_wr_2_s) (void *, bus_space_handle_t, bus_size_t, const u_int16_t *, bus_size_t); void (*bs_wr_4_s) (void *, bus_space_handle_t, bus_size_t, const u_int32_t *, bus_size_t); void (*bs_wr_8_s) (void *, bus_space_handle_t, bus_size_t, const u_int64_t *, bus_size_t); /* peek */ int (*bs_peek_1)(void *, bus_space_handle_t, bus_size_t , uint8_t *); int (*bs_peek_2)(void *, bus_space_handle_t, bus_size_t , uint16_t *); int (*bs_peek_4)(void *, bus_space_handle_t, bus_size_t , uint32_t *); int (*bs_peek_8)(void *, bus_space_handle_t, bus_size_t , uint64_t *); /* poke */ int (*bs_poke_1)(void *, bus_space_handle_t, bus_size_t, uint8_t); int (*bs_poke_2)(void *, bus_space_handle_t, bus_size_t, uint16_t); int (*bs_poke_4)(void *, bus_space_handle_t, bus_size_t, uint32_t); int (*bs_poke_8)(void *, bus_space_handle_t, bus_size_t, uint64_t); }; /* * Utility macros; INTERNAL USE ONLY. */ #define __bs_c(a,b) __CONCAT(a,b) #define __bs_opname(op,size) __bs_c(__bs_c(__bs_c(bs_,op),_),size) #define __bs_rs(sz, t, h, o) \ (*(t)->__bs_opname(r,sz))((t)->bs_cookie, h, o) #define __bs_ws(sz, t, h, o, v) \ (*(t)->__bs_opname(w,sz))((t)->bs_cookie, h, o, v) #define __bs_nonsingle(type, sz, t, h, o, a, c) \ (*(t)->__bs_opname(type,sz))((t)->bs_cookie, h, o, a, c) #define __bs_set(type, sz, t, h, o, v, c) \ (*(t)->__bs_opname(type,sz))((t)->bs_cookie, h, o, v, c) #define __bs_copy(sz, t, h1, o1, h2, o2, cnt) \ (*(t)->__bs_opname(c,sz))((t)->bs_cookie, h1, o1, h2, o2, cnt) #define __bs_opname_s(op,size) __bs_c(__bs_c(__bs_c(__bs_c(bs_,op),_),size),_s) #define __bs_rs_s(sz, t, h, o) \ (*(t)->__bs_opname_s(r,sz))((t)->bs_cookie, h, o) #define __bs_ws_s(sz, t, h, o, v) \ (*(t)->__bs_opname_s(w,sz))((t)->bs_cookie, h, o, v) #define __bs_peek(sz, t, h, o, vp) \ (*(t)->__bs_opname(peek, sz))((t)->bs_cookie, h, o, vp) #define __bs_poke(sz, t, h, o, v) \ (*(t)->__bs_opname(poke, sz))((t)->bs_cookie, h, o, v) #define __bs_nonsingle_s(type, sz, t, h, o, a, c) \ (*(t)->__bs_opname_s(type,sz))((t)->bs_cookie, h, o, a, c) /* * Mapping and unmapping operations. */ #define bus_space_map(t, a, s, c, hp) \ (*(t)->bs_map)((t)->bs_cookie, (a), (s), (c), (hp)) #define bus_space_unmap(t, h, s) \ (*(t)->bs_unmap)((t)->bs_cookie, (h), (s)) #define bus_space_subregion(t, h, o, s, hp) \ (*(t)->bs_subregion)((t)->bs_cookie, (h), (o), (s), (hp)) /* * Allocation and deallocation operations. */ #define bus_space_alloc(t, rs, re, s, a, b, c, ap, hp) \ (*(t)->bs_alloc)((t)->bs_cookie, (rs), (re), (s), (a), (b), \ (c), (ap), (hp)) #define bus_space_free(t, h, s) \ (*(t)->bs_free)((t)->bs_cookie, (h), (s)) /* * Bus barrier operations. */ #define bus_space_barrier(t, h, o, l, f) \ (*(t)->bs_barrier)((t)->bs_cookie, (h), (o), (l), (f)) /* * Bus read (single) operations. */ #define bus_space_read_1(t, h, o) __bs_rs(1,(t),(h),(o)) #define bus_space_read_2(t, h, o) __bs_rs(2,(t),(h),(o)) #define bus_space_read_4(t, h, o) __bs_rs(4,(t),(h),(o)) #define bus_space_read_8(t, h, o) __bs_rs(8,(t),(h),(o)) #define bus_space_read_stream_1(t, h, o) __bs_rs_s(1,(t), (h), (o)) #define bus_space_read_stream_2(t, h, o) __bs_rs_s(2,(t), (h), (o)) #define bus_space_read_stream_4(t, h, o) __bs_rs_s(4,(t), (h), (o)) #define bus_space_read_stream_8(t, h, o) __bs_rs_s(8,(t), (h), (o)) /* * Bus read multiple operations. */ #define bus_space_read_multi_1(t, h, o, a, c) \ __bs_nonsingle(rm,1,(t),(h),(o),(a),(c)) #define bus_space_read_multi_2(t, h, o, a, c) \ __bs_nonsingle(rm,2,(t),(h),(o),(a),(c)) #define bus_space_read_multi_4(t, h, o, a, c) \ __bs_nonsingle(rm,4,(t),(h),(o),(a),(c)) #define bus_space_read_multi_8(t, h, o, a, c) \ __bs_nonsingle(rm,8,(t),(h),(o),(a),(c)) #define bus_space_read_multi_stream_1(t, h, o, a, c) \ __bs_nonsingle_s(rm,1,(t),(h),(o),(a),(c)) #define bus_space_read_multi_stream_2(t, h, o, a, c) \ __bs_nonsingle_s(rm,2,(t),(h),(o),(a),(c)) #define bus_space_read_multi_stream_4(t, h, o, a, c) \ __bs_nonsingle_s(rm,4,(t),(h),(o),(a),(c)) #define bus_space_read_multi_stream_8(t, h, o, a, c) \ __bs_nonsingle_s(rm,8,(t),(h),(o),(a),(c)) /* * Bus read region operations. */ #define bus_space_read_region_1(t, h, o, a, c) \ __bs_nonsingle(rr,1,(t),(h),(o),(a),(c)) #define bus_space_read_region_2(t, h, o, a, c) \ __bs_nonsingle(rr,2,(t),(h),(o),(a),(c)) #define bus_space_read_region_4(t, h, o, a, c) \ __bs_nonsingle(rr,4,(t),(h),(o),(a),(c)) #define bus_space_read_region_8(t, h, o, a, c) \ __bs_nonsingle(rr,8,(t),(h),(o),(a),(c)) #define bus_space_read_region_stream_1(t, h, o, a, c) \ __bs_nonsingle_s(rr,1,(t),(h),(o),(a),(c)) #define bus_space_read_region_stream_2(t, h, o, a, c) \ __bs_nonsingle_s(rr,2,(t),(h),(o),(a),(c)) #define bus_space_read_region_stream_4(t, h, o, a, c) \ __bs_nonsingle_s(rr,4,(t),(h),(o),(a),(c)) #define bus_space_read_region_stream_8(t, h, o, a, c) \ __bs_nonsingle_s(rr,8,(t),(h),(o),(a),(c)) /* * Bus write (single) operations. */ #define bus_space_write_1(t, h, o, v) __bs_ws(1,(t),(h),(o),(v)) #define bus_space_write_2(t, h, o, v) __bs_ws(2,(t),(h),(o),(v)) #define bus_space_write_4(t, h, o, v) __bs_ws(4,(t),(h),(o),(v)) #define bus_space_write_8(t, h, o, v) __bs_ws(8,(t),(h),(o),(v)) #define bus_space_write_stream_1(t, h, o, v) __bs_ws_s(1,(t),(h),(o),(v)) #define bus_space_write_stream_2(t, h, o, v) __bs_ws_s(2,(t),(h),(o),(v)) #define bus_space_write_stream_4(t, h, o, v) __bs_ws_s(4,(t),(h),(o),(v)) #define bus_space_write_stream_8(t, h, o, v) __bs_ws_s(8,(t),(h),(o),(v)) /* * Bus write multiple operations. */ #define bus_space_write_multi_1(t, h, o, a, c) \ __bs_nonsingle(wm,1,(t),(h),(o),(a),(c)) #define bus_space_write_multi_2(t, h, o, a, c) \ __bs_nonsingle(wm,2,(t),(h),(o),(a),(c)) #define bus_space_write_multi_4(t, h, o, a, c) \ __bs_nonsingle(wm,4,(t),(h),(o),(a),(c)) #define bus_space_write_multi_8(t, h, o, a, c) \ __bs_nonsingle(wm,8,(t),(h),(o),(a),(c)) #define bus_space_write_multi_stream_1(t, h, o, a, c) \ __bs_nonsingle_s(wm,1,(t),(h),(o),(a),(c)) #define bus_space_write_multi_stream_2(t, h, o, a, c) \ __bs_nonsingle_s(wm,2,(t),(h),(o),(a),(c)) #define bus_space_write_multi_stream_4(t, h, o, a, c) \ __bs_nonsingle_s(wm,4,(t),(h),(o),(a),(c)) #define bus_space_write_multi_stream_8(t, h, o, a, c) \ __bs_nonsingle_s(wm,8,(t),(h),(o),(a),(c)) /* * Bus write region operations. */ #define bus_space_write_region_1(t, h, o, a, c) \ __bs_nonsingle(wr,1,(t),(h),(o),(a),(c)) #define bus_space_write_region_2(t, h, o, a, c) \ __bs_nonsingle(wr,2,(t),(h),(o),(a),(c)) #define bus_space_write_region_4(t, h, o, a, c) \ __bs_nonsingle(wr,4,(t),(h),(o),(a),(c)) #define bus_space_write_region_8(t, h, o, a, c) \ __bs_nonsingle(wr,8,(t),(h),(o),(a),(c)) #define bus_space_write_region_stream_1(t, h, o, a, c) \ __bs_nonsingle_s(wr,1,(t),(h),(o),(a),(c)) #define bus_space_write_region_stream_2(t, h, o, a, c) \ __bs_nonsingle_s(wr,2,(t),(h),(o),(a),(c)) #define bus_space_write_region_stream_4(t, h, o, a, c) \ __bs_nonsingle_s(wr,4,(t),(h),(o),(a),(c)) #define bus_space_write_region_stream_8(t, h, o, a, c) \ __bs_nonsingle_s(wr,8,(t),(h),(o),(a),(c)) /* * Set multiple operations. */ #define bus_space_set_multi_1(t, h, o, v, c) \ __bs_set(sm,1,(t),(h),(o),(v),(c)) #define bus_space_set_multi_2(t, h, o, v, c) \ __bs_set(sm,2,(t),(h),(o),(v),(c)) #define bus_space_set_multi_4(t, h, o, v, c) \ __bs_set(sm,4,(t),(h),(o),(v),(c)) #define bus_space_set_multi_8(t, h, o, v, c) \ __bs_set(sm,8,(t),(h),(o),(v),(c)) /* * Set region operations. */ #define bus_space_set_region_1(t, h, o, v, c) \ __bs_set(sr,1,(t),(h),(o),(v),(c)) #define bus_space_set_region_2(t, h, o, v, c) \ __bs_set(sr,2,(t),(h),(o),(v),(c)) #define bus_space_set_region_4(t, h, o, v, c) \ __bs_set(sr,4,(t),(h),(o),(v),(c)) #define bus_space_set_region_8(t, h, o, v, c) \ __bs_set(sr,8,(t),(h),(o),(v),(c)) /* * Copy operations. */ #define bus_space_copy_region_1(t, h1, o1, h2, o2, c) \ __bs_copy(1, t, h1, o1, h2, o2, c) #define bus_space_copy_region_2(t, h1, o1, h2, o2, c) \ __bs_copy(2, t, h1, o1, h2, o2, c) #define bus_space_copy_region_4(t, h1, o1, h2, o2, c) \ __bs_copy(4, t, h1, o1, h2, o2, c) #define bus_space_copy_region_8(t, h1, o1, h2, o2, c) \ __bs_copy(8, t, h1, o1, h2, o2, c) /* * Poke (checked write) operations. */ #define bus_space_poke_1(t, h, o, v) __bs_poke(1, (t), (h), (o), (v)) #define bus_space_poke_2(t, h, o, v) __bs_poke(2, (t), (h), (o), (v)) #define bus_space_poke_4(t, h, o, v) __bs_poke(4, (t), (h), (o), (v)) #define bus_space_poke_8(t, h, o, v) __bs_poke(8, (t), (h), (o), (v)) /* * Peek (checked read) operations. */ #define bus_space_peek_1(t, h, o, vp) __bs_peek(1, (t), (h), (o), (vp)) #define bus_space_peek_2(t, h, o, vp) __bs_peek(2, (t), (h), (o), (vp)) #define bus_space_peek_4(t, h, o, vp) __bs_peek(4, (t), (h), (o), (vp)) #define bus_space_peek_8(t, h, o, vp) __bs_peek(8, (t), (h), (o), (vp)) #endif #include #endif /* _MACHINE_BUS_H_ */ diff --git a/sys/kern/kern_kcov.c b/sys/kern/kern_kcov.c index cedfa2c081ba..23e0da4cdb79 100644 --- a/sys/kern/kern_kcov.c +++ b/sys/kern/kern_kcov.c @@ -1,581 +1,583 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (C) 2018 The FreeBSD Foundation. All rights reserved. * Copyright (C) 2018, 2019 Andrew Turner * * This software was developed by Mitchell Horne under sponsorship of * the FreeBSD Foundation. * * This software was developed by SRI International and the University of * Cambridge Computer Laboratory under DARPA/AFRL contract FA8750-10-C-0237 * ("CTSRD"), as part of the DARPA CRASH research programme. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ -#define KCSAN_RUNTIME +#ifdef KCSAN +#define SAN_RUNTIME +#endif #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include MALLOC_DEFINE(M_KCOV_INFO, "kcovinfo", "KCOV info type"); #define KCOV_ELEMENT_SIZE sizeof(uint64_t) /* * To know what the code can safely perform at any point in time we use a * state machine. In the normal case the state transitions are: * * OPEN -> READY -> RUNNING -> DYING * | | ^ | ^ ^ * | | +--------+ | | * | +-------------------+ | * +-----------------------------+ * * The states are: * OPEN: The kcov fd has been opened, but no buffer is available to store * coverage data. * READY: The buffer to store coverage data has been allocated. Userspace * can set this by using ioctl(fd, KIOSETBUFSIZE, entries);. When * this has been set the buffer can be written to by the kernel, * and mmaped by userspace. * RUNNING: The coverage probes are able to store coverage data in the buffer. * This is entered with ioctl(fd, KIOENABLE, mode);. The READY state * can be exited by ioctl(fd, KIODISABLE); or exiting the thread to * return to the READY state to allow tracing to be reused, or by * closing the kcov fd to enter the DYING state. * DYING: The fd has been closed. All states can enter into this state when * userspace closes the kcov fd. * * We need to be careful when moving into and out of the RUNNING state. As * an interrupt may happen while this is happening the ordering of memory * operations is important so struct kcov_info is valid for the tracing * functions. * * When moving into the RUNNING state prior stores to struct kcov_info need * to be observed before the state is set. This allows for interrupts that * may call into one of the coverage functions to fire at any point while * being enabled and see a consistent struct kcov_info. * * When moving out of the RUNNING state any later stores to struct kcov_info * need to be observed after the state is set. As with entering this is to * present a consistent struct kcov_info to interrupts. */ typedef enum { KCOV_STATE_INVALID, KCOV_STATE_OPEN, /* The device is open, but with no buffer */ KCOV_STATE_READY, /* The buffer has been allocated */ KCOV_STATE_RUNNING, /* Recording trace data */ KCOV_STATE_DYING, /* The fd was closed */ } kcov_state_t; /* * (l) Set while holding the kcov_lock mutex and not in the RUNNING state. * (o) Only set once while in the OPEN state. Cleaned up while in the DYING * state, and with no thread associated with the struct kcov_info. * (s) Set atomically to enter or exit the RUNNING state, non-atomically * otherwise. See above for a description of the other constraints while * moving into or out of the RUNNING state. */ struct kcov_info { struct thread *thread; /* (l) */ vm_object_t bufobj; /* (o) */ vm_offset_t kvaddr; /* (o) */ size_t entries; /* (o) */ size_t bufsize; /* (o) */ kcov_state_t state; /* (s) */ int mode; /* (l) */ }; /* Prototypes */ static d_open_t kcov_open; static d_close_t kcov_close; static d_mmap_single_t kcov_mmap_single; static d_ioctl_t kcov_ioctl; static int kcov_alloc(struct kcov_info *info, size_t entries); static void kcov_free(struct kcov_info *info); static void kcov_init(const void *unused); static struct cdevsw kcov_cdevsw = { .d_version = D_VERSION, .d_open = kcov_open, .d_close = kcov_close, .d_mmap_single = kcov_mmap_single, .d_ioctl = kcov_ioctl, .d_name = "kcov", }; SYSCTL_NODE(_kern, OID_AUTO, kcov, CTLFLAG_RW | CTLFLAG_MPSAFE, 0, "Kernel coverage"); static u_int kcov_max_entries = KCOV_MAXENTRIES; SYSCTL_UINT(_kern_kcov, OID_AUTO, max_entries, CTLFLAG_RW, &kcov_max_entries, 0, "Maximum number of entries in the kcov buffer"); static struct mtx kcov_lock; static int active_count; static struct kcov_info * get_kinfo(struct thread *td) { struct kcov_info *info; /* We might have a NULL thread when releasing the secondary CPUs */ if (td == NULL) return (NULL); /* * We are in an interrupt, stop tracing as it is not explicitly * part of a syscall. */ if (td->td_intr_nesting_level > 0 || td->td_intr_frame != NULL) return (NULL); /* * If info is NULL or the state is not running we are not tracing. */ info = td->td_kcov_info; if (info == NULL || atomic_load_acq_int(&info->state) != KCOV_STATE_RUNNING) return (NULL); return (info); } static void trace_pc(uintptr_t ret) { struct thread *td; struct kcov_info *info; uint64_t *buf, index; td = curthread; info = get_kinfo(td); if (info == NULL) return; /* * Check we are in the PC-trace mode. */ if (info->mode != KCOV_MODE_TRACE_PC) return; KASSERT(info->kvaddr != 0, ("__sanitizer_cov_trace_pc: NULL buf while running")); buf = (uint64_t *)info->kvaddr; /* The first entry of the buffer holds the index */ index = buf[0]; if (index + 2 > info->entries) return; buf[index + 1] = ret; buf[0] = index + 1; } static bool trace_cmp(uint64_t type, uint64_t arg1, uint64_t arg2, uint64_t ret) { struct thread *td; struct kcov_info *info; uint64_t *buf, index; td = curthread; info = get_kinfo(td); if (info == NULL) return (false); /* * Check we are in the comparison-trace mode. */ if (info->mode != KCOV_MODE_TRACE_CMP) return (false); KASSERT(info->kvaddr != 0, ("__sanitizer_cov_trace_pc: NULL buf while running")); buf = (uint64_t *)info->kvaddr; /* The first entry of the buffer holds the index */ index = buf[0]; /* Check we have space to store all elements */ if (index * 4 + 4 + 1 > info->entries) return (false); while (1) { buf[index * 4 + 1] = type; buf[index * 4 + 2] = arg1; buf[index * 4 + 3] = arg2; buf[index * 4 + 4] = ret; if (atomic_cmpset_64(&buf[0], index, index + 1)) break; buf[0] = index; } return (true); } /* * The fd is being closed, cleanup everything we can. */ static void kcov_mmap_cleanup(void *arg) { struct kcov_info *info = arg; struct thread *thread; mtx_lock_spin(&kcov_lock); /* * Move to KCOV_STATE_DYING to stop adding new entries. * * If the thread is running we need to wait until thread exit to * clean up as it may currently be adding a new entry. If this is * the case being in KCOV_STATE_DYING will signal that the buffer * needs to be cleaned up. */ atomic_store_int(&info->state, KCOV_STATE_DYING); atomic_thread_fence_seq_cst(); thread = info->thread; mtx_unlock_spin(&kcov_lock); if (thread != NULL) return; /* * We can safely clean up the info struct as it is in the * KCOV_STATE_DYING state with no thread associated. * * The KCOV_STATE_DYING stops new threads from using it. * The lack of a thread means nothing is currently using the buffers. */ kcov_free(info); } static int kcov_open(struct cdev *dev, int oflags, int devtype, struct thread *td) { struct kcov_info *info; int error; info = malloc(sizeof(struct kcov_info), M_KCOV_INFO, M_ZERO | M_WAITOK); info->state = KCOV_STATE_OPEN; info->thread = NULL; info->mode = -1; if ((error = devfs_set_cdevpriv(info, kcov_mmap_cleanup)) != 0) kcov_mmap_cleanup(info); return (error); } static int kcov_close(struct cdev *dev, int fflag, int devtype, struct thread *td) { struct kcov_info *info; int error; if ((error = devfs_get_cdevpriv((void **)&info)) != 0) return (error); KASSERT(info != NULL, ("kcov_close with no kcov_info structure")); /* Trying to close, but haven't disabled */ if (info->state == KCOV_STATE_RUNNING) return (EBUSY); return (0); } static int kcov_mmap_single(struct cdev *dev, vm_ooffset_t *offset, vm_size_t size, struct vm_object **object, int nprot) { struct kcov_info *info; int error; if ((nprot & (PROT_EXEC | PROT_READ | PROT_WRITE)) != (PROT_READ | PROT_WRITE)) return (EINVAL); if ((error = devfs_get_cdevpriv((void **)&info)) != 0) return (error); if (info->kvaddr == 0 || size / KCOV_ELEMENT_SIZE != info->entries) return (EINVAL); vm_object_reference(info->bufobj); *offset = 0; *object = info->bufobj; return (0); } static int kcov_alloc(struct kcov_info *info, size_t entries) { size_t n, pages; vm_page_t m; KASSERT(info->kvaddr == 0, ("kcov_alloc: Already have a buffer")); KASSERT(info->state == KCOV_STATE_OPEN, ("kcov_alloc: Not in open state (%x)", info->state)); if (entries < 2 || entries > kcov_max_entries) return (EINVAL); /* Align to page size so mmap can't access other kernel memory */ info->bufsize = roundup2(entries * KCOV_ELEMENT_SIZE, PAGE_SIZE); pages = info->bufsize / PAGE_SIZE; if ((info->kvaddr = kva_alloc(info->bufsize)) == 0) return (ENOMEM); info->bufobj = vm_pager_allocate(OBJT_PHYS, 0, info->bufsize, PROT_READ | PROT_WRITE, 0, curthread->td_ucred); VM_OBJECT_WLOCK(info->bufobj); for (n = 0; n < pages; n++) { m = vm_page_grab(info->bufobj, n, VM_ALLOC_ZERO | VM_ALLOC_WIRED); vm_page_valid(m); vm_page_xunbusy(m); pmap_qenter(info->kvaddr + n * PAGE_SIZE, &m, 1); } VM_OBJECT_WUNLOCK(info->bufobj); info->entries = entries; return (0); } static void kcov_free(struct kcov_info *info) { vm_page_t m; size_t i; if (info->kvaddr != 0) { pmap_qremove(info->kvaddr, info->bufsize / PAGE_SIZE); kva_free(info->kvaddr, info->bufsize); } if (info->bufobj != NULL) { VM_OBJECT_WLOCK(info->bufobj); m = vm_page_lookup(info->bufobj, 0); for (i = 0; i < info->bufsize / PAGE_SIZE; i++) { vm_page_unwire_noq(m); m = vm_page_next(m); } VM_OBJECT_WUNLOCK(info->bufobj); vm_object_deallocate(info->bufobj); } free(info, M_KCOV_INFO); } static int kcov_ioctl(struct cdev *dev, u_long cmd, caddr_t data, int fflag __unused, struct thread *td) { struct kcov_info *info; int mode, error; if ((error = devfs_get_cdevpriv((void **)&info)) != 0) return (error); if (cmd == KIOSETBUFSIZE) { /* * Set the size of the coverage buffer. Should be called * before enabling coverage collection for that thread. */ if (info->state != KCOV_STATE_OPEN) { return (EBUSY); } error = kcov_alloc(info, *(u_int *)data); if (error == 0) info->state = KCOV_STATE_READY; return (error); } mtx_lock_spin(&kcov_lock); switch (cmd) { case KIOENABLE: if (info->state != KCOV_STATE_READY) { error = EBUSY; break; } if (td->td_kcov_info != NULL) { error = EINVAL; break; } mode = *(int *)data; if (mode != KCOV_MODE_TRACE_PC && mode != KCOV_MODE_TRACE_CMP) { error = EINVAL; break; } /* Lets hope nobody opens this 2 billion times */ KASSERT(active_count < INT_MAX, ("%s: Open too many times", __func__)); active_count++; if (active_count == 1) { cov_register_pc(&trace_pc); cov_register_cmp(&trace_cmp); } KASSERT(info->thread == NULL, ("Enabling kcov when already enabled")); info->thread = td; info->mode = mode; /* * Ensure the mode has been set before starting coverage * tracing. */ atomic_store_rel_int(&info->state, KCOV_STATE_RUNNING); td->td_kcov_info = info; break; case KIODISABLE: /* Only the currently enabled thread may disable itself */ if (info->state != KCOV_STATE_RUNNING || info != td->td_kcov_info) { error = EINVAL; break; } KASSERT(active_count > 0, ("%s: Open count is zero", __func__)); active_count--; if (active_count == 0) { cov_unregister_pc(); cov_unregister_cmp(); } td->td_kcov_info = NULL; atomic_store_int(&info->state, KCOV_STATE_READY); /* * Ensure we have exited the READY state before clearing the * rest of the info struct. */ atomic_thread_fence_rel(); info->mode = -1; info->thread = NULL; break; default: error = EINVAL; break; } mtx_unlock_spin(&kcov_lock); return (error); } static void kcov_thread_dtor(void *arg __unused, struct thread *td) { struct kcov_info *info; info = td->td_kcov_info; if (info == NULL) return; mtx_lock_spin(&kcov_lock); KASSERT(active_count > 0, ("%s: Open count is zero", __func__)); active_count--; if (active_count == 0) { cov_unregister_pc(); cov_unregister_cmp(); } td->td_kcov_info = NULL; if (info->state != KCOV_STATE_DYING) { /* * The kcov file is still open. Mark it as unused and * wait for it to be closed before cleaning up. */ atomic_store_int(&info->state, KCOV_STATE_READY); atomic_thread_fence_seq_cst(); /* This info struct is unused */ info->thread = NULL; mtx_unlock_spin(&kcov_lock); return; } mtx_unlock_spin(&kcov_lock); /* * We can safely clean up the info struct as it is in the * KCOV_STATE_DYING state where the info struct is associated with * the current thread that's about to exit. * * The KCOV_STATE_DYING stops new threads from using it. * It also stops the current thread from trying to use the info struct. */ kcov_free(info); } static void kcov_init(const void *unused) { struct make_dev_args args; struct cdev *dev; mtx_init(&kcov_lock, "kcov lock", NULL, MTX_SPIN); make_dev_args_init(&args); args.mda_devsw = &kcov_cdevsw; args.mda_uid = UID_ROOT; args.mda_gid = GID_WHEEL; args.mda_mode = 0600; if (make_dev_s(&args, &dev, "kcov") != 0) { printf("%s", "Failed to create kcov device"); return; } EVENTHANDLER_REGISTER(thread_dtor, kcov_thread_dtor, NULL, EVENTHANDLER_PRI_ANY); } SYSINIT(kcovdev, SI_SUB_LAST, SI_ORDER_ANY, kcov_init, NULL); diff --git a/sys/kern/subr_coverage.c b/sys/kern/subr_coverage.c index f3ab27c2ca3d..9a719bcaecad 100644 --- a/sys/kern/subr_coverage.c +++ b/sys/kern/subr_coverage.c @@ -1,239 +1,241 @@ /*- * SPDX-License-Identifier: BSD-2-Clause-FreeBSD * * Copyright (C) 2018 The FreeBSD Foundation. All rights reserved. * Copyright (C) 2018, 2019 Andrew Turner * * This software was developed by Mitchell Horne under sponsorship of * the FreeBSD Foundation. * * This software was developed by SRI International and the University of * Cambridge Computer Laboratory under DARPA/AFRL contract FA8750-10-C-0237 * ("CTSRD"), as part of the DARPA CRASH research programme. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ -#define KCSAN_RUNTIME +#ifdef KCSAN +#define SAN_RUNTIME +#endif #include __FBSDID("$FreeBSD$"); #include #include #include void __sanitizer_cov_trace_pc(void); void __sanitizer_cov_trace_cmp1(uint8_t, uint8_t); void __sanitizer_cov_trace_cmp2(uint16_t, uint16_t); void __sanitizer_cov_trace_cmp4(uint32_t, uint32_t); void __sanitizer_cov_trace_cmp8(uint64_t, uint64_t); void __sanitizer_cov_trace_const_cmp1(uint8_t, uint8_t); void __sanitizer_cov_trace_const_cmp2(uint16_t, uint16_t); void __sanitizer_cov_trace_const_cmp4(uint32_t, uint32_t); void __sanitizer_cov_trace_const_cmp8(uint64_t, uint64_t); void __sanitizer_cov_trace_switch(uint64_t, uint64_t *); static cov_trace_pc_t cov_trace_pc; static cov_trace_cmp_t cov_trace_cmp; void cov_register_pc(cov_trace_pc_t trace_pc) { atomic_store_ptr(&cov_trace_pc, trace_pc); } void cov_unregister_pc(void) { atomic_store_ptr(&cov_trace_pc, NULL); } void cov_register_cmp(cov_trace_cmp_t trace_cmp) { atomic_store_ptr(&cov_trace_cmp, trace_cmp); } void cov_unregister_cmp(void) { atomic_store_ptr(&cov_trace_cmp, NULL); } /* * Main entry point. A call to this function will be inserted * at every edge, and if coverage is enabled for the thread * this function will add the PC to the buffer. */ void __sanitizer_cov_trace_pc(void) { cov_trace_pc_t trace_pc; trace_pc = atomic_load_ptr(&cov_trace_pc); if (trace_pc != NULL) trace_pc((uint64_t)__builtin_return_address(0)); } /* * Comparison entry points. When the kernel performs a comparison * operation the compiler inserts a call to one of the following * functions to record the operation. */ void __sanitizer_cov_trace_cmp1(uint8_t arg1, uint8_t arg2) { cov_trace_cmp_t trace_cmp; trace_cmp = atomic_load_ptr(&cov_trace_cmp); if (trace_cmp != NULL) trace_cmp(COV_CMP_SIZE(0), arg1, arg2, (uint64_t)__builtin_return_address(0)); } void __sanitizer_cov_trace_cmp2(uint16_t arg1, uint16_t arg2) { cov_trace_cmp_t trace_cmp; trace_cmp = atomic_load_ptr(&cov_trace_cmp); if (trace_cmp != NULL) trace_cmp(COV_CMP_SIZE(1), arg1, arg2, (uint64_t)__builtin_return_address(0)); } void __sanitizer_cov_trace_cmp4(uint32_t arg1, uint32_t arg2) { cov_trace_cmp_t trace_cmp; trace_cmp = atomic_load_ptr(&cov_trace_cmp); if (trace_cmp != NULL) trace_cmp(COV_CMP_SIZE(2), arg1, arg2, (uint64_t)__builtin_return_address(0)); } void __sanitizer_cov_trace_cmp8(uint64_t arg1, uint64_t arg2) { cov_trace_cmp_t trace_cmp; trace_cmp = atomic_load_ptr(&cov_trace_cmp); if (trace_cmp != NULL) trace_cmp(COV_CMP_SIZE(3), arg1, arg2, (uint64_t)__builtin_return_address(0)); } void __sanitizer_cov_trace_const_cmp1(uint8_t arg1, uint8_t arg2) { cov_trace_cmp_t trace_cmp; trace_cmp = atomic_load_ptr(&cov_trace_cmp); if (trace_cmp != NULL) trace_cmp(COV_CMP_SIZE(0) | COV_CMP_CONST, arg1, arg2, (uint64_t)__builtin_return_address(0)); } void __sanitizer_cov_trace_const_cmp2(uint16_t arg1, uint16_t arg2) { cov_trace_cmp_t trace_cmp; trace_cmp = atomic_load_ptr(&cov_trace_cmp); if (trace_cmp != NULL) trace_cmp(COV_CMP_SIZE(1) | COV_CMP_CONST, arg1, arg2, (uint64_t)__builtin_return_address(0)); } void __sanitizer_cov_trace_const_cmp4(uint32_t arg1, uint32_t arg2) { cov_trace_cmp_t trace_cmp; trace_cmp = atomic_load_ptr(&cov_trace_cmp); if (trace_cmp != NULL) trace_cmp(COV_CMP_SIZE(2) | COV_CMP_CONST, arg1, arg2, (uint64_t)__builtin_return_address(0)); } void __sanitizer_cov_trace_const_cmp8(uint64_t arg1, uint64_t arg2) { cov_trace_cmp_t trace_cmp; trace_cmp = atomic_load_ptr(&cov_trace_cmp); if (trace_cmp != NULL) trace_cmp(COV_CMP_SIZE(3) | COV_CMP_CONST, arg1, arg2, (uint64_t)__builtin_return_address(0)); } /* * val is the switch operand * cases[0] is the number of case constants * cases[1] is the size of val in bits * cases[2..n] are the case constants */ void __sanitizer_cov_trace_switch(uint64_t val, uint64_t *cases) { uint64_t i, count, ret, type; cov_trace_cmp_t trace_cmp; trace_cmp = atomic_load_ptr(&cov_trace_cmp); if (trace_cmp == NULL) return; count = cases[0]; ret = (uint64_t)__builtin_return_address(0); switch (cases[1]) { case 8: type = COV_CMP_SIZE(0); break; case 16: type = COV_CMP_SIZE(1); break; case 32: type = COV_CMP_SIZE(2); break; case 64: type = COV_CMP_SIZE(3); break; default: return; } val |= COV_CMP_CONST; for (i = 0; i < count; i++) if (!trace_cmp(type, val, cases[i + 2], ret)) return; } diff --git a/sys/kern/subr_csan.c b/sys/kern/subr_csan.c index ec2fd23729b2..06b0b6ebb020 100644 --- a/sys/kern/subr_csan.c +++ b/sys/kern/subr_csan.c @@ -1,910 +1,918 @@ /* $NetBSD: subr_csan.c,v 1.5 2019/11/15 08:11:37 maxv Exp $ */ /* * Copyright (c) 2019 The NetBSD Foundation, Inc. * All rights reserved. * Copyright (c) 2019 Andrew Turner * * This code is derived from software contributed to The NetBSD Foundation * by Maxime Villard. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ -#define KCSAN_RUNTIME +#define SAN_RUNTIME #include "opt_ddb.h" #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #ifdef KCSAN_PANIC #define REPORT panic #else #define REPORT printf #endif typedef struct { uintptr_t addr; uint32_t size; bool write:1; bool atomic:1; uintptr_t pc; } csan_cell_t; typedef struct { bool inited; uint32_t cnt; csan_cell_t cell; } csan_cpu_t; static csan_cpu_t kcsan_cpus[MAXCPU]; static bool kcsan_enabled __read_mostly; #define __RET_ADDR (uintptr_t)__builtin_return_address(0) #define KCSAN_NACCESSES 1024 #define KCSAN_DELAY 10 /* 10 microseconds */ /* -------------------------------------------------------------------------- */ /* The MD code. */ #include /* -------------------------------------------------------------------------- */ static void kcsan_enable(void *dummy __unused) { printf("Enabling KCSCAN, expect reduced performance.\n"); kcsan_enabled = true; } SYSINIT(kcsan_enable, SI_SUB_SMP, SI_ORDER_SECOND, kcsan_enable, NULL); void kcsan_cpu_init(u_int cpu) { kcsan_cpus[cpu].inited = true; } /* -------------------------------------------------------------------------- */ static inline void kcsan_report(csan_cell_t *new, u_int newcpu, csan_cell_t *old, u_int oldcpu) { const char *newsym, *oldsym; #ifdef DDB c_db_sym_t sym; db_expr_t offset; sym = db_search_symbol((vm_offset_t)new->pc, DB_STGY_PROC, &offset); db_symbol_values(sym, &newsym, NULL); sym = db_search_symbol((vm_offset_t)old->pc, DB_STGY_PROC, &offset); db_symbol_values(sym, &oldsym, NULL); #else newsym = ""; oldsym = ""; #endif REPORT("CSan: Racy Access " "[Cpu%u %s%s Addr=%p Size=%u PC=%p<%s>] " "[Cpu%u %s%s Addr=%p Size=%u PC=%p<%s>]\n", newcpu, (new->atomic ? "Atomic " : ""), (new->write ? "Write" : "Read"), (void *)new->addr, new->size, (void *)new->pc, newsym, oldcpu, (old->atomic ? "Atomic " : ""), (old->write ? "Write" : "Read"), (void *)old->addr, old->size, (void *)old->pc, oldsym); kcsan_md_unwind(); } static inline bool kcsan_access_is_atomic(csan_cell_t *new, csan_cell_t *old) { if (new->write && !new->atomic) return false; if (old->write && !old->atomic) return false; return true; } static inline void kcsan_access(uintptr_t addr, size_t size, bool write, bool atomic, uintptr_t pc) { csan_cell_t old, new; csan_cpu_t *cpu; uint64_t intr; size_t i; if (__predict_false(!kcsan_enabled)) return; if (__predict_false(kcsan_md_unsupported((vm_offset_t)addr))) return; if (KERNEL_PANICKED()) return; new.addr = addr; new.size = size; new.write = write; new.atomic = atomic; new.pc = pc; CPU_FOREACH(i) { __builtin_memcpy(&old, &kcsan_cpus[i].cell, sizeof(old)); if (old.addr + old.size <= new.addr) continue; if (new.addr + new.size <= old.addr) continue; if (__predict_true(!old.write && !new.write)) continue; if (__predict_true(kcsan_access_is_atomic(&new, &old))) continue; kcsan_report(&new, PCPU_GET(cpuid), &old, i); break; } if (__predict_false(!kcsan_md_is_avail())) return; kcsan_md_disable_intrs(&intr); cpu = &kcsan_cpus[PCPU_GET(cpuid)]; if (__predict_false(!cpu->inited)) goto out; cpu->cnt = (cpu->cnt + 1) % KCSAN_NACCESSES; if (__predict_true(cpu->cnt != 0)) goto out; __builtin_memcpy(&cpu->cell, &new, sizeof(new)); kcsan_md_delay(KCSAN_DELAY); __builtin_memset(&cpu->cell, 0, sizeof(new)); out: kcsan_md_enable_intrs(&intr); } #define CSAN_READ(size) \ void __tsan_read##size(uintptr_t); \ void __tsan_read##size(uintptr_t addr) \ { \ kcsan_access(addr, size, false, false, __RET_ADDR); \ } \ void __tsan_unaligned_read##size(uintptr_t); \ void __tsan_unaligned_read##size(uintptr_t addr) \ { \ kcsan_access(addr, size, false, false, __RET_ADDR); \ } CSAN_READ(1) CSAN_READ(2) CSAN_READ(4) CSAN_READ(8) CSAN_READ(16) #define CSAN_WRITE(size) \ void __tsan_write##size(uintptr_t); \ void __tsan_write##size(uintptr_t addr) \ { \ kcsan_access(addr, size, true, false, __RET_ADDR); \ } \ void __tsan_unaligned_write##size(uintptr_t); \ void __tsan_unaligned_write##size(uintptr_t addr) \ { \ kcsan_access(addr, size, true, false, __RET_ADDR); \ } CSAN_WRITE(1) CSAN_WRITE(2) CSAN_WRITE(4) CSAN_WRITE(8) CSAN_WRITE(16) void __tsan_read_range(uintptr_t, size_t); void __tsan_write_range(uintptr_t, size_t); void __tsan_read_range(uintptr_t addr, size_t size) { kcsan_access(addr, size, false, false, __RET_ADDR); } void __tsan_write_range(uintptr_t addr, size_t size) { kcsan_access(addr, size, true, false, __RET_ADDR); } void __tsan_init(void); void __tsan_func_entry(void *); void __tsan_func_exit(void); void __tsan_init(void) { } void __tsan_func_entry(void *call_pc) { } void __tsan_func_exit(void) { } /* -------------------------------------------------------------------------- */ void * kcsan_memcpy(void *dst, const void *src, size_t len) { kcsan_access((uintptr_t)src, len, false, false, __RET_ADDR); kcsan_access((uintptr_t)dst, len, true, false, __RET_ADDR); return __builtin_memcpy(dst, src, len); } int kcsan_memcmp(const void *b1, const void *b2, size_t len) { kcsan_access((uintptr_t)b1, len, false, false, __RET_ADDR); kcsan_access((uintptr_t)b2, len, false, false, __RET_ADDR); return __builtin_memcmp(b1, b2, len); } void * kcsan_memset(void *b, int c, size_t len) { kcsan_access((uintptr_t)b, len, true, false, __RET_ADDR); return __builtin_memset(b, c, len); } void * kcsan_memmove(void *dst, const void *src, size_t len) { kcsan_access((uintptr_t)src, len, false, false, __RET_ADDR); kcsan_access((uintptr_t)dst, len, true, false, __RET_ADDR); return __builtin_memmove(dst, src, len); } char * kcsan_strcpy(char *dst, const char *src) { char *save = dst; while (1) { kcsan_access((uintptr_t)src, 1, false, false, __RET_ADDR); kcsan_access((uintptr_t)dst, 1, true, false, __RET_ADDR); *dst = *src; if (*src == '\0') break; src++, dst++; } return save; } int kcsan_strcmp(const char *s1, const char *s2) { while (1) { kcsan_access((uintptr_t)s1, 1, false, false, __RET_ADDR); kcsan_access((uintptr_t)s2, 1, false, false, __RET_ADDR); if (*s1 != *s2) break; if (*s1 == '\0') return 0; s1++, s2++; } return (*(const unsigned char *)s1 - *(const unsigned char *)s2); } size_t kcsan_strlen(const char *str) { const char *s; s = str; while (1) { kcsan_access((uintptr_t)s, 1, false, false, __RET_ADDR); if (*s == '\0') break; s++; } return (s - str); } #undef copyin #undef copyin_nofault #undef copyinstr #undef copyout #undef copyout_nofault int kcsan_copyin(const void *uaddr, void *kaddr, size_t len) { kcsan_access((uintptr_t)kaddr, len, true, false, __RET_ADDR); return copyin(uaddr, kaddr, len); } int kcsan_copyinstr(const void *uaddr, void *kaddr, size_t len, size_t *done) { kcsan_access((uintptr_t)kaddr, len, true, false, __RET_ADDR); return copyinstr(uaddr, kaddr, len, done); } int kcsan_copyout(const void *kaddr, void *uaddr, size_t len) { kcsan_access((uintptr_t)kaddr, len, false, false, __RET_ADDR); return copyout(kaddr, uaddr, len); } /* -------------------------------------------------------------------------- */ #include +#define ATOMIC_SAN_PREFIX kcsan #include #define _CSAN_ATOMIC_FUNC_ADD(name, type) \ void kcsan_atomic_add_##name(volatile type *ptr, type val) \ { \ kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \ __RET_ADDR); \ atomic_add_##name(ptr, val); \ } #define CSAN_ATOMIC_FUNC_ADD(name, type) \ _CSAN_ATOMIC_FUNC_ADD(name, type) \ _CSAN_ATOMIC_FUNC_ADD(acq_##name, type) \ _CSAN_ATOMIC_FUNC_ADD(rel_##name, type) #define _CSAN_ATOMIC_FUNC_CLEAR(name, type) \ void kcsan_atomic_clear_##name(volatile type *ptr, type val) \ { \ kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \ __RET_ADDR); \ atomic_clear_##name(ptr, val); \ } #define CSAN_ATOMIC_FUNC_CLEAR(name, type) \ _CSAN_ATOMIC_FUNC_CLEAR(name, type) \ _CSAN_ATOMIC_FUNC_CLEAR(acq_##name, type) \ _CSAN_ATOMIC_FUNC_CLEAR(rel_##name, type) #define _CSAN_ATOMIC_FUNC_CMPSET(name, type) \ int kcsan_atomic_cmpset_##name(volatile type *ptr, type val1, \ type val2) \ { \ kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \ __RET_ADDR); \ return (atomic_cmpset_##name(ptr, val1, val2)); \ } #define CSAN_ATOMIC_FUNC_CMPSET(name, type) \ _CSAN_ATOMIC_FUNC_CMPSET(name, type) \ _CSAN_ATOMIC_FUNC_CMPSET(acq_##name, type) \ _CSAN_ATOMIC_FUNC_CMPSET(rel_##name, type) #define _CSAN_ATOMIC_FUNC_FCMPSET(name, type) \ int kcsan_atomic_fcmpset_##name(volatile type *ptr, type *val1, \ type val2) \ { \ kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \ __RET_ADDR); \ return (atomic_fcmpset_##name(ptr, val1, val2)); \ } #define CSAN_ATOMIC_FUNC_FCMPSET(name, type) \ _CSAN_ATOMIC_FUNC_FCMPSET(name, type) \ _CSAN_ATOMIC_FUNC_FCMPSET(acq_##name, type) \ _CSAN_ATOMIC_FUNC_FCMPSET(rel_##name, type) #define CSAN_ATOMIC_FUNC_FETCHADD(name, type) \ type kcsan_atomic_fetchadd_##name(volatile type *ptr, type val) \ { \ kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \ __RET_ADDR); \ return (atomic_fetchadd_##name(ptr, val)); \ } #define _CSAN_ATOMIC_FUNC_LOAD(name, type) \ type kcsan_atomic_load_##name(volatile type *ptr) \ { \ kcsan_access((uintptr_t)ptr, sizeof(type), false, true, \ __RET_ADDR); \ return (atomic_load_##name(ptr)); \ } #define CSAN_ATOMIC_FUNC_LOAD(name, type) \ _CSAN_ATOMIC_FUNC_LOAD(name, type) \ _CSAN_ATOMIC_FUNC_LOAD(acq_##name, type) \ #define CSAN_ATOMIC_FUNC_READANDCLEAR(name, type) \ type kcsan_atomic_readandclear_##name(volatile type *ptr) \ { \ kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \ __RET_ADDR); \ return (atomic_readandclear_##name(ptr)); \ } #define _CSAN_ATOMIC_FUNC_SET(name, type) \ void kcsan_atomic_set_##name(volatile type *ptr, type val) \ { \ kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \ __RET_ADDR); \ atomic_set_##name(ptr, val); \ } #define CSAN_ATOMIC_FUNC_SET(name, type) \ _CSAN_ATOMIC_FUNC_SET(name, type) \ _CSAN_ATOMIC_FUNC_SET(acq_##name, type) \ _CSAN_ATOMIC_FUNC_SET(rel_##name, type) #define _CSAN_ATOMIC_FUNC_SUBTRACT(name, type) \ void kcsan_atomic_subtract_##name(volatile type *ptr, type val) \ { \ kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \ __RET_ADDR); \ atomic_subtract_##name(ptr, val); \ } #define CSAN_ATOMIC_FUNC_SUBTRACT(name, type) \ _CSAN_ATOMIC_FUNC_SUBTRACT(name, type) \ _CSAN_ATOMIC_FUNC_SUBTRACT(acq_##name, type) \ _CSAN_ATOMIC_FUNC_SUBTRACT(rel_##name, type) #define _CSAN_ATOMIC_FUNC_STORE(name, type) \ void kcsan_atomic_store_##name(volatile type *ptr, type val) \ { \ kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \ __RET_ADDR); \ atomic_store_##name(ptr, val); \ } #define CSAN_ATOMIC_FUNC_STORE(name, type) \ _CSAN_ATOMIC_FUNC_STORE(name, type) \ _CSAN_ATOMIC_FUNC_STORE(rel_##name, type) #define CSAN_ATOMIC_FUNC_SWAP(name, type) \ type kcsan_atomic_swap_##name(volatile type *ptr, type val) \ { \ kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \ __RET_ADDR); \ return(atomic_swap_##name(ptr, val)); \ } #define CSAN_ATOMIC_FUNC_TESTANDCLEAR(name, type) \ int kcsan_atomic_testandclear_##name(volatile type *ptr, u_int val) \ { \ kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \ __RET_ADDR); \ return(atomic_testandclear_##name(ptr, val)); \ } #define CSAN_ATOMIC_FUNC_TESTANDSET(name, type) \ int kcsan_atomic_testandset_##name(volatile type *ptr, u_int val) \ { \ kcsan_access((uintptr_t)ptr, sizeof(type), true, true, \ __RET_ADDR); \ return (atomic_testandset_##name(ptr, val)); \ } CSAN_ATOMIC_FUNC_ADD(8, uint8_t) CSAN_ATOMIC_FUNC_CLEAR(8, uint8_t) CSAN_ATOMIC_FUNC_CMPSET(8, uint8_t) CSAN_ATOMIC_FUNC_FCMPSET(8, uint8_t) CSAN_ATOMIC_FUNC_LOAD(8, uint8_t) CSAN_ATOMIC_FUNC_SET(8, uint8_t) CSAN_ATOMIC_FUNC_SUBTRACT(8, uint8_t) _CSAN_ATOMIC_FUNC_STORE(8, uint8_t) #if 0 CSAN_ATOMIC_FUNC_FETCHADD(8, uint8_t) CSAN_ATOMIC_FUNC_READANDCLEAR(8, uint8_t) CSAN_ATOMIC_FUNC_SWAP(8, uint8_t) CSAN_ATOMIC_FUNC_TESTANDCLEAR(8, uint8_t) CSAN_ATOMIC_FUNC_TESTANDSET(8, uint8_t) #endif CSAN_ATOMIC_FUNC_ADD(16, uint16_t) CSAN_ATOMIC_FUNC_CLEAR(16, uint16_t) CSAN_ATOMIC_FUNC_CMPSET(16, uint16_t) CSAN_ATOMIC_FUNC_FCMPSET(16, uint16_t) CSAN_ATOMIC_FUNC_LOAD(16, uint16_t) CSAN_ATOMIC_FUNC_SET(16, uint16_t) CSAN_ATOMIC_FUNC_SUBTRACT(16, uint16_t) _CSAN_ATOMIC_FUNC_STORE(16, uint16_t) #if 0 CSAN_ATOMIC_FUNC_FETCHADD(16, uint16_t) CSAN_ATOMIC_FUNC_READANDCLEAR(16, uint16_t) CSAN_ATOMIC_FUNC_SWAP(16, uint16_t) CSAN_ATOMIC_FUNC_TESTANDCLEAR(16, uint16_t) CSAN_ATOMIC_FUNC_TESTANDSET(16, uint16_t) #endif CSAN_ATOMIC_FUNC_ADD(32, uint32_t) CSAN_ATOMIC_FUNC_CLEAR(32, uint32_t) CSAN_ATOMIC_FUNC_CMPSET(32, uint32_t) CSAN_ATOMIC_FUNC_FCMPSET(32, uint32_t) CSAN_ATOMIC_FUNC_FETCHADD(32, uint32_t) CSAN_ATOMIC_FUNC_LOAD(32, uint32_t) CSAN_ATOMIC_FUNC_READANDCLEAR(32, uint32_t) CSAN_ATOMIC_FUNC_SET(32, uint32_t) CSAN_ATOMIC_FUNC_SUBTRACT(32, uint32_t) CSAN_ATOMIC_FUNC_STORE(32, uint32_t) CSAN_ATOMIC_FUNC_SWAP(32, uint32_t) #if !defined(__aarch64__) CSAN_ATOMIC_FUNC_TESTANDCLEAR(32, uint32_t) CSAN_ATOMIC_FUNC_TESTANDSET(32, uint32_t) #endif CSAN_ATOMIC_FUNC_ADD(64, uint64_t) CSAN_ATOMIC_FUNC_CLEAR(64, uint64_t) CSAN_ATOMIC_FUNC_CMPSET(64, uint64_t) CSAN_ATOMIC_FUNC_FCMPSET(64, uint64_t) CSAN_ATOMIC_FUNC_FETCHADD(64, uint64_t) CSAN_ATOMIC_FUNC_LOAD(64, uint64_t) CSAN_ATOMIC_FUNC_READANDCLEAR(64, uint64_t) CSAN_ATOMIC_FUNC_SET(64, uint64_t) CSAN_ATOMIC_FUNC_SUBTRACT(64, uint64_t) CSAN_ATOMIC_FUNC_STORE(64, uint64_t) CSAN_ATOMIC_FUNC_SWAP(64, uint64_t) #if !defined(__aarch64__) CSAN_ATOMIC_FUNC_TESTANDCLEAR(64, uint64_t) CSAN_ATOMIC_FUNC_TESTANDSET(64, uint64_t) #endif CSAN_ATOMIC_FUNC_ADD(char, uint8_t) CSAN_ATOMIC_FUNC_CLEAR(char, uint8_t) CSAN_ATOMIC_FUNC_CMPSET(char, uint8_t) CSAN_ATOMIC_FUNC_FCMPSET(char, uint8_t) CSAN_ATOMIC_FUNC_LOAD(char, uint8_t) CSAN_ATOMIC_FUNC_SET(char, uint8_t) CSAN_ATOMIC_FUNC_SUBTRACT(char, uint8_t) _CSAN_ATOMIC_FUNC_STORE(char, uint8_t) #if 0 CSAN_ATOMIC_FUNC_FETCHADD(char, uint8_t) CSAN_ATOMIC_FUNC_READANDCLEAR(char, uint8_t) CSAN_ATOMIC_FUNC_SWAP(char, uint8_t) CSAN_ATOMIC_FUNC_TESTANDCLEAR(char, uint8_t) CSAN_ATOMIC_FUNC_TESTANDSET(char, uint8_t) #endif CSAN_ATOMIC_FUNC_ADD(short, uint16_t) CSAN_ATOMIC_FUNC_CLEAR(short, uint16_t) CSAN_ATOMIC_FUNC_CMPSET(short, uint16_t) CSAN_ATOMIC_FUNC_FCMPSET(short, uint16_t) CSAN_ATOMIC_FUNC_LOAD(short, uint16_t) CSAN_ATOMIC_FUNC_SET(short, uint16_t) CSAN_ATOMIC_FUNC_SUBTRACT(short, uint16_t) _CSAN_ATOMIC_FUNC_STORE(short, uint16_t) #if 0 CSAN_ATOMIC_FUNC_FETCHADD(short, uint16_t) CSAN_ATOMIC_FUNC_READANDCLEAR(short, uint16_t) CSAN_ATOMIC_FUNC_SWAP(short, uint16_t) CSAN_ATOMIC_FUNC_TESTANDCLEAR(short, uint16_t) CSAN_ATOMIC_FUNC_TESTANDSET(short, uint16_t) #endif CSAN_ATOMIC_FUNC_ADD(int, u_int) CSAN_ATOMIC_FUNC_CLEAR(int, u_int) CSAN_ATOMIC_FUNC_CMPSET(int, u_int) CSAN_ATOMIC_FUNC_FCMPSET(int, u_int) CSAN_ATOMIC_FUNC_FETCHADD(int, u_int) CSAN_ATOMIC_FUNC_LOAD(int, u_int) CSAN_ATOMIC_FUNC_READANDCLEAR(int, u_int) CSAN_ATOMIC_FUNC_SET(int, u_int) CSAN_ATOMIC_FUNC_SUBTRACT(int, u_int) CSAN_ATOMIC_FUNC_STORE(int, u_int) CSAN_ATOMIC_FUNC_SWAP(int, u_int) #if !defined(__aarch64__) CSAN_ATOMIC_FUNC_TESTANDCLEAR(int, u_int) CSAN_ATOMIC_FUNC_TESTANDSET(int, u_int) #endif CSAN_ATOMIC_FUNC_ADD(long, u_long) CSAN_ATOMIC_FUNC_CLEAR(long, u_long) CSAN_ATOMIC_FUNC_CMPSET(long, u_long) CSAN_ATOMIC_FUNC_FCMPSET(long, u_long) CSAN_ATOMIC_FUNC_FETCHADD(long, u_long) CSAN_ATOMIC_FUNC_LOAD(long, u_long) CSAN_ATOMIC_FUNC_READANDCLEAR(long, u_long) CSAN_ATOMIC_FUNC_SET(long, u_long) CSAN_ATOMIC_FUNC_SUBTRACT(long, u_long) CSAN_ATOMIC_FUNC_STORE(long, u_long) CSAN_ATOMIC_FUNC_SWAP(long, u_long) #if !defined(__aarch64__) CSAN_ATOMIC_FUNC_TESTANDCLEAR(long, u_long) CSAN_ATOMIC_FUNC_TESTANDSET(long, u_long) CSAN_ATOMIC_FUNC_TESTANDSET(acq_long, u_long) #endif CSAN_ATOMIC_FUNC_ADD(ptr, uintptr_t) CSAN_ATOMIC_FUNC_CLEAR(ptr, uintptr_t) CSAN_ATOMIC_FUNC_CMPSET(ptr, uintptr_t) CSAN_ATOMIC_FUNC_FCMPSET(ptr, uintptr_t) #if !defined(__amd64__) CSAN_ATOMIC_FUNC_FETCHADD(ptr, uintptr_t) #endif CSAN_ATOMIC_FUNC_LOAD(ptr, uintptr_t) CSAN_ATOMIC_FUNC_READANDCLEAR(ptr, uintptr_t) CSAN_ATOMIC_FUNC_SET(ptr, uintptr_t) CSAN_ATOMIC_FUNC_SUBTRACT(ptr, uintptr_t) CSAN_ATOMIC_FUNC_STORE(ptr, uintptr_t) CSAN_ATOMIC_FUNC_SWAP(ptr, uintptr_t) #if 0 CSAN_ATOMIC_FUNC_TESTANDCLEAR(ptr, uintptr_t) CSAN_ATOMIC_FUNC_TESTANDSET(ptr, uintptr_t) #endif #define CSAN_ATOMIC_FUNC_THREAD_FENCE(name) \ void kcsan_atomic_thread_fence_##name(void) \ { \ atomic_thread_fence_##name(); \ } CSAN_ATOMIC_FUNC_THREAD_FENCE(acq) CSAN_ATOMIC_FUNC_THREAD_FENCE(acq_rel) CSAN_ATOMIC_FUNC_THREAD_FENCE(rel) CSAN_ATOMIC_FUNC_THREAD_FENCE(seq_cst) +void +kcsan_atomic_interrupt_fence(void) +{ + atomic_interrupt_fence(); +} + /* -------------------------------------------------------------------------- */ #include #include +#define BUS_SAN_PREFIX kcsan #include int kcsan_bus_space_map(bus_space_tag_t tag, bus_addr_t hnd, bus_size_t size, int flags, bus_space_handle_t *handlep) { return (bus_space_map(tag, hnd, size, flags, handlep)); } void kcsan_bus_space_unmap(bus_space_tag_t tag, bus_space_handle_t hnd, bus_size_t size) { bus_space_unmap(tag, hnd, size); } int kcsan_bus_space_subregion(bus_space_tag_t tag, bus_space_handle_t hnd, bus_size_t offset, bus_size_t size, bus_space_handle_t *handlep) { return (bus_space_subregion(tag, hnd, offset, size, handlep)); } #if !defined(__amd64__) int kcsan_bus_space_alloc(bus_space_tag_t tag, bus_addr_t reg_start, bus_addr_t reg_end, bus_size_t size, bus_size_t alignment, bus_size_t boundary, int flags, bus_addr_t *addrp, bus_space_handle_t *handlep) { return (bus_space_alloc(tag, reg_start, reg_end, size, alignment, boundary, flags, addrp, handlep)); } #endif void kcsan_bus_space_free(bus_space_tag_t tag, bus_space_handle_t hnd, bus_size_t size) { bus_space_free(tag, hnd, size); } void kcsan_bus_space_barrier(bus_space_tag_t tag, bus_space_handle_t hnd, bus_size_t offset, bus_size_t size, int flags) { bus_space_barrier(tag, hnd, offset, size, flags); } #define CSAN_BUS_READ_FUNC(func, width, type) \ type kcsan_bus_space_read##func##_##width(bus_space_tag_t tag, \ bus_space_handle_t hnd, bus_size_t offset) \ { \ return (bus_space_read##func##_##width(tag, hnd, \ offset)); \ } \ #define CSAN_BUS_READ_PTR_FUNC(func, width, type) \ void kcsan_bus_space_read_##func##_##width(bus_space_tag_t tag, \ bus_space_handle_t hnd, bus_size_t size, type *buf, \ bus_size_t count) \ { \ kcsan_access((uintptr_t)buf, sizeof(type) * count, \ false, false, __RET_ADDR); \ bus_space_read_##func##_##width(tag, hnd, size, buf, \ count); \ } CSAN_BUS_READ_FUNC(, 1, uint8_t) CSAN_BUS_READ_FUNC(_stream, 1, uint8_t) CSAN_BUS_READ_PTR_FUNC(multi, 1, uint8_t) CSAN_BUS_READ_PTR_FUNC(multi_stream, 1, uint8_t) CSAN_BUS_READ_PTR_FUNC(region, 1, uint8_t) CSAN_BUS_READ_PTR_FUNC(region_stream, 1, uint8_t) CSAN_BUS_READ_FUNC(, 2, uint16_t) CSAN_BUS_READ_FUNC(_stream, 2, uint16_t) CSAN_BUS_READ_PTR_FUNC(multi, 2, uint16_t) CSAN_BUS_READ_PTR_FUNC(multi_stream, 2, uint16_t) CSAN_BUS_READ_PTR_FUNC(region, 2, uint16_t) CSAN_BUS_READ_PTR_FUNC(region_stream, 2, uint16_t) CSAN_BUS_READ_FUNC(, 4, uint32_t) CSAN_BUS_READ_FUNC(_stream, 4, uint32_t) CSAN_BUS_READ_PTR_FUNC(multi, 4, uint32_t) CSAN_BUS_READ_PTR_FUNC(multi_stream, 4, uint32_t) CSAN_BUS_READ_PTR_FUNC(region, 4, uint32_t) CSAN_BUS_READ_PTR_FUNC(region_stream, 4, uint32_t) CSAN_BUS_READ_FUNC(, 8, uint64_t) #if defined(__aarch64__) CSAN_BUS_READ_FUNC(_stream, 8, uint64_t) CSAN_BUS_READ_PTR_FUNC(multi, 8, uint64_t) CSAN_BUS_READ_PTR_FUNC(multi_stream, 8, uint64_t) CSAN_BUS_READ_PTR_FUNC(region, 8, uint64_t) CSAN_BUS_READ_PTR_FUNC(region_stream, 8, uint64_t) #endif #define CSAN_BUS_WRITE_FUNC(func, width, type) \ void kcsan_bus_space_write##func##_##width(bus_space_tag_t tag, \ bus_space_handle_t hnd, bus_size_t offset, type value) \ { \ bus_space_write##func##_##width(tag, hnd, offset, value); \ } \ #define CSAN_BUS_WRITE_PTR_FUNC(func, width, type) \ void kcsan_bus_space_write_##func##_##width(bus_space_tag_t tag, \ bus_space_handle_t hnd, bus_size_t size, const type *buf, \ bus_size_t count) \ { \ kcsan_access((uintptr_t)buf, sizeof(type) * count, \ true, false, __RET_ADDR); \ bus_space_write_##func##_##width(tag, hnd, size, buf, \ count); \ } CSAN_BUS_WRITE_FUNC(, 1, uint8_t) CSAN_BUS_WRITE_FUNC(_stream, 1, uint8_t) CSAN_BUS_WRITE_PTR_FUNC(multi, 1, uint8_t) CSAN_BUS_WRITE_PTR_FUNC(multi_stream, 1, uint8_t) CSAN_BUS_WRITE_PTR_FUNC(region, 1, uint8_t) CSAN_BUS_WRITE_PTR_FUNC(region_stream, 1, uint8_t) CSAN_BUS_WRITE_FUNC(, 2, uint16_t) CSAN_BUS_WRITE_FUNC(_stream, 2, uint16_t) CSAN_BUS_WRITE_PTR_FUNC(multi, 2, uint16_t) CSAN_BUS_WRITE_PTR_FUNC(multi_stream, 2, uint16_t) CSAN_BUS_WRITE_PTR_FUNC(region, 2, uint16_t) CSAN_BUS_WRITE_PTR_FUNC(region_stream, 2, uint16_t) CSAN_BUS_WRITE_FUNC(, 4, uint32_t) CSAN_BUS_WRITE_FUNC(_stream, 4, uint32_t) CSAN_BUS_WRITE_PTR_FUNC(multi, 4, uint32_t) CSAN_BUS_WRITE_PTR_FUNC(multi_stream, 4, uint32_t) CSAN_BUS_WRITE_PTR_FUNC(region, 4, uint32_t) CSAN_BUS_WRITE_PTR_FUNC(region_stream, 4, uint32_t) CSAN_BUS_WRITE_FUNC(, 8, uint64_t) #if defined(__aarch64__) CSAN_BUS_WRITE_FUNC(_stream, 8, uint64_t) CSAN_BUS_WRITE_PTR_FUNC(multi, 8, uint64_t) CSAN_BUS_WRITE_PTR_FUNC(multi_stream, 8, uint64_t) CSAN_BUS_WRITE_PTR_FUNC(region, 8, uint64_t) CSAN_BUS_WRITE_PTR_FUNC(region_stream, 8, uint64_t) #endif #define CSAN_BUS_SET_FUNC(func, width, type) \ void kcsan_bus_space_set_##func##_##width(bus_space_tag_t tag, \ bus_space_handle_t hnd, bus_size_t offset, type value, \ bus_size_t count) \ { \ bus_space_set_##func##_##width(tag, hnd, offset, value, \ count); \ } CSAN_BUS_SET_FUNC(multi, 1, uint8_t) CSAN_BUS_SET_FUNC(region, 1, uint8_t) #if !defined(__aarch64__) CSAN_BUS_SET_FUNC(multi_stream, 1, uint8_t) CSAN_BUS_SET_FUNC(region_stream, 1, uint8_t) #endif CSAN_BUS_SET_FUNC(multi, 2, uint16_t) CSAN_BUS_SET_FUNC(region, 2, uint16_t) #if !defined(__aarch64__) CSAN_BUS_SET_FUNC(multi_stream, 2, uint16_t) CSAN_BUS_SET_FUNC(region_stream, 2, uint16_t) #endif CSAN_BUS_SET_FUNC(multi, 4, uint32_t) CSAN_BUS_SET_FUNC(region, 4, uint32_t) #if !defined(__aarch64__) CSAN_BUS_SET_FUNC(multi_stream, 4, uint32_t) CSAN_BUS_SET_FUNC(region_stream, 4, uint32_t) #endif #if !defined(__amd64__) CSAN_BUS_SET_FUNC(multi, 8, uint64_t) CSAN_BUS_SET_FUNC(region, 8, uint64_t) #if !defined(__aarch64__) CSAN_BUS_SET_FUNC(multi_stream, 8, uint64_t) CSAN_BUS_SET_FUNC(region_stream, 8, uint64_t) #endif #endif #define CSAN_BUS_PEEK_FUNC(width, type) \ int kcsan_bus_space_peek_##width(bus_space_tag_t tag, \ bus_space_handle_t hnd, bus_size_t offset, type *value) \ { \ kcsan_access((uintptr_t)value, sizeof(type), true, false, \ __RET_ADDR); \ return (bus_space_peek_##width(tag, hnd, offset, value)); \ } CSAN_BUS_PEEK_FUNC(1, uint8_t) CSAN_BUS_PEEK_FUNC(2, uint16_t) CSAN_BUS_PEEK_FUNC(4, uint32_t) #if !defined(__i386__) CSAN_BUS_PEEK_FUNC(8, uint64_t) #endif #define CSAN_BUS_POKE_FUNC(width, type) \ int kcsan_bus_space_poke_##width(bus_space_tag_t tag, \ bus_space_handle_t hnd, bus_size_t offset, type value) \ { \ return (bus_space_poke_##width(tag, hnd, offset, value)); \ } CSAN_BUS_POKE_FUNC(1, uint8_t) CSAN_BUS_POKE_FUNC(2, uint16_t) CSAN_BUS_POKE_FUNC(4, uint32_t) #if !defined(__i386__) CSAN_BUS_POKE_FUNC(8, uint64_t) #endif diff --git a/sys/sys/atomic_san.h b/sys/sys/atomic_san.h index c2b963ae8b92..5d10f58f7565 100644 --- a/sys/sys/atomic_san.h +++ b/sys/sys/atomic_san.h @@ -1,380 +1,402 @@ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2019 Andrew Turner + * Copyright (c) 2021 The FreeBSD Foundation * * This software was developed by SRI International and the University of * Cambridge Computer Laboratory (Department of Computer Science and * Technology) under DARPA contract HR0011-18-C-0016 ("ECATS"), as part of the * DARPA SSITH research programme. * + * Portions of this software were written by Mark Johnston under sponsorship + * by the FreeBSD Foundation. + * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #ifndef _SYS_ATOMIC_SAN_H_ #define _SYS_ATOMIC_SAN_H_ #ifndef _MACHINE_ATOMIC_H_ #error do not include this header, use machine/atomic.h #endif -#define KCSAN_ATOMIC_FUNC_1(op, name, type) \ - void kcsan_atomic_##op##_##name(volatile type *, type); \ - void kcsan_atomic_##op##_acq_##name(volatile type *, type); \ - void kcsan_atomic_##op##_rel_##name(volatile type *, type) +#ifndef ATOMIC_SAN_PREFIX +#error No sanitizer prefix available +#endif + +#define ATOMIC_SAN_FUNC_1(sp, op, name, type) \ + void sp##_atomic_##op##_##name(volatile type *, type); \ + void sp##_atomic_##op##_acq_##name(volatile type *, type); \ + void sp##_atomic_##op##_rel_##name(volatile type *, type) -#define KCSAN_ATOMIC_CMPSET(name, type) \ - int kcsan_atomic_cmpset_##name(volatile type *, type, type); \ - int kcsan_atomic_cmpset_acq_##name(volatile type *, type, type); \ - int kcsan_atomic_cmpset_rel_##name(volatile type *, type, type) +#define ATOMIC_SAN_CMPSET(sp, name, type) \ + int sp##_atomic_cmpset_##name(volatile type *, type, type); \ + int sp##_atomic_cmpset_acq_##name(volatile type *, type, type); \ + int sp##_atomic_cmpset_rel_##name(volatile type *, type, type) -#define KCSAN_ATOMIC_FCMPSET(name, type) \ - int kcsan_atomic_fcmpset_##name(volatile type *, type *, type); \ - int kcsan_atomic_fcmpset_acq_##name(volatile type *, type *, type); \ - int kcsan_atomic_fcmpset_rel_##name(volatile type *, type *, type) +#define ATOMIC_SAN_FCMPSET(sp, name, type) \ + int sp##_atomic_fcmpset_##name(volatile type *, type *, type); \ + int sp##_atomic_fcmpset_acq_##name(volatile type *, type *, type); \ + int sp##_atomic_fcmpset_rel_##name(volatile type *, type *, type) -#define KCSAN_ATOMIC_READ(op, name, type) \ - type kcsan_atomic_##op##_##name(volatile type *, type) +#define ATOMIC_SAN_READ(sp, op, name, type) \ + type sp##_atomic_##op##_##name(volatile type *, type) -#define KCSAN_ATOMIC_READANDCLEAR(name, type) \ - type kcsan_atomic_readandclear_##name(volatile type *) +#define ATOMIC_SAN_READANDCLEAR(sp, name, type) \ + type sp##_atomic_readandclear_##name(volatile type *) -#define KCSAN_ATOMIC_LOAD(name, type) \ - type kcsan_atomic_load_##name(volatile type *); \ - type kcsan_atomic_load_acq_##name(volatile type *) +#define ATOMIC_SAN_LOAD(sp, name, type) \ + type sp##_atomic_load_##name(volatile type *); \ + type sp##_atomic_load_acq_##name(volatile type *) -#define KCSAN_ATOMIC_STORE(name, type) \ - void kcsan_atomic_store_##name(volatile type *, type); \ - void kcsan_atomic_store_rel_##name(volatile type *, type) +#define ATOMIC_SAN_STORE(sp, name, type) \ + void sp##_atomic_store_##name(volatile type *, type); \ + void sp##_atomic_store_rel_##name(volatile type *, type) -#define KCSAN_ATOMIC_TEST(op, name, type) \ - int kcsan_atomic_##op##_##name(volatile type *, u_int); \ - int kcsan_atomic_##op##_acq_##name(volatile type *, u_int) +#define ATOMIC_SAN_TEST(sp, op, name, type) \ + int sp##_atomic_##op##_##name(volatile type *, u_int); \ + int sp##_atomic_##op##_acq_##name(volatile type *, u_int) -#define KCSAN_ATOMIC_FUNCS(name, type) \ - KCSAN_ATOMIC_FUNC_1(add, name, type); \ - KCSAN_ATOMIC_FUNC_1(clear, name, type); \ - KCSAN_ATOMIC_CMPSET(name, type); \ - KCSAN_ATOMIC_FCMPSET(name, type); \ - KCSAN_ATOMIC_READ(fetchadd, name, type); \ - KCSAN_ATOMIC_LOAD(name, type); \ - KCSAN_ATOMIC_READANDCLEAR(name, type); \ - KCSAN_ATOMIC_FUNC_1(set, name, type); \ - KCSAN_ATOMIC_FUNC_1(subtract, name, type); \ - KCSAN_ATOMIC_STORE(name, type); \ - KCSAN_ATOMIC_READ(swap, name, type); \ - KCSAN_ATOMIC_TEST(testandclear, name, type); \ - KCSAN_ATOMIC_TEST(testandset, name, type) +#define ATOMIC_SAN_THREAD_FENCE(sp) \ + void sp##_atomic_thread_fence_acq(void); \ + void sp##_atomic_thread_fence_rel(void); \ + void sp##_atomic_thread_fence_acq_rel(void); \ + void sp##_atomic_thread_fence_seq_cst(void); \ + void sp##_atomic_interrupt_fence(void) -KCSAN_ATOMIC_FUNCS(char, uint8_t); -KCSAN_ATOMIC_FUNCS(short, uint16_t); -KCSAN_ATOMIC_FUNCS(int, u_int); -KCSAN_ATOMIC_FUNCS(long, u_long); -KCSAN_ATOMIC_FUNCS(ptr, uintptr_t); -KCSAN_ATOMIC_FUNCS(8, uint8_t); -KCSAN_ATOMIC_FUNCS(16, uint16_t); -KCSAN_ATOMIC_FUNCS(32, uint32_t); -KCSAN_ATOMIC_FUNCS(64, uint64_t); +#define _ATOMIC_SAN_FUNCS(sp, name, type) \ + ATOMIC_SAN_FUNC_1(sp, add, name, type); \ + ATOMIC_SAN_FUNC_1(sp, clear, name, type); \ + ATOMIC_SAN_CMPSET(sp, name, type); \ + ATOMIC_SAN_FCMPSET(sp, name, type); \ + ATOMIC_SAN_READ(sp, fetchadd, name, type); \ + ATOMIC_SAN_LOAD(sp, name, type); \ + ATOMIC_SAN_READANDCLEAR(sp, name, type); \ + ATOMIC_SAN_FUNC_1(sp, set, name, type); \ + ATOMIC_SAN_FUNC_1(sp, subtract, name, type); \ + ATOMIC_SAN_STORE(sp, name, type); \ + ATOMIC_SAN_READ(sp, swap, name, type); \ + ATOMIC_SAN_TEST(sp, testandclear, name, type); \ + ATOMIC_SAN_TEST(sp, testandset, name, type); \ + ATOMIC_SAN_THREAD_FENCE(sp); -void kcsan_atomic_thread_fence_acq(void); -void kcsan_atomic_thread_fence_acq_rel(void); -void kcsan_atomic_thread_fence_rel(void); -void kcsan_atomic_thread_fence_seq_cst(void); +#define ATOMIC_SAN_FUNCS(name, type) \ + _ATOMIC_SAN_FUNCS(ATOMIC_SAN_PREFIX, name, type) -#ifndef KCSAN_RUNTIME +ATOMIC_SAN_FUNCS(char, uint8_t); +ATOMIC_SAN_FUNCS(short, uint16_t); +ATOMIC_SAN_FUNCS(int, u_int); +ATOMIC_SAN_FUNCS(long, u_long); +ATOMIC_SAN_FUNCS(ptr, uintptr_t); +ATOMIC_SAN_FUNCS(8, uint8_t); +ATOMIC_SAN_FUNCS(16, uint16_t); +ATOMIC_SAN_FUNCS(32, uint32_t); +ATOMIC_SAN_FUNCS(64, uint64_t); + +#ifndef SAN_RUNTIME + +/* + * Redirect uses of an atomic(9) function to the sanitizer's interceptor. + * For instance, KASAN callers of atomic_add_char() will be redirected to + * kasan_atomic_add_char(). + */ +#define ATOMIC_SAN(func) \ + __CONCAT(ATOMIC_SAN_PREFIX, __CONCAT(_atomic_, func)) -#define atomic_add_char kcsan_atomic_add_char -#define atomic_add_acq_char kcsan_atomic_add_acq_char -#define atomic_add_rel_char kcsan_atomic_add_rel_char -#define atomic_clear_char kcsan_atomic_clear_char -#define atomic_clear_acq_char kcsan_atomic_clear_acq_char -#define atomic_clear_rel_char kcsan_atomic_clear_rel_char -#define atomic_cmpset_char kcsan_atomic_cmpset_char -#define atomic_cmpset_acq_char kcsan_atomic_cmpset_acq_char -#define atomic_cmpset_rel_char kcsan_atomic_cmpset_rel_char -#define atomic_fcmpset_char kcsan_atomic_fcmpset_char -#define atomic_fcmpset_acq_char kcsan_atomic_fcmpset_acq_char -#define atomic_fcmpset_rel_char kcsan_atomic_fcmpset_rel_char -#define atomic_fetchadd_char kcsan_atomic_fetchadd_char -#define atomic_load_char kcsan_atomic_load_char -#define atomic_load_acq_char kcsan_atomic_load_acq_char -#define atomic_readandclear_char kcsan_atomic_readandclear_char -#define atomic_set_char kcsan_atomic_set_char -#define atomic_set_acq_char kcsan_atomic_set_acq_char -#define atomic_set_rel_char kcsan_atomic_set_rel_char -#define atomic_subtract_char kcsan_atomic_subtract_char -#define atomic_subtract_acq_char kcsan_atomic_subtract_acq_char -#define atomic_subtract_rel_char kcsan_atomic_subtract_rel_char -#define atomic_store_char kcsan_atomic_store_char -#define atomic_store_rel_char kcsan_atomic_store_rel_char -#define atomic_swap_char kcsan_atomic_swap_char -#define atomic_testandclear_char kcsan_atomic_testandclear_char -#define atomic_testandset_char kcsan_atomic_testandset_char +#define atomic_add_char ATOMIC_SAN(add_char) +#define atomic_add_acq_char ATOMIC_SAN(add_acq_char) +#define atomic_add_rel_char ATOMIC_SAN(add_rel_char) +#define atomic_clear_char ATOMIC_SAN(clear_char) +#define atomic_clear_acq_char ATOMIC_SAN(clear_acq_char) +#define atomic_clear_rel_char ATOMIC_SAN(clear_rel_char) +#define atomic_cmpset_char ATOMIC_SAN(cmpset_char) +#define atomic_cmpset_acq_char ATOMIC_SAN(cmpset_acq_char) +#define atomic_cmpset_rel_char ATOMIC_SAN(cmpset_rel_char) +#define atomic_fcmpset_char ATOMIC_SAN(fcmpset_char) +#define atomic_fcmpset_acq_char ATOMIC_SAN(fcmpset_acq_char) +#define atomic_fcmpset_rel_char ATOMIC_SAN(fcmpset_rel_char) +#define atomic_fetchadd_char ATOMIC_SAN(fetchadd_char) +#define atomic_load_char ATOMIC_SAN(load_char) +#define atomic_load_acq_char ATOMIC_SAN(load_acq_char) +#define atomic_readandclear_char ATOMIC_SAN(readandclear_char) +#define atomic_set_char ATOMIC_SAN(set_char) +#define atomic_set_acq_char ATOMIC_SAN(set_acq_char) +#define atomic_set_rel_char ATOMIC_SAN(set_rel_char) +#define atomic_subtract_char ATOMIC_SAN(subtract_char) +#define atomic_subtract_acq_char ATOMIC_SAN(subtract_acq_char) +#define atomic_subtract_rel_char ATOMIC_SAN(subtract_rel_char) +#define atomic_store_char ATOMIC_SAN(store_char) +#define atomic_store_rel_char ATOMIC_SAN(store_rel_char) +#define atomic_swap_char ATOMIC_SAN(swap_char) +#define atomic_testandclear_char ATOMIC_SAN(testandclear_char) +#define atomic_testandset_char ATOMIC_SAN(testandset_char) -#define atomic_add_short kcsan_atomic_add_short -#define atomic_add_acq_short kcsan_atomic_add_acq_short -#define atomic_add_rel_short kcsan_atomic_add_rel_short -#define atomic_clear_short kcsan_atomic_clear_short -#define atomic_clear_acq_short kcsan_atomic_clear_acq_short -#define atomic_clear_rel_short kcsan_atomic_clear_rel_short -#define atomic_cmpset_short kcsan_atomic_cmpset_short -#define atomic_cmpset_acq_short kcsan_atomic_cmpset_acq_short -#define atomic_cmpset_rel_short kcsan_atomic_cmpset_rel_short -#define atomic_fcmpset_short kcsan_atomic_fcmpset_short -#define atomic_fcmpset_acq_short kcsan_atomic_fcmpset_acq_short -#define atomic_fcmpset_rel_short kcsan_atomic_fcmpset_rel_short -#define atomic_fetchadd_short kcsan_atomic_fetchadd_short -#define atomic_load_short kcsan_atomic_load_short -#define atomic_load_acq_short kcsan_atomic_load_acq_short -#define atomic_readandclear_short kcsan_atomic_readandclear_short -#define atomic_set_short kcsan_atomic_set_short -#define atomic_set_acq_short kcsan_atomic_set_acq_short -#define atomic_set_rel_short kcsan_atomic_set_rel_short -#define atomic_subtract_short kcsan_atomic_subtract_short -#define atomic_subtract_acq_short kcsan_atomic_subtract_acq_short -#define atomic_subtract_rel_short kcsan_atomic_subtract_rel_short -#define atomic_store_short kcsan_atomic_store_short -#define atomic_store_rel_short kcsan_atomic_store_rel_short -#define atomic_swap_short kcsan_atomic_swap_short -#define atomic_testandclear_short kcsan_atomic_testandclear_short -#define atomic_testandset_short kcsan_atomic_testandset_short +#define atomic_add_short ATOMIC_SAN(add_short) +#define atomic_add_acq_short ATOMIC_SAN(add_acq_short) +#define atomic_add_rel_short ATOMIC_SAN(add_rel_short) +#define atomic_clear_short ATOMIC_SAN(clear_short) +#define atomic_clear_acq_short ATOMIC_SAN(clear_acq_short) +#define atomic_clear_rel_short ATOMIC_SAN(clear_rel_short) +#define atomic_cmpset_short ATOMIC_SAN(cmpset_short) +#define atomic_cmpset_acq_short ATOMIC_SAN(cmpset_acq_short) +#define atomic_cmpset_rel_short ATOMIC_SAN(cmpset_rel_short) +#define atomic_fcmpset_short ATOMIC_SAN(fcmpset_short) +#define atomic_fcmpset_acq_short ATOMIC_SAN(fcmpset_acq_short) +#define atomic_fcmpset_rel_short ATOMIC_SAN(fcmpset_rel_short) +#define atomic_fetchadd_short ATOMIC_SAN(fetchadd_short) +#define atomic_load_short ATOMIC_SAN(load_short) +#define atomic_load_acq_short ATOMIC_SAN(load_acq_short) +#define atomic_readandclear_short ATOMIC_SAN(readandclear_short) +#define atomic_set_short ATOMIC_SAN(set_short) +#define atomic_set_acq_short ATOMIC_SAN(set_acq_short) +#define atomic_set_rel_short ATOMIC_SAN(set_rel_short) +#define atomic_subtract_short ATOMIC_SAN(subtract_short) +#define atomic_subtract_acq_short ATOMIC_SAN(subtract_acq_short) +#define atomic_subtract_rel_short ATOMIC_SAN(subtract_rel_short) +#define atomic_store_short ATOMIC_SAN(store_short) +#define atomic_store_rel_short ATOMIC_SAN(store_rel_short) +#define atomic_swap_short ATOMIC_SAN(swap_short) +#define atomic_testandclear_short ATOMIC_SAN(testandclear_short) +#define atomic_testandset_short ATOMIC_SAN(testandset_short) -#define atomic_add_int kcsan_atomic_add_int -#define atomic_add_acq_int kcsan_atomic_add_acq_int -#define atomic_add_rel_int kcsan_atomic_add_rel_int -#define atomic_clear_int kcsan_atomic_clear_int -#define atomic_clear_acq_int kcsan_atomic_clear_acq_int -#define atomic_clear_rel_int kcsan_atomic_clear_rel_int -#define atomic_cmpset_int kcsan_atomic_cmpset_int -#define atomic_cmpset_acq_int kcsan_atomic_cmpset_acq_int -#define atomic_cmpset_rel_int kcsan_atomic_cmpset_rel_int -#define atomic_fcmpset_int kcsan_atomic_fcmpset_int -#define atomic_fcmpset_acq_int kcsan_atomic_fcmpset_acq_int -#define atomic_fcmpset_rel_int kcsan_atomic_fcmpset_rel_int -#define atomic_fetchadd_int kcsan_atomic_fetchadd_int -#define atomic_load_int kcsan_atomic_load_int -#define atomic_load_acq_int kcsan_atomic_load_acq_int -#define atomic_readandclear_int kcsan_atomic_readandclear_int -#define atomic_set_int kcsan_atomic_set_int -#define atomic_set_acq_int kcsan_atomic_set_acq_int -#define atomic_set_rel_int kcsan_atomic_set_rel_int -#define atomic_subtract_int kcsan_atomic_subtract_int -#define atomic_subtract_acq_int kcsan_atomic_subtract_acq_int -#define atomic_subtract_rel_int kcsan_atomic_subtract_rel_int -#define atomic_store_int kcsan_atomic_store_int -#define atomic_store_rel_int kcsan_atomic_store_rel_int -#define atomic_swap_int kcsan_atomic_swap_int -#define atomic_testandclear_int kcsan_atomic_testandclear_int -#define atomic_testandset_int kcsan_atomic_testandset_int +#define atomic_add_int ATOMIC_SAN(add_int) +#define atomic_add_acq_int ATOMIC_SAN(add_acq_int) +#define atomic_add_rel_int ATOMIC_SAN(add_rel_int) +#define atomic_clear_int ATOMIC_SAN(clear_int) +#define atomic_clear_acq_int ATOMIC_SAN(clear_acq_int) +#define atomic_clear_rel_int ATOMIC_SAN(clear_rel_int) +#define atomic_cmpset_int ATOMIC_SAN(cmpset_int) +#define atomic_cmpset_acq_int ATOMIC_SAN(cmpset_acq_int) +#define atomic_cmpset_rel_int ATOMIC_SAN(cmpset_rel_int) +#define atomic_fcmpset_int ATOMIC_SAN(fcmpset_int) +#define atomic_fcmpset_acq_int ATOMIC_SAN(fcmpset_acq_int) +#define atomic_fcmpset_rel_int ATOMIC_SAN(fcmpset_rel_int) +#define atomic_fetchadd_int ATOMIC_SAN(fetchadd_int) +#define atomic_load_int ATOMIC_SAN(load_int) +#define atomic_load_acq_int ATOMIC_SAN(load_acq_int) +#define atomic_readandclear_int ATOMIC_SAN(readandclear_int) +#define atomic_set_int ATOMIC_SAN(set_int) +#define atomic_set_acq_int ATOMIC_SAN(set_acq_int) +#define atomic_set_rel_int ATOMIC_SAN(set_rel_int) +#define atomic_subtract_int ATOMIC_SAN(subtract_int) +#define atomic_subtract_acq_int ATOMIC_SAN(subtract_acq_int) +#define atomic_subtract_rel_int ATOMIC_SAN(subtract_rel_int) +#define atomic_store_int ATOMIC_SAN(store_int) +#define atomic_store_rel_int ATOMIC_SAN(store_rel_int) +#define atomic_swap_int ATOMIC_SAN(swap_int) +#define atomic_testandclear_int ATOMIC_SAN(testandclear_int) +#define atomic_testandset_int ATOMIC_SAN(testandset_int) -#define atomic_add_long kcsan_atomic_add_long -#define atomic_add_acq_long kcsan_atomic_add_acq_long -#define atomic_add_rel_long kcsan_atomic_add_rel_long -#define atomic_clear_long kcsan_atomic_clear_long -#define atomic_clear_acq_long kcsan_atomic_clear_acq_long -#define atomic_clear_rel_long kcsan_atomic_clear_rel_long -#define atomic_cmpset_long kcsan_atomic_cmpset_long -#define atomic_cmpset_acq_long kcsan_atomic_cmpset_acq_long -#define atomic_cmpset_rel_long kcsan_atomic_cmpset_rel_long -#define atomic_fcmpset_long kcsan_atomic_fcmpset_long -#define atomic_fcmpset_acq_long kcsan_atomic_fcmpset_acq_long -#define atomic_fcmpset_rel_long kcsan_atomic_fcmpset_rel_long -#define atomic_fetchadd_long kcsan_atomic_fetchadd_long -#define atomic_load_long kcsan_atomic_load_long -#define atomic_load_acq_long kcsan_atomic_load_acq_long -#define atomic_readandclear_long kcsan_atomic_readandclear_long -#define atomic_set_long kcsan_atomic_set_long -#define atomic_set_acq_long kcsan_atomic_set_acq_long -#define atomic_set_rel_long kcsan_atomic_set_rel_long -#define atomic_subtract_long kcsan_atomic_subtract_long -#define atomic_subtract_acq_long kcsan_atomic_subtract_acq_long -#define atomic_subtract_rel_long kcsan_atomic_subtract_rel_long -#define atomic_store_long kcsan_atomic_store_long -#define atomic_store_rel_long kcsan_atomic_store_rel_long -#define atomic_swap_long kcsan_atomic_swap_long -#define atomic_testandclear_long kcsan_atomic_testandclear_long -#define atomic_testandset_long kcsan_atomic_testandset_long -#define atomic_testandset_acq_long kcsan_atomic_testandset_acq_long +#define atomic_add_long ATOMIC_SAN(add_long) +#define atomic_add_acq_long ATOMIC_SAN(add_acq_long) +#define atomic_add_rel_long ATOMIC_SAN(add_rel_long) +#define atomic_clear_long ATOMIC_SAN(clear_long) +#define atomic_clear_acq_long ATOMIC_SAN(clear_acq_long) +#define atomic_clear_rel_long ATOMIC_SAN(clear_rel_long) +#define atomic_cmpset_long ATOMIC_SAN(cmpset_long) +#define atomic_cmpset_acq_long ATOMIC_SAN(cmpset_acq_long) +#define atomic_cmpset_rel_long ATOMIC_SAN(cmpset_rel_long) +#define atomic_fcmpset_long ATOMIC_SAN(fcmpset_long) +#define atomic_fcmpset_acq_long ATOMIC_SAN(fcmpset_acq_long) +#define atomic_fcmpset_rel_long ATOMIC_SAN(fcmpset_rel_long) +#define atomic_fetchadd_long ATOMIC_SAN(fetchadd_long) +#define atomic_load_long ATOMIC_SAN(load_long) +#define atomic_load_acq_long ATOMIC_SAN(load_acq_long) +#define atomic_readandclear_long ATOMIC_SAN(readandclear_long) +#define atomic_set_long ATOMIC_SAN(set_long) +#define atomic_set_acq_long ATOMIC_SAN(set_acq_long) +#define atomic_set_rel_long ATOMIC_SAN(set_rel_long) +#define atomic_subtract_long ATOMIC_SAN(subtract_long) +#define atomic_subtract_acq_long ATOMIC_SAN(subtract_acq_long) +#define atomic_subtract_rel_long ATOMIC_SAN(subtract_rel_long) +#define atomic_store_long ATOMIC_SAN(store_long) +#define atomic_store_rel_long ATOMIC_SAN(store_rel_long) +#define atomic_swap_long ATOMIC_SAN(swap_long) +#define atomic_testandclear_long ATOMIC_SAN(testandclear_long) +#define atomic_testandset_long ATOMIC_SAN(testandset_long) +#define atomic_testandset_acq_long ATOMIC_SAN(testandset_acq_long) -#define atomic_add_ptr kcsan_atomic_add_ptr -#define atomic_add_acq_ptr kcsan_atomic_add_acq_ptr -#define atomic_add_rel_ptr kcsan_atomic_add_rel_ptr -#define atomic_clear_ptr kcsan_atomic_clear_ptr -#define atomic_clear_acq_ptr kcsan_atomic_clear_acq_ptr -#define atomic_clear_rel_ptr kcsan_atomic_clear_rel_ptr -#define atomic_cmpset_ptr kcsan_atomic_cmpset_ptr -#define atomic_cmpset_acq_ptr kcsan_atomic_cmpset_acq_ptr -#define atomic_cmpset_rel_ptr kcsan_atomic_cmpset_rel_ptr -#define atomic_fcmpset_ptr kcsan_atomic_fcmpset_ptr -#define atomic_fcmpset_acq_ptr kcsan_atomic_fcmpset_acq_ptr -#define atomic_fcmpset_rel_ptr kcsan_atomic_fcmpset_rel_ptr -#define atomic_fetchadd_ptr kcsan_atomic_fetchadd_ptr +#define atomic_add_ptr ATOMIC_SAN(add_ptr) +#define atomic_add_acq_ptr ATOMIC_SAN(add_acq_ptr) +#define atomic_add_rel_ptr ATOMIC_SAN(add_rel_ptr) +#define atomic_clear_ptr ATOMIC_SAN(clear_ptr) +#define atomic_clear_acq_ptr ATOMIC_SAN(clear_acq_ptr) +#define atomic_clear_rel_ptr ATOMIC_SAN(clear_rel_ptr) +#define atomic_cmpset_ptr ATOMIC_SAN(cmpset_ptr) +#define atomic_cmpset_acq_ptr ATOMIC_SAN(cmpset_acq_ptr) +#define atomic_cmpset_rel_ptr ATOMIC_SAN(cmpset_rel_ptr) +#define atomic_fcmpset_ptr ATOMIC_SAN(fcmpset_ptr) +#define atomic_fcmpset_acq_ptr ATOMIC_SAN(fcmpset_acq_ptr) +#define atomic_fcmpset_rel_ptr ATOMIC_SAN(fcmpset_rel_ptr) +#define atomic_fetchadd_ptr ATOMIC_SAN(fetchadd_ptr) #define atomic_load_ptr(x) ({ \ __typeof(*x) __retptr; \ - __retptr = (void *)kcsan_atomic_load_ptr((volatile uintptr_t *)(x)); \ + __retptr = (void *)ATOMIC_SAN(load_ptr)((volatile uintptr_t *)(x)); \ __retptr; \ }) -#define atomic_load_acq_ptr kcsan_atomic_load_acq_ptr +#define atomic_load_acq_ptr ATOMIC_SAN(load_acq_ptr) #define atomic_load_consume_ptr(x) ({ \ __typeof(*x) __retptr; \ - __retptr = (void *)kcsan_atomic_load_acq_ptr((volatile uintptr_t *)(x));\ + __retptr = (void *)atomic_load_acq_ptr((volatile uintptr_t *)(x));\ __retptr; \ }) -#define atomic_readandclear_ptr kcsan_atomic_readandclear_ptr -#define atomic_set_ptr kcsan_atomic_set_ptr -#define atomic_set_acq_ptr kcsan_atomic_set_acq_ptr -#define atomic_set_rel_ptr kcsan_atomic_set_rel_ptr -#define atomic_subtract_ptr kcsan_atomic_subtract_ptr -#define atomic_subtract_acq_ptr kcsan_atomic_subtract_acq_ptr -#define atomic_subtract_rel_ptr kcsan_atomic_subtract_rel_ptr +#define atomic_readandclear_ptr ATOMIC_SAN(readandclear_ptr) +#define atomic_set_ptr ATOMIC_SAN(set_ptr) +#define atomic_set_acq_ptr ATOMIC_SAN(set_acq_ptr) +#define atomic_set_rel_ptr ATOMIC_SAN(set_rel_ptr) +#define atomic_subtract_ptr ATOMIC_SAN(subtract_ptr) +#define atomic_subtract_acq_ptr ATOMIC_SAN(subtract_acq_ptr) +#define atomic_subtract_rel_ptr ATOMIC_SAN(subtract_rel_ptr) #define atomic_store_ptr(x, v) ({ \ __typeof(*x) __value = (v); \ - kcsan_atomic_store_ptr((volatile uintptr_t *)(x), (uintptr_t)(__value));\ + ATOMIC_SAN(store_ptr)((volatile uintptr_t *)(x), (uintptr_t)(__value));\ }) -#define atomic_store_rel_ptr kcsan_atomic_store_rel_ptr -#define atomic_swap_ptr kcsan_atomic_swap_ptr -#define atomic_testandclear_ptr kcsan_atomic_testandclear_ptr -#define atomic_testandset_ptr kcsan_atomic_testandset_ptr +#define atomic_store_rel_ptr ATOMIC_SAN(store_rel_ptr) +#define atomic_swap_ptr ATOMIC_SAN(swap_ptr) +#define atomic_testandclear_ptr ATOMIC_SAN(testandclear_ptr) +#define atomic_testandset_ptr ATOMIC_SAN(testandset_ptr) -#define atomic_add_8 kcsan_atomic_add_8 -#define atomic_add_acq_8 kcsan_atomic_add_acq_8 -#define atomic_add_rel_8 kcsan_atomic_add_rel_8 -#define atomic_clear_8 kcsan_atomic_clear_8 -#define atomic_clear_acq_8 kcsan_atomic_clear_acq_8 -#define atomic_clear_rel_8 kcsan_atomic_clear_rel_8 -#define atomic_cmpset_8 kcsan_atomic_cmpset_8 -#define atomic_cmpset_acq_8 kcsan_atomic_cmpset_acq_8 -#define atomic_cmpset_rel_8 kcsan_atomic_cmpset_rel_8 -#define atomic_fcmpset_8 kcsan_atomic_fcmpset_8 -#define atomic_fcmpset_acq_8 kcsan_atomic_fcmpset_acq_8 -#define atomic_fcmpset_rel_8 kcsan_atomic_fcmpset_rel_8 -#define atomic_fetchadd_8 kcsan_atomic_fetchadd_8 -#define atomic_load_8 kcsan_atomic_load_8 -#define atomic_load_acq_8 kcsan_atomic_load_acq_8 -#define atomic_readandclear_8 kcsan_atomic_readandclear_8 -#define atomic_set_8 kcsan_atomic_set_8 -#define atomic_set_acq_8 kcsan_atomic_set_acq_8 -#define atomic_set_rel_8 kcsan_atomic_set_rel_8 -#define atomic_subtract_8 kcsan_atomic_subtract_8 -#define atomic_subtract_acq_8 kcsan_atomic_subtract_acq_8 -#define atomic_subtract_rel_8 kcsan_atomic_subtract_rel_8 -#define atomic_store_8 kcsan_atomic_store_8 -#define atomic_store_rel_8 kcsan_atomic_store_rel_8 -#define atomic_swap_8 kcsan_atomic_swap_8 -#define atomic_testandclear_8 kcsan_atomic_testandclear_8 -#define atomic_testandset_8 kcsan_atomic_testandset_8 +#define atomic_add_8 ATOMIC_SAN(add_8) +#define atomic_add_acq_8 ATOMIC_SAN(add_acq_8) +#define atomic_add_rel_8 ATOMIC_SAN(add_rel_8) +#define atomic_clear_8 ATOMIC_SAN(clear_8) +#define atomic_clear_acq_8 ATOMIC_SAN(clear_acq_8) +#define atomic_clear_rel_8 ATOMIC_SAN(clear_rel_8) +#define atomic_cmpset_8 ATOMIC_SAN(cmpset_8) +#define atomic_cmpset_acq_8 ATOMIC_SAN(cmpset_acq_8) +#define atomic_cmpset_rel_8 ATOMIC_SAN(cmpset_rel_8) +#define atomic_fcmpset_8 ATOMIC_SAN(fcmpset_8) +#define atomic_fcmpset_acq_8 ATOMIC_SAN(fcmpset_acq_8) +#define atomic_fcmpset_rel_8 ATOMIC_SAN(fcmpset_rel_8) +#define atomic_fetchadd_8 ATOMIC_SAN(fetchadd_8) +#define atomic_load_8 ATOMIC_SAN(load_8) +#define atomic_load_acq_8 ATOMIC_SAN(load_acq_8) +#define atomic_readandclear_8 ATOMIC_SAN(readandclear_8) +#define atomic_set_8 ATOMIC_SAN(set_8) +#define atomic_set_acq_8 ATOMIC_SAN(set_acq_8) +#define atomic_set_rel_8 ATOMIC_SAN(set_rel_8) +#define atomic_subtract_8 ATOMIC_SAN(subtract_8) +#define atomic_subtract_acq_8 ATOMIC_SAN(subtract_acq_8) +#define atomic_subtract_rel_8 ATOMIC_SAN(subtract_rel_8) +#define atomic_store_8 ATOMIC_SAN(store_8) +#define atomic_store_rel_8 ATOMIC_SAN(store_rel_8) +#define atomic_swap_8 ATOMIC_SAN(swap_8) +#define atomic_testandclear_8 ATOMIC_SAN(testandclear_8) +#define atomic_testandset_8 ATOMIC_SAN(testandset_8) -#define atomic_add_16 kcsan_atomic_add_16 -#define atomic_add_acq_16 kcsan_atomic_add_acq_16 -#define atomic_add_rel_16 kcsan_atomic_add_rel_16 -#define atomic_clear_16 kcsan_atomic_clear_16 -#define atomic_clear_acq_16 kcsan_atomic_clear_acq_16 -#define atomic_clear_rel_16 kcsan_atomic_clear_rel_16 -#define atomic_cmpset_16 kcsan_atomic_cmpset_16 -#define atomic_cmpset_acq_16 kcsan_atomic_cmpset_acq_16 -#define atomic_cmpset_rel_16 kcsan_atomic_cmpset_rel_16 -#define atomic_fcmpset_16 kcsan_atomic_fcmpset_16 -#define atomic_fcmpset_acq_16 kcsan_atomic_fcmpset_acq_16 -#define atomic_fcmpset_rel_16 kcsan_atomic_fcmpset_rel_16 -#define atomic_fetchadd_16 kcsan_atomic_fetchadd_16 -#define atomic_load_16 kcsan_atomic_load_16 -#define atomic_load_acq_16 kcsan_atomic_load_acq_16 -#define atomic_readandclear_16 kcsan_atomic_readandclear_16 -#define atomic_set_16 kcsan_atomic_set_16 -#define atomic_set_acq_16 kcsan_atomic_set_acq_16 -#define atomic_set_rel_16 kcsan_atomic_set_rel_16 -#define atomic_subtract_16 kcsan_atomic_subtract_16 -#define atomic_subtract_acq_16 kcsan_atomic_subtract_acq_16 -#define atomic_subtract_rel_16 kcsan_atomic_subtract_rel_16 -#define atomic_store_16 kcsan_atomic_store_16 -#define atomic_store_rel_16 kcsan_atomic_store_rel_16 -#define atomic_swap_16 kcsan_atomic_swap_16 -#define atomic_testandclear_16 kcsan_atomic_testandclear_16 -#define atomic_testandset_16 kcsan_atomic_testandset_16 +#define atomic_add_16 ATOMIC_SAN(add_16) +#define atomic_add_acq_16 ATOMIC_SAN(add_acq_16) +#define atomic_add_rel_16 ATOMIC_SAN(add_rel_16) +#define atomic_clear_16 ATOMIC_SAN(clear_16) +#define atomic_clear_acq_16 ATOMIC_SAN(clear_acq_16) +#define atomic_clear_rel_16 ATOMIC_SAN(clear_rel_16) +#define atomic_cmpset_16 ATOMIC_SAN(cmpset_16) +#define atomic_cmpset_acq_16 ATOMIC_SAN(cmpset_acq_16) +#define atomic_cmpset_rel_16 ATOMIC_SAN(cmpset_rel_16) +#define atomic_fcmpset_16 ATOMIC_SAN(fcmpset_16) +#define atomic_fcmpset_acq_16 ATOMIC_SAN(fcmpset_acq_16) +#define atomic_fcmpset_rel_16 ATOMIC_SAN(fcmpset_rel_16) +#define atomic_fetchadd_16 ATOMIC_SAN(fetchadd_16) +#define atomic_load_16 ATOMIC_SAN(load_16) +#define atomic_load_acq_16 ATOMIC_SAN(load_acq_16) +#define atomic_readandclear_16 ATOMIC_SAN(readandclear_16) +#define atomic_set_16 ATOMIC_SAN(set_16) +#define atomic_set_acq_16 ATOMIC_SAN(set_acq_16) +#define atomic_set_rel_16 ATOMIC_SAN(set_rel_16) +#define atomic_subtract_16 ATOMIC_SAN(subtract_16) +#define atomic_subtract_acq_16 ATOMIC_SAN(subtract_acq_16) +#define atomic_subtract_rel_16 ATOMIC_SAN(subtract_rel_16) +#define atomic_store_16 ATOMIC_SAN(store_16) +#define atomic_store_rel_16 ATOMIC_SAN(store_rel_16) +#define atomic_swap_16 ATOMIC_SAN(swap_16) +#define atomic_testandclear_16 ATOMIC_SAN(testandclear_16) +#define atomic_testandset_16 ATOMIC_SAN(testandset_16) -#define atomic_add_32 kcsan_atomic_add_32 -#define atomic_add_acq_32 kcsan_atomic_add_acq_32 -#define atomic_add_rel_32 kcsan_atomic_add_rel_32 -#define atomic_clear_32 kcsan_atomic_clear_32 -#define atomic_clear_acq_32 kcsan_atomic_clear_acq_32 -#define atomic_clear_rel_32 kcsan_atomic_clear_rel_32 -#define atomic_cmpset_32 kcsan_atomic_cmpset_32 -#define atomic_cmpset_acq_32 kcsan_atomic_cmpset_acq_32 -#define atomic_cmpset_rel_32 kcsan_atomic_cmpset_rel_32 -#define atomic_fcmpset_32 kcsan_atomic_fcmpset_32 -#define atomic_fcmpset_acq_32 kcsan_atomic_fcmpset_acq_32 -#define atomic_fcmpset_rel_32 kcsan_atomic_fcmpset_rel_32 -#define atomic_fetchadd_32 kcsan_atomic_fetchadd_32 -#define atomic_load_32 kcsan_atomic_load_32 -#define atomic_load_acq_32 kcsan_atomic_load_acq_32 -#define atomic_readandclear_32 kcsan_atomic_readandclear_32 -#define atomic_set_32 kcsan_atomic_set_32 -#define atomic_set_acq_32 kcsan_atomic_set_acq_32 -#define atomic_set_rel_32 kcsan_atomic_set_rel_32 -#define atomic_subtract_32 kcsan_atomic_subtract_32 -#define atomic_subtract_acq_32 kcsan_atomic_subtract_acq_32 -#define atomic_subtract_rel_32 kcsan_atomic_subtract_rel_32 -#define atomic_store_32 kcsan_atomic_store_32 -#define atomic_store_rel_32 kcsan_atomic_store_rel_32 -#define atomic_swap_32 kcsan_atomic_swap_32 -#define atomic_testandclear_32 kcsan_atomic_testandclear_32 -#define atomic_testandset_32 kcsan_atomic_testandset_32 +#define atomic_add_32 ATOMIC_SAN(add_32) +#define atomic_add_acq_32 ATOMIC_SAN(add_acq_32) +#define atomic_add_rel_32 ATOMIC_SAN(add_rel_32) +#define atomic_clear_32 ATOMIC_SAN(clear_32) +#define atomic_clear_acq_32 ATOMIC_SAN(clear_acq_32) +#define atomic_clear_rel_32 ATOMIC_SAN(clear_rel_32) +#define atomic_cmpset_32 ATOMIC_SAN(cmpset_32) +#define atomic_cmpset_acq_32 ATOMIC_SAN(cmpset_acq_32) +#define atomic_cmpset_rel_32 ATOMIC_SAN(cmpset_rel_32) +#define atomic_fcmpset_32 ATOMIC_SAN(fcmpset_32) +#define atomic_fcmpset_acq_32 ATOMIC_SAN(fcmpset_acq_32) +#define atomic_fcmpset_rel_32 ATOMIC_SAN(fcmpset_rel_32) +#define atomic_fetchadd_32 ATOMIC_SAN(fetchadd_32) +#define atomic_load_32 ATOMIC_SAN(load_32) +#define atomic_load_acq_32 ATOMIC_SAN(load_acq_32) +#define atomic_readandclear_32 ATOMIC_SAN(readandclear_32) +#define atomic_set_32 ATOMIC_SAN(set_32) +#define atomic_set_acq_32 ATOMIC_SAN(set_acq_32) +#define atomic_set_rel_32 ATOMIC_SAN(set_rel_32) +#define atomic_subtract_32 ATOMIC_SAN(subtract_32) +#define atomic_subtract_acq_32 ATOMIC_SAN(subtract_acq_32) +#define atomic_subtract_rel_32 ATOMIC_SAN(subtract_rel_32) +#define atomic_store_32 ATOMIC_SAN(store_32) +#define atomic_store_rel_32 ATOMIC_SAN(store_rel_32) +#define atomic_swap_32 ATOMIC_SAN(swap_32) +#define atomic_testandclear_32 ATOMIC_SAN(testandclear_32) +#define atomic_testandset_32 ATOMIC_SAN(testandset_32) -#define atomic_add_64 kcsan_atomic_add_64 -#define atomic_add_acq_64 kcsan_atomic_add_acq_64 -#define atomic_add_rel_64 kcsan_atomic_add_rel_64 -#define atomic_clear_64 kcsan_atomic_clear_64 -#define atomic_clear_acq_64 kcsan_atomic_clear_acq_64 -#define atomic_clear_rel_64 kcsan_atomic_clear_rel_64 -#define atomic_cmpset_64 kcsan_atomic_cmpset_64 -#define atomic_cmpset_acq_64 kcsan_atomic_cmpset_acq_64 -#define atomic_cmpset_rel_64 kcsan_atomic_cmpset_rel_64 -#define atomic_fcmpset_64 kcsan_atomic_fcmpset_64 -#define atomic_fcmpset_acq_64 kcsan_atomic_fcmpset_acq_64 -#define atomic_fcmpset_rel_64 kcsan_atomic_fcmpset_rel_64 -#define atomic_fetchadd_64 kcsan_atomic_fetchadd_64 -#define atomic_load_64 kcsan_atomic_load_64 -#define atomic_load_acq_64 kcsan_atomic_load_acq_64 -#define atomic_readandclear_64 kcsan_atomic_readandclear_64 -#define atomic_set_64 kcsan_atomic_set_64 -#define atomic_set_acq_64 kcsan_atomic_set_acq_64 -#define atomic_set_rel_64 kcsan_atomic_set_rel_64 -#define atomic_subtract_64 kcsan_atomic_subtract_64 -#define atomic_subtract_acq_64 kcsan_atomic_subtract_acq_64 -#define atomic_subtract_rel_64 kcsan_atomic_subtract_rel_64 -#define atomic_store_64 kcsan_atomic_store_64 -#define atomic_store_rel_64 kcsan_atomic_store_rel_64 -#define atomic_swap_64 kcsan_atomic_swap_64 -#define atomic_testandclear_64 kcsan_atomic_testandclear_64 -#define atomic_testandset_64 kcsan_atomic_testandset_64 +#define atomic_add_64 ATOMIC_SAN(add_64) +#define atomic_add_acq_64 ATOMIC_SAN(add_acq_64) +#define atomic_add_rel_64 ATOMIC_SAN(add_rel_64) +#define atomic_clear_64 ATOMIC_SAN(clear_64) +#define atomic_clear_acq_64 ATOMIC_SAN(clear_acq_64) +#define atomic_clear_rel_64 ATOMIC_SAN(clear_rel_64) +#define atomic_cmpset_64 ATOMIC_SAN(cmpset_64) +#define atomic_cmpset_acq_64 ATOMIC_SAN(cmpset_acq_64) +#define atomic_cmpset_rel_64 ATOMIC_SAN(cmpset_rel_64) +#define atomic_fcmpset_64 ATOMIC_SAN(fcmpset_64) +#define atomic_fcmpset_acq_64 ATOMIC_SAN(fcmpset_acq_64) +#define atomic_fcmpset_rel_64 ATOMIC_SAN(fcmpset_rel_64) +#define atomic_fetchadd_64 ATOMIC_SAN(fetchadd_64) +#define atomic_load_64 ATOMIC_SAN(load_64) +#define atomic_load_acq_64 ATOMIC_SAN(load_acq_64) +#define atomic_readandclear_64 ATOMIC_SAN(readandclear_64) +#define atomic_set_64 ATOMIC_SAN(set_64) +#define atomic_set_acq_64 ATOMIC_SAN(set_acq_64) +#define atomic_set_rel_64 ATOMIC_SAN(set_rel_64) +#define atomic_subtract_64 ATOMIC_SAN(subtract_64) +#define atomic_subtract_acq_64 ATOMIC_SAN(subtract_acq_64) +#define atomic_subtract_rel_64 ATOMIC_SAN(subtract_rel_64) +#define atomic_store_64 ATOMIC_SAN(store_64) +#define atomic_store_rel_64 ATOMIC_SAN(store_rel_64) +#define atomic_swap_64 ATOMIC_SAN(swap_64) +#define atomic_testandclear_64 ATOMIC_SAN(testandclear_64) +#define atomic_testandset_64 ATOMIC_SAN(testandset_64) -#define atomic_thread_fence_acq kcsan_atomic_thread_fence_acq -#define atomic_thread_fence_acq_rel kcsan_atomic_thread_fence_acq_rel -#define atomic_thread_fence_rel kcsan_atomic_thread_fence_rel -#define atomic_thread_fence_seq_cst kcsan_atomic_thread_fence_seq_cst -#define atomic_interrupt_fence __compiler_membar +#define atomic_thread_fence_acq ATOMIC_SAN(thread_fence_acq) +#define atomic_thread_fence_acq_rel ATOMIC_SAN(thread_fence_acq_rel) +#define atomic_thread_fence_rel ATOMIC_SAN(thread_fence_rel) +#define atomic_thread_fence_seq_cst ATOMIC_SAN(thread_fence_seq_cst) +#define atomic_interrupt_fence ATOMIC_SAN(interrupt_fence) -#endif /* !KCSAN_RUNTIME */ +#endif /* !SAN_RUNTIME */ #endif /* !_SYS_ATOMIC_SAN_H_ */ diff --git a/sys/sys/bus_san.h b/sys/sys/bus_san.h index 96b2441fbdbe..05d5ecd4b844 100644 --- a/sys/sys/bus_san.h +++ b/sys/sys/bus_san.h @@ -1,209 +1,229 @@ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2019 Andrew Turner + * Copyright (c) 2021 The FreeBSD Foundation * * This software was developed by SRI International and the University of * Cambridge Computer Laboratory (Department of Computer Science and * Technology) under DARPA contract HR0011-18-C-0016 ("ECATS"), as part of the * DARPA SSITH research programme. * + * Portions of this software were written by Mark Johnston under sponsorship by + * the FreeBSD Foundation. + * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #ifndef _SYS_BUS_SAN_H_ #define _SYS_BUS_SAN_H_ -#define KCSAN_BS_MULTI(rw, width, type) \ - void kcsan_bus_space_##rw##_multi_##width(bus_space_tag_t, \ +#ifndef _MACHINE_BUS_H_ +#error do not include this header, use machine/bus.h +#endif + +#ifndef BUS_SAN_PREFIX +#error No sanitizer prefix defined +#endif + +#define BUS_SAN_MULTI(sp, rw, width, type) \ + void sp##_bus_space_##rw##_multi_##width(bus_space_tag_t, \ bus_space_handle_t, bus_size_t, type *, bus_size_t); \ - void kcsan_bus_space_##rw##_multi_stream_##width(bus_space_tag_t, \ + void sp##_bus_space_##rw##_multi_stream_##width(bus_space_tag_t, \ bus_space_handle_t, bus_size_t, type *, bus_size_t); \ - void kcsan_bus_space_##rw##_region_##width(bus_space_tag_t, \ + void sp##_bus_space_##rw##_region_##width(bus_space_tag_t, \ bus_space_handle_t, bus_size_t, type *, bus_size_t); \ - void kcsan_bus_space_##rw##_region_stream_##width(bus_space_tag_t, \ + void sp##_bus_space_##rw##_region_stream_##width(bus_space_tag_t, \ bus_space_handle_t, bus_size_t, type *, bus_size_t) -#define KCSAN_BS_READ(width, type) \ - type kcsan_bus_space_read_##width(bus_space_tag_t, \ +#define BUS_SAN_READ(sp, width, type) \ + type sp##_bus_space_read_##width(bus_space_tag_t, \ bus_space_handle_t, bus_size_t); \ - type kcsan_bus_space_read_stream_##width(bus_space_tag_t, \ + type sp##_bus_space_read_stream_##width(bus_space_tag_t, \ bus_space_handle_t, bus_size_t); \ - KCSAN_BS_MULTI(read, width, type) + BUS_SAN_MULTI(sp, read, width, type) -#define KCSAN_BS_WRITE(width, type) \ - void kcsan_bus_space_write_##width(bus_space_tag_t, \ +#define BUS_SAN_WRITE(sp, width, type) \ + void sp##_bus_space_write_##width(bus_space_tag_t, \ bus_space_handle_t, bus_size_t, type); \ - void kcsan_bus_space_write_stream_##width(bus_space_tag_t, \ + void sp##_bus_space_write_stream_##width(bus_space_tag_t, \ bus_space_handle_t, bus_size_t, type); \ - KCSAN_BS_MULTI(write, width, const type) + BUS_SAN_MULTI(sp, write, width, const type) -#define KCSAN_BS_SET(width, type) \ - void kcsan_bus_space_set_multi_##width(bus_space_tag_t, \ +#define BUS_SAN_SET(sp, width, type) \ + void sp##_bus_space_set_multi_##width(bus_space_tag_t, \ bus_space_handle_t, bus_size_t, type, bus_size_t); \ - void kcsan_bus_space_set_multi_stream_##width(bus_space_tag_t, \ + void sp##_bus_space_set_multi_stream_##width(bus_space_tag_t, \ bus_space_handle_t, bus_size_t, type, bus_size_t); \ - void kcsan_bus_space_set_region_##width(bus_space_tag_t, \ + void sp##_bus_space_set_region_##width(bus_space_tag_t, \ bus_space_handle_t, bus_size_t, type, bus_size_t); \ - void kcsan_bus_space_set_region_stream_##width(bus_space_tag_t, \ + void sp##_bus_space_set_region_stream_##width(bus_space_tag_t, \ bus_space_handle_t, bus_size_t, type, bus_size_t) -#define KCSAN_BS_COPY(width, type) \ - void kcsan_bus_space_copy_region_##width(bus_space_tag_t, \ +#define BUS_SAN_COPY(sp, width, type) \ + void sp##_bus_space_copy_region_##width(bus_space_tag_t, \ bus_space_handle_t, bus_size_t, bus_space_handle_t, \ bus_size_t, bus_size_t); \ - void kcsan_bus_space_copy_region_stream_##width(bus_space_tag_t, \ + void sp##_bus_space_copy_region_stream_##width(bus_space_tag_t, \ bus_space_handle_t, bus_size_t, bus_space_handle_t, \ bus_size_t, bus_size_t); -#define KCSAN_BS_PEEK(width, type) \ - int kcsan_bus_space_peek_##width(bus_space_tag_t, \ +#define BUS_SAN_PEEK(sp, width, type) \ + int sp##_bus_space_peek_##width(bus_space_tag_t, \ bus_space_handle_t, bus_size_t, type *); -#define KCSAN_BS_POKE(width, type) \ - int kcsan_bus_space_poke_##width(bus_space_tag_t, \ +#define BUS_SAN_POKE(sp, width, type) \ + int sp##_bus_space_poke_##width(bus_space_tag_t, \ bus_space_handle_t, bus_size_t, type); -#define KCSAN_BS(width, type) \ - KCSAN_BS_READ(width, type); \ - KCSAN_BS_WRITE(width, type); \ - KCSAN_BS_SET(width, type); \ - KCSAN_BS_COPY(width, type) \ - KCSAN_BS_PEEK(width, type); \ - KCSAN_BS_POKE(width, type); - -KCSAN_BS(1, uint8_t); -KCSAN_BS(2, uint16_t); -KCSAN_BS(4, uint32_t); -KCSAN_BS(8, uint64_t); - -int kcsan_bus_space_map(bus_space_tag_t, bus_addr_t, bus_size_t, int, - bus_space_handle_t *); -void kcsan_bus_space_unmap(bus_space_tag_t, bus_space_handle_t, bus_size_t); -int kcsan_bus_space_subregion(bus_space_tag_t, bus_space_handle_t, bus_size_t, - bus_size_t, bus_space_handle_t *); -int kcsan_bus_space_alloc(bus_space_tag_t, bus_addr_t, bus_addr_t, - bus_size_t, bus_size_t, bus_size_t, int, bus_addr_t *, - bus_space_handle_t *); -void kcsan_bus_space_free(bus_space_tag_t, bus_space_handle_t, bus_size_t); -void kcsan_bus_space_barrier(bus_space_tag_t, bus_space_handle_t, bus_size_t, - bus_size_t, int); - -#ifndef KCSAN_RUNTIME - -#define bus_space_map kcsan_bus_space_map -#define bus_space_unmap kcsan_bus_space_unmap -#define bus_space_subregion kcsan_bus_space_subregion -#define bus_space_alloc kcsan_bus_space_alloc -#define bus_space_free kcsan_bus_space_free -#define bus_space_barrier kcsan_bus_space_barrier - -#define bus_space_read_1 kcsan_bus_space_read_1 -#define bus_space_read_stream_1 kcsan_bus_space_read_stream_1 -#define bus_space_read_multi_1 kcsan_bus_space_read_multi_1 -#define bus_space_read_multi_stream_1 kcsan_bus_space_read_multi_stream_1 -#define bus_space_read_region_1 kcsan_bus_space_read_region_1 -#define bus_space_read_region_stream_1 kcsan_bus_space_read_region_stream_1 -#define bus_space_write_1 kcsan_bus_space_write_1 -#define bus_space_write_stream_1 kcsan_bus_space_write_stream_1 -#define bus_space_write_multi_1 kcsan_bus_space_write_multi_1 -#define bus_space_write_multi_stream_1 kcsan_bus_space_write_multi_stream_1 -#define bus_space_write_region_1 kcsan_bus_space_write_region_1 -#define bus_space_write_region_stream_1 kcsan_bus_space_write_region_stream_1 -#define bus_space_set_multi_1 kcsan_bus_space_set_multi_1 -#define bus_space_set_multi_stream_1 kcsan_bus_space_set_multi_stream_1 -#define bus_space_set_region_1 kcsan_bus_space_set_region_1 -#define bus_space_set_region_stream_1 kcsan_bus_space_set_region_stream_1 -#define bus_space_copy_multi_1 kcsan_bus_space_copy_multi_1 -#define bus_space_copy_multi_stream_1 kcsan_bus_space_copy_multi_stream_1 -#define bus_space_poke_1 kcsan_bus_space_poke_1 -#define bus_space_peek_1 kcsan_bus_space_peek_1 - -#define bus_space_read_2 kcsan_bus_space_read_2 -#define bus_space_read_stream_2 kcsan_bus_space_read_stream_2 -#define bus_space_read_multi_2 kcsan_bus_space_read_multi_2 -#define bus_space_read_multi_stream_2 kcsan_bus_space_read_multi_stream_2 -#define bus_space_read_region_2 kcsan_bus_space_read_region_2 -#define bus_space_read_region_stream_2 kcsan_bus_space_read_region_stream_2 -#define bus_space_write_2 kcsan_bus_space_write_2 -#define bus_space_write_stream_2 kcsan_bus_space_write_stream_2 -#define bus_space_write_multi_2 kcsan_bus_space_write_multi_2 -#define bus_space_write_multi_stream_2 kcsan_bus_space_write_multi_stream_2 -#define bus_space_write_region_2 kcsan_bus_space_write_region_2 -#define bus_space_write_region_stream_2 kcsan_bus_space_write_region_stream_2 -#define bus_space_set_multi_2 kcsan_bus_space_set_multi_2 -#define bus_space_set_multi_stream_2 kcsan_bus_space_set_multi_stream_2 -#define bus_space_set_region_2 kcsan_bus_space_set_region_2 -#define bus_space_set_region_stream_2 kcsan_bus_space_set_region_stream_2 -#define bus_space_copy_multi_2 kcsan_bus_space_copy_multi_2 -#define bus_space_copy_multi_stream_2 kcsan_bus_space_copy_multi_stream_2 -#define bus_space_poke_2 kcsan_bus_space_poke_2 -#define bus_space_peek_2 kcsan_bus_space_peek_2 - -#define bus_space_read_4 kcsan_bus_space_read_4 -#define bus_space_read_stream_4 kcsan_bus_space_read_stream_4 -#define bus_space_read_multi_4 kcsan_bus_space_read_multi_4 -#define bus_space_read_multi_stream_4 kcsan_bus_space_read_multi_stream_4 -#define bus_space_read_region_4 kcsan_bus_space_read_region_4 -#define bus_space_read_region_stream_4 kcsan_bus_space_read_region_stream_4 -#define bus_space_write_4 kcsan_bus_space_write_4 -#define bus_space_write_stream_4 kcsan_bus_space_write_stream_4 -#define bus_space_write_multi_4 kcsan_bus_space_write_multi_4 -#define bus_space_write_multi_stream_4 kcsan_bus_space_write_multi_stream_4 -#define bus_space_write_region_4 kcsan_bus_space_write_region_4 -#define bus_space_write_region_stream_4 kcsan_bus_space_write_region_stream_4 -#define bus_space_set_multi_4 kcsan_bus_space_set_multi_4 -#define bus_space_set_multi_stream_4 kcsan_bus_space_set_multi_stream_4 -#define bus_space_set_region_4 kcsan_bus_space_set_region_4 -#define bus_space_set_region_stream_4 kcsan_bus_space_set_region_stream_4 -#define bus_space_copy_multi_4 kcsan_bus_space_copy_multi_4 -#define bus_space_copy_multi_stream_4 kcsan_bus_space_copy_multi_stream_4 -#define bus_space_poke_4 kcsan_bus_space_poke_4 -#define bus_space_peek_4 kcsan_bus_space_peek_4 - -#define bus_space_read_8 kcsan_bus_space_read_8 -#define bus_space_read_stream_8 kcsan_bus_space_read_stream_8 -#define bus_space_read_multi_8 kcsan_bus_space_read_multi_8 -#define bus_space_read_multi_stream_8 kcsan_bus_space_read_multi_stream_8 -#define bus_space_read_region_8 kcsan_bus_space_read_region_8 -#define bus_space_read_region_stream_8 kcsan_bus_space_read_region_stream_8 -#define bus_space_write_8 kcsan_bus_space_write_8 -#define bus_space_write_stream_8 kcsan_bus_space_write_stream_8 -#define bus_space_write_multi_8 kcsan_bus_space_write_multi_8 -#define bus_space_write_multi_stream_8 kcsan_bus_space_write_multi_stream_8 -#define bus_space_write_region_8 kcsan_bus_space_write_region_8 -#define bus_space_write_region_stream_8 kcsan_bus_space_write_region_stream_8 -#define bus_space_set_multi_8 kcsan_bus_space_set_multi_8 -#define bus_space_set_multi_stream_8 kcsan_bus_space_set_multi_stream_8 -#define bus_space_set_region_8 kcsan_bus_space_set_region_8 -#define bus_space_set_region_stream_8 kcsan_bus_space_set_region_stream_8 -#define bus_space_copy_multi_8 kcsan_bus_space_copy_multi_8 -#define bus_space_copy_multi_stream_8 kcsan_bus_space_copy_multi_stream_8 -#define bus_space_poke_8 kcsan_bus_space_poke_8 -#define bus_space_peek_8 kcsan_bus_space_peek_8 - +#define BUS_SAN_MISC(sp) \ + int sp##_bus_space_map(bus_space_tag_t, bus_addr_t, bus_size_t, \ + int, bus_space_handle_t *); \ + void sp##_bus_space_unmap(bus_space_tag_t, bus_space_handle_t, \ + bus_size_t); \ + int sp##_bus_space_subregion(bus_space_tag_t, bus_space_handle_t,\ + bus_size_t, bus_size_t, bus_space_handle_t *); \ + int sp##_bus_space_alloc(bus_space_tag_t, bus_addr_t, bus_addr_t,\ + bus_size_t, bus_size_t, bus_size_t, int, bus_addr_t *, \ + bus_space_handle_t *); \ + void sp##_bus_space_free(bus_space_tag_t, bus_space_handle_t, \ + bus_size_t); \ + void sp##_bus_space_barrier(bus_space_tag_t, bus_space_handle_t,\ + bus_size_t, bus_size_t, int); + +#define _BUS_SAN_FUNCS(sp, width, type) \ + BUS_SAN_READ(sp, width, type); \ + BUS_SAN_WRITE(sp, width, type); \ + BUS_SAN_SET(sp, width, type); \ + BUS_SAN_COPY(sp, width, type) \ + BUS_SAN_PEEK(sp, width, type); \ + BUS_SAN_POKE(sp, width, type); \ + BUS_SAN_MISC(sp); + +#define BUS_SAN_FUNCS(width, type) \ + _BUS_SAN_FUNCS(BUS_SAN_PREFIX, width, type) + +BUS_SAN_FUNCS(1, uint8_t); +BUS_SAN_FUNCS(2, uint16_t); +BUS_SAN_FUNCS(4, uint32_t); +BUS_SAN_FUNCS(8, uint64_t); + +#ifndef SAN_RUNTIME + +#define BUS_SAN(func) __CONCAT(BUS_SAN_PREFIX, __CONCAT(_bus_space_, func)) + +#define bus_space_map BUS_SAN(map) +#define bus_space_unmap BUS_SAN(unmap) +#define bus_space_subregion BUS_SAN(subregion) +#define bus_space_alloc BUS_SAN(alloc) +#define bus_space_free BUS_SAN(free) +#define bus_space_barrier BUS_SAN(barrier) + +#define bus_space_read_1 BUS_SAN(read_1) +#define bus_space_read_stream_1 BUS_SAN(read_stream_1) +#define bus_space_read_multi_1 BUS_SAN(read_multi_1) +#define bus_space_read_multi_stream_1 BUS_SAN(read_multi_stream_1) +#define bus_space_read_region_1 BUS_SAN(read_region_1) +#define bus_space_read_region_stream_1 BUS_SAN(read_region_stream_1) +#define bus_space_write_1 BUS_SAN(write_1) +#define bus_space_write_stream_1 BUS_SAN(write_stream_1) +#define bus_space_write_multi_1 BUS_SAN(write_multi_1) +#define bus_space_write_multi_stream_1 BUS_SAN(write_multi_stream_1) +#define bus_space_write_region_1 BUS_SAN(write_region_1) +#define bus_space_write_region_stream_1 BUS_SAN(write_region_stream_1) +#define bus_space_set_multi_1 BUS_SAN(set_multi_1) +#define bus_space_set_multi_stream_1 BUS_SAN(set_multi_stream_1) +#define bus_space_set_region_1 BUS_SAN(set_region_1) +#define bus_space_set_region_stream_1 BUS_SAN(set_region_stream_1) +#define bus_space_copy_multi_1 BUS_SAN(copy_multi_1) +#define bus_space_copy_multi_stream_1 BUS_SAN(copy_multi_stream_1) +#define bus_space_poke_1 BUS_SAN(poke_1) +#define bus_space_peek_1 BUS_SAN(peek_1) + +#define bus_space_read_2 BUS_SAN(read_2) +#define bus_space_read_stream_2 BUS_SAN(read_stream_2) +#define bus_space_read_multi_2 BUS_SAN(read_multi_2) +#define bus_space_read_multi_stream_2 BUS_SAN(read_multi_stream_2) +#define bus_space_read_region_2 BUS_SAN(read_region_2) +#define bus_space_read_region_stream_2 BUS_SAN(read_region_stream_2) +#define bus_space_write_2 BUS_SAN(write_2) +#define bus_space_write_stream_2 BUS_SAN(write_stream_2) +#define bus_space_write_multi_2 BUS_SAN(write_multi_2) +#define bus_space_write_multi_stream_2 BUS_SAN(write_multi_stream_2) +#define bus_space_write_region_2 BUS_SAN(write_region_2) +#define bus_space_write_region_stream_2 BUS_SAN(write_region_stream_2) +#define bus_space_set_multi_2 BUS_SAN(set_multi_2) +#define bus_space_set_multi_stream_2 BUS_SAN(set_multi_stream_2) +#define bus_space_set_region_2 BUS_SAN(set_region_2) +#define bus_space_set_region_stream_2 BUS_SAN(set_region_stream_2) +#define bus_space_copy_multi_2 BUS_SAN(copy_multi_2) +#define bus_space_copy_multi_stream_2 BUS_SAN(copy_multi_stream_2) +#define bus_space_poke_2 BUS_SAN(poke_2) +#define bus_space_peek_2 BUS_SAN(peek_2) + +#define bus_space_read_4 BUS_SAN(read_4) +#define bus_space_read_stream_4 BUS_SAN(read_stream_4) +#define bus_space_read_multi_4 BUS_SAN(read_multi_4) +#define bus_space_read_multi_stream_4 BUS_SAN(read_multi_stream_4) +#define bus_space_read_region_4 BUS_SAN(read_region_4) +#define bus_space_read_region_stream_4 BUS_SAN(read_region_stream_4) +#define bus_space_write_4 BUS_SAN(write_4) +#define bus_space_write_stream_4 BUS_SAN(write_stream_4) +#define bus_space_write_multi_4 BUS_SAN(write_multi_4) +#define bus_space_write_multi_stream_4 BUS_SAN(write_multi_stream_4) +#define bus_space_write_region_4 BUS_SAN(write_region_4) +#define bus_space_write_region_stream_4 BUS_SAN(write_region_stream_4) +#define bus_space_set_multi_4 BUS_SAN(set_multi_4) +#define bus_space_set_multi_stream_4 BUS_SAN(set_multi_stream_4) +#define bus_space_set_region_4 BUS_SAN(set_region_4) +#define bus_space_set_region_stream_4 BUS_SAN(set_region_stream_4) +#define bus_space_copy_multi_4 BUS_SAN(copy_multi_4) +#define bus_space_copy_multi_stream_4 BUS_SAN(copy_multi_stream_4) +#define bus_space_poke_4 BUS_SAN(poke_4) +#define bus_space_peek_4 BUS_SAN(peek_4) + +#define bus_space_read_8 BUS_SAN(read_8) +#define bus_space_read_stream_8 BUS_SAN(read_stream_8) +#define bus_space_read_multi_8 BUS_SAN(read_multi_8) +#define bus_space_read_multi_stream_8 BUS_SAN(read_multi_stream_8) +#define bus_space_read_region_8 BUS_SAN(read_region_8) +#define bus_space_read_region_stream_8 BUS_SAN(read_region_stream_8) +#define bus_space_write_8 BUS_SAN(write_8) +#define bus_space_write_stream_8 BUS_SAN(write_stream_8) +#define bus_space_write_multi_8 BUS_SAN(write_multi_8) +#define bus_space_write_multi_stream_8 BUS_SAN(write_multi_stream_8) +#define bus_space_write_region_8 BUS_SAN(write_region_8) +#define bus_space_write_region_stream_8 BUS_SAN(write_region_stream_8) +#define bus_space_set_multi_8 BUS_SAN(set_multi_8) +#define bus_space_set_multi_stream_8 BUS_SAN(set_multi_stream_8) +#define bus_space_set_region_8 BUS_SAN(set_region_8) +#define bus_space_set_region_stream_8 BUS_SAN(set_region_stream_8) +#define bus_space_copy_multi_8 BUS_SAN(copy_multi_8) +#define bus_space_copy_multi_stream_8 BUS_SAN(copy_multi_stream_8) +#define bus_space_poke_8 BUS_SAN(poke_8) +#define bus_space_peek_8 BUS_SAN(peek_8) #endif /* !KCSAN_RUNTIME */ #endif /* !_SYS_BUS_SAN_H_ */ diff --git a/sys/x86/include/bus.h b/sys/x86/include/bus.h index da2fe237ae99..0ba68e250b3d 100644 --- a/sys/x86/include/bus.h +++ b/sys/x86/include/bus.h @@ -1,1126 +1,1134 @@ /*- * SPDX-License-Identifier: BSD-3-Clause AND BSD-2-Clause-NetBSDE * * Copyright (c) KATO Takenori, 1999. * * All rights reserved. Unpublished rights reserved under the copyright * laws of Japan. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer as * the first lines of this file unmodified. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. The name of the author may not be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * * $FreeBSD$ */ /* $NetBSD: bus.h,v 1.12 1997/10/01 08:25:15 fvdl Exp $ */ /*- * Copyright (c) 1996, 1997 The NetBSD Foundation, Inc. * All rights reserved. * * This code is derived from software contributed to The NetBSD Foundation * by Jason R. Thorpe of the Numerical Aerospace Simulation Facility, * NASA Ames Research Center. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ /*- * Copyright (c) 1996 Charles M. Hannum. All rights reserved. * Copyright (c) 1996 Christopher G. Demetriou. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. All advertising materials mentioning features or use of this software * must display the following acknowledgement: * This product includes software developed by Christopher G. Demetriou * for the NetBSD Project. * 4. The name of the author may not be used to endorse or promote products * derived from this software without specific prior written permission * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -#ifndef _X86_BUS_H_ -#define _X86_BUS_H_ +#ifndef _MACHINE_BUS_H_ +#define _MACHINE_BUS_H_ #include #include #include #ifndef __GNUCLIKE_ASM #error "no assembler code for your compiler" #endif /* * Values for the x86 bus space tag, not to be used directly by MI code. */ #define X86_BUS_SPACE_IO 0 /* space is i/o space */ #define X86_BUS_SPACE_MEM 1 /* space is mem space */ #define BUS_SPACE_MAXSIZE_24BIT 0xFFFFFF #define BUS_SPACE_MAXSIZE_32BIT 0xFFFFFFFF #if defined(__amd64__) #define BUS_SPACE_MAXSIZE 0xFFFFFFFFFFFFFFFFULL #else #define BUS_SPACE_MAXSIZE 0xFFFFFFFF #endif #define BUS_SPACE_MAXADDR_24BIT 0xFFFFFF #define BUS_SPACE_MAXADDR_32BIT 0xFFFFFFFF #if defined(__amd64__) || defined(PAE) #define BUS_SPACE_MAXADDR_48BIT 0xFFFFFFFFFFFFULL #define BUS_SPACE_MAXADDR 0xFFFFFFFFFFFFFFFFULL #else #define BUS_SPACE_MAXADDR 0xFFFFFFFF #endif #define BUS_SPACE_INVALID_DATA (~0) #define BUS_SPACE_UNRESTRICTED (~0) #define BUS_SPACE_BARRIER_READ 0x01 /* force read barrier */ #define BUS_SPACE_BARRIER_WRITE 0x02 /* force write barrier */ -#if defined(KCSAN) && !defined(KCSAN_RUNTIME) +#ifndef SAN_RUNTIME +#if defined(KASAN) +#define BUS_SAN_PREFIX kasan +#elif defined(KCSAN) +#define BUS_SAN_PREFIX kcsan +#endif +#endif + +#ifdef BUS_SAN_PREFIX #include #else /* * Map a region of device bus space into CPU virtual address space. */ int bus_space_map(bus_space_tag_t tag, bus_addr_t addr, bus_size_t size, int flags, bus_space_handle_t *bshp); /* * Unmap a region of device bus space. */ void bus_space_unmap(bus_space_tag_t tag, bus_space_handle_t bsh, bus_size_t size); /* * Get a new handle for a subregion of an already-mapped area of bus space. */ static __inline int bus_space_subregion(bus_space_tag_t t, bus_space_handle_t bsh, bus_size_t offset, bus_size_t size, bus_space_handle_t *nbshp); static __inline int bus_space_subregion(bus_space_tag_t t __unused, bus_space_handle_t bsh, bus_size_t offset, bus_size_t size __unused, bus_space_handle_t *nbshp) { *nbshp = bsh + offset; return (0); } /* * Allocate a region of memory that is accessible to devices in bus space. */ int bus_space_alloc(bus_space_tag_t t, bus_addr_t rstart, bus_addr_t rend, bus_size_t size, bus_size_t align, bus_size_t boundary, int flags, bus_addr_t *addrp, bus_space_handle_t *bshp); /* * Free a region of bus space accessible memory. */ static __inline void bus_space_free(bus_space_tag_t t, bus_space_handle_t bsh, bus_size_t size); static __inline void bus_space_free(bus_space_tag_t t __unused, bus_space_handle_t bsh __unused, bus_size_t size __unused) { } /* * Read a 1, 2, 4, or 8 byte quantity from bus space * described by tag/handle/offset. */ static __inline u_int8_t bus_space_read_1(bus_space_tag_t tag, bus_space_handle_t handle, bus_size_t offset); static __inline u_int16_t bus_space_read_2(bus_space_tag_t tag, bus_space_handle_t handle, bus_size_t offset); static __inline u_int32_t bus_space_read_4(bus_space_tag_t tag, bus_space_handle_t handle, bus_size_t offset); #ifdef __amd64__ static __inline uint64_t bus_space_read_8(bus_space_tag_t tag, bus_space_handle_t handle, bus_size_t offset); #endif static __inline u_int8_t bus_space_read_1(bus_space_tag_t tag, bus_space_handle_t handle, bus_size_t offset) { if (tag == X86_BUS_SPACE_IO) return (inb(handle + offset)); return (*(volatile u_int8_t *)(handle + offset)); } static __inline u_int16_t bus_space_read_2(bus_space_tag_t tag, bus_space_handle_t handle, bus_size_t offset) { if (tag == X86_BUS_SPACE_IO) return (inw(handle + offset)); return (*(volatile u_int16_t *)(handle + offset)); } static __inline u_int32_t bus_space_read_4(bus_space_tag_t tag, bus_space_handle_t handle, bus_size_t offset) { if (tag == X86_BUS_SPACE_IO) return (inl(handle + offset)); return (*(volatile u_int32_t *)(handle + offset)); } #ifdef __amd64__ static __inline uint64_t bus_space_read_8(bus_space_tag_t tag, bus_space_handle_t handle, bus_size_t offset) { if (tag == X86_BUS_SPACE_IO) /* No 8 byte IO space access on x86 */ return (BUS_SPACE_INVALID_DATA); return (*(volatile uint64_t *)(handle + offset)); } #endif /* * Read `count' 1, 2, 4, or 8 byte quantities from bus space * described by tag/handle/offset and copy into buffer provided. */ static __inline void bus_space_read_multi_1(bus_space_tag_t tag, bus_space_handle_t bsh, bus_size_t offset, u_int8_t *addr, size_t count); static __inline void bus_space_read_multi_2(bus_space_tag_t tag, bus_space_handle_t bsh, bus_size_t offset, u_int16_t *addr, size_t count); static __inline void bus_space_read_multi_4(bus_space_tag_t tag, bus_space_handle_t bsh, bus_size_t offset, u_int32_t *addr, size_t count); static __inline void bus_space_read_multi_1(bus_space_tag_t tag, bus_space_handle_t bsh, bus_size_t offset, u_int8_t *addr, size_t count) { if (tag == X86_BUS_SPACE_IO) insb(bsh + offset, addr, count); else { #ifdef __GNUCLIKE_ASM __asm __volatile(" \n\ 1: movb (%2),%%al \n\ stosb \n\ loop 1b" : "=D" (addr), "=c" (count) : "r" (bsh + offset), "0" (addr), "1" (count) : "%eax", "memory"); #endif } } static __inline void bus_space_read_multi_2(bus_space_tag_t tag, bus_space_handle_t bsh, bus_size_t offset, u_int16_t *addr, size_t count) { if (tag == X86_BUS_SPACE_IO) insw(bsh + offset, addr, count); else { #ifdef __GNUCLIKE_ASM __asm __volatile(" \n\ 1: movw (%2),%%ax \n\ stosw \n\ loop 1b" : "=D" (addr), "=c" (count) : "r" (bsh + offset), "0" (addr), "1" (count) : "%eax", "memory"); #endif } } static __inline void bus_space_read_multi_4(bus_space_tag_t tag, bus_space_handle_t bsh, bus_size_t offset, u_int32_t *addr, size_t count) { if (tag == X86_BUS_SPACE_IO) insl(bsh + offset, addr, count); else { #ifdef __GNUCLIKE_ASM __asm __volatile(" \n\ 1: movl (%2),%%eax \n\ stosl \n\ loop 1b" : "=D" (addr), "=c" (count) : "r" (bsh + offset), "0" (addr), "1" (count) : "%eax", "memory"); #endif } } #if 0 /* Cause a link error for bus_space_read_multi_8 */ #define bus_space_read_multi_8 !!! bus_space_read_multi_8 unimplemented !!! #endif /* * Read `count' 1, 2, 4, or 8 byte quantities from bus space * described by tag/handle and starting at `offset' and copy into * buffer provided. */ static __inline void bus_space_read_region_1(bus_space_tag_t tag, bus_space_handle_t bsh, bus_size_t offset, u_int8_t *addr, size_t count); static __inline void bus_space_read_region_2(bus_space_tag_t tag, bus_space_handle_t bsh, bus_size_t offset, u_int16_t *addr, size_t count); static __inline void bus_space_read_region_4(bus_space_tag_t tag, bus_space_handle_t bsh, bus_size_t offset, u_int32_t *addr, size_t count); static __inline void bus_space_read_region_1(bus_space_tag_t tag, bus_space_handle_t bsh, bus_size_t offset, u_int8_t *addr, size_t count) { if (tag == X86_BUS_SPACE_IO) { int _port_ = bsh + offset; #ifdef __GNUCLIKE_ASM __asm __volatile(" \n\ 1: inb %w2,%%al \n\ stosb \n\ incl %2 \n\ loop 1b" : "=D" (addr), "=c" (count), "=d" (_port_) : "0" (addr), "1" (count), "2" (_port_) : "%eax", "memory", "cc"); #endif } else { bus_space_handle_t _port_ = bsh + offset; #ifdef __GNUCLIKE_ASM __asm __volatile(" \n\ repne \n\ movsb" : "=D" (addr), "=c" (count), "=S" (_port_) : "0" (addr), "1" (count), "2" (_port_) : "memory", "cc"); #endif } } static __inline void bus_space_read_region_2(bus_space_tag_t tag, bus_space_handle_t bsh, bus_size_t offset, u_int16_t *addr, size_t count) { if (tag == X86_BUS_SPACE_IO) { int _port_ = bsh + offset; #ifdef __GNUCLIKE_ASM __asm __volatile(" \n\ 1: inw %w2,%%ax \n\ stosw \n\ addl $2,%2 \n\ loop 1b" : "=D" (addr), "=c" (count), "=d" (_port_) : "0" (addr), "1" (count), "2" (_port_) : "%eax", "memory", "cc"); #endif } else { bus_space_handle_t _port_ = bsh + offset; #ifdef __GNUCLIKE_ASM __asm __volatile(" \n\ repne \n\ movsw" : "=D" (addr), "=c" (count), "=S" (_port_) : "0" (addr), "1" (count), "2" (_port_) : "memory", "cc"); #endif } } static __inline void bus_space_read_region_4(bus_space_tag_t tag, bus_space_handle_t bsh, bus_size_t offset, u_int32_t *addr, size_t count) { if (tag == X86_BUS_SPACE_IO) { int _port_ = bsh + offset; #ifdef __GNUCLIKE_ASM __asm __volatile(" \n\ 1: inl %w2,%%eax \n\ stosl \n\ addl $4,%2 \n\ loop 1b" : "=D" (addr), "=c" (count), "=d" (_port_) : "0" (addr), "1" (count), "2" (_port_) : "%eax", "memory", "cc"); #endif } else { bus_space_handle_t _port_ = bsh + offset; #ifdef __GNUCLIKE_ASM __asm __volatile(" \n\ repne \n\ movsl" : "=D" (addr), "=c" (count), "=S" (_port_) : "0" (addr), "1" (count), "2" (_port_) : "memory", "cc"); #endif } } #if 0 /* Cause a link error for bus_space_read_region_8 */ #define bus_space_read_region_8 !!! bus_space_read_region_8 unimplemented !!! #endif /* * Write the 1, 2, 4, or 8 byte value `value' to bus space * described by tag/handle/offset. */ static __inline void bus_space_write_1(bus_space_tag_t tag, bus_space_handle_t bsh, bus_size_t offset, u_int8_t value); static __inline void bus_space_write_2(bus_space_tag_t tag, bus_space_handle_t bsh, bus_size_t offset, u_int16_t value); static __inline void bus_space_write_4(bus_space_tag_t tag, bus_space_handle_t bsh, bus_size_t offset, u_int32_t value); #ifdef __amd64__ static __inline void bus_space_write_8(bus_space_tag_t tag, bus_space_handle_t bsh, bus_size_t offset, uint64_t value); #endif static __inline void bus_space_write_1(bus_space_tag_t tag, bus_space_handle_t bsh, bus_size_t offset, u_int8_t value) { if (tag == X86_BUS_SPACE_IO) outb(bsh + offset, value); else *(volatile u_int8_t *)(bsh + offset) = value; } static __inline void bus_space_write_2(bus_space_tag_t tag, bus_space_handle_t bsh, bus_size_t offset, u_int16_t value) { if (tag == X86_BUS_SPACE_IO) outw(bsh + offset, value); else *(volatile u_int16_t *)(bsh + offset) = value; } static __inline void bus_space_write_4(bus_space_tag_t tag, bus_space_handle_t bsh, bus_size_t offset, u_int32_t value) { if (tag == X86_BUS_SPACE_IO) outl(bsh + offset, value); else *(volatile u_int32_t *)(bsh + offset) = value; } #ifdef __amd64__ static __inline void bus_space_write_8(bus_space_tag_t tag, bus_space_handle_t bsh, bus_size_t offset, uint64_t value) { if (tag == X86_BUS_SPACE_IO) /* No 8 byte IO space access on x86 */ return; else *(volatile uint64_t *)(bsh + offset) = value; } #endif /* * Write `count' 1, 2, 4, or 8 byte quantities from the buffer * provided to bus space described by tag/handle/offset. */ static __inline void bus_space_write_multi_1(bus_space_tag_t tag, bus_space_handle_t bsh, bus_size_t offset, const u_int8_t *addr, size_t count); static __inline void bus_space_write_multi_2(bus_space_tag_t tag, bus_space_handle_t bsh, bus_size_t offset, const u_int16_t *addr, size_t count); static __inline void bus_space_write_multi_4(bus_space_tag_t tag, bus_space_handle_t bsh, bus_size_t offset, const u_int32_t *addr, size_t count); static __inline void bus_space_write_multi_1(bus_space_tag_t tag, bus_space_handle_t bsh, bus_size_t offset, const u_int8_t *addr, size_t count) { if (tag == X86_BUS_SPACE_IO) outsb(bsh + offset, addr, count); else { #ifdef __GNUCLIKE_ASM __asm __volatile(" \n\ 1: lodsb \n\ movb %%al,(%2) \n\ loop 1b" : "=S" (addr), "=c" (count) : "r" (bsh + offset), "0" (addr), "1" (count) : "%eax", "memory", "cc"); #endif } } static __inline void bus_space_write_multi_2(bus_space_tag_t tag, bus_space_handle_t bsh, bus_size_t offset, const u_int16_t *addr, size_t count) { if (tag == X86_BUS_SPACE_IO) outsw(bsh + offset, addr, count); else { #ifdef __GNUCLIKE_ASM __asm __volatile(" \n\ 1: lodsw \n\ movw %%ax,(%2) \n\ loop 1b" : "=S" (addr), "=c" (count) : "r" (bsh + offset), "0" (addr), "1" (count) : "%eax", "memory", "cc"); #endif } } static __inline void bus_space_write_multi_4(bus_space_tag_t tag, bus_space_handle_t bsh, bus_size_t offset, const u_int32_t *addr, size_t count) { if (tag == X86_BUS_SPACE_IO) outsl(bsh + offset, addr, count); else { #ifdef __GNUCLIKE_ASM __asm __volatile(" \n\ 1: lodsl \n\ movl %%eax,(%2) \n\ loop 1b" : "=S" (addr), "=c" (count) : "r" (bsh + offset), "0" (addr), "1" (count) : "%eax", "memory", "cc"); #endif } } #if 0 /* Cause a link error for bus_space_write_multi_8 */ #define bus_space_write_multi_8(t, h, o, a, c) \ !!! bus_space_write_multi_8 unimplemented !!! #endif /* * Write `count' 1, 2, 4, or 8 byte quantities from the buffer provided * to bus space described by tag/handle starting at `offset'. */ static __inline void bus_space_write_region_1(bus_space_tag_t tag, bus_space_handle_t bsh, bus_size_t offset, const u_int8_t *addr, size_t count); static __inline void bus_space_write_region_2(bus_space_tag_t tag, bus_space_handle_t bsh, bus_size_t offset, const u_int16_t *addr, size_t count); static __inline void bus_space_write_region_4(bus_space_tag_t tag, bus_space_handle_t bsh, bus_size_t offset, const u_int32_t *addr, size_t count); static __inline void bus_space_write_region_1(bus_space_tag_t tag, bus_space_handle_t bsh, bus_size_t offset, const u_int8_t *addr, size_t count) { if (tag == X86_BUS_SPACE_IO) { int _port_ = bsh + offset; #ifdef __GNUCLIKE_ASM __asm __volatile(" \n\ 1: lodsb \n\ outb %%al,%w0 \n\ incl %0 \n\ loop 1b" : "=d" (_port_), "=S" (addr), "=c" (count) : "0" (_port_), "1" (addr), "2" (count) : "%eax", "memory", "cc"); #endif } else { bus_space_handle_t _port_ = bsh + offset; #ifdef __GNUCLIKE_ASM __asm __volatile(" \n\ repne \n\ movsb" : "=D" (_port_), "=S" (addr), "=c" (count) : "0" (_port_), "1" (addr), "2" (count) : "memory", "cc"); #endif } } static __inline void bus_space_write_region_2(bus_space_tag_t tag, bus_space_handle_t bsh, bus_size_t offset, const u_int16_t *addr, size_t count) { if (tag == X86_BUS_SPACE_IO) { int _port_ = bsh + offset; #ifdef __GNUCLIKE_ASM __asm __volatile(" \n\ 1: lodsw \n\ outw %%ax,%w0 \n\ addl $2,%0 \n\ loop 1b" : "=d" (_port_), "=S" (addr), "=c" (count) : "0" (_port_), "1" (addr), "2" (count) : "%eax", "memory", "cc"); #endif } else { bus_space_handle_t _port_ = bsh + offset; #ifdef __GNUCLIKE_ASM __asm __volatile(" \n\ repne \n\ movsw" : "=D" (_port_), "=S" (addr), "=c" (count) : "0" (_port_), "1" (addr), "2" (count) : "memory", "cc"); #endif } } static __inline void bus_space_write_region_4(bus_space_tag_t tag, bus_space_handle_t bsh, bus_size_t offset, const u_int32_t *addr, size_t count) { if (tag == X86_BUS_SPACE_IO) { int _port_ = bsh + offset; #ifdef __GNUCLIKE_ASM __asm __volatile(" \n\ 1: lodsl \n\ outl %%eax,%w0 \n\ addl $4,%0 \n\ loop 1b" : "=d" (_port_), "=S" (addr), "=c" (count) : "0" (_port_), "1" (addr), "2" (count) : "%eax", "memory", "cc"); #endif } else { bus_space_handle_t _port_ = bsh + offset; #ifdef __GNUCLIKE_ASM __asm __volatile(" \n\ repne \n\ movsl" : "=D" (_port_), "=S" (addr), "=c" (count) : "0" (_port_), "1" (addr), "2" (count) : "memory", "cc"); #endif } } #if 0 /* Cause a link error for bus_space_write_region_8 */ #define bus_space_write_region_8 \ !!! bus_space_write_region_8 unimplemented !!! #endif /* * Write the 1, 2, 4, or 8 byte value `val' to bus space described * by tag/handle/offset `count' times. */ static __inline void bus_space_set_multi_1(bus_space_tag_t tag, bus_space_handle_t bsh, bus_size_t offset, u_int8_t value, size_t count); static __inline void bus_space_set_multi_2(bus_space_tag_t tag, bus_space_handle_t bsh, bus_size_t offset, u_int16_t value, size_t count); static __inline void bus_space_set_multi_4(bus_space_tag_t tag, bus_space_handle_t bsh, bus_size_t offset, u_int32_t value, size_t count); static __inline void bus_space_set_multi_1(bus_space_tag_t tag, bus_space_handle_t bsh, bus_size_t offset, u_int8_t value, size_t count) { bus_space_handle_t addr = bsh + offset; if (tag == X86_BUS_SPACE_IO) while (count--) outb(addr, value); else while (count--) *(volatile u_int8_t *)(addr) = value; } static __inline void bus_space_set_multi_2(bus_space_tag_t tag, bus_space_handle_t bsh, bus_size_t offset, u_int16_t value, size_t count) { bus_space_handle_t addr = bsh + offset; if (tag == X86_BUS_SPACE_IO) while (count--) outw(addr, value); else while (count--) *(volatile u_int16_t *)(addr) = value; } static __inline void bus_space_set_multi_4(bus_space_tag_t tag, bus_space_handle_t bsh, bus_size_t offset, u_int32_t value, size_t count) { bus_space_handle_t addr = bsh + offset; if (tag == X86_BUS_SPACE_IO) while (count--) outl(addr, value); else while (count--) *(volatile u_int32_t *)(addr) = value; } #if 0 /* Cause a link error for bus_space_set_multi_8 */ #define bus_space_set_multi_8 !!! bus_space_set_multi_8 unimplemented !!! #endif /* * Write `count' 1, 2, 4, or 8 byte value `val' to bus space described * by tag/handle starting at `offset'. */ static __inline void bus_space_set_region_1(bus_space_tag_t tag, bus_space_handle_t bsh, bus_size_t offset, u_int8_t value, size_t count); static __inline void bus_space_set_region_2(bus_space_tag_t tag, bus_space_handle_t bsh, bus_size_t offset, u_int16_t value, size_t count); static __inline void bus_space_set_region_4(bus_space_tag_t tag, bus_space_handle_t bsh, bus_size_t offset, u_int32_t value, size_t count); static __inline void bus_space_set_region_1(bus_space_tag_t tag, bus_space_handle_t bsh, bus_size_t offset, u_int8_t value, size_t count) { bus_space_handle_t addr = bsh + offset; if (tag == X86_BUS_SPACE_IO) for (; count != 0; count--, addr++) outb(addr, value); else for (; count != 0; count--, addr++) *(volatile u_int8_t *)(addr) = value; } static __inline void bus_space_set_region_2(bus_space_tag_t tag, bus_space_handle_t bsh, bus_size_t offset, u_int16_t value, size_t count) { bus_space_handle_t addr = bsh + offset; if (tag == X86_BUS_SPACE_IO) for (; count != 0; count--, addr += 2) outw(addr, value); else for (; count != 0; count--, addr += 2) *(volatile u_int16_t *)(addr) = value; } static __inline void bus_space_set_region_4(bus_space_tag_t tag, bus_space_handle_t bsh, bus_size_t offset, u_int32_t value, size_t count) { bus_space_handle_t addr = bsh + offset; if (tag == X86_BUS_SPACE_IO) for (; count != 0; count--, addr += 4) outl(addr, value); else for (; count != 0; count--, addr += 4) *(volatile u_int32_t *)(addr) = value; } #if 0 /* Cause a link error for bus_space_set_region_8 */ #define bus_space_set_region_8 !!! bus_space_set_region_8 unimplemented !!! #endif /* * Copy `count' 1, 2, 4, or 8 byte values from bus space starting * at tag/bsh1/off1 to bus space starting at tag/bsh2/off2. */ static __inline void bus_space_copy_region_1(bus_space_tag_t tag, bus_space_handle_t bsh1, bus_size_t off1, bus_space_handle_t bsh2, bus_size_t off2, size_t count); static __inline void bus_space_copy_region_2(bus_space_tag_t tag, bus_space_handle_t bsh1, bus_size_t off1, bus_space_handle_t bsh2, bus_size_t off2, size_t count); static __inline void bus_space_copy_region_4(bus_space_tag_t tag, bus_space_handle_t bsh1, bus_size_t off1, bus_space_handle_t bsh2, bus_size_t off2, size_t count); static __inline void bus_space_copy_region_1(bus_space_tag_t tag, bus_space_handle_t bsh1, bus_size_t off1, bus_space_handle_t bsh2, bus_size_t off2, size_t count) { bus_space_handle_t addr1 = bsh1 + off1; bus_space_handle_t addr2 = bsh2 + off2; if (tag == X86_BUS_SPACE_IO) { if (addr1 >= addr2) { /* src after dest: copy forward */ for (; count != 0; count--, addr1++, addr2++) outb(addr2, inb(addr1)); } else { /* dest after src: copy backwards */ for (addr1 += (count - 1), addr2 += (count - 1); count != 0; count--, addr1--, addr2--) outb(addr2, inb(addr1)); } } else { if (addr1 >= addr2) { /* src after dest: copy forward */ for (; count != 0; count--, addr1++, addr2++) *(volatile u_int8_t *)(addr2) = *(volatile u_int8_t *)(addr1); } else { /* dest after src: copy backwards */ for (addr1 += (count - 1), addr2 += (count - 1); count != 0; count--, addr1--, addr2--) *(volatile u_int8_t *)(addr2) = *(volatile u_int8_t *)(addr1); } } } static __inline void bus_space_copy_region_2(bus_space_tag_t tag, bus_space_handle_t bsh1, bus_size_t off1, bus_space_handle_t bsh2, bus_size_t off2, size_t count) { bus_space_handle_t addr1 = bsh1 + off1; bus_space_handle_t addr2 = bsh2 + off2; if (tag == X86_BUS_SPACE_IO) { if (addr1 >= addr2) { /* src after dest: copy forward */ for (; count != 0; count--, addr1 += 2, addr2 += 2) outw(addr2, inw(addr1)); } else { /* dest after src: copy backwards */ for (addr1 += 2 * (count - 1), addr2 += 2 * (count - 1); count != 0; count--, addr1 -= 2, addr2 -= 2) outw(addr2, inw(addr1)); } } else { if (addr1 >= addr2) { /* src after dest: copy forward */ for (; count != 0; count--, addr1 += 2, addr2 += 2) *(volatile u_int16_t *)(addr2) = *(volatile u_int16_t *)(addr1); } else { /* dest after src: copy backwards */ for (addr1 += 2 * (count - 1), addr2 += 2 * (count - 1); count != 0; count--, addr1 -= 2, addr2 -= 2) *(volatile u_int16_t *)(addr2) = *(volatile u_int16_t *)(addr1); } } } static __inline void bus_space_copy_region_4(bus_space_tag_t tag, bus_space_handle_t bsh1, bus_size_t off1, bus_space_handle_t bsh2, bus_size_t off2, size_t count) { bus_space_handle_t addr1 = bsh1 + off1; bus_space_handle_t addr2 = bsh2 + off2; if (tag == X86_BUS_SPACE_IO) { if (addr1 >= addr2) { /* src after dest: copy forward */ for (; count != 0; count--, addr1 += 4, addr2 += 4) outl(addr2, inl(addr1)); } else { /* dest after src: copy backwards */ for (addr1 += 4 * (count - 1), addr2 += 4 * (count - 1); count != 0; count--, addr1 -= 4, addr2 -= 4) outl(addr2, inl(addr1)); } } else { if (addr1 >= addr2) { /* src after dest: copy forward */ for (; count != 0; count--, addr1 += 4, addr2 += 4) *(volatile u_int32_t *)(addr2) = *(volatile u_int32_t *)(addr1); } else { /* dest after src: copy backwards */ for (addr1 += 4 * (count - 1), addr2 += 4 * (count - 1); count != 0; count--, addr1 -= 4, addr2 -= 4) *(volatile u_int32_t *)(addr2) = *(volatile u_int32_t *)(addr1); } } } #if 0 /* Cause a link error for bus_space_copy_8 */ #define bus_space_copy_region_8 !!! bus_space_copy_region_8 unimplemented !!! #endif /* * Bus read/write barrier methods. * * void bus_space_barrier(bus_space_tag_t tag, bus_space_handle_t bsh, * bus_size_t offset, bus_size_t len, int flags); * * * Note that BUS_SPACE_BARRIER_WRITE doesn't do anything other than * prevent reordering by the compiler; all Intel x86 processors currently * retire operations outside the CPU in program order. */ static __inline void bus_space_barrier(bus_space_tag_t tag __unused, bus_space_handle_t bsh __unused, bus_size_t offset __unused, bus_size_t len __unused, int flags) { #ifdef __GNUCLIKE_ASM if (flags & BUS_SPACE_BARRIER_READ) #ifdef __amd64__ __asm __volatile("lock; addl $0,0(%%rsp)" : : : "memory"); #else __asm __volatile("lock; addl $0,0(%%esp)" : : : "memory"); #endif else __compiler_membar(); #endif } #ifdef BUS_SPACE_NO_LEGACY #undef inb #undef outb #define inb(a) compiler_error #define inw(a) compiler_error #define inl(a) compiler_error #define outb(a, b) compiler_error #define outw(a, b) compiler_error #define outl(a, b) compiler_error #endif /* * Stream accesses are the same as normal accesses on x86; there are no * supported bus systems with an endianess different from the host one. */ #define bus_space_read_stream_1(t, h, o) bus_space_read_1((t), (h), (o)) #define bus_space_read_stream_2(t, h, o) bus_space_read_2((t), (h), (o)) #define bus_space_read_stream_4(t, h, o) bus_space_read_4((t), (h), (o)) #define bus_space_read_multi_stream_1(t, h, o, a, c) \ bus_space_read_multi_1((t), (h), (o), (a), (c)) #define bus_space_read_multi_stream_2(t, h, o, a, c) \ bus_space_read_multi_2((t), (h), (o), (a), (c)) #define bus_space_read_multi_stream_4(t, h, o, a, c) \ bus_space_read_multi_4((t), (h), (o), (a), (c)) #define bus_space_write_stream_1(t, h, o, v) \ bus_space_write_1((t), (h), (o), (v)) #define bus_space_write_stream_2(t, h, o, v) \ bus_space_write_2((t), (h), (o), (v)) #define bus_space_write_stream_4(t, h, o, v) \ bus_space_write_4((t), (h), (o), (v)) #define bus_space_write_multi_stream_1(t, h, o, a, c) \ bus_space_write_multi_1((t), (h), (o), (a), (c)) #define bus_space_write_multi_stream_2(t, h, o, a, c) \ bus_space_write_multi_2((t), (h), (o), (a), (c)) #define bus_space_write_multi_stream_4(t, h, o, a, c) \ bus_space_write_multi_4((t), (h), (o), (a), (c)) #define bus_space_set_multi_stream_1(t, h, o, v, c) \ bus_space_set_multi_1((t), (h), (o), (v), (c)) #define bus_space_set_multi_stream_2(t, h, o, v, c) \ bus_space_set_multi_2((t), (h), (o), (v), (c)) #define bus_space_set_multi_stream_4(t, h, o, v, c) \ bus_space_set_multi_4((t), (h), (o), (v), (c)) #define bus_space_read_region_stream_1(t, h, o, a, c) \ bus_space_read_region_1((t), (h), (o), (a), (c)) #define bus_space_read_region_stream_2(t, h, o, a, c) \ bus_space_read_region_2((t), (h), (o), (a), (c)) #define bus_space_read_region_stream_4(t, h, o, a, c) \ bus_space_read_region_4((t), (h), (o), (a), (c)) #define bus_space_write_region_stream_1(t, h, o, a, c) \ bus_space_write_region_1((t), (h), (o), (a), (c)) #define bus_space_write_region_stream_2(t, h, o, a, c) \ bus_space_write_region_2((t), (h), (o), (a), (c)) #define bus_space_write_region_stream_4(t, h, o, a, c) \ bus_space_write_region_4((t), (h), (o), (a), (c)) #define bus_space_set_region_stream_1(t, h, o, v, c) \ bus_space_set_region_1((t), (h), (o), (v), (c)) #define bus_space_set_region_stream_2(t, h, o, v, c) \ bus_space_set_region_2((t), (h), (o), (v), (c)) #define bus_space_set_region_stream_4(t, h, o, v, c) \ bus_space_set_region_4((t), (h), (o), (v), (c)) #define bus_space_copy_region_stream_1(t, h1, o1, h2, o2, c) \ bus_space_copy_region_1((t), (h1), (o1), (h2), (o2), (c)) #define bus_space_copy_region_stream_2(t, h1, o1, h2, o2, c) \ bus_space_copy_region_2((t), (h1), (o1), (h2), (o2), (c)) #define bus_space_copy_region_stream_4(t, h1, o1, h2, o2, c) \ bus_space_copy_region_4((t), (h1), (o1), (h2), (o2), (c)) #define BUS_PEEK_FUNC(width, type) \ static inline int \ bus_space_peek_##width(bus_space_tag_t tag, \ bus_space_handle_t hnd, bus_size_t offset, type *value) \ { \ type tmp; \ tmp = bus_space_read_##width(tag, hnd, offset); \ *value = (type)tmp; \ return (0); \ } BUS_PEEK_FUNC(1, uint8_t) BUS_PEEK_FUNC(2, uint16_t) BUS_PEEK_FUNC(4, uint32_t) #ifdef __amd64__ BUS_PEEK_FUNC(8, uint64_t) #endif #define BUS_POKE_FUNC(width, type) \ static inline int \ bus_space_poke_##width(bus_space_tag_t tag, \ bus_space_handle_t hnd, bus_size_t offset, type value) \ { \ bus_space_write_##width(tag, hnd, offset, value); \ return (0); \ } BUS_POKE_FUNC(1, uint8_t) BUS_POKE_FUNC(2, uint16_t) BUS_POKE_FUNC(4, uint32_t) #ifdef __amd64__ BUS_POKE_FUNC(8, uint64_t) #endif -#endif /* KCSAN && !KCSAN_RUNTIME */ +#endif /* !BUS_SAN_PREFIX */ -#endif /* _X86_BUS_H_ */ +#endif /* !_MACHINE_BUS_H_ */ diff --git a/sys/x86/x86/bus_machdep.c b/sys/x86/x86/bus_machdep.c index 67edbb976415..3dc203f65694 100644 --- a/sys/x86/x86/bus_machdep.c +++ b/sys/x86/x86/bus_machdep.c @@ -1,61 +1,63 @@ /*- * Copyright (c) 2015 Marcel Moolenaar * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -#define KCSAN_RUNTIME +#if defined(KASAN) || defined(KCSAN) +#define SAN_RUNTIME +#endif #include __FBSDID("$FreeBSD$"); #include #include #include #include #include /* * Implementation of bus_space_map(), which effectively is a thin * wrapper around pmap_mapdev() for memory mapped I/O space. It's * implemented here and not in to avoid pollution. */ int bus_space_map(bus_space_tag_t tag, bus_addr_t addr, bus_size_t size, int flags __unused, bus_space_handle_t *bshp) { *bshp = (tag == X86_BUS_SPACE_MEM) ? (uintptr_t)pmap_mapdev(addr, size) : addr; return (0); } void bus_space_unmap(bus_space_tag_t tag, bus_space_handle_t bsh, bus_size_t size) { if (tag == X86_BUS_SPACE_MEM) pmap_unmapdev(bsh, size); }